feat(lifecycle): add stage 4 authoritative unbind and durable task routing

Unify broker-mode binding changes in database transactions: advance device
generation, invalidate pending authorizations/tasks and prior pairing-code
epochs, and persist audit plus invalidation outbox. Add authenticated device
unbind alongside robot/admin paths. Pairing codes are hashed, generation-
fenced and consumed atomically with binding.

Replay Redis invalidation effects without undoing committed unbind. Guard
online cleanup and delayed kill events by generation. Introduce a separate
router admission credential with no platform-event injection authority.
Router verifies signed nonce-bound internal admission on handshake, task
traffic and periodic checks; reject stale sessions and control-plane outages.

Persist task owner/device/generation/session/worker and original message
mapping. Sign routing metadata, verify late results, claim replies once and
mark ambiguous/crashed tasks unknown instead of replaying writes. Fence
notifications across same-user rebind. Release in-memory slots/waiters even
when database or Redis cleanup fails.

Validation: 213 passed, 3 skipped (real PostgreSQL not configured), 30 existing
uvicorn/websockets deprecation warnings. New lifecycle coverage: 28 tests,
including real loopback WebSocket isolation, reverse completion, missed kill,
periodic unbind checks, outbox recovery, transaction rollback and cleanup
failures. Syntax, docs and staged diff checks pass.

Default legacy mode remains unchanged. Broker mode now requires coordinated
router admission configuration and a v2 client handshake. No production ECS
configuration, real credentials or client installer was changed. PostgreSQL,
real Redis multi-node, ARM64 and actual Work/client rollout remain stage 5.
This commit is contained in:
ex_tianzy
2026-11-04 11:43:53 +08:00
parent 93bfbca83a
commit d1ab834ecb
13 changed files with 1128 additions and 29 deletions
+58 -5
View File
@@ -3,6 +3,8 @@ import contextlib
import json
import secrets
import time
import hashlib
import httpx
from contextlib import asynccontextmanager
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
@@ -10,7 +12,7 @@ from redis.asyncio import Redis
from redis.exceptions import WatchError
from .config import Settings
from .crypto import fingerprint, raw_public, verified, verify_device
from .crypto import canonical, fingerprint, load_public, raw_public, verified, verify_device
async def compare_delete(redis, key, expected):
@@ -41,16 +43,38 @@ async def compare_expire(redis, key, expected, ttl):
return False
def create_app(settings=None, redis=None):
def create_app(settings=None, redis=None, admission_transport=None):
s = settings or Settings(role="router")
r = redis or Redis.from_url(s.redis_url, decode_responses=True)
ca = s.ca_cert.read_bytes()
active = set()
internal = (httpx.AsyncClient(base_url=s.connector_internal_url.rstrip('/'),timeout=3,
trust_env=False,follow_redirects=False,transport=admission_transport) if s.credential_mode == "broker" else None)
connector_public = load_public(s.connector_public_key) if internal else None
async def admission(host, owner, generation, *, turn=None, session=None, expected_fp=None):
nonce = secrets.token_hex(16)
response = await internal.post("/internal/devices/"+host+"/admission",
headers={"X-Router-Admission-Secret":s.router_admission_secret},
json={"nonce":nonce,"turn_id":turn,"session_id":session})
if response.status_code != 200:
raise ValueError("device admission rejected")
data = verified(response.json(),connector_public)
if (data["nonce"] != nonce or data["host_id"] != host or data["owner"] != owner
or data["binding_generation"] != generation or data["environment"] != s.environment
or data["app_id"] != s.feishu_app_id or data["protocol_version"] != 2
or data["turn_id"] != turn or data["session_id"] != session
or not time.time() <= data["expires_at"] <= time.time()+10
or data["fingerprint"] != expected_fp):
raise ValueError("stale device admission")
return data
@asynccontextmanager
async def lifespan(app):
await r.ping()
yield
if internal:
await internal.aclose()
if redis is None:
await r.aclose()
@@ -101,6 +125,14 @@ def create_app(settings=None, redis=None):
fp = fingerprint(hello["cert"])
if await r.get("hn:device:" + host) != fp or await r.exists("hn:revoked:" + host):
raise ValueError("unregistered/revoked")
generation = owner = None
if internal:
generation, owner = proof["binding_generation"], proof["binding_owner"]
if (type(generation) is not int or generation < 0 or type(proof.get("protocol_version")) is not int
or proof.get("protocol_version") != 2
or proof.get("environment") != s.environment or proof.get("app_id") != s.feishu_app_id):
raise ValueError("invalid binding handshake")
await admission(host,owner,generation,expected_fp=fp)
session = proof["session_id"]
presence = json.dumps({"cert": hello["cert"], "proof": hello["proof"], "challenge": challenge})
if not await r.set("hn:online:" + host, presence, nx=True, ex=30):
@@ -108,6 +140,8 @@ def create_app(settings=None, redis=None):
active.add(session)
pub = r.pubsub()
await pub.subscribe("hn:down:" + host, "hn:kill:" + host)
if internal:
await admission(host,owner,generation,expected_fp=fp)
await ws.send_json({"type": "connected", "session_id": session})
async def downstream():
@@ -115,13 +149,28 @@ def create_app(settings=None, redis=None):
if msg["type"] != "message":
continue
if msg["channel"] == "hn:kill:" + host:
if internal:
event = json.loads(msg["data"])
if generation >= event["binding_generation"]:
continue
await ws.close(1008)
return
if internal:
packet = json.loads(msg["data"])
routing = verified(packet["routing"],connector_public)
if (routing["host_id"] != host or routing["session_id"] != session
or routing["environment"] != s.environment or routing["app_id"] != s.feishu_app_id
or routing["binding_generation"] != generation
or routing["envelope_hash"] != hashlib.sha256(canonical(packet["envelope"])).hexdigest()):
raise ValueError("invalid instruction routing")
await admission(host,owner,generation,turn=routing["turn_id"],session=session,expected_fp=fp)
await ws.send_text(msg["data"])
async def heartbeat():
while True:
await asyncio.sleep(8)
if internal:
await admission(host,owner,generation,expected_fp=fp)
if await r.exists("hn:revoked:" + host) or not await compare_expire(r, "hn:online:" + host, presence, 30):
await ws.close(1008)
return
@@ -142,6 +191,8 @@ def create_app(settings=None, redis=None):
verified(envelope, public)
if envelope["host_id"] != host or envelope["session_id"] != session:
raise ValueError("wrong reply destination")
if internal:
await admission(host,owner,generation,turn=turn,session=session,expected_fp=fp)
await r.publish("hn:replies", json.dumps(data))
tasks = [asyncio.create_task(x()) for x in (downstream, heartbeat, upstream)]
@@ -155,13 +206,15 @@ def create_app(settings=None, redis=None):
with contextlib.suppress(Exception):
await ws.close(1011)
finally:
active.discard(session)
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
if pub:
await pub.aclose()
with contextlib.suppress(Exception):
await pub.aclose()
if presence:
await compare_delete(r, "hn:online:" + host, presence)
active.discard(session)
with contextlib.suppress(Exception):
await compare_delete(r, "hn:online:" + host, presence)
return app