feat(lifecycle): add stage 4 authoritative unbind and durable task routing

Unify broker-mode binding changes in database transactions: advance device
generation, invalidate pending authorizations/tasks and prior pairing-code
epochs, and persist audit plus invalidation outbox. Add authenticated device
unbind alongside robot/admin paths. Pairing codes are hashed, generation-
fenced and consumed atomically with binding.

Replay Redis invalidation effects without undoing committed unbind. Guard
online cleanup and delayed kill events by generation. Introduce a separate
router admission credential with no platform-event injection authority.
Router verifies signed nonce-bound internal admission on handshake, task
traffic and periodic checks; reject stale sessions and control-plane outages.

Persist task owner/device/generation/session/worker and original message
mapping. Sign routing metadata, verify late results, claim replies once and
mark ambiguous/crashed tasks unknown instead of replaying writes. Fence
notifications across same-user rebind. Release in-memory slots/waiters even
when database or Redis cleanup fails.

Validation: 213 passed, 3 skipped (real PostgreSQL not configured), 30 existing
uvicorn/websockets deprecation warnings. New lifecycle coverage: 28 tests,
including real loopback WebSocket isolation, reverse completion, missed kill,
periodic unbind checks, outbox recovery, transaction rollback and cleanup
failures. Syntax, docs and staged diff checks pass.

Default legacy mode remains unchanged. Broker mode now requires coordinated
router admission configuration and a v2 client handshake. No production ECS
configuration, real credentials or client installer was changed. PostgreSQL,
real Redis multi-node, ARM64 and actual Work/client rollout remain stage 5.
This commit is contained in:
ex_tianzy
2026-11-04 11:43:53 +08:00
parent 93bfbca83a
commit d1ab834ecb
13 changed files with 1128 additions and 29 deletions
+159 -17
View File
@@ -24,7 +24,7 @@ from redis.asyncio import Redis
from .common import BodyLimitMiddleware, no_cache, peer, rate_limit, require_secret
from .config import Settings
from .crypto import (fingerprint, load_private, open_envelope, raw_public, seal,
from .crypto import (canonical, fingerprint, load_private, open_envelope, raw_public, seal,
sign, verified, verify_device)
from .oauth import OAuth
from .platform import Feishu, PlatformError
@@ -62,6 +62,8 @@ class Broker:
self.s, self.r, self.store = settings, redis, store
self.signer = load_private(settings.connector_key)
self.pending: dict[str, Pending] = {}
self.worker_id = uuid.uuid4().hex
self.lifecycle = None
async def replies(self, pub):
async for message in pub.listen():
@@ -79,12 +81,14 @@ class Broker:
result = plain.get("result", "")
if not isinstance(result, str) or len(result) > self.s.max_result:
continue
if self.lifecycle and not self.lifecycle.complete(data["turn_id"],item.host,item.session,self.worker_id):
continue
item.future.set_result({"status": "completed", "turn_id": data["turn_id"], "result": result})
except Exception:
# Invalid packets never consume a pending result; do not log packet contents.
continue
async def dispatch(self, owner, text, turn):
async def dispatch(self, owner, text, turn, *, chat_id="", source_message_id=""):
device = self.store.by_owner(owner)
if not device:
return {"status": "unavailable", "turn_id": turn, "result": "未绑定设备或设备不可用。"}
@@ -104,36 +108,75 @@ class Broker:
if proof["host_id"] != host or proof["challenge"] != presence["challenge"]:
raise ValueError("invalid device session")
session = proof["session_id"]
if self.lifecycle and (proof.get("binding_owner") != owner
or proof.get("binding_generation") != device["binding_generation"]):
raise ValueError("stale binding session")
except Exception:
self.store.audit(owner, "dispatch", host, "identity_rejected")
raise HTTPException(409, "device identity rejected") from None
if not self.store.begin_turn(turn, owner, host):
return {"status": "duplicate", "turn_id": turn, "result": "该请求已接收,不会重复执行。"}
task = None
if self.lifecycle:
try:
task = self.lifecycle.create_task(turn,owner,host,device["binding_generation"],session,
self.worker_id,time.time()+self.s.request_timeout,chat_id,source_message_id)
except ValueError:
self.store.finish(turn,"unbound")
return {"status":"unbound","turn_id":turn,"result":"绑定已变化,任务未下发。"}
private = x25519.X25519PrivateKey.generate()
future = asyncio.get_running_loop().create_future()
self.pending[turn] = Pending(future, private, public, host, session)
envelope = seal({"type": "instruction", "turn_id": turn, "owner": owner,
payload = {"type": "instruction", "turn_id": turn, "owner": owner,
"text": text, "read_only": True, "channel": "im",
"reply_key": raw_public(private.public_key())}, proof["kx"], self.signer,
host_id=host, session_id=session)
"reply_key": raw_public(private.public_key())}
if task:
payload["task_context"] = {key:task[key] for key in ("environment","app_id","owner","host_id","binding_generation",
"conversation_id","chat_id","source_message_id")}
payload["task_context"].update(task_id=turn,channel="work-dm")
try:
envelope = seal(payload, proof["kx"], self.signer,host_id=host,session_id=session)
except Exception:
self.pending.pop(turn,None)
self.store.finish(turn,"unknown")
if task:
self.lifecycle.transition(turn,"inflight","unknown")
raise HTTPException(409,"invalid_device_crypto_session") from None
key = f"hn:inflight:{host}:{turn}"
try:
await self.r.set(key, session, ex=self.s.request_timeout + 5)
count = await self.r.publish("hn:down:" + host, json.dumps({"type": "instruction", "envelope": envelope}))
packet = {"type":"instruction","envelope":envelope}
if task:
packet["routing"] = sign({"turn_id":turn,"host_id":host,"session_id":session,
"environment":self.s.environment,"app_id":self.s.feishu_app_id,
"binding_generation":task["binding_generation"],
"envelope_hash":hashlib.sha256(canonical(envelope)).hexdigest()}, self.signer)
count = await self.r.publish("hn:down:" + host, json.dumps(packet))
if not count:
self.store.finish(turn, "offline")
if task:
self.lifecycle.transition(turn,"inflight","offline")
return {"status": "offline", "turn_id": turn, "result": "你的电脑当前离线。"}
result = await asyncio.wait_for(future, timeout=self.s.request_timeout)
self.store.finish(turn, "completed")
self.store.audit(owner, "instruction", host, "completed")
self.store.finish(turn, result.get("status","unknown") if task else "completed")
if task and not source_message_id:
self.lifecycle.transition(turn,"completed","finished")
self.store.audit(owner, "instruction", host, result.get("status","unknown") if task else "completed")
return result
except asyncio.TimeoutError:
self.store.finish(turn, "unknown")
if task:
self.lifecycle.transition(turn,"inflight","unknown")
return {"status": "unknown", "turn_id": turn,
"result": "等待结果超时,执行状态未知;不要直接重发有副作用的任务。"}
finally:
self.pending.pop(turn, None)
await self.r.delete(key)
if task:
# Cancellation, publish failures and connector shutdown never replay work.
with contextlib.suppress(Exception):
self.lifecycle.transition(turn,"inflight","unknown")
with contextlib.suppress(Exception):
await self.r.delete(key)
def create_app(settings=None, redis=None, platform=None):
@@ -149,9 +192,15 @@ def create_app(settings=None, redis=None, platform=None):
credential_store = CredentialStore(store, CredentialKeyring.from_file(s.credential_keyring_file))
credential_broker = CredentialBroker(credential_store, r, f, environment=s.environment, app_id=s.feishu_app_id)
oauth = CredentialAPI(s, r, store, credential_broker)
from .binding_state import BindingState
from .device_lifecycle import DeviceLifecycle
store.lifecycle = BindingState(store,s.environment,s.feishu_app_id)
broker.lifecycle = store.lifecycle
lifecycle = DeviceLifecycle(store.lifecycle,r,broker)
else:
oauth = OAuth(s, r, f)
oauth.store = store
lifecycle = None
from .notifications import router as notification_router
background = set()
@@ -165,6 +214,8 @@ def create_app(settings=None, redis=None, platform=None):
@asynccontextmanager
async def lifespan(app):
await r.ping()
if lifecycle:
await lifecycle.maintain()
for row in store.db.execute("SELECT host_id,fingerprint,revoked FROM devices").fetchall():
await r.set("hn:device:" + row["host_id"], row["fingerprint"])
if row["revoked"]:
@@ -180,12 +231,25 @@ def create_app(settings=None, redis=None, platform=None):
async with oauth.lock:
oauth.prune()
sweep_task = asyncio.create_task(sweep())
async def maintenance():
while True:
await asyncio.sleep(5)
try:
await lifecycle.maintain()
except Exception:
# No credentials or raw exception text in background logs.
pass
lifecycle_task = asyncio.create_task(maintenance()) if lifecycle else None
try:
yield
finally:
for task in [reply_task, sweep_task, *background]:
work = [reply_task,sweep_task,*background] + ([lifecycle_task] if lifecycle_task else [])
for task in work:
task.cancel()
await asyncio.gather(reply_task, sweep_task, *background, return_exceptions=True)
await asyncio.gather(*work, return_exceptions=True)
if lifecycle:
with contextlib.suppress(Exception):
await r.delete("hn:task-worker:"+broker.worker_id)
await pub.aclose()
await f.close()
store.close()
@@ -201,6 +265,28 @@ def create_app(settings=None, redis=None, platform=None):
app.include_router(web_sessions.router)
app.state.store, app.state.broker, app.state.oauth = store, broker, oauth
app.state.platform, app.state.redis = f, r
app.state.lifecycle = lifecycle
if lifecycle:
from .device_lifecycle import AdmissionRequest
from .credential_api import DeviceRequest
@app.post("/work/v2/device/unbind")
async def device_unbind(body: DeviceRequest, request: Request):
await oauth.authenticate(body,request)
try:
store.lifecycle.invalidate_host(body.host_id,"unbind",expected_owner=body.binding_owner,
expected_generation=body.binding_generation)
except ValueError:
raise HTTPException(409,"binding_changed_retry") from None
lifecycle.cancel_waiters()
propagated = await lifecycle.flush()
return {"status":"unbound","binding_generation":store.device(body.host_id)["binding_generation"],
"propagation":"published" if propagated else "pending"}
@app.post("/internal/devices/{host}/admission")
async def admission(host: str, body: AdmissionRequest, request: Request):
require_secret(request.headers.get("X-Router-Admission-Secret"),s.router_admission_secret)
return sign(await lifecycle.admission(host,body),broker.signer)
@app.exception_handler(RequestValidationError)
async def validation_error(request, exc):
@@ -261,6 +347,14 @@ def create_app(settings=None, redis=None, platform=None):
admin(request)
if not store.device(host):
raise HTTPException(404, "device not found")
if lifecycle:
try:
store.revoke(host)
except ValueError:
raise HTTPException(409,"binding_changed_retry") from None
lifecycle.cancel_waiters()
propagated = await lifecycle.flush()
return {"status":"revoked","propagation":"published" if propagated else "pending"}
# Persistent source of truth first; router block + disconnect follows.
store.revoke(host)
await r.incr("hn:web-generation:" + host)
@@ -277,6 +371,8 @@ def create_app(settings=None, redis=None, platform=None):
async def reconcile(request: Request):
"""Rebuild router admission metadata after Redis recovery; no dialogue data."""
admin(request)
if lifecycle:
await lifecycle.maintain()
rows = store.db.execute("SELECT host_id,fingerprint,revoked FROM devices").fetchall()
for row in rows:
await r.set("hn:device:" + row["host_id"], row["fingerprint"])
@@ -290,6 +386,15 @@ def create_app(settings=None, redis=None, platform=None):
device = store.device(host)
if not device:
raise HTTPException(404, "device not found")
if lifecycle:
try:
store.lifecycle.invalidate_host(host,"unbind")
except ValueError:
raise HTTPException(409,"binding_changed_retry") from None
lifecycle.cancel_waiters()
propagated = await lifecycle.flush()
return {"status":"unbound","host_id":host,"revoked":bool(device["revoked"]),
"propagation":"published" if propagated else "pending"}
if device["owner"]:
store.unbind(device["owner"])
await r.incr("hn:web-generation:" + host)
@@ -306,7 +411,13 @@ def create_app(settings=None, redis=None, platform=None):
return "你已绑定电脑,请先发送 /解绑。"
await rate_limit(r, "binding:" + owner, 5)
code = secrets.token_urlsafe(12)
await r.set("hn:bind:" + hashlib.sha256(code.encode()).hexdigest(), owner, ex=300)
if lifecycle:
try:
store.lifecycle.issue_code(owner,code)
except ValueError:
return "你已绑定电脑,请先发送 /解绑。"
else:
await r.set("hn:bind:" + hashlib.sha256(code.encode()).hexdigest(), owner, ex=300)
return "请在本人电脑的桥接客户端输入绑定码(5分钟有效):" + code
@app.post("/device/bind")
@@ -325,6 +436,16 @@ def create_app(settings=None, redis=None, platform=None):
raise ValueError("invalid code")
except Exception:
raise HTTPException(403, "invalid device proof") from None
if lifecycle:
try:
owner = store.lifecycle.consume_code(body.host_id,code)
except LookupError:
raise HTTPException(404,"binding code unavailable") from None
except (*INTEGRITY_ERRORS,ValueError):
raise HTTPException(409,"owner/device already bound; request new code") from None
return sign({"host_id":body.host_id,"owner":owner,
"binding_generation":store.device(body.host_id)["binding_generation"],
"ts":int(time.time())},broker.signer)
key = "hn:bind:" + hashlib.sha256(code.encode()).hexdigest()
owner = await r.get(key)
if not owner:
@@ -365,11 +486,15 @@ def create_app(settings=None, redis=None, platform=None):
owner = sender.get("sender_id", {}).get("open_id")
message_id = message.get("message_id", "")
if (message.get("chat_type") != "p2p" or message.get("message_type") != "text"
or sender.get("sender_type") != "user" or not owner
or sender.get("sender_type") != "user" or not isinstance(owner,str)
or not 0 < len(owner) <= 128 or any(ord(c) < 32 for c in owner)
or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", message_id)):
return
if header.get("app_id") != s.feishu_app_id:
return
if lifecycle and (not isinstance(message.get("chat_id"),str) or not 0 < len(message["chat_id"]) <= 128
or any(ord(c) < 32 for c in message["chat_id"])):
return
try:
text = json.loads(message["content"])["text"].strip()
if not text or len(text) > 8000:
@@ -383,7 +508,10 @@ def create_app(settings=None, redis=None, platform=None):
elif text == "/解绑":
old = store.by_owner(owner)
store.unbind(owner)
if old:
if lifecycle:
lifecycle.cancel_waiters()
await lifecycle.flush()
elif old:
await r.incr("hn:web-generation:" + old["host_id"])
await r.publish("hn:kill:" + old["host_id"], "unbind")
for item in broker.pending.values():
@@ -391,11 +519,25 @@ def create_app(settings=None, redis=None, platform=None):
item.future.set_result({"status": "unbound", "result": "设备已解绑,结果已丢弃。"})
result = "已解绑;重新绑定后请更新本机绑定信息。"
elif text == "/主机名" or text.startswith("/回显 "):
result = (await broker.dispatch(owner, text, "msg_" + hashlib.sha256(message_id.encode()).hexdigest()))["result"]
result = (await broker.dispatch(owner, text, "msg_" + hashlib.sha256(message_id.encode()).hexdigest(),
**({"chat_id":message["chat_id"],"source_message_id":message_id} if lifecycle else {})))["result"]
else:
instruction = text if text.startswith("/助手 ") else "/助手 " + text
result = (await broker.dispatch(owner, instruction, "msg_" + hashlib.sha256(message_id.encode()).hexdigest()))["result"]
await f.reply(message_id, result[:s.max_result])
result = (await broker.dispatch(owner, instruction, "msg_" + hashlib.sha256(message_id.encode()).hexdigest(),
**({"chat_id":message["chat_id"],"source_message_id":message_id} if lifecycle else {})))["result"]
turn = "msg_"+hashlib.sha256(message_id.encode()).hexdigest()
task = store.lifecycle.task(turn) if lifecycle else None
if task and not store.lifecycle.claim_reply(turn,owner,message_id):
store.finish(event_turn,"result_discarded")
return
try:
await f.reply(message_id, result[:s.max_result])
except Exception:
if task:
store.lifecycle.transition(turn,"replying","reply_unknown")
raise
if task:
store.lifecycle.transition(turn,"replying","replied")
store.finish(event_turn, "replied")
except Exception:
store.audit(owner, "event", message_id, "failed_or_unknown")