feat(lifecycle): add stage 4 authoritative unbind and durable task routing
Unify broker-mode binding changes in database transactions: advance device generation, invalidate pending authorizations/tasks and prior pairing-code epochs, and persist audit plus invalidation outbox. Add authenticated device unbind alongside robot/admin paths. Pairing codes are hashed, generation- fenced and consumed atomically with binding. Replay Redis invalidation effects without undoing committed unbind. Guard online cleanup and delayed kill events by generation. Introduce a separate router admission credential with no platform-event injection authority. Router verifies signed nonce-bound internal admission on handshake, task traffic and periodic checks; reject stale sessions and control-plane outages. Persist task owner/device/generation/session/worker and original message mapping. Sign routing metadata, verify late results, claim replies once and mark ambiguous/crashed tasks unknown instead of replaying writes. Fence notifications across same-user rebind. Release in-memory slots/waiters even when database or Redis cleanup fails. Validation: 213 passed, 3 skipped (real PostgreSQL not configured), 30 existing uvicorn/websockets deprecation warnings. New lifecycle coverage: 28 tests, including real loopback WebSocket isolation, reverse completion, missed kill, periodic unbind checks, outbox recovery, transaction rollback and cleanup failures. Syntax, docs and staged diff checks pass. Default legacy mode remains unchanged. Broker mode now requires coordinated router admission configuration and a v2 client handshake. No production ECS configuration, real credentials or client installer was changed. PostgreSQL, real Redis multi-node, ARM64 and actual Work/client rollout remain stage 5.
This commit is contained in:
+159
-17
@@ -24,7 +24,7 @@ from redis.asyncio import Redis
|
||||
|
||||
from .common import BodyLimitMiddleware, no_cache, peer, rate_limit, require_secret
|
||||
from .config import Settings
|
||||
from .crypto import (fingerprint, load_private, open_envelope, raw_public, seal,
|
||||
from .crypto import (canonical, fingerprint, load_private, open_envelope, raw_public, seal,
|
||||
sign, verified, verify_device)
|
||||
from .oauth import OAuth
|
||||
from .platform import Feishu, PlatformError
|
||||
@@ -62,6 +62,8 @@ class Broker:
|
||||
self.s, self.r, self.store = settings, redis, store
|
||||
self.signer = load_private(settings.connector_key)
|
||||
self.pending: dict[str, Pending] = {}
|
||||
self.worker_id = uuid.uuid4().hex
|
||||
self.lifecycle = None
|
||||
|
||||
async def replies(self, pub):
|
||||
async for message in pub.listen():
|
||||
@@ -79,12 +81,14 @@ class Broker:
|
||||
result = plain.get("result", "")
|
||||
if not isinstance(result, str) or len(result) > self.s.max_result:
|
||||
continue
|
||||
if self.lifecycle and not self.lifecycle.complete(data["turn_id"],item.host,item.session,self.worker_id):
|
||||
continue
|
||||
item.future.set_result({"status": "completed", "turn_id": data["turn_id"], "result": result})
|
||||
except Exception:
|
||||
# Invalid packets never consume a pending result; do not log packet contents.
|
||||
continue
|
||||
|
||||
async def dispatch(self, owner, text, turn):
|
||||
async def dispatch(self, owner, text, turn, *, chat_id="", source_message_id=""):
|
||||
device = self.store.by_owner(owner)
|
||||
if not device:
|
||||
return {"status": "unavailable", "turn_id": turn, "result": "未绑定设备或设备不可用。"}
|
||||
@@ -104,36 +108,75 @@ class Broker:
|
||||
if proof["host_id"] != host or proof["challenge"] != presence["challenge"]:
|
||||
raise ValueError("invalid device session")
|
||||
session = proof["session_id"]
|
||||
if self.lifecycle and (proof.get("binding_owner") != owner
|
||||
or proof.get("binding_generation") != device["binding_generation"]):
|
||||
raise ValueError("stale binding session")
|
||||
except Exception:
|
||||
self.store.audit(owner, "dispatch", host, "identity_rejected")
|
||||
raise HTTPException(409, "device identity rejected") from None
|
||||
if not self.store.begin_turn(turn, owner, host):
|
||||
return {"status": "duplicate", "turn_id": turn, "result": "该请求已接收,不会重复执行。"}
|
||||
task = None
|
||||
if self.lifecycle:
|
||||
try:
|
||||
task = self.lifecycle.create_task(turn,owner,host,device["binding_generation"],session,
|
||||
self.worker_id,time.time()+self.s.request_timeout,chat_id,source_message_id)
|
||||
except ValueError:
|
||||
self.store.finish(turn,"unbound")
|
||||
return {"status":"unbound","turn_id":turn,"result":"绑定已变化,任务未下发。"}
|
||||
private = x25519.X25519PrivateKey.generate()
|
||||
future = asyncio.get_running_loop().create_future()
|
||||
self.pending[turn] = Pending(future, private, public, host, session)
|
||||
envelope = seal({"type": "instruction", "turn_id": turn, "owner": owner,
|
||||
payload = {"type": "instruction", "turn_id": turn, "owner": owner,
|
||||
"text": text, "read_only": True, "channel": "im",
|
||||
"reply_key": raw_public(private.public_key())}, proof["kx"], self.signer,
|
||||
host_id=host, session_id=session)
|
||||
"reply_key": raw_public(private.public_key())}
|
||||
if task:
|
||||
payload["task_context"] = {key:task[key] for key in ("environment","app_id","owner","host_id","binding_generation",
|
||||
"conversation_id","chat_id","source_message_id")}
|
||||
payload["task_context"].update(task_id=turn,channel="work-dm")
|
||||
try:
|
||||
envelope = seal(payload, proof["kx"], self.signer,host_id=host,session_id=session)
|
||||
except Exception:
|
||||
self.pending.pop(turn,None)
|
||||
self.store.finish(turn,"unknown")
|
||||
if task:
|
||||
self.lifecycle.transition(turn,"inflight","unknown")
|
||||
raise HTTPException(409,"invalid_device_crypto_session") from None
|
||||
key = f"hn:inflight:{host}:{turn}"
|
||||
try:
|
||||
await self.r.set(key, session, ex=self.s.request_timeout + 5)
|
||||
count = await self.r.publish("hn:down:" + host, json.dumps({"type": "instruction", "envelope": envelope}))
|
||||
packet = {"type":"instruction","envelope":envelope}
|
||||
if task:
|
||||
packet["routing"] = sign({"turn_id":turn,"host_id":host,"session_id":session,
|
||||
"environment":self.s.environment,"app_id":self.s.feishu_app_id,
|
||||
"binding_generation":task["binding_generation"],
|
||||
"envelope_hash":hashlib.sha256(canonical(envelope)).hexdigest()}, self.signer)
|
||||
count = await self.r.publish("hn:down:" + host, json.dumps(packet))
|
||||
if not count:
|
||||
self.store.finish(turn, "offline")
|
||||
if task:
|
||||
self.lifecycle.transition(turn,"inflight","offline")
|
||||
return {"status": "offline", "turn_id": turn, "result": "你的电脑当前离线。"}
|
||||
result = await asyncio.wait_for(future, timeout=self.s.request_timeout)
|
||||
self.store.finish(turn, "completed")
|
||||
self.store.audit(owner, "instruction", host, "completed")
|
||||
self.store.finish(turn, result.get("status","unknown") if task else "completed")
|
||||
if task and not source_message_id:
|
||||
self.lifecycle.transition(turn,"completed","finished")
|
||||
self.store.audit(owner, "instruction", host, result.get("status","unknown") if task else "completed")
|
||||
return result
|
||||
except asyncio.TimeoutError:
|
||||
self.store.finish(turn, "unknown")
|
||||
if task:
|
||||
self.lifecycle.transition(turn,"inflight","unknown")
|
||||
return {"status": "unknown", "turn_id": turn,
|
||||
"result": "等待结果超时,执行状态未知;不要直接重发有副作用的任务。"}
|
||||
finally:
|
||||
self.pending.pop(turn, None)
|
||||
await self.r.delete(key)
|
||||
if task:
|
||||
# Cancellation, publish failures and connector shutdown never replay work.
|
||||
with contextlib.suppress(Exception):
|
||||
self.lifecycle.transition(turn,"inflight","unknown")
|
||||
with contextlib.suppress(Exception):
|
||||
await self.r.delete(key)
|
||||
|
||||
|
||||
def create_app(settings=None, redis=None, platform=None):
|
||||
@@ -149,9 +192,15 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
credential_store = CredentialStore(store, CredentialKeyring.from_file(s.credential_keyring_file))
|
||||
credential_broker = CredentialBroker(credential_store, r, f, environment=s.environment, app_id=s.feishu_app_id)
|
||||
oauth = CredentialAPI(s, r, store, credential_broker)
|
||||
from .binding_state import BindingState
|
||||
from .device_lifecycle import DeviceLifecycle
|
||||
store.lifecycle = BindingState(store,s.environment,s.feishu_app_id)
|
||||
broker.lifecycle = store.lifecycle
|
||||
lifecycle = DeviceLifecycle(store.lifecycle,r,broker)
|
||||
else:
|
||||
oauth = OAuth(s, r, f)
|
||||
oauth.store = store
|
||||
lifecycle = None
|
||||
from .notifications import router as notification_router
|
||||
background = set()
|
||||
|
||||
@@ -165,6 +214,8 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
await r.ping()
|
||||
if lifecycle:
|
||||
await lifecycle.maintain()
|
||||
for row in store.db.execute("SELECT host_id,fingerprint,revoked FROM devices").fetchall():
|
||||
await r.set("hn:device:" + row["host_id"], row["fingerprint"])
|
||||
if row["revoked"]:
|
||||
@@ -180,12 +231,25 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
async with oauth.lock:
|
||||
oauth.prune()
|
||||
sweep_task = asyncio.create_task(sweep())
|
||||
async def maintenance():
|
||||
while True:
|
||||
await asyncio.sleep(5)
|
||||
try:
|
||||
await lifecycle.maintain()
|
||||
except Exception:
|
||||
# No credentials or raw exception text in background logs.
|
||||
pass
|
||||
lifecycle_task = asyncio.create_task(maintenance()) if lifecycle else None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for task in [reply_task, sweep_task, *background]:
|
||||
work = [reply_task,sweep_task,*background] + ([lifecycle_task] if lifecycle_task else [])
|
||||
for task in work:
|
||||
task.cancel()
|
||||
await asyncio.gather(reply_task, sweep_task, *background, return_exceptions=True)
|
||||
await asyncio.gather(*work, return_exceptions=True)
|
||||
if lifecycle:
|
||||
with contextlib.suppress(Exception):
|
||||
await r.delete("hn:task-worker:"+broker.worker_id)
|
||||
await pub.aclose()
|
||||
await f.close()
|
||||
store.close()
|
||||
@@ -201,6 +265,28 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
app.include_router(web_sessions.router)
|
||||
app.state.store, app.state.broker, app.state.oauth = store, broker, oauth
|
||||
app.state.platform, app.state.redis = f, r
|
||||
app.state.lifecycle = lifecycle
|
||||
|
||||
if lifecycle:
|
||||
from .device_lifecycle import AdmissionRequest
|
||||
from .credential_api import DeviceRequest
|
||||
@app.post("/work/v2/device/unbind")
|
||||
async def device_unbind(body: DeviceRequest, request: Request):
|
||||
await oauth.authenticate(body,request)
|
||||
try:
|
||||
store.lifecycle.invalidate_host(body.host_id,"unbind",expected_owner=body.binding_owner,
|
||||
expected_generation=body.binding_generation)
|
||||
except ValueError:
|
||||
raise HTTPException(409,"binding_changed_retry") from None
|
||||
lifecycle.cancel_waiters()
|
||||
propagated = await lifecycle.flush()
|
||||
return {"status":"unbound","binding_generation":store.device(body.host_id)["binding_generation"],
|
||||
"propagation":"published" if propagated else "pending"}
|
||||
|
||||
@app.post("/internal/devices/{host}/admission")
|
||||
async def admission(host: str, body: AdmissionRequest, request: Request):
|
||||
require_secret(request.headers.get("X-Router-Admission-Secret"),s.router_admission_secret)
|
||||
return sign(await lifecycle.admission(host,body),broker.signer)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request, exc):
|
||||
@@ -261,6 +347,14 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
admin(request)
|
||||
if not store.device(host):
|
||||
raise HTTPException(404, "device not found")
|
||||
if lifecycle:
|
||||
try:
|
||||
store.revoke(host)
|
||||
except ValueError:
|
||||
raise HTTPException(409,"binding_changed_retry") from None
|
||||
lifecycle.cancel_waiters()
|
||||
propagated = await lifecycle.flush()
|
||||
return {"status":"revoked","propagation":"published" if propagated else "pending"}
|
||||
# Persistent source of truth first; router block + disconnect follows.
|
||||
store.revoke(host)
|
||||
await r.incr("hn:web-generation:" + host)
|
||||
@@ -277,6 +371,8 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
async def reconcile(request: Request):
|
||||
"""Rebuild router admission metadata after Redis recovery; no dialogue data."""
|
||||
admin(request)
|
||||
if lifecycle:
|
||||
await lifecycle.maintain()
|
||||
rows = store.db.execute("SELECT host_id,fingerprint,revoked FROM devices").fetchall()
|
||||
for row in rows:
|
||||
await r.set("hn:device:" + row["host_id"], row["fingerprint"])
|
||||
@@ -290,6 +386,15 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
device = store.device(host)
|
||||
if not device:
|
||||
raise HTTPException(404, "device not found")
|
||||
if lifecycle:
|
||||
try:
|
||||
store.lifecycle.invalidate_host(host,"unbind")
|
||||
except ValueError:
|
||||
raise HTTPException(409,"binding_changed_retry") from None
|
||||
lifecycle.cancel_waiters()
|
||||
propagated = await lifecycle.flush()
|
||||
return {"status":"unbound","host_id":host,"revoked":bool(device["revoked"]),
|
||||
"propagation":"published" if propagated else "pending"}
|
||||
if device["owner"]:
|
||||
store.unbind(device["owner"])
|
||||
await r.incr("hn:web-generation:" + host)
|
||||
@@ -306,7 +411,13 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
return "你已绑定电脑,请先发送 /解绑。"
|
||||
await rate_limit(r, "binding:" + owner, 5)
|
||||
code = secrets.token_urlsafe(12)
|
||||
await r.set("hn:bind:" + hashlib.sha256(code.encode()).hexdigest(), owner, ex=300)
|
||||
if lifecycle:
|
||||
try:
|
||||
store.lifecycle.issue_code(owner,code)
|
||||
except ValueError:
|
||||
return "你已绑定电脑,请先发送 /解绑。"
|
||||
else:
|
||||
await r.set("hn:bind:" + hashlib.sha256(code.encode()).hexdigest(), owner, ex=300)
|
||||
return "请在本人电脑的桥接客户端输入绑定码(5分钟有效):" + code
|
||||
|
||||
@app.post("/device/bind")
|
||||
@@ -325,6 +436,16 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
raise ValueError("invalid code")
|
||||
except Exception:
|
||||
raise HTTPException(403, "invalid device proof") from None
|
||||
if lifecycle:
|
||||
try:
|
||||
owner = store.lifecycle.consume_code(body.host_id,code)
|
||||
except LookupError:
|
||||
raise HTTPException(404,"binding code unavailable") from None
|
||||
except (*INTEGRITY_ERRORS,ValueError):
|
||||
raise HTTPException(409,"owner/device already bound; request new code") from None
|
||||
return sign({"host_id":body.host_id,"owner":owner,
|
||||
"binding_generation":store.device(body.host_id)["binding_generation"],
|
||||
"ts":int(time.time())},broker.signer)
|
||||
key = "hn:bind:" + hashlib.sha256(code.encode()).hexdigest()
|
||||
owner = await r.get(key)
|
||||
if not owner:
|
||||
@@ -365,11 +486,15 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
owner = sender.get("sender_id", {}).get("open_id")
|
||||
message_id = message.get("message_id", "")
|
||||
if (message.get("chat_type") != "p2p" or message.get("message_type") != "text"
|
||||
or sender.get("sender_type") != "user" or not owner
|
||||
or sender.get("sender_type") != "user" or not isinstance(owner,str)
|
||||
or not 0 < len(owner) <= 128 or any(ord(c) < 32 for c in owner)
|
||||
or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", message_id)):
|
||||
return
|
||||
if header.get("app_id") != s.feishu_app_id:
|
||||
return
|
||||
if lifecycle and (not isinstance(message.get("chat_id"),str) or not 0 < len(message["chat_id"]) <= 128
|
||||
or any(ord(c) < 32 for c in message["chat_id"])):
|
||||
return
|
||||
try:
|
||||
text = json.loads(message["content"])["text"].strip()
|
||||
if not text or len(text) > 8000:
|
||||
@@ -383,7 +508,10 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
elif text == "/解绑":
|
||||
old = store.by_owner(owner)
|
||||
store.unbind(owner)
|
||||
if old:
|
||||
if lifecycle:
|
||||
lifecycle.cancel_waiters()
|
||||
await lifecycle.flush()
|
||||
elif old:
|
||||
await r.incr("hn:web-generation:" + old["host_id"])
|
||||
await r.publish("hn:kill:" + old["host_id"], "unbind")
|
||||
for item in broker.pending.values():
|
||||
@@ -391,11 +519,25 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
item.future.set_result({"status": "unbound", "result": "设备已解绑,结果已丢弃。"})
|
||||
result = "已解绑;重新绑定后请更新本机绑定信息。"
|
||||
elif text == "/主机名" or text.startswith("/回显 "):
|
||||
result = (await broker.dispatch(owner, text, "msg_" + hashlib.sha256(message_id.encode()).hexdigest()))["result"]
|
||||
result = (await broker.dispatch(owner, text, "msg_" + hashlib.sha256(message_id.encode()).hexdigest(),
|
||||
**({"chat_id":message["chat_id"],"source_message_id":message_id} if lifecycle else {})))["result"]
|
||||
else:
|
||||
instruction = text if text.startswith("/助手 ") else "/助手 " + text
|
||||
result = (await broker.dispatch(owner, instruction, "msg_" + hashlib.sha256(message_id.encode()).hexdigest()))["result"]
|
||||
await f.reply(message_id, result[:s.max_result])
|
||||
result = (await broker.dispatch(owner, instruction, "msg_" + hashlib.sha256(message_id.encode()).hexdigest(),
|
||||
**({"chat_id":message["chat_id"],"source_message_id":message_id} if lifecycle else {})))["result"]
|
||||
turn = "msg_"+hashlib.sha256(message_id.encode()).hexdigest()
|
||||
task = store.lifecycle.task(turn) if lifecycle else None
|
||||
if task and not store.lifecycle.claim_reply(turn,owner,message_id):
|
||||
store.finish(event_turn,"result_discarded")
|
||||
return
|
||||
try:
|
||||
await f.reply(message_id, result[:s.max_result])
|
||||
except Exception:
|
||||
if task:
|
||||
store.lifecycle.transition(turn,"replying","reply_unknown")
|
||||
raise
|
||||
if task:
|
||||
store.lifecycle.transition(turn,"replying","replied")
|
||||
store.finish(event_turn, "replied")
|
||||
except Exception:
|
||||
store.audit(owner, "event", message_id, "failed_or_unknown")
|
||||
|
||||
Reference in New Issue
Block a user