feat(lifecycle): add stage 4 authoritative unbind and durable task routing

Unify broker-mode binding changes in database transactions: advance device
generation, invalidate pending authorizations/tasks and prior pairing-code
epochs, and persist audit plus invalidation outbox. Add authenticated device
unbind alongside robot/admin paths. Pairing codes are hashed, generation-
fenced and consumed atomically with binding.

Replay Redis invalidation effects without undoing committed unbind. Guard
online cleanup and delayed kill events by generation. Introduce a separate
router admission credential with no platform-event injection authority.
Router verifies signed nonce-bound internal admission on handshake, task
traffic and periodic checks; reject stale sessions and control-plane outages.

Persist task owner/device/generation/session/worker and original message
mapping. Sign routing metadata, verify late results, claim replies once and
mark ambiguous/crashed tasks unknown instead of replaying writes. Fence
notifications across same-user rebind. Release in-memory slots/waiters even
when database or Redis cleanup fails.

Validation: 213 passed, 3 skipped (real PostgreSQL not configured), 30 existing
uvicorn/websockets deprecation warnings. New lifecycle coverage: 28 tests,
including real loopback WebSocket isolation, reverse completion, missed kill,
periodic unbind checks, outbox recovery, transaction rollback and cleanup
failures. Syntax, docs and staged diff checks pass.

Default legacy mode remains unchanged. Broker mode now requires coordinated
router admission configuration and a v2 client handshake. No production ECS
configuration, real credentials or client installer was changed. PostgreSQL,
real Redis multi-node, ARM64 and actual Work/client rollout remain stage 5.
This commit is contained in:
ex_tianzy
2026-11-04 11:43:53 +08:00
parent 93bfbca83a
commit d1ab834ecb
13 changed files with 1128 additions and 29 deletions
+182
View File
@@ -0,0 +1,182 @@
"""Transactional broker-mode binding, pairing, invalidation and task metadata.
No message bodies or credential values are stored here. All mutating binding
paths in a broker connector use this object through Store's lifecycle hook.
"""
import hashlib
import time
import uuid
class BindingState:
def __init__(self, store, environment, app_id):
self.store, self.db, self.environment, self.app_id = store, store.db, environment, app_id
with self.db:
self.db.execute("""CREATE TABLE IF NOT EXISTS work_owner_epochs_v2(
environment TEXT NOT NULL,app_id TEXT NOT NULL,owner TEXT NOT NULL,epoch BIGINT NOT NULL,
PRIMARY KEY(environment,app_id,owner))""")
self.db.execute("""CREATE TABLE IF NOT EXISTS work_binding_codes_v2(
code_hash TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
owner TEXT NOT NULL,owner_epoch BIGINT NOT NULL,expires_at DOUBLE PRECISION NOT NULL,
consumed INTEGER NOT NULL DEFAULT 0)""")
self.db.execute("""CREATE TABLE IF NOT EXISTS work_binding_events_v2(
event_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,host_id TEXT NOT NULL,
owner TEXT NOT NULL,binding_generation BIGINT NOT NULL,reason TEXT NOT NULL,
created_at DOUBLE PRECISION NOT NULL,delivered INTEGER NOT NULL DEFAULT 0)""")
self.db.execute("""CREATE TABLE IF NOT EXISTS work_tasks_v2(
turn_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,owner TEXT NOT NULL,
host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,session_id TEXT NOT NULL,
worker_id TEXT NOT NULL,conversation_id TEXT NOT NULL,chat_id TEXT NOT NULL,
source_message_id TEXT NOT NULL,status TEXT NOT NULL,deadline DOUBLE PRECISION NOT NULL,
created_at DOUBLE PRECISION NOT NULL,updated_at DOUBLE PRECISION NOT NULL)""")
self.db.execute("""CREATE INDEX IF NOT EXISTS work_binding_events_pending_v2
ON work_binding_events_v2(environment,app_id,delivered,created_at)""")
self.db.execute("""CREATE INDEX IF NOT EXISTS work_tasks_active_v2
ON work_tasks_v2(environment,app_id,status,updated_at)""")
self.db.execute("""CREATE INDEX IF NOT EXISTS work_tasks_binding_v2
ON work_tasks_v2(environment,app_id,host_id,binding_generation)""")
def _owner_lock(self, owner, bump=False):
identity = (self.environment, self.app_id, owner)
self.db.execute("INSERT INTO work_owner_epochs_v2 VALUES(?,?,?,0) ON CONFLICT DO NOTHING", identity)
self.db.execute("UPDATE work_owner_epochs_v2 SET epoch=epoch+? WHERE environment=? AND app_id=? AND owner=?",
(1 if bump else 0, *identity))
return self.db.execute("SELECT epoch FROM work_owner_epochs_v2 WHERE environment=? AND app_id=? AND owner=?",
identity).fetchone()["epoch"]
def _bind(self, host, owner):
count = self.db.execute("""UPDATE devices SET owner=?,binding_generation=binding_generation+1
WHERE host_id=? AND owner IS NULL AND revoked=0""", (owner, host)).rowcount
if count != 1:
raise ValueError("device unavailable or already bound")
def bind(self, host, owner):
with self.db:
self._owner_lock(owner)
self._bind(host, owner)
def issue_code(self, owner, code, ttl=300):
with self.db:
epoch = self._owner_lock(owner)
if self.db.execute("SELECT host_id FROM devices WHERE owner=?", (owner,)).fetchone():
raise ValueError("owner already bound")
self.db.execute("INSERT INTO work_binding_codes_v2 VALUES(?,?,?,?,?,?,0)",
(hashlib.sha256(code.encode()).hexdigest(), self.environment, self.app_id, owner, epoch, time.time()+ttl))
def consume_code(self, host, code):
digest = hashlib.sha256(code.encode()).hexdigest()
row = self.db.execute("SELECT * FROM work_binding_codes_v2 WHERE code_hash=? AND environment=? AND app_id=?",
(digest, self.environment, self.app_id)).fetchone()
if not row:
raise LookupError("binding code unavailable")
with self.db:
epoch = self._owner_lock(row["owner"])
count = self.db.execute("""UPDATE work_binding_codes_v2 SET consumed=1
WHERE code_hash=? AND owner_epoch=? AND consumed=0 AND expires_at>?""",
(digest, epoch, time.time())).rowcount
if count != 1:
raise LookupError("binding code unavailable")
self._bind(host, row["owner"])
return row["owner"]
def _cancel(self, device, reason):
owner = device["owner"] or ""
generation = device["binding_generation"]
self.db.execute("""UPDATE work_authorizations_v2 SET status='invalidated'
WHERE environment=? AND app_id=? AND host_id=? AND binding_generation<=?
AND status IN ('waiting','exchanging')""",
(self.environment, self.app_id, device["host_id"], generation))
self.db.execute("""UPDATE work_tasks_v2 SET status=?,updated_at=?
WHERE environment=? AND app_id=? AND host_id=? AND binding_generation<=?
AND status IN ('inflight','completed','replying')""",
("revoked" if reason == "revoke" else "unbound", time.time(), self.environment,
self.app_id, device["host_id"], generation))
event = uuid.uuid4().hex
self.db.execute("INSERT INTO work_binding_events_v2 VALUES(?,?,?,?,?,?,?,?,0)",
(event, self.environment, self.app_id, device["host_id"], owner, generation+1, reason, time.time()))
self.db.execute("INSERT INTO audit(ts,actor,action,resource,result) VALUES(?,?,?,?,?)",
(time.time(), owner or "admin", "binding_"+reason, device["host_id"], "committed"))
def unbind(self, owner):
with self.db:
self._owner_lock(owner, bump=True) # Invalidates pre-unbind codes even if already unbound.
device = self.db.execute("SELECT * FROM devices WHERE owner=?", (owner,)).fetchone()
if device:
self.db.execute("UPDATE devices SET owner=NULL,binding_generation=binding_generation+1 WHERE host_id=?",
(device["host_id"],))
self._cancel(dict(device), "unbind")
def invalidate_host(self, host, reason, *, expected_owner=None, expected_generation=None):
if reason not in {"unbind", "revoke"}:
raise ValueError("invalid binding change")
device = self.store.device(host)
if not device:
raise LookupError("device not found")
if ((expected_owner is not None and device["owner"] != expected_owner)
or (expected_generation is not None and device["binding_generation"] != expected_generation)):
raise ValueError("binding changed; retry")
if reason == "unbind" and not device["owner"]:
return
if reason == "revoke" and device["revoked"]:
return
with self.db:
if device["owner"]:
self._owner_lock(device["owner"], bump=True)
mutation = "owner=NULL" if reason == "unbind" else "revoked=1"
count = self.db.execute(f"""UPDATE devices SET {mutation},binding_generation=binding_generation+1
WHERE host_id=? AND binding_generation=? AND COALESCE(owner,'')=? AND revoked=?""",
(host, device["binding_generation"], device["owner"] or "", device["revoked"])).rowcount
if count != 1:
raise ValueError("binding changed; retry")
self._cancel(device, reason)
def task(self, turn):
row = self.db.execute("SELECT * FROM work_tasks_v2 WHERE turn_id=? AND environment=? AND app_id=?",
(turn, self.environment, self.app_id)).fetchone()
return dict(row) if row else None
def task_current(self, task):
device = self.store.device(task["host_id"]) if task else None
return bool(device and not device["revoked"] and device["owner"] == task["owner"]
and device["binding_generation"] == task["binding_generation"]
and task["deadline"] > time.time())
def create_task(self, turn, owner, host, generation, session, worker, deadline, chat_id="", message_id=""):
conversation = hashlib.sha256((self.environment+"\0"+self.app_id+"\0"+owner+"\0"+chat_id+"\0work-dm").encode()).hexdigest()
with self.db:
self._owner_lock(owner)
device = self.store.device(host)
if not device or device["revoked"] or device["owner"] != owner or device["binding_generation"] != generation:
raise ValueError("binding changed")
count = self.db.execute("""INSERT INTO work_tasks_v2 VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
ON CONFLICT DO NOTHING""", (turn,self.environment,self.app_id,owner,host,generation,session,
worker,conversation,chat_id,message_id,"inflight",deadline,time.time(),time.time())).rowcount
if count != 1:
raise ValueError("task already recorded")
return self.task(turn)
def transition(self, turn, previous, status):
with self.db:
return self.db.execute("""UPDATE work_tasks_v2 SET status=?,updated_at=?
WHERE turn_id=? AND environment=? AND app_id=? AND status=?""",
(status,time.time(),turn,self.environment,self.app_id,previous)).rowcount == 1
def complete(self, turn, host, session, worker):
with self.db:
task = self.task(turn)
if not task:
return False
self._owner_lock(task["owner"])
if not self.task_current(task) or (task["host_id"],task["session_id"],task["worker_id"],task["status"]) != (host,session,worker,"inflight"):
return False
return self.db.execute("UPDATE work_tasks_v2 SET status='completed',updated_at=? WHERE turn_id=? AND status='inflight'",
(time.time(),turn)).rowcount == 1
def claim_reply(self, turn, owner, message_id):
with self.db:
self._owner_lock(owner)
task = self.task(turn)
if not self.task_current(task) or (task["owner"],task["source_message_id"],task["status"]) != (owner,message_id,"completed"):
return False
return self.db.execute("UPDATE work_tasks_v2 SET status='replying',updated_at=? WHERE turn_id=? AND status='completed'",
(time.time(),turn)).rowcount == 1
+12
View File
@@ -44,11 +44,17 @@ class Settings(BaseSettings):
credential_mode: str = "legacy"
credential_keyring_file: Path | None = None
credential_permission_profiles: dict[str, list[str]] = {}
connector_internal_url: str = "http://connector:8000"
router_admission_secret: str = ""
@model_validator(mode="after")
def validate_runtime(self):
if self.credential_mode not in {"legacy", "broker"}:
raise ValueError("invalid credential mode")
if self.credential_mode == "broker" and self.role in {"connector","router"}:
if (len(self.router_admission_secret) < 32 or self.router_admission_secret in {
self.internal_secret,self.admin_secret,self.feishu_app_secret}):
raise ValueError("broker mode requires an independent router admission credential")
for profile, scopes in self.credential_permission_profiles.items():
if (not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", profile) or not scopes or len(scopes) > 128
or any(not re.fullmatch(r"[A-Za-z0-9_:.-]{1,256}", scope) for scope in scopes)):
@@ -63,6 +69,12 @@ class Settings(BaseSettings):
if (authorize.scheme != "https" or not authorize.hostname or authorize.username or authorize.password
or authorize.query or authorize.fragment):
raise ValueError("broker mode requires a fixed HTTPS authorization endpoint")
if self.credential_mode == "broker" and self.role == "router":
target = urlsplit(self.connector_internal_url)
if (not self.feishu_app_id or target.scheme not in {"http","https"}
or not target.hostname or target.username or target.password or target.query or target.fragment
or target.path not in {"","/"}):
raise ValueError("broker router requires authenticated fixed internal connector")
if self.environment not in {"production", "test", "development"}:
raise ValueError("invalid environment")
if self.role == "connector" and min(len(self.admin_secret), len(self.internal_secret)) < 32:
+159 -17
View File
@@ -24,7 +24,7 @@ from redis.asyncio import Redis
from .common import BodyLimitMiddleware, no_cache, peer, rate_limit, require_secret
from .config import Settings
from .crypto import (fingerprint, load_private, open_envelope, raw_public, seal,
from .crypto import (canonical, fingerprint, load_private, open_envelope, raw_public, seal,
sign, verified, verify_device)
from .oauth import OAuth
from .platform import Feishu, PlatformError
@@ -62,6 +62,8 @@ class Broker:
self.s, self.r, self.store = settings, redis, store
self.signer = load_private(settings.connector_key)
self.pending: dict[str, Pending] = {}
self.worker_id = uuid.uuid4().hex
self.lifecycle = None
async def replies(self, pub):
async for message in pub.listen():
@@ -79,12 +81,14 @@ class Broker:
result = plain.get("result", "")
if not isinstance(result, str) or len(result) > self.s.max_result:
continue
if self.lifecycle and not self.lifecycle.complete(data["turn_id"],item.host,item.session,self.worker_id):
continue
item.future.set_result({"status": "completed", "turn_id": data["turn_id"], "result": result})
except Exception:
# Invalid packets never consume a pending result; do not log packet contents.
continue
async def dispatch(self, owner, text, turn):
async def dispatch(self, owner, text, turn, *, chat_id="", source_message_id=""):
device = self.store.by_owner(owner)
if not device:
return {"status": "unavailable", "turn_id": turn, "result": "未绑定设备或设备不可用。"}
@@ -104,36 +108,75 @@ class Broker:
if proof["host_id"] != host or proof["challenge"] != presence["challenge"]:
raise ValueError("invalid device session")
session = proof["session_id"]
if self.lifecycle and (proof.get("binding_owner") != owner
or proof.get("binding_generation") != device["binding_generation"]):
raise ValueError("stale binding session")
except Exception:
self.store.audit(owner, "dispatch", host, "identity_rejected")
raise HTTPException(409, "device identity rejected") from None
if not self.store.begin_turn(turn, owner, host):
return {"status": "duplicate", "turn_id": turn, "result": "该请求已接收,不会重复执行。"}
task = None
if self.lifecycle:
try:
task = self.lifecycle.create_task(turn,owner,host,device["binding_generation"],session,
self.worker_id,time.time()+self.s.request_timeout,chat_id,source_message_id)
except ValueError:
self.store.finish(turn,"unbound")
return {"status":"unbound","turn_id":turn,"result":"绑定已变化,任务未下发。"}
private = x25519.X25519PrivateKey.generate()
future = asyncio.get_running_loop().create_future()
self.pending[turn] = Pending(future, private, public, host, session)
envelope = seal({"type": "instruction", "turn_id": turn, "owner": owner,
payload = {"type": "instruction", "turn_id": turn, "owner": owner,
"text": text, "read_only": True, "channel": "im",
"reply_key": raw_public(private.public_key())}, proof["kx"], self.signer,
host_id=host, session_id=session)
"reply_key": raw_public(private.public_key())}
if task:
payload["task_context"] = {key:task[key] for key in ("environment","app_id","owner","host_id","binding_generation",
"conversation_id","chat_id","source_message_id")}
payload["task_context"].update(task_id=turn,channel="work-dm")
try:
envelope = seal(payload, proof["kx"], self.signer,host_id=host,session_id=session)
except Exception:
self.pending.pop(turn,None)
self.store.finish(turn,"unknown")
if task:
self.lifecycle.transition(turn,"inflight","unknown")
raise HTTPException(409,"invalid_device_crypto_session") from None
key = f"hn:inflight:{host}:{turn}"
try:
await self.r.set(key, session, ex=self.s.request_timeout + 5)
count = await self.r.publish("hn:down:" + host, json.dumps({"type": "instruction", "envelope": envelope}))
packet = {"type":"instruction","envelope":envelope}
if task:
packet["routing"] = sign({"turn_id":turn,"host_id":host,"session_id":session,
"environment":self.s.environment,"app_id":self.s.feishu_app_id,
"binding_generation":task["binding_generation"],
"envelope_hash":hashlib.sha256(canonical(envelope)).hexdigest()}, self.signer)
count = await self.r.publish("hn:down:" + host, json.dumps(packet))
if not count:
self.store.finish(turn, "offline")
if task:
self.lifecycle.transition(turn,"inflight","offline")
return {"status": "offline", "turn_id": turn, "result": "你的电脑当前离线。"}
result = await asyncio.wait_for(future, timeout=self.s.request_timeout)
self.store.finish(turn, "completed")
self.store.audit(owner, "instruction", host, "completed")
self.store.finish(turn, result.get("status","unknown") if task else "completed")
if task and not source_message_id:
self.lifecycle.transition(turn,"completed","finished")
self.store.audit(owner, "instruction", host, result.get("status","unknown") if task else "completed")
return result
except asyncio.TimeoutError:
self.store.finish(turn, "unknown")
if task:
self.lifecycle.transition(turn,"inflight","unknown")
return {"status": "unknown", "turn_id": turn,
"result": "等待结果超时,执行状态未知;不要直接重发有副作用的任务。"}
finally:
self.pending.pop(turn, None)
await self.r.delete(key)
if task:
# Cancellation, publish failures and connector shutdown never replay work.
with contextlib.suppress(Exception):
self.lifecycle.transition(turn,"inflight","unknown")
with contextlib.suppress(Exception):
await self.r.delete(key)
def create_app(settings=None, redis=None, platform=None):
@@ -149,9 +192,15 @@ def create_app(settings=None, redis=None, platform=None):
credential_store = CredentialStore(store, CredentialKeyring.from_file(s.credential_keyring_file))
credential_broker = CredentialBroker(credential_store, r, f, environment=s.environment, app_id=s.feishu_app_id)
oauth = CredentialAPI(s, r, store, credential_broker)
from .binding_state import BindingState
from .device_lifecycle import DeviceLifecycle
store.lifecycle = BindingState(store,s.environment,s.feishu_app_id)
broker.lifecycle = store.lifecycle
lifecycle = DeviceLifecycle(store.lifecycle,r,broker)
else:
oauth = OAuth(s, r, f)
oauth.store = store
lifecycle = None
from .notifications import router as notification_router
background = set()
@@ -165,6 +214,8 @@ def create_app(settings=None, redis=None, platform=None):
@asynccontextmanager
async def lifespan(app):
await r.ping()
if lifecycle:
await lifecycle.maintain()
for row in store.db.execute("SELECT host_id,fingerprint,revoked FROM devices").fetchall():
await r.set("hn:device:" + row["host_id"], row["fingerprint"])
if row["revoked"]:
@@ -180,12 +231,25 @@ def create_app(settings=None, redis=None, platform=None):
async with oauth.lock:
oauth.prune()
sweep_task = asyncio.create_task(sweep())
async def maintenance():
while True:
await asyncio.sleep(5)
try:
await lifecycle.maintain()
except Exception:
# No credentials or raw exception text in background logs.
pass
lifecycle_task = asyncio.create_task(maintenance()) if lifecycle else None
try:
yield
finally:
for task in [reply_task, sweep_task, *background]:
work = [reply_task,sweep_task,*background] + ([lifecycle_task] if lifecycle_task else [])
for task in work:
task.cancel()
await asyncio.gather(reply_task, sweep_task, *background, return_exceptions=True)
await asyncio.gather(*work, return_exceptions=True)
if lifecycle:
with contextlib.suppress(Exception):
await r.delete("hn:task-worker:"+broker.worker_id)
await pub.aclose()
await f.close()
store.close()
@@ -201,6 +265,28 @@ def create_app(settings=None, redis=None, platform=None):
app.include_router(web_sessions.router)
app.state.store, app.state.broker, app.state.oauth = store, broker, oauth
app.state.platform, app.state.redis = f, r
app.state.lifecycle = lifecycle
if lifecycle:
from .device_lifecycle import AdmissionRequest
from .credential_api import DeviceRequest
@app.post("/work/v2/device/unbind")
async def device_unbind(body: DeviceRequest, request: Request):
await oauth.authenticate(body,request)
try:
store.lifecycle.invalidate_host(body.host_id,"unbind",expected_owner=body.binding_owner,
expected_generation=body.binding_generation)
except ValueError:
raise HTTPException(409,"binding_changed_retry") from None
lifecycle.cancel_waiters()
propagated = await lifecycle.flush()
return {"status":"unbound","binding_generation":store.device(body.host_id)["binding_generation"],
"propagation":"published" if propagated else "pending"}
@app.post("/internal/devices/{host}/admission")
async def admission(host: str, body: AdmissionRequest, request: Request):
require_secret(request.headers.get("X-Router-Admission-Secret"),s.router_admission_secret)
return sign(await lifecycle.admission(host,body),broker.signer)
@app.exception_handler(RequestValidationError)
async def validation_error(request, exc):
@@ -261,6 +347,14 @@ def create_app(settings=None, redis=None, platform=None):
admin(request)
if not store.device(host):
raise HTTPException(404, "device not found")
if lifecycle:
try:
store.revoke(host)
except ValueError:
raise HTTPException(409,"binding_changed_retry") from None
lifecycle.cancel_waiters()
propagated = await lifecycle.flush()
return {"status":"revoked","propagation":"published" if propagated else "pending"}
# Persistent source of truth first; router block + disconnect follows.
store.revoke(host)
await r.incr("hn:web-generation:" + host)
@@ -277,6 +371,8 @@ def create_app(settings=None, redis=None, platform=None):
async def reconcile(request: Request):
"""Rebuild router admission metadata after Redis recovery; no dialogue data."""
admin(request)
if lifecycle:
await lifecycle.maintain()
rows = store.db.execute("SELECT host_id,fingerprint,revoked FROM devices").fetchall()
for row in rows:
await r.set("hn:device:" + row["host_id"], row["fingerprint"])
@@ -290,6 +386,15 @@ def create_app(settings=None, redis=None, platform=None):
device = store.device(host)
if not device:
raise HTTPException(404, "device not found")
if lifecycle:
try:
store.lifecycle.invalidate_host(host,"unbind")
except ValueError:
raise HTTPException(409,"binding_changed_retry") from None
lifecycle.cancel_waiters()
propagated = await lifecycle.flush()
return {"status":"unbound","host_id":host,"revoked":bool(device["revoked"]),
"propagation":"published" if propagated else "pending"}
if device["owner"]:
store.unbind(device["owner"])
await r.incr("hn:web-generation:" + host)
@@ -306,7 +411,13 @@ def create_app(settings=None, redis=None, platform=None):
return "你已绑定电脑,请先发送 /解绑。"
await rate_limit(r, "binding:" + owner, 5)
code = secrets.token_urlsafe(12)
await r.set("hn:bind:" + hashlib.sha256(code.encode()).hexdigest(), owner, ex=300)
if lifecycle:
try:
store.lifecycle.issue_code(owner,code)
except ValueError:
return "你已绑定电脑,请先发送 /解绑。"
else:
await r.set("hn:bind:" + hashlib.sha256(code.encode()).hexdigest(), owner, ex=300)
return "请在本人电脑的桥接客户端输入绑定码(5分钟有效):" + code
@app.post("/device/bind")
@@ -325,6 +436,16 @@ def create_app(settings=None, redis=None, platform=None):
raise ValueError("invalid code")
except Exception:
raise HTTPException(403, "invalid device proof") from None
if lifecycle:
try:
owner = store.lifecycle.consume_code(body.host_id,code)
except LookupError:
raise HTTPException(404,"binding code unavailable") from None
except (*INTEGRITY_ERRORS,ValueError):
raise HTTPException(409,"owner/device already bound; request new code") from None
return sign({"host_id":body.host_id,"owner":owner,
"binding_generation":store.device(body.host_id)["binding_generation"],
"ts":int(time.time())},broker.signer)
key = "hn:bind:" + hashlib.sha256(code.encode()).hexdigest()
owner = await r.get(key)
if not owner:
@@ -365,11 +486,15 @@ def create_app(settings=None, redis=None, platform=None):
owner = sender.get("sender_id", {}).get("open_id")
message_id = message.get("message_id", "")
if (message.get("chat_type") != "p2p" or message.get("message_type") != "text"
or sender.get("sender_type") != "user" or not owner
or sender.get("sender_type") != "user" or not isinstance(owner,str)
or not 0 < len(owner) <= 128 or any(ord(c) < 32 for c in owner)
or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", message_id)):
return
if header.get("app_id") != s.feishu_app_id:
return
if lifecycle and (not isinstance(message.get("chat_id"),str) or not 0 < len(message["chat_id"]) <= 128
or any(ord(c) < 32 for c in message["chat_id"])):
return
try:
text = json.loads(message["content"])["text"].strip()
if not text or len(text) > 8000:
@@ -383,7 +508,10 @@ def create_app(settings=None, redis=None, platform=None):
elif text == "/解绑":
old = store.by_owner(owner)
store.unbind(owner)
if old:
if lifecycle:
lifecycle.cancel_waiters()
await lifecycle.flush()
elif old:
await r.incr("hn:web-generation:" + old["host_id"])
await r.publish("hn:kill:" + old["host_id"], "unbind")
for item in broker.pending.values():
@@ -391,11 +519,25 @@ def create_app(settings=None, redis=None, platform=None):
item.future.set_result({"status": "unbound", "result": "设备已解绑,结果已丢弃。"})
result = "已解绑;重新绑定后请更新本机绑定信息。"
elif text == "/主机名" or text.startswith("/回显 "):
result = (await broker.dispatch(owner, text, "msg_" + hashlib.sha256(message_id.encode()).hexdigest()))["result"]
result = (await broker.dispatch(owner, text, "msg_" + hashlib.sha256(message_id.encode()).hexdigest(),
**({"chat_id":message["chat_id"],"source_message_id":message_id} if lifecycle else {})))["result"]
else:
instruction = text if text.startswith("/助手 ") else "/助手 " + text
result = (await broker.dispatch(owner, instruction, "msg_" + hashlib.sha256(message_id.encode()).hexdigest()))["result"]
await f.reply(message_id, result[:s.max_result])
result = (await broker.dispatch(owner, instruction, "msg_" + hashlib.sha256(message_id.encode()).hexdigest(),
**({"chat_id":message["chat_id"],"source_message_id":message_id} if lifecycle else {})))["result"]
turn = "msg_"+hashlib.sha256(message_id.encode()).hexdigest()
task = store.lifecycle.task(turn) if lifecycle else None
if task and not store.lifecycle.claim_reply(turn,owner,message_id):
store.finish(event_turn,"result_discarded")
return
try:
await f.reply(message_id, result[:s.max_result])
except Exception:
if task:
store.lifecycle.transition(turn,"replying","reply_unknown")
raise
if task:
store.lifecycle.transition(turn,"replying","replied")
store.finish(event_turn, "replied")
except Exception:
store.audit(owner, "event", message_id, "failed_or_unknown")
+104
View File
@@ -0,0 +1,104 @@
"""Broker-mode invalidation outbox, worker liveness and internal admission."""
import json
import time
from fastapi import HTTPException
from pydantic import BaseModel, ConfigDict, Field
from .router import compare_delete
class AdmissionRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
nonce: str = Field(pattern=r"^[a-f0-9]{32}$")
turn_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9_-]{1,128}$")
session_id: str | None = Field(default=None, min_length=1, max_length=64)
class DeviceLifecycle:
def __init__(self, state, redis, broker):
self.state, self.r, self.broker = state, redis, broker
async def flush(self):
rows = self.state.db.execute("""SELECT * FROM work_binding_events_v2
WHERE environment=? AND app_id=? AND delivered=0 ORDER BY created_at LIMIT 100""",
(self.state.environment,self.state.app_id)).fetchall()
for event in rows:
try:
device = self.state.store.device(event["host_id"])
if device and device["revoked"]:
await self.r.set("hn:revoked:"+event["host_id"], "1")
key = "hn:online:"+event["host_id"]
presence = await self.r.get(key)
if presence:
try:
generation = json.loads(presence)["proof"]["binding_generation"]
if type(generation) is not int:
generation = -1
except (ValueError, KeyError, TypeError):
generation = -1
if generation < event["binding_generation"]:
await compare_delete(self.r, key, presence)
# Newer sessions ignore a delayed invalidation for older generations.
await self.r.publish("hn:kill:"+event["host_id"], json.dumps({
"reason": event["reason"], "binding_generation": event["binding_generation"]}))
with self.state.db:
self.state.db.execute("UPDATE work_binding_events_v2 SET delivered=1 WHERE event_id=?",
(event["event_id"],))
except Exception:
# DB remains authoritative; leave durable work for another attempt.
return False
self.cancel_waiters()
return not self.state.db.execute("""SELECT event_id FROM work_binding_events_v2
WHERE environment=? AND app_id=? AND delivered=0 LIMIT 1""",
(self.state.environment,self.state.app_id)).fetchone()
def cancel_waiters(self):
for turn, pending in list(self.broker.pending.items()):
task = self.state.task(turn)
if task and task["status"] not in {"inflight", "completed", "replying"} and not pending.future.done():
pending.future.set_result({"status":task["status"], "turn_id":turn,
"result":"任务已结束,旧结果不会继续投递。"})
async def maintain(self):
await self.r.set("hn:task-worker:"+self.broker.worker_id,"alive",ex=30)
await self.flush()
rows = self.state.db.execute("""SELECT * FROM work_tasks_v2
WHERE environment=? AND app_id=? AND status IN ('inflight','completed','replying')
ORDER BY updated_at LIMIT 100""", (self.state.environment,self.state.app_id)).fetchall()
for task in rows:
if task["deadline"] <= time.time() or not await self.r.exists("hn:task-worker:"+task["worker_id"]):
self.state.transition(task["turn_id"],task["status"],
"reply_unknown" if task["status"] == "replying" else "unknown")
else:
# Rotate bounded scanning instead of starving later tasks behind
# the same first 100 healthy workers on every maintenance pass.
with self.state.db:
self.state.db.execute("UPDATE work_tasks_v2 SET updated_at=? WHERE turn_id=? AND status=?",
(time.time(),task["turn_id"],task["status"]))
self.cancel_waiters()
async def admission(self, host, body):
device = self.state.store.device(host)
if not device or device["revoked"] or not device["owner"]:
raise HTTPException(403,"device_unbound_or_revoked")
task = None
if body.turn_id:
task = self.state.task(body.turn_id)
if (not self.state.task_current(task) or task["host_id"] != host
or task["status"] != "inflight" or task["session_id"] != body.session_id
or not await self.r.exists("hn:task-worker:"+task["worker_id"])):
raise HTTPException(403,"task_not_active")
# Re-read after Redis await; never rely on router's cached admission.
latest = self.state.store.device(host)
if (not latest or latest["revoked"] or latest["owner"] != device["owner"]
or latest["binding_generation"] != device["binding_generation"]):
raise HTTPException(403,"stale_binding")
if task:
current = self.state.task(body.turn_id)
if not self.state.task_current(current) or current["status"] != "inflight":
raise HTTPException(403,"task_not_active")
return {"protocol_version":2,"environment":self.state.environment,"app_id":self.state.app_id,
"host_id":host,"owner":latest["owner"],"binding_generation":latest["binding_generation"],
"fingerprint":latest["fingerprint"],"nonce":body.nonce,"turn_id":body.turn_id,
"session_id":body.session_id,"expires_at":time.time()+5}
+9 -1
View File
@@ -21,6 +21,7 @@ class Notification(BaseModel):
proof: dict
request_id: str = Field(pattern=r'^[A-Za-z0-9_-]{8,100}$')
binding_owner: str = Field(min_length=1, max_length=128)
binding_generation: int | None = Field(default=None,strict=True,ge=0)
kind: str = Field(pattern=r'^(text|notice|authorization)$')
title: str = Field(default='', max_length=80)
text: str = Field(default='', max_length=2000)
@@ -41,10 +42,16 @@ def router(settings, redis, store, platform, oauth):
if not settings.feishu_enabled:
raise HTTPException(503, 'platform disabled')
payload = body.model_dump(exclude={'host_id', 'proof'})
if body.binding_generation is None:
payload.pop('binding_generation') # Preserve exact legacy proof contract.
owner = await device_owner(settings, redis, store, body.host_id, body.proof,
'work-notify-v1', payload)
if body.binding_owner != owner:
raise HTTPException(403, 'local binding is stale; pair the client again')
if settings.credential_mode == 'broker':
current = store.device(body.host_id)
if body.binding_generation is None or current['binding_generation'] != body.binding_generation:
raise HTTPException(403,'stale_binding')
digest = hashlib.sha256(canonical(payload)).hexdigest()
previous = store.db.execute('SELECT * FROM notifications WHERE host_id=? AND request_id=?',
(body.host_id, body.request_id)).fetchone()
@@ -106,7 +113,8 @@ def router(settings, redis, store, platform, oauth):
# Recheck the transaction and generation after waiting on app credentials.
oauth.authorization_url_for_notification(owner, body.host_id, body.oauth_state)
device = store.device(body.host_id)
if not device or device['revoked'] or device['owner'] != owner:
if (not device or device['revoked'] or device['owner'] != owner
or (settings.credential_mode == 'broker' and device['binding_generation'] != body.binding_generation)):
status = 'binding_changed'
else:
delivery_id = uuid.uuid5(uuid.NAMESPACE_URL, body.host_id + ':' + body.request_id).hex
+58 -5
View File
@@ -3,6 +3,8 @@ import contextlib
import json
import secrets
import time
import hashlib
import httpx
from contextlib import asynccontextmanager
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
@@ -10,7 +12,7 @@ from redis.asyncio import Redis
from redis.exceptions import WatchError
from .config import Settings
from .crypto import fingerprint, raw_public, verified, verify_device
from .crypto import canonical, fingerprint, load_public, raw_public, verified, verify_device
async def compare_delete(redis, key, expected):
@@ -41,16 +43,38 @@ async def compare_expire(redis, key, expected, ttl):
return False
def create_app(settings=None, redis=None):
def create_app(settings=None, redis=None, admission_transport=None):
s = settings or Settings(role="router")
r = redis or Redis.from_url(s.redis_url, decode_responses=True)
ca = s.ca_cert.read_bytes()
active = set()
internal = (httpx.AsyncClient(base_url=s.connector_internal_url.rstrip('/'),timeout=3,
trust_env=False,follow_redirects=False,transport=admission_transport) if s.credential_mode == "broker" else None)
connector_public = load_public(s.connector_public_key) if internal else None
async def admission(host, owner, generation, *, turn=None, session=None, expected_fp=None):
nonce = secrets.token_hex(16)
response = await internal.post("/internal/devices/"+host+"/admission",
headers={"X-Router-Admission-Secret":s.router_admission_secret},
json={"nonce":nonce,"turn_id":turn,"session_id":session})
if response.status_code != 200:
raise ValueError("device admission rejected")
data = verified(response.json(),connector_public)
if (data["nonce"] != nonce or data["host_id"] != host or data["owner"] != owner
or data["binding_generation"] != generation or data["environment"] != s.environment
or data["app_id"] != s.feishu_app_id or data["protocol_version"] != 2
or data["turn_id"] != turn or data["session_id"] != session
or not time.time() <= data["expires_at"] <= time.time()+10
or data["fingerprint"] != expected_fp):
raise ValueError("stale device admission")
return data
@asynccontextmanager
async def lifespan(app):
await r.ping()
yield
if internal:
await internal.aclose()
if redis is None:
await r.aclose()
@@ -101,6 +125,14 @@ def create_app(settings=None, redis=None):
fp = fingerprint(hello["cert"])
if await r.get("hn:device:" + host) != fp or await r.exists("hn:revoked:" + host):
raise ValueError("unregistered/revoked")
generation = owner = None
if internal:
generation, owner = proof["binding_generation"], proof["binding_owner"]
if (type(generation) is not int or generation < 0 or type(proof.get("protocol_version")) is not int
or proof.get("protocol_version") != 2
or proof.get("environment") != s.environment or proof.get("app_id") != s.feishu_app_id):
raise ValueError("invalid binding handshake")
await admission(host,owner,generation,expected_fp=fp)
session = proof["session_id"]
presence = json.dumps({"cert": hello["cert"], "proof": hello["proof"], "challenge": challenge})
if not await r.set("hn:online:" + host, presence, nx=True, ex=30):
@@ -108,6 +140,8 @@ def create_app(settings=None, redis=None):
active.add(session)
pub = r.pubsub()
await pub.subscribe("hn:down:" + host, "hn:kill:" + host)
if internal:
await admission(host,owner,generation,expected_fp=fp)
await ws.send_json({"type": "connected", "session_id": session})
async def downstream():
@@ -115,13 +149,28 @@ def create_app(settings=None, redis=None):
if msg["type"] != "message":
continue
if msg["channel"] == "hn:kill:" + host:
if internal:
event = json.loads(msg["data"])
if generation >= event["binding_generation"]:
continue
await ws.close(1008)
return
if internal:
packet = json.loads(msg["data"])
routing = verified(packet["routing"],connector_public)
if (routing["host_id"] != host or routing["session_id"] != session
or routing["environment"] != s.environment or routing["app_id"] != s.feishu_app_id
or routing["binding_generation"] != generation
or routing["envelope_hash"] != hashlib.sha256(canonical(packet["envelope"])).hexdigest()):
raise ValueError("invalid instruction routing")
await admission(host,owner,generation,turn=routing["turn_id"],session=session,expected_fp=fp)
await ws.send_text(msg["data"])
async def heartbeat():
while True:
await asyncio.sleep(8)
if internal:
await admission(host,owner,generation,expected_fp=fp)
if await r.exists("hn:revoked:" + host) or not await compare_expire(r, "hn:online:" + host, presence, 30):
await ws.close(1008)
return
@@ -142,6 +191,8 @@ def create_app(settings=None, redis=None):
verified(envelope, public)
if envelope["host_id"] != host or envelope["session_id"] != session:
raise ValueError("wrong reply destination")
if internal:
await admission(host,owner,generation,turn=turn,session=session,expected_fp=fp)
await r.publish("hn:replies", json.dumps(data))
tasks = [asyncio.create_task(x()) for x in (downstream, heartbeat, upstream)]
@@ -155,13 +206,15 @@ def create_app(settings=None, redis=None):
with contextlib.suppress(Exception):
await ws.close(1011)
finally:
active.discard(session)
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
if pub:
await pub.aclose()
with contextlib.suppress(Exception):
await pub.aclose()
if presence:
await compare_delete(r, "hn:online:" + host, presence)
active.discard(session)
with contextlib.suppress(Exception):
await compare_delete(r, "hn:online:" + host, presence)
return app
+7
View File
@@ -32,6 +32,7 @@ class Postgres:
class Store:
def __init__(self, path, database_url=""):
self.lifecycle = None
path.parent.mkdir(parents=True, exist_ok=True)
schema = """
CREATE TABLE IF NOT EXISTS devices(
@@ -71,6 +72,8 @@ class Store:
self.db.execute("INSERT INTO devices(host_id,cert,fingerprint) VALUES(?,?,?)", (host, cert, fp))
def bind(self, host, owner):
if self.lifecycle:
return self.lifecycle.bind(host, owner)
with self.db:
# Conditional update also protects against two connectors racing for the same device.
count = self.db.execute("UPDATE devices SET owner=?,binding_generation=binding_generation+1 WHERE host_id=? AND owner IS NULL AND revoked=0", (owner, host)).rowcount
@@ -78,10 +81,14 @@ class Store:
raise ValueError("device unavailable or already bound")
def unbind(self, owner):
if self.lifecycle:
return self.lifecycle.unbind(owner)
with self.db:
self.db.execute("UPDATE devices SET owner=NULL,binding_generation=binding_generation+1 WHERE owner=?", (owner,))
def revoke(self, host):
if self.lifecycle:
return self.lifecycle.invalidate_host(host, "revoke")
with self.db:
self.db.execute("UPDATE devices SET revoked=1,binding_generation=binding_generation+1 WHERE host_id=? AND revoked=0", (host,))