Unify broker-mode binding changes in database transactions: advance device generation, invalidate pending authorizations/tasks and prior pairing-code epochs, and persist audit plus invalidation outbox. Add authenticated device unbind alongside robot/admin paths. Pairing codes are hashed, generation- fenced and consumed atomically with binding. Replay Redis invalidation effects without undoing committed unbind. Guard online cleanup and delayed kill events by generation. Introduce a separate router admission credential with no platform-event injection authority. Router verifies signed nonce-bound internal admission on handshake, task traffic and periodic checks; reject stale sessions and control-plane outages. Persist task owner/device/generation/session/worker and original message mapping. Sign routing metadata, verify late results, claim replies once and mark ambiguous/crashed tasks unknown instead of replaying writes. Fence notifications across same-user rebind. Release in-memory slots/waiters even when database or Redis cleanup fails. Validation: 213 passed, 3 skipped (real PostgreSQL not configured), 30 existing uvicorn/websockets deprecation warnings. New lifecycle coverage: 28 tests, including real loopback WebSocket isolation, reverse completion, missed kill, periodic unbind checks, outbox recovery, transaction rollback and cleanup failures. Syntax, docs and staged diff checks pass. Default legacy mode remains unchanged. Broker mode now requires coordinated router admission configuration and a v2 client handshake. No production ECS configuration, real credentials or client installer was changed. PostgreSQL, real Redis multi-node, ARM64 and actual Work/client rollout remain stage 5.
105 lines
5.6 KiB
Python
105 lines
5.6 KiB
Python
"""Broker-mode invalidation outbox, worker liveness and internal admission."""
|
|
import json
|
|
import time
|
|
|
|
from fastapi import HTTPException
|
|
from pydantic import BaseModel, ConfigDict, Field
|
|
|
|
from .router import compare_delete
|
|
|
|
|
|
class AdmissionRequest(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
nonce: str = Field(pattern=r"^[a-f0-9]{32}$")
|
|
turn_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9_-]{1,128}$")
|
|
session_id: str | None = Field(default=None, min_length=1, max_length=64)
|
|
|
|
|
|
class DeviceLifecycle:
|
|
def __init__(self, state, redis, broker):
|
|
self.state, self.r, self.broker = state, redis, broker
|
|
|
|
async def flush(self):
|
|
rows = self.state.db.execute("""SELECT * FROM work_binding_events_v2
|
|
WHERE environment=? AND app_id=? AND delivered=0 ORDER BY created_at LIMIT 100""",
|
|
(self.state.environment,self.state.app_id)).fetchall()
|
|
for event in rows:
|
|
try:
|
|
device = self.state.store.device(event["host_id"])
|
|
if device and device["revoked"]:
|
|
await self.r.set("hn:revoked:"+event["host_id"], "1")
|
|
key = "hn:online:"+event["host_id"]
|
|
presence = await self.r.get(key)
|
|
if presence:
|
|
try:
|
|
generation = json.loads(presence)["proof"]["binding_generation"]
|
|
if type(generation) is not int:
|
|
generation = -1
|
|
except (ValueError, KeyError, TypeError):
|
|
generation = -1
|
|
if generation < event["binding_generation"]:
|
|
await compare_delete(self.r, key, presence)
|
|
# Newer sessions ignore a delayed invalidation for older generations.
|
|
await self.r.publish("hn:kill:"+event["host_id"], json.dumps({
|
|
"reason": event["reason"], "binding_generation": event["binding_generation"]}))
|
|
with self.state.db:
|
|
self.state.db.execute("UPDATE work_binding_events_v2 SET delivered=1 WHERE event_id=?",
|
|
(event["event_id"],))
|
|
except Exception:
|
|
# DB remains authoritative; leave durable work for another attempt.
|
|
return False
|
|
self.cancel_waiters()
|
|
return not self.state.db.execute("""SELECT event_id FROM work_binding_events_v2
|
|
WHERE environment=? AND app_id=? AND delivered=0 LIMIT 1""",
|
|
(self.state.environment,self.state.app_id)).fetchone()
|
|
|
|
def cancel_waiters(self):
|
|
for turn, pending in list(self.broker.pending.items()):
|
|
task = self.state.task(turn)
|
|
if task and task["status"] not in {"inflight", "completed", "replying"} and not pending.future.done():
|
|
pending.future.set_result({"status":task["status"], "turn_id":turn,
|
|
"result":"任务已结束,旧结果不会继续投递。"})
|
|
|
|
async def maintain(self):
|
|
await self.r.set("hn:task-worker:"+self.broker.worker_id,"alive",ex=30)
|
|
await self.flush()
|
|
rows = self.state.db.execute("""SELECT * FROM work_tasks_v2
|
|
WHERE environment=? AND app_id=? AND status IN ('inflight','completed','replying')
|
|
ORDER BY updated_at LIMIT 100""", (self.state.environment,self.state.app_id)).fetchall()
|
|
for task in rows:
|
|
if task["deadline"] <= time.time() or not await self.r.exists("hn:task-worker:"+task["worker_id"]):
|
|
self.state.transition(task["turn_id"],task["status"],
|
|
"reply_unknown" if task["status"] == "replying" else "unknown")
|
|
else:
|
|
# Rotate bounded scanning instead of starving later tasks behind
|
|
# the same first 100 healthy workers on every maintenance pass.
|
|
with self.state.db:
|
|
self.state.db.execute("UPDATE work_tasks_v2 SET updated_at=? WHERE turn_id=? AND status=?",
|
|
(time.time(),task["turn_id"],task["status"]))
|
|
self.cancel_waiters()
|
|
|
|
async def admission(self, host, body):
|
|
device = self.state.store.device(host)
|
|
if not device or device["revoked"] or not device["owner"]:
|
|
raise HTTPException(403,"device_unbound_or_revoked")
|
|
task = None
|
|
if body.turn_id:
|
|
task = self.state.task(body.turn_id)
|
|
if (not self.state.task_current(task) or task["host_id"] != host
|
|
or task["status"] != "inflight" or task["session_id"] != body.session_id
|
|
or not await self.r.exists("hn:task-worker:"+task["worker_id"])):
|
|
raise HTTPException(403,"task_not_active")
|
|
# Re-read after Redis await; never rely on router's cached admission.
|
|
latest = self.state.store.device(host)
|
|
if (not latest or latest["revoked"] or latest["owner"] != device["owner"]
|
|
or latest["binding_generation"] != device["binding_generation"]):
|
|
raise HTTPException(403,"stale_binding")
|
|
if task:
|
|
current = self.state.task(body.turn_id)
|
|
if not self.state.task_current(current) or current["status"] != "inflight":
|
|
raise HTTPException(403,"task_not_active")
|
|
return {"protocol_version":2,"environment":self.state.environment,"app_id":self.state.app_id,
|
|
"host_id":host,"owner":latest["owner"],"binding_generation":latest["binding_generation"],
|
|
"fingerprint":latest["fingerprint"],"nonce":body.nonce,"turn_id":body.turn_id,
|
|
"session_id":body.session_id,"expires_at":time.time()+5}
|