feat(lifecycle): add stage 4 authoritative unbind and durable task routing
Unify broker-mode binding changes in database transactions: advance device generation, invalidate pending authorizations/tasks and prior pairing-code epochs, and persist audit plus invalidation outbox. Add authenticated device unbind alongside robot/admin paths. Pairing codes are hashed, generation- fenced and consumed atomically with binding. Replay Redis invalidation effects without undoing committed unbind. Guard online cleanup and delayed kill events by generation. Introduce a separate router admission credential with no platform-event injection authority. Router verifies signed nonce-bound internal admission on handshake, task traffic and periodic checks; reject stale sessions and control-plane outages. Persist task owner/device/generation/session/worker and original message mapping. Sign routing metadata, verify late results, claim replies once and mark ambiguous/crashed tasks unknown instead of replaying writes. Fence notifications across same-user rebind. Release in-memory slots/waiters even when database or Redis cleanup fails. Validation: 213 passed, 3 skipped (real PostgreSQL not configured), 30 existing uvicorn/websockets deprecation warnings. New lifecycle coverage: 28 tests, including real loopback WebSocket isolation, reverse completion, missed kill, periodic unbind checks, outbox recovery, transaction rollback and cleanup failures. Syntax, docs and staged diff checks pass. Default legacy mode remains unchanged. Broker mode now requires coordinated router admission configuration and a v2 client handshake. No production ECS configuration, real credentials or client installer was changed. PostgreSQL, real Redis multi-node, ARM64 and actual Work/client rollout remain stage 5.
This commit is contained in:
@@ -0,0 +1,182 @@
|
||||
"""Transactional broker-mode binding, pairing, invalidation and task metadata.
|
||||
|
||||
No message bodies or credential values are stored here. All mutating binding
|
||||
paths in a broker connector use this object through Store's lifecycle hook.
|
||||
"""
|
||||
import hashlib
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
class BindingState:
|
||||
def __init__(self, store, environment, app_id):
|
||||
self.store, self.db, self.environment, self.app_id = store, store.db, environment, app_id
|
||||
with self.db:
|
||||
self.db.execute("""CREATE TABLE IF NOT EXISTS work_owner_epochs_v2(
|
||||
environment TEXT NOT NULL,app_id TEXT NOT NULL,owner TEXT NOT NULL,epoch BIGINT NOT NULL,
|
||||
PRIMARY KEY(environment,app_id,owner))""")
|
||||
self.db.execute("""CREATE TABLE IF NOT EXISTS work_binding_codes_v2(
|
||||
code_hash TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
|
||||
owner TEXT NOT NULL,owner_epoch BIGINT NOT NULL,expires_at DOUBLE PRECISION NOT NULL,
|
||||
consumed INTEGER NOT NULL DEFAULT 0)""")
|
||||
self.db.execute("""CREATE TABLE IF NOT EXISTS work_binding_events_v2(
|
||||
event_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,host_id TEXT NOT NULL,
|
||||
owner TEXT NOT NULL,binding_generation BIGINT NOT NULL,reason TEXT NOT NULL,
|
||||
created_at DOUBLE PRECISION NOT NULL,delivered INTEGER NOT NULL DEFAULT 0)""")
|
||||
self.db.execute("""CREATE TABLE IF NOT EXISTS work_tasks_v2(
|
||||
turn_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,owner TEXT NOT NULL,
|
||||
host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,session_id TEXT NOT NULL,
|
||||
worker_id TEXT NOT NULL,conversation_id TEXT NOT NULL,chat_id TEXT NOT NULL,
|
||||
source_message_id TEXT NOT NULL,status TEXT NOT NULL,deadline DOUBLE PRECISION NOT NULL,
|
||||
created_at DOUBLE PRECISION NOT NULL,updated_at DOUBLE PRECISION NOT NULL)""")
|
||||
self.db.execute("""CREATE INDEX IF NOT EXISTS work_binding_events_pending_v2
|
||||
ON work_binding_events_v2(environment,app_id,delivered,created_at)""")
|
||||
self.db.execute("""CREATE INDEX IF NOT EXISTS work_tasks_active_v2
|
||||
ON work_tasks_v2(environment,app_id,status,updated_at)""")
|
||||
self.db.execute("""CREATE INDEX IF NOT EXISTS work_tasks_binding_v2
|
||||
ON work_tasks_v2(environment,app_id,host_id,binding_generation)""")
|
||||
|
||||
def _owner_lock(self, owner, bump=False):
|
||||
identity = (self.environment, self.app_id, owner)
|
||||
self.db.execute("INSERT INTO work_owner_epochs_v2 VALUES(?,?,?,0) ON CONFLICT DO NOTHING", identity)
|
||||
self.db.execute("UPDATE work_owner_epochs_v2 SET epoch=epoch+? WHERE environment=? AND app_id=? AND owner=?",
|
||||
(1 if bump else 0, *identity))
|
||||
return self.db.execute("SELECT epoch FROM work_owner_epochs_v2 WHERE environment=? AND app_id=? AND owner=?",
|
||||
identity).fetchone()["epoch"]
|
||||
|
||||
def _bind(self, host, owner):
|
||||
count = self.db.execute("""UPDATE devices SET owner=?,binding_generation=binding_generation+1
|
||||
WHERE host_id=? AND owner IS NULL AND revoked=0""", (owner, host)).rowcount
|
||||
if count != 1:
|
||||
raise ValueError("device unavailable or already bound")
|
||||
|
||||
def bind(self, host, owner):
|
||||
with self.db:
|
||||
self._owner_lock(owner)
|
||||
self._bind(host, owner)
|
||||
|
||||
def issue_code(self, owner, code, ttl=300):
|
||||
with self.db:
|
||||
epoch = self._owner_lock(owner)
|
||||
if self.db.execute("SELECT host_id FROM devices WHERE owner=?", (owner,)).fetchone():
|
||||
raise ValueError("owner already bound")
|
||||
self.db.execute("INSERT INTO work_binding_codes_v2 VALUES(?,?,?,?,?,?,0)",
|
||||
(hashlib.sha256(code.encode()).hexdigest(), self.environment, self.app_id, owner, epoch, time.time()+ttl))
|
||||
|
||||
def consume_code(self, host, code):
|
||||
digest = hashlib.sha256(code.encode()).hexdigest()
|
||||
row = self.db.execute("SELECT * FROM work_binding_codes_v2 WHERE code_hash=? AND environment=? AND app_id=?",
|
||||
(digest, self.environment, self.app_id)).fetchone()
|
||||
if not row:
|
||||
raise LookupError("binding code unavailable")
|
||||
with self.db:
|
||||
epoch = self._owner_lock(row["owner"])
|
||||
count = self.db.execute("""UPDATE work_binding_codes_v2 SET consumed=1
|
||||
WHERE code_hash=? AND owner_epoch=? AND consumed=0 AND expires_at>?""",
|
||||
(digest, epoch, time.time())).rowcount
|
||||
if count != 1:
|
||||
raise LookupError("binding code unavailable")
|
||||
self._bind(host, row["owner"])
|
||||
return row["owner"]
|
||||
|
||||
def _cancel(self, device, reason):
|
||||
owner = device["owner"] or ""
|
||||
generation = device["binding_generation"]
|
||||
self.db.execute("""UPDATE work_authorizations_v2 SET status='invalidated'
|
||||
WHERE environment=? AND app_id=? AND host_id=? AND binding_generation<=?
|
||||
AND status IN ('waiting','exchanging')""",
|
||||
(self.environment, self.app_id, device["host_id"], generation))
|
||||
self.db.execute("""UPDATE work_tasks_v2 SET status=?,updated_at=?
|
||||
WHERE environment=? AND app_id=? AND host_id=? AND binding_generation<=?
|
||||
AND status IN ('inflight','completed','replying')""",
|
||||
("revoked" if reason == "revoke" else "unbound", time.time(), self.environment,
|
||||
self.app_id, device["host_id"], generation))
|
||||
event = uuid.uuid4().hex
|
||||
self.db.execute("INSERT INTO work_binding_events_v2 VALUES(?,?,?,?,?,?,?,?,0)",
|
||||
(event, self.environment, self.app_id, device["host_id"], owner, generation+1, reason, time.time()))
|
||||
self.db.execute("INSERT INTO audit(ts,actor,action,resource,result) VALUES(?,?,?,?,?)",
|
||||
(time.time(), owner or "admin", "binding_"+reason, device["host_id"], "committed"))
|
||||
|
||||
def unbind(self, owner):
|
||||
with self.db:
|
||||
self._owner_lock(owner, bump=True) # Invalidates pre-unbind codes even if already unbound.
|
||||
device = self.db.execute("SELECT * FROM devices WHERE owner=?", (owner,)).fetchone()
|
||||
if device:
|
||||
self.db.execute("UPDATE devices SET owner=NULL,binding_generation=binding_generation+1 WHERE host_id=?",
|
||||
(device["host_id"],))
|
||||
self._cancel(dict(device), "unbind")
|
||||
|
||||
def invalidate_host(self, host, reason, *, expected_owner=None, expected_generation=None):
|
||||
if reason not in {"unbind", "revoke"}:
|
||||
raise ValueError("invalid binding change")
|
||||
device = self.store.device(host)
|
||||
if not device:
|
||||
raise LookupError("device not found")
|
||||
if ((expected_owner is not None and device["owner"] != expected_owner)
|
||||
or (expected_generation is not None and device["binding_generation"] != expected_generation)):
|
||||
raise ValueError("binding changed; retry")
|
||||
if reason == "unbind" and not device["owner"]:
|
||||
return
|
||||
if reason == "revoke" and device["revoked"]:
|
||||
return
|
||||
with self.db:
|
||||
if device["owner"]:
|
||||
self._owner_lock(device["owner"], bump=True)
|
||||
mutation = "owner=NULL" if reason == "unbind" else "revoked=1"
|
||||
count = self.db.execute(f"""UPDATE devices SET {mutation},binding_generation=binding_generation+1
|
||||
WHERE host_id=? AND binding_generation=? AND COALESCE(owner,'')=? AND revoked=?""",
|
||||
(host, device["binding_generation"], device["owner"] or "", device["revoked"])).rowcount
|
||||
if count != 1:
|
||||
raise ValueError("binding changed; retry")
|
||||
self._cancel(device, reason)
|
||||
|
||||
def task(self, turn):
|
||||
row = self.db.execute("SELECT * FROM work_tasks_v2 WHERE turn_id=? AND environment=? AND app_id=?",
|
||||
(turn, self.environment, self.app_id)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def task_current(self, task):
|
||||
device = self.store.device(task["host_id"]) if task else None
|
||||
return bool(device and not device["revoked"] and device["owner"] == task["owner"]
|
||||
and device["binding_generation"] == task["binding_generation"]
|
||||
and task["deadline"] > time.time())
|
||||
|
||||
def create_task(self, turn, owner, host, generation, session, worker, deadline, chat_id="", message_id=""):
|
||||
conversation = hashlib.sha256((self.environment+"\0"+self.app_id+"\0"+owner+"\0"+chat_id+"\0work-dm").encode()).hexdigest()
|
||||
with self.db:
|
||||
self._owner_lock(owner)
|
||||
device = self.store.device(host)
|
||||
if not device or device["revoked"] or device["owner"] != owner or device["binding_generation"] != generation:
|
||||
raise ValueError("binding changed")
|
||||
count = self.db.execute("""INSERT INTO work_tasks_v2 VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT DO NOTHING""", (turn,self.environment,self.app_id,owner,host,generation,session,
|
||||
worker,conversation,chat_id,message_id,"inflight",deadline,time.time(),time.time())).rowcount
|
||||
if count != 1:
|
||||
raise ValueError("task already recorded")
|
||||
return self.task(turn)
|
||||
|
||||
def transition(self, turn, previous, status):
|
||||
with self.db:
|
||||
return self.db.execute("""UPDATE work_tasks_v2 SET status=?,updated_at=?
|
||||
WHERE turn_id=? AND environment=? AND app_id=? AND status=?""",
|
||||
(status,time.time(),turn,self.environment,self.app_id,previous)).rowcount == 1
|
||||
|
||||
def complete(self, turn, host, session, worker):
|
||||
with self.db:
|
||||
task = self.task(turn)
|
||||
if not task:
|
||||
return False
|
||||
self._owner_lock(task["owner"])
|
||||
if not self.task_current(task) or (task["host_id"],task["session_id"],task["worker_id"],task["status"]) != (host,session,worker,"inflight"):
|
||||
return False
|
||||
return self.db.execute("UPDATE work_tasks_v2 SET status='completed',updated_at=? WHERE turn_id=? AND status='inflight'",
|
||||
(time.time(),turn)).rowcount == 1
|
||||
|
||||
def claim_reply(self, turn, owner, message_id):
|
||||
with self.db:
|
||||
self._owner_lock(owner)
|
||||
task = self.task(turn)
|
||||
if not self.task_current(task) or (task["owner"],task["source_message_id"],task["status"]) != (owner,message_id,"completed"):
|
||||
return False
|
||||
return self.db.execute("UPDATE work_tasks_v2 SET status='replying',updated_at=? WHERE turn_id=? AND status='completed'",
|
||||
(time.time(),turn)).rowcount == 1
|
||||
Reference in New Issue
Block a user