feat(lifecycle): add stage 4 authoritative unbind and durable task routing

Unify broker-mode binding changes in database transactions: advance device
generation, invalidate pending authorizations/tasks and prior pairing-code
epochs, and persist audit plus invalidation outbox. Add authenticated device
unbind alongside robot/admin paths. Pairing codes are hashed, generation-
fenced and consumed atomically with binding.

Replay Redis invalidation effects without undoing committed unbind. Guard
online cleanup and delayed kill events by generation. Introduce a separate
router admission credential with no platform-event injection authority.
Router verifies signed nonce-bound internal admission on handshake, task
traffic and periodic checks; reject stale sessions and control-plane outages.

Persist task owner/device/generation/session/worker and original message
mapping. Sign routing metadata, verify late results, claim replies once and
mark ambiguous/crashed tasks unknown instead of replaying writes. Fence
notifications across same-user rebind. Release in-memory slots/waiters even
when database or Redis cleanup fails.

Validation: 213 passed, 3 skipped (real PostgreSQL not configured), 30 existing
uvicorn/websockets deprecation warnings. New lifecycle coverage: 28 tests,
including real loopback WebSocket isolation, reverse completion, missed kill,
periodic unbind checks, outbox recovery, transaction rollback and cleanup
failures. Syntax, docs and staged diff checks pass.

Default legacy mode remains unchanged. Broker mode now requires coordinated
router admission configuration and a v2 client handshake. No production ECS
configuration, real credentials or client installer was changed. PostgreSQL,
real Redis multi-node, ARM64 and actual Work/client rollout remain stage 5.
This commit is contained in:
ex_tianzy
2026-11-04 11:43:53 +08:00
parent 93bfbca83a
commit d1ab834ecb
13 changed files with 1128 additions and 29 deletions
+479
View File
@@ -0,0 +1,479 @@
"""Broker-mode invalidation and real loopback WS task routing (no real Work)."""
import asyncio
import hashlib
import json
import secrets
import socket
import time
import httpx
import pytest
import pytest_asyncio
import uvicorn
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import x25519
from websockets.asyncio.client import connect
from websockets.exceptions import ConnectionClosed
from test_credential_api import env, wire, start, authorize
from relay.crypto import canonical, load_private, load_public, open_envelope, raw_public, seal, sign
from relay.device_lifecycle import AdmissionRequest
from relay.router import create_app as router_app
@pytest_asyncio.fixture
async def routed(env, tmp_path):
public_path = tmp_path / "connector.pub"
public_path.write_bytes(load_private(env["settings"].connector_key).public_key().public_bytes(
serialization.Encoding.PEM,serialization.PublicFormat.SubjectPublicKeyInfo))
settings = env["settings"].model_copy(update={"role":"router","connector_public_key":public_path,
"connector_internal_url":"http://connector:8000"})
app = router_app(settings,env["redis"],admission_transport=httpx.ASGITransport(env["app"]))
sock = socket.socket()
sock.bind(("127.0.0.1",0))
sock.listen(128)
port = sock.getsockname()[1]
server = uvicorn.Server(uvicorn.Config(app,log_level="critical",access_log=False,lifespan="on"))
server_task = asyncio.create_task(server.serve(sockets=[sock]))
try:
for _ in range(200):
if server.started:
break
if server_task.done():
server_task.result()
await asyncio.sleep(0.01)
assert server.started
value = {**env,"url":f"ws://127.0.0.1:{port}/device/ws","public":load_public(public_path),"sessions":[],"router":app}
yield value
finally:
for ws in locals().get("value",{}).get("sessions",[]):
await ws.close()
server.should_exit = True
await asyncio.wait_for(server_task,5)
async def device(env, host="a"*32, *, generation=None, legacy=False):
ws = await connect(env["url"],proxy=None)
env["sessions"].append(ws)
challenge = json.loads(await ws.recv())["challenge"]
row = env["store"].device(host)
ephemeral = x25519.X25519PrivateKey.generate()
session = secrets.token_hex(16)
plain = {"host_id":host,"challenge":challenge,"session_id":session,
"kx":raw_public(ephemeral.public_key()),"ts":int(time.time())}
if not legacy:
plain.update(protocol_version=2,environment=env["settings"].environment,
app_id=env["settings"].feishu_app_id,binding_owner=row["owner"] or "ou_a",
binding_generation=row["binding_generation"] if generation is None else generation)
await ws.send(json.dumps({"host_id":host,"cert":row["cert"],"proof":sign(plain,env["keys"][host])}))
accepted = json.loads(await asyncio.wait_for(ws.recv(),3))
assert accepted["type"] == "connected"
return ws,session,ephemeral
async def instruction(env, connection, host="a"*32):
ws,session,ephemeral = connection
packet = json.loads(await asyncio.wait_for(ws.recv(),3))
payload = open_envelope(packet["envelope"],ephemeral,env["public"],host_id=host,session_id=session)
return payload
async def reply(env, connection, payload, text, host="a"*32):
ws,session,_ = connection
envelope = seal({"type":"result","turn_id":payload["turn_id"],"result":text},
payload["reply_key"],env["keys"][host],host_id=host,session_id=session)
await ws.send(json.dumps({"type":"reply","turn_id":payload["turn_id"],"envelope":envelope}))
async def wait_for_reply(platform, count):
for _ in range(200):
if len(platform.replies) >= count:
return
await asyncio.sleep(0.01)
assert len(platform.replies) >= count
def event(env, message_id, text, chat="chat-a", owner="ou_a"):
return {"header":{"event_type":"im.message.receive_v1","app_id":env["settings"].feishu_app_id},
"event":{"sender":{"sender_type":"user","sender_id":{"open_id":owner}},
"message":{"message_id":message_id,"chat_type":"p2p","chat_id":chat,"message_type":"text",
"content":json.dumps({"text":text})}}}
async def send_event(env, data):
response = await env["client"].post("/internal/events",json=data,
headers={"X-Internal-Secret":env["settings"].internal_secret})
assert response.status_code == 200
async def test_unbind_commits_and_defers_redis_side_effects(env,monkeypatch):
await authorize(env)
pending = await start(env)
with monkeypatch.context() as patch:
async def fail(*args,**kwargs):
raise ConnectionError("synthetic redis outage")
patch.setattr(env["redis"],"publish",fail)
response = await env["client"].post("/admin/devices/"+"a"*32+"/unbind",
headers={"X-Admin-Secret":env["settings"].admin_secret})
assert response.status_code == 200 and response.json()["propagation"] == "pending"
assert env["store"].device("a"*32)["owner"] is None
assert env["api"]._row(pending["transaction_id"])["status"] == "invalidated"
path = "/work/v2/credentials/access-token"
assert (await env["client"].post(path,json=wire(env,path,permission_profile="product-v1"))).status_code == 403
assert await env["app"].state.lifecycle.flush()
assert env["store"].db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2 WHERE delivered=0").fetchone()["n"] == 0
async def test_codes_are_one_time_and_pre_unbind_codes_stay_invalid_after_rebind(env):
state = env["store"].lifecycle
env["store"].unbind("ou_a")
state.issue_code("ou_a","fake-code-old")
env["store"].unbind("ou_a")
with pytest.raises(LookupError):
state.consume_code("a"*32,"fake-code-old")
state.issue_code("ou_a","fake-code-new")
assert state.consume_code("a"*32,"fake-code-new") == "ou_a"
with pytest.raises(LookupError):
state.consume_code("a"*32,"fake-code-new")
dump = "\n".join(env["store"].db.iterdump())
assert "fake-code-old" not in dump and "fake-code-new" not in dump
async def test_failed_database_invalidation_rolls_back_binding_and_outbox(env,monkeypatch):
state = env["store"].lifecycle
original = state._cancel
def fail(device,reason):
original(device,reason)
raise RuntimeError("synthetic transaction abort")
monkeypatch.setattr(state,"_cancel",fail)
with pytest.raises(RuntimeError):
env["store"].unbind("ou_a")
current = env["store"].device("a"*32)
assert current["owner"] == "ou_a" and current["binding_generation"] == 1
assert state.db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2").fetchone()["n"] == 0
async def test_unbind_outbox_is_persistent_and_reprocessed_by_second_connector(env):
from relay.connector import create_app
env["store"].unbind("ou_a")
second = create_app(env["settings"],env["redis"],env["platform"])
try:
assert second.state.store.device("a"*32)["owner"] is None
assert await second.state.lifecycle.flush()
assert second.state.store.db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2 WHERE delivered=0").fetchone()["n"] == 0
finally:
second.state.store.close()
async def test_new_online_generation_not_removed_by_old_outbox(env):
env["store"].unbind("ou_a")
env["store"].bind("a"*32,"ou_a")
generation = env["store"].device("a"*32)["binding_generation"]
presence = json.dumps({"proof":{"binding_generation":generation}})
await env["redis"].set("hn:online:"+"a"*32,presence)
assert await env["app"].state.lifecycle.flush()
assert await env["redis"].get("hn:online:"+"a"*32) == presence
def notification(env, *, generation=None, include_generation=True):
host = "a"*32
payload = {"request_id":secrets.token_hex(8),"kind":"text","text":"synthetic notice","title":"",
"oauth_state":"","binding_owner":"ou_a"}
if include_generation:
payload["binding_generation"] = env["store"].device(host)["binding_generation"] if generation is None else generation
proof = sign({"host_id":host,"purpose":"work-notify-v1","ts":int(time.time()),"nonce":secrets.token_hex(16),
"payload_hash":hashlib.sha256(canonical(payload)).hexdigest()},env["keys"][host])
return {**payload,"host_id":host,"proof":proof}
async def test_notifications_require_current_generation_in_broker_mode(env):
stale = notification(env)
env["store"].unbind("ou_a")
env["store"].bind("a"*32,"ou_a")
assert (await env["client"].post("/device/notifications",json=stale)).status_code == 403
assert (await env["client"].post("/device/notifications",json=notification(env,include_generation=False))).status_code == 403
result = await env["client"].post("/device/notifications",json=notification(env))
assert result.json()["status"] == "sent" and len(env["platform"].sent) == 1
async def test_notification_rebind_during_app_token_wait_does_not_send(env,monkeypatch):
async def rebind():
env["store"].unbind("ou_a")
env["store"].bind("a"*32,"ou_a")
return "fake-app-token"
monkeypatch.setattr(env["platform"],"tenant_token",rebind)
result = await env["client"].post("/device/notifications",json=notification(env))
assert result.json()["status"] == "binding_changed" and not env["platform"].sent
async def test_internal_admission_requires_secret_and_current_binding(env):
path = "/internal/devices/"+"a"*32+"/admission"
body = {"nonce":secrets.token_hex(16)}
assert (await env["client"].post(path,json=body)).status_code == 401
assert (await env["client"].post(path,json=body,
headers={"X-Internal-Secret":env["settings"].internal_secret})).status_code == 401
headers = {"X-Router-Admission-Secret":env["settings"].router_admission_secret}
assert (await env["client"].post(path,json=body,headers=headers)).status_code == 200
env["store"].unbind("ou_a")
assert (await env["client"].post(path,json=body,headers=headers)).status_code == 403
async def test_router_credential_cannot_inject_platform_events(env):
result = await env["client"].post("/internal/events",json=event(env,"om-injected","hello"),
headers={"X-Internal-Secret":env["settings"].router_admission_secret})
assert result.status_code == 401
async def test_device_can_only_unbind_its_current_authenticated_binding(env):
path = "/work/v2/device/unbind"
invalid = wire(env,path)
invalid["proof"] = {}
assert (await env["client"].post(path,json=invalid)).status_code == 403
result = await env["client"].post(path,json=wire(env,path))
assert result.status_code == 200 and result.json()["status"] == "unbound"
assert env["store"].device("a"*32)["owner"] is None
assert env["store"].device("b"*32)["owner"] == "ou_b"
async def test_authenticated_unbind_snapshot_cannot_clear_a_new_binding(env):
state = env["store"].lifecycle
old = env["store"].device("a"*32)
env["store"].unbind("ou_a")
env["store"].bind("a"*32,"ou_a")
with pytest.raises(ValueError):
state.invalidate_host("a"*32,"unbind",expected_owner="ou_a",expected_generation=old["binding_generation"])
assert env["store"].device("a"*32)["owner"] == "ou_a"
async def test_v2_router_rejects_old_handshake_and_unbound_reconnect(routed):
with pytest.raises(ConnectionClosed):
await device(routed,legacy=True)
routed["store"].unbind("ou_a")
with pytest.raises(ConnectionClosed):
await device(routed)
async def test_two_devices_reverse_results_match_own_tasks(routed):
a = await device(routed)
b = await device(routed,"b"*32)
broker = routed["app"].state.broker
ta = asyncio.create_task(broker.dispatch("ou_a","a","turn-a",chat_id="chat-a",source_message_id="om-a"))
tb = asyncio.create_task(broker.dispatch("ou_b","b","turn-b",chat_id="chat-b",source_message_id="om-b"))
pa,pb = await asyncio.gather(instruction(routed,a),instruction(routed,b,"b"*32))
assert pa["task_context"]["source_message_id"] == "om-a"
assert pb["task_context"]["owner"] == "ou_b"
assert pa["task_context"]["conversation_id"] != pb["task_context"]["conversation_id"]
await reply(routed,b,pb,"answer-b","b"*32)
await reply(routed,a,pa,"answer-a")
ra,rb = await asyncio.gather(ta,tb)
assert (ra["result"],rb["result"]) == ("answer-a","answer-b")
state = routed["store"].lifecycle
assert not state.claim_reply("turn-a","ou_a","om-b")
assert state.claim_reply("turn-a","ou_a","om-a")
assert not state.claim_reply("turn-a","ou_a","om-a")
async def test_same_owner_two_conversations_reply_to_original_message(routed):
connection = await device(routed)
await send_event(routed,event(routed,"om-first","first","chat-first"))
first = await instruction(routed,connection)
await send_event(routed,event(routed,"om-second","second","chat-second"))
second = await instruction(routed,connection)
assert first["task_context"]["conversation_id"] != second["task_context"]["conversation_id"]
await reply(routed,connection,second,"second-answer")
await reply(routed,connection,first,"first-answer")
await wait_for_reply(routed["platform"],2)
assert routed["platform"].replies == [("om-second","second-answer"),("om-first","first-answer")]
await send_event(routed,event(routed,"om-first","first","chat-first"))
await asyncio.sleep(0.05)
assert len(routed["platform"].replies) == 2
async def test_robot_unbind_discards_previous_result_and_closes_connection(routed):
connection = await device(routed)
await send_event(routed,event(routed,"om-old-task","slow"))
payload = await instruction(routed,connection)
await send_event(routed,event(routed,"om-unbind","/解绑"))
await wait_for_reply(routed["platform"],1)
assert routed["platform"].replies[0][0] == "om-unbind"
assert routed["store"].device("a"*32)["owner"] is None
assert routed["store"].lifecycle.task(payload["turn_id"])["status"] == "unbound"
with pytest.raises(ConnectionClosed):
await asyncio.wait_for(connection[0].recv(),2)
await asyncio.sleep(0.05)
assert all(message != "om-old-task" for message,_ in routed["platform"].replies)
async def test_revoke_has_durable_state_and_blocks_connection(routed):
connection = await device(routed)
response = await routed["client"].post("/admin/devices/"+"a"*32+"/revoke",
headers={"X-Admin-Secret":routed["settings"].admin_secret})
assert response.status_code == 200
assert routed["store"].device("a"*32)["revoked"] == 1
with pytest.raises(ConnectionClosed):
await asyncio.wait_for(connection[0].recv(),2)
async def test_delayed_kill_for_old_generation_does_not_disconnect_new_session(routed):
old = await device(routed)
routed["store"].unbind("ou_a")
await routed["app"].state.lifecycle.flush()
with pytest.raises(ConnectionClosed):
await asyncio.wait_for(old[0].recv(),2)
routed["store"].bind("a"*32,"ou_a")
fresh = await device(routed)
with routed["store"].db:
routed["store"].db.execute("UPDATE work_binding_events_v2 SET delivered=0")
await routed["app"].state.lifecycle.flush()
task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","hello","fresh-task"))
packet = await instruction(routed,fresh)
await reply(routed,fresh,packet,"fresh-answer")
assert (await task)["result"] == "fresh-answer"
async def test_lost_disconnect_notification_still_rejects_old_reply(routed):
connection = await device(routed)
task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","slow","lost-disconnect"))
packet = await instruction(routed,connection)
routed["store"].unbind("ou_a") # Deliberately do not flush PubSub notification.
await reply(routed,connection,packet,"must-not-arrive")
with pytest.raises(ConnectionClosed):
await asyncio.wait_for(connection[0].recv(),2)
routed["app"].state.lifecycle.cancel_waiters()
assert (await task)["status"] == "unbound"
async def test_idle_connection_rechecks_database_even_when_pubsub_flush_fails(routed,monkeypatch):
connection = await device(routed)
async def unavailable():
return False
monkeypatch.setattr(routed["app"].state.lifecycle,"flush",unavailable)
routed["store"].unbind("ou_a")
# No client cooperation or kill publication: router's 8-second control check
# must close the old session from authoritative database state.
with pytest.raises(ConnectionClosed):
await asyncio.wait_for(connection[0].recv(),12)
async def test_control_plane_outage_does_not_forward_new_task(routed,monkeypatch):
from fastapi import HTTPException
connection = await device(routed)
async def unavailable(*args,**kwargs):
raise HTTPException(503,"synthetic_control_outage")
monkeypatch.setattr(routed["app"].state.lifecycle,"admission",unavailable)
task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","do not deliver","control-outage"))
try:
with pytest.raises(ConnectionClosed):
await asyncio.wait_for(connection[0].recv(),3)
finally:
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert routed["store"].lifecycle.task("control-outage")["status"] == "unknown"
async def test_dispatch_cleanup_releases_waiter_despite_database_failure(routed,monkeypatch):
connection = await device(routed)
broker = routed["app"].state.broker
task = asyncio.create_task(broker.dispatch("ou_a","pending","cleanup-task"))
await instruction(routed,connection)
def fail(*args,**kwargs):
raise OSError("synthetic cleanup database failure")
monkeypatch.setattr(routed["store"].lifecycle,"transition",fail)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert "cleanup-task" not in broker.pending
assert not await routed["redis"].exists("hn:inflight:"+"a"*32+":cleanup-task")
async def test_router_releases_connection_slot_despite_redis_cleanup_failure(routed,monkeypatch):
import relay.router as module
connection = await device(routed)
async def fail(*args,**kwargs):
raise ConnectionError("synthetic cleanup Redis failure")
monkeypatch.setattr(module,"compare_delete",fail)
await connection[0].close()
async with httpx.AsyncClient(transport=httpx.ASGITransport(routed["router"]),base_url="http://router") as client:
for _ in range(100):
result = await client.get("/ready")
if result.json()["connections"] == 0:
break
await asyncio.sleep(0.01)
assert result.json()["connections"] == 0
async def test_platform_reply_unknown_not_replayed_on_duplicate_event(routed,monkeypatch):
attempts = []
async def ambiguous(message,text):
attempts.append(message)
raise httpx.ReadTimeout("synthetic platform timeout")
monkeypatch.setattr(routed["platform"],"reply",ambiguous)
connection = await device(routed)
data = event(routed,"om-reply-unknown","hello")
await send_event(routed,data)
packet = await instruction(routed,connection)
await reply(routed,connection,packet,"answer")
for _ in range(100):
if routed["store"].lifecycle.task(packet["turn_id"])["status"] == "reply_unknown":
break
await asyncio.sleep(0.01)
assert routed["store"].lifecycle.task(packet["turn_id"])["status"] == "reply_unknown"
await send_event(routed,data)
await asyncio.sleep(0.05)
assert attempts == ["om-reply-unknown"]
async def test_binding_api_validates_signature_before_consuming_code(env):
state = env["store"].lifecycle
env["store"].unbind("ou_a")
state.issue_code("ou_a","fake-binding-once")
host = "a"*32
proof = sign({"host_id":host,"code":"fake-binding-once","ts":int(time.time())},env["keys"][host])
invalid = {**proof,"signature":"bad"}
assert (await env["client"].post("/device/bind",json={"host_id":host,"proof":invalid})).status_code == 403
response = await env["client"].post("/device/bind",json={"host_id":host,"proof":proof})
assert response.status_code == 200 and response.json()["binding_generation"] == 3
assert (await env["client"].post("/device/bind",json={"host_id":host,"proof":proof})).status_code == 404
async def test_dead_worker_tasks_unknown_and_never_replayed(env):
state = env["store"].lifecycle
row = env["store"].device("a"*32)
state.create_task("abandoned-task","ou_a","a"*32,row["binding_generation"],"session","dead-worker",time.time()+300,"chat","om-abandoned")
await env["app"].state.lifecycle.maintain()
assert state.task("abandoned-task")["status"] == "unknown"
assert not state.complete("abandoned-task","a"*32,"session","dead-worker")
assert not state.claim_reply("abandoned-task","ou_a","om-abandoned")
with pytest.raises(ValueError):
state.create_task("abandoned-task","ou_a","a"*32,row["binding_generation"],"new","new-worker",time.time()+300)
async def test_task_unknown_after_deadline_and_replying_after_crash(env):
state = env["store"].lifecycle
gen = env["store"].device("a"*32)["binding_generation"]
state.create_task("expired-task","ou_a","a"*32,gen,"s","missing",time.time()-1)
state.create_task("reply-crash","ou_a","a"*32,gen,"s","missing",time.time()+300)
state.transition("reply-crash","inflight","replying")
await env["app"].state.lifecycle.maintain()
assert state.task("expired-task")["status"] == "unknown"
assert state.task("reply-crash")["status"] == "reply_unknown"
async def test_wrong_session_and_old_generation_cannot_complete_task(env):
state = env["store"].lifecycle
gen = env["store"].device("a"*32)["binding_generation"]
state.create_task("old-generation","ou_a","a"*32,gen,"session","worker",time.time()+300)
assert not state.complete("old-generation","a"*32,"other-session","worker")
env["store"].unbind("ou_a")
env["store"].bind("a"*32,"ou_a")
assert not state.complete("old-generation","a"*32,"session","worker")
async def test_admission_denies_dead_tasks_without_exposing_credentials(env):
gen = env["store"].device("a"*32)["binding_generation"]
env["store"].lifecycle.create_task("task-check","ou_a","a"*32,gen,"s","dead",time.time()+300)
response = await env["client"].post("/internal/devices/"+"a"*32+"/admission",
headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret},
json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"})
assert response.status_code == 403 and "token" not in response.text.lower()