"""Broker-mode invalidation and real loopback WS task routing (no real Work).""" import asyncio import hashlib import json import secrets import socket import time import httpx import pytest import pytest_asyncio import uvicorn from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import x25519 from websockets.asyncio.client import connect from websockets.exceptions import ConnectionClosed from test_credential_api import env, wire, start, authorize from relay.crypto import canonical, load_private, load_public, open_envelope, raw_public, seal, sign from relay.device_lifecycle import AdmissionRequest from relay.router import create_app as router_app @pytest_asyncio.fixture async def routed(env, tmp_path): public_path = tmp_path / "connector.pub" public_path.write_bytes(load_private(env["settings"].connector_key).public_key().public_bytes( serialization.Encoding.PEM,serialization.PublicFormat.SubjectPublicKeyInfo)) settings = env["settings"].model_copy(update={"role":"router","connector_public_key":public_path, "connector_internal_url":"http://connector:8000"}) app = router_app(settings,env["redis"],admission_transport=httpx.ASGITransport(env["app"])) sock = socket.socket() sock.bind(("127.0.0.1",0)) sock.listen(128) port = sock.getsockname()[1] server = uvicorn.Server(uvicorn.Config(app,log_level="critical",access_log=False,lifespan="on")) server_task = asyncio.create_task(server.serve(sockets=[sock])) try: for _ in range(200): if server.started: break if server_task.done(): server_task.result() await asyncio.sleep(0.01) assert server.started value = {**env,"url":f"ws://127.0.0.1:{port}/device/ws","public":load_public(public_path),"sessions":[],"router":app} yield value finally: for ws in locals().get("value",{}).get("sessions",[]): await ws.close() server.should_exit = True await asyncio.wait_for(server_task,5) async def device(env, host="a"*32, *, generation=None, legacy=False, work_files=False): ws = await connect(env["url"],proxy=None) env["sessions"].append(ws) challenge = json.loads(await ws.recv())["challenge"] row = env["store"].device(host) ephemeral = x25519.X25519PrivateKey.generate() session = secrets.token_hex(16) plain = {"host_id":host,"challenge":challenge,"session_id":session, "kx":raw_public(ephemeral.public_key()),"ts":int(time.time())} if not legacy: plain.update(protocol_version=2,environment=env["settings"].environment, app_id=env["settings"].feishu_app_id,binding_owner=row["owner"] or "ou_a", binding_generation=row["binding_generation"] if generation is None else generation) if work_files:plain['work_files']='work-files-v1' await ws.send(json.dumps({"host_id":host,"cert":row["cert"],"proof":sign(plain,env["keys"][host])})) accepted = json.loads(await asyncio.wait_for(ws.recv(),3)) assert accepted["type"] == "connected" return ws,session,ephemeral async def instruction(env, connection, host="a"*32): ws,session,ephemeral = connection packet = json.loads(await asyncio.wait_for(ws.recv(),3)) payload = open_envelope(packet["envelope"],ephemeral,env["public"],host_id=host,session_id=session) return payload async def reply(env, connection, payload, text, host="a"*32): ws,session,_ = connection envelope = seal({"type":"result","turn_id":payload["turn_id"],"result":text}, payload["reply_key"],env["keys"][host],host_id=host,session_id=session) await ws.send(json.dumps({"type":"reply","turn_id":payload["turn_id"],"envelope":envelope})) async def wait_for_reply(platform, count): for _ in range(200): if len(platform.replies) >= count: return await asyncio.sleep(0.01) assert len(platform.replies) >= count def event(env, message_id, text, chat="chat-a", owner="ou_a"): return {"header":{"event_type":"im.message.receive_v1","app_id":env["settings"].feishu_app_id}, "event":{"sender":{"sender_type":"user","sender_id":{"open_id":owner}}, "message":{"message_id":message_id,"chat_type":"p2p","chat_id":chat,"message_type":"text", "content":json.dumps({"text":text})}}} async def send_event(env, data): response = await env["client"].post("/internal/events",json=data, headers={"X-Internal-Secret":env["settings"].internal_secret}) assert response.status_code == 200 async def test_unbind_commits_and_defers_redis_side_effects(env,monkeypatch): await authorize(env) pending = await start(env) with monkeypatch.context() as patch: async def fail(*args,**kwargs): raise ConnectionError("synthetic redis outage") patch.setattr(env["redis"],"publish",fail) response = await env["client"].post("/admin/devices/"+"a"*32+"/unbind", headers={"X-Admin-Secret":env["settings"].admin_secret}) assert response.status_code == 200 and response.json()["propagation"] == "pending" assert env["store"].device("a"*32)["owner"] is None assert env["api"]._row(pending["transaction_id"])["status"] == "invalidated" path = "/work/v2/credentials/access-token" assert (await env["client"].post(path,json=wire(env,path,permission_profile="product-v1"))).status_code == 403 assert await env["app"].state.lifecycle.flush() assert env["store"].db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2 WHERE delivered=0").fetchone()["n"] == 0 async def test_codes_are_one_time_and_pre_unbind_codes_stay_invalid_after_rebind(env): state = env["store"].lifecycle env["store"].unbind("ou_a") state.issue_code("ou_a","fake-code-old") env["store"].unbind("ou_a") with pytest.raises(LookupError): state.consume_code("a"*32,"fake-code-old") state.issue_code("ou_a","fake-code-new") assert state.consume_code("a"*32,"fake-code-new") == "ou_a" with pytest.raises(LookupError): state.consume_code("a"*32,"fake-code-new") dump = "\n".join(env["store"].db.iterdump()) assert "fake-code-old" not in dump and "fake-code-new" not in dump async def test_failed_database_invalidation_rolls_back_binding_and_outbox(env,monkeypatch): state = env["store"].lifecycle original = state._cancel def fail(device,reason): original(device,reason) raise RuntimeError("synthetic transaction abort") monkeypatch.setattr(state,"_cancel",fail) with pytest.raises(RuntimeError): env["store"].unbind("ou_a") current = env["store"].device("a"*32) assert current["owner"] == "ou_a" and current["binding_generation"] == 1 assert state.db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2").fetchone()["n"] == 0 async def test_unbind_outbox_is_persistent_and_reprocessed_by_second_connector(env): from relay.connector import create_app env["store"].unbind("ou_a") second = create_app(env["settings"],env["redis"],env["platform"]) try: assert second.state.store.device("a"*32)["owner"] is None assert await second.state.lifecycle.flush() assert second.state.store.db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2 WHERE delivered=0").fetchone()["n"] == 0 finally: second.state.store.close() async def test_new_online_generation_not_removed_by_old_outbox(env): env["store"].unbind("ou_a") env["store"].bind("a"*32,"ou_a") generation = env["store"].device("a"*32)["binding_generation"] presence = json.dumps({"proof":{"binding_generation":generation}}) await env["redis"].set("hn:online:"+"a"*32,presence) assert await env["app"].state.lifecycle.flush() assert await env["redis"].get("hn:online:"+"a"*32) == presence def notification(env, *, generation=None, include_generation=True): host = "a"*32 payload = {"request_id":secrets.token_hex(8),"kind":"text","text":"synthetic notice","title":"", "oauth_state":"","binding_owner":"ou_a"} if include_generation: payload["binding_generation"] = env["store"].device(host)["binding_generation"] if generation is None else generation proof = sign({"host_id":host,"purpose":"work-notify-v1","ts":int(time.time()),"nonce":secrets.token_hex(16), "payload_hash":hashlib.sha256(canonical(payload)).hexdigest()},env["keys"][host]) return {**payload,"host_id":host,"proof":proof} async def test_notifications_require_current_generation_in_broker_mode(env): stale = notification(env) env["store"].unbind("ou_a") env["store"].bind("a"*32,"ou_a") assert (await env["client"].post("/device/notifications",json=stale)).status_code == 403 assert (await env["client"].post("/device/notifications",json=notification(env,include_generation=False))).status_code == 403 result = await env["client"].post("/device/notifications",json=notification(env)) assert result.json()["status"] == "sent" and len(env["platform"].sent) == 1 async def test_notification_rebind_during_app_token_wait_does_not_send(env,monkeypatch): async def rebind(): env["store"].unbind("ou_a") env["store"].bind("a"*32,"ou_a") return "fake-app-token" monkeypatch.setattr(env["platform"],"tenant_token",rebind) result = await env["client"].post("/device/notifications",json=notification(env)) assert result.json()["status"] == "binding_changed" and not env["platform"].sent async def test_internal_admission_requires_secret_and_current_binding(env): path = "/internal/devices/"+"a"*32+"/admission" body = {"nonce":secrets.token_hex(16)} assert (await env["client"].post(path,json=body)).status_code == 401 assert (await env["client"].post(path,json=body, headers={"X-Internal-Secret":env["settings"].internal_secret})).status_code == 401 headers = {"X-Router-Admission-Secret":env["settings"].router_admission_secret} assert (await env["client"].post(path,json=body,headers=headers)).status_code == 200 env["store"].unbind("ou_a") assert (await env["client"].post(path,json=body,headers=headers)).status_code == 403 async def test_router_credential_cannot_inject_platform_events(env): result = await env["client"].post("/internal/events",json=event(env,"om-injected","hello"), headers={"X-Internal-Secret":env["settings"].router_admission_secret}) assert result.status_code == 401 async def test_device_can_only_unbind_its_current_authenticated_binding(env): path = "/work/v2/device/unbind" invalid = wire(env,path) invalid["proof"] = {} assert (await env["client"].post(path,json=invalid)).status_code == 403 result = await env["client"].post(path,json=wire(env,path)) assert result.status_code == 200 and result.json()["status"] == "unbound" assert env["store"].device("a"*32)["owner"] is None assert env["store"].device("b"*32)["owner"] == "ou_b" async def test_authenticated_unbind_snapshot_cannot_clear_a_new_binding(env): state = env["store"].lifecycle old = env["store"].device("a"*32) env["store"].unbind("ou_a") env["store"].bind("a"*32,"ou_a") with pytest.raises(ValueError): state.invalidate_host("a"*32,"unbind",expected_owner="ou_a",expected_generation=old["binding_generation"]) assert env["store"].device("a"*32)["owner"] == "ou_a" async def test_v2_router_rejects_old_handshake_and_unbound_reconnect(routed): with pytest.raises(ConnectionClosed): await device(routed,legacy=True) routed["store"].unbind("ou_a") with pytest.raises(ConnectionClosed): await device(routed) async def test_two_devices_reverse_results_match_own_tasks(routed): a = await device(routed) b = await device(routed,"b"*32) broker = routed["app"].state.broker ta = asyncio.create_task(broker.dispatch("ou_a","a","turn-a",chat_id="chat-a",source_message_id="om-a")) tb = asyncio.create_task(broker.dispatch("ou_b","b","turn-b",chat_id="chat-b",source_message_id="om-b")) pa,pb = await asyncio.gather(instruction(routed,a),instruction(routed,b,"b"*32)) assert pa["task_context"]["source_message_id"] == "om-a" assert pb["task_context"]["owner"] == "ou_b" assert pa["task_context"]["conversation_id"] != pb["task_context"]["conversation_id"] await reply(routed,b,pb,"answer-b","b"*32) await reply(routed,a,pa,"answer-a") ra,rb = await asyncio.gather(ta,tb) assert (ra["result"],rb["result"]) == ("answer-a","answer-b") state = routed["store"].lifecycle assert not state.claim_reply("turn-a","ou_a","om-b") assert state.claim_reply("turn-a","ou_a","om-a") assert not state.claim_reply("turn-a","ou_a","om-a") async def test_same_owner_two_conversations_reply_to_original_message(routed): connection = await device(routed) await send_event(routed,event(routed,"om-first","first","chat-first")) first = await instruction(routed,connection) await send_event(routed,event(routed,"om-second","second","chat-second")) second = await instruction(routed,connection) assert first["task_context"]["conversation_id"] != second["task_context"]["conversation_id"] await reply(routed,connection,second,"second-answer") await reply(routed,connection,first,"first-answer") await wait_for_reply(routed["platform"],2) assert routed["platform"].replies == [("om-second","second-answer"),("om-first","first-answer")] await send_event(routed,event(routed,"om-first","first","chat-first")) await asyncio.sleep(0.05) assert len(routed["platform"].replies) == 2 async def test_robot_unbind_discards_previous_result_and_closes_connection(routed): connection = await device(routed) await send_event(routed,event(routed,"om-old-task","slow")) payload = await instruction(routed,connection) await send_event(routed,event(routed,"om-unbind","/解绑")) await wait_for_reply(routed["platform"],1) assert routed["platform"].replies[0][0] == "om-unbind" assert routed["store"].device("a"*32)["owner"] is None assert routed["store"].lifecycle.task(payload["turn_id"])["status"] == "unbound" with pytest.raises(ConnectionClosed): await asyncio.wait_for(connection[0].recv(),2) await asyncio.sleep(0.05) assert all(message != "om-old-task" for message,_ in routed["platform"].replies) async def test_revoke_has_durable_state_and_blocks_connection(routed): connection = await device(routed) response = await routed["client"].post("/admin/devices/"+"a"*32+"/revoke", headers={"X-Admin-Secret":routed["settings"].admin_secret}) assert response.status_code == 200 assert routed["store"].device("a"*32)["revoked"] == 1 with pytest.raises(ConnectionClosed): await asyncio.wait_for(connection[0].recv(),2) async def test_delayed_kill_for_old_generation_does_not_disconnect_new_session(routed): old = await device(routed) routed["store"].unbind("ou_a") await routed["app"].state.lifecycle.flush() with pytest.raises(ConnectionClosed): await asyncio.wait_for(old[0].recv(),2) routed["store"].bind("a"*32,"ou_a") fresh = await device(routed) with routed["store"].db: routed["store"].db.execute("UPDATE work_binding_events_v2 SET delivered=0") await routed["app"].state.lifecycle.flush() task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","hello","fresh-task")) packet = await instruction(routed,fresh) await reply(routed,fresh,packet,"fresh-answer") assert (await task)["result"] == "fresh-answer" async def test_lost_disconnect_notification_still_rejects_old_reply(routed): connection = await device(routed) task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","slow","lost-disconnect")) packet = await instruction(routed,connection) routed["store"].unbind("ou_a") # Deliberately do not flush PubSub notification. await reply(routed,connection,packet,"must-not-arrive") with pytest.raises(ConnectionClosed): await asyncio.wait_for(connection[0].recv(),2) routed["app"].state.lifecycle.cancel_waiters() assert (await task)["status"] == "unbound" async def test_idle_connection_rechecks_database_even_when_pubsub_flush_fails(routed,monkeypatch): connection = await device(routed) async def unavailable(): return False monkeypatch.setattr(routed["app"].state.lifecycle,"flush",unavailable) routed["store"].unbind("ou_a") # No client cooperation or kill publication: router's 8-second control check # must close the old session from authoritative database state. with pytest.raises(ConnectionClosed): await asyncio.wait_for(connection[0].recv(),12) async def test_control_plane_outage_does_not_forward_new_task(routed,monkeypatch): from fastapi import HTTPException connection = await device(routed) async def unavailable(*args,**kwargs): raise HTTPException(503,"synthetic_control_outage") monkeypatch.setattr(routed["app"].state.lifecycle,"admission",unavailable) task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","do not deliver","control-outage")) try: with pytest.raises(ConnectionClosed): await asyncio.wait_for(connection[0].recv(),3) finally: task.cancel() with pytest.raises(asyncio.CancelledError): await task assert routed["store"].lifecycle.task("control-outage")["status"] == "unknown" async def test_dispatch_cleanup_releases_waiter_despite_database_failure(routed,monkeypatch): connection = await device(routed) broker = routed["app"].state.broker task = asyncio.create_task(broker.dispatch("ou_a","pending","cleanup-task")) await instruction(routed,connection) def fail(*args,**kwargs): raise OSError("synthetic cleanup database failure") monkeypatch.setattr(routed["store"].lifecycle,"transition",fail) task.cancel() with pytest.raises(asyncio.CancelledError): await task assert "cleanup-task" not in broker.pending assert not await routed["redis"].exists("hn:inflight:"+"a"*32+":cleanup-task") async def test_router_releases_connection_slot_despite_redis_cleanup_failure(routed,monkeypatch): import relay.router as module connection = await device(routed) async def fail(*args,**kwargs): raise ConnectionError("synthetic cleanup Redis failure") monkeypatch.setattr(module,"compare_delete",fail) await connection[0].close() async with httpx.AsyncClient(transport=httpx.ASGITransport(routed["router"]),base_url="http://router") as client: for _ in range(100): result = await client.get("/ready") if result.json()["connections"] == 0: break await asyncio.sleep(0.01) assert result.json()["connections"] == 0 async def test_platform_reply_unknown_not_replayed_on_duplicate_event(routed,monkeypatch): attempts = [] async def ambiguous(message,text): attempts.append(message) raise httpx.ReadTimeout("synthetic platform timeout") monkeypatch.setattr(routed["platform"],"reply",ambiguous) connection = await device(routed) data = event(routed,"om-reply-unknown","hello") await send_event(routed,data) packet = await instruction(routed,connection) await reply(routed,connection,packet,"answer") for _ in range(100): if routed["store"].lifecycle.task(packet["turn_id"])["status"] == "reply_unknown": break await asyncio.sleep(0.01) assert routed["store"].lifecycle.task(packet["turn_id"])["status"] == "reply_unknown" await send_event(routed,data) await asyncio.sleep(0.05) assert attempts == ["om-reply-unknown"] async def test_binding_api_validates_signature_before_consuming_code(env): state = env["store"].lifecycle env["store"].unbind("ou_a") state.issue_code("ou_a","fake-binding-once") host = "a"*32 proof = sign({"host_id":host,"code":"fake-binding-once","ts":int(time.time())},env["keys"][host]) invalid = {**proof,"signature":"bad"} assert (await env["client"].post("/device/bind",json={"host_id":host,"proof":invalid})).status_code == 403 response = await env["client"].post("/device/bind",json={"host_id":host,"proof":proof}) assert response.status_code == 200 and response.json()["binding_generation"] == 3 assert (await env["client"].post("/device/bind",json={"host_id":host,"proof":proof})).status_code == 404 async def test_dead_worker_tasks_unknown_and_never_replayed(env): state = env["store"].lifecycle row = env["store"].device("a"*32) state.create_task("abandoned-task","ou_a","a"*32,row["binding_generation"],"session","dead-worker",time.time()+300,"chat","om-abandoned") await env["app"].state.lifecycle.maintain() assert state.task("abandoned-task")["status"] == "unknown" assert not state.complete("abandoned-task","a"*32,"session","dead-worker") assert not state.claim_reply("abandoned-task","ou_a","om-abandoned") with pytest.raises(ValueError): state.create_task("abandoned-task","ou_a","a"*32,row["binding_generation"],"new","new-worker",time.time()+300) async def test_task_unknown_after_deadline_and_replying_after_crash(env): state = env["store"].lifecycle gen = env["store"].device("a"*32)["binding_generation"] state.create_task("expired-task","ou_a","a"*32,gen,"s","missing",time.time()-1) state.create_task("reply-crash","ou_a","a"*32,gen,"s","missing",time.time()+300) state.transition("reply-crash","inflight","replying") await env["app"].state.lifecycle.maintain() assert state.task("expired-task")["status"] == "unknown" assert state.task("reply-crash")["status"] == "reply_unknown" async def test_wrong_session_and_old_generation_cannot_complete_task(env): state = env["store"].lifecycle gen = env["store"].device("a"*32)["binding_generation"] state.create_task("old-generation","ou_a","a"*32,gen,"session","worker",time.time()+300) assert not state.complete("old-generation","a"*32,"other-session","worker") env["store"].unbind("ou_a") env["store"].bind("a"*32,"ou_a") assert not state.complete("old-generation","a"*32,"session","worker") async def test_admission_denies_dead_tasks_without_exposing_credentials(env): gen = env["store"].device("a"*32)["binding_generation"] env["store"].lifecycle.create_task("task-check","ou_a","a"*32,gen,"s","dead",time.time()+300) response = await env["client"].post("/internal/devices/"+"a"*32+"/admission", headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret}, json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"}) assert response.status_code == 403 and "token" not in response.text.lower() async def test_native_progress_keeps_task_until_final_and_original_reply_once(routed): routed['settings'].request_timeout=1 connection=await device(routed) await send_event(routed,event(routed,'om-progress','synthetic slow task')) payload=await instruction(routed,connection) ws,session,_=connection for _ in range(4): envelope=seal({'type':'progress','turn_id':payload['turn_id']},payload['reply_key'], routed['keys']['a'*32],host_id='a'*32,session_id=session) await ws.send(json.dumps({'type':'reply','turn_id':payload['turn_id'],'envelope':envelope})) await asyncio.sleep(.4) assert not routed['platform'].replies assert routed['store'].lifecycle.task(payload['turn_id'])['status']=='inflight' await reply(routed,connection,payload,'synthetic final') await wait_for_reply(routed['platform'],1) assert routed['platform'].replies==[('om-progress','synthetic final')] await send_event(routed,event(routed,'om-progress','synthetic slow task')) await asyncio.sleep(.05) assert len(routed['platform'].replies)==1 async def test_progress_wait_cannot_extend_hard_deadline_or_hide_idle_failure(): from relay.connector import Broker,Pending from types import SimpleNamespace broker=object.__new__(Broker) broker.s=SimpleNamespace(task_tracking_timeout=.12,request_timeout=.08) item=Pending(asyncio.get_running_loop().create_future(),None,None,'host','session') async def pulses(): while True: item.progress.set() await asyncio.sleep(.02) pulse=asyncio.create_task(pulses()) try: with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item) assert not item.future.done() finally: pulse.cancel() try:await pulse except asyncio.CancelledError:pass broker.s.task_tracking_timeout=1 with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item) async def test_unbound_or_wrong_session_progress_cannot_keep_task_alive(env): state=env['store'].lifecycle state.create_task('progress-bound','ou_a','a'*32,1,'session-a','worker-a',time.time()+60) assert state.progress('progress-bound','a'*32,'session-a','worker-a') assert not state.progress('progress-bound','a'*32,'session-b','worker-a') assert not state.progress('progress-bound','a'*32,'session-a','worker-b') env['store'].unbind('ou_a') assert not state.progress('progress-bound','a'*32,'session-a','worker-a')