Files

534 lines
25 KiB
Python

"""Broker-mode invalidation and real loopback WS task routing (no real Work)."""
import asyncio
import hashlib
import json
import secrets
import socket
import time
import httpx
import pytest
import pytest_asyncio
import uvicorn
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import x25519
from websockets.asyncio.client import connect
from websockets.exceptions import ConnectionClosed
from test_credential_api import env, wire, start, authorize
from relay.crypto import canonical, load_private, load_public, open_envelope, raw_public, seal, sign
from relay.device_lifecycle import AdmissionRequest
from relay.router import create_app as router_app
@pytest_asyncio.fixture
async def routed(env, tmp_path):
public_path = tmp_path / "connector.pub"
public_path.write_bytes(load_private(env["settings"].connector_key).public_key().public_bytes(
serialization.Encoding.PEM,serialization.PublicFormat.SubjectPublicKeyInfo))
settings = env["settings"].model_copy(update={"role":"router","connector_public_key":public_path,
"connector_internal_url":"http://connector:8000"})
app = router_app(settings,env["redis"],admission_transport=httpx.ASGITransport(env["app"]))
sock = socket.socket()
sock.bind(("127.0.0.1",0))
sock.listen(128)
port = sock.getsockname()[1]
server = uvicorn.Server(uvicorn.Config(app,log_level="critical",access_log=False,lifespan="on"))
server_task = asyncio.create_task(server.serve(sockets=[sock]))
try:
for _ in range(200):
if server.started:
break
if server_task.done():
server_task.result()
await asyncio.sleep(0.01)
assert server.started
value = {**env,"url":f"ws://127.0.0.1:{port}/device/ws","public":load_public(public_path),"sessions":[],"router":app}
yield value
finally:
for ws in locals().get("value",{}).get("sessions",[]):
await ws.close()
server.should_exit = True
await asyncio.wait_for(server_task,5)
async def device(env, host="a"*32, *, generation=None, legacy=False, work_files=False):
ws = await connect(env["url"],proxy=None)
env["sessions"].append(ws)
challenge = json.loads(await ws.recv())["challenge"]
row = env["store"].device(host)
ephemeral = x25519.X25519PrivateKey.generate()
session = secrets.token_hex(16)
plain = {"host_id":host,"challenge":challenge,"session_id":session,
"kx":raw_public(ephemeral.public_key()),"ts":int(time.time())}
if not legacy:
plain.update(protocol_version=2,environment=env["settings"].environment,
app_id=env["settings"].feishu_app_id,binding_owner=row["owner"] or "ou_a",
binding_generation=row["binding_generation"] if generation is None else generation)
if work_files:plain['work_files']='work-files-v1'
await ws.send(json.dumps({"host_id":host,"cert":row["cert"],"proof":sign(plain,env["keys"][host])}))
accepted = json.loads(await asyncio.wait_for(ws.recv(),3))
assert accepted["type"] == "connected"
return ws,session,ephemeral
async def instruction(env, connection, host="a"*32):
ws,session,ephemeral = connection
packet = json.loads(await asyncio.wait_for(ws.recv(),3))
payload = open_envelope(packet["envelope"],ephemeral,env["public"],host_id=host,session_id=session)
return payload
async def reply(env, connection, payload, text, host="a"*32):
ws,session,_ = connection
envelope = seal({"type":"result","turn_id":payload["turn_id"],"result":text},
payload["reply_key"],env["keys"][host],host_id=host,session_id=session)
await ws.send(json.dumps({"type":"reply","turn_id":payload["turn_id"],"envelope":envelope}))
async def wait_for_reply(platform, count):
for _ in range(200):
if len(platform.replies) >= count:
return
await asyncio.sleep(0.01)
assert len(platform.replies) >= count
def event(env, message_id, text, chat="chat-a", owner="ou_a"):
return {"header":{"event_type":"im.message.receive_v1","app_id":env["settings"].feishu_app_id},
"event":{"sender":{"sender_type":"user","sender_id":{"open_id":owner}},
"message":{"message_id":message_id,"chat_type":"p2p","chat_id":chat,"message_type":"text",
"content":json.dumps({"text":text})}}}
async def send_event(env, data):
response = await env["client"].post("/internal/events",json=data,
headers={"X-Internal-Secret":env["settings"].internal_secret})
assert response.status_code == 200
async def test_unbind_commits_and_defers_redis_side_effects(env,monkeypatch):
await authorize(env)
pending = await start(env)
with monkeypatch.context() as patch:
async def fail(*args,**kwargs):
raise ConnectionError("synthetic redis outage")
patch.setattr(env["redis"],"publish",fail)
response = await env["client"].post("/admin/devices/"+"a"*32+"/unbind",
headers={"X-Admin-Secret":env["settings"].admin_secret})
assert response.status_code == 200 and response.json()["propagation"] == "pending"
assert env["store"].device("a"*32)["owner"] is None
assert env["api"]._row(pending["transaction_id"])["status"] == "invalidated"
path = "/work/v2/credentials/access-token"
assert (await env["client"].post(path,json=wire(env,path,permission_profile="product-v1"))).status_code == 403
assert await env["app"].state.lifecycle.flush()
assert env["store"].db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2 WHERE delivered=0").fetchone()["n"] == 0
async def test_codes_are_one_time_and_pre_unbind_codes_stay_invalid_after_rebind(env):
state = env["store"].lifecycle
env["store"].unbind("ou_a")
state.issue_code("ou_a","fake-code-old")
env["store"].unbind("ou_a")
with pytest.raises(LookupError):
state.consume_code("a"*32,"fake-code-old")
state.issue_code("ou_a","fake-code-new")
assert state.consume_code("a"*32,"fake-code-new") == "ou_a"
with pytest.raises(LookupError):
state.consume_code("a"*32,"fake-code-new")
dump = "\n".join(env["store"].db.iterdump())
assert "fake-code-old" not in dump and "fake-code-new" not in dump
async def test_failed_database_invalidation_rolls_back_binding_and_outbox(env,monkeypatch):
state = env["store"].lifecycle
original = state._cancel
def fail(device,reason):
original(device,reason)
raise RuntimeError("synthetic transaction abort")
monkeypatch.setattr(state,"_cancel",fail)
with pytest.raises(RuntimeError):
env["store"].unbind("ou_a")
current = env["store"].device("a"*32)
assert current["owner"] == "ou_a" and current["binding_generation"] == 1
assert state.db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2").fetchone()["n"] == 0
async def test_unbind_outbox_is_persistent_and_reprocessed_by_second_connector(env):
from relay.connector import create_app
env["store"].unbind("ou_a")
second = create_app(env["settings"],env["redis"],env["platform"])
try:
assert second.state.store.device("a"*32)["owner"] is None
assert await second.state.lifecycle.flush()
assert second.state.store.db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2 WHERE delivered=0").fetchone()["n"] == 0
finally:
second.state.store.close()
async def test_new_online_generation_not_removed_by_old_outbox(env):
env["store"].unbind("ou_a")
env["store"].bind("a"*32,"ou_a")
generation = env["store"].device("a"*32)["binding_generation"]
presence = json.dumps({"proof":{"binding_generation":generation}})
await env["redis"].set("hn:online:"+"a"*32,presence)
assert await env["app"].state.lifecycle.flush()
assert await env["redis"].get("hn:online:"+"a"*32) == presence
def notification(env, *, generation=None, include_generation=True):
host = "a"*32
payload = {"request_id":secrets.token_hex(8),"kind":"text","text":"synthetic notice","title":"",
"oauth_state":"","binding_owner":"ou_a"}
if include_generation:
payload["binding_generation"] = env["store"].device(host)["binding_generation"] if generation is None else generation
proof = sign({"host_id":host,"purpose":"work-notify-v1","ts":int(time.time()),"nonce":secrets.token_hex(16),
"payload_hash":hashlib.sha256(canonical(payload)).hexdigest()},env["keys"][host])
return {**payload,"host_id":host,"proof":proof}
async def test_notifications_require_current_generation_in_broker_mode(env):
stale = notification(env)
env["store"].unbind("ou_a")
env["store"].bind("a"*32,"ou_a")
assert (await env["client"].post("/device/notifications",json=stale)).status_code == 403
assert (await env["client"].post("/device/notifications",json=notification(env,include_generation=False))).status_code == 403
result = await env["client"].post("/device/notifications",json=notification(env))
assert result.json()["status"] == "sent" and len(env["platform"].sent) == 1
async def test_notification_rebind_during_app_token_wait_does_not_send(env,monkeypatch):
async def rebind():
env["store"].unbind("ou_a")
env["store"].bind("a"*32,"ou_a")
return "fake-app-token"
monkeypatch.setattr(env["platform"],"tenant_token",rebind)
result = await env["client"].post("/device/notifications",json=notification(env))
assert result.json()["status"] == "binding_changed" and not env["platform"].sent
async def test_internal_admission_requires_secret_and_current_binding(env):
path = "/internal/devices/"+"a"*32+"/admission"
body = {"nonce":secrets.token_hex(16)}
assert (await env["client"].post(path,json=body)).status_code == 401
assert (await env["client"].post(path,json=body,
headers={"X-Internal-Secret":env["settings"].internal_secret})).status_code == 401
headers = {"X-Router-Admission-Secret":env["settings"].router_admission_secret}
assert (await env["client"].post(path,json=body,headers=headers)).status_code == 200
env["store"].unbind("ou_a")
assert (await env["client"].post(path,json=body,headers=headers)).status_code == 403
async def test_router_credential_cannot_inject_platform_events(env):
result = await env["client"].post("/internal/events",json=event(env,"om-injected","hello"),
headers={"X-Internal-Secret":env["settings"].router_admission_secret})
assert result.status_code == 401
async def test_device_can_only_unbind_its_current_authenticated_binding(env):
path = "/work/v2/device/unbind"
invalid = wire(env,path)
invalid["proof"] = {}
assert (await env["client"].post(path,json=invalid)).status_code == 403
result = await env["client"].post(path,json=wire(env,path))
assert result.status_code == 200 and result.json()["status"] == "unbound"
assert env["store"].device("a"*32)["owner"] is None
assert env["store"].device("b"*32)["owner"] == "ou_b"
async def test_authenticated_unbind_snapshot_cannot_clear_a_new_binding(env):
state = env["store"].lifecycle
old = env["store"].device("a"*32)
env["store"].unbind("ou_a")
env["store"].bind("a"*32,"ou_a")
with pytest.raises(ValueError):
state.invalidate_host("a"*32,"unbind",expected_owner="ou_a",expected_generation=old["binding_generation"])
assert env["store"].device("a"*32)["owner"] == "ou_a"
async def test_v2_router_rejects_old_handshake_and_unbound_reconnect(routed):
with pytest.raises(ConnectionClosed):
await device(routed,legacy=True)
routed["store"].unbind("ou_a")
with pytest.raises(ConnectionClosed):
await device(routed)
async def test_two_devices_reverse_results_match_own_tasks(routed):
a = await device(routed)
b = await device(routed,"b"*32)
broker = routed["app"].state.broker
ta = asyncio.create_task(broker.dispatch("ou_a","a","turn-a",chat_id="chat-a",source_message_id="om-a"))
tb = asyncio.create_task(broker.dispatch("ou_b","b","turn-b",chat_id="chat-b",source_message_id="om-b"))
pa,pb = await asyncio.gather(instruction(routed,a),instruction(routed,b,"b"*32))
assert pa["task_context"]["source_message_id"] == "om-a"
assert pb["task_context"]["owner"] == "ou_b"
assert pa["task_context"]["conversation_id"] != pb["task_context"]["conversation_id"]
await reply(routed,b,pb,"answer-b","b"*32)
await reply(routed,a,pa,"answer-a")
ra,rb = await asyncio.gather(ta,tb)
assert (ra["result"],rb["result"]) == ("answer-a","answer-b")
state = routed["store"].lifecycle
assert not state.claim_reply("turn-a","ou_a","om-b")
assert state.claim_reply("turn-a","ou_a","om-a")
assert not state.claim_reply("turn-a","ou_a","om-a")
async def test_same_owner_two_conversations_reply_to_original_message(routed):
connection = await device(routed)
await send_event(routed,event(routed,"om-first","first","chat-first"))
first = await instruction(routed,connection)
await send_event(routed,event(routed,"om-second","second","chat-second"))
second = await instruction(routed,connection)
assert first["task_context"]["conversation_id"] != second["task_context"]["conversation_id"]
await reply(routed,connection,second,"second-answer")
await reply(routed,connection,first,"first-answer")
await wait_for_reply(routed["platform"],2)
assert routed["platform"].replies == [("om-second","second-answer"),("om-first","first-answer")]
await send_event(routed,event(routed,"om-first","first","chat-first"))
await asyncio.sleep(0.05)
assert len(routed["platform"].replies) == 2
async def test_robot_unbind_discards_previous_result_and_closes_connection(routed):
connection = await device(routed)
await send_event(routed,event(routed,"om-old-task","slow"))
payload = await instruction(routed,connection)
await send_event(routed,event(routed,"om-unbind","/解绑"))
await wait_for_reply(routed["platform"],1)
assert routed["platform"].replies[0][0] == "om-unbind"
assert routed["store"].device("a"*32)["owner"] is None
assert routed["store"].lifecycle.task(payload["turn_id"])["status"] == "unbound"
with pytest.raises(ConnectionClosed):
await asyncio.wait_for(connection[0].recv(),2)
await asyncio.sleep(0.05)
assert all(message != "om-old-task" for message,_ in routed["platform"].replies)
async def test_revoke_has_durable_state_and_blocks_connection(routed):
connection = await device(routed)
response = await routed["client"].post("/admin/devices/"+"a"*32+"/revoke",
headers={"X-Admin-Secret":routed["settings"].admin_secret})
assert response.status_code == 200
assert routed["store"].device("a"*32)["revoked"] == 1
with pytest.raises(ConnectionClosed):
await asyncio.wait_for(connection[0].recv(),2)
async def test_delayed_kill_for_old_generation_does_not_disconnect_new_session(routed):
old = await device(routed)
routed["store"].unbind("ou_a")
await routed["app"].state.lifecycle.flush()
with pytest.raises(ConnectionClosed):
await asyncio.wait_for(old[0].recv(),2)
routed["store"].bind("a"*32,"ou_a")
fresh = await device(routed)
with routed["store"].db:
routed["store"].db.execute("UPDATE work_binding_events_v2 SET delivered=0")
await routed["app"].state.lifecycle.flush()
task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","hello","fresh-task"))
packet = await instruction(routed,fresh)
await reply(routed,fresh,packet,"fresh-answer")
assert (await task)["result"] == "fresh-answer"
async def test_lost_disconnect_notification_still_rejects_old_reply(routed):
connection = await device(routed)
task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","slow","lost-disconnect"))
packet = await instruction(routed,connection)
routed["store"].unbind("ou_a") # Deliberately do not flush PubSub notification.
await reply(routed,connection,packet,"must-not-arrive")
with pytest.raises(ConnectionClosed):
await asyncio.wait_for(connection[0].recv(),2)
routed["app"].state.lifecycle.cancel_waiters()
assert (await task)["status"] == "unbound"
async def test_idle_connection_rechecks_database_even_when_pubsub_flush_fails(routed,monkeypatch):
connection = await device(routed)
async def unavailable():
return False
monkeypatch.setattr(routed["app"].state.lifecycle,"flush",unavailable)
routed["store"].unbind("ou_a")
# No client cooperation or kill publication: router's 8-second control check
# must close the old session from authoritative database state.
with pytest.raises(ConnectionClosed):
await asyncio.wait_for(connection[0].recv(),12)
async def test_control_plane_outage_does_not_forward_new_task(routed,monkeypatch):
from fastapi import HTTPException
connection = await device(routed)
async def unavailable(*args,**kwargs):
raise HTTPException(503,"synthetic_control_outage")
monkeypatch.setattr(routed["app"].state.lifecycle,"admission",unavailable)
task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","do not deliver","control-outage"))
try:
with pytest.raises(ConnectionClosed):
await asyncio.wait_for(connection[0].recv(),3)
finally:
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert routed["store"].lifecycle.task("control-outage")["status"] == "unknown"
async def test_dispatch_cleanup_releases_waiter_despite_database_failure(routed,monkeypatch):
connection = await device(routed)
broker = routed["app"].state.broker
task = asyncio.create_task(broker.dispatch("ou_a","pending","cleanup-task"))
await instruction(routed,connection)
def fail(*args,**kwargs):
raise OSError("synthetic cleanup database failure")
monkeypatch.setattr(routed["store"].lifecycle,"transition",fail)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert "cleanup-task" not in broker.pending
assert not await routed["redis"].exists("hn:inflight:"+"a"*32+":cleanup-task")
async def test_router_releases_connection_slot_despite_redis_cleanup_failure(routed,monkeypatch):
import relay.router as module
connection = await device(routed)
async def fail(*args,**kwargs):
raise ConnectionError("synthetic cleanup Redis failure")
monkeypatch.setattr(module,"compare_delete",fail)
await connection[0].close()
async with httpx.AsyncClient(transport=httpx.ASGITransport(routed["router"]),base_url="http://router") as client:
for _ in range(100):
result = await client.get("/ready")
if result.json()["connections"] == 0:
break
await asyncio.sleep(0.01)
assert result.json()["connections"] == 0
async def test_platform_reply_unknown_not_replayed_on_duplicate_event(routed,monkeypatch):
attempts = []
async def ambiguous(message,text):
attempts.append(message)
raise httpx.ReadTimeout("synthetic platform timeout")
monkeypatch.setattr(routed["platform"],"reply",ambiguous)
connection = await device(routed)
data = event(routed,"om-reply-unknown","hello")
await send_event(routed,data)
packet = await instruction(routed,connection)
await reply(routed,connection,packet,"answer")
for _ in range(100):
if routed["store"].lifecycle.task(packet["turn_id"])["status"] == "reply_unknown":
break
await asyncio.sleep(0.01)
assert routed["store"].lifecycle.task(packet["turn_id"])["status"] == "reply_unknown"
await send_event(routed,data)
await asyncio.sleep(0.05)
assert attempts == ["om-reply-unknown"]
async def test_binding_api_validates_signature_before_consuming_code(env):
state = env["store"].lifecycle
env["store"].unbind("ou_a")
state.issue_code("ou_a","fake-binding-once")
host = "a"*32
proof = sign({"host_id":host,"code":"fake-binding-once","ts":int(time.time())},env["keys"][host])
invalid = {**proof,"signature":"bad"}
assert (await env["client"].post("/device/bind",json={"host_id":host,"proof":invalid})).status_code == 403
response = await env["client"].post("/device/bind",json={"host_id":host,"proof":proof})
assert response.status_code == 200 and response.json()["binding_generation"] == 3
assert (await env["client"].post("/device/bind",json={"host_id":host,"proof":proof})).status_code == 404
async def test_dead_worker_tasks_unknown_and_never_replayed(env):
state = env["store"].lifecycle
row = env["store"].device("a"*32)
state.create_task("abandoned-task","ou_a","a"*32,row["binding_generation"],"session","dead-worker",time.time()+300,"chat","om-abandoned")
await env["app"].state.lifecycle.maintain()
assert state.task("abandoned-task")["status"] == "unknown"
assert not state.complete("abandoned-task","a"*32,"session","dead-worker")
assert not state.claim_reply("abandoned-task","ou_a","om-abandoned")
with pytest.raises(ValueError):
state.create_task("abandoned-task","ou_a","a"*32,row["binding_generation"],"new","new-worker",time.time()+300)
async def test_task_unknown_after_deadline_and_replying_after_crash(env):
state = env["store"].lifecycle
gen = env["store"].device("a"*32)["binding_generation"]
state.create_task("expired-task","ou_a","a"*32,gen,"s","missing",time.time()-1)
state.create_task("reply-crash","ou_a","a"*32,gen,"s","missing",time.time()+300)
state.transition("reply-crash","inflight","replying")
await env["app"].state.lifecycle.maintain()
assert state.task("expired-task")["status"] == "unknown"
assert state.task("reply-crash")["status"] == "reply_unknown"
async def test_wrong_session_and_old_generation_cannot_complete_task(env):
state = env["store"].lifecycle
gen = env["store"].device("a"*32)["binding_generation"]
state.create_task("old-generation","ou_a","a"*32,gen,"session","worker",time.time()+300)
assert not state.complete("old-generation","a"*32,"other-session","worker")
env["store"].unbind("ou_a")
env["store"].bind("a"*32,"ou_a")
assert not state.complete("old-generation","a"*32,"session","worker")
async def test_admission_denies_dead_tasks_without_exposing_credentials(env):
gen = env["store"].device("a"*32)["binding_generation"]
env["store"].lifecycle.create_task("task-check","ou_a","a"*32,gen,"s","dead",time.time()+300)
response = await env["client"].post("/internal/devices/"+"a"*32+"/admission",
headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret},
json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"})
assert response.status_code == 403 and "token" not in response.text.lower()
async def test_native_progress_keeps_task_until_final_and_original_reply_once(routed):
routed['settings'].request_timeout=1
connection=await device(routed)
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
payload=await instruction(routed,connection)
ws,session,_=connection
for _ in range(4):
envelope=seal({'type':'progress','turn_id':payload['turn_id']},payload['reply_key'],
routed['keys']['a'*32],host_id='a'*32,session_id=session)
await ws.send(json.dumps({'type':'reply','turn_id':payload['turn_id'],'envelope':envelope}))
await asyncio.sleep(.4)
assert not routed['platform'].replies
assert routed['store'].lifecycle.task(payload['turn_id'])['status']=='inflight'
await reply(routed,connection,payload,'synthetic final')
await wait_for_reply(routed['platform'],1)
assert routed['platform'].replies==[('om-progress','synthetic final')]
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
await asyncio.sleep(.05)
assert len(routed['platform'].replies)==1
async def test_progress_wait_cannot_extend_hard_deadline_or_hide_idle_failure():
from relay.connector import Broker,Pending
from types import SimpleNamespace
broker=object.__new__(Broker)
broker.s=SimpleNamespace(task_tracking_timeout=.12,request_timeout=.08)
item=Pending(asyncio.get_running_loop().create_future(),None,None,'host','session')
async def pulses():
while True:
item.progress.set()
await asyncio.sleep(.02)
pulse=asyncio.create_task(pulses())
try:
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
assert not item.future.done()
finally:
pulse.cancel()
try:await pulse
except asyncio.CancelledError:pass
broker.s.task_tracking_timeout=1
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
async def test_unbound_or_wrong_session_progress_cannot_keep_task_alive(env):
state=env['store'].lifecycle
state.create_task('progress-bound','ou_a','a'*32,1,'session-a','worker-a',time.time()+60)
assert state.progress('progress-bound','a'*32,'session-a','worker-a')
assert not state.progress('progress-bound','a'*32,'session-b','worker-a')
assert not state.progress('progress-bound','a'*32,'session-a','worker-b')
env['store'].unbind('ou_a')
assert not state.progress('progress-bound','a'*32,'session-a','worker-a')