Unify broker-mode binding changes in database transactions: advance device generation, invalidate pending authorizations/tasks and prior pairing-code epochs, and persist audit plus invalidation outbox. Add authenticated device unbind alongside robot/admin paths. Pairing codes are hashed, generation- fenced and consumed atomically with binding. Replay Redis invalidation effects without undoing committed unbind. Guard online cleanup and delayed kill events by generation. Introduce a separate router admission credential with no platform-event injection authority. Router verifies signed nonce-bound internal admission on handshake, task traffic and periodic checks; reject stale sessions and control-plane outages. Persist task owner/device/generation/session/worker and original message mapping. Sign routing metadata, verify late results, claim replies once and mark ambiguous/crashed tasks unknown instead of replaying writes. Fence notifications across same-user rebind. Release in-memory slots/waiters even when database or Redis cleanup fails. Validation: 213 passed, 3 skipped (real PostgreSQL not configured), 30 existing uvicorn/websockets deprecation warnings. New lifecycle coverage: 28 tests, including real loopback WebSocket isolation, reverse completion, missed kill, periodic unbind checks, outbox recovery, transaction rollback and cleanup failures. Syntax, docs and staged diff checks pass. Default legacy mode remains unchanged. Broker mode now requires coordinated router admission configuration and a v2 client handshake. No production ECS configuration, real credentials or client installer was changed. PostgreSQL, real Redis multi-node, ARM64 and actual Work/client rollout remain stage 5.
480 lines
22 KiB
Python
480 lines
22 KiB
Python
"""Broker-mode invalidation and real loopback WS task routing (no real Work)."""
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import secrets
|
|
import socket
|
|
import time
|
|
|
|
import httpx
|
|
import pytest
|
|
import pytest_asyncio
|
|
import uvicorn
|
|
from cryptography.hazmat.primitives import serialization
|
|
from cryptography.hazmat.primitives.asymmetric import x25519
|
|
from websockets.asyncio.client import connect
|
|
from websockets.exceptions import ConnectionClosed
|
|
|
|
from test_credential_api import env, wire, start, authorize
|
|
from relay.crypto import canonical, load_private, load_public, open_envelope, raw_public, seal, sign
|
|
from relay.device_lifecycle import AdmissionRequest
|
|
from relay.router import create_app as router_app
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def routed(env, tmp_path):
|
|
public_path = tmp_path / "connector.pub"
|
|
public_path.write_bytes(load_private(env["settings"].connector_key).public_key().public_bytes(
|
|
serialization.Encoding.PEM,serialization.PublicFormat.SubjectPublicKeyInfo))
|
|
settings = env["settings"].model_copy(update={"role":"router","connector_public_key":public_path,
|
|
"connector_internal_url":"http://connector:8000"})
|
|
app = router_app(settings,env["redis"],admission_transport=httpx.ASGITransport(env["app"]))
|
|
sock = socket.socket()
|
|
sock.bind(("127.0.0.1",0))
|
|
sock.listen(128)
|
|
port = sock.getsockname()[1]
|
|
server = uvicorn.Server(uvicorn.Config(app,log_level="critical",access_log=False,lifespan="on"))
|
|
server_task = asyncio.create_task(server.serve(sockets=[sock]))
|
|
try:
|
|
for _ in range(200):
|
|
if server.started:
|
|
break
|
|
if server_task.done():
|
|
server_task.result()
|
|
await asyncio.sleep(0.01)
|
|
assert server.started
|
|
value = {**env,"url":f"ws://127.0.0.1:{port}/device/ws","public":load_public(public_path),"sessions":[],"router":app}
|
|
yield value
|
|
finally:
|
|
for ws in locals().get("value",{}).get("sessions",[]):
|
|
await ws.close()
|
|
server.should_exit = True
|
|
await asyncio.wait_for(server_task,5)
|
|
|
|
|
|
async def device(env, host="a"*32, *, generation=None, legacy=False):
|
|
ws = await connect(env["url"],proxy=None)
|
|
env["sessions"].append(ws)
|
|
challenge = json.loads(await ws.recv())["challenge"]
|
|
row = env["store"].device(host)
|
|
ephemeral = x25519.X25519PrivateKey.generate()
|
|
session = secrets.token_hex(16)
|
|
plain = {"host_id":host,"challenge":challenge,"session_id":session,
|
|
"kx":raw_public(ephemeral.public_key()),"ts":int(time.time())}
|
|
if not legacy:
|
|
plain.update(protocol_version=2,environment=env["settings"].environment,
|
|
app_id=env["settings"].feishu_app_id,binding_owner=row["owner"] or "ou_a",
|
|
binding_generation=row["binding_generation"] if generation is None else generation)
|
|
await ws.send(json.dumps({"host_id":host,"cert":row["cert"],"proof":sign(plain,env["keys"][host])}))
|
|
accepted = json.loads(await asyncio.wait_for(ws.recv(),3))
|
|
assert accepted["type"] == "connected"
|
|
return ws,session,ephemeral
|
|
|
|
|
|
async def instruction(env, connection, host="a"*32):
|
|
ws,session,ephemeral = connection
|
|
packet = json.loads(await asyncio.wait_for(ws.recv(),3))
|
|
payload = open_envelope(packet["envelope"],ephemeral,env["public"],host_id=host,session_id=session)
|
|
return payload
|
|
|
|
|
|
async def reply(env, connection, payload, text, host="a"*32):
|
|
ws,session,_ = connection
|
|
envelope = seal({"type":"result","turn_id":payload["turn_id"],"result":text},
|
|
payload["reply_key"],env["keys"][host],host_id=host,session_id=session)
|
|
await ws.send(json.dumps({"type":"reply","turn_id":payload["turn_id"],"envelope":envelope}))
|
|
|
|
|
|
async def wait_for_reply(platform, count):
|
|
for _ in range(200):
|
|
if len(platform.replies) >= count:
|
|
return
|
|
await asyncio.sleep(0.01)
|
|
assert len(platform.replies) >= count
|
|
|
|
|
|
def event(env, message_id, text, chat="chat-a", owner="ou_a"):
|
|
return {"header":{"event_type":"im.message.receive_v1","app_id":env["settings"].feishu_app_id},
|
|
"event":{"sender":{"sender_type":"user","sender_id":{"open_id":owner}},
|
|
"message":{"message_id":message_id,"chat_type":"p2p","chat_id":chat,"message_type":"text",
|
|
"content":json.dumps({"text":text})}}}
|
|
|
|
|
|
async def send_event(env, data):
|
|
response = await env["client"].post("/internal/events",json=data,
|
|
headers={"X-Internal-Secret":env["settings"].internal_secret})
|
|
assert response.status_code == 200
|
|
|
|
|
|
async def test_unbind_commits_and_defers_redis_side_effects(env,monkeypatch):
|
|
await authorize(env)
|
|
pending = await start(env)
|
|
with monkeypatch.context() as patch:
|
|
async def fail(*args,**kwargs):
|
|
raise ConnectionError("synthetic redis outage")
|
|
patch.setattr(env["redis"],"publish",fail)
|
|
response = await env["client"].post("/admin/devices/"+"a"*32+"/unbind",
|
|
headers={"X-Admin-Secret":env["settings"].admin_secret})
|
|
assert response.status_code == 200 and response.json()["propagation"] == "pending"
|
|
assert env["store"].device("a"*32)["owner"] is None
|
|
assert env["api"]._row(pending["transaction_id"])["status"] == "invalidated"
|
|
path = "/work/v2/credentials/access-token"
|
|
assert (await env["client"].post(path,json=wire(env,path,permission_profile="product-v1"))).status_code == 403
|
|
assert await env["app"].state.lifecycle.flush()
|
|
assert env["store"].db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2 WHERE delivered=0").fetchone()["n"] == 0
|
|
|
|
|
|
async def test_codes_are_one_time_and_pre_unbind_codes_stay_invalid_after_rebind(env):
|
|
state = env["store"].lifecycle
|
|
env["store"].unbind("ou_a")
|
|
state.issue_code("ou_a","fake-code-old")
|
|
env["store"].unbind("ou_a")
|
|
with pytest.raises(LookupError):
|
|
state.consume_code("a"*32,"fake-code-old")
|
|
state.issue_code("ou_a","fake-code-new")
|
|
assert state.consume_code("a"*32,"fake-code-new") == "ou_a"
|
|
with pytest.raises(LookupError):
|
|
state.consume_code("a"*32,"fake-code-new")
|
|
dump = "\n".join(env["store"].db.iterdump())
|
|
assert "fake-code-old" not in dump and "fake-code-new" not in dump
|
|
|
|
|
|
async def test_failed_database_invalidation_rolls_back_binding_and_outbox(env,monkeypatch):
|
|
state = env["store"].lifecycle
|
|
original = state._cancel
|
|
def fail(device,reason):
|
|
original(device,reason)
|
|
raise RuntimeError("synthetic transaction abort")
|
|
monkeypatch.setattr(state,"_cancel",fail)
|
|
with pytest.raises(RuntimeError):
|
|
env["store"].unbind("ou_a")
|
|
current = env["store"].device("a"*32)
|
|
assert current["owner"] == "ou_a" and current["binding_generation"] == 1
|
|
assert state.db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2").fetchone()["n"] == 0
|
|
|
|
|
|
async def test_unbind_outbox_is_persistent_and_reprocessed_by_second_connector(env):
|
|
from relay.connector import create_app
|
|
env["store"].unbind("ou_a")
|
|
second = create_app(env["settings"],env["redis"],env["platform"])
|
|
try:
|
|
assert second.state.store.device("a"*32)["owner"] is None
|
|
assert await second.state.lifecycle.flush()
|
|
assert second.state.store.db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2 WHERE delivered=0").fetchone()["n"] == 0
|
|
finally:
|
|
second.state.store.close()
|
|
|
|
|
|
async def test_new_online_generation_not_removed_by_old_outbox(env):
|
|
env["store"].unbind("ou_a")
|
|
env["store"].bind("a"*32,"ou_a")
|
|
generation = env["store"].device("a"*32)["binding_generation"]
|
|
presence = json.dumps({"proof":{"binding_generation":generation}})
|
|
await env["redis"].set("hn:online:"+"a"*32,presence)
|
|
assert await env["app"].state.lifecycle.flush()
|
|
assert await env["redis"].get("hn:online:"+"a"*32) == presence
|
|
|
|
|
|
def notification(env, *, generation=None, include_generation=True):
|
|
host = "a"*32
|
|
payload = {"request_id":secrets.token_hex(8),"kind":"text","text":"synthetic notice","title":"",
|
|
"oauth_state":"","binding_owner":"ou_a"}
|
|
if include_generation:
|
|
payload["binding_generation"] = env["store"].device(host)["binding_generation"] if generation is None else generation
|
|
proof = sign({"host_id":host,"purpose":"work-notify-v1","ts":int(time.time()),"nonce":secrets.token_hex(16),
|
|
"payload_hash":hashlib.sha256(canonical(payload)).hexdigest()},env["keys"][host])
|
|
return {**payload,"host_id":host,"proof":proof}
|
|
|
|
|
|
async def test_notifications_require_current_generation_in_broker_mode(env):
|
|
stale = notification(env)
|
|
env["store"].unbind("ou_a")
|
|
env["store"].bind("a"*32,"ou_a")
|
|
assert (await env["client"].post("/device/notifications",json=stale)).status_code == 403
|
|
assert (await env["client"].post("/device/notifications",json=notification(env,include_generation=False))).status_code == 403
|
|
result = await env["client"].post("/device/notifications",json=notification(env))
|
|
assert result.json()["status"] == "sent" and len(env["platform"].sent) == 1
|
|
|
|
|
|
async def test_notification_rebind_during_app_token_wait_does_not_send(env,monkeypatch):
|
|
async def rebind():
|
|
env["store"].unbind("ou_a")
|
|
env["store"].bind("a"*32,"ou_a")
|
|
return "fake-app-token"
|
|
monkeypatch.setattr(env["platform"],"tenant_token",rebind)
|
|
result = await env["client"].post("/device/notifications",json=notification(env))
|
|
assert result.json()["status"] == "binding_changed" and not env["platform"].sent
|
|
|
|
|
|
async def test_internal_admission_requires_secret_and_current_binding(env):
|
|
path = "/internal/devices/"+"a"*32+"/admission"
|
|
body = {"nonce":secrets.token_hex(16)}
|
|
assert (await env["client"].post(path,json=body)).status_code == 401
|
|
assert (await env["client"].post(path,json=body,
|
|
headers={"X-Internal-Secret":env["settings"].internal_secret})).status_code == 401
|
|
headers = {"X-Router-Admission-Secret":env["settings"].router_admission_secret}
|
|
assert (await env["client"].post(path,json=body,headers=headers)).status_code == 200
|
|
env["store"].unbind("ou_a")
|
|
assert (await env["client"].post(path,json=body,headers=headers)).status_code == 403
|
|
|
|
|
|
async def test_router_credential_cannot_inject_platform_events(env):
|
|
result = await env["client"].post("/internal/events",json=event(env,"om-injected","hello"),
|
|
headers={"X-Internal-Secret":env["settings"].router_admission_secret})
|
|
assert result.status_code == 401
|
|
|
|
|
|
async def test_device_can_only_unbind_its_current_authenticated_binding(env):
|
|
path = "/work/v2/device/unbind"
|
|
invalid = wire(env,path)
|
|
invalid["proof"] = {}
|
|
assert (await env["client"].post(path,json=invalid)).status_code == 403
|
|
result = await env["client"].post(path,json=wire(env,path))
|
|
assert result.status_code == 200 and result.json()["status"] == "unbound"
|
|
assert env["store"].device("a"*32)["owner"] is None
|
|
assert env["store"].device("b"*32)["owner"] == "ou_b"
|
|
|
|
|
|
async def test_authenticated_unbind_snapshot_cannot_clear_a_new_binding(env):
|
|
state = env["store"].lifecycle
|
|
old = env["store"].device("a"*32)
|
|
env["store"].unbind("ou_a")
|
|
env["store"].bind("a"*32,"ou_a")
|
|
with pytest.raises(ValueError):
|
|
state.invalidate_host("a"*32,"unbind",expected_owner="ou_a",expected_generation=old["binding_generation"])
|
|
assert env["store"].device("a"*32)["owner"] == "ou_a"
|
|
|
|
|
|
async def test_v2_router_rejects_old_handshake_and_unbound_reconnect(routed):
|
|
with pytest.raises(ConnectionClosed):
|
|
await device(routed,legacy=True)
|
|
routed["store"].unbind("ou_a")
|
|
with pytest.raises(ConnectionClosed):
|
|
await device(routed)
|
|
|
|
|
|
async def test_two_devices_reverse_results_match_own_tasks(routed):
|
|
a = await device(routed)
|
|
b = await device(routed,"b"*32)
|
|
broker = routed["app"].state.broker
|
|
ta = asyncio.create_task(broker.dispatch("ou_a","a","turn-a",chat_id="chat-a",source_message_id="om-a"))
|
|
tb = asyncio.create_task(broker.dispatch("ou_b","b","turn-b",chat_id="chat-b",source_message_id="om-b"))
|
|
pa,pb = await asyncio.gather(instruction(routed,a),instruction(routed,b,"b"*32))
|
|
assert pa["task_context"]["source_message_id"] == "om-a"
|
|
assert pb["task_context"]["owner"] == "ou_b"
|
|
assert pa["task_context"]["conversation_id"] != pb["task_context"]["conversation_id"]
|
|
await reply(routed,b,pb,"answer-b","b"*32)
|
|
await reply(routed,a,pa,"answer-a")
|
|
ra,rb = await asyncio.gather(ta,tb)
|
|
assert (ra["result"],rb["result"]) == ("answer-a","answer-b")
|
|
state = routed["store"].lifecycle
|
|
assert not state.claim_reply("turn-a","ou_a","om-b")
|
|
assert state.claim_reply("turn-a","ou_a","om-a")
|
|
assert not state.claim_reply("turn-a","ou_a","om-a")
|
|
|
|
|
|
async def test_same_owner_two_conversations_reply_to_original_message(routed):
|
|
connection = await device(routed)
|
|
await send_event(routed,event(routed,"om-first","first","chat-first"))
|
|
first = await instruction(routed,connection)
|
|
await send_event(routed,event(routed,"om-second","second","chat-second"))
|
|
second = await instruction(routed,connection)
|
|
assert first["task_context"]["conversation_id"] != second["task_context"]["conversation_id"]
|
|
await reply(routed,connection,second,"second-answer")
|
|
await reply(routed,connection,first,"first-answer")
|
|
await wait_for_reply(routed["platform"],2)
|
|
assert routed["platform"].replies == [("om-second","second-answer"),("om-first","first-answer")]
|
|
await send_event(routed,event(routed,"om-first","first","chat-first"))
|
|
await asyncio.sleep(0.05)
|
|
assert len(routed["platform"].replies) == 2
|
|
|
|
|
|
async def test_robot_unbind_discards_previous_result_and_closes_connection(routed):
|
|
connection = await device(routed)
|
|
await send_event(routed,event(routed,"om-old-task","slow"))
|
|
payload = await instruction(routed,connection)
|
|
await send_event(routed,event(routed,"om-unbind","/解绑"))
|
|
await wait_for_reply(routed["platform"],1)
|
|
assert routed["platform"].replies[0][0] == "om-unbind"
|
|
assert routed["store"].device("a"*32)["owner"] is None
|
|
assert routed["store"].lifecycle.task(payload["turn_id"])["status"] == "unbound"
|
|
with pytest.raises(ConnectionClosed):
|
|
await asyncio.wait_for(connection[0].recv(),2)
|
|
await asyncio.sleep(0.05)
|
|
assert all(message != "om-old-task" for message,_ in routed["platform"].replies)
|
|
|
|
|
|
async def test_revoke_has_durable_state_and_blocks_connection(routed):
|
|
connection = await device(routed)
|
|
response = await routed["client"].post("/admin/devices/"+"a"*32+"/revoke",
|
|
headers={"X-Admin-Secret":routed["settings"].admin_secret})
|
|
assert response.status_code == 200
|
|
assert routed["store"].device("a"*32)["revoked"] == 1
|
|
with pytest.raises(ConnectionClosed):
|
|
await asyncio.wait_for(connection[0].recv(),2)
|
|
|
|
|
|
async def test_delayed_kill_for_old_generation_does_not_disconnect_new_session(routed):
|
|
old = await device(routed)
|
|
routed["store"].unbind("ou_a")
|
|
await routed["app"].state.lifecycle.flush()
|
|
with pytest.raises(ConnectionClosed):
|
|
await asyncio.wait_for(old[0].recv(),2)
|
|
routed["store"].bind("a"*32,"ou_a")
|
|
fresh = await device(routed)
|
|
with routed["store"].db:
|
|
routed["store"].db.execute("UPDATE work_binding_events_v2 SET delivered=0")
|
|
await routed["app"].state.lifecycle.flush()
|
|
task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","hello","fresh-task"))
|
|
packet = await instruction(routed,fresh)
|
|
await reply(routed,fresh,packet,"fresh-answer")
|
|
assert (await task)["result"] == "fresh-answer"
|
|
|
|
|
|
async def test_lost_disconnect_notification_still_rejects_old_reply(routed):
|
|
connection = await device(routed)
|
|
task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","slow","lost-disconnect"))
|
|
packet = await instruction(routed,connection)
|
|
routed["store"].unbind("ou_a") # Deliberately do not flush PubSub notification.
|
|
await reply(routed,connection,packet,"must-not-arrive")
|
|
with pytest.raises(ConnectionClosed):
|
|
await asyncio.wait_for(connection[0].recv(),2)
|
|
routed["app"].state.lifecycle.cancel_waiters()
|
|
assert (await task)["status"] == "unbound"
|
|
|
|
|
|
async def test_idle_connection_rechecks_database_even_when_pubsub_flush_fails(routed,monkeypatch):
|
|
connection = await device(routed)
|
|
async def unavailable():
|
|
return False
|
|
monkeypatch.setattr(routed["app"].state.lifecycle,"flush",unavailable)
|
|
routed["store"].unbind("ou_a")
|
|
# No client cooperation or kill publication: router's 8-second control check
|
|
# must close the old session from authoritative database state.
|
|
with pytest.raises(ConnectionClosed):
|
|
await asyncio.wait_for(connection[0].recv(),12)
|
|
|
|
|
|
async def test_control_plane_outage_does_not_forward_new_task(routed,monkeypatch):
|
|
from fastapi import HTTPException
|
|
connection = await device(routed)
|
|
async def unavailable(*args,**kwargs):
|
|
raise HTTPException(503,"synthetic_control_outage")
|
|
monkeypatch.setattr(routed["app"].state.lifecycle,"admission",unavailable)
|
|
task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","do not deliver","control-outage"))
|
|
try:
|
|
with pytest.raises(ConnectionClosed):
|
|
await asyncio.wait_for(connection[0].recv(),3)
|
|
finally:
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
assert routed["store"].lifecycle.task("control-outage")["status"] == "unknown"
|
|
|
|
|
|
async def test_dispatch_cleanup_releases_waiter_despite_database_failure(routed,monkeypatch):
|
|
connection = await device(routed)
|
|
broker = routed["app"].state.broker
|
|
task = asyncio.create_task(broker.dispatch("ou_a","pending","cleanup-task"))
|
|
await instruction(routed,connection)
|
|
def fail(*args,**kwargs):
|
|
raise OSError("synthetic cleanup database failure")
|
|
monkeypatch.setattr(routed["store"].lifecycle,"transition",fail)
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
assert "cleanup-task" not in broker.pending
|
|
assert not await routed["redis"].exists("hn:inflight:"+"a"*32+":cleanup-task")
|
|
|
|
|
|
async def test_router_releases_connection_slot_despite_redis_cleanup_failure(routed,monkeypatch):
|
|
import relay.router as module
|
|
connection = await device(routed)
|
|
async def fail(*args,**kwargs):
|
|
raise ConnectionError("synthetic cleanup Redis failure")
|
|
monkeypatch.setattr(module,"compare_delete",fail)
|
|
await connection[0].close()
|
|
async with httpx.AsyncClient(transport=httpx.ASGITransport(routed["router"]),base_url="http://router") as client:
|
|
for _ in range(100):
|
|
result = await client.get("/ready")
|
|
if result.json()["connections"] == 0:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert result.json()["connections"] == 0
|
|
|
|
|
|
async def test_platform_reply_unknown_not_replayed_on_duplicate_event(routed,monkeypatch):
|
|
attempts = []
|
|
async def ambiguous(message,text):
|
|
attempts.append(message)
|
|
raise httpx.ReadTimeout("synthetic platform timeout")
|
|
monkeypatch.setattr(routed["platform"],"reply",ambiguous)
|
|
connection = await device(routed)
|
|
data = event(routed,"om-reply-unknown","hello")
|
|
await send_event(routed,data)
|
|
packet = await instruction(routed,connection)
|
|
await reply(routed,connection,packet,"answer")
|
|
for _ in range(100):
|
|
if routed["store"].lifecycle.task(packet["turn_id"])["status"] == "reply_unknown":
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert routed["store"].lifecycle.task(packet["turn_id"])["status"] == "reply_unknown"
|
|
await send_event(routed,data)
|
|
await asyncio.sleep(0.05)
|
|
assert attempts == ["om-reply-unknown"]
|
|
|
|
|
|
async def test_binding_api_validates_signature_before_consuming_code(env):
|
|
state = env["store"].lifecycle
|
|
env["store"].unbind("ou_a")
|
|
state.issue_code("ou_a","fake-binding-once")
|
|
host = "a"*32
|
|
proof = sign({"host_id":host,"code":"fake-binding-once","ts":int(time.time())},env["keys"][host])
|
|
invalid = {**proof,"signature":"bad"}
|
|
assert (await env["client"].post("/device/bind",json={"host_id":host,"proof":invalid})).status_code == 403
|
|
response = await env["client"].post("/device/bind",json={"host_id":host,"proof":proof})
|
|
assert response.status_code == 200 and response.json()["binding_generation"] == 3
|
|
assert (await env["client"].post("/device/bind",json={"host_id":host,"proof":proof})).status_code == 404
|
|
|
|
|
|
async def test_dead_worker_tasks_unknown_and_never_replayed(env):
|
|
state = env["store"].lifecycle
|
|
row = env["store"].device("a"*32)
|
|
state.create_task("abandoned-task","ou_a","a"*32,row["binding_generation"],"session","dead-worker",time.time()+300,"chat","om-abandoned")
|
|
await env["app"].state.lifecycle.maintain()
|
|
assert state.task("abandoned-task")["status"] == "unknown"
|
|
assert not state.complete("abandoned-task","a"*32,"session","dead-worker")
|
|
assert not state.claim_reply("abandoned-task","ou_a","om-abandoned")
|
|
with pytest.raises(ValueError):
|
|
state.create_task("abandoned-task","ou_a","a"*32,row["binding_generation"],"new","new-worker",time.time()+300)
|
|
|
|
|
|
async def test_task_unknown_after_deadline_and_replying_after_crash(env):
|
|
state = env["store"].lifecycle
|
|
gen = env["store"].device("a"*32)["binding_generation"]
|
|
state.create_task("expired-task","ou_a","a"*32,gen,"s","missing",time.time()-1)
|
|
state.create_task("reply-crash","ou_a","a"*32,gen,"s","missing",time.time()+300)
|
|
state.transition("reply-crash","inflight","replying")
|
|
await env["app"].state.lifecycle.maintain()
|
|
assert state.task("expired-task")["status"] == "unknown"
|
|
assert state.task("reply-crash")["status"] == "reply_unknown"
|
|
|
|
|
|
async def test_wrong_session_and_old_generation_cannot_complete_task(env):
|
|
state = env["store"].lifecycle
|
|
gen = env["store"].device("a"*32)["binding_generation"]
|
|
state.create_task("old-generation","ou_a","a"*32,gen,"session","worker",time.time()+300)
|
|
assert not state.complete("old-generation","a"*32,"other-session","worker")
|
|
env["store"].unbind("ou_a")
|
|
env["store"].bind("a"*32,"ou_a")
|
|
assert not state.complete("old-generation","a"*32,"session","worker")
|
|
|
|
|
|
async def test_admission_denies_dead_tasks_without_exposing_credentials(env):
|
|
gen = env["store"].device("a"*32)["binding_generation"]
|
|
env["store"].lifecycle.create_task("task-check","ou_a","a"*32,gen,"s","dead",time.time()+300)
|
|
response = await env["client"].post("/internal/devices/"+"a"*32+"/admission",
|
|
headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret},
|
|
json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"})
|
|
assert response.status_code == 403 and "token" not in response.text.lower()
|