534 lines
25 KiB
Python
534 lines
25 KiB
Python
"""Broker-mode invalidation and real loopback WS task routing (no real Work)."""
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import secrets
|
|
import socket
|
|
import time
|
|
|
|
import httpx
|
|
import pytest
|
|
import pytest_asyncio
|
|
import uvicorn
|
|
from cryptography.hazmat.primitives import serialization
|
|
from cryptography.hazmat.primitives.asymmetric import x25519
|
|
from websockets.asyncio.client import connect
|
|
from websockets.exceptions import ConnectionClosed
|
|
|
|
from test_credential_api import env, wire, start, authorize
|
|
from relay.crypto import canonical, load_private, load_public, open_envelope, raw_public, seal, sign
|
|
from relay.device_lifecycle import AdmissionRequest
|
|
from relay.router import create_app as router_app
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def routed(env, tmp_path):
|
|
public_path = tmp_path / "connector.pub"
|
|
public_path.write_bytes(load_private(env["settings"].connector_key).public_key().public_bytes(
|
|
serialization.Encoding.PEM,serialization.PublicFormat.SubjectPublicKeyInfo))
|
|
settings = env["settings"].model_copy(update={"role":"router","connector_public_key":public_path,
|
|
"connector_internal_url":"http://connector:8000"})
|
|
app = router_app(settings,env["redis"],admission_transport=httpx.ASGITransport(env["app"]))
|
|
sock = socket.socket()
|
|
sock.bind(("127.0.0.1",0))
|
|
sock.listen(128)
|
|
port = sock.getsockname()[1]
|
|
server = uvicorn.Server(uvicorn.Config(app,log_level="critical",access_log=False,lifespan="on"))
|
|
server_task = asyncio.create_task(server.serve(sockets=[sock]))
|
|
try:
|
|
for _ in range(200):
|
|
if server.started:
|
|
break
|
|
if server_task.done():
|
|
server_task.result()
|
|
await asyncio.sleep(0.01)
|
|
assert server.started
|
|
value = {**env,"url":f"ws://127.0.0.1:{port}/device/ws","public":load_public(public_path),"sessions":[],"router":app}
|
|
yield value
|
|
finally:
|
|
for ws in locals().get("value",{}).get("sessions",[]):
|
|
await ws.close()
|
|
server.should_exit = True
|
|
await asyncio.wait_for(server_task,5)
|
|
|
|
|
|
async def device(env, host="a"*32, *, generation=None, legacy=False, work_files=False):
|
|
ws = await connect(env["url"],proxy=None)
|
|
env["sessions"].append(ws)
|
|
challenge = json.loads(await ws.recv())["challenge"]
|
|
row = env["store"].device(host)
|
|
ephemeral = x25519.X25519PrivateKey.generate()
|
|
session = secrets.token_hex(16)
|
|
plain = {"host_id":host,"challenge":challenge,"session_id":session,
|
|
"kx":raw_public(ephemeral.public_key()),"ts":int(time.time())}
|
|
if not legacy:
|
|
plain.update(protocol_version=2,environment=env["settings"].environment,
|
|
app_id=env["settings"].feishu_app_id,binding_owner=row["owner"] or "ou_a",
|
|
binding_generation=row["binding_generation"] if generation is None else generation)
|
|
if work_files:plain['work_files']='work-files-v1'
|
|
await ws.send(json.dumps({"host_id":host,"cert":row["cert"],"proof":sign(plain,env["keys"][host])}))
|
|
accepted = json.loads(await asyncio.wait_for(ws.recv(),3))
|
|
assert accepted["type"] == "connected"
|
|
return ws,session,ephemeral
|
|
|
|
|
|
async def instruction(env, connection, host="a"*32):
|
|
ws,session,ephemeral = connection
|
|
packet = json.loads(await asyncio.wait_for(ws.recv(),3))
|
|
payload = open_envelope(packet["envelope"],ephemeral,env["public"],host_id=host,session_id=session)
|
|
return payload
|
|
|
|
|
|
async def reply(env, connection, payload, text, host="a"*32):
|
|
ws,session,_ = connection
|
|
envelope = seal({"type":"result","turn_id":payload["turn_id"],"result":text},
|
|
payload["reply_key"],env["keys"][host],host_id=host,session_id=session)
|
|
await ws.send(json.dumps({"type":"reply","turn_id":payload["turn_id"],"envelope":envelope}))
|
|
|
|
|
|
async def wait_for_reply(platform, count):
|
|
for _ in range(200):
|
|
if len(platform.replies) >= count:
|
|
return
|
|
await asyncio.sleep(0.01)
|
|
assert len(platform.replies) >= count
|
|
|
|
|
|
def event(env, message_id, text, chat="chat-a", owner="ou_a"):
|
|
return {"header":{"event_type":"im.message.receive_v1","app_id":env["settings"].feishu_app_id},
|
|
"event":{"sender":{"sender_type":"user","sender_id":{"open_id":owner}},
|
|
"message":{"message_id":message_id,"chat_type":"p2p","chat_id":chat,"message_type":"text",
|
|
"content":json.dumps({"text":text})}}}
|
|
|
|
|
|
async def send_event(env, data):
|
|
response = await env["client"].post("/internal/events",json=data,
|
|
headers={"X-Internal-Secret":env["settings"].internal_secret})
|
|
assert response.status_code == 200
|
|
|
|
|
|
async def test_unbind_commits_and_defers_redis_side_effects(env,monkeypatch):
|
|
await authorize(env)
|
|
pending = await start(env)
|
|
with monkeypatch.context() as patch:
|
|
async def fail(*args,**kwargs):
|
|
raise ConnectionError("synthetic redis outage")
|
|
patch.setattr(env["redis"],"publish",fail)
|
|
response = await env["client"].post("/admin/devices/"+"a"*32+"/unbind",
|
|
headers={"X-Admin-Secret":env["settings"].admin_secret})
|
|
assert response.status_code == 200 and response.json()["propagation"] == "pending"
|
|
assert env["store"].device("a"*32)["owner"] is None
|
|
assert env["api"]._row(pending["transaction_id"])["status"] == "invalidated"
|
|
path = "/work/v2/credentials/access-token"
|
|
assert (await env["client"].post(path,json=wire(env,path,permission_profile="product-v1"))).status_code == 403
|
|
assert await env["app"].state.lifecycle.flush()
|
|
assert env["store"].db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2 WHERE delivered=0").fetchone()["n"] == 0
|
|
|
|
|
|
async def test_codes_are_one_time_and_pre_unbind_codes_stay_invalid_after_rebind(env):
|
|
state = env["store"].lifecycle
|
|
env["store"].unbind("ou_a")
|
|
state.issue_code("ou_a","fake-code-old")
|
|
env["store"].unbind("ou_a")
|
|
with pytest.raises(LookupError):
|
|
state.consume_code("a"*32,"fake-code-old")
|
|
state.issue_code("ou_a","fake-code-new")
|
|
assert state.consume_code("a"*32,"fake-code-new") == "ou_a"
|
|
with pytest.raises(LookupError):
|
|
state.consume_code("a"*32,"fake-code-new")
|
|
dump = "\n".join(env["store"].db.iterdump())
|
|
assert "fake-code-old" not in dump and "fake-code-new" not in dump
|
|
|
|
|
|
async def test_failed_database_invalidation_rolls_back_binding_and_outbox(env,monkeypatch):
|
|
state = env["store"].lifecycle
|
|
original = state._cancel
|
|
def fail(device,reason):
|
|
original(device,reason)
|
|
raise RuntimeError("synthetic transaction abort")
|
|
monkeypatch.setattr(state,"_cancel",fail)
|
|
with pytest.raises(RuntimeError):
|
|
env["store"].unbind("ou_a")
|
|
current = env["store"].device("a"*32)
|
|
assert current["owner"] == "ou_a" and current["binding_generation"] == 1
|
|
assert state.db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2").fetchone()["n"] == 0
|
|
|
|
|
|
async def test_unbind_outbox_is_persistent_and_reprocessed_by_second_connector(env):
|
|
from relay.connector import create_app
|
|
env["store"].unbind("ou_a")
|
|
second = create_app(env["settings"],env["redis"],env["platform"])
|
|
try:
|
|
assert second.state.store.device("a"*32)["owner"] is None
|
|
assert await second.state.lifecycle.flush()
|
|
assert second.state.store.db.execute("SELECT COUNT(*) AS n FROM work_binding_events_v2 WHERE delivered=0").fetchone()["n"] == 0
|
|
finally:
|
|
second.state.store.close()
|
|
|
|
|
|
async def test_new_online_generation_not_removed_by_old_outbox(env):
|
|
env["store"].unbind("ou_a")
|
|
env["store"].bind("a"*32,"ou_a")
|
|
generation = env["store"].device("a"*32)["binding_generation"]
|
|
presence = json.dumps({"proof":{"binding_generation":generation}})
|
|
await env["redis"].set("hn:online:"+"a"*32,presence)
|
|
assert await env["app"].state.lifecycle.flush()
|
|
assert await env["redis"].get("hn:online:"+"a"*32) == presence
|
|
|
|
|
|
def notification(env, *, generation=None, include_generation=True):
|
|
host = "a"*32
|
|
payload = {"request_id":secrets.token_hex(8),"kind":"text","text":"synthetic notice","title":"",
|
|
"oauth_state":"","binding_owner":"ou_a"}
|
|
if include_generation:
|
|
payload["binding_generation"] = env["store"].device(host)["binding_generation"] if generation is None else generation
|
|
proof = sign({"host_id":host,"purpose":"work-notify-v1","ts":int(time.time()),"nonce":secrets.token_hex(16),
|
|
"payload_hash":hashlib.sha256(canonical(payload)).hexdigest()},env["keys"][host])
|
|
return {**payload,"host_id":host,"proof":proof}
|
|
|
|
|
|
async def test_notifications_require_current_generation_in_broker_mode(env):
|
|
stale = notification(env)
|
|
env["store"].unbind("ou_a")
|
|
env["store"].bind("a"*32,"ou_a")
|
|
assert (await env["client"].post("/device/notifications",json=stale)).status_code == 403
|
|
assert (await env["client"].post("/device/notifications",json=notification(env,include_generation=False))).status_code == 403
|
|
result = await env["client"].post("/device/notifications",json=notification(env))
|
|
assert result.json()["status"] == "sent" and len(env["platform"].sent) == 1
|
|
|
|
|
|
async def test_notification_rebind_during_app_token_wait_does_not_send(env,monkeypatch):
|
|
async def rebind():
|
|
env["store"].unbind("ou_a")
|
|
env["store"].bind("a"*32,"ou_a")
|
|
return "fake-app-token"
|
|
monkeypatch.setattr(env["platform"],"tenant_token",rebind)
|
|
result = await env["client"].post("/device/notifications",json=notification(env))
|
|
assert result.json()["status"] == "binding_changed" and not env["platform"].sent
|
|
|
|
|
|
async def test_internal_admission_requires_secret_and_current_binding(env):
|
|
path = "/internal/devices/"+"a"*32+"/admission"
|
|
body = {"nonce":secrets.token_hex(16)}
|
|
assert (await env["client"].post(path,json=body)).status_code == 401
|
|
assert (await env["client"].post(path,json=body,
|
|
headers={"X-Internal-Secret":env["settings"].internal_secret})).status_code == 401
|
|
headers = {"X-Router-Admission-Secret":env["settings"].router_admission_secret}
|
|
assert (await env["client"].post(path,json=body,headers=headers)).status_code == 200
|
|
env["store"].unbind("ou_a")
|
|
assert (await env["client"].post(path,json=body,headers=headers)).status_code == 403
|
|
|
|
|
|
async def test_router_credential_cannot_inject_platform_events(env):
|
|
result = await env["client"].post("/internal/events",json=event(env,"om-injected","hello"),
|
|
headers={"X-Internal-Secret":env["settings"].router_admission_secret})
|
|
assert result.status_code == 401
|
|
|
|
|
|
async def test_device_can_only_unbind_its_current_authenticated_binding(env):
|
|
path = "/work/v2/device/unbind"
|
|
invalid = wire(env,path)
|
|
invalid["proof"] = {}
|
|
assert (await env["client"].post(path,json=invalid)).status_code == 403
|
|
result = await env["client"].post(path,json=wire(env,path))
|
|
assert result.status_code == 200 and result.json()["status"] == "unbound"
|
|
assert env["store"].device("a"*32)["owner"] is None
|
|
assert env["store"].device("b"*32)["owner"] == "ou_b"
|
|
|
|
|
|
async def test_authenticated_unbind_snapshot_cannot_clear_a_new_binding(env):
|
|
state = env["store"].lifecycle
|
|
old = env["store"].device("a"*32)
|
|
env["store"].unbind("ou_a")
|
|
env["store"].bind("a"*32,"ou_a")
|
|
with pytest.raises(ValueError):
|
|
state.invalidate_host("a"*32,"unbind",expected_owner="ou_a",expected_generation=old["binding_generation"])
|
|
assert env["store"].device("a"*32)["owner"] == "ou_a"
|
|
|
|
|
|
async def test_v2_router_rejects_old_handshake_and_unbound_reconnect(routed):
|
|
with pytest.raises(ConnectionClosed):
|
|
await device(routed,legacy=True)
|
|
routed["store"].unbind("ou_a")
|
|
with pytest.raises(ConnectionClosed):
|
|
await device(routed)
|
|
|
|
|
|
async def test_two_devices_reverse_results_match_own_tasks(routed):
|
|
a = await device(routed)
|
|
b = await device(routed,"b"*32)
|
|
broker = routed["app"].state.broker
|
|
ta = asyncio.create_task(broker.dispatch("ou_a","a","turn-a",chat_id="chat-a",source_message_id="om-a"))
|
|
tb = asyncio.create_task(broker.dispatch("ou_b","b","turn-b",chat_id="chat-b",source_message_id="om-b"))
|
|
pa,pb = await asyncio.gather(instruction(routed,a),instruction(routed,b,"b"*32))
|
|
assert pa["task_context"]["source_message_id"] == "om-a"
|
|
assert pb["task_context"]["owner"] == "ou_b"
|
|
assert pa["task_context"]["conversation_id"] != pb["task_context"]["conversation_id"]
|
|
await reply(routed,b,pb,"answer-b","b"*32)
|
|
await reply(routed,a,pa,"answer-a")
|
|
ra,rb = await asyncio.gather(ta,tb)
|
|
assert (ra["result"],rb["result"]) == ("answer-a","answer-b")
|
|
state = routed["store"].lifecycle
|
|
assert not state.claim_reply("turn-a","ou_a","om-b")
|
|
assert state.claim_reply("turn-a","ou_a","om-a")
|
|
assert not state.claim_reply("turn-a","ou_a","om-a")
|
|
|
|
|
|
async def test_same_owner_two_conversations_reply_to_original_message(routed):
|
|
connection = await device(routed)
|
|
await send_event(routed,event(routed,"om-first","first","chat-first"))
|
|
first = await instruction(routed,connection)
|
|
await send_event(routed,event(routed,"om-second","second","chat-second"))
|
|
second = await instruction(routed,connection)
|
|
assert first["task_context"]["conversation_id"] != second["task_context"]["conversation_id"]
|
|
await reply(routed,connection,second,"second-answer")
|
|
await reply(routed,connection,first,"first-answer")
|
|
await wait_for_reply(routed["platform"],2)
|
|
assert routed["platform"].replies == [("om-second","second-answer"),("om-first","first-answer")]
|
|
await send_event(routed,event(routed,"om-first","first","chat-first"))
|
|
await asyncio.sleep(0.05)
|
|
assert len(routed["platform"].replies) == 2
|
|
|
|
|
|
async def test_robot_unbind_discards_previous_result_and_closes_connection(routed):
|
|
connection = await device(routed)
|
|
await send_event(routed,event(routed,"om-old-task","slow"))
|
|
payload = await instruction(routed,connection)
|
|
await send_event(routed,event(routed,"om-unbind","/解绑"))
|
|
await wait_for_reply(routed["platform"],1)
|
|
assert routed["platform"].replies[0][0] == "om-unbind"
|
|
assert routed["store"].device("a"*32)["owner"] is None
|
|
assert routed["store"].lifecycle.task(payload["turn_id"])["status"] == "unbound"
|
|
with pytest.raises(ConnectionClosed):
|
|
await asyncio.wait_for(connection[0].recv(),2)
|
|
await asyncio.sleep(0.05)
|
|
assert all(message != "om-old-task" for message,_ in routed["platform"].replies)
|
|
|
|
|
|
async def test_revoke_has_durable_state_and_blocks_connection(routed):
|
|
connection = await device(routed)
|
|
response = await routed["client"].post("/admin/devices/"+"a"*32+"/revoke",
|
|
headers={"X-Admin-Secret":routed["settings"].admin_secret})
|
|
assert response.status_code == 200
|
|
assert routed["store"].device("a"*32)["revoked"] == 1
|
|
with pytest.raises(ConnectionClosed):
|
|
await asyncio.wait_for(connection[0].recv(),2)
|
|
|
|
|
|
async def test_delayed_kill_for_old_generation_does_not_disconnect_new_session(routed):
|
|
old = await device(routed)
|
|
routed["store"].unbind("ou_a")
|
|
await routed["app"].state.lifecycle.flush()
|
|
with pytest.raises(ConnectionClosed):
|
|
await asyncio.wait_for(old[0].recv(),2)
|
|
routed["store"].bind("a"*32,"ou_a")
|
|
fresh = await device(routed)
|
|
with routed["store"].db:
|
|
routed["store"].db.execute("UPDATE work_binding_events_v2 SET delivered=0")
|
|
await routed["app"].state.lifecycle.flush()
|
|
task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","hello","fresh-task"))
|
|
packet = await instruction(routed,fresh)
|
|
await reply(routed,fresh,packet,"fresh-answer")
|
|
assert (await task)["result"] == "fresh-answer"
|
|
|
|
|
|
async def test_lost_disconnect_notification_still_rejects_old_reply(routed):
|
|
connection = await device(routed)
|
|
task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","slow","lost-disconnect"))
|
|
packet = await instruction(routed,connection)
|
|
routed["store"].unbind("ou_a") # Deliberately do not flush PubSub notification.
|
|
await reply(routed,connection,packet,"must-not-arrive")
|
|
with pytest.raises(ConnectionClosed):
|
|
await asyncio.wait_for(connection[0].recv(),2)
|
|
routed["app"].state.lifecycle.cancel_waiters()
|
|
assert (await task)["status"] == "unbound"
|
|
|
|
|
|
async def test_idle_connection_rechecks_database_even_when_pubsub_flush_fails(routed,monkeypatch):
|
|
connection = await device(routed)
|
|
async def unavailable():
|
|
return False
|
|
monkeypatch.setattr(routed["app"].state.lifecycle,"flush",unavailable)
|
|
routed["store"].unbind("ou_a")
|
|
# No client cooperation or kill publication: router's 8-second control check
|
|
# must close the old session from authoritative database state.
|
|
with pytest.raises(ConnectionClosed):
|
|
await asyncio.wait_for(connection[0].recv(),12)
|
|
|
|
|
|
async def test_control_plane_outage_does_not_forward_new_task(routed,monkeypatch):
|
|
from fastapi import HTTPException
|
|
connection = await device(routed)
|
|
async def unavailable(*args,**kwargs):
|
|
raise HTTPException(503,"synthetic_control_outage")
|
|
monkeypatch.setattr(routed["app"].state.lifecycle,"admission",unavailable)
|
|
task = asyncio.create_task(routed["app"].state.broker.dispatch("ou_a","do not deliver","control-outage"))
|
|
try:
|
|
with pytest.raises(ConnectionClosed):
|
|
await asyncio.wait_for(connection[0].recv(),3)
|
|
finally:
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
assert routed["store"].lifecycle.task("control-outage")["status"] == "unknown"
|
|
|
|
|
|
async def test_dispatch_cleanup_releases_waiter_despite_database_failure(routed,monkeypatch):
|
|
connection = await device(routed)
|
|
broker = routed["app"].state.broker
|
|
task = asyncio.create_task(broker.dispatch("ou_a","pending","cleanup-task"))
|
|
await instruction(routed,connection)
|
|
def fail(*args,**kwargs):
|
|
raise OSError("synthetic cleanup database failure")
|
|
monkeypatch.setattr(routed["store"].lifecycle,"transition",fail)
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
assert "cleanup-task" not in broker.pending
|
|
assert not await routed["redis"].exists("hn:inflight:"+"a"*32+":cleanup-task")
|
|
|
|
|
|
async def test_router_releases_connection_slot_despite_redis_cleanup_failure(routed,monkeypatch):
|
|
import relay.router as module
|
|
connection = await device(routed)
|
|
async def fail(*args,**kwargs):
|
|
raise ConnectionError("synthetic cleanup Redis failure")
|
|
monkeypatch.setattr(module,"compare_delete",fail)
|
|
await connection[0].close()
|
|
async with httpx.AsyncClient(transport=httpx.ASGITransport(routed["router"]),base_url="http://router") as client:
|
|
for _ in range(100):
|
|
result = await client.get("/ready")
|
|
if result.json()["connections"] == 0:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert result.json()["connections"] == 0
|
|
|
|
|
|
async def test_platform_reply_unknown_not_replayed_on_duplicate_event(routed,monkeypatch):
|
|
attempts = []
|
|
async def ambiguous(message,text):
|
|
attempts.append(message)
|
|
raise httpx.ReadTimeout("synthetic platform timeout")
|
|
monkeypatch.setattr(routed["platform"],"reply",ambiguous)
|
|
connection = await device(routed)
|
|
data = event(routed,"om-reply-unknown","hello")
|
|
await send_event(routed,data)
|
|
packet = await instruction(routed,connection)
|
|
await reply(routed,connection,packet,"answer")
|
|
for _ in range(100):
|
|
if routed["store"].lifecycle.task(packet["turn_id"])["status"] == "reply_unknown":
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert routed["store"].lifecycle.task(packet["turn_id"])["status"] == "reply_unknown"
|
|
await send_event(routed,data)
|
|
await asyncio.sleep(0.05)
|
|
assert attempts == ["om-reply-unknown"]
|
|
|
|
|
|
async def test_binding_api_validates_signature_before_consuming_code(env):
|
|
state = env["store"].lifecycle
|
|
env["store"].unbind("ou_a")
|
|
state.issue_code("ou_a","fake-binding-once")
|
|
host = "a"*32
|
|
proof = sign({"host_id":host,"code":"fake-binding-once","ts":int(time.time())},env["keys"][host])
|
|
invalid = {**proof,"signature":"bad"}
|
|
assert (await env["client"].post("/device/bind",json={"host_id":host,"proof":invalid})).status_code == 403
|
|
response = await env["client"].post("/device/bind",json={"host_id":host,"proof":proof})
|
|
assert response.status_code == 200 and response.json()["binding_generation"] == 3
|
|
assert (await env["client"].post("/device/bind",json={"host_id":host,"proof":proof})).status_code == 404
|
|
|
|
|
|
async def test_dead_worker_tasks_unknown_and_never_replayed(env):
|
|
state = env["store"].lifecycle
|
|
row = env["store"].device("a"*32)
|
|
state.create_task("abandoned-task","ou_a","a"*32,row["binding_generation"],"session","dead-worker",time.time()+300,"chat","om-abandoned")
|
|
await env["app"].state.lifecycle.maintain()
|
|
assert state.task("abandoned-task")["status"] == "unknown"
|
|
assert not state.complete("abandoned-task","a"*32,"session","dead-worker")
|
|
assert not state.claim_reply("abandoned-task","ou_a","om-abandoned")
|
|
with pytest.raises(ValueError):
|
|
state.create_task("abandoned-task","ou_a","a"*32,row["binding_generation"],"new","new-worker",time.time()+300)
|
|
|
|
|
|
async def test_task_unknown_after_deadline_and_replying_after_crash(env):
|
|
state = env["store"].lifecycle
|
|
gen = env["store"].device("a"*32)["binding_generation"]
|
|
state.create_task("expired-task","ou_a","a"*32,gen,"s","missing",time.time()-1)
|
|
state.create_task("reply-crash","ou_a","a"*32,gen,"s","missing",time.time()+300)
|
|
state.transition("reply-crash","inflight","replying")
|
|
await env["app"].state.lifecycle.maintain()
|
|
assert state.task("expired-task")["status"] == "unknown"
|
|
assert state.task("reply-crash")["status"] == "reply_unknown"
|
|
|
|
|
|
async def test_wrong_session_and_old_generation_cannot_complete_task(env):
|
|
state = env["store"].lifecycle
|
|
gen = env["store"].device("a"*32)["binding_generation"]
|
|
state.create_task("old-generation","ou_a","a"*32,gen,"session","worker",time.time()+300)
|
|
assert not state.complete("old-generation","a"*32,"other-session","worker")
|
|
env["store"].unbind("ou_a")
|
|
env["store"].bind("a"*32,"ou_a")
|
|
assert not state.complete("old-generation","a"*32,"session","worker")
|
|
|
|
|
|
async def test_admission_denies_dead_tasks_without_exposing_credentials(env):
|
|
gen = env["store"].device("a"*32)["binding_generation"]
|
|
env["store"].lifecycle.create_task("task-check","ou_a","a"*32,gen,"s","dead",time.time()+300)
|
|
response = await env["client"].post("/internal/devices/"+"a"*32+"/admission",
|
|
headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret},
|
|
json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"})
|
|
assert response.status_code == 403 and "token" not in response.text.lower()
|
|
|
|
|
|
async def test_native_progress_keeps_task_until_final_and_original_reply_once(routed):
|
|
routed['settings'].request_timeout=1
|
|
connection=await device(routed)
|
|
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
|
|
payload=await instruction(routed,connection)
|
|
ws,session,_=connection
|
|
for _ in range(4):
|
|
envelope=seal({'type':'progress','turn_id':payload['turn_id']},payload['reply_key'],
|
|
routed['keys']['a'*32],host_id='a'*32,session_id=session)
|
|
await ws.send(json.dumps({'type':'reply','turn_id':payload['turn_id'],'envelope':envelope}))
|
|
await asyncio.sleep(.4)
|
|
assert not routed['platform'].replies
|
|
assert routed['store'].lifecycle.task(payload['turn_id'])['status']=='inflight'
|
|
await reply(routed,connection,payload,'synthetic final')
|
|
await wait_for_reply(routed['platform'],1)
|
|
assert routed['platform'].replies==[('om-progress','synthetic final')]
|
|
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
|
|
await asyncio.sleep(.05)
|
|
assert len(routed['platform'].replies)==1
|
|
|
|
|
|
async def test_progress_wait_cannot_extend_hard_deadline_or_hide_idle_failure():
|
|
from relay.connector import Broker,Pending
|
|
from types import SimpleNamespace
|
|
broker=object.__new__(Broker)
|
|
broker.s=SimpleNamespace(task_tracking_timeout=.12,request_timeout=.08)
|
|
item=Pending(asyncio.get_running_loop().create_future(),None,None,'host','session')
|
|
async def pulses():
|
|
while True:
|
|
item.progress.set()
|
|
await asyncio.sleep(.02)
|
|
pulse=asyncio.create_task(pulses())
|
|
try:
|
|
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
|
|
assert not item.future.done()
|
|
finally:
|
|
pulse.cancel()
|
|
try:await pulse
|
|
except asyncio.CancelledError:pass
|
|
broker.s.task_tracking_timeout=1
|
|
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
|
|
|
|
|
|
async def test_unbound_or_wrong_session_progress_cannot_keep_task_alive(env):
|
|
state=env['store'].lifecycle
|
|
state.create_task('progress-bound','ou_a','a'*32,1,'session-a','worker-a',time.time()+60)
|
|
assert state.progress('progress-bound','a'*32,'session-a','worker-a')
|
|
assert not state.progress('progress-bound','a'*32,'session-b','worker-a')
|
|
assert not state.progress('progress-bound','a'*32,'session-a','worker-b')
|
|
env['store'].unbind('ou_a')
|
|
assert not state.progress('progress-bound','a'*32,'session-a','worker-a')
|