feat(tasks): track authenticated native progress until final reply
Renew broker idle waits only for encrypted current-task progress, keep an absolute deadline and binding/session/worker fences, and retain original-message reply deduplication. Add guarded prepare-update staging that reuses the active broker key material and application configuration. Validation: 216 passed, 2 skipped; guarded update and long-task routing covered with synthetic Docker configuration and loopback WebSocket tests. Real ECS build and acceptance remain pending.
This commit is contained in:
@@ -477,3 +477,56 @@ async def test_admission_denies_dead_tasks_without_exposing_credentials(env):
|
||||
headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret},
|
||||
json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"})
|
||||
assert response.status_code == 403 and "token" not in response.text.lower()
|
||||
|
||||
|
||||
async def test_native_progress_keeps_task_until_final_and_original_reply_once(routed):
|
||||
routed['settings'].request_timeout=1
|
||||
connection=await device(routed)
|
||||
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
|
||||
payload=await instruction(routed,connection)
|
||||
ws,session,_=connection
|
||||
for _ in range(4):
|
||||
envelope=seal({'type':'progress','turn_id':payload['turn_id']},payload['reply_key'],
|
||||
routed['keys']['a'*32],host_id='a'*32,session_id=session)
|
||||
await ws.send(json.dumps({'type':'reply','turn_id':payload['turn_id'],'envelope':envelope}))
|
||||
await asyncio.sleep(.4)
|
||||
assert not routed['platform'].replies
|
||||
assert routed['store'].lifecycle.task(payload['turn_id'])['status']=='inflight'
|
||||
await reply(routed,connection,payload,'synthetic final')
|
||||
await wait_for_reply(routed['platform'],1)
|
||||
assert routed['platform'].replies==[('om-progress','synthetic final')]
|
||||
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
|
||||
await asyncio.sleep(.05)
|
||||
assert len(routed['platform'].replies)==1
|
||||
|
||||
|
||||
async def test_progress_wait_cannot_extend_hard_deadline_or_hide_idle_failure():
|
||||
from relay.connector import Broker,Pending
|
||||
from types import SimpleNamespace
|
||||
broker=object.__new__(Broker)
|
||||
broker.s=SimpleNamespace(task_tracking_timeout=.12,request_timeout=.08)
|
||||
item=Pending(asyncio.get_running_loop().create_future(),None,None,'host','session')
|
||||
async def pulses():
|
||||
while True:
|
||||
item.progress.set()
|
||||
await asyncio.sleep(.02)
|
||||
pulse=asyncio.create_task(pulses())
|
||||
try:
|
||||
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
|
||||
assert not item.future.done()
|
||||
finally:
|
||||
pulse.cancel()
|
||||
try:await pulse
|
||||
except asyncio.CancelledError:pass
|
||||
broker.s.task_tracking_timeout=1
|
||||
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
|
||||
|
||||
|
||||
async def test_unbound_or_wrong_session_progress_cannot_keep_task_alive(env):
|
||||
state=env['store'].lifecycle
|
||||
state.create_task('progress-bound','ou_a','a'*32,1,'session-a','worker-a',time.time()+60)
|
||||
assert state.progress('progress-bound','a'*32,'session-a','worker-a')
|
||||
assert not state.progress('progress-bound','a'*32,'session-b','worker-a')
|
||||
assert not state.progress('progress-bound','a'*32,'session-a','worker-b')
|
||||
env['store'].unbind('ou_a')
|
||||
assert not state.progress('progress-bound','a'*32,'session-a','worker-a')
|
||||
|
||||
Reference in New Issue
Block a user