feat(tasks): track authenticated native progress until final reply

Renew broker idle waits only for encrypted current-task progress, keep an absolute deadline and binding/session/worker fences, and retain original-message reply deduplication. Add guarded prepare-update staging that reuses the active broker key material and application configuration. Validation: 216 passed, 2 skipped; guarded update and long-task routing covered with synthetic Docker configuration and loopback WebSocket tests. Real ECS build and acceptance remain pending.
This commit is contained in:
ex_tianzy
2026-11-04 18:12:12 +08:00
parent ce965aaf58
commit 660d7a48d2
7 changed files with 225 additions and 4 deletions
+37
View File
@@ -510,3 +510,40 @@ def test_overlay_validator_rejects_database_volume_or_network_change(scenario,fi
merged[field] = {"unexpected":{}}
with pytest.raises(deploy.DeployError):
deploy.validate_overlay(base,merged,plan["candidate_image"])
def test_prepare_update_preserves_key_material_and_parent_plan_without_restart(scenario):
parent=prepared(scenario)
root=scenario['output']
original_key=(root/'credential-keyring.json').read_bytes()
original_plan=(root/'plan.json').read_bytes()
image='sha256:'+'a'*64
for name in ('connector','router'):scenario['roles'][name]['image']=image
deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']})
output=root.parent/'update'
args=argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output))
scenario['calls'].clear()
deploy.prepare_update(args)
updated=deploy.load_plan(output)
assert (output/'credential-keyring.json').read_bytes()==original_key
assert (root/'plan.json').read_bytes()==original_plan
assert updated['base_image']==image and updated['roles']['connector']['image']==image
assert updated['candidate_image']!=parent['candidate_image']
assert updated['update_preserves_credentials'] is True
overlay=json.loads((output/'compose.broker.json').read_text())
old=json.loads((root/'compose.broker.json').read_text())
assert overlay['services']['connector']['environment']==old['services']['connector']['environment']
assert not any('up' in call or 'build' in call or 'tag' in call for call in scenario['calls'])
def test_prepare_update_rejects_partial_switch_and_existing_output(scenario):
parent=prepared(scenario);root=scenario['output'];image='sha256:'+'a'*64
deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']})
scenario['roles']['connector']['image']=image
output=root.parent/'update'
args=argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output))
with pytest.raises(deploy.DeployError,match='both services'):deploy.prepare_update(args)
assert not output.exists()
scenario['roles']['router']['image']=image
args.output=str(root)
with pytest.raises(deploy.DeployError,match='new absolute'):deploy.prepare_update(args)
+53
View File
@@ -477,3 +477,56 @@ async def test_admission_denies_dead_tasks_without_exposing_credentials(env):
headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret},
json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"})
assert response.status_code == 403 and "token" not in response.text.lower()
async def test_native_progress_keeps_task_until_final_and_original_reply_once(routed):
routed['settings'].request_timeout=1
connection=await device(routed)
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
payload=await instruction(routed,connection)
ws,session,_=connection
for _ in range(4):
envelope=seal({'type':'progress','turn_id':payload['turn_id']},payload['reply_key'],
routed['keys']['a'*32],host_id='a'*32,session_id=session)
await ws.send(json.dumps({'type':'reply','turn_id':payload['turn_id'],'envelope':envelope}))
await asyncio.sleep(.4)
assert not routed['platform'].replies
assert routed['store'].lifecycle.task(payload['turn_id'])['status']=='inflight'
await reply(routed,connection,payload,'synthetic final')
await wait_for_reply(routed['platform'],1)
assert routed['platform'].replies==[('om-progress','synthetic final')]
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
await asyncio.sleep(.05)
assert len(routed['platform'].replies)==1
async def test_progress_wait_cannot_extend_hard_deadline_or_hide_idle_failure():
from relay.connector import Broker,Pending
from types import SimpleNamespace
broker=object.__new__(Broker)
broker.s=SimpleNamespace(task_tracking_timeout=.12,request_timeout=.08)
item=Pending(asyncio.get_running_loop().create_future(),None,None,'host','session')
async def pulses():
while True:
item.progress.set()
await asyncio.sleep(.02)
pulse=asyncio.create_task(pulses())
try:
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
assert not item.future.done()
finally:
pulse.cancel()
try:await pulse
except asyncio.CancelledError:pass
broker.s.task_tracking_timeout=1
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
async def test_unbound_or_wrong_session_progress_cannot_keep_task_alive(env):
state=env['store'].lifecycle
state.create_task('progress-bound','ou_a','a'*32,1,'session-a','worker-a',time.time()+60)
assert state.progress('progress-bound','a'*32,'session-a','worker-a')
assert not state.progress('progress-bound','a'*32,'session-b','worker-a')
assert not state.progress('progress-bound','a'*32,'session-a','worker-b')
env['store'].unbind('ou_a')
assert not state.progress('progress-bound','a'*32,'session-a','worker-a')