feat(tasks): persist encrypted result handoff and recover received tasks
Add signed device-scoped task state/result endpoints and AES-GCM result outbox. Atomically claim original-message replies, reject cross-user/stale/expired tasks, preserve unknown platform receipts without blind replay, and retain ambiguous ciphertext for at most seven days. Maintain legacy synchronous compatibility and clarify stale deployment-plan diagnostics. Validation: ECS suite 227 passed/2 skipped, later outbox guards 12 passed, deployment diagnostics 54 passed. Real ECS/PostgreSQL/platform failure acceptance remains pending.
This commit is contained in:
@@ -0,0 +1,159 @@
|
||||
import asyncio
|
||||
import time
|
||||
import pytest
|
||||
from test_credential_api import env, wire
|
||||
from relay.task_delivery import TaskDelivery
|
||||
|
||||
|
||||
def task(env,turn='task-durable',owner='ou_a',host='a'*32):
|
||||
device=env['store'].device(host)
|
||||
return env['store'].lifecycle.create_task(turn,owner,host,device['binding_generation'],
|
||||
'old-session','old-worker',time.time()+600,'chat-synthetic','om-synthetic')
|
||||
|
||||
|
||||
async def upload(env,turn='task-durable',text='synthetic-private-result'):
|
||||
path='/work/v2/tasks/result'
|
||||
return await env['client'].post(path,json=wire(env,path,task_id=turn,result=text))
|
||||
|
||||
|
||||
async def test_result_is_encrypted_durable_and_delivered_once_after_worker_loss(env):
|
||||
task(env)
|
||||
api=env['app'].state.deliveries
|
||||
first=await upload(env)
|
||||
assert first.status_code==200 and first.json()['result_received']
|
||||
row=env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchone()
|
||||
assert 'synthetic-private-result' not in str(dict(row)) and row['status']=='ready'
|
||||
# Recreate application delivery service: no process-local future or keys required.
|
||||
recovered=TaskDelivery(env['api'],env['store'].lifecycle,env['app'].state.broker,env['platform'])
|
||||
await recovered.flush()
|
||||
assert env['platform'].replies==[('om-synthetic','synthetic-private-result')]
|
||||
retry=await upload(env)
|
||||
assert retry.json()['status']=='replied'
|
||||
await recovered.flush()
|
||||
assert len(env['platform'].replies)==1
|
||||
assert env['store'].db.execute('SELECT ciphertext FROM work_result_outbox_v2').fetchone()['ciphertext']==''
|
||||
|
||||
|
||||
async def test_result_conflict_and_foreign_task_are_rejected(env):
|
||||
task(env)
|
||||
assert (await upload(env)).status_code==200
|
||||
assert (await upload(env,text='different-result')).status_code==409
|
||||
task(env,'foreign-task','ou_b','b'*32)
|
||||
assert (await upload(env,'foreign-task')).status_code==403
|
||||
assert len(env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall())==1
|
||||
|
||||
|
||||
async def test_unbind_before_flush_discards_without_platform_send(env):
|
||||
task(env)
|
||||
await upload(env)
|
||||
env['store'].unbind('ou_a')
|
||||
await env['app'].state.deliveries.flush()
|
||||
assert not env['platform'].replies
|
||||
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='discarded'
|
||||
|
||||
|
||||
async def test_ambiguous_platform_send_never_retries(env,monkeypatch):
|
||||
task(env);await upload(env)
|
||||
calls=[]
|
||||
async def uncertain(*args):
|
||||
calls.append(args)
|
||||
raise ConnectionError('synthetic receipt loss')
|
||||
monkeypatch.setattr(env['platform'],'reply',uncertain)
|
||||
await env['app'].state.deliveries.flush()
|
||||
await env['app'].state.deliveries.flush()
|
||||
assert len(calls)==1
|
||||
assert (await upload(env)).json()['status']=='unknown'
|
||||
|
||||
|
||||
async def test_live_sender_is_not_retired_by_other_worker_and_dead_sender_not_replayed(env):
|
||||
task(env);await upload(env)
|
||||
db=env['store'].db
|
||||
with db:db.execute("UPDATE work_result_outbox_v2 SET status='sending',worker_id='sender-a',updated_at=0")
|
||||
await env['redis'].set('hn:task-worker:sender-a','alive',ex=30)
|
||||
api=env['app'].state.deliveries
|
||||
await api.recover_interrupted_sends()
|
||||
assert db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='sending'
|
||||
await env['redis'].delete('hn:task-worker:sender-a')
|
||||
await api.recover_interrupted_sends();await api.flush()
|
||||
assert db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='unknown'
|
||||
assert not env['platform'].replies
|
||||
|
||||
|
||||
async def test_task_status_uses_signed_device_binding_and_rejects_expired_tasks(env):
|
||||
task(env)
|
||||
path='/work/v2/tasks/state'
|
||||
r=await env['client'].post(path,json=wire(env,path,task_id='task-durable'))
|
||||
assert r.status_code==200 and not r.json()['result_received']
|
||||
with env['store'].db:env['store'].db.execute('UPDATE work_tasks_v2 SET deadline=?',(time.time()-1,))
|
||||
assert (await env['client'].post(path,json=wire(env,path,task_id='task-durable'))).status_code==403
|
||||
|
||||
|
||||
async def test_result_receipt_loss_retry_does_not_reexecute_or_duplicate_and_rebind_blocks(env):
|
||||
task(env)
|
||||
assert (await upload(env)).status_code==200
|
||||
# Lost HTTP receipt: the same payload is retried against durable storage.
|
||||
assert (await upload(env)).status_code==200
|
||||
assert len(env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall())==1
|
||||
env['store'].unbind('ou_a');env['store'].bind('a'*32,'ou_a')
|
||||
assert (await upload(env)).status_code==403
|
||||
await env['app'].state.deliveries.flush()
|
||||
assert not env['platform'].replies
|
||||
|
||||
|
||||
async def test_worker_restart_marks_task_unknown_but_same_binding_can_finish(env):
|
||||
task(env)
|
||||
await env['app'].state.lifecycle.maintain()
|
||||
assert env['store'].lifecycle.task('task-durable')['status']=='unknown'
|
||||
assert (await upload(env)).status_code==200
|
||||
await env['app'].state.deliveries.flush()
|
||||
assert env['platform'].replies==[('om-synthetic','synthetic-private-result')]
|
||||
|
||||
|
||||
async def test_send_preflight_failure_retries_only_before_sending_and_rechecks_unbind(env,monkeypatch):
|
||||
task(env);await upload(env)
|
||||
api=env['app'].state.deliveries
|
||||
original=env['platform'].tenant_token
|
||||
async def unavailable():raise ConnectionError('synthetic preflight outage')
|
||||
monkeypatch.setattr(env['platform'],'tenant_token',unavailable)
|
||||
await api.flush()
|
||||
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='ready'
|
||||
async def invalidate():
|
||||
env['store'].unbind('ou_a')
|
||||
return 'fake-token'
|
||||
monkeypatch.setattr(env['platform'],'tenant_token',invalidate)
|
||||
await api.flush()
|
||||
assert not env['platform'].replies
|
||||
|
||||
|
||||
async def test_parallel_outbox_sweepers_claim_only_one_platform_send(env,monkeypatch):
|
||||
task(env);await upload(env)
|
||||
api=env['app'].state.deliveries
|
||||
sends=[];entered=asyncio.Event();release=asyncio.Event()
|
||||
async def slow(message,text):
|
||||
sends.append((message,text));entered.set();await release.wait()
|
||||
monkeypatch.setattr(env['platform'],'reply',slow)
|
||||
first=asyncio.create_task(api.flush())
|
||||
await entered.wait()
|
||||
await api.flush()
|
||||
assert len(sends)==1
|
||||
release.set();await first
|
||||
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='replied'
|
||||
|
||||
|
||||
async def test_ambiguous_result_is_encrypted_retained_then_expires_without_resend(env,monkeypatch):
|
||||
task(env);await upload(env)
|
||||
async def fail(*args):raise ConnectionError('synthetic ambiguous')
|
||||
monkeypatch.setattr(env['platform'],'reply',fail)
|
||||
api=env['app'].state.deliveries;await api.flush()
|
||||
row=env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchone()
|
||||
assert row['status']=='unknown' and row['ciphertext'] and 'synthetic-private-result' not in str(dict(row))
|
||||
with env['store'].db:env['store'].db.execute('UPDATE work_result_outbox_v2 SET updated_at=?',(time.time()-8*86400,))
|
||||
await api.flush()
|
||||
assert env['store'].db.execute('SELECT ciphertext FROM work_result_outbox_v2').fetchone()['ciphertext']==''
|
||||
|
||||
|
||||
async def test_completed_legacy_reply_cannot_be_uploaded_again_via_new_protocol(env):
|
||||
task(env)
|
||||
with env['store'].db:env['store'].db.execute("UPDATE work_tasks_v2 SET status='replied'")
|
||||
assert (await upload(env)).status_code==403
|
||||
assert not env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall()
|
||||
Reference in New Issue
Block a user