194 lines
9.0 KiB
Python
194 lines
9.0 KiB
Python
import asyncio
|
|
import time
|
|
import pytest
|
|
from test_credential_api import env, wire
|
|
from relay.task_delivery import TaskDelivery
|
|
|
|
|
|
def task(env,turn='task-durable',owner='ou_a',host='a'*32):
|
|
device=env['store'].device(host)
|
|
return env['store'].lifecycle.create_task(turn,owner,host,device['binding_generation'],
|
|
'old-session','old-worker',time.time()+600,'chat-synthetic','om-synthetic')
|
|
|
|
|
|
async def upload(env,turn='task-durable',text='synthetic-private-result'):
|
|
path='/work/v2/tasks/result'
|
|
return await env['client'].post(path,json=wire(env,path,task_id=turn,result=text))
|
|
|
|
|
|
async def test_execution_status_order_replay_terminal_and_delivery_are_separate(env):
|
|
task(env)
|
|
path='/work/v2/tasks/progress'
|
|
async def report(state,sequence,turn='task-durable'):
|
|
return await env['client'].post(path,json=wire(env,path,task_id=turn,
|
|
execution_state=state,status_sequence=sequence))
|
|
assert (await report('running',1)).json()['execution_state']=='running'
|
|
waiting=await report('waiting_for_approval',2)
|
|
assert waiting.json()['status_sequence']==2
|
|
assert (await report('running',1)).json()['execution_state']=='waiting_for_approval'
|
|
assert (await report('waiting_for_approval',2)).status_code==200
|
|
assert (await report('running',2)).status_code==409
|
|
completed=await report('completed',3)
|
|
assert completed.json()['execution_state']=='completed'
|
|
assert not completed.json()['result_received']
|
|
assert (await report('running',4)).status_code==409
|
|
await upload(env)
|
|
assert (await report('completed',3)).json()['result_received']
|
|
|
|
|
|
async def test_execution_state_auth_binding_and_parameter_validation(env):
|
|
task(env)
|
|
path='/work/v2/tasks/progress'
|
|
missing=await env['client'].post(path,json=wire(env,path,task_id='task-durable',execution_state='running'))
|
|
assert missing.status_code==422
|
|
task(env,'foreign-task','ou_b','b'*32)
|
|
foreign=await env['client'].post(path,json=wire(env,path,task_id='foreign-task',execution_state='running',status_sequence=1))
|
|
assert foreign.status_code==403
|
|
env['store'].unbind('ou_a')
|
|
revoked=await env['client'].post(path,json=wire(env,path,task_id='task-durable',execution_state='running',status_sequence=1))
|
|
assert revoked.status_code==403
|
|
assert not env['store'].db.execute('SELECT * FROM work_execution_state_v3').fetchall()
|
|
|
|
|
|
async def test_result_is_encrypted_durable_and_delivered_once_after_worker_loss(env):
|
|
task(env)
|
|
api=env['app'].state.deliveries
|
|
first=await upload(env)
|
|
assert first.status_code==200 and first.json()['result_received']
|
|
row=env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchone()
|
|
assert 'synthetic-private-result' not in str(dict(row)) and row['status']=='ready'
|
|
# Recreate application delivery service: no process-local future or keys required.
|
|
recovered=TaskDelivery(env['api'],env['store'].lifecycle,env['app'].state.broker,env['platform'])
|
|
await recovered.flush()
|
|
assert env['platform'].replies==[('om-synthetic','synthetic-private-result')]
|
|
retry=await upload(env)
|
|
assert retry.json()['status']=='replied'
|
|
await recovered.flush()
|
|
assert len(env['platform'].replies)==1
|
|
assert env['store'].db.execute('SELECT ciphertext FROM work_result_outbox_v2').fetchone()['ciphertext']==''
|
|
|
|
|
|
async def test_result_conflict_and_foreign_task_are_rejected(env):
|
|
task(env)
|
|
assert (await upload(env)).status_code==200
|
|
assert (await upload(env,text='different-result')).status_code==409
|
|
task(env,'foreign-task','ou_b','b'*32)
|
|
assert (await upload(env,'foreign-task')).status_code==403
|
|
assert len(env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall())==1
|
|
|
|
|
|
async def test_unbind_before_flush_discards_without_platform_send(env):
|
|
task(env)
|
|
await upload(env)
|
|
env['store'].unbind('ou_a')
|
|
await env['app'].state.deliveries.flush()
|
|
assert not env['platform'].replies
|
|
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='discarded'
|
|
|
|
|
|
async def test_ambiguous_platform_send_never_retries(env,monkeypatch):
|
|
task(env);await upload(env)
|
|
calls=[]
|
|
async def uncertain(*args):
|
|
calls.append(args)
|
|
raise ConnectionError('synthetic receipt loss')
|
|
monkeypatch.setattr(env['platform'],'reply',uncertain)
|
|
await env['app'].state.deliveries.flush()
|
|
await env['app'].state.deliveries.flush()
|
|
assert len(calls)==1
|
|
assert (await upload(env)).json()['status']=='unknown'
|
|
|
|
|
|
async def test_live_sender_is_not_retired_by_other_worker_and_dead_sender_not_replayed(env):
|
|
task(env);await upload(env)
|
|
db=env['store'].db
|
|
with db:db.execute("UPDATE work_result_outbox_v2 SET status='sending',worker_id='sender-a',updated_at=0")
|
|
await env['redis'].set('hn:task-worker:sender-a','alive',ex=30)
|
|
api=env['app'].state.deliveries
|
|
await api.recover_interrupted_sends()
|
|
assert db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='sending'
|
|
await env['redis'].delete('hn:task-worker:sender-a')
|
|
await api.recover_interrupted_sends();await api.flush()
|
|
assert db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='unknown'
|
|
assert not env['platform'].replies
|
|
|
|
|
|
async def test_task_status_uses_signed_device_binding_and_rejects_expired_tasks(env):
|
|
task(env)
|
|
path='/work/v2/tasks/state'
|
|
r=await env['client'].post(path,json=wire(env,path,task_id='task-durable'))
|
|
assert r.status_code==200 and not r.json()['result_received']
|
|
with env['store'].db:env['store'].db.execute('UPDATE work_tasks_v2 SET deadline=?',(time.time()-1,))
|
|
assert (await env['client'].post(path,json=wire(env,path,task_id='task-durable'))).status_code==403
|
|
|
|
|
|
async def test_result_receipt_loss_retry_does_not_reexecute_or_duplicate_and_rebind_blocks(env):
|
|
task(env)
|
|
assert (await upload(env)).status_code==200
|
|
# Lost HTTP receipt: the same payload is retried against durable storage.
|
|
assert (await upload(env)).status_code==200
|
|
assert len(env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall())==1
|
|
env['store'].unbind('ou_a');env['store'].bind('a'*32,'ou_a')
|
|
assert (await upload(env)).status_code==403
|
|
await env['app'].state.deliveries.flush()
|
|
assert not env['platform'].replies
|
|
|
|
|
|
async def test_worker_restart_marks_task_unknown_but_same_binding_can_finish(env):
|
|
task(env)
|
|
await env['app'].state.lifecycle.maintain()
|
|
assert env['store'].lifecycle.task('task-durable')['status']=='unknown'
|
|
assert (await upload(env)).status_code==200
|
|
await env['app'].state.deliveries.flush()
|
|
assert env['platform'].replies==[('om-synthetic','synthetic-private-result')]
|
|
|
|
|
|
async def test_send_preflight_failure_retries_only_before_sending_and_rechecks_unbind(env,monkeypatch):
|
|
task(env);await upload(env)
|
|
api=env['app'].state.deliveries
|
|
original=env['platform'].tenant_token
|
|
async def unavailable():raise ConnectionError('synthetic preflight outage')
|
|
monkeypatch.setattr(env['platform'],'tenant_token',unavailable)
|
|
await api.flush()
|
|
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='ready'
|
|
async def invalidate():
|
|
env['store'].unbind('ou_a')
|
|
return 'fake-token'
|
|
monkeypatch.setattr(env['platform'],'tenant_token',invalidate)
|
|
await api.flush()
|
|
assert not env['platform'].replies
|
|
|
|
|
|
async def test_parallel_outbox_sweepers_claim_only_one_platform_send(env,monkeypatch):
|
|
task(env);await upload(env)
|
|
api=env['app'].state.deliveries
|
|
sends=[];entered=asyncio.Event();release=asyncio.Event()
|
|
async def slow(message,text):
|
|
sends.append((message,text));entered.set();await release.wait()
|
|
monkeypatch.setattr(env['platform'],'reply',slow)
|
|
first=asyncio.create_task(api.flush())
|
|
await entered.wait()
|
|
await api.flush()
|
|
assert len(sends)==1
|
|
release.set();await first
|
|
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='replied'
|
|
|
|
|
|
async def test_ambiguous_result_is_encrypted_retained_then_expires_without_resend(env,monkeypatch):
|
|
task(env);await upload(env)
|
|
async def fail(*args):raise ConnectionError('synthetic ambiguous')
|
|
monkeypatch.setattr(env['platform'],'reply',fail)
|
|
api=env['app'].state.deliveries;await api.flush()
|
|
row=env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchone()
|
|
assert row['status']=='unknown' and row['ciphertext'] and 'synthetic-private-result' not in str(dict(row))
|
|
with env['store'].db:env['store'].db.execute('UPDATE work_result_outbox_v2 SET updated_at=?',(time.time()-8*86400,))
|
|
await api.flush()
|
|
assert env['store'].db.execute('SELECT ciphertext FROM work_result_outbox_v2').fetchone()['ciphertext']==''
|
|
|
|
|
|
async def test_completed_legacy_reply_cannot_be_uploaded_again_via_new_protocol(env):
|
|
task(env)
|
|
with env['store'].db:env['store'].db.execute("UPDATE work_tasks_v2 SET status='replied'")
|
|
assert (await upload(env)).status_code==403
|
|
assert not env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall()
|