Files

194 lines
9.0 KiB
Python

import asyncio
import time
import pytest
from test_credential_api import env, wire
from relay.task_delivery import TaskDelivery
def task(env,turn='task-durable',owner='ou_a',host='a'*32):
device=env['store'].device(host)
return env['store'].lifecycle.create_task(turn,owner,host,device['binding_generation'],
'old-session','old-worker',time.time()+600,'chat-synthetic','om-synthetic')
async def upload(env,turn='task-durable',text='synthetic-private-result'):
path='/work/v2/tasks/result'
return await env['client'].post(path,json=wire(env,path,task_id=turn,result=text))
async def test_execution_status_order_replay_terminal_and_delivery_are_separate(env):
task(env)
path='/work/v2/tasks/progress'
async def report(state,sequence,turn='task-durable'):
return await env['client'].post(path,json=wire(env,path,task_id=turn,
execution_state=state,status_sequence=sequence))
assert (await report('running',1)).json()['execution_state']=='running'
waiting=await report('waiting_for_approval',2)
assert waiting.json()['status_sequence']==2
assert (await report('running',1)).json()['execution_state']=='waiting_for_approval'
assert (await report('waiting_for_approval',2)).status_code==200
assert (await report('running',2)).status_code==409
completed=await report('completed',3)
assert completed.json()['execution_state']=='completed'
assert not completed.json()['result_received']
assert (await report('running',4)).status_code==409
await upload(env)
assert (await report('completed',3)).json()['result_received']
async def test_execution_state_auth_binding_and_parameter_validation(env):
task(env)
path='/work/v2/tasks/progress'
missing=await env['client'].post(path,json=wire(env,path,task_id='task-durable',execution_state='running'))
assert missing.status_code==422
task(env,'foreign-task','ou_b','b'*32)
foreign=await env['client'].post(path,json=wire(env,path,task_id='foreign-task',execution_state='running',status_sequence=1))
assert foreign.status_code==403
env['store'].unbind('ou_a')
revoked=await env['client'].post(path,json=wire(env,path,task_id='task-durable',execution_state='running',status_sequence=1))
assert revoked.status_code==403
assert not env['store'].db.execute('SELECT * FROM work_execution_state_v3').fetchall()
async def test_result_is_encrypted_durable_and_delivered_once_after_worker_loss(env):
task(env)
api=env['app'].state.deliveries
first=await upload(env)
assert first.status_code==200 and first.json()['result_received']
row=env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchone()
assert 'synthetic-private-result' not in str(dict(row)) and row['status']=='ready'
# Recreate application delivery service: no process-local future or keys required.
recovered=TaskDelivery(env['api'],env['store'].lifecycle,env['app'].state.broker,env['platform'])
await recovered.flush()
assert env['platform'].replies==[('om-synthetic','synthetic-private-result')]
retry=await upload(env)
assert retry.json()['status']=='replied'
await recovered.flush()
assert len(env['platform'].replies)==1
assert env['store'].db.execute('SELECT ciphertext FROM work_result_outbox_v2').fetchone()['ciphertext']==''
async def test_result_conflict_and_foreign_task_are_rejected(env):
task(env)
assert (await upload(env)).status_code==200
assert (await upload(env,text='different-result')).status_code==409
task(env,'foreign-task','ou_b','b'*32)
assert (await upload(env,'foreign-task')).status_code==403
assert len(env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall())==1
async def test_unbind_before_flush_discards_without_platform_send(env):
task(env)
await upload(env)
env['store'].unbind('ou_a')
await env['app'].state.deliveries.flush()
assert not env['platform'].replies
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='discarded'
async def test_ambiguous_platform_send_never_retries(env,monkeypatch):
task(env);await upload(env)
calls=[]
async def uncertain(*args):
calls.append(args)
raise ConnectionError('synthetic receipt loss')
monkeypatch.setattr(env['platform'],'reply',uncertain)
await env['app'].state.deliveries.flush()
await env['app'].state.deliveries.flush()
assert len(calls)==1
assert (await upload(env)).json()['status']=='unknown'
async def test_live_sender_is_not_retired_by_other_worker_and_dead_sender_not_replayed(env):
task(env);await upload(env)
db=env['store'].db
with db:db.execute("UPDATE work_result_outbox_v2 SET status='sending',worker_id='sender-a',updated_at=0")
await env['redis'].set('hn:task-worker:sender-a','alive',ex=30)
api=env['app'].state.deliveries
await api.recover_interrupted_sends()
assert db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='sending'
await env['redis'].delete('hn:task-worker:sender-a')
await api.recover_interrupted_sends();await api.flush()
assert db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='unknown'
assert not env['platform'].replies
async def test_task_status_uses_signed_device_binding_and_rejects_expired_tasks(env):
task(env)
path='/work/v2/tasks/state'
r=await env['client'].post(path,json=wire(env,path,task_id='task-durable'))
assert r.status_code==200 and not r.json()['result_received']
with env['store'].db:env['store'].db.execute('UPDATE work_tasks_v2 SET deadline=?',(time.time()-1,))
assert (await env['client'].post(path,json=wire(env,path,task_id='task-durable'))).status_code==403
async def test_result_receipt_loss_retry_does_not_reexecute_or_duplicate_and_rebind_blocks(env):
task(env)
assert (await upload(env)).status_code==200
# Lost HTTP receipt: the same payload is retried against durable storage.
assert (await upload(env)).status_code==200
assert len(env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall())==1
env['store'].unbind('ou_a');env['store'].bind('a'*32,'ou_a')
assert (await upload(env)).status_code==403
await env['app'].state.deliveries.flush()
assert not env['platform'].replies
async def test_worker_restart_marks_task_unknown_but_same_binding_can_finish(env):
task(env)
await env['app'].state.lifecycle.maintain()
assert env['store'].lifecycle.task('task-durable')['status']=='unknown'
assert (await upload(env)).status_code==200
await env['app'].state.deliveries.flush()
assert env['platform'].replies==[('om-synthetic','synthetic-private-result')]
async def test_send_preflight_failure_retries_only_before_sending_and_rechecks_unbind(env,monkeypatch):
task(env);await upload(env)
api=env['app'].state.deliveries
original=env['platform'].tenant_token
async def unavailable():raise ConnectionError('synthetic preflight outage')
monkeypatch.setattr(env['platform'],'tenant_token',unavailable)
await api.flush()
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='ready'
async def invalidate():
env['store'].unbind('ou_a')
return 'fake-token'
monkeypatch.setattr(env['platform'],'tenant_token',invalidate)
await api.flush()
assert not env['platform'].replies
async def test_parallel_outbox_sweepers_claim_only_one_platform_send(env,monkeypatch):
task(env);await upload(env)
api=env['app'].state.deliveries
sends=[];entered=asyncio.Event();release=asyncio.Event()
async def slow(message,text):
sends.append((message,text));entered.set();await release.wait()
monkeypatch.setattr(env['platform'],'reply',slow)
first=asyncio.create_task(api.flush())
await entered.wait()
await api.flush()
assert len(sends)==1
release.set();await first
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='replied'
async def test_ambiguous_result_is_encrypted_retained_then_expires_without_resend(env,monkeypatch):
task(env);await upload(env)
async def fail(*args):raise ConnectionError('synthetic ambiguous')
monkeypatch.setattr(env['platform'],'reply',fail)
api=env['app'].state.deliveries;await api.flush()
row=env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchone()
assert row['status']=='unknown' and row['ciphertext'] and 'synthetic-private-result' not in str(dict(row))
with env['store'].db:env['store'].db.execute('UPDATE work_result_outbox_v2 SET updated_at=?',(time.time()-8*86400,))
await api.flush()
assert env['store'].db.execute('SELECT ciphertext FROM work_result_outbox_v2').fetchone()['ciphertext']==''
async def test_completed_legacy_reply_cannot_be_uploaded_again_via_new_protocol(env):
task(env)
with env['store'].db:env['store'].db.execute("UPDATE work_tasks_v2 SET status='replied'")
assert (await upload(env)).status_code==403
assert not env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall()