feat(work): add task-scoped attachment intake and durable file delivery
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from test_credential_api import env,wire
|
||||
from relay.work_files import WorkFiles
|
||||
|
||||
pytestmark=pytest.mark.parametrize('env',[True],indirect=True)
|
||||
|
||||
|
||||
def create_task(env,name='task-files',owner='ou_a',host='a'*32,chat='chat-files'):
|
||||
generation=env['store'].device(host)['binding_generation']
|
||||
return env['store'].lifecycle.create_task(name,owner,host,generation,'session','worker',time.time()+600,chat,'om-original')
|
||||
|
||||
|
||||
def fake_platform(env,monkeypatch):
|
||||
files={};calls=[]
|
||||
async def download(message,key,limit):calls.append(('download',message,key));return b'PRIVATE_MATERIAL_'+key.encode()
|
||||
async def upload(name,raw,mime,*,token):
|
||||
calls.append(('upload',name));key='file_'+hashlib.sha256(raw).hexdigest();files[key]=raw;return key
|
||||
async def reply(message,key,delivery,*,token):calls.append(('reply_file',message,key,delivery));return 'om-file-reply'
|
||||
monkeypatch.setattr(env['platform'],'download_message_file',download,raising=False)
|
||||
monkeypatch.setattr(env['platform'],'upload_file',upload,raising=False)
|
||||
monkeypatch.setattr(env['platform'],'reply_file',reply,raising=False)
|
||||
return files,calls
|
||||
|
||||
|
||||
async def upload(env,task='task-files',raw=b'PRIVATE_GENERATED_RESULT',artifact='artifact-01',**overrides):
|
||||
path='/work/v2/files/upload'
|
||||
data=dict(task_id=task,artifact_id=artifact,name='result.txt',sha256=hashlib.sha256(raw).hexdigest(),size=len(raw));data.update(overrides)
|
||||
body=wire(env,path,**data)
|
||||
return await env['client'].post(path,content=raw,headers={'X-Hainuo-File-Request':base64.urlsafe_b64encode(json.dumps(body).encode()).decode()})
|
||||
|
||||
|
||||
async def test_material_event_is_staged_without_model_and_duplicate_does_not_redownload(env,monkeypatch):
|
||||
_,calls=fake_platform(env,monkeypatch)
|
||||
event={'header':{'event_type':'im.message.receive_v1','app_id':'cli_test'},'event':{
|
||||
'sender':{'sender_type':'user','sender_id':{'open_id':'ou_a'}},
|
||||
'message':{'chat_type':'p2p','chat_id':'chat-files','message_type':'file','message_id':'om-input',
|
||||
'content':json.dumps({'file_name':'source.txt','file_key':'file-source'})}}}
|
||||
for _ in range(2):
|
||||
assert (await env['client'].post('/internal/events',json=event,headers={'X-Internal-Secret':env['settings'].internal_secret})).status_code==200
|
||||
await asyncio.sleep(.02)
|
||||
assert len([x for x in calls if x[0]=='download'])==1
|
||||
row=env['store'].db.execute('SELECT * FROM work_files_v1').fetchone()
|
||||
assert row['status']=='staged' and row['task_id']==''
|
||||
assert 'PRIVATE_MATERIAL'not in str(dict(row))
|
||||
assert env['store'].db.execute('SELECT COUNT(*) AS n FROM work_tasks_v2').fetchone()['n']==0
|
||||
assert len(env['platform'].replies)==1
|
||||
|
||||
|
||||
async def test_claim_download_and_cross_owner_rejection(env,monkeypatch):
|
||||
fake_platform(env,monkeypatch);api=env['app'].state.files
|
||||
incoming=await api.receive('ou_a','chat-files','om-input',{'file_name':'source.txt','file_key':'file-source'})
|
||||
task=create_task(env);api.claim(task)
|
||||
path='/work/v2/files/list';listed=await env['client'].post(path,json=wire(env,path,task_id='task-files'))
|
||||
assert listed.json()['files'][0]['file_id']==incoming['file_id']
|
||||
path='/work/v2/files/download';body=wire(env,path,task_id='task-files',file_id=incoming['file_id'])
|
||||
received=await env['client'].post(path,json=body)
|
||||
assert received.content==b'PRIVATE_MATERIAL_file-source'
|
||||
assert received.headers['X-Hainuo-File-SHA256']==hashlib.sha256(received.content).hexdigest()
|
||||
bad=await env['client'].post(path,json=wire(env,path,host='b'*32,task_id='task-files',file_id=incoming['file_id']))
|
||||
assert bad.status_code==403
|
||||
assert (await env['client'].post(path,json=body)).status_code==409
|
||||
|
||||
|
||||
async def test_artifact_handoff_encrypted_idempotent_and_replies_to_original_only(env,monkeypatch):
|
||||
files,calls=fake_platform(env,monkeypatch);create_task(env)
|
||||
first=await upload(env);assert first.status_code==200 and first.json()['received']
|
||||
assert (await upload(env)).json()['file_id']==first.json()['file_id']
|
||||
row=env['store'].db.execute("SELECT * FROM work_files_v1 WHERE direction='output'").fetchone()
|
||||
assert 'PRIVATE_GENERATED_RESULT'not in str(dict(row))
|
||||
assert (await upload(env,raw=b'changed')).status_code==409
|
||||
api=WorkFiles(env['settings'],env['api'],env['store'].lifecycle,env['platform'])
|
||||
await api.flush();await api.flush();await api.flush()
|
||||
assert len([x for x in calls if x[0]=='upload'])==1
|
||||
replies=[x for x in calls if x[0]=='reply_file'];assert len(replies)==1 and replies[0][1]=='om-original'
|
||||
assert files[replies[0][2]]==b'PRIVATE_GENERATED_RESULT'
|
||||
assert env['store'].db.execute("SELECT status,ciphertext FROM work_files_v1").fetchone()['status']=='sent'
|
||||
|
||||
|
||||
async def test_bad_digest_filename_size_and_stale_binding_are_rejected(env,monkeypatch):
|
||||
fake_platform(env,monkeypatch);create_task(env)
|
||||
assert (await upload(env,sha256='0'*64)).status_code==409
|
||||
assert (await upload(env,name='../secret.txt')).status_code==422
|
||||
assert (await upload(env,size=env['settings'].work_file_max_bytes+1)).status_code==413
|
||||
env['store'].unbind('ou_a')
|
||||
assert (await upload(env)).status_code==403
|
||||
assert env['store'].db.execute('SELECT COUNT(*) AS n FROM work_files_v1').fetchone()['n']==0
|
||||
|
||||
|
||||
async def test_ambiguous_platform_upload_or_send_never_repeats(env,monkeypatch):
|
||||
_,calls=fake_platform(env,monkeypatch);create_task(env);assert (await upload(env)).status_code==200
|
||||
async def uncertain(*a,**kw):calls.append(('uncertain',));raise ConnectionError('PRIVATE_PLATFORM_ERROR')
|
||||
monkeypatch.setattr(env['platform'],'upload_file',uncertain)
|
||||
api=env['app'].state.files;await api.flush();await api.flush()
|
||||
assert len([x for x in calls if x[0]=='uncertain'])==1
|
||||
assert env['store'].db.execute('SELECT status FROM work_files_v1').fetchone()['status']=='unknown'
|
||||
|
||||
|
||||
async def test_unbind_before_delivery_discards_without_platform_upload(env,monkeypatch):
|
||||
_,calls=fake_platform(env,monkeypatch);create_task(env);await upload(env)
|
||||
env['store'].unbind('ou_a');await env['app'].state.files.flush()
|
||||
assert not calls
|
||||
row=env['store'].db.execute('SELECT status,ciphertext FROM work_files_v1').fetchone()
|
||||
assert row['status']=='discarded'and row['ciphertext']==''
|
||||
|
||||
|
||||
async def test_file_body_limit_is_separate_from_json_limit(env,monkeypatch):
|
||||
fake_platform(env,monkeypatch);create_task(env)
|
||||
raw=b'x'*(env['settings'].max_body+1)
|
||||
assert (await upload(env,raw=raw)).status_code==200
|
||||
response=await env['client'].post('/work/v2/files/list',content=b'x'*(env['settings'].max_body+1))
|
||||
assert response.status_code==413
|
||||
|
||||
|
||||
async def test_receiving_material_blocks_dispatch_and_failed_input_is_explicit(env,monkeypatch):
|
||||
fake_platform(env,monkeypatch);api=env['app'].state.files;entered=asyncio.Event();resume=asyncio.Event()
|
||||
async def slow(*args):entered.set();await resume.wait();raise ConnectionError('synthetic download failed')
|
||||
monkeypatch.setattr(env['platform'],'download_message_file',slow)
|
||||
receiving=asyncio.create_task(api.receive('ou_a','chat-files','om-slow',{'file_name':'source.txt','file_key':'file-slow'}))
|
||||
await entered.wait()
|
||||
assert api.pending_count('ou_a','a'*32,env['store'].device('a'*32)['binding_generation'],'chat-files',True)==1
|
||||
task=create_task(env)
|
||||
with pytest.raises(ValueError,match='receiving'):api.claim(task)
|
||||
resume.set()
|
||||
with pytest.raises(ConnectionError):await receiving
|
||||
api.claim(task)
|
||||
row=env['store'].db.execute('SELECT status,task_id,ciphertext FROM work_files_v1').fetchone()
|
||||
assert row['status']=='failed_assigned'and row['task_id']=='task-files'and row['ciphertext']==''
|
||||
|
||||
|
||||
async def test_owner_quota_blocks_storage_without_upload_or_download(env,monkeypatch):
|
||||
_,calls=fake_platform(env,monkeypatch);api=env['app'].state.files;create_task(env)
|
||||
monkeypatch.setattr(env['settings'],'work_file_owner_quota_bytes',10)
|
||||
response=await upload(env)
|
||||
assert response.status_code==413 and not calls
|
||||
from fastapi import HTTPException
|
||||
with pytest.raises(HTTPException)as error:await api.receive('ou_a','chat-files','om-quota',{'file_name':'source.txt','file_key':'file-source'})
|
||||
assert error.value.status_code==413 and not calls
|
||||
|
||||
|
||||
async def test_lost_platform_message_receipt_is_unknown_without_repeat(env,monkeypatch):
|
||||
_,calls=fake_platform(env,monkeypatch);create_task(env);await upload(env)
|
||||
api=env['app'].state.files;await api.flush()
|
||||
async def unknown(*args,**kw):calls.append(('reply_unknown',));raise ConnectionError('lost message receipt')
|
||||
monkeypatch.setattr(env['platform'],'reply_file',unknown)
|
||||
await api.flush();await api.flush()
|
||||
assert len([x for x in calls if x[0]=='reply_unknown'])==1
|
||||
assert env['store'].db.execute('SELECT status FROM work_files_v1').fetchone()['status']=='unknown'
|
||||
Reference in New Issue
Block a user