Files
ecs-relay-service/deployable/tests/test_work_files.py
T

155 lines
8.8 KiB
Python

import asyncio
import base64
import hashlib
import json
import time
from pathlib import Path
import pytest
from test_credential_api import env,wire
from relay.work_files import WorkFiles
pytestmark=pytest.mark.parametrize('env',[True],indirect=True)
def create_task(env,name='task-files',owner='ou_a',host='a'*32,chat='chat-files'):
generation=env['store'].device(host)['binding_generation']
return env['store'].lifecycle.create_task(name,owner,host,generation,'session','worker',time.time()+600,chat,'om-original')
def fake_platform(env,monkeypatch):
files={};calls=[]
async def download(message,key,limit):calls.append(('download',message,key));return b'PRIVATE_MATERIAL_'+key.encode()
async def upload(name,raw,mime,*,token):
calls.append(('upload',name));key='file_'+hashlib.sha256(raw).hexdigest();files[key]=raw;return key
async def reply(message,key,delivery,*,token):calls.append(('reply_file',message,key,delivery));return 'om-file-reply'
monkeypatch.setattr(env['platform'],'download_message_file',download,raising=False)
monkeypatch.setattr(env['platform'],'upload_file',upload,raising=False)
monkeypatch.setattr(env['platform'],'reply_file',reply,raising=False)
return files,calls
async def upload(env,task='task-files',raw=b'PRIVATE_GENERATED_RESULT',artifact='artifact-01',**overrides):
path='/work/v2/files/upload'
data=dict(task_id=task,artifact_id=artifact,name='result.txt',sha256=hashlib.sha256(raw).hexdigest(),size=len(raw));data.update(overrides)
body=wire(env,path,**data)
return await env['client'].post(path,content=raw,headers={'X-Hainuo-File-Request':base64.urlsafe_b64encode(json.dumps(body).encode()).decode()})
async def test_material_event_is_staged_without_model_and_duplicate_does_not_redownload(env,monkeypatch):
_,calls=fake_platform(env,monkeypatch)
event={'header':{'event_type':'im.message.receive_v1','app_id':'cli_test'},'event':{
'sender':{'sender_type':'user','sender_id':{'open_id':'ou_a'}},
'message':{'chat_type':'p2p','chat_id':'chat-files','message_type':'file','message_id':'om-input',
'content':json.dumps({'file_name':'source.txt','file_key':'file-source'})}}}
for _ in range(2):
assert (await env['client'].post('/internal/events',json=event,headers={'X-Internal-Secret':env['settings'].internal_secret})).status_code==200
await asyncio.sleep(.02)
assert len([x for x in calls if x[0]=='download'])==1
row=env['store'].db.execute('SELECT * FROM work_files_v1').fetchone()
assert row['status']=='staged' and row['task_id']==''
assert 'PRIVATE_MATERIAL'not in str(dict(row))
assert env['store'].db.execute('SELECT COUNT(*) AS n FROM work_tasks_v2').fetchone()['n']==0
assert len(env['platform'].replies)==1
async def test_claim_download_and_cross_owner_rejection(env,monkeypatch):
fake_platform(env,monkeypatch);api=env['app'].state.files
incoming=await api.receive('ou_a','chat-files','om-input',{'file_name':'source.txt','file_key':'file-source'})
task=create_task(env);api.claim(task)
path='/work/v2/files/list';listed=await env['client'].post(path,json=wire(env,path,task_id='task-files'))
assert listed.json()['files'][0]['file_id']==incoming['file_id']
path='/work/v2/files/download';body=wire(env,path,task_id='task-files',file_id=incoming['file_id'])
received=await env['client'].post(path,json=body)
assert received.content==b'PRIVATE_MATERIAL_file-source'
assert received.headers['X-Hainuo-File-SHA256']==hashlib.sha256(received.content).hexdigest()
bad=await env['client'].post(path,json=wire(env,path,host='b'*32,task_id='task-files',file_id=incoming['file_id']))
assert bad.status_code==403
assert (await env['client'].post(path,json=body)).status_code==409
async def test_artifact_handoff_encrypted_idempotent_and_replies_to_original_only(env,monkeypatch):
files,calls=fake_platform(env,monkeypatch);create_task(env)
first=await upload(env);assert first.status_code==200 and first.json()['received']
assert (await upload(env)).json()['file_id']==first.json()['file_id']
row=env['store'].db.execute("SELECT * FROM work_files_v1 WHERE direction='output'").fetchone()
assert 'PRIVATE_GENERATED_RESULT'not in str(dict(row))
assert (await upload(env,raw=b'changed')).status_code==409
api=WorkFiles(env['settings'],env['api'],env['store'].lifecycle,env['platform'])
await api.flush();await api.flush();await api.flush()
assert len([x for x in calls if x[0]=='upload'])==1
replies=[x for x in calls if x[0]=='reply_file'];assert len(replies)==1 and replies[0][1]=='om-original'
assert files[replies[0][2]]==b'PRIVATE_GENERATED_RESULT'
assert env['store'].db.execute("SELECT status,ciphertext FROM work_files_v1").fetchone()['status']=='sent'
async def test_bad_digest_filename_size_and_stale_binding_are_rejected(env,monkeypatch):
fake_platform(env,monkeypatch);create_task(env)
assert (await upload(env,sha256='0'*64)).status_code==409
assert (await upload(env,name='../secret.txt')).status_code==422
assert (await upload(env,size=env['settings'].work_file_max_bytes+1)).status_code==413
env['store'].unbind('ou_a')
assert (await upload(env)).status_code==403
assert env['store'].db.execute('SELECT COUNT(*) AS n FROM work_files_v1').fetchone()['n']==0
async def test_ambiguous_platform_upload_or_send_never_repeats(env,monkeypatch):
_,calls=fake_platform(env,monkeypatch);create_task(env);assert (await upload(env)).status_code==200
async def uncertain(*a,**kw):calls.append(('uncertain',));raise ConnectionError('PRIVATE_PLATFORM_ERROR')
monkeypatch.setattr(env['platform'],'upload_file',uncertain)
api=env['app'].state.files;await api.flush();await api.flush()
assert len([x for x in calls if x[0]=='uncertain'])==1
assert env['store'].db.execute('SELECT status FROM work_files_v1').fetchone()['status']=='unknown'
async def test_unbind_before_delivery_discards_without_platform_upload(env,monkeypatch):
_,calls=fake_platform(env,monkeypatch);create_task(env);await upload(env)
env['store'].unbind('ou_a');await env['app'].state.files.flush()
assert not calls
row=env['store'].db.execute('SELECT status,ciphertext FROM work_files_v1').fetchone()
assert row['status']=='discarded'and row['ciphertext']==''
async def test_file_body_limit_is_separate_from_json_limit(env,monkeypatch):
fake_platform(env,monkeypatch);create_task(env)
raw=b'x'*(env['settings'].max_body+1)
assert (await upload(env,raw=raw)).status_code==200
response=await env['client'].post('/work/v2/files/list',content=b'x'*(env['settings'].max_body+1))
assert response.status_code==413
async def test_receiving_material_blocks_dispatch_and_failed_input_is_explicit(env,monkeypatch):
fake_platform(env,monkeypatch);api=env['app'].state.files;entered=asyncio.Event();resume=asyncio.Event()
async def slow(*args):entered.set();await resume.wait();raise ConnectionError('synthetic download failed')
monkeypatch.setattr(env['platform'],'download_message_file',slow)
receiving=asyncio.create_task(api.receive('ou_a','chat-files','om-slow',{'file_name':'source.txt','file_key':'file-slow'}))
await entered.wait()
assert api.pending_count('ou_a','a'*32,env['store'].device('a'*32)['binding_generation'],'chat-files',True)==1
task=create_task(env)
with pytest.raises(ValueError,match='receiving'):api.claim(task)
resume.set()
with pytest.raises(ConnectionError):await receiving
api.claim(task)
row=env['store'].db.execute('SELECT status,task_id,ciphertext FROM work_files_v1').fetchone()
assert row['status']=='failed_assigned'and row['task_id']=='task-files'and row['ciphertext']==''
async def test_owner_quota_blocks_storage_without_upload_or_download(env,monkeypatch):
_,calls=fake_platform(env,monkeypatch);api=env['app'].state.files;create_task(env)
monkeypatch.setattr(env['settings'],'work_file_owner_quota_bytes',10)
response=await upload(env)
assert response.status_code==413 and not calls
from fastapi import HTTPException
with pytest.raises(HTTPException)as error:await api.receive('ou_a','chat-files','om-quota',{'file_name':'source.txt','file_key':'file-source'})
assert error.value.status_code==413 and not calls
async def test_lost_platform_message_receipt_is_unknown_without_repeat(env,monkeypatch):
_,calls=fake_platform(env,monkeypatch);create_task(env);await upload(env)
api=env['app'].state.files;await api.flush()
async def unknown(*args,**kw):calls.append(('reply_unknown',));raise ConnectionError('lost message receipt')
monkeypatch.setattr(env['platform'],'reply_file',unknown)
await api.flush();await api.flush()
assert len([x for x in calls if x[0]=='reply_unknown'])==1
assert env['store'].db.execute('SELECT status FROM work_files_v1').fetchone()['status']=='unknown'