feat(work): add task-scoped attachment intake and durable file delivery
This commit is contained in:
@@ -643,3 +643,18 @@ def test_private_approved_profile_excludes_pending_review_scopes():
|
||||
approved=set(json.loads((ROOT/'examples/work-product-profiles.approved.json').read_text())['current-config-v1'])
|
||||
assert all_scopes-approved=={'contact:department.base:readonly','contact:department.organize:readonly','task:tasklist:write'}
|
||||
assert len(approved)==20 and {'drive:drive','wiki:wiki','bitable:app','task:task:write'}<=approved
|
||||
|
||||
|
||||
def test_prepare_update_file_feature_is_explicit_and_rollback_restores_parent_flag(scenario):
|
||||
parent=prepared(scenario);root=scenario['output'];image='sha256:'+'a'*64
|
||||
for name in ('connector','router'):scenario['roles'][name]['image']=image
|
||||
deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']})
|
||||
output=root.parent/'update-files'
|
||||
args=argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output),enable_work_files=True)
|
||||
deploy.prepare_update(args)
|
||||
current=json.loads((output/'compose.broker.json').read_text())
|
||||
rollback=json.loads((output/'compose.rollback.json').read_text())
|
||||
old=json.loads((root/'compose.broker.json').read_text())
|
||||
assert current['services']['connector']['environment']['HN_WORK_FILES_ENABLED']=='true'
|
||||
assert rollback['services']['connector']['environment']==old['services']['connector']['environment']
|
||||
assert (output/'credential-keyring.json').read_bytes()==(root/'credential-keyring.json').read_bytes()
|
||||
|
||||
@@ -74,7 +74,7 @@ class Platform:
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def env(tmp_path):
|
||||
async def env(tmp_path,request):
|
||||
ca_key, ca = issue_ca()
|
||||
ca_path = tmp_path / "ca.pem"
|
||||
ca_path.write_bytes(ca.public_bytes(serialization.Encoding.PEM))
|
||||
@@ -89,7 +89,7 @@ async def env(tmp_path):
|
||||
feishu_app_id="cli_test", feishu_app_secret="fake-app-secret", credential_mode="broker",
|
||||
credential_keyring_file=keyring, credential_permission_profiles={"product-v1": ["read"], "product-v2": ["read", "write"]},
|
||||
public_url="https://relay.example.test", feishu_api_base="https://work.example.test",
|
||||
feishu_authorize_url="https://work.example.test/open-apis/authen/v1/index")
|
||||
feishu_authorize_url="https://work.example.test/open-apis/authen/v1/index",work_files_enabled=bool(getattr(request,'param',False)))
|
||||
redis = fakeredis.aioredis.FakeRedis(server=fakeredis.FakeServer(), decode_responses=True)
|
||||
platform = Platform()
|
||||
app = create_app(settings, redis, platform)
|
||||
|
||||
@@ -52,7 +52,7 @@ async def routed(env, tmp_path):
|
||||
await asyncio.wait_for(server_task,5)
|
||||
|
||||
|
||||
async def device(env, host="a"*32, *, generation=None, legacy=False):
|
||||
async def device(env, host="a"*32, *, generation=None, legacy=False, work_files=False):
|
||||
ws = await connect(env["url"],proxy=None)
|
||||
env["sessions"].append(ws)
|
||||
challenge = json.loads(await ws.recv())["challenge"]
|
||||
@@ -65,6 +65,7 @@ async def device(env, host="a"*32, *, generation=None, legacy=False):
|
||||
plain.update(protocol_version=2,environment=env["settings"].environment,
|
||||
app_id=env["settings"].feishu_app_id,binding_owner=row["owner"] or "ou_a",
|
||||
binding_generation=row["binding_generation"] if generation is None else generation)
|
||||
if work_files:plain['work_files']='work-files-v1'
|
||||
await ws.send(json.dumps({"host_id":host,"cert":row["cert"],"proof":sign(plain,env["keys"][host])}))
|
||||
accepted = json.loads(await asyncio.wait_for(ws.recv(),3))
|
||||
assert accepted["type"] == "connected"
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from test_credential_api import env,wire
|
||||
from relay.work_files import WorkFiles
|
||||
|
||||
pytestmark=pytest.mark.parametrize('env',[True],indirect=True)
|
||||
|
||||
|
||||
def create_task(env,name='task-files',owner='ou_a',host='a'*32,chat='chat-files'):
|
||||
generation=env['store'].device(host)['binding_generation']
|
||||
return env['store'].lifecycle.create_task(name,owner,host,generation,'session','worker',time.time()+600,chat,'om-original')
|
||||
|
||||
|
||||
def fake_platform(env,monkeypatch):
|
||||
files={};calls=[]
|
||||
async def download(message,key,limit):calls.append(('download',message,key));return b'PRIVATE_MATERIAL_'+key.encode()
|
||||
async def upload(name,raw,mime,*,token):
|
||||
calls.append(('upload',name));key='file_'+hashlib.sha256(raw).hexdigest();files[key]=raw;return key
|
||||
async def reply(message,key,delivery,*,token):calls.append(('reply_file',message,key,delivery));return 'om-file-reply'
|
||||
monkeypatch.setattr(env['platform'],'download_message_file',download,raising=False)
|
||||
monkeypatch.setattr(env['platform'],'upload_file',upload,raising=False)
|
||||
monkeypatch.setattr(env['platform'],'reply_file',reply,raising=False)
|
||||
return files,calls
|
||||
|
||||
|
||||
async def upload(env,task='task-files',raw=b'PRIVATE_GENERATED_RESULT',artifact='artifact-01',**overrides):
|
||||
path='/work/v2/files/upload'
|
||||
data=dict(task_id=task,artifact_id=artifact,name='result.txt',sha256=hashlib.sha256(raw).hexdigest(),size=len(raw));data.update(overrides)
|
||||
body=wire(env,path,**data)
|
||||
return await env['client'].post(path,content=raw,headers={'X-Hainuo-File-Request':base64.urlsafe_b64encode(json.dumps(body).encode()).decode()})
|
||||
|
||||
|
||||
async def test_material_event_is_staged_without_model_and_duplicate_does_not_redownload(env,monkeypatch):
|
||||
_,calls=fake_platform(env,monkeypatch)
|
||||
event={'header':{'event_type':'im.message.receive_v1','app_id':'cli_test'},'event':{
|
||||
'sender':{'sender_type':'user','sender_id':{'open_id':'ou_a'}},
|
||||
'message':{'chat_type':'p2p','chat_id':'chat-files','message_type':'file','message_id':'om-input',
|
||||
'content':json.dumps({'file_name':'source.txt','file_key':'file-source'})}}}
|
||||
for _ in range(2):
|
||||
assert (await env['client'].post('/internal/events',json=event,headers={'X-Internal-Secret':env['settings'].internal_secret})).status_code==200
|
||||
await asyncio.sleep(.02)
|
||||
assert len([x for x in calls if x[0]=='download'])==1
|
||||
row=env['store'].db.execute('SELECT * FROM work_files_v1').fetchone()
|
||||
assert row['status']=='staged' and row['task_id']==''
|
||||
assert 'PRIVATE_MATERIAL'not in str(dict(row))
|
||||
assert env['store'].db.execute('SELECT COUNT(*) AS n FROM work_tasks_v2').fetchone()['n']==0
|
||||
assert len(env['platform'].replies)==1
|
||||
|
||||
|
||||
async def test_claim_download_and_cross_owner_rejection(env,monkeypatch):
|
||||
fake_platform(env,monkeypatch);api=env['app'].state.files
|
||||
incoming=await api.receive('ou_a','chat-files','om-input',{'file_name':'source.txt','file_key':'file-source'})
|
||||
task=create_task(env);api.claim(task)
|
||||
path='/work/v2/files/list';listed=await env['client'].post(path,json=wire(env,path,task_id='task-files'))
|
||||
assert listed.json()['files'][0]['file_id']==incoming['file_id']
|
||||
path='/work/v2/files/download';body=wire(env,path,task_id='task-files',file_id=incoming['file_id'])
|
||||
received=await env['client'].post(path,json=body)
|
||||
assert received.content==b'PRIVATE_MATERIAL_file-source'
|
||||
assert received.headers['X-Hainuo-File-SHA256']==hashlib.sha256(received.content).hexdigest()
|
||||
bad=await env['client'].post(path,json=wire(env,path,host='b'*32,task_id='task-files',file_id=incoming['file_id']))
|
||||
assert bad.status_code==403
|
||||
assert (await env['client'].post(path,json=body)).status_code==409
|
||||
|
||||
|
||||
async def test_artifact_handoff_encrypted_idempotent_and_replies_to_original_only(env,monkeypatch):
|
||||
files,calls=fake_platform(env,monkeypatch);create_task(env)
|
||||
first=await upload(env);assert first.status_code==200 and first.json()['received']
|
||||
assert (await upload(env)).json()['file_id']==first.json()['file_id']
|
||||
row=env['store'].db.execute("SELECT * FROM work_files_v1 WHERE direction='output'").fetchone()
|
||||
assert 'PRIVATE_GENERATED_RESULT'not in str(dict(row))
|
||||
assert (await upload(env,raw=b'changed')).status_code==409
|
||||
api=WorkFiles(env['settings'],env['api'],env['store'].lifecycle,env['platform'])
|
||||
await api.flush();await api.flush();await api.flush()
|
||||
assert len([x for x in calls if x[0]=='upload'])==1
|
||||
replies=[x for x in calls if x[0]=='reply_file'];assert len(replies)==1 and replies[0][1]=='om-original'
|
||||
assert files[replies[0][2]]==b'PRIVATE_GENERATED_RESULT'
|
||||
assert env['store'].db.execute("SELECT status,ciphertext FROM work_files_v1").fetchone()['status']=='sent'
|
||||
|
||||
|
||||
async def test_bad_digest_filename_size_and_stale_binding_are_rejected(env,monkeypatch):
|
||||
fake_platform(env,monkeypatch);create_task(env)
|
||||
assert (await upload(env,sha256='0'*64)).status_code==409
|
||||
assert (await upload(env,name='../secret.txt')).status_code==422
|
||||
assert (await upload(env,size=env['settings'].work_file_max_bytes+1)).status_code==413
|
||||
env['store'].unbind('ou_a')
|
||||
assert (await upload(env)).status_code==403
|
||||
assert env['store'].db.execute('SELECT COUNT(*) AS n FROM work_files_v1').fetchone()['n']==0
|
||||
|
||||
|
||||
async def test_ambiguous_platform_upload_or_send_never_repeats(env,monkeypatch):
|
||||
_,calls=fake_platform(env,monkeypatch);create_task(env);assert (await upload(env)).status_code==200
|
||||
async def uncertain(*a,**kw):calls.append(('uncertain',));raise ConnectionError('PRIVATE_PLATFORM_ERROR')
|
||||
monkeypatch.setattr(env['platform'],'upload_file',uncertain)
|
||||
api=env['app'].state.files;await api.flush();await api.flush()
|
||||
assert len([x for x in calls if x[0]=='uncertain'])==1
|
||||
assert env['store'].db.execute('SELECT status FROM work_files_v1').fetchone()['status']=='unknown'
|
||||
|
||||
|
||||
async def test_unbind_before_delivery_discards_without_platform_upload(env,monkeypatch):
|
||||
_,calls=fake_platform(env,monkeypatch);create_task(env);await upload(env)
|
||||
env['store'].unbind('ou_a');await env['app'].state.files.flush()
|
||||
assert not calls
|
||||
row=env['store'].db.execute('SELECT status,ciphertext FROM work_files_v1').fetchone()
|
||||
assert row['status']=='discarded'and row['ciphertext']==''
|
||||
|
||||
|
||||
async def test_file_body_limit_is_separate_from_json_limit(env,monkeypatch):
|
||||
fake_platform(env,monkeypatch);create_task(env)
|
||||
raw=b'x'*(env['settings'].max_body+1)
|
||||
assert (await upload(env,raw=raw)).status_code==200
|
||||
response=await env['client'].post('/work/v2/files/list',content=b'x'*(env['settings'].max_body+1))
|
||||
assert response.status_code==413
|
||||
|
||||
|
||||
async def test_receiving_material_blocks_dispatch_and_failed_input_is_explicit(env,monkeypatch):
|
||||
fake_platform(env,monkeypatch);api=env['app'].state.files;entered=asyncio.Event();resume=asyncio.Event()
|
||||
async def slow(*args):entered.set();await resume.wait();raise ConnectionError('synthetic download failed')
|
||||
monkeypatch.setattr(env['platform'],'download_message_file',slow)
|
||||
receiving=asyncio.create_task(api.receive('ou_a','chat-files','om-slow',{'file_name':'source.txt','file_key':'file-slow'}))
|
||||
await entered.wait()
|
||||
assert api.pending_count('ou_a','a'*32,env['store'].device('a'*32)['binding_generation'],'chat-files',True)==1
|
||||
task=create_task(env)
|
||||
with pytest.raises(ValueError,match='receiving'):api.claim(task)
|
||||
resume.set()
|
||||
with pytest.raises(ConnectionError):await receiving
|
||||
api.claim(task)
|
||||
row=env['store'].db.execute('SELECT status,task_id,ciphertext FROM work_files_v1').fetchone()
|
||||
assert row['status']=='failed_assigned'and row['task_id']=='task-files'and row['ciphertext']==''
|
||||
|
||||
|
||||
async def test_owner_quota_blocks_storage_without_upload_or_download(env,monkeypatch):
|
||||
_,calls=fake_platform(env,monkeypatch);api=env['app'].state.files;create_task(env)
|
||||
monkeypatch.setattr(env['settings'],'work_file_owner_quota_bytes',10)
|
||||
response=await upload(env)
|
||||
assert response.status_code==413 and not calls
|
||||
from fastapi import HTTPException
|
||||
with pytest.raises(HTTPException)as error:await api.receive('ou_a','chat-files','om-quota',{'file_name':'source.txt','file_key':'file-source'})
|
||||
assert error.value.status_code==413 and not calls
|
||||
|
||||
|
||||
async def test_lost_platform_message_receipt_is_unknown_without_repeat(env,monkeypatch):
|
||||
_,calls=fake_platform(env,monkeypatch);create_task(env);await upload(env)
|
||||
api=env['app'].state.files;await api.flush()
|
||||
async def unknown(*args,**kw):calls.append(('reply_unknown',));raise ConnectionError('lost message receipt')
|
||||
monkeypatch.setattr(env['platform'],'reply_file',unknown)
|
||||
await api.flush();await api.flush()
|
||||
assert len([x for x in calls if x[0]=='reply_unknown'])==1
|
||||
assert env['store'].db.execute('SELECT status FROM work_files_v1').fetchone()['status']=='unknown'
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Optional cross-checkout test using actual PC transport/signatures, no real Work."""
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from test_credential_api import env
|
||||
from test_work_files import create_task,fake_platform
|
||||
from relay.crypto import pem_private
|
||||
|
||||
pytestmark=pytest.mark.parametrize('env',[True],indirect=True)
|
||||
|
||||
|
||||
async def test_three_materials_native_output_lost_receipt_and_original_file_reply(env,tmp_path,monkeypatch):
|
||||
source=os.environ.get('HN_PC_FILES_SOURCE')
|
||||
if not source:pytest.skip('explicit PC source checkout required')
|
||||
source=Path(source);package=source/'integrations/work-relay/service/relay'
|
||||
spec=importlib.util.spec_from_file_location('_pc_work_file_chain',package/'__init__.py',submodule_search_locations=[str(package)])
|
||||
module=importlib.util.module_from_spec(spec);sys.modules[spec.name]=module;spec.loader.exec_module(module)
|
||||
files_module=importlib.import_module(spec.name+'.work_files')
|
||||
credentials=importlib.import_module(spec.name+'.credential_provider')
|
||||
office_spec=importlib.util.spec_from_file_location('_office_chain',source/'capability-slim/runtime/office_workflows.py')
|
||||
office=importlib.util.module_from_spec(office_spec);office_spec.loader.exec_module(office)
|
||||
remote_files,platform_calls=fake_platform(env,monkeypatch)
|
||||
samples=os.environ.get('HN_PC_SAMPLE_DIR');native_python=os.environ.get('HN_OFFICE_PYTHON')
|
||||
names=['材料.docx','表格.xlsx','参考.pdf']if samples else['one.txt','two.csv','three.md']
|
||||
if samples:
|
||||
paths=[Path(samples)/n for n in ['报告模板.docx','材料一.xlsx','文本材料.pdf']]
|
||||
raw_materials=[p.read_bytes()for p in paths]
|
||||
else:raw_materials=[b'synthetic material 1',b'column,value\nx,1',b'# source']
|
||||
mapping={f'file-input-{i}':content for i,content in enumerate(raw_materials)}
|
||||
async def download(message,key,limit):return mapping[key]
|
||||
monkeypatch.setattr(env['platform'],'download_message_file',download)
|
||||
api=env['app'].state.files
|
||||
for i,name in enumerate(names):await api.receive('ou_a','chat-files',f'om-input-{i}',{'file_name':name,'file_key':f'file-input-{i}'})
|
||||
task=create_task(env);api.claim(task)
|
||||
context={k:task[k]for k in ['environment','app_id','owner','host_id','binding_generation','chat_id','source_message_id','conversation_id']}
|
||||
context.update(task_id='task-files',channel='work-dm')
|
||||
device=tmp_path/'pc-device';device.mkdir();(device/'device.json').write_text(json.dumps({'host_id':'a'*32,'owner':'ou_a','binding_generation':task['binding_generation']}))
|
||||
(device/'device.key').write_bytes(pem_private(env['keys']['a'*32]))
|
||||
loop=asyncio.get_running_loop();requests=[];lost=[False]
|
||||
def forward(request):
|
||||
requests.append(request.url.path)
|
||||
future=asyncio.run_coroutine_threadsafe(env['client'].request(request.method,request.url.path,
|
||||
content=request.read(),headers=dict(request.headers)),loop)
|
||||
response=future.result(20)
|
||||
if request.url.path.endswith('/upload')and not lost[0]:
|
||||
lost[0]=True;raise httpx.ReadError('synthetic lost upload receipt')
|
||||
return httpx.Response(response.status_code,headers=dict(response.headers),content=response.content)
|
||||
transport=httpx.Client(base_url='https://relay.example.test',transport=httpx.MockTransport(forward))
|
||||
provider=object.__new__(credentials.CredentialProvider)
|
||||
provider.directory=device;provider.host='a'*32;provider.owner='ou_a';provider.wall=time.time;provider.http=transport
|
||||
provider.release={'environment':'test','app_id':'cli_test'};provider.cache_key=('synthetic',)
|
||||
client=SimpleNamespace(provider=provider,http=transport)
|
||||
home=tmp_path/'pc-home'
|
||||
try:
|
||||
staged=await asyncio.to_thread(files_module.stage_materials,client,context,home)
|
||||
assert len(staged['materials'])==3 and all(m['status']=='ready'for m in staged['materials'])
|
||||
assert sorted(Path(m['path']).read_bytes()for m in staged['materials'])==sorted(raw_materials)
|
||||
root=files_module.task_workspace(context,home)
|
||||
if native_python and samples:
|
||||
doc=next(m['path']for m in staged['materials']if m['name'].endswith('.docx'))
|
||||
output=root/'outputs/converted.pdf'
|
||||
request={'action':'export_pdf','arguments':{'source':doc,'output':str(output)}}
|
||||
completed=await asyncio.to_thread(subprocess.run,[native_python,'-X','utf8',str(source/'capability-slim/runtime/office_workflows.py')],
|
||||
input=json.dumps(request).encode(),capture_output=True,timeout=120)
|
||||
assert completed.returncode==0,completed.stderr
|
||||
assert json.loads(completed.stdout)['status']=='completed',completed.stdout
|
||||
assert output.read_bytes().startswith(b'%PDF')
|
||||
else:
|
||||
output=root/'outputs/result.txt';output.write_text('synthetic final output')
|
||||
office.register_artifacts(root,[output])
|
||||
first=await asyncio.to_thread(files_module.deliver_artifacts,client,context,home)
|
||||
assert first['unknown']==1
|
||||
second=await asyncio.to_thread(files_module.deliver_artifacts,client,context,home)
|
||||
assert second['received']==1 and second['unknown']==0
|
||||
assert requests.count('/work/v2/files/upload')==1 # Read state instead of resending bytes.
|
||||
await api.flush();await api.flush();await api.flush()
|
||||
sent=[x for x in platform_calls if x[0]=='reply_file']
|
||||
assert len(sent)==1 and sent[0][1]=='om-original'
|
||||
assert remote_files[sent[0][2]]==output.read_bytes()
|
||||
assert len([x for x in platform_calls if x[0]=='upload'])==1
|
||||
finally:transport.close()
|
||||
@@ -0,0 +1,49 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from test_credential_api import env
|
||||
from test_device_lifecycle import routed,device,instruction,reply,send_event,event,wait_for_reply
|
||||
from test_work_files import fake_platform
|
||||
|
||||
pytestmark=pytest.mark.parametrize('env',[True],indirect=True)
|
||||
|
||||
|
||||
def file_event(env,message_id='om-file',chat='chat-a',owner='ou_a'):
|
||||
value=event(env,message_id,'',chat=chat,owner=owner)
|
||||
value['event']['message'].update(message_type='file',content=json.dumps({'file_name':'source.txt','file_key':'file-source'}))
|
||||
return value
|
||||
|
||||
|
||||
async def test_file_then_instruction_uses_encrypted_routing_and_capability_gate(routed,monkeypatch):
|
||||
env=routed;_,calls=fake_platform(env,monkeypatch)
|
||||
connection=await device(env,work_files=True)
|
||||
await send_event(env,file_event(env));await wait_for_reply(env['platform'],1)
|
||||
await send_event(env,event(env,'om-instruction','请处理刚才的材料'))
|
||||
payload=await instruction(env,connection)
|
||||
assert payload['file_transfer']=='work-files-v1'
|
||||
task_id='msg_'+hashlib.sha256(b'om-instruction').hexdigest()
|
||||
assert payload['task_context']['task_id']==task_id
|
||||
rows=env['store'].db.execute('SELECT task_id,status FROM work_files_v1').fetchall()
|
||||
assert len(rows)==1 and rows[0]['task_id']==task_id and rows[0]['status']=='assigned'
|
||||
await reply(env,connection,payload,'synthetic result')
|
||||
await wait_for_reply(env['platform'],2)
|
||||
assert len([x for x in calls if x[0]=='download'])==1
|
||||
|
||||
|
||||
async def test_old_client_does_not_receive_a_task_that_silently_omits_materials(routed,monkeypatch):
|
||||
env=routed;fake_platform(env,monkeypatch);connection=await device(env)
|
||||
await send_event(env,file_event(env));await wait_for_reply(env['platform'],1)
|
||||
await send_event(env,event(env,'om-instruction','请处理材料'));await wait_for_reply(env['platform'],2)
|
||||
assert '升级' in env['platform'].replies[-1][1]
|
||||
assert env['store'].db.execute('SELECT status FROM work_files_v1').fetchone()['status']=='staged'
|
||||
assert env['store'].db.execute('SELECT COUNT(*) AS n FROM work_tasks_v2').fetchone()['n']==0
|
||||
|
||||
|
||||
async def test_invalid_upload_metadata_is_rejected_before_binary_body_is_read(env):
|
||||
consumed=[]
|
||||
async def content():
|
||||
consumed.append(True);yield b'not read'
|
||||
response=await env['client'].post('/work/v2/files/upload',content=content())
|
||||
assert response.status_code==422 and not consumed
|
||||
Reference in New Issue
Block a user