Files
ecs-relay-service/deployable/tests/test_approval_integration.py
T

351 lines
19 KiB
Python

"""Two-repo offline approval integration: real proofs/SQL, synthetic platform only."""
import asyncio
import importlib
import importlib.util
import json
import os
from pathlib import Path
import secrets
import sys
import threading
import time
import httpx
import pytest
import pytest_asyncio
from test_credential_api import env,wire,authorize,Platform
from relay.connector import create_app
from relay.crypto import pem_private
from relay.approval_store import ApprovalStore
from relay.store import Store
from relay.binding_state import BindingState
from relay.approval_worker import ApprovalWorker
PC_SERVICE=Path(os.environ.get('HAINUO_APPROVAL_PC_SOURCE',str(
Path(__file__).resolve().parents[3]/'hainuo-build-phase-A-local-20261116')))/'integrations/work-relay/service/relay'
if not (PC_SERVICE/'approval_transport.py').is_file():
pytest.skip('paired PC source missing; set HAINUO_APPROVAL_PC_SOURCE to run joint acceptance',allow_module_level=True)
# An explicit checkout selection keeps imports independent of cwd and installed
# runtime. Missing paired sources are reported as skipped, never a joint pass.
spec=importlib.util.spec_from_file_location('approval_pc',PC_SERVICE/'__init__.py',
submodule_search_locations=[str(PC_SERVICE)])
pc=importlib.util.module_from_spec(spec);sys.modules['approval_pc']=pc;spec.loader.exec_module(pc)
pc_broker=importlib.import_module('approval_pc.broker_client')
pc_gate=importlib.import_module('approval_pc.approval_transport')
pc_config=importlib.import_module('approval_pc.release_config')
pc_context=importlib.import_module('approval_pc.task_context')
pc_errors=importlib.import_module('approval_pc.work_errors')
class CardPlatform(Platform):
def __init__(self):
super().__init__();self.cards=[];self.updates=[];self.lose_card_receipt=False
async def reply_task_card(self,source,card,delivery_id,*,token):
self.cards.append((source,card,delivery_id))
if self.lose_card_receipt:raise ConnectionError('synthetic lost card receipt')
return 'om_approval_'+str(len(self.cards))
async def update_task_card(self,message_id,card,*,token):
self.updates.append((message_id,card))
@pytest_asyncio.fixture
async def approval_env(env):
settings=env['settings'].model_copy(update=dict(work_remote_approvals_enabled=True,
feishu_tenant_key='tenant_synthetic',feishu_verification_token='verification-synthetic'))
platform=CardPlatform();app=create_app(settings,env['redis'],platform)
async with app.router.lifespan_context(app):
async with httpx.AsyncClient(transport=httpx.ASGITransport(app),base_url=settings.public_url)as client:
current={**env,'settings':settings,'platform':platform,'app':app,'client':client,
'store':app.state.store,'api':app.state.oauth}
await authorize(current)
yield current
@pytest_asyncio.fixture
async def pair(approval_env,tmp_path,monkeypatch):
e=approval_env;directory=tmp_path/'pc';directory.mkdir()
host='a'*32;device=e['store'].device(host)
(directory/'server.pem').write_bytes(e['settings'].ca_cert.read_bytes())
(directory/'device.key').write_bytes(pem_private(e['keys'][host]))
(directory/'device.json').write_text(json.dumps({'host_id':host,'owner':'ou_a',
'binding_generation':device['binding_generation']}))
release=pc_config.validate_config(dict(public_url=e['settings'].public_url,credential_mode='broker',
app_id='cli_test',environment='test',permission_profile='product-v1',
work_api_base='https://work.example.test',remote_approvals=True))
monkeypatch.setattr(pc_config,'load_config',lambda:release)
monkeypatch.setattr(pc_broker,'load_config',lambda:release)
loop=asyncio.get_running_loop();faults=set();calls=[];trace=[];stop=threading.Event()
def forward(request):
async def dispatch():
response=await e['client'].request(request.method,request.url.path,
headers=dict(request.headers),content=request.content)
return httpx.Response(response.status_code,headers=response.headers,content=response.content,request=request)
response=asyncio.run_coroutine_threadsafe(dispatch(),loop).result(timeout=10)
trace.append((request.url.path,response.status_code))
if request.url.path.endswith('/consume')and 'consume_receipt_lost'in faults:
raise httpx.ReadTimeout('synthetic lost consume response')
if request.url.path.endswith('/status')and '/approvals/'in request.url.path and 'poll_lost'in faults:
raise httpx.ConnectError('synthetic offline')
return response
transport=httpx.MockTransport(forward)
actual_transport=pc_gate.ApprovalTransport
monkeypatch.setattr(pc_gate,'ApprovalTransport',lambda d,r:actual_transport(d,r,http_transport=transport))
def platform(request):
assert request.url.host=='work.example.test'
assert request.headers['authorization'].startswith('Bearer fake-access')
calls.append(request)
if 'business_receipt_lost'in faults:raise httpx.ReadTimeout('synthetic lost business response')
return httpx.Response(200,json={'code':0,'data':{'event':{'event_id':'synthetic-event'}}})
client=pc_broker.BrokerWorkClient(directory,transport=transport,api_transport=httpx.MockTransport(platform))
task=e['store'].lifecycle.create_task(('msg_'+'a'*64),'ou_a',host,device['binding_generation'],
'synthetic-session',e['app'].state.broker.worker_id,time.time()+600,'oc_synthetic','om_source')
context={k:task[k]for k in ('environment','app_id','owner','host_id','binding_generation',
'conversation_id','chat_id','source_message_id')}
context.update(task_id=task['turn_id'],channel='work-dm')
marker=pc_context.CURRENT_TASK.set(context);control=pc_context.CURRENT_TASK_STOP.set(stop.is_set)
jobs=[]
def launch(operation='operation-synthetic'):
nonlocal loop
loop=asyncio.get_running_loop()
job=asyncio.create_task(asyncio.to_thread(client.api,'POST',
'/open-apis/calendar/v4/calendars/cal_synthetic/events',{},
{'summary':'合成测试:不创建真实日程'},operation))
jobs.append(job);return job
async def card(bound=True):
deadline=time.monotonic()+10
while time.monotonic()<deadline:
for job in jobs:
if job.done():raise AssertionError(('task settled before card',job.result()))
rows=e['store'].db.execute('SELECT * FROM work_approvals_v3').fetchall()
if rows and e['platform'].cards and (not bound or rows[0]['card_message_id']):return dict(rows[0])
await asyncio.sleep(.01)
raise AssertionError(('synthetic approval card was not delivered/bound',trace,
[(r['state'],bool(r['card_message_id']))for r in rows],len(e['platform'].cards)))
async def click(choice='allow',actor='ou_a',event_id=None,mutate=None):
row=await card();value=e['platform'].cards[0][1]['elements'][-1]['actions'][0]['value']
body={'schema':'2.0','header':{'event_type':'card.action.trigger','app_id':'cli_test',
'tenant_key':'tenant_synthetic','token':'verification-synthetic',
'create_time':str(int(time.time()*1000000)),'event_id':event_id or secrets.token_hex(16)},
'event':{'host':'im_message','operator':{'open_id':actor,'tenant_key':'tenant_synthetic'},
'context':{'open_message_id':row['card_message_id'],'open_chat_id':'oc_synthetic'},
'action':{'tag':'button','value':{**value,'choice':choice}}}}
if mutate:mutate(body)
response=await e['client'].post('/internal/approval-callback',json=body,
headers={'X-Internal-Secret':e['settings'].internal_secret})
return response,body
try:yield dict(env=e,client=client,directory=directory,context=context,stop=stop,faults=faults,
calls=calls,launch=launch,card=card,click=click,transport=transport,platform=platform)
finally:
stop.set()
if jobs:await asyncio.wait_for(asyncio.gather(*jobs,return_exceptions=True),timeout=15)
pc_context.CURRENT_TASK_STOP.reset(control);pc_context.CURRENT_TASK.reset(marker)
client.close()
async def test_allow_callback_replay_and_local_repeat_execute_once(pair):
p=pair;job=p['launch']();row=await p['card']()
assert not p['calls'] and pc_gate.approval_pending(p['context'])
response,body=await p['click']();assert response.status_code==200
result=await asyncio.wait_for(job,10)
assert result['status']=='completed'and len(p['calls'])==1
replay=await p['env']['client'].post('/internal/approval-callback',json=body,
headers={'X-Internal-Secret':p['env']['settings'].internal_secret})
assert replay.status_code==200
duplicate=await p['launch']()
assert duplicate['status']=='duplicate'and len(p['calls'])==1
assert not pc_gate.approval_pending(p['context'])
async def test_deny_never_sends_or_reprompts_same_operation(pair):
p=pair;job=p['launch']();await p['click']('deny')
with pytest.raises(pc_errors.WorkError,match='拒绝'):await asyncio.wait_for(job,10)
assert not p['calls']
assert (await p['launch']())['previous_status']=='not_executed'
assert len(p['env']['platform'].cards)==1
@pytest.mark.parametrize('fault',['consume_receipt_lost','poll_lost','business_receipt_lost'])
async def test_ambiguous_response_never_replays_after_reopen(pair,fault):
p=pair;p['faults'].add(fault);job=p['launch']();await p['click']()
if fault=='business_receipt_lost':
assert (await asyncio.wait_for(job,10))['status']=='unknown'
else:
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
expected=1 if fault=='business_receipt_lost'else 0
assert len(p['calls'])==expected
reopened=pc_broker.BrokerWorkClient(p['directory'],transport=p['transport'],
api_transport=httpx.MockTransport(p['platform']))
try:
result=await asyncio.to_thread(reopened.api,'POST',
'/open-apis/calendar/v4/calendars/cal_synthetic/events',{},
{'summary':'合成测试:不创建真实日程'},'operation-synthetic')
assert result['status']=='duplicate'and len(p['calls'])==expected
finally:reopened.close()
if fault=='consume_receipt_lost':
e=p['env'];connection=Store(e['settings'].data_dir/'metadata.sqlite')
try:
restored=ApprovalStore(BindingState(connection,'test','cli_test'),e['api'].broker.store.keyring)
row=connection.db.execute('SELECT state FROM work_approvals_v3').fetchone()
assert row['state']=='consumed'
finally:connection.close()
@pytest.mark.parametrize('failure',['stop','rebind','terminal','expired'])
async def test_stop_binding_terminal_and_expiry_do_not_execute(pair,failure):
p=pair;job=p['launch']();row=await p['card']();e=p['env']
if failure=='stop':p['stop'].set()
elif failure=='rebind':
value=json.loads((p['directory']/'device.json').read_text());value['binding_generation']+=1
(p['directory']/'device.json').write_text(json.dumps(value))
elif failure=='terminal':
with e['store'].db:e['store'].db.execute('INSERT INTO work_execution_state_v3 VALUES(?,?,?,?)',
(('msg_'+'a'*64),'failed',1,time.time()))
else:
with e['store'].db:e['store'].db.execute('UPDATE work_approvals_v3 SET expires_at=0')
if failure in {'terminal','expired'}:await p['click']()
try:
result=await asyncio.wait_for(job,10)
assert result['status']!='completed'
except pc_errors.WorkError:pass
assert not p['calls']
async def test_other_operator_cannot_approve_but_owner_can(pair):
p=pair;job=p['launch']();response,_=await p['click'](actor='ou_b')
assert response.json()['toast']['type']=='warning'and not p['calls']
assert p['env']['store'].db.execute('SELECT state FROM work_approvals_v3').fetchone()['state']=='pending'
await p['click']();assert (await asyncio.wait_for(job,10))['status']=='completed'
assert len(p['calls'])==1
async def test_card_send_receipt_loss_never_sends_second_card(pair):
p=pair;p['env']['platform'].lose_card_receipt=True;job=p['launch']();await p['card'](bound=False)
deadline=time.monotonic()+5
while time.monotonic()<deadline:
row=p['env']['store'].db.execute('SELECT state FROM work_approval_card_outbox_v1').fetchone()
if row and row['state']=='unknown':break
await asyncio.sleep(.01)
assert row['state']=='unknown'
await p['env']['app'].state.approvals.maintain()
await p['env']['app'].state.approvals.maintain()
assert len(p['env']['platform'].cards)==1
p['stop'].set()
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
assert not p['calls']
async def test_unsigned_rpc_proof_replay_and_other_task_are_rejected(pair):
p=pair;job=p['launch']();row=await p['card']();e=p['env'];path='/work/v2/approvals/status'
fields=dict(task_id=('msg_'+'a'*64),approval_id=row['approval_id'],operation_digest=row['operation_digest'])
unsigned=await e['client'].post(path,json=fields);assert unsigned.status_code==422
signed=wire(e,path,**fields)
assert (await e['client'].post(path,json=signed)).status_code==200
assert (await e['client'].post(path,json=signed)).status_code==409
other=await e['client'].post(path,json=wire(e,path,**{**fields,'task_id':'other-task'}))
assert other.status_code==409 and not p['calls']
p['stop'].set()
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
@pytest.mark.parametrize('field',['app','tenant','token','timestamp','chat','card','nonce','value_identity'])
async def test_forged_callback_cannot_approve(pair,field):
p=pair;job=p['launch']();await p['card']()
def mutate(body):
if field=='app':body['header']['app_id']='another-app'
elif field=='tenant':body['header']['tenant_key']='another-tenant'
elif field=='token':body['header']['token']='wrong-verification'
elif field=='timestamp':body['header']['create_time']='1'
elif field=='chat':body['event']['context']['open_chat_id']='oc_other'
elif field=='card':body['event']['context']['open_message_id']='om_other'
elif field=='nonce':body['event']['action']['value']['nonce']='A'*43
else:body['event']['action']['value']['actor_open_id']='ou_a'
response,_=await p['click'](mutate=mutate)
assert response.json()['toast']['type']=='warning'
assert p['env']['store'].db.execute('SELECT state FROM work_approvals_v3').fetchone()['state']=='pending'
assert not p['calls']
p['stop'].set()
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
async def test_conflicting_clicks_and_concurrent_consume_grant_at_most_once(pair):
p=pair;job=p['launch']();row=await p['card']()
# Prevent PC consumption while testing two consumers of the same grant.
response,_=await p['click']('allow')
conflict,_=await p['click']('deny')
assert response.json()['toast']['type']=='info'
assert conflict.json()['toast']['type']=='warning'
e=p['env'];path='/work/v2/approvals/consume'
fields=dict(task_id=('msg_'+'a'*64),approval_id=row['approval_id'],operation_digest=row['operation_digest'])
responses=await asyncio.gather(*(e['client'].post(path,json=wire(e,path,**fields))for _ in range(2)))
assert sum(r.json().get('execute')is True for r in responses)<=1
try:result=await asyncio.wait_for(job,10)
except pc_errors.WorkError:result=None
assert len(p['calls'])<=1
assert sum(r.json().get('execute')is True for r in responses)+len(p['calls'])==1
async def test_expiry_removes_buttons_and_update_receipt_loss_is_retryable(pair,monkeypatch):
p=pair;job=p['launch']();row=await p['card']();e=p['env']
with e['store'].db:e['store'].db.execute('UPDATE work_approvals_v3 SET expires_at=0')
await e['app'].state.approvals.worker.call('close_unavailable')
sender=e['app'].state.approvals.sender;original=e['platform'].update_task_card
attempts=[]
async def lost_once(*args,**kwargs):
await original(*args,**kwargs);attempts.append(1)
if len(attempts)==1:raise ConnectionError('synthetic update receipt loss')
monkeypatch.setattr(e['platform'],'update_task_card',lost_once)
with pytest.raises(ConnectionError):await sender.update_closed(row['approval_id'])
assert await sender.update_closed(row['approval_id'])=={'state':'updated'}
assert await sender.update_closed(row['approval_id'])=={'state':'no_closed_card'}
assert len(e['platform'].cards)==1 and len(attempts)==2
assert all('"button"'not in json.dumps(card)for _,card in e['platform'].updates)
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
assert not p['calls']
async def test_reviewed_read_bypasses_confirmation(pair):
p=pair
result=await asyncio.to_thread(p['client'].api,'GET','/open-apis/calendar/v4/calendars')
assert result['status']=='completed'and len(p['calls'])==1
assert not p['env']['platform'].cards
assert not p['env']['store'].db.execute('SELECT * FROM work_approvals_v3').fetchall()
async def test_default_off_does_not_register_approval_or_callback_routes(env):
status=await env['client'].post('/work/v2/status',json=wire(env,'/work/v2/status'))
assert status.status_code==200 and status.json()['approval_protocol']is None
for path in ('/work/v2/approvals/create','/internal/approval-callback'):
assert (await env['client'].post(path,json={})).status_code==404
assert not env['store'].db.execute("SELECT name FROM sqlite_master WHERE name='work_approvals_v3'").fetchall()
async def test_approval_worker_recreation_preserves_original_pending_card(pair):
p=pair;job=p['launch']();row=await p['card']();e=p['env']
api=e['app'].state.approvals;original=api.worker
restored=ApprovalWorker(e['settings'],e['api'].broker.store.keyring)
await restored.start()
api.worker=restored;api.sender.approvals=restored
try:
await api.maintain()
assert len(e['platform'].cards)==1
response,_=await p['click']()
assert response.json()['toast']['type']=='info'
assert (await asyncio.wait_for(job,10))['status']=='completed'
assert len(p['calls'])==1
saved=e['store'].db.execute('SELECT approval_id,state,card_message_id FROM work_approvals_v3').fetchone()
assert (saved['approval_id'],saved['state'],saved['card_message_id'])==(
row['approval_id'],'consumed',row['card_message_id'])
finally:
# Drain only this adapter's tracked card work before closing its
# connection. App lifespan still owns the original worker shutdown.
deadline=time.monotonic()+5
while time.monotonic()<deadline:
if not api.scheduled:break
await asyncio.sleep(.01)
api.worker=original;api.sender.approvals=original
await restored.close()