351 lines
19 KiB
Python
351 lines
19 KiB
Python
"""Two-repo offline approval integration: real proofs/SQL, synthetic platform only."""
|
|
import asyncio
|
|
import importlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import secrets
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
import httpx
|
|
import pytest
|
|
import pytest_asyncio
|
|
|
|
from test_credential_api import env,wire,authorize,Platform
|
|
from relay.connector import create_app
|
|
from relay.crypto import pem_private
|
|
from relay.approval_store import ApprovalStore
|
|
from relay.store import Store
|
|
from relay.binding_state import BindingState
|
|
from relay.approval_worker import ApprovalWorker
|
|
|
|
|
|
PC_SERVICE=Path(os.environ.get('HAINUO_APPROVAL_PC_SOURCE',str(
|
|
Path(__file__).resolve().parents[3]/'hainuo-build-phase-A-local-20261116')))/'integrations/work-relay/service/relay'
|
|
if not (PC_SERVICE/'approval_transport.py').is_file():
|
|
pytest.skip('paired PC source missing; set HAINUO_APPROVAL_PC_SOURCE to run joint acceptance',allow_module_level=True)
|
|
# An explicit checkout selection keeps imports independent of cwd and installed
|
|
# runtime. Missing paired sources are reported as skipped, never a joint pass.
|
|
spec=importlib.util.spec_from_file_location('approval_pc',PC_SERVICE/'__init__.py',
|
|
submodule_search_locations=[str(PC_SERVICE)])
|
|
pc=importlib.util.module_from_spec(spec);sys.modules['approval_pc']=pc;spec.loader.exec_module(pc)
|
|
pc_broker=importlib.import_module('approval_pc.broker_client')
|
|
pc_gate=importlib.import_module('approval_pc.approval_transport')
|
|
pc_config=importlib.import_module('approval_pc.release_config')
|
|
pc_context=importlib.import_module('approval_pc.task_context')
|
|
pc_errors=importlib.import_module('approval_pc.work_errors')
|
|
|
|
|
|
class CardPlatform(Platform):
|
|
def __init__(self):
|
|
super().__init__();self.cards=[];self.updates=[];self.lose_card_receipt=False
|
|
|
|
async def reply_task_card(self,source,card,delivery_id,*,token):
|
|
self.cards.append((source,card,delivery_id))
|
|
if self.lose_card_receipt:raise ConnectionError('synthetic lost card receipt')
|
|
return 'om_approval_'+str(len(self.cards))
|
|
|
|
async def update_task_card(self,message_id,card,*,token):
|
|
self.updates.append((message_id,card))
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def approval_env(env):
|
|
settings=env['settings'].model_copy(update=dict(work_remote_approvals_enabled=True,
|
|
feishu_tenant_key='tenant_synthetic',feishu_verification_token='verification-synthetic'))
|
|
platform=CardPlatform();app=create_app(settings,env['redis'],platform)
|
|
async with app.router.lifespan_context(app):
|
|
async with httpx.AsyncClient(transport=httpx.ASGITransport(app),base_url=settings.public_url)as client:
|
|
current={**env,'settings':settings,'platform':platform,'app':app,'client':client,
|
|
'store':app.state.store,'api':app.state.oauth}
|
|
await authorize(current)
|
|
yield current
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def pair(approval_env,tmp_path,monkeypatch):
|
|
e=approval_env;directory=tmp_path/'pc';directory.mkdir()
|
|
host='a'*32;device=e['store'].device(host)
|
|
(directory/'server.pem').write_bytes(e['settings'].ca_cert.read_bytes())
|
|
(directory/'device.key').write_bytes(pem_private(e['keys'][host]))
|
|
(directory/'device.json').write_text(json.dumps({'host_id':host,'owner':'ou_a',
|
|
'binding_generation':device['binding_generation']}))
|
|
release=pc_config.validate_config(dict(public_url=e['settings'].public_url,credential_mode='broker',
|
|
app_id='cli_test',environment='test',permission_profile='product-v1',
|
|
work_api_base='https://work.example.test',remote_approvals=True))
|
|
monkeypatch.setattr(pc_config,'load_config',lambda:release)
|
|
monkeypatch.setattr(pc_broker,'load_config',lambda:release)
|
|
loop=asyncio.get_running_loop();faults=set();calls=[];trace=[];stop=threading.Event()
|
|
def forward(request):
|
|
async def dispatch():
|
|
response=await e['client'].request(request.method,request.url.path,
|
|
headers=dict(request.headers),content=request.content)
|
|
return httpx.Response(response.status_code,headers=response.headers,content=response.content,request=request)
|
|
response=asyncio.run_coroutine_threadsafe(dispatch(),loop).result(timeout=10)
|
|
trace.append((request.url.path,response.status_code))
|
|
if request.url.path.endswith('/consume')and 'consume_receipt_lost'in faults:
|
|
raise httpx.ReadTimeout('synthetic lost consume response')
|
|
if request.url.path.endswith('/status')and '/approvals/'in request.url.path and 'poll_lost'in faults:
|
|
raise httpx.ConnectError('synthetic offline')
|
|
return response
|
|
transport=httpx.MockTransport(forward)
|
|
actual_transport=pc_gate.ApprovalTransport
|
|
monkeypatch.setattr(pc_gate,'ApprovalTransport',lambda d,r:actual_transport(d,r,http_transport=transport))
|
|
def platform(request):
|
|
assert request.url.host=='work.example.test'
|
|
assert request.headers['authorization'].startswith('Bearer fake-access')
|
|
calls.append(request)
|
|
if 'business_receipt_lost'in faults:raise httpx.ReadTimeout('synthetic lost business response')
|
|
return httpx.Response(200,json={'code':0,'data':{'event':{'event_id':'synthetic-event'}}})
|
|
client=pc_broker.BrokerWorkClient(directory,transport=transport,api_transport=httpx.MockTransport(platform))
|
|
task=e['store'].lifecycle.create_task('task-approval','ou_a',host,device['binding_generation'],
|
|
'synthetic-session',e['app'].state.broker.worker_id,time.time()+600,'oc_synthetic','om_source')
|
|
context={k:task[k]for k in ('environment','app_id','owner','host_id','binding_generation',
|
|
'conversation_id','chat_id','source_message_id')}
|
|
context.update(task_id=task['turn_id'],channel='work-dm')
|
|
marker=pc_context.CURRENT_TASK.set(context);control=pc_context.CURRENT_TASK_STOP.set(stop.is_set)
|
|
jobs=[]
|
|
def launch(operation='operation-synthetic'):
|
|
nonlocal loop
|
|
loop=asyncio.get_running_loop()
|
|
job=asyncio.create_task(asyncio.to_thread(client.api,'POST',
|
|
'/open-apis/calendar/v4/calendars/cal_synthetic/events',{},
|
|
{'summary':'合成测试:不创建真实日程'},operation))
|
|
jobs.append(job);return job
|
|
async def card(bound=True):
|
|
deadline=time.monotonic()+10
|
|
while time.monotonic()<deadline:
|
|
for job in jobs:
|
|
if job.done():raise AssertionError(('task settled before card',job.result()))
|
|
rows=e['store'].db.execute('SELECT * FROM work_approvals_v3').fetchall()
|
|
if rows and e['platform'].cards and (not bound or rows[0]['card_message_id']):return dict(rows[0])
|
|
await asyncio.sleep(.01)
|
|
raise AssertionError(('synthetic approval card was not delivered/bound',trace,
|
|
[(r['state'],bool(r['card_message_id']))for r in rows],len(e['platform'].cards)))
|
|
async def click(choice='allow',actor='ou_a',event_id=None,mutate=None):
|
|
row=await card();value=e['platform'].cards[0][1]['elements'][-1]['actions'][0]['value']
|
|
body={'schema':'2.0','header':{'event_type':'card.action.trigger','app_id':'cli_test',
|
|
'tenant_key':'tenant_synthetic','token':'verification-synthetic',
|
|
'create_time':str(int(time.time()*1000000)),'event_id':event_id or secrets.token_hex(16)},
|
|
'event':{'host':'im_message','operator':{'open_id':actor,'tenant_key':'tenant_synthetic'},
|
|
'context':{'open_message_id':row['card_message_id'],'open_chat_id':'oc_synthetic'},
|
|
'action':{'tag':'button','value':{**value,'choice':choice}}}}
|
|
if mutate:mutate(body)
|
|
response=await e['client'].post('/internal/approval-callback',json=body,
|
|
headers={'X-Internal-Secret':e['settings'].internal_secret})
|
|
return response,body
|
|
try:yield dict(env=e,client=client,directory=directory,context=context,stop=stop,faults=faults,
|
|
calls=calls,launch=launch,card=card,click=click,transport=transport,platform=platform)
|
|
finally:
|
|
stop.set()
|
|
if jobs:await asyncio.wait_for(asyncio.gather(*jobs,return_exceptions=True),timeout=15)
|
|
pc_context.CURRENT_TASK_STOP.reset(control);pc_context.CURRENT_TASK.reset(marker)
|
|
client.close()
|
|
|
|
|
|
async def test_allow_callback_replay_and_local_repeat_execute_once(pair):
|
|
p=pair;job=p['launch']();row=await p['card']()
|
|
assert not p['calls'] and pc_gate.approval_pending(p['context'])
|
|
response,body=await p['click']();assert response.status_code==200
|
|
result=await asyncio.wait_for(job,10)
|
|
assert result['status']=='completed'and len(p['calls'])==1
|
|
replay=await p['env']['client'].post('/internal/approval-callback',json=body,
|
|
headers={'X-Internal-Secret':p['env']['settings'].internal_secret})
|
|
assert replay.status_code==200
|
|
duplicate=await p['launch']()
|
|
assert duplicate['status']=='duplicate'and len(p['calls'])==1
|
|
assert not pc_gate.approval_pending(p['context'])
|
|
|
|
|
|
async def test_deny_never_sends_or_reprompts_same_operation(pair):
|
|
p=pair;job=p['launch']();await p['click']('deny')
|
|
with pytest.raises(pc_errors.WorkError,match='拒绝'):await asyncio.wait_for(job,10)
|
|
assert not p['calls']
|
|
assert (await p['launch']())['previous_status']=='not_executed'
|
|
assert len(p['env']['platform'].cards)==1
|
|
|
|
|
|
@pytest.mark.parametrize('fault',['consume_receipt_lost','poll_lost','business_receipt_lost'])
|
|
async def test_ambiguous_response_never_replays_after_reopen(pair,fault):
|
|
p=pair;p['faults'].add(fault);job=p['launch']();await p['click']()
|
|
if fault=='business_receipt_lost':
|
|
assert (await asyncio.wait_for(job,10))['status']=='unknown'
|
|
else:
|
|
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
|
|
expected=1 if fault=='business_receipt_lost'else 0
|
|
assert len(p['calls'])==expected
|
|
reopened=pc_broker.BrokerWorkClient(p['directory'],transport=p['transport'],
|
|
api_transport=httpx.MockTransport(p['platform']))
|
|
try:
|
|
result=await asyncio.to_thread(reopened.api,'POST',
|
|
'/open-apis/calendar/v4/calendars/cal_synthetic/events',{},
|
|
{'summary':'合成测试:不创建真实日程'},'operation-synthetic')
|
|
assert result['status']=='duplicate'and len(p['calls'])==expected
|
|
finally:reopened.close()
|
|
if fault=='consume_receipt_lost':
|
|
e=p['env'];connection=Store(e['settings'].data_dir/'metadata.sqlite')
|
|
try:
|
|
restored=ApprovalStore(BindingState(connection,'test','cli_test'),e['api'].broker.store.keyring)
|
|
row=connection.db.execute('SELECT state FROM work_approvals_v3').fetchone()
|
|
assert row['state']=='consumed'
|
|
finally:connection.close()
|
|
|
|
|
|
@pytest.mark.parametrize('failure',['stop','rebind','terminal','expired'])
|
|
async def test_stop_binding_terminal_and_expiry_do_not_execute(pair,failure):
|
|
p=pair;job=p['launch']();row=await p['card']();e=p['env']
|
|
if failure=='stop':p['stop'].set()
|
|
elif failure=='rebind':
|
|
value=json.loads((p['directory']/'device.json').read_text());value['binding_generation']+=1
|
|
(p['directory']/'device.json').write_text(json.dumps(value))
|
|
elif failure=='terminal':
|
|
with e['store'].db:e['store'].db.execute('INSERT INTO work_execution_state_v3 VALUES(?,?,?,?)',
|
|
('task-approval','failed',1,time.time()))
|
|
else:
|
|
with e['store'].db:e['store'].db.execute('UPDATE work_approvals_v3 SET expires_at=0')
|
|
if failure in {'terminal','expired'}:await p['click']()
|
|
try:
|
|
result=await asyncio.wait_for(job,10)
|
|
assert result['status']!='completed'
|
|
except pc_errors.WorkError:pass
|
|
assert not p['calls']
|
|
|
|
|
|
async def test_other_operator_cannot_approve_but_owner_can(pair):
|
|
p=pair;job=p['launch']();response,_=await p['click'](actor='ou_b')
|
|
assert response.json()['toast']['type']=='warning'and not p['calls']
|
|
assert p['env']['store'].db.execute('SELECT state FROM work_approvals_v3').fetchone()['state']=='pending'
|
|
await p['click']();assert (await asyncio.wait_for(job,10))['status']=='completed'
|
|
assert len(p['calls'])==1
|
|
|
|
|
|
async def test_card_send_receipt_loss_never_sends_second_card(pair):
|
|
p=pair;p['env']['platform'].lose_card_receipt=True;job=p['launch']();await p['card'](bound=False)
|
|
deadline=time.monotonic()+5
|
|
while time.monotonic()<deadline:
|
|
row=p['env']['store'].db.execute('SELECT state FROM work_approval_card_outbox_v1').fetchone()
|
|
if row and row['state']=='unknown':break
|
|
await asyncio.sleep(.01)
|
|
assert row['state']=='unknown'
|
|
await p['env']['app'].state.approvals.maintain()
|
|
await p['env']['app'].state.approvals.maintain()
|
|
assert len(p['env']['platform'].cards)==1
|
|
p['stop'].set()
|
|
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
|
|
assert not p['calls']
|
|
|
|
|
|
async def test_unsigned_rpc_proof_replay_and_other_task_are_rejected(pair):
|
|
p=pair;job=p['launch']();row=await p['card']();e=p['env'];path='/work/v2/approvals/status'
|
|
fields=dict(task_id='task-approval',approval_id=row['approval_id'],operation_digest=row['operation_digest'])
|
|
unsigned=await e['client'].post(path,json=fields);assert unsigned.status_code==422
|
|
signed=wire(e,path,**fields)
|
|
assert (await e['client'].post(path,json=signed)).status_code==200
|
|
assert (await e['client'].post(path,json=signed)).status_code==409
|
|
other=await e['client'].post(path,json=wire(e,path,**{**fields,'task_id':'other-task'}))
|
|
assert other.status_code==409 and not p['calls']
|
|
p['stop'].set()
|
|
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
|
|
|
|
|
|
@pytest.mark.parametrize('field',['app','tenant','token','timestamp','chat','card','nonce','value_identity'])
|
|
async def test_forged_callback_cannot_approve(pair,field):
|
|
p=pair;job=p['launch']();await p['card']()
|
|
def mutate(body):
|
|
if field=='app':body['header']['app_id']='another-app'
|
|
elif field=='tenant':body['header']['tenant_key']='another-tenant'
|
|
elif field=='token':body['header']['token']='wrong-verification'
|
|
elif field=='timestamp':body['header']['create_time']='1'
|
|
elif field=='chat':body['event']['context']['open_chat_id']='oc_other'
|
|
elif field=='card':body['event']['context']['open_message_id']='om_other'
|
|
elif field=='nonce':body['event']['action']['value']['nonce']='A'*43
|
|
else:body['event']['action']['value']['actor_open_id']='ou_a'
|
|
response,_=await p['click'](mutate=mutate)
|
|
assert response.json()['toast']['type']=='warning'
|
|
assert p['env']['store'].db.execute('SELECT state FROM work_approvals_v3').fetchone()['state']=='pending'
|
|
assert not p['calls']
|
|
p['stop'].set()
|
|
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
|
|
|
|
|
|
async def test_conflicting_clicks_and_concurrent_consume_grant_at_most_once(pair):
|
|
p=pair;job=p['launch']();row=await p['card']()
|
|
# Prevent PC consumption while testing two consumers of the same grant.
|
|
response,_=await p['click']('allow')
|
|
conflict,_=await p['click']('deny')
|
|
assert response.json()['toast']['type']=='info'
|
|
assert conflict.json()['toast']['type']=='warning'
|
|
e=p['env'];path='/work/v2/approvals/consume'
|
|
fields=dict(task_id='task-approval',approval_id=row['approval_id'],operation_digest=row['operation_digest'])
|
|
responses=await asyncio.gather(*(e['client'].post(path,json=wire(e,path,**fields))for _ in range(2)))
|
|
assert sum(r.json().get('execute')is True for r in responses)<=1
|
|
try:result=await asyncio.wait_for(job,10)
|
|
except pc_errors.WorkError:result=None
|
|
assert len(p['calls'])<=1
|
|
assert sum(r.json().get('execute')is True for r in responses)+len(p['calls'])==1
|
|
|
|
|
|
async def test_expiry_removes_buttons_and_update_receipt_loss_is_retryable(pair,monkeypatch):
|
|
p=pair;job=p['launch']();row=await p['card']();e=p['env']
|
|
with e['store'].db:e['store'].db.execute('UPDATE work_approvals_v3 SET expires_at=0')
|
|
await e['app'].state.approvals.worker.call('close_unavailable')
|
|
sender=e['app'].state.approvals.sender;original=e['platform'].update_task_card
|
|
attempts=[]
|
|
async def lost_once(*args,**kwargs):
|
|
await original(*args,**kwargs);attempts.append(1)
|
|
if len(attempts)==1:raise ConnectionError('synthetic update receipt loss')
|
|
monkeypatch.setattr(e['platform'],'update_task_card',lost_once)
|
|
with pytest.raises(ConnectionError):await sender.update_closed(row['approval_id'])
|
|
assert await sender.update_closed(row['approval_id'])=={'state':'updated'}
|
|
assert await sender.update_closed(row['approval_id'])=={'state':'no_closed_card'}
|
|
assert len(e['platform'].cards)==1 and len(attempts)==2
|
|
assert all('"button"'not in json.dumps(card)for _,card in e['platform'].updates)
|
|
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
|
|
assert not p['calls']
|
|
|
|
|
|
async def test_reviewed_read_bypasses_confirmation(pair):
|
|
p=pair
|
|
result=await asyncio.to_thread(p['client'].api,'GET','/open-apis/calendar/v4/calendars')
|
|
assert result['status']=='completed'and len(p['calls'])==1
|
|
assert not p['env']['platform'].cards
|
|
assert not p['env']['store'].db.execute('SELECT * FROM work_approvals_v3').fetchall()
|
|
|
|
|
|
async def test_default_off_does_not_register_approval_or_callback_routes(env):
|
|
status=await env['client'].post('/work/v2/status',json=wire(env,'/work/v2/status'))
|
|
assert status.status_code==200 and status.json()['approval_protocol']is None
|
|
for path in ('/work/v2/approvals/create','/internal/approval-callback'):
|
|
assert (await env['client'].post(path,json={})).status_code==404
|
|
assert not env['store'].db.execute("SELECT name FROM sqlite_master WHERE name='work_approvals_v3'").fetchall()
|
|
|
|
|
|
async def test_approval_worker_recreation_preserves_original_pending_card(pair):
|
|
p=pair;job=p['launch']();row=await p['card']();e=p['env']
|
|
api=e['app'].state.approvals;original=api.worker
|
|
restored=ApprovalWorker(e['settings'],e['api'].broker.store.keyring)
|
|
await restored.start()
|
|
api.worker=restored;api.sender.approvals=restored
|
|
try:
|
|
await api.maintain()
|
|
assert len(e['platform'].cards)==1
|
|
response,_=await p['click']()
|
|
assert response.json()['toast']['type']=='info'
|
|
assert (await asyncio.wait_for(job,10))['status']=='completed'
|
|
assert len(p['calls'])==1
|
|
saved=e['store'].db.execute('SELECT approval_id,state,card_message_id FROM work_approvals_v3').fetchone()
|
|
assert (saved['approval_id'],saved['state'],saved['card_message_id'])==(
|
|
row['approval_id'],'consumed',row['card_message_id'])
|
|
finally:
|
|
# Drain only this adapter's tracked card work before closing its
|
|
# connection. App lifespan still owns the original worker shutdown.
|
|
deadline=time.monotonic()+5
|
|
while time.monotonic()<deadline:
|
|
if not api.scheduled:break
|
|
await asyncio.sleep(.01)
|
|
api.worker=original;api.sender.approvals=original
|
|
await restored.close()
|