feat(work): stage authenticated remote approvals with scoped execution fencing

This commit is contained in:
ex_tianzy
2026-11-17 11:43:33 +08:00
parent d2881b82b0
commit c80c6a0d27
19 changed files with 1586 additions and 25 deletions
@@ -0,0 +1,350 @@
"""Two-repo offline approval integration: real proofs/SQL, synthetic platform only."""
import asyncio
import importlib
import importlib.util
import json
import os
from pathlib import Path
import secrets
import sys
import threading
import time
import httpx
import pytest
import pytest_asyncio
from test_credential_api import env,wire,authorize,Platform
from relay.connector import create_app
from relay.crypto import pem_private
from relay.approval_store import ApprovalStore
from relay.store import Store
from relay.binding_state import BindingState
from relay.approval_worker import ApprovalWorker
PC_SERVICE=Path(os.environ.get('HAINUO_APPROVAL_PC_SOURCE',str(
Path(__file__).resolve().parents[3]/'hainuo-build-phase-A-local-20261116')))/'integrations/work-relay/service/relay'
if not (PC_SERVICE/'approval_transport.py').is_file():
pytest.skip('paired PC source missing; set HAINUO_APPROVAL_PC_SOURCE to run joint acceptance',allow_module_level=True)
# An explicit checkout selection keeps imports independent of cwd and installed
# runtime. Missing paired sources are reported as skipped, never a joint pass.
spec=importlib.util.spec_from_file_location('approval_pc',PC_SERVICE/'__init__.py',
submodule_search_locations=[str(PC_SERVICE)])
pc=importlib.util.module_from_spec(spec);sys.modules['approval_pc']=pc;spec.loader.exec_module(pc)
pc_broker=importlib.import_module('approval_pc.broker_client')
pc_gate=importlib.import_module('approval_pc.approval_transport')
pc_config=importlib.import_module('approval_pc.release_config')
pc_context=importlib.import_module('approval_pc.task_context')
pc_errors=importlib.import_module('approval_pc.work_errors')
class CardPlatform(Platform):
def __init__(self):
super().__init__();self.cards=[];self.updates=[];self.lose_card_receipt=False
async def reply_task_card(self,source,card,delivery_id,*,token):
self.cards.append((source,card,delivery_id))
if self.lose_card_receipt:raise ConnectionError('synthetic lost card receipt')
return 'om_approval_'+str(len(self.cards))
async def update_task_card(self,message_id,card,*,token):
self.updates.append((message_id,card))
@pytest_asyncio.fixture
async def approval_env(env):
settings=env['settings'].model_copy(update=dict(work_remote_approvals_enabled=True,
feishu_tenant_key='tenant_synthetic',feishu_verification_token='verification-synthetic'))
platform=CardPlatform();app=create_app(settings,env['redis'],platform)
async with app.router.lifespan_context(app):
async with httpx.AsyncClient(transport=httpx.ASGITransport(app),base_url=settings.public_url)as client:
current={**env,'settings':settings,'platform':platform,'app':app,'client':client,
'store':app.state.store,'api':app.state.oauth}
await authorize(current)
yield current
@pytest_asyncio.fixture
async def pair(approval_env,tmp_path,monkeypatch):
e=approval_env;directory=tmp_path/'pc';directory.mkdir()
host='a'*32;device=e['store'].device(host)
(directory/'server.pem').write_bytes(e['settings'].ca_cert.read_bytes())
(directory/'device.key').write_bytes(pem_private(e['keys'][host]))
(directory/'device.json').write_text(json.dumps({'host_id':host,'owner':'ou_a',
'binding_generation':device['binding_generation']}))
release=pc_config.validate_config(dict(public_url=e['settings'].public_url,credential_mode='broker',
app_id='cli_test',environment='test',permission_profile='product-v1',
work_api_base='https://work.example.test',remote_approvals=True))
monkeypatch.setattr(pc_config,'load_config',lambda:release)
monkeypatch.setattr(pc_broker,'load_config',lambda:release)
loop=asyncio.get_running_loop();faults=set();calls=[];trace=[];stop=threading.Event()
def forward(request):
async def dispatch():
response=await e['client'].request(request.method,request.url.path,
headers=dict(request.headers),content=request.content)
return httpx.Response(response.status_code,headers=response.headers,content=response.content,request=request)
response=asyncio.run_coroutine_threadsafe(dispatch(),loop).result(timeout=10)
trace.append((request.url.path,response.status_code))
if request.url.path.endswith('/consume')and 'consume_receipt_lost'in faults:
raise httpx.ReadTimeout('synthetic lost consume response')
if request.url.path.endswith('/status')and '/approvals/'in request.url.path and 'poll_lost'in faults:
raise httpx.ConnectError('synthetic offline')
return response
transport=httpx.MockTransport(forward)
actual_transport=pc_gate.ApprovalTransport
monkeypatch.setattr(pc_gate,'ApprovalTransport',lambda d,r:actual_transport(d,r,http_transport=transport))
def platform(request):
assert request.url.host=='work.example.test'
assert request.headers['authorization'].startswith('Bearer fake-access')
calls.append(request)
if 'business_receipt_lost'in faults:raise httpx.ReadTimeout('synthetic lost business response')
return httpx.Response(200,json={'code':0,'data':{'event':{'event_id':'synthetic-event'}}})
client=pc_broker.BrokerWorkClient(directory,transport=transport,api_transport=httpx.MockTransport(platform))
task=e['store'].lifecycle.create_task('task-approval','ou_a',host,device['binding_generation'],
'synthetic-session',e['app'].state.broker.worker_id,time.time()+600,'oc_synthetic','om_source')
context={k:task[k]for k in ('environment','app_id','owner','host_id','binding_generation',
'conversation_id','chat_id','source_message_id')}
context.update(task_id=task['turn_id'],channel='work-dm')
marker=pc_context.CURRENT_TASK.set(context);control=pc_context.CURRENT_TASK_STOP.set(stop.is_set)
jobs=[]
def launch(operation='operation-synthetic'):
nonlocal loop
loop=asyncio.get_running_loop()
job=asyncio.create_task(asyncio.to_thread(client.api,'POST',
'/open-apis/calendar/v4/calendars/cal_synthetic/events',{},
{'summary':'合成测试:不创建真实日程'},operation))
jobs.append(job);return job
async def card(bound=True):
deadline=time.monotonic()+10
while time.monotonic()<deadline:
for job in jobs:
if job.done():raise AssertionError(('task settled before card',job.result()))
rows=e['store'].db.execute('SELECT * FROM work_approvals_v3').fetchall()
if rows and e['platform'].cards and (not bound or rows[0]['card_message_id']):return dict(rows[0])
await asyncio.sleep(.01)
raise AssertionError(('synthetic approval card was not delivered/bound',trace,
[(r['state'],bool(r['card_message_id']))for r in rows],len(e['platform'].cards)))
async def click(choice='allow',actor='ou_a',event_id=None,mutate=None):
row=await card();value=e['platform'].cards[0][1]['elements'][-1]['actions'][0]['value']
body={'schema':'2.0','header':{'event_type':'card.action.trigger','app_id':'cli_test',
'tenant_key':'tenant_synthetic','token':'verification-synthetic',
'create_time':str(int(time.time()*1000000)),'event_id':event_id or secrets.token_hex(16)},
'event':{'host':'im_message','operator':{'open_id':actor,'tenant_key':'tenant_synthetic'},
'context':{'open_message_id':row['card_message_id'],'open_chat_id':'oc_synthetic'},
'action':{'tag':'button','value':{**value,'choice':choice}}}}
if mutate:mutate(body)
response=await e['client'].post('/internal/approval-callback',json=body,
headers={'X-Internal-Secret':e['settings'].internal_secret})
return response,body
try:yield dict(env=e,client=client,directory=directory,context=context,stop=stop,faults=faults,
calls=calls,launch=launch,card=card,click=click,transport=transport,platform=platform)
finally:
stop.set()
if jobs:await asyncio.wait_for(asyncio.gather(*jobs,return_exceptions=True),timeout=15)
pc_context.CURRENT_TASK_STOP.reset(control);pc_context.CURRENT_TASK.reset(marker)
client.close()
async def test_allow_callback_replay_and_local_repeat_execute_once(pair):
p=pair;job=p['launch']();row=await p['card']()
assert not p['calls'] and pc_gate.approval_pending(p['context'])
response,body=await p['click']();assert response.status_code==200
result=await asyncio.wait_for(job,10)
assert result['status']=='completed'and len(p['calls'])==1
replay=await p['env']['client'].post('/internal/approval-callback',json=body,
headers={'X-Internal-Secret':p['env']['settings'].internal_secret})
assert replay.status_code==200
duplicate=await p['launch']()
assert duplicate['status']=='duplicate'and len(p['calls'])==1
assert not pc_gate.approval_pending(p['context'])
async def test_deny_never_sends_or_reprompts_same_operation(pair):
p=pair;job=p['launch']();await p['click']('deny')
with pytest.raises(pc_errors.WorkError,match='拒绝'):await asyncio.wait_for(job,10)
assert not p['calls']
assert (await p['launch']())['previous_status']=='not_executed'
assert len(p['env']['platform'].cards)==1
@pytest.mark.parametrize('fault',['consume_receipt_lost','poll_lost','business_receipt_lost'])
async def test_ambiguous_response_never_replays_after_reopen(pair,fault):
p=pair;p['faults'].add(fault);job=p['launch']();await p['click']()
if fault=='business_receipt_lost':
assert (await asyncio.wait_for(job,10))['status']=='unknown'
else:
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
expected=1 if fault=='business_receipt_lost'else 0
assert len(p['calls'])==expected
reopened=pc_broker.BrokerWorkClient(p['directory'],transport=p['transport'],
api_transport=httpx.MockTransport(p['platform']))
try:
result=await asyncio.to_thread(reopened.api,'POST',
'/open-apis/calendar/v4/calendars/cal_synthetic/events',{},
{'summary':'合成测试:不创建真实日程'},'operation-synthetic')
assert result['status']=='duplicate'and len(p['calls'])==expected
finally:reopened.close()
if fault=='consume_receipt_lost':
e=p['env'];connection=Store(e['settings'].data_dir/'metadata.sqlite')
try:
restored=ApprovalStore(BindingState(connection,'test','cli_test'),e['api'].broker.store.keyring)
row=connection.db.execute('SELECT state FROM work_approvals_v3').fetchone()
assert row['state']=='consumed'
finally:connection.close()
@pytest.mark.parametrize('failure',['stop','rebind','terminal','expired'])
async def test_stop_binding_terminal_and_expiry_do_not_execute(pair,failure):
p=pair;job=p['launch']();row=await p['card']();e=p['env']
if failure=='stop':p['stop'].set()
elif failure=='rebind':
value=json.loads((p['directory']/'device.json').read_text());value['binding_generation']+=1
(p['directory']/'device.json').write_text(json.dumps(value))
elif failure=='terminal':
with e['store'].db:e['store'].db.execute('INSERT INTO work_execution_state_v3 VALUES(?,?,?,?)',
('task-approval','failed',1,time.time()))
else:
with e['store'].db:e['store'].db.execute('UPDATE work_approvals_v3 SET expires_at=0')
if failure in {'terminal','expired'}:await p['click']()
try:
result=await asyncio.wait_for(job,10)
assert result['status']!='completed'
except pc_errors.WorkError:pass
assert not p['calls']
async def test_other_operator_cannot_approve_but_owner_can(pair):
p=pair;job=p['launch']();response,_=await p['click'](actor='ou_b')
assert response.json()['toast']['type']=='warning'and not p['calls']
assert p['env']['store'].db.execute('SELECT state FROM work_approvals_v3').fetchone()['state']=='pending'
await p['click']();assert (await asyncio.wait_for(job,10))['status']=='completed'
assert len(p['calls'])==1
async def test_card_send_receipt_loss_never_sends_second_card(pair):
p=pair;p['env']['platform'].lose_card_receipt=True;job=p['launch']();await p['card'](bound=False)
deadline=time.monotonic()+5
while time.monotonic()<deadline:
row=p['env']['store'].db.execute('SELECT state FROM work_approval_card_outbox_v1').fetchone()
if row and row['state']=='unknown':break
await asyncio.sleep(.01)
assert row['state']=='unknown'
await p['env']['app'].state.approvals.maintain()
await p['env']['app'].state.approvals.maintain()
assert len(p['env']['platform'].cards)==1
p['stop'].set()
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
assert not p['calls']
async def test_unsigned_rpc_proof_replay_and_other_task_are_rejected(pair):
p=pair;job=p['launch']();row=await p['card']();e=p['env'];path='/work/v2/approvals/status'
fields=dict(task_id='task-approval',approval_id=row['approval_id'],operation_digest=row['operation_digest'])
unsigned=await e['client'].post(path,json=fields);assert unsigned.status_code==422
signed=wire(e,path,**fields)
assert (await e['client'].post(path,json=signed)).status_code==200
assert (await e['client'].post(path,json=signed)).status_code==409
other=await e['client'].post(path,json=wire(e,path,**{**fields,'task_id':'other-task'}))
assert other.status_code==409 and not p['calls']
p['stop'].set()
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
@pytest.mark.parametrize('field',['app','tenant','token','timestamp','chat','card','nonce','value_identity'])
async def test_forged_callback_cannot_approve(pair,field):
p=pair;job=p['launch']();await p['card']()
def mutate(body):
if field=='app':body['header']['app_id']='another-app'
elif field=='tenant':body['header']['tenant_key']='another-tenant'
elif field=='token':body['header']['token']='wrong-verification'
elif field=='timestamp':body['header']['create_time']='1'
elif field=='chat':body['event']['context']['open_chat_id']='oc_other'
elif field=='card':body['event']['context']['open_message_id']='om_other'
elif field=='nonce':body['event']['action']['value']['nonce']='A'*43
else:body['event']['action']['value']['actor_open_id']='ou_a'
response,_=await p['click'](mutate=mutate)
assert response.json()['toast']['type']=='warning'
assert p['env']['store'].db.execute('SELECT state FROM work_approvals_v3').fetchone()['state']=='pending'
assert not p['calls']
p['stop'].set()
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
async def test_conflicting_clicks_and_concurrent_consume_grant_at_most_once(pair):
p=pair;job=p['launch']();row=await p['card']()
# Prevent PC consumption while testing two consumers of the same grant.
response,_=await p['click']('allow')
conflict,_=await p['click']('deny')
assert response.json()['toast']['type']=='info'
assert conflict.json()['toast']['type']=='warning'
e=p['env'];path='/work/v2/approvals/consume'
fields=dict(task_id='task-approval',approval_id=row['approval_id'],operation_digest=row['operation_digest'])
responses=await asyncio.gather(*(e['client'].post(path,json=wire(e,path,**fields))for _ in range(2)))
assert sum(r.json().get('execute')is True for r in responses)<=1
try:result=await asyncio.wait_for(job,10)
except pc_errors.WorkError:result=None
assert len(p['calls'])<=1
assert sum(r.json().get('execute')is True for r in responses)+len(p['calls'])==1
async def test_expiry_removes_buttons_and_update_receipt_loss_is_retryable(pair,monkeypatch):
p=pair;job=p['launch']();row=await p['card']();e=p['env']
with e['store'].db:e['store'].db.execute('UPDATE work_approvals_v3 SET expires_at=0')
await e['app'].state.approvals.worker.call('close_unavailable')
sender=e['app'].state.approvals.sender;original=e['platform'].update_task_card
attempts=[]
async def lost_once(*args,**kwargs):
await original(*args,**kwargs);attempts.append(1)
if len(attempts)==1:raise ConnectionError('synthetic update receipt loss')
monkeypatch.setattr(e['platform'],'update_task_card',lost_once)
with pytest.raises(ConnectionError):await sender.update_closed(row['approval_id'])
assert await sender.update_closed(row['approval_id'])=={'state':'updated'}
assert await sender.update_closed(row['approval_id'])=={'state':'no_closed_card'}
assert len(e['platform'].cards)==1 and len(attempts)==2
assert all('"button"'not in json.dumps(card)for _,card in e['platform'].updates)
with pytest.raises(pc_errors.WorkError):await asyncio.wait_for(job,10)
assert not p['calls']
async def test_reviewed_read_bypasses_confirmation(pair):
p=pair
result=await asyncio.to_thread(p['client'].api,'GET','/open-apis/calendar/v4/calendars')
assert result['status']=='completed'and len(p['calls'])==1
assert not p['env']['platform'].cards
assert not p['env']['store'].db.execute('SELECT * FROM work_approvals_v3').fetchall()
async def test_default_off_does_not_register_approval_or_callback_routes(env):
status=await env['client'].post('/work/v2/status',json=wire(env,'/work/v2/status'))
assert status.status_code==200 and status.json()['approval_protocol']is None
for path in ('/work/v2/approvals/create','/internal/approval-callback'):
assert (await env['client'].post(path,json={})).status_code==404
assert not env['store'].db.execute("SELECT name FROM sqlite_master WHERE name='work_approvals_v3'").fetchall()
async def test_approval_worker_recreation_preserves_original_pending_card(pair):
p=pair;job=p['launch']();row=await p['card']();e=p['env']
api=e['app'].state.approvals;original=api.worker
restored=ApprovalWorker(e['settings'],e['api'].broker.store.keyring)
await restored.start()
api.worker=restored;api.sender.approvals=restored
try:
await api.maintain()
assert len(e['platform'].cards)==1
response,_=await p['click']()
assert response.json()['toast']['type']=='info'
assert (await asyncio.wait_for(job,10))['status']=='completed'
assert len(p['calls'])==1
saved=e['store'].db.execute('SELECT approval_id,state,card_message_id FROM work_approvals_v3').fetchone()
assert (saved['approval_id'],saved['state'],saved['card_message_id'])==(
row['approval_id'],'consumed',row['card_message_id'])
finally:
# Drain only this adapter's tracked card work before closing its
# connection. App lifespan still owns the original worker shutdown.
deadline=time.monotonic()+5
while time.monotonic()<deadline:
if not api.scheduled:break
await asyncio.sleep(.01)
api.worker=original;api.sender.approvals=original
await restored.close()
+87
View File
@@ -672,3 +672,90 @@ def test_prepare_update_cards_are_opt_in_and_rollback_keeps_parent_environment(s
assert current['services']['connector']['environment']['HN_WORK_TASK_CARDS_ENABLED']=='true'
assert rollback['services']['connector']['environment']==old['services']['connector']['environment']
assert (output/'credential-keyring.json').read_bytes()==(root/'credential-keyring.json').read_bytes()
def approval_update(scenario,**changes):
parent=prepared(scenario);root=scenario['output'];image='sha256:'+'a'*64
for name in ('connector','router'):scenario['roles'][name]['image']=image
deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']})
env=scenario['base']['services']['connector']['environment']
scenario['base']['services']['longconn']['environment'].update(
HN_FEISHU_APP_ID=env['HN_FEISHU_APP_ID'],HN_INTERNAL_SECRET=env['HN_INTERNAL_SECRET'],HN_FEISHU_EVENT_SOURCE='longconn')
identity=root.parent/'callback-private.json'
identity.write_text(json.dumps({'tenant_key':'tenant_test','verification_token':'synthetic-verify-token'}))
identity.chmod(0o600)
args=argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(root.parent/'approvals'),
enable_remote_approvals=True,callback_identity=str(identity))
for key,value in changes.items():setattr(args,key,value)
return args
def test_approval_update_preserves_runtime_ancestry_and_original_longconn_rollback(scenario):
args=approval_update(scenario);deploy.prepare_update(args)
root=Path(args.output);plan=deploy.load_plan(root)
candidate=json.loads((root/'compose.broker.json').read_text());rollback=json.loads((root/'compose.rollback.json').read_text())
assert deploy.targets(plan)==('connector','router','longconn')
assert plan['longconn_base_image']==scenario['roles']['longconn']['image']
assert candidate['services']['longconn']['image']!=candidate['services']['connector']['image']
assert candidate['services']['longconn']['environment']['HN_CREDENTIAL_MODE']=='broker'
assert rollback['services']['longconn']['image']==scenario['roles']['longconn']['image']
assert rollback['services']['longconn']['environment']['HN_CREDENTIAL_MODE']=='legacy'
assert rollback['services']['longconn']['environment']['HN_WORK_REMOTE_APPROVALS_ENABLED']=='false'
assert (root/'credential-keyring.json').read_bytes()==(Path(args.plan)/'credential-keyring.json').read_bytes()
def test_approval_runtime_checks_three_roles_and_sdk_without_network(scenario):
args=approval_update(scenario);deploy.prepare_update(args);root=Path(args.output);plan=deploy.load_plan(root)
scenario['calls'].clear();deploy.runtime_check(root,plan)
probes=[c for c in scenario['calls']if c[:2]==['docker','run']]
assert len(probes)==3
assert all('--network'in c and c[c.index('--network')+1]=='none'for c in probes)
assert any(plan['longconn_candidate_image']in c and 'register_p2_card_action_trigger'in c[-1]for c in probes)
assert not any('synthetic-verify-token'in part for c in probes for part in c)
def test_approval_activation_targets_original_three_services_only(scenario):
args=approval_update(scenario);deploy.prepare_update(args);root=Path(args.output);plan=deploy.load_plan(root)
image='sha256:'+'a'*64
deploy.write_json(root/'candidate.json',{'image':image,'tag':plan['candidate_image']})
deploy.write_json(root/'candidate-longconn.json',{'image':image,'tag':plan['longconn_candidate_image']})
scenario['calls'].clear();deploy.activate(root,plan,True)
up=[c for c in scenario['calls']if 'up'in c]
assert len(up)==1 and up[0][-3:]==['connector','router','longconn']
assert '--no-deps'in up[0]and '--pull'in up[0]
def test_approval_overlay_rejects_longconn_command_or_network_change(scenario):
args=approval_update(scenario);deploy.prepare_update(args);root=Path(args.output);plan=deploy.load_plan(root)
base=deploy.run(deploy.compose(plan)+['config','--format','json'],True)
merged=deploy.run(deploy.compose(plan,root/'compose.broker.json')+['config','--format','json'],True)
merged['services']['longconn']['entrypoint']=['sh','-c','unreviewed']
with pytest.raises(deploy.DeployError,match='unrelated longconn'):
deploy.validate_overlay(base,merged,plan['candidate_image'],approval_plan=plan)
def test_approval_update_rejects_different_app_or_missing_identity(scenario):
args=approval_update(scenario)
scenario['base']['services']['longconn']['environment']['HN_FEISHU_APP_ID']='other'
with pytest.raises(deploy.DeployError,match='does not match'):deploy.prepare_update(args)
assert not Path(args.output).exists()
def test_approval_build_reuses_distinct_longconn_base_and_saves_both_receipts(scenario):
args=approval_update(scenario);deploy.prepare_update(args);root=Path(args.output);plan=deploy.load_plan(root)
scenario['calls'].clear();deploy.build(root,plan)
commands=[c for c in scenario['calls']if c[:2]==['docker','build']]
assert len(commands)==2
assert 'BASE_IMAGE='+scenario['roles']['longconn']['image']in commands[1]
assert plan['longconn_candidate_image']in commands[1]
assert all('--network'in c and c[c.index('--network')+1]=='none'for c in commands)
assert (root/'candidate.json').is_file()and(root/'candidate-longconn.json').is_file()
def test_approval_cli_rollback_restores_original_longconn_too(scenario,monkeypatch):
args=approval_update(scenario);deploy.prepare_update(args);root=Path(args.output)
monkeypatch.setattr(deploy.sys,'argv',['broker_deploy.py','rollback','--plan',str(root),'--stop-broker-use'])
scenario['calls'].clear();deploy.main()
commands=[c for c in scenario['calls']if 'up'in c]
assert len(commands)==1 and commands[0][-3:]==['connector','router','longconn']
assert str(root/'compose.rollback.json')in commands[0]