fix(work): fence repeated approval intents and audit state changes atomically

This commit is contained in:
ex_tianzy
2026-11-17 15:21:54 +08:00
parent 03d34f1d55
commit 75c609ded6
15 changed files with 638 additions and 10 deletions
+119
View File
@@ -0,0 +1,119 @@
"""M6 audit and cleanup on temporary encrypted approval stores only."""
import json
import pytest
from test_credential_api import env
from test_approval_integration import approval_env,pair
from relay.approval_store import ApprovalStore,ApprovalError
async def test_audit_is_minimal_and_repeated_decisions_do_not_duplicate(pair):
p=pair;job=p['launch']();row=await p['card']();response,body=await p['click']()
await job
e=p['env'];await e['client'].post('/internal/approval-callback',json=body,
headers={'X-Internal-Secret':e['settings'].internal_secret})
events=[dict(r)for r in e['store'].db.execute('SELECT * FROM work_approval_audit_v1').fetchall()]
assert {r['event']for r in events}>={'created','card_bound','approved','consumed'}
assert sum(r['event']=='approved'for r in events)==1
assert sum(r['event']=='consumed'for r in events)==1
encoded=json.dumps(events)
for secret in ('ou_a','oc_synthetic','om_source','合成测试','fake-access','verification-synthetic',row['ciphertext']):
assert secret not in encoded
assert all(len(r['actor_scope'])==64 and len(r['device_scope'])==64 for r in events)
assert all(r['action_class']=='work_api_write'for r in events)
async def test_closed_payload_purge_keeps_replay_tombstone_and_active_payload(pair):
p=pair;job=p['launch']();row=await p['card']();await p['click']('deny')
from test_approval_integration import pc_errors
with pytest.raises(pc_errors.WorkError):await job
e=p['env'];now=__import__('time').time()
with e['store'].db:e['store'].db.execute('UPDATE work_approvals_v3 SET updated_at=?',(now-31*86400,))
api=ApprovalStore(e['store'].lifecycle,e['api'].broker.store.keyring,clock=lambda:now)
active=api.create('task-approval','ou_a','a'*32,1,'operation-active',{'kind':'synthetic'},'合成动作','仅测试')
with e['store'].db:e['store'].db.execute('UPDATE work_approvals_v3 SET updated_at=? WHERE approval_id=?',
(now-32*86400,active['approval_id']))
result=api.maintain_retention()
assert result['payloads_purged']==1 and result['tombstones_preserved']
saved=e['store'].db.execute('SELECT * FROM work_approvals_v3').fetchone()
assert saved['state']=='rejected'and saved['ciphertext']==''and saved['nonce_digest']==''
assert api.maintain_retention()['payloads_purged']==0
status=api.status(row['approval_id'],'task-approval','ou_a','a'*32,1,row['operation_digest'])
assert status['state']=='rejected'
assert api.consume(row['approval_id'],'task-approval','ou_a','a'*32,1,row['operation_digest'])['execute']is False
action=json.loads(e['api'].broker.store.keyring.decrypt(row['key_id'],row['ciphertext'],api._aad(row)))['action']
assert api.create('task-approval','ou_a','a'*32,1,'operation-synthetic',action,'同动作','不得重新放行')['state']=='rejected'
# A pre-existing pending approval is not selected even when timestamp is old.
assert api.maintain_retention()['payloads_purged']==0
assert e['store'].db.execute('SELECT ciphertext FROM work_approvals_v3 WHERE approval_id=?',
(active['approval_id'],)).fetchone()['ciphertext']
async def test_retention_is_environment_scoped_and_bounded(pair):
p=pair;job=p['launch']();row=await p['card']();await p['click']('deny')
from test_approval_integration import pc_errors
with pytest.raises(pc_errors.WorkError):await job
e=p['env'];now=__import__('time').time()
api=ApprovalStore(e['store'].lifecycle,e['api'].broker.store.keyring,clock=lambda:now)
with e['store'].db:
e['store'].db.execute('UPDATE work_approvals_v3 SET updated_at=?',(now-100*86400,))
e['store'].db.execute('UPDATE work_approval_audit_v1 SET created_at=?',(now-100*86400,))
result=api.maintain_retention(limit=1)
assert result['audit_events_removed']==1
assert e['store'].db.execute('SELECT COUNT(*) n FROM work_approvals_v3').fetchone()['n']==1
assert e['store'].db.execute('SELECT COUNT(*) n FROM work_approval_callbacks_v1').fetchone()['n']==1
for kwargs in ({'payload_days':0},{'audit_days':366},{'payload_days':91,'audit_days':90},{'limit':100}):
with pytest.raises(ApprovalError):api.maintain_retention(**kwargs)
async def test_audit_write_failure_rolls_back_consume_and_never_sends(pair,monkeypatch):
from relay.approval_audit import ApprovalAudit
from test_approval_integration import pc_errors
original=ApprovalAudit.record
def fail_consume(self,row,event,action_class=None):
if event=='consumed':raise RuntimeError('synthetic audit storage failure')
return original(self,row,event,action_class)
monkeypatch.setattr(ApprovalAudit,'record',fail_consume)
p=pair;job=p['launch']();await p['click']()
with pytest.raises(pc_errors.WorkError):await job
e=p['env']
assert not p['calls']
assert e['store'].db.execute("SELECT state FROM work_approvals_v3").fetchone()['state']!='consumed'
assert not e['store'].db.execute("SELECT * FROM work_approval_audit_v1 WHERE event='consumed'").fetchall()
async def test_retention_never_touches_another_environment(pair):
from test_approval_integration import pc_errors
p=pair;job=p['launch']();await p['click']('deny')
with pytest.raises(pc_errors.WorkError):await job
e=p['env'];now=__import__('time').time()
with e['store'].db:e['store'].db.execute("UPDATE work_approvals_v3 SET environment='development',updated_at=?",(now-100*86400,))
api=ApprovalStore(e['store'].lifecycle,e['api'].broker.store.keyring,clock=lambda:now)
assert api.maintain_retention()['payloads_purged']==0
assert e['store'].db.execute('SELECT ciphertext FROM work_approvals_v3').fetchone()['ciphertext']
def test_retention_configuration_is_explicit_and_ordered():
from relay.config import Settings
from pydantic import ValidationError
settings=Settings(role='test',environment='test')
assert settings.approval_retention_enabled is False
assert (settings.approval_payload_retention_days,settings.approval_audit_retention_days)==(30,90)
for changes in ({'approval_retention_enabled':True},
{'approval_payload_retention_days':91,'approval_audit_retention_days':90},
{'approval_audit_retention_days':366}):
with pytest.raises(ValidationError):Settings(role='test',environment='test',**changes)
async def test_consumed_but_result_unknown_is_not_purged(pair):
p=pair;job=p['launch']();await p['click']();await job
e=p['env'];now=__import__('time').time()
api=ApprovalStore(e['store'].lifecycle,e['api'].broker.store.keyring,clock=lambda:now)
with e['store'].db:
e['store'].db.execute('UPDATE work_approvals_v3 SET updated_at=?',(now-100*86400,))
e['store'].db.execute("UPDATE work_tasks_v2 SET status='unknown'")
assert api.maintain_retention()['payloads_purged']==0
assert e['store'].db.execute('SELECT ciphertext FROM work_approvals_v3').fetchone()['ciphertext']
with e['store'].db:e['store'].db.execute("UPDATE work_tasks_v2 SET status='completed'")
# Bounded keyset scan wraps on the next maintenance pass.
outcomes=[api.maintain_retention()['payloads_purged']for _ in range(2)]
assert sum(outcomes)==1
+109
View File
@@ -0,0 +1,109 @@
"""Opt-in isolated-schema PostgreSQL race test; never uses production config."""
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import os
from pathlib import Path
import secrets
import time
import threading
import pytest
import psycopg
from psycopg.conninfo import make_conninfo
from relay.store import Store
from relay.binding_state import BindingState
from relay.credential_store import CredentialKeyring
from relay.approval_store import ApprovalStore,VerifiedDecision,ApprovalError
from relay.crypto import canonical
pytestmark=pytest.mark.skipif(not os.environ.get('HN_TEST_DATABASE_URL'),reason='isolated PostgreSQL test URL not supplied')
@pytest.mark.parametrize('winner',['race','allow','deny'])
@pytest.mark.parametrize('consumers',[4,16])
def test_concurrent_postgres_decisions_consumption_and_reopen(tmp_path,winner,consumers):
schema='approval_test_'+secrets.token_hex(10)
target=os.environ['HN_TEST_DATABASE_URL']
control=psycopg.connect(target,autocommit=True)
control.execute('CREATE SCHEMA '+schema)
dsn=make_conninfo(target,options='-c search_path='+schema,connect_timeout=5)
keyring=CredentialKeyring('synthetic',{ 'synthetic':secrets.token_bytes(32)})
connections=[]
def connect():
store=Store(tmp_path/'unused.sqlite',dsn)
store.db.execute("SET lock_timeout TO '2s'")
store.db.execute("SET statement_timeout TO '3s'")
state=BindingState(store,'test','cli_pg_test')
return store,state,ApprovalStore(state,keyring)
try:
store,state,approvals=connect();connections.append(store)
with store.db:
store.db.execute("INSERT INTO devices VALUES(?,?,?,?,?,?)",('a'*32,'synthetic-cert','synthetic-fingerprint','ou_pg',0,1))
state.create_task('pg-task','ou_pg','a'*32,1,'session','worker',time.time()+300,'oc_pg','om_pg_source')
action={'kind':'synthetic','value':'no real side effect'}
item=approvals.create('pg-task','ou_pg','a'*32,1,'pg-operation',action,'合成并发确认','不执行真实业务')
material=approvals.card_material(item['approval_id']);assert approvals.claim_card(item['approval_id'])
approvals.bind_card(item['approval_id'],'om_pg_card')
def make_decision(choice):
fields=dict(environment='test',app_id='cli_pg_test',actor_open_id='ou_pg',chat_id='oc_pg',
card_message_id='om_pg_card',approval_id=item['approval_id'],nonce=material['nonce'],choice=choice)
return VerifiedDecision(**fields,event_id=secrets.token_hex(16),event_digest=hashlib.sha256(canonical(fields)).hexdigest())
if winner!='race':approvals.decide(make_decision(winner))
decision_gate=threading.Barrier(2,timeout=10)
def decide(choice):
connection,_,ledger=connect()
try:
decision_gate.wait()
try:return ledger.decide(make_decision(choice))['state']
except ApprovalError:return 'conflict'
finally:connection.close()
with ThreadPoolExecutor(max_workers=2)as executor:
decisions=list(executor.map(decide,('allow','deny')))
row=store.db.execute('SELECT state FROM work_approvals_v3').fetchone()
assert row['state']in {'approved','rejected'}and decisions.count('conflict')==1
if winner=='allow':
# A real PostgreSQL statement error must roll back the earlier
# consumed UPDATE in the same transaction, not merely hide a reply.
store.db.execute("""CREATE FUNCTION fail_approval_audit() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN IF NEW.event='consumed' THEN RAISE EXCEPTION 'synthetic audit storage failure';
END IF; RETURN NEW; END $$""")
store.db.execute('''CREATE TRIGGER synthetic_audit_failure BEFORE INSERT ON work_approval_audit_v1
FOR EACH ROW EXECUTE FUNCTION fail_approval_audit()''')
with pytest.raises(psycopg.Error):
approvals.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])
assert store.db.execute('SELECT state FROM work_approvals_v3').fetchone()['state']=='approved'
assert not store.db.execute("SELECT * FROM work_approval_audit_v1 WHERE event='consumed'").fetchall()
store.db.execute('DROP TRIGGER synthetic_audit_failure ON work_approval_audit_v1')
store.db.execute('DROP FUNCTION fail_approval_audit()')
consume_gate=threading.Barrier(consumers,timeout=20)
def consume(_):
connection,_,ledger=connect()
try:
consume_gate.wait()
return ledger.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])['execute']
finally:connection.close()
with ThreadPoolExecutor(max_workers=consumers)as executor:executed=list(executor.map(consume,range(consumers)))
assert sum(executed)==(1 if row['state']=='approved'else 0)
restored,_,ledger=connect();connections.append(restored)
assert ledger.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])['execute']is False
assert store.db.execute("SELECT COUNT(*) n FROM work_approval_audit_v1 WHERE event IN ('approved','rejected')").fetchone()['n']==1
state.create_task('pg-intent-task','ou_pg','a'*32,1,'session','worker',time.time()+300,'oc_pg','om_pg_intent')
intent_gate=threading.Barrier(2,timeout=10)
def create_same_intent(operation):
connection,_,ledger=connect()
try:
intent_gate.wait()
try:
ledger.create('pg-intent-task','ou_pg','a'*32,1,operation,action,'相同合成动作','不执行真实业务')
return 'created'
except ApprovalError:return 'conflict'
finally:connection.close()
with ThreadPoolExecutor(max_workers=2)as executor:
outcomes=list(executor.map(create_same_intent,('pg-intent-one','pg-intent-two')))
assert outcomes.count('created')==1 and outcomes.count('conflict')==1
assert store.db.execute("SELECT COUNT(*) n FROM work_approvals_v3 WHERE task_id='pg-intent-task'").fetchone()['n']==1
finally:
for connection in connections:connection.close()
# The unique generated test schema is the only deletion target.
control.execute('DROP SCHEMA '+schema+' CASCADE');control.close()
@@ -0,0 +1,62 @@
"""Adversarial M6 checks, no real platform operations."""
import hashlib
import json
import pytest
from test_credential_api import env,wire
from test_approval_integration import approval_env,pair,pc_errors
from relay.approval_store import ApprovalStore,ApprovalError
from relay.approval_policy import describe_action
from relay.crypto import canonical
@pytest.mark.parametrize('choice',[None,'deny','allow'])
async def test_same_intent_cannot_get_new_card_by_changing_operation_id(pair,choice):
p=pair;job=p['launch']();row=await p['card']()
if choice:
await p['click'](choice)
if choice=='deny':
with pytest.raises(pc_errors.WorkError):await job
else:await job
e=p['env'];ledger=ApprovalStore(e['store'].lifecycle,e['api'].broker.store.keyring)
action=json.loads(ledger.keyring.decrypt(row['key_id'],row['ciphertext'],ledger._aad(row)))['action']
other='new-operation-id'
action['params']['idempotency_key']=hashlib.sha256(canonical(['ou_a',other])).hexdigest()
path='/work/v2/approvals/create'
response=await e['client'].post(path,json=wire(e,path,task_id='task-approval',operation_id=other,action=action,ttl=120))
assert response.status_code==409
assert e['store'].db.execute('SELECT COUNT(*) n FROM work_approvals_v3').fetchone()['n']==1
assert len(p['calls'])==(1 if choice=='allow'else 0)
if not choice:
p['stop'].set()
with pytest.raises(pc_errors.WorkError):await job
def test_confirmation_identifies_actual_calendar_not_just_events_suffix():
def action(calendar):return {'kind':'work-api-v1','method':'POST',
'path':'/open-apis/calendar/v4/calendars/'+calendar+'/events','params':{},'body':{'summary':'同一标题'}}
first=describe_action(action('calendar_a'))
second=describe_action(action('calendar_b'))
assert 'calendar_a'in first[1]and 'calendar_b'in second[1]and first!=second
def test_long_target_is_rejected_instead_of_silently_truncated():
with pytest.raises(ApprovalError):describe_action({'kind':'work-api-v1','method':'DELETE',
'path':'/open-apis/drive/v1/files/'+'x'*400,'params':{},'body':{}})
async def test_changed_action_same_id_rejected_but_distinct_new_action_can_confirm(pair):
p=pair;job=p['launch']();row=await p['card']();await p['click']('deny')
with pytest.raises(pc_errors.WorkError):await job
e=p['env'];ledger=ApprovalStore(e['store'].lifecycle,e['api'].broker.store.keyring)
action=json.loads(ledger.keyring.decrypt(row['key_id'],row['ciphertext'],ledger._aad(row)))['action']
action['body']['summary']='真正不同的合成动作'
path='/work/v2/approvals/create'
same=await e['client'].post(path,json=wire(e,path,task_id='task-approval',
operation_id='operation-synthetic',action=action,ttl=120))
assert same.status_code==409
other='distinct-operation'
action['params']['idempotency_key']=hashlib.sha256(canonical(['ou_a',other])).hexdigest()
distinct=await e['client'].post(path,json=wire(e,path,task_id='task-approval',
operation_id=other,action=action,ttl=120))
assert distinct.status_code==200 and distinct.json()['state']=='pending'
assert not p['calls']
+23
View File
@@ -698,12 +698,35 @@ def test_approval_update_preserves_runtime_ancestry_and_original_longconn_rollba
assert plan['longconn_base_image']==scenario['roles']['longconn']['image']
assert candidate['services']['longconn']['image']!=candidate['services']['connector']['image']
assert candidate['services']['longconn']['environment']['HN_CREDENTIAL_MODE']=='broker'
assert candidate['services']['connector']['environment']['HN_APPROVAL_RETENTION_ENABLED']=='false'
assert rollback['services']['longconn']['image']==scenario['roles']['longconn']['image']
assert rollback['services']['longconn']['environment']['HN_CREDENTIAL_MODE']=='legacy'
assert rollback['services']['longconn']['environment']['HN_WORK_REMOTE_APPROVALS_ENABLED']=='false'
assert (root/'credential-keyring.json').read_bytes()==(Path(args.plan)/'credential-keyring.json').read_bytes()
def test_second_approval_update_uses_active_parent_images_and_retains_off(scenario):
args=approval_update(scenario);deploy.prepare_update(args)
root=Path(args.output);parent=deploy.load_plan(root)
connector='sha256:'+'c'*64;longconn='sha256:'+'d'*64
for name in ('connector','router'):scenario['roles'][name]['image']=connector
scenario['roles']['longconn']['image']=longconn
deploy.write_json(root/'candidate.json',{'image':connector,'tag':parent['candidate_image']})
deploy.write_json(root/'candidate-longconn.json',{'image':longconn,'tag':parent['longconn_candidate_image']})
output=root.parent/'approvals-m6-fix'
deploy.prepare_update(argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output),
enable_remote_approvals=True,callback_identity=args.callback_identity))
result=deploy.load_plan(output)
candidate=json.loads((output/'compose.broker.json').read_text())
rollback=json.loads((output/'compose.rollback.json').read_text())
assert result['base_image']==connector and result['longconn_base_image']==longconn
assert candidate['services']['connector']['environment']['HN_APPROVAL_RETENTION_ENABLED']=='false'
assert rollback['services']['connector']['image']==connector
assert rollback['services']['longconn']['image']==longconn
assert rollback['services']['longconn']['environment']['HN_WORK_REMOTE_APPROVALS_ENABLED']=='true'
assert (output/'credential-keyring.json').read_bytes()==(root/'credential-keyring.json').read_bytes()
def test_approval_runtime_checks_three_roles_and_sdk_without_network(scenario):
args=approval_update(scenario);deploy.prepare_update(args);root=Path(args.output);plan=deploy.load_plan(root)
scenario['calls'].clear();deploy.runtime_check(root,plan)