fix(work): fence repeated approval intents and audit state changes atomically
This commit is contained in:
@@ -0,0 +1,119 @@
|
||||
"""M6 audit and cleanup on temporary encrypted approval stores only."""
|
||||
import json
|
||||
import pytest
|
||||
from test_credential_api import env
|
||||
from test_approval_integration import approval_env,pair
|
||||
from relay.approval_store import ApprovalStore,ApprovalError
|
||||
|
||||
|
||||
async def test_audit_is_minimal_and_repeated_decisions_do_not_duplicate(pair):
|
||||
p=pair;job=p['launch']();row=await p['card']();response,body=await p['click']()
|
||||
await job
|
||||
e=p['env'];await e['client'].post('/internal/approval-callback',json=body,
|
||||
headers={'X-Internal-Secret':e['settings'].internal_secret})
|
||||
events=[dict(r)for r in e['store'].db.execute('SELECT * FROM work_approval_audit_v1').fetchall()]
|
||||
assert {r['event']for r in events}>={'created','card_bound','approved','consumed'}
|
||||
assert sum(r['event']=='approved'for r in events)==1
|
||||
assert sum(r['event']=='consumed'for r in events)==1
|
||||
encoded=json.dumps(events)
|
||||
for secret in ('ou_a','oc_synthetic','om_source','合成测试','fake-access','verification-synthetic',row['ciphertext']):
|
||||
assert secret not in encoded
|
||||
assert all(len(r['actor_scope'])==64 and len(r['device_scope'])==64 for r in events)
|
||||
assert all(r['action_class']=='work_api_write'for r in events)
|
||||
|
||||
|
||||
async def test_closed_payload_purge_keeps_replay_tombstone_and_active_payload(pair):
|
||||
p=pair;job=p['launch']();row=await p['card']();await p['click']('deny')
|
||||
from test_approval_integration import pc_errors
|
||||
with pytest.raises(pc_errors.WorkError):await job
|
||||
e=p['env'];now=__import__('time').time()
|
||||
with e['store'].db:e['store'].db.execute('UPDATE work_approvals_v3 SET updated_at=?',(now-31*86400,))
|
||||
api=ApprovalStore(e['store'].lifecycle,e['api'].broker.store.keyring,clock=lambda:now)
|
||||
active=api.create('task-approval','ou_a','a'*32,1,'operation-active',{'kind':'synthetic'},'合成动作','仅测试')
|
||||
with e['store'].db:e['store'].db.execute('UPDATE work_approvals_v3 SET updated_at=? WHERE approval_id=?',
|
||||
(now-32*86400,active['approval_id']))
|
||||
result=api.maintain_retention()
|
||||
assert result['payloads_purged']==1 and result['tombstones_preserved']
|
||||
saved=e['store'].db.execute('SELECT * FROM work_approvals_v3').fetchone()
|
||||
assert saved['state']=='rejected'and saved['ciphertext']==''and saved['nonce_digest']==''
|
||||
assert api.maintain_retention()['payloads_purged']==0
|
||||
status=api.status(row['approval_id'],'task-approval','ou_a','a'*32,1,row['operation_digest'])
|
||||
assert status['state']=='rejected'
|
||||
assert api.consume(row['approval_id'],'task-approval','ou_a','a'*32,1,row['operation_digest'])['execute']is False
|
||||
action=json.loads(e['api'].broker.store.keyring.decrypt(row['key_id'],row['ciphertext'],api._aad(row)))['action']
|
||||
assert api.create('task-approval','ou_a','a'*32,1,'operation-synthetic',action,'同动作','不得重新放行')['state']=='rejected'
|
||||
# A pre-existing pending approval is not selected even when timestamp is old.
|
||||
assert api.maintain_retention()['payloads_purged']==0
|
||||
assert e['store'].db.execute('SELECT ciphertext FROM work_approvals_v3 WHERE approval_id=?',
|
||||
(active['approval_id'],)).fetchone()['ciphertext']
|
||||
|
||||
|
||||
async def test_retention_is_environment_scoped_and_bounded(pair):
|
||||
p=pair;job=p['launch']();row=await p['card']();await p['click']('deny')
|
||||
from test_approval_integration import pc_errors
|
||||
with pytest.raises(pc_errors.WorkError):await job
|
||||
e=p['env'];now=__import__('time').time()
|
||||
api=ApprovalStore(e['store'].lifecycle,e['api'].broker.store.keyring,clock=lambda:now)
|
||||
with e['store'].db:
|
||||
e['store'].db.execute('UPDATE work_approvals_v3 SET updated_at=?',(now-100*86400,))
|
||||
e['store'].db.execute('UPDATE work_approval_audit_v1 SET created_at=?',(now-100*86400,))
|
||||
result=api.maintain_retention(limit=1)
|
||||
assert result['audit_events_removed']==1
|
||||
assert e['store'].db.execute('SELECT COUNT(*) n FROM work_approvals_v3').fetchone()['n']==1
|
||||
assert e['store'].db.execute('SELECT COUNT(*) n FROM work_approval_callbacks_v1').fetchone()['n']==1
|
||||
for kwargs in ({'payload_days':0},{'audit_days':366},{'payload_days':91,'audit_days':90},{'limit':100}):
|
||||
with pytest.raises(ApprovalError):api.maintain_retention(**kwargs)
|
||||
|
||||
|
||||
async def test_audit_write_failure_rolls_back_consume_and_never_sends(pair,monkeypatch):
|
||||
from relay.approval_audit import ApprovalAudit
|
||||
from test_approval_integration import pc_errors
|
||||
original=ApprovalAudit.record
|
||||
def fail_consume(self,row,event,action_class=None):
|
||||
if event=='consumed':raise RuntimeError('synthetic audit storage failure')
|
||||
return original(self,row,event,action_class)
|
||||
monkeypatch.setattr(ApprovalAudit,'record',fail_consume)
|
||||
p=pair;job=p['launch']();await p['click']()
|
||||
with pytest.raises(pc_errors.WorkError):await job
|
||||
e=p['env']
|
||||
assert not p['calls']
|
||||
assert e['store'].db.execute("SELECT state FROM work_approvals_v3").fetchone()['state']!='consumed'
|
||||
assert not e['store'].db.execute("SELECT * FROM work_approval_audit_v1 WHERE event='consumed'").fetchall()
|
||||
|
||||
|
||||
async def test_retention_never_touches_another_environment(pair):
|
||||
from test_approval_integration import pc_errors
|
||||
p=pair;job=p['launch']();await p['click']('deny')
|
||||
with pytest.raises(pc_errors.WorkError):await job
|
||||
e=p['env'];now=__import__('time').time()
|
||||
with e['store'].db:e['store'].db.execute("UPDATE work_approvals_v3 SET environment='development',updated_at=?",(now-100*86400,))
|
||||
api=ApprovalStore(e['store'].lifecycle,e['api'].broker.store.keyring,clock=lambda:now)
|
||||
assert api.maintain_retention()['payloads_purged']==0
|
||||
assert e['store'].db.execute('SELECT ciphertext FROM work_approvals_v3').fetchone()['ciphertext']
|
||||
|
||||
|
||||
def test_retention_configuration_is_explicit_and_ordered():
|
||||
from relay.config import Settings
|
||||
from pydantic import ValidationError
|
||||
settings=Settings(role='test',environment='test')
|
||||
assert settings.approval_retention_enabled is False
|
||||
assert (settings.approval_payload_retention_days,settings.approval_audit_retention_days)==(30,90)
|
||||
for changes in ({'approval_retention_enabled':True},
|
||||
{'approval_payload_retention_days':91,'approval_audit_retention_days':90},
|
||||
{'approval_audit_retention_days':366}):
|
||||
with pytest.raises(ValidationError):Settings(role='test',environment='test',**changes)
|
||||
|
||||
|
||||
async def test_consumed_but_result_unknown_is_not_purged(pair):
|
||||
p=pair;job=p['launch']();await p['click']();await job
|
||||
e=p['env'];now=__import__('time').time()
|
||||
api=ApprovalStore(e['store'].lifecycle,e['api'].broker.store.keyring,clock=lambda:now)
|
||||
with e['store'].db:
|
||||
e['store'].db.execute('UPDATE work_approvals_v3 SET updated_at=?',(now-100*86400,))
|
||||
e['store'].db.execute("UPDATE work_tasks_v2 SET status='unknown'")
|
||||
assert api.maintain_retention()['payloads_purged']==0
|
||||
assert e['store'].db.execute('SELECT ciphertext FROM work_approvals_v3').fetchone()['ciphertext']
|
||||
with e['store'].db:e['store'].db.execute("UPDATE work_tasks_v2 SET status='completed'")
|
||||
# Bounded keyset scan wraps on the next maintenance pass.
|
||||
outcomes=[api.maintain_retention()['payloads_purged']for _ in range(2)]
|
||||
assert sum(outcomes)==1
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Opt-in isolated-schema PostgreSQL race test; never uses production config."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import time
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
import psycopg
|
||||
from psycopg.conninfo import make_conninfo
|
||||
from relay.store import Store
|
||||
from relay.binding_state import BindingState
|
||||
from relay.credential_store import CredentialKeyring
|
||||
from relay.approval_store import ApprovalStore,VerifiedDecision,ApprovalError
|
||||
from relay.crypto import canonical
|
||||
|
||||
pytestmark=pytest.mark.skipif(not os.environ.get('HN_TEST_DATABASE_URL'),reason='isolated PostgreSQL test URL not supplied')
|
||||
|
||||
|
||||
@pytest.mark.parametrize('winner',['race','allow','deny'])
|
||||
@pytest.mark.parametrize('consumers',[4,16])
|
||||
def test_concurrent_postgres_decisions_consumption_and_reopen(tmp_path,winner,consumers):
|
||||
schema='approval_test_'+secrets.token_hex(10)
|
||||
target=os.environ['HN_TEST_DATABASE_URL']
|
||||
control=psycopg.connect(target,autocommit=True)
|
||||
control.execute('CREATE SCHEMA '+schema)
|
||||
dsn=make_conninfo(target,options='-c search_path='+schema,connect_timeout=5)
|
||||
keyring=CredentialKeyring('synthetic',{ 'synthetic':secrets.token_bytes(32)})
|
||||
connections=[]
|
||||
def connect():
|
||||
store=Store(tmp_path/'unused.sqlite',dsn)
|
||||
store.db.execute("SET lock_timeout TO '2s'")
|
||||
store.db.execute("SET statement_timeout TO '3s'")
|
||||
state=BindingState(store,'test','cli_pg_test')
|
||||
return store,state,ApprovalStore(state,keyring)
|
||||
try:
|
||||
store,state,approvals=connect();connections.append(store)
|
||||
with store.db:
|
||||
store.db.execute("INSERT INTO devices VALUES(?,?,?,?,?,?)",('a'*32,'synthetic-cert','synthetic-fingerprint','ou_pg',0,1))
|
||||
state.create_task('pg-task','ou_pg','a'*32,1,'session','worker',time.time()+300,'oc_pg','om_pg_source')
|
||||
action={'kind':'synthetic','value':'no real side effect'}
|
||||
item=approvals.create('pg-task','ou_pg','a'*32,1,'pg-operation',action,'合成并发确认','不执行真实业务')
|
||||
material=approvals.card_material(item['approval_id']);assert approvals.claim_card(item['approval_id'])
|
||||
approvals.bind_card(item['approval_id'],'om_pg_card')
|
||||
def make_decision(choice):
|
||||
fields=dict(environment='test',app_id='cli_pg_test',actor_open_id='ou_pg',chat_id='oc_pg',
|
||||
card_message_id='om_pg_card',approval_id=item['approval_id'],nonce=material['nonce'],choice=choice)
|
||||
return VerifiedDecision(**fields,event_id=secrets.token_hex(16),event_digest=hashlib.sha256(canonical(fields)).hexdigest())
|
||||
if winner!='race':approvals.decide(make_decision(winner))
|
||||
decision_gate=threading.Barrier(2,timeout=10)
|
||||
def decide(choice):
|
||||
connection,_,ledger=connect()
|
||||
try:
|
||||
decision_gate.wait()
|
||||
try:return ledger.decide(make_decision(choice))['state']
|
||||
except ApprovalError:return 'conflict'
|
||||
finally:connection.close()
|
||||
with ThreadPoolExecutor(max_workers=2)as executor:
|
||||
decisions=list(executor.map(decide,('allow','deny')))
|
||||
row=store.db.execute('SELECT state FROM work_approvals_v3').fetchone()
|
||||
assert row['state']in {'approved','rejected'}and decisions.count('conflict')==1
|
||||
if winner=='allow':
|
||||
# A real PostgreSQL statement error must roll back the earlier
|
||||
# consumed UPDATE in the same transaction, not merely hide a reply.
|
||||
store.db.execute("""CREATE FUNCTION fail_approval_audit() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN IF NEW.event='consumed' THEN RAISE EXCEPTION 'synthetic audit storage failure';
|
||||
END IF; RETURN NEW; END $$""")
|
||||
store.db.execute('''CREATE TRIGGER synthetic_audit_failure BEFORE INSERT ON work_approval_audit_v1
|
||||
FOR EACH ROW EXECUTE FUNCTION fail_approval_audit()''')
|
||||
with pytest.raises(psycopg.Error):
|
||||
approvals.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])
|
||||
assert store.db.execute('SELECT state FROM work_approvals_v3').fetchone()['state']=='approved'
|
||||
assert not store.db.execute("SELECT * FROM work_approval_audit_v1 WHERE event='consumed'").fetchall()
|
||||
store.db.execute('DROP TRIGGER synthetic_audit_failure ON work_approval_audit_v1')
|
||||
store.db.execute('DROP FUNCTION fail_approval_audit()')
|
||||
consume_gate=threading.Barrier(consumers,timeout=20)
|
||||
def consume(_):
|
||||
connection,_,ledger=connect()
|
||||
try:
|
||||
consume_gate.wait()
|
||||
return ledger.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])['execute']
|
||||
finally:connection.close()
|
||||
with ThreadPoolExecutor(max_workers=consumers)as executor:executed=list(executor.map(consume,range(consumers)))
|
||||
assert sum(executed)==(1 if row['state']=='approved'else 0)
|
||||
restored,_,ledger=connect();connections.append(restored)
|
||||
assert ledger.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])['execute']is False
|
||||
assert store.db.execute("SELECT COUNT(*) n FROM work_approval_audit_v1 WHERE event IN ('approved','rejected')").fetchone()['n']==1
|
||||
state.create_task('pg-intent-task','ou_pg','a'*32,1,'session','worker',time.time()+300,'oc_pg','om_pg_intent')
|
||||
intent_gate=threading.Barrier(2,timeout=10)
|
||||
def create_same_intent(operation):
|
||||
connection,_,ledger=connect()
|
||||
try:
|
||||
intent_gate.wait()
|
||||
try:
|
||||
ledger.create('pg-intent-task','ou_pg','a'*32,1,operation,action,'相同合成动作','不执行真实业务')
|
||||
return 'created'
|
||||
except ApprovalError:return 'conflict'
|
||||
finally:connection.close()
|
||||
with ThreadPoolExecutor(max_workers=2)as executor:
|
||||
outcomes=list(executor.map(create_same_intent,('pg-intent-one','pg-intent-two')))
|
||||
assert outcomes.count('created')==1 and outcomes.count('conflict')==1
|
||||
assert store.db.execute("SELECT COUNT(*) n FROM work_approvals_v3 WHERE task_id='pg-intent-task'").fetchone()['n']==1
|
||||
finally:
|
||||
for connection in connections:connection.close()
|
||||
# The unique generated test schema is the only deletion target.
|
||||
control.execute('DROP SCHEMA '+schema+' CASCADE');control.close()
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Adversarial M6 checks, no real platform operations."""
|
||||
import hashlib
|
||||
import json
|
||||
import pytest
|
||||
from test_credential_api import env,wire
|
||||
from test_approval_integration import approval_env,pair,pc_errors
|
||||
from relay.approval_store import ApprovalStore,ApprovalError
|
||||
from relay.approval_policy import describe_action
|
||||
from relay.crypto import canonical
|
||||
|
||||
|
||||
@pytest.mark.parametrize('choice',[None,'deny','allow'])
|
||||
async def test_same_intent_cannot_get_new_card_by_changing_operation_id(pair,choice):
|
||||
p=pair;job=p['launch']();row=await p['card']()
|
||||
if choice:
|
||||
await p['click'](choice)
|
||||
if choice=='deny':
|
||||
with pytest.raises(pc_errors.WorkError):await job
|
||||
else:await job
|
||||
e=p['env'];ledger=ApprovalStore(e['store'].lifecycle,e['api'].broker.store.keyring)
|
||||
action=json.loads(ledger.keyring.decrypt(row['key_id'],row['ciphertext'],ledger._aad(row)))['action']
|
||||
other='new-operation-id'
|
||||
action['params']['idempotency_key']=hashlib.sha256(canonical(['ou_a',other])).hexdigest()
|
||||
path='/work/v2/approvals/create'
|
||||
response=await e['client'].post(path,json=wire(e,path,task_id='task-approval',operation_id=other,action=action,ttl=120))
|
||||
assert response.status_code==409
|
||||
assert e['store'].db.execute('SELECT COUNT(*) n FROM work_approvals_v3').fetchone()['n']==1
|
||||
assert len(p['calls'])==(1 if choice=='allow'else 0)
|
||||
if not choice:
|
||||
p['stop'].set()
|
||||
with pytest.raises(pc_errors.WorkError):await job
|
||||
|
||||
|
||||
def test_confirmation_identifies_actual_calendar_not_just_events_suffix():
|
||||
def action(calendar):return {'kind':'work-api-v1','method':'POST',
|
||||
'path':'/open-apis/calendar/v4/calendars/'+calendar+'/events','params':{},'body':{'summary':'同一标题'}}
|
||||
first=describe_action(action('calendar_a'))
|
||||
second=describe_action(action('calendar_b'))
|
||||
assert 'calendar_a'in first[1]and 'calendar_b'in second[1]and first!=second
|
||||
|
||||
|
||||
def test_long_target_is_rejected_instead_of_silently_truncated():
|
||||
with pytest.raises(ApprovalError):describe_action({'kind':'work-api-v1','method':'DELETE',
|
||||
'path':'/open-apis/drive/v1/files/'+'x'*400,'params':{},'body':{}})
|
||||
|
||||
|
||||
async def test_changed_action_same_id_rejected_but_distinct_new_action_can_confirm(pair):
|
||||
p=pair;job=p['launch']();row=await p['card']();await p['click']('deny')
|
||||
with pytest.raises(pc_errors.WorkError):await job
|
||||
e=p['env'];ledger=ApprovalStore(e['store'].lifecycle,e['api'].broker.store.keyring)
|
||||
action=json.loads(ledger.keyring.decrypt(row['key_id'],row['ciphertext'],ledger._aad(row)))['action']
|
||||
action['body']['summary']='真正不同的合成动作'
|
||||
path='/work/v2/approvals/create'
|
||||
same=await e['client'].post(path,json=wire(e,path,task_id='task-approval',
|
||||
operation_id='operation-synthetic',action=action,ttl=120))
|
||||
assert same.status_code==409
|
||||
other='distinct-operation'
|
||||
action['params']['idempotency_key']=hashlib.sha256(canonical(['ou_a',other])).hexdigest()
|
||||
distinct=await e['client'].post(path,json=wire(e,path,task_id='task-approval',
|
||||
operation_id=other,action=action,ttl=120))
|
||||
assert distinct.status_code==200 and distinct.json()['state']=='pending'
|
||||
assert not p['calls']
|
||||
@@ -698,12 +698,35 @@ def test_approval_update_preserves_runtime_ancestry_and_original_longconn_rollba
|
||||
assert plan['longconn_base_image']==scenario['roles']['longconn']['image']
|
||||
assert candidate['services']['longconn']['image']!=candidate['services']['connector']['image']
|
||||
assert candidate['services']['longconn']['environment']['HN_CREDENTIAL_MODE']=='broker'
|
||||
assert candidate['services']['connector']['environment']['HN_APPROVAL_RETENTION_ENABLED']=='false'
|
||||
assert rollback['services']['longconn']['image']==scenario['roles']['longconn']['image']
|
||||
assert rollback['services']['longconn']['environment']['HN_CREDENTIAL_MODE']=='legacy'
|
||||
assert rollback['services']['longconn']['environment']['HN_WORK_REMOTE_APPROVALS_ENABLED']=='false'
|
||||
assert (root/'credential-keyring.json').read_bytes()==(Path(args.plan)/'credential-keyring.json').read_bytes()
|
||||
|
||||
|
||||
def test_second_approval_update_uses_active_parent_images_and_retains_off(scenario):
|
||||
args=approval_update(scenario);deploy.prepare_update(args)
|
||||
root=Path(args.output);parent=deploy.load_plan(root)
|
||||
connector='sha256:'+'c'*64;longconn='sha256:'+'d'*64
|
||||
for name in ('connector','router'):scenario['roles'][name]['image']=connector
|
||||
scenario['roles']['longconn']['image']=longconn
|
||||
deploy.write_json(root/'candidate.json',{'image':connector,'tag':parent['candidate_image']})
|
||||
deploy.write_json(root/'candidate-longconn.json',{'image':longconn,'tag':parent['longconn_candidate_image']})
|
||||
output=root.parent/'approvals-m6-fix'
|
||||
deploy.prepare_update(argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output),
|
||||
enable_remote_approvals=True,callback_identity=args.callback_identity))
|
||||
result=deploy.load_plan(output)
|
||||
candidate=json.loads((output/'compose.broker.json').read_text())
|
||||
rollback=json.loads((output/'compose.rollback.json').read_text())
|
||||
assert result['base_image']==connector and result['longconn_base_image']==longconn
|
||||
assert candidate['services']['connector']['environment']['HN_APPROVAL_RETENTION_ENABLED']=='false'
|
||||
assert rollback['services']['connector']['image']==connector
|
||||
assert rollback['services']['longconn']['image']==longconn
|
||||
assert rollback['services']['longconn']['environment']['HN_WORK_REMOTE_APPROVALS_ENABLED']=='true'
|
||||
assert (output/'credential-keyring.json').read_bytes()==(root/'credential-keyring.json').read_bytes()
|
||||
|
||||
|
||||
def test_approval_runtime_checks_three_roles_and_sdk_without_network(scenario):
|
||||
args=approval_update(scenario);deploy.prepare_update(args);root=Path(args.output);plan=deploy.load_plan(root)
|
||||
scenario['calls'].clear();deploy.runtime_check(root,plan)
|
||||
|
||||
Reference in New Issue
Block a user