fix(work): fence repeated approval intents and audit state changes atomically
This commit is contained in:
@@ -0,0 +1,109 @@
|
||||
"""Opt-in isolated-schema PostgreSQL race test; never uses production config."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import time
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
import psycopg
|
||||
from psycopg.conninfo import make_conninfo
|
||||
from relay.store import Store
|
||||
from relay.binding_state import BindingState
|
||||
from relay.credential_store import CredentialKeyring
|
||||
from relay.approval_store import ApprovalStore,VerifiedDecision,ApprovalError
|
||||
from relay.crypto import canonical
|
||||
|
||||
pytestmark=pytest.mark.skipif(not os.environ.get('HN_TEST_DATABASE_URL'),reason='isolated PostgreSQL test URL not supplied')
|
||||
|
||||
|
||||
@pytest.mark.parametrize('winner',['race','allow','deny'])
|
||||
@pytest.mark.parametrize('consumers',[4,16])
|
||||
def test_concurrent_postgres_decisions_consumption_and_reopen(tmp_path,winner,consumers):
|
||||
schema='approval_test_'+secrets.token_hex(10)
|
||||
target=os.environ['HN_TEST_DATABASE_URL']
|
||||
control=psycopg.connect(target,autocommit=True)
|
||||
control.execute('CREATE SCHEMA '+schema)
|
||||
dsn=make_conninfo(target,options='-c search_path='+schema,connect_timeout=5)
|
||||
keyring=CredentialKeyring('synthetic',{ 'synthetic':secrets.token_bytes(32)})
|
||||
connections=[]
|
||||
def connect():
|
||||
store=Store(tmp_path/'unused.sqlite',dsn)
|
||||
store.db.execute("SET lock_timeout TO '2s'")
|
||||
store.db.execute("SET statement_timeout TO '3s'")
|
||||
state=BindingState(store,'test','cli_pg_test')
|
||||
return store,state,ApprovalStore(state,keyring)
|
||||
try:
|
||||
store,state,approvals=connect();connections.append(store)
|
||||
with store.db:
|
||||
store.db.execute("INSERT INTO devices VALUES(?,?,?,?,?,?)",('a'*32,'synthetic-cert','synthetic-fingerprint','ou_pg',0,1))
|
||||
state.create_task('pg-task','ou_pg','a'*32,1,'session','worker',time.time()+300,'oc_pg','om_pg_source')
|
||||
action={'kind':'synthetic','value':'no real side effect'}
|
||||
item=approvals.create('pg-task','ou_pg','a'*32,1,'pg-operation',action,'合成并发确认','不执行真实业务')
|
||||
material=approvals.card_material(item['approval_id']);assert approvals.claim_card(item['approval_id'])
|
||||
approvals.bind_card(item['approval_id'],'om_pg_card')
|
||||
def make_decision(choice):
|
||||
fields=dict(environment='test',app_id='cli_pg_test',actor_open_id='ou_pg',chat_id='oc_pg',
|
||||
card_message_id='om_pg_card',approval_id=item['approval_id'],nonce=material['nonce'],choice=choice)
|
||||
return VerifiedDecision(**fields,event_id=secrets.token_hex(16),event_digest=hashlib.sha256(canonical(fields)).hexdigest())
|
||||
if winner!='race':approvals.decide(make_decision(winner))
|
||||
decision_gate=threading.Barrier(2,timeout=10)
|
||||
def decide(choice):
|
||||
connection,_,ledger=connect()
|
||||
try:
|
||||
decision_gate.wait()
|
||||
try:return ledger.decide(make_decision(choice))['state']
|
||||
except ApprovalError:return 'conflict'
|
||||
finally:connection.close()
|
||||
with ThreadPoolExecutor(max_workers=2)as executor:
|
||||
decisions=list(executor.map(decide,('allow','deny')))
|
||||
row=store.db.execute('SELECT state FROM work_approvals_v3').fetchone()
|
||||
assert row['state']in {'approved','rejected'}and decisions.count('conflict')==1
|
||||
if winner=='allow':
|
||||
# A real PostgreSQL statement error must roll back the earlier
|
||||
# consumed UPDATE in the same transaction, not merely hide a reply.
|
||||
store.db.execute("""CREATE FUNCTION fail_approval_audit() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN IF NEW.event='consumed' THEN RAISE EXCEPTION 'synthetic audit storage failure';
|
||||
END IF; RETURN NEW; END $$""")
|
||||
store.db.execute('''CREATE TRIGGER synthetic_audit_failure BEFORE INSERT ON work_approval_audit_v1
|
||||
FOR EACH ROW EXECUTE FUNCTION fail_approval_audit()''')
|
||||
with pytest.raises(psycopg.Error):
|
||||
approvals.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])
|
||||
assert store.db.execute('SELECT state FROM work_approvals_v3').fetchone()['state']=='approved'
|
||||
assert not store.db.execute("SELECT * FROM work_approval_audit_v1 WHERE event='consumed'").fetchall()
|
||||
store.db.execute('DROP TRIGGER synthetic_audit_failure ON work_approval_audit_v1')
|
||||
store.db.execute('DROP FUNCTION fail_approval_audit()')
|
||||
consume_gate=threading.Barrier(consumers,timeout=20)
|
||||
def consume(_):
|
||||
connection,_,ledger=connect()
|
||||
try:
|
||||
consume_gate.wait()
|
||||
return ledger.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])['execute']
|
||||
finally:connection.close()
|
||||
with ThreadPoolExecutor(max_workers=consumers)as executor:executed=list(executor.map(consume,range(consumers)))
|
||||
assert sum(executed)==(1 if row['state']=='approved'else 0)
|
||||
restored,_,ledger=connect();connections.append(restored)
|
||||
assert ledger.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])['execute']is False
|
||||
assert store.db.execute("SELECT COUNT(*) n FROM work_approval_audit_v1 WHERE event IN ('approved','rejected')").fetchone()['n']==1
|
||||
state.create_task('pg-intent-task','ou_pg','a'*32,1,'session','worker',time.time()+300,'oc_pg','om_pg_intent')
|
||||
intent_gate=threading.Barrier(2,timeout=10)
|
||||
def create_same_intent(operation):
|
||||
connection,_,ledger=connect()
|
||||
try:
|
||||
intent_gate.wait()
|
||||
try:
|
||||
ledger.create('pg-intent-task','ou_pg','a'*32,1,operation,action,'相同合成动作','不执行真实业务')
|
||||
return 'created'
|
||||
except ApprovalError:return 'conflict'
|
||||
finally:connection.close()
|
||||
with ThreadPoolExecutor(max_workers=2)as executor:
|
||||
outcomes=list(executor.map(create_same_intent,('pg-intent-one','pg-intent-two')))
|
||||
assert outcomes.count('created')==1 and outcomes.count('conflict')==1
|
||||
assert store.db.execute("SELECT COUNT(*) n FROM work_approvals_v3 WHERE task_id='pg-intent-task'").fetchone()['n']==1
|
||||
finally:
|
||||
for connection in connections:connection.close()
|
||||
# The unique generated test schema is the only deletion target.
|
||||
control.execute('DROP SCHEMA '+schema+' CASCADE');control.close()
|
||||
Reference in New Issue
Block a user