Files
ecs-relay-service/deployable/tests/test_approval_postgres.py
T

110 lines
6.4 KiB
Python

"""Opt-in isolated-schema PostgreSQL race test; never uses production config."""
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import os
from pathlib import Path
import secrets
import time
import threading
import pytest
import psycopg
from psycopg.conninfo import make_conninfo
from relay.store import Store
from relay.binding_state import BindingState
from relay.credential_store import CredentialKeyring
from relay.approval_store import ApprovalStore,VerifiedDecision,ApprovalError
from relay.crypto import canonical
pytestmark=pytest.mark.skipif(not os.environ.get('HN_TEST_DATABASE_URL'),reason='isolated PostgreSQL test URL not supplied')
@pytest.mark.parametrize('winner',['race','allow','deny'])
@pytest.mark.parametrize('consumers',[4,16])
def test_concurrent_postgres_decisions_consumption_and_reopen(tmp_path,winner,consumers):
schema='approval_test_'+secrets.token_hex(10)
target=os.environ['HN_TEST_DATABASE_URL']
control=psycopg.connect(target,autocommit=True)
control.execute('CREATE SCHEMA '+schema)
dsn=make_conninfo(target,options='-c search_path='+schema,connect_timeout=5)
keyring=CredentialKeyring('synthetic',{ 'synthetic':secrets.token_bytes(32)})
connections=[]
def connect():
store=Store(tmp_path/'unused.sqlite',dsn)
store.db.execute("SET lock_timeout TO '2s'")
store.db.execute("SET statement_timeout TO '3s'")
state=BindingState(store,'test','cli_pg_test')
return store,state,ApprovalStore(state,keyring)
try:
store,state,approvals=connect();connections.append(store)
with store.db:
store.db.execute("INSERT INTO devices VALUES(?,?,?,?,?,?)",('a'*32,'synthetic-cert','synthetic-fingerprint','ou_pg',0,1))
state.create_task('pg-task','ou_pg','a'*32,1,'session','worker',time.time()+300,'oc_pg','om_pg_source')
action={'kind':'synthetic','value':'no real side effect'}
item=approvals.create('pg-task','ou_pg','a'*32,1,'pg-operation',action,'合成并发确认','不执行真实业务')
material=approvals.card_material(item['approval_id']);assert approvals.claim_card(item['approval_id'])
approvals.bind_card(item['approval_id'],'om_pg_card')
def make_decision(choice):
fields=dict(environment='test',app_id='cli_pg_test',actor_open_id='ou_pg',chat_id='oc_pg',
card_message_id='om_pg_card',approval_id=item['approval_id'],nonce=material['nonce'],choice=choice)
return VerifiedDecision(**fields,event_id=secrets.token_hex(16),event_digest=hashlib.sha256(canonical(fields)).hexdigest())
if winner!='race':approvals.decide(make_decision(winner))
decision_gate=threading.Barrier(2,timeout=10)
def decide(choice):
connection,_,ledger=connect()
try:
decision_gate.wait()
try:return ledger.decide(make_decision(choice))['state']
except ApprovalError:return 'conflict'
finally:connection.close()
with ThreadPoolExecutor(max_workers=2)as executor:
decisions=list(executor.map(decide,('allow','deny')))
row=store.db.execute('SELECT state FROM work_approvals_v3').fetchone()
assert row['state']in {'approved','rejected'}and decisions.count('conflict')==1
if winner=='allow':
# A real PostgreSQL statement error must roll back the earlier
# consumed UPDATE in the same transaction, not merely hide a reply.
store.db.execute("""CREATE FUNCTION fail_approval_audit() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN IF NEW.event='consumed' THEN RAISE EXCEPTION 'synthetic audit storage failure';
END IF; RETURN NEW; END $$""")
store.db.execute('''CREATE TRIGGER synthetic_audit_failure BEFORE INSERT ON work_approval_audit_v1
FOR EACH ROW EXECUTE FUNCTION fail_approval_audit()''')
with pytest.raises(psycopg.Error):
approvals.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])
assert store.db.execute('SELECT state FROM work_approvals_v3').fetchone()['state']=='approved'
assert not store.db.execute("SELECT * FROM work_approval_audit_v1 WHERE event='consumed'").fetchall()
store.db.execute('DROP TRIGGER synthetic_audit_failure ON work_approval_audit_v1')
store.db.execute('DROP FUNCTION fail_approval_audit()')
consume_gate=threading.Barrier(consumers,timeout=20)
def consume(_):
connection,_,ledger=connect()
try:
consume_gate.wait()
return ledger.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])['execute']
finally:connection.close()
with ThreadPoolExecutor(max_workers=consumers)as executor:executed=list(executor.map(consume,range(consumers)))
assert sum(executed)==(1 if row['state']=='approved'else 0)
restored,_,ledger=connect();connections.append(restored)
assert ledger.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])['execute']is False
assert store.db.execute("SELECT COUNT(*) n FROM work_approval_audit_v1 WHERE event IN ('approved','rejected')").fetchone()['n']==1
state.create_task('pg-intent-task','ou_pg','a'*32,1,'session','worker',time.time()+300,'oc_pg','om_pg_intent')
intent_gate=threading.Barrier(2,timeout=10)
def create_same_intent(operation):
connection,_,ledger=connect()
try:
intent_gate.wait()
try:
ledger.create('pg-intent-task','ou_pg','a'*32,1,operation,action,'相同合成动作','不执行真实业务')
return 'created'
except ApprovalError:return 'conflict'
finally:connection.close()
with ThreadPoolExecutor(max_workers=2)as executor:
outcomes=list(executor.map(create_same_intent,('pg-intent-one','pg-intent-two')))
assert outcomes.count('created')==1 and outcomes.count('conflict')==1
assert store.db.execute("SELECT COUNT(*) n FROM work_approvals_v3 WHERE task_id='pg-intent-task'").fetchone()['n']==1
finally:
for connection in connections:connection.close()
# The unique generated test schema is the only deletion target.
control.execute('DROP SCHEMA '+schema+' CASCADE');control.close()