110 lines
6.4 KiB
Python
110 lines
6.4 KiB
Python
"""Opt-in isolated-schema PostgreSQL race test; never uses production config."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import secrets
|
|
import time
|
|
import threading
|
|
|
|
import pytest
|
|
import psycopg
|
|
from psycopg.conninfo import make_conninfo
|
|
from relay.store import Store
|
|
from relay.binding_state import BindingState
|
|
from relay.credential_store import CredentialKeyring
|
|
from relay.approval_store import ApprovalStore,VerifiedDecision,ApprovalError
|
|
from relay.crypto import canonical
|
|
|
|
pytestmark=pytest.mark.skipif(not os.environ.get('HN_TEST_DATABASE_URL'),reason='isolated PostgreSQL test URL not supplied')
|
|
|
|
|
|
@pytest.mark.parametrize('winner',['race','allow','deny'])
|
|
@pytest.mark.parametrize('consumers',[4,16])
|
|
def test_concurrent_postgres_decisions_consumption_and_reopen(tmp_path,winner,consumers):
|
|
schema='approval_test_'+secrets.token_hex(10)
|
|
target=os.environ['HN_TEST_DATABASE_URL']
|
|
control=psycopg.connect(target,autocommit=True)
|
|
control.execute('CREATE SCHEMA '+schema)
|
|
dsn=make_conninfo(target,options='-c search_path='+schema,connect_timeout=5)
|
|
keyring=CredentialKeyring('synthetic',{ 'synthetic':secrets.token_bytes(32)})
|
|
connections=[]
|
|
def connect():
|
|
store=Store(tmp_path/'unused.sqlite',dsn)
|
|
store.db.execute("SET lock_timeout TO '2s'")
|
|
store.db.execute("SET statement_timeout TO '3s'")
|
|
state=BindingState(store,'test','cli_pg_test')
|
|
return store,state,ApprovalStore(state,keyring)
|
|
try:
|
|
store,state,approvals=connect();connections.append(store)
|
|
with store.db:
|
|
store.db.execute("INSERT INTO devices VALUES(?,?,?,?,?,?)",('a'*32,'synthetic-cert','synthetic-fingerprint','ou_pg',0,1))
|
|
state.create_task('pg-task','ou_pg','a'*32,1,'session','worker',time.time()+300,'oc_pg','om_pg_source')
|
|
action={'kind':'synthetic','value':'no real side effect'}
|
|
item=approvals.create('pg-task','ou_pg','a'*32,1,'pg-operation',action,'合成并发确认','不执行真实业务')
|
|
material=approvals.card_material(item['approval_id']);assert approvals.claim_card(item['approval_id'])
|
|
approvals.bind_card(item['approval_id'],'om_pg_card')
|
|
def make_decision(choice):
|
|
fields=dict(environment='test',app_id='cli_pg_test',actor_open_id='ou_pg',chat_id='oc_pg',
|
|
card_message_id='om_pg_card',approval_id=item['approval_id'],nonce=material['nonce'],choice=choice)
|
|
return VerifiedDecision(**fields,event_id=secrets.token_hex(16),event_digest=hashlib.sha256(canonical(fields)).hexdigest())
|
|
if winner!='race':approvals.decide(make_decision(winner))
|
|
decision_gate=threading.Barrier(2,timeout=10)
|
|
def decide(choice):
|
|
connection,_,ledger=connect()
|
|
try:
|
|
decision_gate.wait()
|
|
try:return ledger.decide(make_decision(choice))['state']
|
|
except ApprovalError:return 'conflict'
|
|
finally:connection.close()
|
|
with ThreadPoolExecutor(max_workers=2)as executor:
|
|
decisions=list(executor.map(decide,('allow','deny')))
|
|
row=store.db.execute('SELECT state FROM work_approvals_v3').fetchone()
|
|
assert row['state']in {'approved','rejected'}and decisions.count('conflict')==1
|
|
if winner=='allow':
|
|
# A real PostgreSQL statement error must roll back the earlier
|
|
# consumed UPDATE in the same transaction, not merely hide a reply.
|
|
store.db.execute("""CREATE FUNCTION fail_approval_audit() RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN IF NEW.event='consumed' THEN RAISE EXCEPTION 'synthetic audit storage failure';
|
|
END IF; RETURN NEW; END $$""")
|
|
store.db.execute('''CREATE TRIGGER synthetic_audit_failure BEFORE INSERT ON work_approval_audit_v1
|
|
FOR EACH ROW EXECUTE FUNCTION fail_approval_audit()''')
|
|
with pytest.raises(psycopg.Error):
|
|
approvals.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])
|
|
assert store.db.execute('SELECT state FROM work_approvals_v3').fetchone()['state']=='approved'
|
|
assert not store.db.execute("SELECT * FROM work_approval_audit_v1 WHERE event='consumed'").fetchall()
|
|
store.db.execute('DROP TRIGGER synthetic_audit_failure ON work_approval_audit_v1')
|
|
store.db.execute('DROP FUNCTION fail_approval_audit()')
|
|
consume_gate=threading.Barrier(consumers,timeout=20)
|
|
def consume(_):
|
|
connection,_,ledger=connect()
|
|
try:
|
|
consume_gate.wait()
|
|
return ledger.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])['execute']
|
|
finally:connection.close()
|
|
with ThreadPoolExecutor(max_workers=consumers)as executor:executed=list(executor.map(consume,range(consumers)))
|
|
assert sum(executed)==(1 if row['state']=='approved'else 0)
|
|
restored,_,ledger=connect();connections.append(restored)
|
|
assert ledger.consume(item['approval_id'],'pg-task','ou_pg','a'*32,1,item['operation_digest'])['execute']is False
|
|
assert store.db.execute("SELECT COUNT(*) n FROM work_approval_audit_v1 WHERE event IN ('approved','rejected')").fetchone()['n']==1
|
|
state.create_task('pg-intent-task','ou_pg','a'*32,1,'session','worker',time.time()+300,'oc_pg','om_pg_intent')
|
|
intent_gate=threading.Barrier(2,timeout=10)
|
|
def create_same_intent(operation):
|
|
connection,_,ledger=connect()
|
|
try:
|
|
intent_gate.wait()
|
|
try:
|
|
ledger.create('pg-intent-task','ou_pg','a'*32,1,operation,action,'相同合成动作','不执行真实业务')
|
|
return 'created'
|
|
except ApprovalError:return 'conflict'
|
|
finally:connection.close()
|
|
with ThreadPoolExecutor(max_workers=2)as executor:
|
|
outcomes=list(executor.map(create_same_intent,('pg-intent-one','pg-intent-two')))
|
|
assert outcomes.count('created')==1 and outcomes.count('conflict')==1
|
|
assert store.db.execute("SELECT COUNT(*) n FROM work_approvals_v3 WHERE task_id='pg-intent-task'").fetchone()['n']==1
|
|
finally:
|
|
for connection in connections:connection.close()
|
|
# The unique generated test schema is the only deletion target.
|
|
control.execute('DROP SCHEMA '+schema+' CASCADE');control.close()
|