feat(credentials): add stage 2 centralized broker and refresh fencing

Add an internal CredentialBroker that serializes refresh and authorization
by environment/app/user using renewable Redis leases. Re-read credentials
under the lease and persist rotated pairs with expected-version CAS before
returning an explicit access-only grant. Verify the actual platform user,
remaining lifetime and scopes; never return refresh tokens from the broker.

Persist source-version refresh attempts before platform calls. Timeouts,
cancellation, lease loss, malformed responses and ambiguous database errors
fence reuse of the old rotating refresh token, including after restart.
A new authorization can establish a new version without stale overwrite.

Add a strict, separate private Work v1 OIDC adapter while preserving legacy
OAuth method behavior. Enable fakeredis Lua test support for Redis lease
renewal tests. Document stage boundaries, recovery limitations and testing.

Validation: 147 passed, 2 skipped (real PostgreSQL not configured), with
15 existing uvicorn/websockets deprecation warnings. New broker/adapter
coverage: 45 tests. AST, runtime opt-in boundary and diff checks pass.
An earlier Windows temp-file replace failure in a legacy client test passed
both isolated rerun and the final full suite; no client code was changed.

No public credential/OAuth routes are wired, no production database/key is
initialized, and no real Work authorization, refresh or ECS deployment was
performed. Authenticated access-token issuance, state transactions, binding
checks and unbind integration remain subsequent stages.
This commit is contained in:
ex_tianzy
2026-11-04 10:30:27 +08:00
parent 71b9678d22
commit 02d433250f
8 changed files with 788 additions and 5 deletions
+454
View File
@@ -0,0 +1,454 @@
"""Synthetic credentials only: central refresh fencing and access-only boundary."""
import asyncio
import dataclasses
import json
import os
import sys
from pathlib import Path
import fakeredis
import fakeredis.aioredis
import httpx
import pytest
import pytest_asyncio
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from relay.credential_broker import (AuthorizationRequired, BrokerUnavailable, CredentialBroker,
PermissionRequired, RefreshUnknown)
from relay.credential_store import CredentialKeyring, CredentialStore
from relay.platform import Feishu, PlatformError
from relay.config import Settings
from relay.store import Store
NOW = 2000000000
def response(suffix="new", **changes):
return {**dict(user_access_token="fake-access-" + suffix, refresh_token="fake-refresh-" + suffix,
expires_in=3600, refresh_expires_in=86400, scope="read write"), **changes}
class Platform:
def __init__(self):
self.calls = 0
self.exchanges = 0
self.result = response()
self.owner = "alice"
self.action = None
self.delay = 0
self.user_info_error = False
async def credential_refresh(self, token):
assert token.startswith("fake-refresh-")
self.calls += 1
if self.delay:
await asyncio.sleep(self.delay)
if self.action:
await self.action()
return self.result
async def credential_exchange(self, code):
self.exchanges += 1
if self.delay:
await asyncio.sleep(self.delay)
if self.action:
await self.action()
return self.result
async def user_info(self, token):
if self.user_info_error:
raise RuntimeError("fake-access-secret-in-upstream-error")
return {"open_id": self.owner}
@pytest_asyncio.fixture
async def setup(tmp_path):
db = Store(tmp_path / "metadata.sqlite")
vault = CredentialStore(db, CredentialKeyring("test-key", {"test-key": os.urandom(32)}))
redis = fakeredis.aioredis.FakeRedis(server=fakeredis.FakeServer(), decode_responses=True)
platform = Platform()
broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app",
clock=lambda: NOW, lock_wait_seconds=1)
yield db, vault, redis, platform, broker
await redis.aclose()
db.close()
def seed(vault, *, owner="alice", app="app", environment="test", remaining=30, refresh_remaining=86400):
return vault.save(environment, app, owner, expected_version=0, access_token="fake-access-old",
refresh_token="fake-refresh-old", access_expires_at=NOW + remaining,
refresh_expires_at=NOW + refresh_remaining, scopes=["read", "write"])
async def test_access_only_remaining_lifetime_and_scope(setup):
db, vault, redis, platform, broker = setup
seed(vault, remaining=601)
grant = await broker.get_access("alice", required_scopes=["read"])
assert grant.expires_in == 601 and grant.credential_version == 1
assert set(dataclasses.asdict(grant)) == {"access_token", "expires_at", "expires_in", "credential_version", "scopes"}
assert "fake-access" not in repr(grant)
assert not hasattr(grant, "refresh_token") and platform.calls == 0
with pytest.raises(PermissionRequired):
await broker.get_access("alice", required_scopes=["admin"])
assert platform.calls == 0
async def test_missing_expired_and_cross_identity(setup):
_, vault, redis, platform, broker = setup
seed(vault, refresh_remaining=-1)
for candidate, owner in [(broker, "bob"), (broker, "alice"),
(CredentialBroker(vault, redis, platform, environment="production", app_id="app", clock=lambda: NOW), "alice"),
(CredentialBroker(vault, redis, platform, environment="test", app_id="other", clock=lambda: NOW), "alice")]:
with pytest.raises(AuthorizationRequired):
await candidate.get_access(owner)
assert platform.calls == 0
async def test_many_instances_share_one_refresh(setup):
_, vault, redis, platform, broker = setup
seed(vault)
platform.delay = 0.05
peers = [CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW,
lock_wait_seconds=2) for _ in range(8)]
results = await asyncio.gather(*(peer.get_access("alice") for peer in peers))
assert platform.calls == 1
assert all(item.credential_version == 2 for item in results)
assert vault.get("test", "app", "alice").refresh_token == "fake-refresh-new"
assert vault.refresh_status("test", "app", "alice", 1) == "completed"
async def test_separate_redis_clients_and_database_connections(setup, tmp_path):
db, vault, redis, platform, broker = setup
seed(vault)
other_db = Store(tmp_path / "metadata.sqlite")
other_redis = fakeredis.aioredis.FakeRedis(server=redis.connection_pool.connection_kwargs["server"],
decode_responses=True)
try:
peer = CredentialBroker(CredentialStore(other_db, vault.keyring), other_redis, platform,
environment="test", app_id="app", clock=lambda: NOW)
platform.delay = 0.05
first, second = await asyncio.gather(broker.get_access("alice"), peer.get_access("alice"))
assert platform.calls == 1 and first == second
finally:
other_db.close()
await other_redis.aclose()
async def test_authorization_checks_actual_user_and_no_refresh_output(setup):
_, vault, _, platform, broker = setup
platform.owner = "bob"
with pytest.raises(BrokerUnavailable, match="authorization did not complete"):
await broker.authorize("alice", "fake-code")
assert vault.get("test", "app", "alice") is None
platform.owner = "alice"
grant = await broker.authorize("alice", "fake-code")
assert grant.credential_version == 1 and not hasattr(grant, "refresh_token")
@pytest.mark.parametrize("changes", [dict(refresh_token=""), dict(expires_in=None),
dict(refresh_expires_in=0), dict(expires_in=True), dict(expires_in=float("nan")),
dict(scope=[1]), dict(user_access_token=""), dict(refresh_expires_in=10**400)])
async def test_bad_refresh_response_is_fenced(setup, changes):
_, vault, _, platform, broker = setup
seed(vault)
platform.result = response(**changes)
with pytest.raises(RefreshUnknown):
await broker.get_access("alice")
assert vault.get("test", "app", "alice").version == 1
assert vault.refresh_status("test", "app", "alice", 1) == "unknown"
with pytest.raises(RefreshUnknown):
await broker.get_access("alice")
assert platform.calls == 1
@pytest.mark.parametrize("failure", ["timeout", "identity", "userinfo", "save", "lost-lease"])
async def test_refresh_failures_do_not_reuse_old_refresh(setup, monkeypatch, failure):
_, vault, redis, platform, broker = setup
seed(vault)
async def action():
if failure == "timeout":
raise httpx.ReadTimeout("fake-refresh-private-error")
if failure == "lost-lease":
await redis.delete(broker._lock_key("alice"))
platform.action = action
if failure == "identity":
platform.owner = "bob"
if failure == "userinfo":
platform.user_info_error = True
if failure == "save":
def broken(*args, **kwargs):
raise OSError("fake-refresh-private-error")
monkeypatch.setattr(vault, "save", broken)
with pytest.raises(RefreshUnknown) as error:
await broker.get_access("alice")
assert "fake-" not in str(error.value)
with pytest.raises(RefreshUnknown):
await broker.get_access("alice")
assert platform.calls == 1
async def test_crash_started_marker_survives_restart_and_new_authorization_recovers(setup, tmp_path):
_, vault, redis, platform, broker = setup
seed(vault)
assert vault.begin_refresh("test", "app", "alice", 1, "crashed-attempt")
other = Store(tmp_path / "metadata.sqlite")
try:
peer = CredentialBroker(CredentialStore(other, vault.keyring), redis, platform,
environment="test", app_id="app", clock=lambda: NOW)
with pytest.raises(RefreshUnknown):
await peer.get_access("alice")
assert platform.calls == 0
grant = await peer.authorize("alice", "new-test-code")
assert grant.credential_version == 2
assert (await broker.get_access("alice")).credential_version == 2
finally:
other.close()
async def test_refresh_and_authorization_share_lock_and_latest_pair_wins(setup):
_, vault, _, platform, broker = setup
seed(vault)
platform.delay = 0.05
refresh = asyncio.create_task(broker.get_access("alice"))
while platform.calls == 0:
await asyncio.sleep(0.001)
authorize = asyncio.create_task(broker.authorize("alice", "new-test-code"))
refreshed, authorized = await asyncio.gather(refresh, authorize)
assert refreshed.credential_version == 2 and authorized.credential_version == 3
assert vault.get("test", "app", "alice").version == 3
async def test_stale_external_writer_not_overwritten(setup):
_, vault, _, platform, broker = setup
seed(vault)
async def external_write():
vault.save("test", "app", "alice", expected_version=1, access_token="fake-access-reauthed",
refresh_token="fake-refresh-reauthed", access_expires_at=NOW + 3600,
refresh_expires_at=NOW + 86400, scopes=["read"])
platform.action = external_write
with pytest.raises(RefreshUnknown):
await broker.get_access("alice")
assert vault.get("test", "app", "alice").refresh_token == "fake-refresh-reauthed"
async def test_authorization_cas_does_not_overwrite_external_replacement(setup):
_, vault, _, platform, broker = setup
seed(vault)
async def external_write():
vault.save("test", "app", "alice", expected_version=1, access_token="fake-access-other",
refresh_token="fake-refresh-other", access_expires_at=NOW + 3600,
refresh_expires_at=NOW + 86400, scopes=["read"])
platform.action = external_write
with pytest.raises(BrokerUnavailable, match="version changed"):
await broker.authorize("alice", "synthetic-code")
assert vault.get("test", "app", "alice").refresh_token == "fake-refresh-other"
async def test_renewal_keeps_lease_during_slow_platform_call(setup):
_, vault, redis, platform, _ = setup
seed(vault)
broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW,
lease_seconds=0.3, lock_wait_seconds=2)
platform.delay = 0.7
results = await asyncio.gather(broker.get_access("alice"), broker.get_access("alice"))
assert platform.calls == 1 and all(r.credential_version == 2 for r in results)
async def test_cancellation_and_timeout_fence_rotation(setup):
_, vault, _, platform, broker = setup
seed(vault)
platform.delay = 2
task = asyncio.create_task(broker.get_access("alice"))
while platform.calls == 0:
await asyncio.sleep(0.001)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
with pytest.raises(RefreshUnknown):
await broker.get_access("alice")
assert platform.calls == 1
async def test_platform_deadline_fences_refresh(setup):
_, vault, redis, platform, _ = setup
seed(vault)
platform.delay = 0.2
broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW,
platform_timeout=0.02)
with pytest.raises(RefreshUnknown):
await broker.get_access("alice")
assert vault.refresh_status("test", "app", "alice", 1) == "unknown"
with pytest.raises(RefreshUnknown):
await broker.get_access("alice")
assert platform.calls == 1
async def test_renewal_failure_prevents_publishing_result(setup, monkeypatch):
_, vault, redis, platform, _ = setup
seed(vault)
original = redis.lock
def failing_lock(*args, **kwargs):
lock = original(*args, **kwargs)
async def fail(*args, **kwargs):
raise ConnectionError("synthetic renew failure")
lock.extend = fail
return lock
monkeypatch.setattr(redis, "lock", failing_lock)
platform.delay = 0.2
broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW,
lease_seconds=0.15)
with pytest.raises(RefreshUnknown):
await broker.get_access("alice")
assert vault.get("test", "app", "alice").version == 1
assert vault.refresh_status("test", "app", "alice", 1) == "unknown"
async def test_failed_unknown_write_keeps_started_marker_as_crash_fence(setup, monkeypatch):
_, vault, _, platform, broker = setup
seed(vault)
async def fail():
raise httpx.ReadTimeout("synthetic upstream failure")
def broken(*args, **kwargs):
raise OSError("synthetic DB failure")
platform.action = fail
monkeypatch.setattr(vault, "finish_refresh", broken)
with pytest.raises(RefreshUnknown):
await broker.get_access("alice")
assert vault.refresh_status("test", "app", "alice", 1) == "started"
with pytest.raises(RefreshUnknown):
await broker.get_access("alice")
assert platform.calls == 1
async def test_committed_pair_not_lost_if_database_ack_raises(setup, monkeypatch):
_, vault, _, platform, broker = setup
seed(vault)
original = vault.save
def commit_then_fail(*args, **kwargs):
original(*args, **kwargs)
raise OSError("synthetic post-commit disconnect")
monkeypatch.setattr(vault, "save", commit_then_fail)
with pytest.raises(RefreshUnknown):
await broker.get_access("alice")
grant = await broker.get_access("alice")
assert grant.credential_version == 2 and platform.calls == 1
async def test_missing_scope_during_refresh_retains_verified_scopes(setup):
_, vault, _, platform, broker = setup
seed(vault)
del platform.result["scope"]
grant = await broker.get_access("alice", required_scopes=["read"])
assert grant.scopes == ("read", "write")
async def test_refresh_with_reduced_scopes_persists_pair_but_denies_missing_capability(setup):
_, vault, _, platform, broker = setup
seed(vault)
platform.result["scope"] = "read"
with pytest.raises(PermissionRequired):
await broker.get_access("alice", required_scopes=["write"])
saved = vault.get("test", "app", "alice")
assert saved.version == 2 and saved.refresh_token == "fake-refresh-new"
assert (await broker.get_access("alice", required_scopes=["read"])).scopes == ("read",)
assert platform.calls == 1
async def test_authorization_requires_scope_and_preserves_existing_on_failure(setup):
_, vault, _, platform, broker = setup
seed(vault)
del platform.result["scope"]
with pytest.raises(BrokerUnavailable):
await broker.authorize("alice", "fake-code")
assert vault.get("test", "app", "alice").version == 1
async def test_successful_persistence_survives_failed_completion_marker(setup, monkeypatch):
_, vault, _, platform, broker = setup
seed(vault)
def broken(*args, **kwargs):
raise OSError("synthetic database error")
monkeypatch.setattr(vault, "finish_refresh", broken)
grant = await broker.get_access("alice")
assert (await broker.get_access("alice")) == grant
assert platform.calls == 1
async def test_distributed_lock_busy_and_redis_failure_fail_closed(setup):
_, vault, redis, platform, _ = setup
seed(vault)
broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW,
lock_wait_seconds=0.05)
lock = redis.lock(broker._lock_key("alice"), timeout=5, thread_local=False)
await lock.acquire()
try:
with pytest.raises(BrokerUnavailable):
await broker.get_access("alice")
finally:
await lock.release()
assert platform.calls == 0
async def test_redis_unavailable_does_not_call_platform(setup, monkeypatch):
_, vault, redis, platform, broker = setup
seed(vault)
original = redis.lock
def failing_lock(*args, **kwargs):
lock = original(*args, **kwargs)
async def fail(*args, **kwargs):
raise ConnectionError("synthetic redis failure")
lock.acquire = fail
return lock
monkeypatch.setattr(redis, "lock", failing_lock)
with pytest.raises(BrokerUnavailable):
await broker.get_access("alice")
assert platform.calls == 0
@pytest.mark.parametrize("bad", [0, -1, True, float("inf")])
def test_invalid_configuration_rejected(bad):
# Constructor validation is independent of any network or key material.
with pytest.raises(Exception, match="invalid broker configuration"):
CredentialBroker(None, None, None, environment="test", app_id="app", lease_seconds=bad)
@pytest.mark.parametrize("kind", ["exchange", "refresh"])
async def test_private_oidc_adapter_and_scope_preservation(kind):
calls = []
async def handler(request):
calls.append((request.url.path, json.loads(request.content)))
if request.url.path.endswith("app_access_token/internal"):
return httpx.Response(200, json={"code": 0, "app_access_token": "fake-app-token", "expire": 7200})
return httpx.Response(200, json={"code": 0, "data": {
"access_token": "fake-access", "refresh_token": "fake-refresh", "expire": 3600,
"refresh_expires_in": 86400, "scope": "read"}})
settings = Settings(environment="test", admin_secret="a"*40, internal_secret="b"*40)
platform = Feishu(settings, transport=httpx.MockTransport(handler))
try:
method = platform.credential_exchange if kind == "exchange" else platform.credential_refresh
result = await method("synthetic-input")
assert result["scope"] == "read" and result["expires_in"] == 3600
expected_path = "access_token" if kind == "exchange" else "refresh_access_token"
assert calls[-1][0] == "/open-apis/authen/v1/oidc/" + expected_path
assert calls[-1][1]["grant_type"] == ("authorization_code" if kind == "exchange" else "refresh_token")
finally:
await platform.close()
@pytest.mark.parametrize("status,body", [(500, {"code": 0}), (200, {"data": {}}),
(200, {"code": True, "data": {}}), (200, {"code": 20026, "msg": "fake-secret"}),
(200, {"code": 0, "data": {"access_token": "fake-access"}})])
async def test_adapter_rejects_error_contract_without_leaking_body(status, body):
async def handler(request):
if request.url.path.endswith("app_access_token/internal"):
return httpx.Response(200, json={"code": 0, "app_access_token": "fake-app-token", "expire": 7200})
return httpx.Response(status, json=body)
platform = Feishu(Settings(environment="test", admin_secret="a"*40, internal_secret="b"*40),
transport=httpx.MockTransport(handler))
try:
with pytest.raises(PlatformError) as error:
await platform.credential_refresh("fake-refresh")
assert str(error.value) == "credential platform request failed"
finally:
await platform.close()