feat(credentials): add stage 2 centralized broker and refresh fencing
Add an internal CredentialBroker that serializes refresh and authorization by environment/app/user using renewable Redis leases. Re-read credentials under the lease and persist rotated pairs with expected-version CAS before returning an explicit access-only grant. Verify the actual platform user, remaining lifetime and scopes; never return refresh tokens from the broker. Persist source-version refresh attempts before platform calls. Timeouts, cancellation, lease loss, malformed responses and ambiguous database errors fence reuse of the old rotating refresh token, including after restart. A new authorization can establish a new version without stale overwrite. Add a strict, separate private Work v1 OIDC adapter while preserving legacy OAuth method behavior. Enable fakeredis Lua test support for Redis lease renewal tests. Document stage boundaries, recovery limitations and testing. Validation: 147 passed, 2 skipped (real PostgreSQL not configured), with 15 existing uvicorn/websockets deprecation warnings. New broker/adapter coverage: 45 tests. AST, runtime opt-in boundary and diff checks pass. An earlier Windows temp-file replace failure in a legacy client test passed both isolated rerun and the final full suite; no client code was changed. No public credential/OAuth routes are wired, no production database/key is initialized, and no real Work authorization, refresh or ECS deployment was performed. Authenticated access-token issuance, state transactions, binding checks and unbind integration remain subsequent stages.
This commit is contained in:
@@ -0,0 +1,454 @@
|
||||
"""Synthetic credentials only: central refresh fencing and access-only boundary."""
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import fakeredis
|
||||
import fakeredis.aioredis
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from relay.credential_broker import (AuthorizationRequired, BrokerUnavailable, CredentialBroker,
|
||||
PermissionRequired, RefreshUnknown)
|
||||
from relay.credential_store import CredentialKeyring, CredentialStore
|
||||
from relay.platform import Feishu, PlatformError
|
||||
from relay.config import Settings
|
||||
from relay.store import Store
|
||||
|
||||
|
||||
NOW = 2000000000
|
||||
|
||||
|
||||
def response(suffix="new", **changes):
|
||||
return {**dict(user_access_token="fake-access-" + suffix, refresh_token="fake-refresh-" + suffix,
|
||||
expires_in=3600, refresh_expires_in=86400, scope="read write"), **changes}
|
||||
|
||||
|
||||
class Platform:
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
self.exchanges = 0
|
||||
self.result = response()
|
||||
self.owner = "alice"
|
||||
self.action = None
|
||||
self.delay = 0
|
||||
self.user_info_error = False
|
||||
|
||||
async def credential_refresh(self, token):
|
||||
assert token.startswith("fake-refresh-")
|
||||
self.calls += 1
|
||||
if self.delay:
|
||||
await asyncio.sleep(self.delay)
|
||||
if self.action:
|
||||
await self.action()
|
||||
return self.result
|
||||
|
||||
async def credential_exchange(self, code):
|
||||
self.exchanges += 1
|
||||
if self.delay:
|
||||
await asyncio.sleep(self.delay)
|
||||
if self.action:
|
||||
await self.action()
|
||||
return self.result
|
||||
|
||||
async def user_info(self, token):
|
||||
if self.user_info_error:
|
||||
raise RuntimeError("fake-access-secret-in-upstream-error")
|
||||
return {"open_id": self.owner}
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def setup(tmp_path):
|
||||
db = Store(tmp_path / "metadata.sqlite")
|
||||
vault = CredentialStore(db, CredentialKeyring("test-key", {"test-key": os.urandom(32)}))
|
||||
redis = fakeredis.aioredis.FakeRedis(server=fakeredis.FakeServer(), decode_responses=True)
|
||||
platform = Platform()
|
||||
broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app",
|
||||
clock=lambda: NOW, lock_wait_seconds=1)
|
||||
yield db, vault, redis, platform, broker
|
||||
await redis.aclose()
|
||||
db.close()
|
||||
|
||||
|
||||
def seed(vault, *, owner="alice", app="app", environment="test", remaining=30, refresh_remaining=86400):
|
||||
return vault.save(environment, app, owner, expected_version=0, access_token="fake-access-old",
|
||||
refresh_token="fake-refresh-old", access_expires_at=NOW + remaining,
|
||||
refresh_expires_at=NOW + refresh_remaining, scopes=["read", "write"])
|
||||
|
||||
|
||||
async def test_access_only_remaining_lifetime_and_scope(setup):
|
||||
db, vault, redis, platform, broker = setup
|
||||
seed(vault, remaining=601)
|
||||
grant = await broker.get_access("alice", required_scopes=["read"])
|
||||
assert grant.expires_in == 601 and grant.credential_version == 1
|
||||
assert set(dataclasses.asdict(grant)) == {"access_token", "expires_at", "expires_in", "credential_version", "scopes"}
|
||||
assert "fake-access" not in repr(grant)
|
||||
assert not hasattr(grant, "refresh_token") and platform.calls == 0
|
||||
with pytest.raises(PermissionRequired):
|
||||
await broker.get_access("alice", required_scopes=["admin"])
|
||||
assert platform.calls == 0
|
||||
|
||||
|
||||
async def test_missing_expired_and_cross_identity(setup):
|
||||
_, vault, redis, platform, broker = setup
|
||||
seed(vault, refresh_remaining=-1)
|
||||
for candidate, owner in [(broker, "bob"), (broker, "alice"),
|
||||
(CredentialBroker(vault, redis, platform, environment="production", app_id="app", clock=lambda: NOW), "alice"),
|
||||
(CredentialBroker(vault, redis, platform, environment="test", app_id="other", clock=lambda: NOW), "alice")]:
|
||||
with pytest.raises(AuthorizationRequired):
|
||||
await candidate.get_access(owner)
|
||||
assert platform.calls == 0
|
||||
|
||||
|
||||
async def test_many_instances_share_one_refresh(setup):
|
||||
_, vault, redis, platform, broker = setup
|
||||
seed(vault)
|
||||
platform.delay = 0.05
|
||||
peers = [CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW,
|
||||
lock_wait_seconds=2) for _ in range(8)]
|
||||
results = await asyncio.gather(*(peer.get_access("alice") for peer in peers))
|
||||
assert platform.calls == 1
|
||||
assert all(item.credential_version == 2 for item in results)
|
||||
assert vault.get("test", "app", "alice").refresh_token == "fake-refresh-new"
|
||||
assert vault.refresh_status("test", "app", "alice", 1) == "completed"
|
||||
|
||||
|
||||
async def test_separate_redis_clients_and_database_connections(setup, tmp_path):
|
||||
db, vault, redis, platform, broker = setup
|
||||
seed(vault)
|
||||
other_db = Store(tmp_path / "metadata.sqlite")
|
||||
other_redis = fakeredis.aioredis.FakeRedis(server=redis.connection_pool.connection_kwargs["server"],
|
||||
decode_responses=True)
|
||||
try:
|
||||
peer = CredentialBroker(CredentialStore(other_db, vault.keyring), other_redis, platform,
|
||||
environment="test", app_id="app", clock=lambda: NOW)
|
||||
platform.delay = 0.05
|
||||
first, second = await asyncio.gather(broker.get_access("alice"), peer.get_access("alice"))
|
||||
assert platform.calls == 1 and first == second
|
||||
finally:
|
||||
other_db.close()
|
||||
await other_redis.aclose()
|
||||
|
||||
|
||||
async def test_authorization_checks_actual_user_and_no_refresh_output(setup):
|
||||
_, vault, _, platform, broker = setup
|
||||
platform.owner = "bob"
|
||||
with pytest.raises(BrokerUnavailable, match="authorization did not complete"):
|
||||
await broker.authorize("alice", "fake-code")
|
||||
assert vault.get("test", "app", "alice") is None
|
||||
platform.owner = "alice"
|
||||
grant = await broker.authorize("alice", "fake-code")
|
||||
assert grant.credential_version == 1 and not hasattr(grant, "refresh_token")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("changes", [dict(refresh_token=""), dict(expires_in=None),
|
||||
dict(refresh_expires_in=0), dict(expires_in=True), dict(expires_in=float("nan")),
|
||||
dict(scope=[1]), dict(user_access_token=""), dict(refresh_expires_in=10**400)])
|
||||
async def test_bad_refresh_response_is_fenced(setup, changes):
|
||||
_, vault, _, platform, broker = setup
|
||||
seed(vault)
|
||||
platform.result = response(**changes)
|
||||
with pytest.raises(RefreshUnknown):
|
||||
await broker.get_access("alice")
|
||||
assert vault.get("test", "app", "alice").version == 1
|
||||
assert vault.refresh_status("test", "app", "alice", 1) == "unknown"
|
||||
with pytest.raises(RefreshUnknown):
|
||||
await broker.get_access("alice")
|
||||
assert platform.calls == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["timeout", "identity", "userinfo", "save", "lost-lease"])
|
||||
async def test_refresh_failures_do_not_reuse_old_refresh(setup, monkeypatch, failure):
|
||||
_, vault, redis, platform, broker = setup
|
||||
seed(vault)
|
||||
async def action():
|
||||
if failure == "timeout":
|
||||
raise httpx.ReadTimeout("fake-refresh-private-error")
|
||||
if failure == "lost-lease":
|
||||
await redis.delete(broker._lock_key("alice"))
|
||||
platform.action = action
|
||||
if failure == "identity":
|
||||
platform.owner = "bob"
|
||||
if failure == "userinfo":
|
||||
platform.user_info_error = True
|
||||
if failure == "save":
|
||||
def broken(*args, **kwargs):
|
||||
raise OSError("fake-refresh-private-error")
|
||||
monkeypatch.setattr(vault, "save", broken)
|
||||
with pytest.raises(RefreshUnknown) as error:
|
||||
await broker.get_access("alice")
|
||||
assert "fake-" not in str(error.value)
|
||||
with pytest.raises(RefreshUnknown):
|
||||
await broker.get_access("alice")
|
||||
assert platform.calls == 1
|
||||
|
||||
|
||||
async def test_crash_started_marker_survives_restart_and_new_authorization_recovers(setup, tmp_path):
|
||||
_, vault, redis, platform, broker = setup
|
||||
seed(vault)
|
||||
assert vault.begin_refresh("test", "app", "alice", 1, "crashed-attempt")
|
||||
other = Store(tmp_path / "metadata.sqlite")
|
||||
try:
|
||||
peer = CredentialBroker(CredentialStore(other, vault.keyring), redis, platform,
|
||||
environment="test", app_id="app", clock=lambda: NOW)
|
||||
with pytest.raises(RefreshUnknown):
|
||||
await peer.get_access("alice")
|
||||
assert platform.calls == 0
|
||||
grant = await peer.authorize("alice", "new-test-code")
|
||||
assert grant.credential_version == 2
|
||||
assert (await broker.get_access("alice")).credential_version == 2
|
||||
finally:
|
||||
other.close()
|
||||
|
||||
|
||||
async def test_refresh_and_authorization_share_lock_and_latest_pair_wins(setup):
|
||||
_, vault, _, platform, broker = setup
|
||||
seed(vault)
|
||||
platform.delay = 0.05
|
||||
refresh = asyncio.create_task(broker.get_access("alice"))
|
||||
while platform.calls == 0:
|
||||
await asyncio.sleep(0.001)
|
||||
authorize = asyncio.create_task(broker.authorize("alice", "new-test-code"))
|
||||
refreshed, authorized = await asyncio.gather(refresh, authorize)
|
||||
assert refreshed.credential_version == 2 and authorized.credential_version == 3
|
||||
assert vault.get("test", "app", "alice").version == 3
|
||||
|
||||
|
||||
async def test_stale_external_writer_not_overwritten(setup):
|
||||
_, vault, _, platform, broker = setup
|
||||
seed(vault)
|
||||
async def external_write():
|
||||
vault.save("test", "app", "alice", expected_version=1, access_token="fake-access-reauthed",
|
||||
refresh_token="fake-refresh-reauthed", access_expires_at=NOW + 3600,
|
||||
refresh_expires_at=NOW + 86400, scopes=["read"])
|
||||
platform.action = external_write
|
||||
with pytest.raises(RefreshUnknown):
|
||||
await broker.get_access("alice")
|
||||
assert vault.get("test", "app", "alice").refresh_token == "fake-refresh-reauthed"
|
||||
|
||||
|
||||
async def test_authorization_cas_does_not_overwrite_external_replacement(setup):
|
||||
_, vault, _, platform, broker = setup
|
||||
seed(vault)
|
||||
async def external_write():
|
||||
vault.save("test", "app", "alice", expected_version=1, access_token="fake-access-other",
|
||||
refresh_token="fake-refresh-other", access_expires_at=NOW + 3600,
|
||||
refresh_expires_at=NOW + 86400, scopes=["read"])
|
||||
platform.action = external_write
|
||||
with pytest.raises(BrokerUnavailable, match="version changed"):
|
||||
await broker.authorize("alice", "synthetic-code")
|
||||
assert vault.get("test", "app", "alice").refresh_token == "fake-refresh-other"
|
||||
|
||||
|
||||
async def test_renewal_keeps_lease_during_slow_platform_call(setup):
|
||||
_, vault, redis, platform, _ = setup
|
||||
seed(vault)
|
||||
broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW,
|
||||
lease_seconds=0.3, lock_wait_seconds=2)
|
||||
platform.delay = 0.7
|
||||
results = await asyncio.gather(broker.get_access("alice"), broker.get_access("alice"))
|
||||
assert platform.calls == 1 and all(r.credential_version == 2 for r in results)
|
||||
|
||||
|
||||
async def test_cancellation_and_timeout_fence_rotation(setup):
|
||||
_, vault, _, platform, broker = setup
|
||||
seed(vault)
|
||||
platform.delay = 2
|
||||
task = asyncio.create_task(broker.get_access("alice"))
|
||||
while platform.calls == 0:
|
||||
await asyncio.sleep(0.001)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
with pytest.raises(RefreshUnknown):
|
||||
await broker.get_access("alice")
|
||||
assert platform.calls == 1
|
||||
|
||||
|
||||
async def test_platform_deadline_fences_refresh(setup):
|
||||
_, vault, redis, platform, _ = setup
|
||||
seed(vault)
|
||||
platform.delay = 0.2
|
||||
broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW,
|
||||
platform_timeout=0.02)
|
||||
with pytest.raises(RefreshUnknown):
|
||||
await broker.get_access("alice")
|
||||
assert vault.refresh_status("test", "app", "alice", 1) == "unknown"
|
||||
with pytest.raises(RefreshUnknown):
|
||||
await broker.get_access("alice")
|
||||
assert platform.calls == 1
|
||||
|
||||
|
||||
async def test_renewal_failure_prevents_publishing_result(setup, monkeypatch):
|
||||
_, vault, redis, platform, _ = setup
|
||||
seed(vault)
|
||||
original = redis.lock
|
||||
def failing_lock(*args, **kwargs):
|
||||
lock = original(*args, **kwargs)
|
||||
async def fail(*args, **kwargs):
|
||||
raise ConnectionError("synthetic renew failure")
|
||||
lock.extend = fail
|
||||
return lock
|
||||
monkeypatch.setattr(redis, "lock", failing_lock)
|
||||
platform.delay = 0.2
|
||||
broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW,
|
||||
lease_seconds=0.15)
|
||||
with pytest.raises(RefreshUnknown):
|
||||
await broker.get_access("alice")
|
||||
assert vault.get("test", "app", "alice").version == 1
|
||||
assert vault.refresh_status("test", "app", "alice", 1) == "unknown"
|
||||
|
||||
|
||||
async def test_failed_unknown_write_keeps_started_marker_as_crash_fence(setup, monkeypatch):
|
||||
_, vault, _, platform, broker = setup
|
||||
seed(vault)
|
||||
async def fail():
|
||||
raise httpx.ReadTimeout("synthetic upstream failure")
|
||||
def broken(*args, **kwargs):
|
||||
raise OSError("synthetic DB failure")
|
||||
platform.action = fail
|
||||
monkeypatch.setattr(vault, "finish_refresh", broken)
|
||||
with pytest.raises(RefreshUnknown):
|
||||
await broker.get_access("alice")
|
||||
assert vault.refresh_status("test", "app", "alice", 1) == "started"
|
||||
with pytest.raises(RefreshUnknown):
|
||||
await broker.get_access("alice")
|
||||
assert platform.calls == 1
|
||||
|
||||
|
||||
async def test_committed_pair_not_lost_if_database_ack_raises(setup, monkeypatch):
|
||||
_, vault, _, platform, broker = setup
|
||||
seed(vault)
|
||||
original = vault.save
|
||||
def commit_then_fail(*args, **kwargs):
|
||||
original(*args, **kwargs)
|
||||
raise OSError("synthetic post-commit disconnect")
|
||||
monkeypatch.setattr(vault, "save", commit_then_fail)
|
||||
with pytest.raises(RefreshUnknown):
|
||||
await broker.get_access("alice")
|
||||
grant = await broker.get_access("alice")
|
||||
assert grant.credential_version == 2 and platform.calls == 1
|
||||
|
||||
|
||||
async def test_missing_scope_during_refresh_retains_verified_scopes(setup):
|
||||
_, vault, _, platform, broker = setup
|
||||
seed(vault)
|
||||
del platform.result["scope"]
|
||||
grant = await broker.get_access("alice", required_scopes=["read"])
|
||||
assert grant.scopes == ("read", "write")
|
||||
|
||||
|
||||
async def test_refresh_with_reduced_scopes_persists_pair_but_denies_missing_capability(setup):
|
||||
_, vault, _, platform, broker = setup
|
||||
seed(vault)
|
||||
platform.result["scope"] = "read"
|
||||
with pytest.raises(PermissionRequired):
|
||||
await broker.get_access("alice", required_scopes=["write"])
|
||||
saved = vault.get("test", "app", "alice")
|
||||
assert saved.version == 2 and saved.refresh_token == "fake-refresh-new"
|
||||
assert (await broker.get_access("alice", required_scopes=["read"])).scopes == ("read",)
|
||||
assert platform.calls == 1
|
||||
|
||||
|
||||
async def test_authorization_requires_scope_and_preserves_existing_on_failure(setup):
|
||||
_, vault, _, platform, broker = setup
|
||||
seed(vault)
|
||||
del platform.result["scope"]
|
||||
with pytest.raises(BrokerUnavailable):
|
||||
await broker.authorize("alice", "fake-code")
|
||||
assert vault.get("test", "app", "alice").version == 1
|
||||
|
||||
|
||||
async def test_successful_persistence_survives_failed_completion_marker(setup, monkeypatch):
|
||||
_, vault, _, platform, broker = setup
|
||||
seed(vault)
|
||||
def broken(*args, **kwargs):
|
||||
raise OSError("synthetic database error")
|
||||
monkeypatch.setattr(vault, "finish_refresh", broken)
|
||||
grant = await broker.get_access("alice")
|
||||
assert (await broker.get_access("alice")) == grant
|
||||
assert platform.calls == 1
|
||||
|
||||
|
||||
async def test_distributed_lock_busy_and_redis_failure_fail_closed(setup):
|
||||
_, vault, redis, platform, _ = setup
|
||||
seed(vault)
|
||||
broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW,
|
||||
lock_wait_seconds=0.05)
|
||||
lock = redis.lock(broker._lock_key("alice"), timeout=5, thread_local=False)
|
||||
await lock.acquire()
|
||||
try:
|
||||
with pytest.raises(BrokerUnavailable):
|
||||
await broker.get_access("alice")
|
||||
finally:
|
||||
await lock.release()
|
||||
assert platform.calls == 0
|
||||
|
||||
|
||||
async def test_redis_unavailable_does_not_call_platform(setup, monkeypatch):
|
||||
_, vault, redis, platform, broker = setup
|
||||
seed(vault)
|
||||
original = redis.lock
|
||||
def failing_lock(*args, **kwargs):
|
||||
lock = original(*args, **kwargs)
|
||||
async def fail(*args, **kwargs):
|
||||
raise ConnectionError("synthetic redis failure")
|
||||
lock.acquire = fail
|
||||
return lock
|
||||
monkeypatch.setattr(redis, "lock", failing_lock)
|
||||
with pytest.raises(BrokerUnavailable):
|
||||
await broker.get_access("alice")
|
||||
assert platform.calls == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad", [0, -1, True, float("inf")])
|
||||
def test_invalid_configuration_rejected(bad):
|
||||
# Constructor validation is independent of any network or key material.
|
||||
with pytest.raises(Exception, match="invalid broker configuration"):
|
||||
CredentialBroker(None, None, None, environment="test", app_id="app", lease_seconds=bad)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["exchange", "refresh"])
|
||||
async def test_private_oidc_adapter_and_scope_preservation(kind):
|
||||
calls = []
|
||||
async def handler(request):
|
||||
calls.append((request.url.path, json.loads(request.content)))
|
||||
if request.url.path.endswith("app_access_token/internal"):
|
||||
return httpx.Response(200, json={"code": 0, "app_access_token": "fake-app-token", "expire": 7200})
|
||||
return httpx.Response(200, json={"code": 0, "data": {
|
||||
"access_token": "fake-access", "refresh_token": "fake-refresh", "expire": 3600,
|
||||
"refresh_expires_in": 86400, "scope": "read"}})
|
||||
settings = Settings(environment="test", admin_secret="a"*40, internal_secret="b"*40)
|
||||
platform = Feishu(settings, transport=httpx.MockTransport(handler))
|
||||
try:
|
||||
method = platform.credential_exchange if kind == "exchange" else platform.credential_refresh
|
||||
result = await method("synthetic-input")
|
||||
assert result["scope"] == "read" and result["expires_in"] == 3600
|
||||
expected_path = "access_token" if kind == "exchange" else "refresh_access_token"
|
||||
assert calls[-1][0] == "/open-apis/authen/v1/oidc/" + expected_path
|
||||
assert calls[-1][1]["grant_type"] == ("authorization_code" if kind == "exchange" else "refresh_token")
|
||||
finally:
|
||||
await platform.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status,body", [(500, {"code": 0}), (200, {"data": {}}),
|
||||
(200, {"code": True, "data": {}}), (200, {"code": 20026, "msg": "fake-secret"}),
|
||||
(200, {"code": 0, "data": {"access_token": "fake-access"}})])
|
||||
async def test_adapter_rejects_error_contract_without_leaking_body(status, body):
|
||||
async def handler(request):
|
||||
if request.url.path.endswith("app_access_token/internal"):
|
||||
return httpx.Response(200, json={"code": 0, "app_access_token": "fake-app-token", "expire": 7200})
|
||||
return httpx.Response(status, json=body)
|
||||
platform = Feishu(Settings(environment="test", admin_secret="a"*40, internal_secret="b"*40),
|
||||
transport=httpx.MockTransport(handler))
|
||||
try:
|
||||
with pytest.raises(PlatformError) as error:
|
||||
await platform.credential_refresh("fake-refresh")
|
||||
assert str(error.value) == "credential platform request failed"
|
||||
finally:
|
||||
await platform.close()
|
||||
Reference in New Issue
Block a user