Files
ecs-relay-service/deployable/relay/credential_broker.py
T
ex_tianzy 02d433250f feat(credentials): add stage 2 centralized broker and refresh fencing
Add an internal CredentialBroker that serializes refresh and authorization
by environment/app/user using renewable Redis leases. Re-read credentials
under the lease and persist rotated pairs with expected-version CAS before
returning an explicit access-only grant. Verify the actual platform user,
remaining lifetime and scopes; never return refresh tokens from the broker.

Persist source-version refresh attempts before platform calls. Timeouts,
cancellation, lease loss, malformed responses and ambiguous database errors
fence reuse of the old rotating refresh token, including after restart.
A new authorization can establish a new version without stale overwrite.

Add a strict, separate private Work v1 OIDC adapter while preserving legacy
OAuth method behavior. Enable fakeredis Lua test support for Redis lease
renewal tests. Document stage boundaries, recovery limitations and testing.

Validation: 147 passed, 2 skipped (real PostgreSQL not configured), with
15 existing uvicorn/websockets deprecation warnings. New broker/adapter
coverage: 45 tests. AST, runtime opt-in boundary and diff checks pass.
An earlier Windows temp-file replace failure in a legacy client test passed
both isolated rerun and the final full suite; no client code was changed.

No public credential/OAuth routes are wired, no production database/key is
initialized, and no real Work authorization, refresh or ECS deployment was
performed. Authenticated access-token issuance, state transactions, binding
checks and unbind integration remain subsequent stages.
2026-11-04 10:30:27 +08:00

227 lines
10 KiB
Python

"""Internal centralized credentials; stage 2 has no public route or device auth.
The future authenticated route supplies the verified owner's platform open_id.
Never expose this object directly as a tool or accept a model-selected owner.
"""
import asyncio
import contextlib
import hashlib
import json
import math
import secrets
import time
from contextlib import asynccontextmanager
from dataclasses import dataclass, field
from .credential_store import CredentialConflict, CredentialError, _identifier
class BrokerUnavailable(CredentialError):
pass
class AuthorizationRequired(CredentialError):
pass
class PermissionRequired(CredentialError):
pass
class RefreshUnknown(CredentialError):
pass
@dataclass(frozen=True)
class AccessGrant:
"""Explicit client-safe shape: no refresh token or internal credential object."""
access_token: str = field(repr=False)
expires_at: float
expires_in: int
credential_version: int
scopes: tuple[str, ...]
class _Lease:
def __init__(self, lock, ttl):
self.lock, self.ttl = lock, ttl
self.lost = False
async def renew(self):
try:
while True:
await asyncio.sleep(self.ttl / 3)
await self.lock.extend(self.ttl, replace_ttl=True)
except asyncio.CancelledError:
raise
except Exception:
self.lost = True
async def check(self):
try:
owned = not self.lost and await self.lock.owned()
except Exception:
owned = False
self.lost = True
if not owned:
raise BrokerUnavailable("credential lease unavailable")
class CredentialBroker:
def __init__(self, store, redis, platform, *, environment, app_id,
minimum_validity=120, lease_seconds=15, lock_wait_seconds=5,
platform_timeout=30, clock=time.time):
self.store, self.redis, self.platform = store, redis, platform
self.environment, self.app_id = _identifier(environment), _identifier(app_id)
for value in (minimum_validity, lease_seconds, lock_wait_seconds, platform_timeout):
if (type(value) not in (int, float) or not math.isfinite(value) or value <= 0):
raise CredentialError("invalid broker configuration")
self.minimum_validity = minimum_validity
self.lease_seconds, self.lock_wait_seconds = lease_seconds, lock_wait_seconds
self.platform_timeout, self.clock = platform_timeout, clock
def _identity(self, owner):
return self.environment, self.app_id, _identifier(owner)
def _lock_key(self, owner):
identity = json.dumps(self._identity(owner), ensure_ascii=False, separators=(",", ":"))
return "hn:credentials:v2:lock:" + hashlib.sha256(identity.encode()).hexdigest()
@asynccontextmanager
async def _lease(self, owner):
lock = self.redis.lock(self._lock_key(owner), timeout=self.lease_seconds,
blocking_timeout=self.lock_wait_seconds,
sleep=min(0.05, self.lock_wait_seconds), thread_local=False)
try:
acquired = await lock.acquire()
except Exception:
raise BrokerUnavailable("credential lease unavailable") from None
if not acquired:
raise BrokerUnavailable("credential lease busy")
lease = _Lease(lock, self.lease_seconds)
task = asyncio.create_task(lease.renew())
try:
yield lease
finally:
task.cancel()
await asyncio.gather(task, return_exceptions=True)
with contextlib.suppress(Exception):
await lock.release()
@staticmethod
def _scopes(value):
if isinstance(value, str):
value = value.split()
if (not isinstance(value, (list, tuple)) or len(value) > 1024
or any(not isinstance(s, str) or not s or len(s) > 256 for s in value)):
raise CredentialError("invalid credential scope response")
return tuple(sorted(set(value)))
def _payload(self, result, started_at, previous=None):
if not isinstance(result, dict):
raise CredentialError("invalid credential response")
durations = []
for name in ("expires_in", "refresh_expires_in"):
value = result.get(name)
if (type(value) not in (int, float) or not 0 < value < 253402300799
or not math.isfinite(value)):
raise CredentialError("invalid credential response")
durations.append(value)
# Explicit scopes are mandatory for new authorization. A refresh response
# without scope retains the previous verified set, never expands it.
scopes = self._scopes(result["scope"]) if "scope" in result else (
previous.scopes if previous is not None else None)
if scopes is None:
raise CredentialError("credential scopes missing")
payload = self.store._payload(result.get("user_access_token"), result.get("refresh_token"),
started_at + durations[0], started_at + durations[1], scopes)
if min(payload["access_expires_at"], payload["refresh_expires_at"]) <= self.clock():
raise CredentialError("credential response already expired")
return payload
def _grant(self, credential, required_scopes):
if not set(required_scopes).issubset(credential.scopes):
raise PermissionRequired("permission update required")
remaining = int(credential.access_expires_at - self.clock())
if remaining <= 0:
raise AuthorizationRequired("authorization required")
return AccessGrant(credential.access_token, credential.access_expires_at,
remaining, credential.version, credential.scopes)
async def authorize(self, owner, code):
"""Only callable after the future OAuth transaction authenticates owner/state."""
identity = self._identity(owner)
if not isinstance(code, str) or not code or len(code) > 4096:
raise CredentialError("invalid authorization input")
async with self._lease(owner) as lease:
current = self.store.get(*identity)
expected_version = current.version if current else 0
started = self.clock()
try:
result = await asyncio.wait_for(self.platform.credential_exchange(code), self.platform_timeout)
payload = self._payload(result, started)
info = await asyncio.wait_for(self.platform.user_info(payload["access_token"]), self.platform_timeout)
if not isinstance(info, dict) or info.get("open_id") != owner:
raise CredentialError("authorization identity mismatch")
await lease.check()
version = self.store.save(*identity, expected_version=expected_version, **payload)
except CredentialConflict:
raise BrokerUnavailable("credential version changed") from None
except asyncio.CancelledError:
raise
except Exception:
raise BrokerUnavailable("authorization did not complete") from None
# Construct explicitly from verified payload, never serialize result.
from .credential_store import UserCredential
credential = UserCredential(version=version, **{**payload, "scopes": tuple(payload["scopes"])})
return self._grant(credential, ())
async def get_access(self, owner, *, required_scopes=()):
identity = self._identity(owner)
required = self._scopes(required_scopes)
# Lock even cache hits: authorization replacement and refresh share one owner.
async with self._lease(owner) as lease:
current = self.store.get(*identity)
if current is None:
raise AuthorizationRequired("authorization required")
if self.store.refresh_status(*identity, current.version) is not None:
raise RefreshUnknown("refresh outcome unknown; authorize again")
if not set(required).issubset(current.scopes):
raise PermissionRequired("permission update required")
if current.access_expires_at - self.clock() > self.minimum_validity:
await lease.check()
return self._grant(current, required)
if current.refresh_expires_at <= self.clock():
raise AuthorizationRequired("authorization required")
await lease.check()
attempt = secrets.token_hex(16)
if not self.store.begin_refresh(*identity, current.version, attempt):
raise RefreshUnknown("refresh outcome unknown; authorize again")
started = self.clock()
try:
await lease.check()
result = await asyncio.wait_for(self.platform.credential_refresh(current.refresh_token),
self.platform_timeout)
payload = self._payload(result, started, previous=current)
info = await asyncio.wait_for(self.platform.user_info(payload["access_token"]), self.platform_timeout)
if not isinstance(info, dict) or info.get("open_id") != owner:
raise CredentialError("refresh identity mismatch")
await lease.check()
version = self.store.save(*identity, expected_version=current.version, **payload)
except BaseException as error:
# Persisted started records also fence retries if this update fails
# or the worker crashes. No automatic reuse of a rotating refresh.
with contextlib.suppress(Exception):
self.store.finish_refresh(*identity, current.version, attempt, "unknown")
if isinstance(error, asyncio.CancelledError):
raise
if not isinstance(error, Exception):
raise
raise RefreshUnknown("refresh outcome unknown; authorize again") from None
# A failed completion marker must not undo the already durable new pair.
with contextlib.suppress(Exception):
self.store.finish_refresh(*identity, current.version, attempt, "completed")
from .credential_store import UserCredential
credential = UserCredential(version=version, **{**payload, "scopes": tuple(payload["scopes"])})
return self._grant(credential, required)