From 02d433250fac263602a3a609a0e1972c4d575763 Mon Sep 17 00:00:00 2001 From: ex_tianzy Date: Wed, 4 Nov 2026 10:30:27 +0800 Subject: [PATCH] feat(credentials): add stage 2 centralized broker and refresh fencing Add an internal CredentialBroker that serializes refresh and authorization by environment/app/user using renewable Redis leases. Re-read credentials under the lease and persist rotated pairs with expected-version CAS before returning an explicit access-only grant. Verify the actual platform user, remaining lifetime and scopes; never return refresh tokens from the broker. Persist source-version refresh attempts before platform calls. Timeouts, cancellation, lease loss, malformed responses and ambiguous database errors fence reuse of the old rotating refresh token, including after restart. A new authorization can establish a new version without stale overwrite. Add a strict, separate private Work v1 OIDC adapter while preserving legacy OAuth method behavior. Enable fakeredis Lua test support for Redis lease renewal tests. Document stage boundaries, recovery limitations and testing. Validation: 147 passed, 2 skipped (real PostgreSQL not configured), with 15 existing uvicorn/websockets deprecation warnings. New broker/adapter coverage: 45 tests. AST, runtime opt-in boundary and diff checks pass. An earlier Windows temp-file replace failure in a legacy client test passed both isolated rerun and the final full suite; no client code was changed. No public credential/OAuth routes are wired, no production database/key is initialized, and no real Work authorization, refresh or ECS deployment was performed. Authenticated access-token issuance, state transactions, binding checks and unbind integration remain subsequent stages. --- deployable/CREDENTIAL_MIGRATION.md | 38 +- deployable/README.md | 2 +- deployable/pyproject.toml | 2 +- deployable/relay/credential_broker.py | 226 ++++++++++ deployable/relay/credential_store.py | 34 +- deployable/relay/platform.py | 31 ++ deployable/tests/test_credential_broker.py | 454 +++++++++++++++++++++ deployable/tests/test_credential_store.py | 6 + 8 files changed, 788 insertions(+), 5 deletions(-) create mode 100644 deployable/relay/credential_broker.py create mode 100644 deployable/tests/test_credential_broker.py diff --git a/deployable/CREDENTIAL_MIGRATION.md b/deployable/CREDENTIAL_MIGRATION.md index 0a63294..05a1e94 100644 --- a/deployable/CREDENTIAL_MIGRATION.md +++ b/deployable/CREDENTIAL_MIGRATION.md @@ -8,8 +8,8 @@ | 阶段 | 范围 | 完成门槛 | 状态 | | --- | --- | --- | --- | -| 1 | 服务端加密凭据存储基础 | 密文持久化、应用/用户/环境隔离、版本冲突、密钥轮换与篡改检测测试;不接入旧路由 | 本次交付 | -| 2 | 集中凭据Broker及刷新 | 应用/用户锁、持锁重读、版本CAS、响应校验、刷新unknown;新授权不受旧刷新覆盖 | 待开发 | +| 1 | 服务端加密凭据存储基础 | 密文持久化、应用/用户/环境隔离、版本冲突、密钥轮换与篡改检测测试;不接入旧路由 | 已完成本地开发测试,提交71b9678 | +| 2 | 集中凭据Broker及刷新 | 应用/用户锁、持锁重读、版本CAS、响应校验、刷新unknown;新授权不受旧刷新覆盖 | 本次完成本地开发测试 | | 3 | OAuth与认证领取 | 共享state事务、实际用户核验、绑定代次、只返回access的独立DTO及领取认证 | 待开发 | | 4 | ECS权威解绑与消息任务关联 | 停发Token、失效事务/任务、连接和旧码联动,多实例/异常恢复 | 待开发 | | 5 | 配置部署、迁移与旧接口退出 | 密钥挂载、产品权限配置、客户端对接、停用旧刷新/claim及代理、ARM64和真实平台验收 | 待开发 | @@ -60,3 +60,37 @@ python -m pytest deployable/tests deployment-source/tests -q - connector未引用新增模块,未新增公开路由,未生成真实密钥或初始化现有数据库。 测试只使用合成凭据、本地临时文件和平台桩,没有进行现网授权、刷新或消息推送。真实PostgreSQL、Docker/ARM64容器运行及现机业务验收尚未执行;不得将本地回归标为服务器上线完成。 + +## 阶段2:集中凭据Broker与刷新(2026-11-04) + +### 本次改造 + +- 新增内部`CredentialBroker`,按环境/应用/已核验owner读取加密凭据,保留原凭据实际剩余期限。`AccessGrant`只包含access token、到期时间、剩余秒数、版本和实际Scope,不含refresh token,repr也隐藏access token。 +- 按环境/应用/owner取Redis可续期租约,刷新与新授权共用同一锁;持锁重读当前凭据,只在access即将过期时刷新。锁等待有上限,慢请求超过等待预算时其他调用返回busy,不自动并发刷新。 +- 新增持久`work_refresh_attempts_v2`,按源凭据版本唯一。上游调用前记录started;超时、取消、响应缺字段、身份不符、失锁或数据库失败后标为unknown;即使标记失败留下started,也阻止重复使用同一旧refresh。 +- 新凭据经user_info核验实际用户后,以CAS原子保存access/refresh和版本,再返回access;后续领取不再触发同一版本刷新。重开数据库可保留刷新未知状态;重新授权写新版本后恢复。 +- 新授权与刷新共享互斥与版本检查,拒绝旧结果覆盖外部已写入的新凭据。刷新Scope缺失仅继承原已核验Scope,不扩权;显式返回缩减Scope时保存新凭据但拒绝缺失能力。新授权必须返回实际Scope。 +- `platform.py`增加独立的`credential_exchange/credential_refresh`适配,使用现有私有化v1 OIDC端点。严格检查HTTP状态、顶层业务码和必要字段;不改变旧`exchange/refresh/normalize`调用契约,不直接复制agent-hub默认v3接口。 +- 新增平台调用超时、Redis失败、租约失效、身份与Scope核验,错误采用固定消息,不输出平台原始凭据或错误正文。 + +### 调用边界与后续集成要求 + +Broker仍是内部模块,未由connector实例化,没有新HTTP接口或后台调度。`owner`参数暂按应用范围内已核验的open_id使用,只能来自后续可信设备/授权事务映射;不得把此内部方法直接暴露给模型、客户端自选用户或无认证HTTP路由。阶段3仍需完成state、绑定代次、领取认证、产品权限配置及公开响应白名单。 + +此阶段不改变旧OAuth向PC交付refresh的行为,也不读取旧PC凭据。部署阶段必须关闭旧刷新并避免双主,不能仅发布此模块就宣称refresh只在ECS。 + +当前get_access连有效Token读取也经过短租约,便于与新授权/刷新串行;客户端将缓存access以降低领取压力。阶段3需在发放前复核绑定和代次,阶段4实现解绑联动。数据库失效状态、已发Bearer残余有效、Scope审批和平台撤权均不由本阶段替代。 + +已有Redis租约和CAS仍无法消除“平台已轮换、服务端未可靠保存”的故障窗口:持续未知时需重新授权,不自动重试旧refresh。数据库记录可恢复到新版本时后续领取复用,不因调用方未收到响应而再次刷新。记录清理必须保留当前源版本的重试屏障,后续运维策略另行定义,不能简单按短TTL删除started/unknown。 + +### 测试环境及结果 + +测试extra改为`fakeredis[lua]`,在本机隔离测试环境安装Lua支持,用FakeRedis实际执行锁的Lua脚本及续租,不将Python本地锁冒充Redis分布式锁。Lua支持仅为测试依赖,生产使用Redis自身脚本,不增加生产Lupa依赖。离线测试环境需事先准备对应测试extra依赖。 + +最后一轮完整回归:**147 passed,2 skipped,15 warnings**(29.57秒)。新增Broker/平台适配测试45项通过;真实PostgreSQL的2个用例因未配置`HN_TEST_DATABASE_URL`而跳过,扩充的PostgreSQL用例含刷新记录唯一性和状态更新但尚未实测。 + +本地覆盖:多Broker实例与独立Redis客户端/数据库连接并发只刷新一次、续租覆盖慢平台调用、失败续租不发布结果、实际剩余期限、access-only对象、身份错误、授权/刷新串行、陈旧CAS、Scope减少、异常字段、进程取消、调用超时、崩溃started记录重开保留、刷新状态写失败、新凭据提交后确认丢失、Redis不可用及旧接口回归。 + +首轮完整回归有一条原有PC测试遇到Windows临时文件replace拒绝访问(WinError 5);该用例单独重跑和最后完整回归均通过,未修改该客户端实现,具体系统原因未确定。warnings仍为uvicorn/websockets旧API弃用提示。 + +未进行真实Redis多进程/多节点压测、真实PostgreSQL、Docker/ARM64、现机部署或真实授权/刷新。FakeRedis共享实例测试证明本地协议并发行为,不外推为生产集群验收。 diff --git a/deployable/README.md b/deployable/README.md index afcc3ca..be35489 100644 --- a/deployable/README.md +++ b/deployable/README.md @@ -1,6 +1,6 @@ # 海诺中继服务部署项目 -集中凭据改造按阶段交付,见 [改造与阶段记录](CREDENTIAL_MIGRATION.md)。阶段1仅提供独立加密存储基础,未启用Token集中管理或更改当前客户端协议;不能据此判断现网已切换。 +集中凭据改造按阶段交付,见 [改造与阶段记录](CREDENTIAL_MIGRATION.md)。阶段1已提供加密存储基础,阶段2已提供内部Broker与集中刷新;尚未接入公开OAuth/领取路由或改变当前客户端协议,不能据此判断现网已切换。 2026-10-28 新增[客户端主动消息与授权卡片](NOTIFICATIONS.md),本地代码已实现,现网部署和卡片验收待完成。 diff --git a/deployable/pyproject.toml b/deployable/pyproject.toml index 3bf5e2c..3b087f3 100644 --- a/deployable/pyproject.toml +++ b/deployable/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ ] [project.optional-dependencies] -test = ["pytest==8.4.2", "pytest-asyncio==1.2.0", "fakeredis==2.32.0"] +test = ["pytest==8.4.2", "pytest-asyncio==1.2.0", "fakeredis[lua]==2.32.0"] longconn = ["lark-oapi==1.5.3"] [project.scripts] diff --git a/deployable/relay/credential_broker.py b/deployable/relay/credential_broker.py new file mode 100644 index 0000000..b731d65 --- /dev/null +++ b/deployable/relay/credential_broker.py @@ -0,0 +1,226 @@ +"""Internal centralized credentials; stage 2 has no public route or device auth. + +The future authenticated route supplies the verified owner's platform open_id. +Never expose this object directly as a tool or accept a model-selected owner. +""" +import asyncio +import contextlib +import hashlib +import json +import math +import secrets +import time +from contextlib import asynccontextmanager +from dataclasses import dataclass, field + +from .credential_store import CredentialConflict, CredentialError, _identifier + + +class BrokerUnavailable(CredentialError): + pass + + +class AuthorizationRequired(CredentialError): + pass + + +class PermissionRequired(CredentialError): + pass + + +class RefreshUnknown(CredentialError): + pass + + +@dataclass(frozen=True) +class AccessGrant: + """Explicit client-safe shape: no refresh token or internal credential object.""" + access_token: str = field(repr=False) + expires_at: float + expires_in: int + credential_version: int + scopes: tuple[str, ...] + + +class _Lease: + def __init__(self, lock, ttl): + self.lock, self.ttl = lock, ttl + self.lost = False + + async def renew(self): + try: + while True: + await asyncio.sleep(self.ttl / 3) + await self.lock.extend(self.ttl, replace_ttl=True) + except asyncio.CancelledError: + raise + except Exception: + self.lost = True + + async def check(self): + try: + owned = not self.lost and await self.lock.owned() + except Exception: + owned = False + self.lost = True + if not owned: + raise BrokerUnavailable("credential lease unavailable") + + +class CredentialBroker: + def __init__(self, store, redis, platform, *, environment, app_id, + minimum_validity=120, lease_seconds=15, lock_wait_seconds=5, + platform_timeout=30, clock=time.time): + self.store, self.redis, self.platform = store, redis, platform + self.environment, self.app_id = _identifier(environment), _identifier(app_id) + for value in (minimum_validity, lease_seconds, lock_wait_seconds, platform_timeout): + if (type(value) not in (int, float) or not math.isfinite(value) or value <= 0): + raise CredentialError("invalid broker configuration") + self.minimum_validity = minimum_validity + self.lease_seconds, self.lock_wait_seconds = lease_seconds, lock_wait_seconds + self.platform_timeout, self.clock = platform_timeout, clock + + def _identity(self, owner): + return self.environment, self.app_id, _identifier(owner) + + def _lock_key(self, owner): + identity = json.dumps(self._identity(owner), ensure_ascii=False, separators=(",", ":")) + return "hn:credentials:v2:lock:" + hashlib.sha256(identity.encode()).hexdigest() + + @asynccontextmanager + async def _lease(self, owner): + lock = self.redis.lock(self._lock_key(owner), timeout=self.lease_seconds, + blocking_timeout=self.lock_wait_seconds, + sleep=min(0.05, self.lock_wait_seconds), thread_local=False) + try: + acquired = await lock.acquire() + except Exception: + raise BrokerUnavailable("credential lease unavailable") from None + if not acquired: + raise BrokerUnavailable("credential lease busy") + lease = _Lease(lock, self.lease_seconds) + task = asyncio.create_task(lease.renew()) + try: + yield lease + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + with contextlib.suppress(Exception): + await lock.release() + + @staticmethod + def _scopes(value): + if isinstance(value, str): + value = value.split() + if (not isinstance(value, (list, tuple)) or len(value) > 1024 + or any(not isinstance(s, str) or not s or len(s) > 256 for s in value)): + raise CredentialError("invalid credential scope response") + return tuple(sorted(set(value))) + + def _payload(self, result, started_at, previous=None): + if not isinstance(result, dict): + raise CredentialError("invalid credential response") + durations = [] + for name in ("expires_in", "refresh_expires_in"): + value = result.get(name) + if (type(value) not in (int, float) or not 0 < value < 253402300799 + or not math.isfinite(value)): + raise CredentialError("invalid credential response") + durations.append(value) + # Explicit scopes are mandatory for new authorization. A refresh response + # without scope retains the previous verified set, never expands it. + scopes = self._scopes(result["scope"]) if "scope" in result else ( + previous.scopes if previous is not None else None) + if scopes is None: + raise CredentialError("credential scopes missing") + payload = self.store._payload(result.get("user_access_token"), result.get("refresh_token"), + started_at + durations[0], started_at + durations[1], scopes) + if min(payload["access_expires_at"], payload["refresh_expires_at"]) <= self.clock(): + raise CredentialError("credential response already expired") + return payload + + def _grant(self, credential, required_scopes): + if not set(required_scopes).issubset(credential.scopes): + raise PermissionRequired("permission update required") + remaining = int(credential.access_expires_at - self.clock()) + if remaining <= 0: + raise AuthorizationRequired("authorization required") + return AccessGrant(credential.access_token, credential.access_expires_at, + remaining, credential.version, credential.scopes) + + async def authorize(self, owner, code): + """Only callable after the future OAuth transaction authenticates owner/state.""" + identity = self._identity(owner) + if not isinstance(code, str) or not code or len(code) > 4096: + raise CredentialError("invalid authorization input") + async with self._lease(owner) as lease: + current = self.store.get(*identity) + expected_version = current.version if current else 0 + started = self.clock() + try: + result = await asyncio.wait_for(self.platform.credential_exchange(code), self.platform_timeout) + payload = self._payload(result, started) + info = await asyncio.wait_for(self.platform.user_info(payload["access_token"]), self.platform_timeout) + if not isinstance(info, dict) or info.get("open_id") != owner: + raise CredentialError("authorization identity mismatch") + await lease.check() + version = self.store.save(*identity, expected_version=expected_version, **payload) + except CredentialConflict: + raise BrokerUnavailable("credential version changed") from None + except asyncio.CancelledError: + raise + except Exception: + raise BrokerUnavailable("authorization did not complete") from None + # Construct explicitly from verified payload, never serialize result. + from .credential_store import UserCredential + credential = UserCredential(version=version, **{**payload, "scopes": tuple(payload["scopes"])}) + return self._grant(credential, ()) + + async def get_access(self, owner, *, required_scopes=()): + identity = self._identity(owner) + required = self._scopes(required_scopes) + # Lock even cache hits: authorization replacement and refresh share one owner. + async with self._lease(owner) as lease: + current = self.store.get(*identity) + if current is None: + raise AuthorizationRequired("authorization required") + if self.store.refresh_status(*identity, current.version) is not None: + raise RefreshUnknown("refresh outcome unknown; authorize again") + if not set(required).issubset(current.scopes): + raise PermissionRequired("permission update required") + if current.access_expires_at - self.clock() > self.minimum_validity: + await lease.check() + return self._grant(current, required) + if current.refresh_expires_at <= self.clock(): + raise AuthorizationRequired("authorization required") + await lease.check() + attempt = secrets.token_hex(16) + if not self.store.begin_refresh(*identity, current.version, attempt): + raise RefreshUnknown("refresh outcome unknown; authorize again") + started = self.clock() + try: + await lease.check() + result = await asyncio.wait_for(self.platform.credential_refresh(current.refresh_token), + self.platform_timeout) + payload = self._payload(result, started, previous=current) + info = await asyncio.wait_for(self.platform.user_info(payload["access_token"]), self.platform_timeout) + if not isinstance(info, dict) or info.get("open_id") != owner: + raise CredentialError("refresh identity mismatch") + await lease.check() + version = self.store.save(*identity, expected_version=current.version, **payload) + except BaseException as error: + # Persisted started records also fence retries if this update fails + # or the worker crashes. No automatic reuse of a rotating refresh. + with contextlib.suppress(Exception): + self.store.finish_refresh(*identity, current.version, attempt, "unknown") + if isinstance(error, asyncio.CancelledError): + raise + if not isinstance(error, Exception): + raise + raise RefreshUnknown("refresh outcome unknown; authorize again") from None + # A failed completion marker must not undo the already durable new pair. + with contextlib.suppress(Exception): + self.store.finish_refresh(*identity, current.version, attempt, "completed") + from .credential_store import UserCredential + credential = UserCredential(version=version, **{**payload, "scopes": tuple(payload["scopes"])}) + return self._grant(credential, required) diff --git a/deployable/relay/credential_store.py b/deployable/relay/credential_store.py index ac86304..15251ef 100644 --- a/deployable/relay/credential_store.py +++ b/deployable/relay/credential_store.py @@ -1,4 +1,4 @@ -"""Server-only encrypted user credentials. Not wired to public routes in stage 1.""" +"""Server-only encrypted credentials and durable refresh fencing, opt-in only.""" import base64 import json import math @@ -106,6 +106,38 @@ class CredentialStore: credential_version INTEGER NOT NULL, key_id TEXT NOT NULL, ciphertext TEXT NOT NULL, updated_at DOUBLE PRECISION NOT NULL, PRIMARY KEY(environment, app_id, principal_id))""") + self.db.execute("""CREATE TABLE IF NOT EXISTS work_refresh_attempts_v2( + environment TEXT NOT NULL, app_id TEXT NOT NULL, principal_id TEXT NOT NULL, + source_version INTEGER NOT NULL, attempt_id TEXT NOT NULL, + status TEXT NOT NULL, updated_at DOUBLE PRECISION NOT NULL, + PRIMARY KEY(environment, app_id, principal_id, source_version))""") + + def refresh_status(self, environment, app_id, principal_id, version): + identity = tuple(_identifier(v) for v in (environment, app_id, principal_id)) + row = self.db.execute("""SELECT status FROM work_refresh_attempts_v2 + WHERE environment=? AND app_id=? AND principal_id=? AND source_version=?""", + (*identity, version)).fetchone() + return row["status"] if row else None + + def begin_refresh(self, environment, app_id, principal_id, version, attempt_id): + identity = tuple(_identifier(v) for v in (environment, app_id, principal_id)) + _identifier(attempt_id) + with self.db: + count = self.db.execute("""INSERT INTO work_refresh_attempts_v2 + VALUES(?,?,?,?,?,?,?) ON CONFLICT DO NOTHING""", + (*identity, version, attempt_id, "started", time.time())).rowcount + return count == 1 + + def finish_refresh(self, environment, app_id, principal_id, version, attempt_id, status): + identity = tuple(_identifier(v) for v in (environment, app_id, principal_id)) + if status not in {"completed", "unknown"}: + raise CredentialError("invalid refresh status") + with self.db: + count = self.db.execute("""UPDATE work_refresh_attempts_v2 SET status=?,updated_at=? + WHERE environment=? AND app_id=? AND principal_id=? AND source_version=? + AND attempt_id=? AND status='started'""", + (status, time.time(), *identity, version, attempt_id)).rowcount + return count == 1 @staticmethod def _aad(identity, version, key_id): diff --git a/deployable/relay/platform.py b/deployable/relay/platform.py index 4e4dc83..0524d22 100644 --- a/deployable/relay/platform.py +++ b/deployable/relay/platform.py @@ -84,6 +84,37 @@ class Feishu: headers={"Authorization": f"Bearer {token}"}) return data["data"] + async def _credential_request(self, grant): + """Strict stage-2 adapter; legacy OAuth normalization remains unchanged. + + Missing rotation fields are not silently defaulted; the Broker validates + them and treats an ambiguous refresh as unknown rather than replaying it. + """ + try: + token = await self.app_token() + path = ("/open-apis/authen/v1/oidc/access_token" if grant["grant_type"] == "authorization_code" + else "/open-apis/authen/v1/oidc/refresh_access_token") + response = await self.http.post(path, headers={"Authorization": f"Bearer {token}"}, json=grant) + response.raise_for_status() + body = response.json() + if not isinstance(body, dict) or type(body.get("code")) is not int or body["code"] != 0: + raise ValueError() + data = body["data"] + result = {"user_access_token": data["access_token"], "refresh_token": data["refresh_token"], + "expires_in": data.get("expires_in", data.get("expire")), + "refresh_expires_in": data.get("refresh_expires_in")} + if "scope" in data: + result["scope"] = data["scope"] + return result + except Exception: + raise PlatformError("credential platform request failed") from None + + async def credential_exchange(self, code): + return await self._credential_request({"grant_type": "authorization_code", "code": code}) + + async def credential_refresh(self, refresh_token): + return await self._credential_request({"grant_type": "refresh_token", "refresh_token": refresh_token}) + async def reply(self, message_id, text): # Reply to the original message only: no caller-controlled recipient. token = await self.tenant_token() diff --git a/deployable/tests/test_credential_broker.py b/deployable/tests/test_credential_broker.py new file mode 100644 index 0000000..9396e5d --- /dev/null +++ b/deployable/tests/test_credential_broker.py @@ -0,0 +1,454 @@ +"""Synthetic credentials only: central refresh fencing and access-only boundary.""" +import asyncio +import dataclasses +import json +import os +import sys +from pathlib import Path + +import fakeredis +import fakeredis.aioredis +import httpx +import pytest +import pytest_asyncio + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from relay.credential_broker import (AuthorizationRequired, BrokerUnavailable, CredentialBroker, + PermissionRequired, RefreshUnknown) +from relay.credential_store import CredentialKeyring, CredentialStore +from relay.platform import Feishu, PlatformError +from relay.config import Settings +from relay.store import Store + + +NOW = 2000000000 + + +def response(suffix="new", **changes): + return {**dict(user_access_token="fake-access-" + suffix, refresh_token="fake-refresh-" + suffix, + expires_in=3600, refresh_expires_in=86400, scope="read write"), **changes} + + +class Platform: + def __init__(self): + self.calls = 0 + self.exchanges = 0 + self.result = response() + self.owner = "alice" + self.action = None + self.delay = 0 + self.user_info_error = False + + async def credential_refresh(self, token): + assert token.startswith("fake-refresh-") + self.calls += 1 + if self.delay: + await asyncio.sleep(self.delay) + if self.action: + await self.action() + return self.result + + async def credential_exchange(self, code): + self.exchanges += 1 + if self.delay: + await asyncio.sleep(self.delay) + if self.action: + await self.action() + return self.result + + async def user_info(self, token): + if self.user_info_error: + raise RuntimeError("fake-access-secret-in-upstream-error") + return {"open_id": self.owner} + + +@pytest_asyncio.fixture +async def setup(tmp_path): + db = Store(tmp_path / "metadata.sqlite") + vault = CredentialStore(db, CredentialKeyring("test-key", {"test-key": os.urandom(32)})) + redis = fakeredis.aioredis.FakeRedis(server=fakeredis.FakeServer(), decode_responses=True) + platform = Platform() + broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", + clock=lambda: NOW, lock_wait_seconds=1) + yield db, vault, redis, platform, broker + await redis.aclose() + db.close() + + +def seed(vault, *, owner="alice", app="app", environment="test", remaining=30, refresh_remaining=86400): + return vault.save(environment, app, owner, expected_version=0, access_token="fake-access-old", + refresh_token="fake-refresh-old", access_expires_at=NOW + remaining, + refresh_expires_at=NOW + refresh_remaining, scopes=["read", "write"]) + + +async def test_access_only_remaining_lifetime_and_scope(setup): + db, vault, redis, platform, broker = setup + seed(vault, remaining=601) + grant = await broker.get_access("alice", required_scopes=["read"]) + assert grant.expires_in == 601 and grant.credential_version == 1 + assert set(dataclasses.asdict(grant)) == {"access_token", "expires_at", "expires_in", "credential_version", "scopes"} + assert "fake-access" not in repr(grant) + assert not hasattr(grant, "refresh_token") and platform.calls == 0 + with pytest.raises(PermissionRequired): + await broker.get_access("alice", required_scopes=["admin"]) + assert platform.calls == 0 + + +async def test_missing_expired_and_cross_identity(setup): + _, vault, redis, platform, broker = setup + seed(vault, refresh_remaining=-1) + for candidate, owner in [(broker, "bob"), (broker, "alice"), + (CredentialBroker(vault, redis, platform, environment="production", app_id="app", clock=lambda: NOW), "alice"), + (CredentialBroker(vault, redis, platform, environment="test", app_id="other", clock=lambda: NOW), "alice")]: + with pytest.raises(AuthorizationRequired): + await candidate.get_access(owner) + assert platform.calls == 0 + + +async def test_many_instances_share_one_refresh(setup): + _, vault, redis, platform, broker = setup + seed(vault) + platform.delay = 0.05 + peers = [CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW, + lock_wait_seconds=2) for _ in range(8)] + results = await asyncio.gather(*(peer.get_access("alice") for peer in peers)) + assert platform.calls == 1 + assert all(item.credential_version == 2 for item in results) + assert vault.get("test", "app", "alice").refresh_token == "fake-refresh-new" + assert vault.refresh_status("test", "app", "alice", 1) == "completed" + + +async def test_separate_redis_clients_and_database_connections(setup, tmp_path): + db, vault, redis, platform, broker = setup + seed(vault) + other_db = Store(tmp_path / "metadata.sqlite") + other_redis = fakeredis.aioredis.FakeRedis(server=redis.connection_pool.connection_kwargs["server"], + decode_responses=True) + try: + peer = CredentialBroker(CredentialStore(other_db, vault.keyring), other_redis, platform, + environment="test", app_id="app", clock=lambda: NOW) + platform.delay = 0.05 + first, second = await asyncio.gather(broker.get_access("alice"), peer.get_access("alice")) + assert platform.calls == 1 and first == second + finally: + other_db.close() + await other_redis.aclose() + + +async def test_authorization_checks_actual_user_and_no_refresh_output(setup): + _, vault, _, platform, broker = setup + platform.owner = "bob" + with pytest.raises(BrokerUnavailable, match="authorization did not complete"): + await broker.authorize("alice", "fake-code") + assert vault.get("test", "app", "alice") is None + platform.owner = "alice" + grant = await broker.authorize("alice", "fake-code") + assert grant.credential_version == 1 and not hasattr(grant, "refresh_token") + + +@pytest.mark.parametrize("changes", [dict(refresh_token=""), dict(expires_in=None), + dict(refresh_expires_in=0), dict(expires_in=True), dict(expires_in=float("nan")), + dict(scope=[1]), dict(user_access_token=""), dict(refresh_expires_in=10**400)]) +async def test_bad_refresh_response_is_fenced(setup, changes): + _, vault, _, platform, broker = setup + seed(vault) + platform.result = response(**changes) + with pytest.raises(RefreshUnknown): + await broker.get_access("alice") + assert vault.get("test", "app", "alice").version == 1 + assert vault.refresh_status("test", "app", "alice", 1) == "unknown" + with pytest.raises(RefreshUnknown): + await broker.get_access("alice") + assert platform.calls == 1 + + +@pytest.mark.parametrize("failure", ["timeout", "identity", "userinfo", "save", "lost-lease"]) +async def test_refresh_failures_do_not_reuse_old_refresh(setup, monkeypatch, failure): + _, vault, redis, platform, broker = setup + seed(vault) + async def action(): + if failure == "timeout": + raise httpx.ReadTimeout("fake-refresh-private-error") + if failure == "lost-lease": + await redis.delete(broker._lock_key("alice")) + platform.action = action + if failure == "identity": + platform.owner = "bob" + if failure == "userinfo": + platform.user_info_error = True + if failure == "save": + def broken(*args, **kwargs): + raise OSError("fake-refresh-private-error") + monkeypatch.setattr(vault, "save", broken) + with pytest.raises(RefreshUnknown) as error: + await broker.get_access("alice") + assert "fake-" not in str(error.value) + with pytest.raises(RefreshUnknown): + await broker.get_access("alice") + assert platform.calls == 1 + + +async def test_crash_started_marker_survives_restart_and_new_authorization_recovers(setup, tmp_path): + _, vault, redis, platform, broker = setup + seed(vault) + assert vault.begin_refresh("test", "app", "alice", 1, "crashed-attempt") + other = Store(tmp_path / "metadata.sqlite") + try: + peer = CredentialBroker(CredentialStore(other, vault.keyring), redis, platform, + environment="test", app_id="app", clock=lambda: NOW) + with pytest.raises(RefreshUnknown): + await peer.get_access("alice") + assert platform.calls == 0 + grant = await peer.authorize("alice", "new-test-code") + assert grant.credential_version == 2 + assert (await broker.get_access("alice")).credential_version == 2 + finally: + other.close() + + +async def test_refresh_and_authorization_share_lock_and_latest_pair_wins(setup): + _, vault, _, platform, broker = setup + seed(vault) + platform.delay = 0.05 + refresh = asyncio.create_task(broker.get_access("alice")) + while platform.calls == 0: + await asyncio.sleep(0.001) + authorize = asyncio.create_task(broker.authorize("alice", "new-test-code")) + refreshed, authorized = await asyncio.gather(refresh, authorize) + assert refreshed.credential_version == 2 and authorized.credential_version == 3 + assert vault.get("test", "app", "alice").version == 3 + + +async def test_stale_external_writer_not_overwritten(setup): + _, vault, _, platform, broker = setup + seed(vault) + async def external_write(): + vault.save("test", "app", "alice", expected_version=1, access_token="fake-access-reauthed", + refresh_token="fake-refresh-reauthed", access_expires_at=NOW + 3600, + refresh_expires_at=NOW + 86400, scopes=["read"]) + platform.action = external_write + with pytest.raises(RefreshUnknown): + await broker.get_access("alice") + assert vault.get("test", "app", "alice").refresh_token == "fake-refresh-reauthed" + + +async def test_authorization_cas_does_not_overwrite_external_replacement(setup): + _, vault, _, platform, broker = setup + seed(vault) + async def external_write(): + vault.save("test", "app", "alice", expected_version=1, access_token="fake-access-other", + refresh_token="fake-refresh-other", access_expires_at=NOW + 3600, + refresh_expires_at=NOW + 86400, scopes=["read"]) + platform.action = external_write + with pytest.raises(BrokerUnavailable, match="version changed"): + await broker.authorize("alice", "synthetic-code") + assert vault.get("test", "app", "alice").refresh_token == "fake-refresh-other" + + +async def test_renewal_keeps_lease_during_slow_platform_call(setup): + _, vault, redis, platform, _ = setup + seed(vault) + broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW, + lease_seconds=0.3, lock_wait_seconds=2) + platform.delay = 0.7 + results = await asyncio.gather(broker.get_access("alice"), broker.get_access("alice")) + assert platform.calls == 1 and all(r.credential_version == 2 for r in results) + + +async def test_cancellation_and_timeout_fence_rotation(setup): + _, vault, _, platform, broker = setup + seed(vault) + platform.delay = 2 + task = asyncio.create_task(broker.get_access("alice")) + while platform.calls == 0: + await asyncio.sleep(0.001) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + with pytest.raises(RefreshUnknown): + await broker.get_access("alice") + assert platform.calls == 1 + + +async def test_platform_deadline_fences_refresh(setup): + _, vault, redis, platform, _ = setup + seed(vault) + platform.delay = 0.2 + broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW, + platform_timeout=0.02) + with pytest.raises(RefreshUnknown): + await broker.get_access("alice") + assert vault.refresh_status("test", "app", "alice", 1) == "unknown" + with pytest.raises(RefreshUnknown): + await broker.get_access("alice") + assert platform.calls == 1 + + +async def test_renewal_failure_prevents_publishing_result(setup, monkeypatch): + _, vault, redis, platform, _ = setup + seed(vault) + original = redis.lock + def failing_lock(*args, **kwargs): + lock = original(*args, **kwargs) + async def fail(*args, **kwargs): + raise ConnectionError("synthetic renew failure") + lock.extend = fail + return lock + monkeypatch.setattr(redis, "lock", failing_lock) + platform.delay = 0.2 + broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW, + lease_seconds=0.15) + with pytest.raises(RefreshUnknown): + await broker.get_access("alice") + assert vault.get("test", "app", "alice").version == 1 + assert vault.refresh_status("test", "app", "alice", 1) == "unknown" + + +async def test_failed_unknown_write_keeps_started_marker_as_crash_fence(setup, monkeypatch): + _, vault, _, platform, broker = setup + seed(vault) + async def fail(): + raise httpx.ReadTimeout("synthetic upstream failure") + def broken(*args, **kwargs): + raise OSError("synthetic DB failure") + platform.action = fail + monkeypatch.setattr(vault, "finish_refresh", broken) + with pytest.raises(RefreshUnknown): + await broker.get_access("alice") + assert vault.refresh_status("test", "app", "alice", 1) == "started" + with pytest.raises(RefreshUnknown): + await broker.get_access("alice") + assert platform.calls == 1 + + +async def test_committed_pair_not_lost_if_database_ack_raises(setup, monkeypatch): + _, vault, _, platform, broker = setup + seed(vault) + original = vault.save + def commit_then_fail(*args, **kwargs): + original(*args, **kwargs) + raise OSError("synthetic post-commit disconnect") + monkeypatch.setattr(vault, "save", commit_then_fail) + with pytest.raises(RefreshUnknown): + await broker.get_access("alice") + grant = await broker.get_access("alice") + assert grant.credential_version == 2 and platform.calls == 1 + + +async def test_missing_scope_during_refresh_retains_verified_scopes(setup): + _, vault, _, platform, broker = setup + seed(vault) + del platform.result["scope"] + grant = await broker.get_access("alice", required_scopes=["read"]) + assert grant.scopes == ("read", "write") + + +async def test_refresh_with_reduced_scopes_persists_pair_but_denies_missing_capability(setup): + _, vault, _, platform, broker = setup + seed(vault) + platform.result["scope"] = "read" + with pytest.raises(PermissionRequired): + await broker.get_access("alice", required_scopes=["write"]) + saved = vault.get("test", "app", "alice") + assert saved.version == 2 and saved.refresh_token == "fake-refresh-new" + assert (await broker.get_access("alice", required_scopes=["read"])).scopes == ("read",) + assert platform.calls == 1 + + +async def test_authorization_requires_scope_and_preserves_existing_on_failure(setup): + _, vault, _, platform, broker = setup + seed(vault) + del platform.result["scope"] + with pytest.raises(BrokerUnavailable): + await broker.authorize("alice", "fake-code") + assert vault.get("test", "app", "alice").version == 1 + + +async def test_successful_persistence_survives_failed_completion_marker(setup, monkeypatch): + _, vault, _, platform, broker = setup + seed(vault) + def broken(*args, **kwargs): + raise OSError("synthetic database error") + monkeypatch.setattr(vault, "finish_refresh", broken) + grant = await broker.get_access("alice") + assert (await broker.get_access("alice")) == grant + assert platform.calls == 1 + + +async def test_distributed_lock_busy_and_redis_failure_fail_closed(setup): + _, vault, redis, platform, _ = setup + seed(vault) + broker = CredentialBroker(vault, redis, platform, environment="test", app_id="app", clock=lambda: NOW, + lock_wait_seconds=0.05) + lock = redis.lock(broker._lock_key("alice"), timeout=5, thread_local=False) + await lock.acquire() + try: + with pytest.raises(BrokerUnavailable): + await broker.get_access("alice") + finally: + await lock.release() + assert platform.calls == 0 + + +async def test_redis_unavailable_does_not_call_platform(setup, monkeypatch): + _, vault, redis, platform, broker = setup + seed(vault) + original = redis.lock + def failing_lock(*args, **kwargs): + lock = original(*args, **kwargs) + async def fail(*args, **kwargs): + raise ConnectionError("synthetic redis failure") + lock.acquire = fail + return lock + monkeypatch.setattr(redis, "lock", failing_lock) + with pytest.raises(BrokerUnavailable): + await broker.get_access("alice") + assert platform.calls == 0 + + +@pytest.mark.parametrize("bad", [0, -1, True, float("inf")]) +def test_invalid_configuration_rejected(bad): + # Constructor validation is independent of any network or key material. + with pytest.raises(Exception, match="invalid broker configuration"): + CredentialBroker(None, None, None, environment="test", app_id="app", lease_seconds=bad) + + +@pytest.mark.parametrize("kind", ["exchange", "refresh"]) +async def test_private_oidc_adapter_and_scope_preservation(kind): + calls = [] + async def handler(request): + calls.append((request.url.path, json.loads(request.content))) + if request.url.path.endswith("app_access_token/internal"): + return httpx.Response(200, json={"code": 0, "app_access_token": "fake-app-token", "expire": 7200}) + return httpx.Response(200, json={"code": 0, "data": { + "access_token": "fake-access", "refresh_token": "fake-refresh", "expire": 3600, + "refresh_expires_in": 86400, "scope": "read"}}) + settings = Settings(environment="test", admin_secret="a"*40, internal_secret="b"*40) + platform = Feishu(settings, transport=httpx.MockTransport(handler)) + try: + method = platform.credential_exchange if kind == "exchange" else platform.credential_refresh + result = await method("synthetic-input") + assert result["scope"] == "read" and result["expires_in"] == 3600 + expected_path = "access_token" if kind == "exchange" else "refresh_access_token" + assert calls[-1][0] == "/open-apis/authen/v1/oidc/" + expected_path + assert calls[-1][1]["grant_type"] == ("authorization_code" if kind == "exchange" else "refresh_token") + finally: + await platform.close() + + +@pytest.mark.parametrize("status,body", [(500, {"code": 0}), (200, {"data": {}}), + (200, {"code": True, "data": {}}), (200, {"code": 20026, "msg": "fake-secret"}), + (200, {"code": 0, "data": {"access_token": "fake-access"}})]) +async def test_adapter_rejects_error_contract_without_leaking_body(status, body): + async def handler(request): + if request.url.path.endswith("app_access_token/internal"): + return httpx.Response(200, json={"code": 0, "app_access_token": "fake-app-token", "expire": 7200}) + return httpx.Response(status, json=body) + platform = Feishu(Settings(environment="test", admin_secret="a"*40, internal_secret="b"*40), + transport=httpx.MockTransport(handler)) + try: + with pytest.raises(PlatformError) as error: + await platform.credential_refresh("fake-refresh") + assert str(error.value) == "credential platform request failed" + finally: + await platform.close() diff --git a/deployable/tests/test_credential_store.py b/deployable/tests/test_credential_store.py index 0e89968..9d0543d 100644 --- a/deployable/tests/test_credential_store.py +++ b/deployable/tests/test_credential_store.py @@ -208,6 +208,12 @@ def test_postgres_encryption_cas_and_schema_reentry(tmp_path): with pytest.raises(CredentialConflict): vault.save("test", "app", "alice", expected_version=1, **payload("stale")) assert vault.get("test", "app", "alice").refresh_token == "test-refresh-new" + assert vault.begin_refresh("test", "app", "alice", 2, "pg-attempt") + assert not vault.begin_refresh("test", "app", "alice", 2, "pg-duplicate") + assert vault.refresh_status("test", "app", "alice", 2) == "started" + assert vault.finish_refresh("test", "app", "alice", 2, "pg-attempt", "unknown") + assert not vault.finish_refresh("test", "app", "alice", 2, "pg-attempt", "completed") + assert vault.refresh_status("test", "app", "alice", 2) == "unknown" row = store.db.execute("SELECT ciphertext FROM work_credentials_v2").fetchone() assert "test-refresh" not in row["ciphertext"] finally: