diff --git a/deployable/CREDENTIAL_MIGRATION.md b/deployable/CREDENTIAL_MIGRATION.md new file mode 100644 index 0000000..0a63294 --- /dev/null +++ b/deployable/CREDENTIAL_MIGRATION.md @@ -0,0 +1,62 @@ +# 集中凭据改造:分阶段交付记录 + +目标:ECS保存并刷新用户access/refresh token;客户端仅认证领取本人access token,MCP、Skill及业务接口在客户端执行并直连私有化Work。refresh token不下发客户端。 + +本文件描述开发阶段,不能作为当前现网已切换的证明。每阶段分别开发、测试、提交master并推送;实际ECS部署及客户端升级另有版本和现场验收。 + +## 阶段划分 + +| 阶段 | 范围 | 完成门槛 | 状态 | +| --- | --- | --- | --- | +| 1 | 服务端加密凭据存储基础 | 密文持久化、应用/用户/环境隔离、版本冲突、密钥轮换与篡改检测测试;不接入旧路由 | 本次交付 | +| 2 | 集中凭据Broker及刷新 | 应用/用户锁、持锁重读、版本CAS、响应校验、刷新unknown;新授权不受旧刷新覆盖 | 待开发 | +| 3 | OAuth与认证领取 | 共享state事务、实际用户核验、绑定代次、只返回access的独立DTO及领取认证 | 待开发 | +| 4 | ECS权威解绑与消息任务关联 | 停发Token、失效事务/任务、连接和旧码联动,多实例/异常恢复 | 待开发 | +| 5 | 配置部署、迁移与旧接口退出 | 密钥挂载、产品权限配置、客户端对接、停用旧刷新/claim及代理、ARM64和真实平台验收 | 待开发 | + +产品经理决定功能和Scope清单。阶段1不预设具体权限,不生成真实凭据、不读取现有设备、不调用平台。 + +## 阶段1:加密存储基础 + +新增`relay/credential_store.py`和测试,使用现有`Store`数据库连接,支持SQLite本地测试和PostgreSQL部署。只有显式构造`CredentialStore`时初始化新增表`work_credentials_v2`,当前connector不导入/启用该模块。 + +### 存储与加密 + +- 主键为environment/app_id/principal_id。access token、refresh token、实际Scope、到期时间一起作为AES-256-GCM密文保存;数据库元数据只有身份、凭据版本、key_id和更新时间。 +- AAD绑定格式版本、身份三元组、凭据版本和key_id,防止跨用户/应用/环境调换密文。每次写入生成随机96位nonce。 +- 调用方提交expected_version:0仅允许首次创建;后续更新必须匹配数据库版本。冲突抛固定异常,不自动覆盖、重试或刷新上游。 +- keyring包含active_key_id及保留解密用的旧密钥。新写入采用当前密钥;旧记录必须先由有旧密钥的服务读取并以CAS重写后才可移除旧密钥。改active_key_id不表示全量密文已迁移。 +- 存储对象仅供ECS内部使用,包含refresh token,绝不能直接序列化为客户端响应。后续阶段必须用独立字段白名单DTO,客户端只收到access token。 + +### 密钥管理 + +`CredentialKeyring.from_file(Path(...))`读取JSON文件,格式为active_key_id和keys映射,每项keys值为32字节随机密钥的标准Base64编码。当前不提供默认密钥、环境配置或自动启动接线,不能只上线此提交就开始存生产凭据。 + +部署阶段需由运维将密钥文件放在Git、镜像、数据库和普通备份之外的受控位置,以最小OS权限挂载,仅凭据服务可读;Windows文件ACL和Linux权限由部署检查落实,模块不声称已校验ACL。密钥与加密数据库备份分开保护。不要把真实密钥作为命令行参数、日志、工单或文档示例。 + +### 兼容与未完成边界 + +现有设备/绑定、数据库旧表、OAuth路由、通知和客户端协议均未修改;原始快照目录保留。阶段1没有发Token接口、后台刷新、绑定代次或解绑联动;旧实现仍可能把refresh下发PC,不能把本提交视为已经满足最终凭据边界。 + +版本CAS用于避免陈旧写入覆盖,并不能实现跨平台刷新exactly-once。AEAD用于检测密文篡改,不保证数据库整体历史回滚检测。PostgreSQL真实环境、密钥轮换运维和加密备份恢复必须另行验收。 + +### 复核命令 + +在仓库根设置`PYTHONPATH=deployable`后执行: + +```text +python -m pytest deployable/tests/test_credential_store.py -q +python -m pytest deployable/tests deployment-source/tests -q +``` + +真实PostgreSQL用例只有在测试环境设置`HN_TEST_DATABASE_URL`时执行;它创建并清理独立随机schema,不把测试指向业务库。未提供时明确skipped,不冒充实机通过。 + +### 本阶段验证记录(2026-11-04) + +- Windows / Python 3.12.10,使用本机已有隔离测试依赖环境,`PYTHONPATH=deployable`。 +- 最后一轮完整本地回归:**102 passed,2 skipped,15 warnings**(42秒)。其中新增存储测试23项通过,新增真实PostgreSQL测试1项跳过;原有真实PostgreSQL测试也因未配置连接而跳过。 +- warnings为现有uvicorn/websockets旧接口弃用提示,没有新增测试失败。新增模块AST语法与`git diff --check`通过。 +- 覆盖密文/WAL不含测试Token、重开连接持久性、环境/应用/用户密文调换拒绝、AAD/认证标签篡改、错误密钥、旧密钥移除、轮换重写、陈旧CAS和双连接并发写入、异常输入与固定错误信息。 +- connector未引用新增模块,未新增公开路由,未生成真实密钥或初始化现有数据库。 + +测试只使用合成凭据、本地临时文件和平台桩,没有进行现网授权、刷新或消息推送。真实PostgreSQL、Docker/ARM64容器运行及现机业务验收尚未执行;不得将本地回归标为服务器上线完成。 diff --git a/deployable/README.md b/deployable/README.md index af7894a..afcc3ca 100644 --- a/deployable/README.md +++ b/deployable/README.md @@ -1,5 +1,7 @@ # 海诺中继服务部署项目 +集中凭据改造按阶段交付,见 [改造与阶段记录](CREDENTIAL_MIGRATION.md)。阶段1仅提供独立加密存储基础,未启用Token集中管理或更改当前客户端协议;不能据此判断现网已切换。 + 2026-10-28 新增[客户端主动消息与授权卡片](NOTIFICATIONS.md),本地代码已实现,现网部署和卡片验收待完成。 本目录用于新环境部署,源码对应 2026-10-27 ECS connector/router/longconn 导出版本。协议和业务源码保持原样。仓库其他快照目录用于追溯,不需要手工合并。 diff --git a/deployable/relay/credential_store.py b/deployable/relay/credential_store.py new file mode 100644 index 0000000..ac86304 --- /dev/null +++ b/deployable/relay/credential_store.py @@ -0,0 +1,177 @@ +"""Server-only encrypted user credentials. Not wired to public routes in stage 1.""" +import base64 +import json +import math +import re +import time +from dataclasses import dataclass, field +from pathlib import Path + +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from .store import INTEGRITY_ERRORS + + +class CredentialError(RuntimeError): + """Safe, fixed messages only; never attach platform responses or key material.""" + + +class CredentialConflict(CredentialError): + pass + + +def _identifier(value): + if not isinstance(value, str) or not value or len(value) > 256: + raise CredentialError("invalid credential identity") + if any(ord(c) < 32 for c in value): + raise CredentialError("invalid credential identity") + return value + + +def _key_id(value): + if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", value): + raise CredentialError("invalid credential key configuration") + return value + + +class CredentialKeyring: + """AES-256-GCM keyring; active key writes, retained keys decrypt old versions.""" + + def __init__(self, active_key_id, keys): + self.active_key_id = _key_id(active_key_id) + if not isinstance(keys, dict) or not keys or len(keys) > 32: + raise CredentialError("invalid credential key configuration") + self._keys = {} + for name, key in keys.items(): + if not isinstance(key, bytes) or len(key) != 32: + raise CredentialError("invalid credential key configuration") + self._keys[_key_id(name)] = key + if self.active_key_id not in self._keys: + raise CredentialError("invalid credential key configuration") + + @classmethod + def from_file(cls, path: Path): + """Load an operator-protected, separately backed-up JSON key file.""" + try: + if path.stat().st_size > 16384: + raise ValueError() + data = json.loads(path.read_text(encoding="utf-8")) + if set(data) != {"active_key_id", "keys"} or not isinstance(data["keys"], dict): + raise ValueError() + keys = {name: base64.b64decode(value, validate=True) + for name, value in data["keys"].items()} + return cls(data["active_key_id"], keys) + except (OSError, ValueError, TypeError, KeyError, CredentialError): + raise CredentialError("cannot load credential key configuration") from None + + def encrypt(self, plaintext, aad): + import secrets + nonce = secrets.token_bytes(12) + ciphertext = AESGCM(self._keys[self.active_key_id]).encrypt(nonce, plaintext, aad) + return self.active_key_id, base64.b64encode(nonce + ciphertext).decode("ascii") + + def decrypt(self, key_id, ciphertext, aad): + try: + raw = base64.b64decode(ciphertext, validate=True) + if len(raw) < 28: + raise ValueError() + return AESGCM(self._keys[key_id]).decrypt(raw[:12], raw[12:], aad) + except Exception: + raise CredentialError("credential decryption failed") from None + + +@dataclass(frozen=True) +class UserCredential: + version: int + access_expires_at: float + refresh_expires_at: float + scopes: tuple[str, ...] + access_token: str = field(repr=False) + refresh_token: str = field(repr=False) + + +class CredentialStore: + """Opt-in additive schema on the connector Store connection (SQLite or PG). + + Caller must supply the last read version. Never retries a conflicting write. + This is internal storage, not a response DTO: never serialize it to clients. + """ + + def __init__(self, store, keyring: CredentialKeyring): + self.db = store.db + self.keyring = keyring + with self.db: + self.db.execute("""CREATE TABLE IF NOT EXISTS work_credentials_v2( + environment TEXT NOT NULL, app_id TEXT NOT NULL, principal_id TEXT NOT NULL, + credential_version INTEGER NOT NULL, key_id TEXT NOT NULL, + ciphertext TEXT NOT NULL, updated_at DOUBLE PRECISION NOT NULL, + PRIMARY KEY(environment, app_id, principal_id))""") + + @staticmethod + def _aad(identity, version, key_id): + # JSON array preserves identity boundaries (no delimiter collisions). + return json.dumps(["hainuo-user-credential-v1", *identity, version, key_id], + ensure_ascii=False, separators=(",", ":")).encode("utf-8") + + @staticmethod + def _payload(access_token, refresh_token, access_expires_at, refresh_expires_at, scopes): + for token in (access_token, refresh_token): + if not isinstance(token, str) or not token or len(token) > 16384: + raise CredentialError("invalid credential payload") + for expiry in (access_expires_at, refresh_expires_at): + if (isinstance(expiry, bool) or not isinstance(expiry, (int, float)) + or expiry <= 0 or expiry > 253402300799 or not math.isfinite(expiry)): + raise CredentialError("invalid credential payload") + if (not isinstance(scopes, (list, tuple)) or len(scopes) > 1024 + or any(not isinstance(s, str) or not s or len(s) > 256 for s in scopes)): + raise CredentialError("invalid credential payload") + return {"access_token": access_token, "refresh_token": refresh_token, + "access_expires_at": access_expires_at, "refresh_expires_at": refresh_expires_at, + "scopes": sorted(set(scopes))} + + def save(self, environment, app_id, principal_id, *, expected_version, access_token, + refresh_token, access_expires_at, refresh_expires_at, scopes): + identity = tuple(_identifier(v) for v in (environment, app_id, principal_id)) + if type(expected_version) is not int or expected_version < 0 or expected_version >= 2147483647: + raise CredentialError("invalid credential version") + payload = self._payload(access_token, refresh_token, access_expires_at, refresh_expires_at, scopes) + version = expected_version + 1 + key_id = self.keyring.active_key_id + _, ciphertext = self.keyring.encrypt( + json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8"), + self._aad(identity, version, key_id)) + try: + with self.db: + if expected_version == 0: + self.db.execute("INSERT INTO work_credentials_v2 VALUES(?,?,?,?,?,?,?)", + (*identity, version, key_id, ciphertext, time.time())) + else: + count = self.db.execute("""UPDATE work_credentials_v2 + SET credential_version=?,key_id=?,ciphertext=?,updated_at=? + WHERE environment=? AND app_id=? AND principal_id=? AND credential_version=?""", + (version, key_id, ciphertext, time.time(), *identity, expected_version)).rowcount + if count != 1: + raise CredentialConflict("credential version conflict") + except INTEGRITY_ERRORS: + raise CredentialConflict("credential version conflict") from None + return version + + def get(self, environment, app_id, principal_id): + identity = tuple(_identifier(v) for v in (environment, app_id, principal_id)) + row = self.db.execute("""SELECT credential_version,key_id,ciphertext FROM work_credentials_v2 + WHERE environment=? AND app_id=? AND principal_id=?""", identity).fetchone() + if row is None: + return None + try: + version = row["credential_version"] + raw = self.keyring.decrypt(row["key_id"], row["ciphertext"], + self._aad(identity, version, row["key_id"])) + data = json.loads(raw) + payload = self._payload(**data) + return UserCredential(version=version, access_token=payload["access_token"], + refresh_token=payload["refresh_token"], + access_expires_at=payload["access_expires_at"], + refresh_expires_at=payload["refresh_expires_at"], + scopes=tuple(payload["scopes"])) + except Exception: + raise CredentialError("credential decryption failed") from None diff --git a/deployable/tests/test_credential_store.py b/deployable/tests/test_credential_store.py new file mode 100644 index 0000000..0e89968 --- /dev/null +++ b/deployable/tests/test_credential_store.py @@ -0,0 +1,217 @@ +"""Stage 1 confidentiality, isolation, persistence, rotation and write fencing.""" +import base64 +import json +import os +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from threading import Barrier + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from relay.credential_store import CredentialConflict, CredentialError, CredentialKeyring, CredentialStore +from relay.store import Store + + +def payload(suffix="a"): + return dict(access_token="test-access-" + suffix, refresh_token="test-refresh-" + suffix, + access_expires_at=2000000000, refresh_expires_at=2100000000, + scopes=["scope:b", "scope:a", "scope:a"]) + + +@pytest.fixture +def vault(tmp_path): + store = Store(tmp_path / "metadata.sqlite") + keys = CredentialKeyring("k1", {"k1": os.urandom(32)}) + vault = CredentialStore(store, keys) + yield store, keys, vault + store.close() + + +def test_encrypted_persistent_and_no_secret_repr(vault, tmp_path): + store, keys, vault = vault + assert vault.get("test", "app", "alice") is None + assert vault.save("test", "app", "alice", expected_version=0, **payload()) == 1 + value = vault.get("test", "app", "alice") + assert value.refresh_token == payload()["refresh_token"] + assert value.scopes == ("scope:a", "scope:b") + assert "test-access" not in repr(value) and "test-refresh" not in repr(value) + dump = "\n".join(store.db.iterdump()) + assert "test-access" not in dump and "test-refresh" not in dump and "scope:a" not in dump + other = Store(tmp_path / "metadata.sqlite") + try: + assert CredentialStore(other, keys).get("test", "app", "alice") == value + finally: + other.close() + for path in tmp_path.glob("metadata.sqlite*"): + assert b"test-refresh" not in path.read_bytes() + + +def test_identity_isolation_and_ciphertext_substitution(vault): + store, keys, vault = vault + for identity in [("test", "app", "alice"), ("test", "app", "bob"), + ("test", "other-app", "alice"), ("production", "app", "alice")]: + vault.save(*identity, expected_version=0, **payload(identity[-1])) + row = store.db.execute("SELECT ciphertext FROM work_credentials_v2 WHERE principal_id=? AND app_id=? AND environment=?", + ("alice", "app", "test")).fetchone() + for where, args in [("principal_id=? AND app_id=? AND environment=?", ("bob", "app", "test")), + ("principal_id=? AND app_id=? AND environment=?", ("alice", "other-app", "test")), + ("principal_id=? AND app_id=? AND environment=?", ("alice", "app", "production"))]: + with store.db: + store.db.execute("UPDATE work_credentials_v2 SET ciphertext=? WHERE " + where, (row["ciphertext"], *args)) + for identity in [("test", "app", "bob"), ("test", "other-app", "alice"), ("production", "app", "alice")]: + with pytest.raises(CredentialError, match="decryption failed"): + vault.get(*identity) + + +@pytest.mark.parametrize("column,value", [("ciphertext", "broken"), ("key_id", "missing"), ("credential_version", 9)]) +def test_tampering_fails_closed(vault, column, value): + store, keys, vault = vault + vault.save("test", "app", "alice", expected_version=0, **payload()) + with store.db: + store.db.execute(f"UPDATE work_credentials_v2 SET {column}=?", (value,)) + with pytest.raises(CredentialError, match="decryption failed") as error: + vault.get("test", "app", "alice") + assert "test-refresh" not in str(error.value) + + +def test_modified_authenticated_ciphertext_and_wrong_key(vault): + store, _, vault = vault + vault.save("test", "app", "alice", expected_version=0, **payload()) + wrong = CredentialStore(store, CredentialKeyring("k1", {"k1": os.urandom(32)})) + with pytest.raises(CredentialError, match="decryption failed"): + wrong.get("test", "app", "alice") + row = store.db.execute("SELECT ciphertext FROM work_credentials_v2").fetchone() + raw = bytearray(base64.b64decode(row["ciphertext"])) + raw[-1] ^= 1 + with store.db: + store.db.execute("UPDATE work_credentials_v2 SET ciphertext=?", + (base64.b64encode(raw).decode(),)) + with pytest.raises(CredentialError, match="decryption failed"): + vault.get("test", "app", "alice") + + +def test_cas_rejects_stale_create_update_and_missing_record(vault): + _, _, vault = vault + vault.save("test", "app", "alice", expected_version=0, **payload()) + with pytest.raises(CredentialConflict): + vault.save("test", "app", "alice", expected_version=0, **payload("stale")) + assert vault.save("test", "app", "alice", expected_version=1, **payload("new")) == 2 + with pytest.raises(CredentialConflict): + vault.save("test", "app", "alice", expected_version=1, **payload("stale")) + with pytest.raises(CredentialConflict): + vault.save("test", "app", "missing", expected_version=1, **payload()) + assert vault.get("test", "app", "alice").refresh_token == "test-refresh-new" + + +def test_two_connections_only_one_writer_wins(vault, tmp_path): + _, keys, vault = vault + vault.save("test", "app", "alice", expected_version=0, **payload()) + barrier = Barrier(2) + + def write(suffix): + store = Store(tmp_path / "metadata.sqlite") + try: + candidate = CredentialStore(store, keys) + assert candidate.get("test", "app", "alice").version == 1 + barrier.wait(timeout=10) + try: + candidate.save("test", "app", "alice", expected_version=1, **payload(suffix)) + return "saved" + except CredentialConflict: + return "conflict" + finally: + store.close() + + with ThreadPoolExecutor(max_workers=2) as pool: + assert sorted(pool.map(write, ["one", "two"])) == ["conflict", "saved"] + assert vault.get("test", "app", "alice").version == 2 + + +def test_key_rotation_and_removed_key_fail_closed(vault): + store, keys, vault = vault + vault.save("test", "app", "alice", expected_version=0, **payload()) + new_key = os.urandom(32) + rotated = CredentialStore(store, CredentialKeyring("k2", {**keys._keys, "k2": new_key})) + old = rotated.get("test", "app", "alice") + assert old.version == 1 + no_old_key = CredentialStore(store, CredentialKeyring("k2", {"k2": new_key})) + with pytest.raises(CredentialError): + no_old_key.get("test", "app", "alice") + rotated.save("test", "app", "alice", expected_version=1, **payload("rotated")) + assert no_old_key.get("test", "app", "alice").refresh_token == "test-refresh-rotated" + + +@pytest.mark.parametrize("change", [dict(refresh_token=""), dict(access_token=None), + dict(access_expires_at=float("nan")), dict(refresh_expires_at=-1), dict(scopes="scope:a"), + dict(scopes=[1]), dict(access_expires_at=True), dict(refresh_expires_at=10**400)]) +def test_invalid_payload_does_not_write(vault, change): + _, _, vault = vault + with pytest.raises(CredentialError): + vault.save("test", "app", "alice", expected_version=0, **{**payload(), **change}) + assert vault.get("test", "app", "alice") is None + + +@pytest.mark.parametrize("version", [-1, True, "1", 2147483647]) +def test_invalid_version_rejected(vault, version): + _, _, vault = vault + with pytest.raises(CredentialError, match="invalid credential version"): + vault.save("test", "app", "alice", expected_version=version, **payload()) + + +def test_empty_identity_and_missing_key_file_fail_closed(vault, tmp_path): + _, _, vault = vault + with pytest.raises(CredentialError, match="invalid credential identity"): + vault.save("test", "app", "", expected_version=0, **payload()) + with pytest.raises(CredentialError, match="cannot load"): + CredentialKeyring.from_file(tmp_path / "missing.json") + + +def test_key_file_validation_and_safe_errors(tmp_path): + path = tmp_path / "keyring.json" + key = os.urandom(32) + path.write_text(json.dumps({"active_key_id": "k1", "keys": {"k1": base64.b64encode(key).decode()}})) + ring = CredentialKeyring.from_file(path) + key_id, cipher = ring.encrypt(b"test-secret", b"context") + assert ring.decrypt(key_id, cipher, b"context") == b"test-secret" + for content in ["not-json-test-secret", '{"active_key_id":"k1","keys":{"k1":"test-secret"}}', + '{"active_key_id":"missing","keys":{}}', '[]']: + path.write_text(content) + with pytest.raises(CredentialError, match="cannot load") as error: + CredentialKeyring.from_file(path) + assert "test-secret" not in str(error.value) + + +@pytest.mark.skipif(not os.environ.get("HN_TEST_DATABASE_URL"), reason="real PostgreSQL URL not supplied") +def test_postgres_encryption_cas_and_schema_reentry(tmp_path): + """Runs in a unique test schema only, never against existing production tables.""" + import uuid + import psycopg + from psycopg import sql + from psycopg.conninfo import make_conninfo + + url = os.environ["HN_TEST_DATABASE_URL"] + schema = "hn_credential_test_" + uuid.uuid4().hex + admin = psycopg.connect(url, autocommit=True) + admin.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + store = None + try: + store = Store(tmp_path / "unused.sqlite", make_conninfo(url, options=f"-c search_path={schema}")) + keys = CredentialKeyring("test-key", {"test-key": os.urandom(32)}) + vault = CredentialStore(store, keys) + CredentialStore(store, keys) # Additive initialization is repeatable. + vault.save("test", "app", "alice", expected_version=0, **payload()) + with pytest.raises(CredentialConflict): + vault.save("test", "app", "alice", expected_version=0, **payload("duplicate")) + assert vault.save("test", "app", "alice", expected_version=1, **payload("new")) == 2 + with pytest.raises(CredentialConflict): + vault.save("test", "app", "alice", expected_version=1, **payload("stale")) + assert vault.get("test", "app", "alice").refresh_token == "test-refresh-new" + row = store.db.execute("SELECT ciphertext FROM work_credentials_v2").fetchone() + assert "test-refresh" not in row["ciphertext"] + finally: + if store: + store.close() + admin.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) + admin.close()