diff --git a/deployable/CREDENTIAL_MIGRATION.md b/deployable/CREDENTIAL_MIGRATION.md index 7893abf..aeb2112 100644 --- a/deployable/CREDENTIAL_MIGRATION.md +++ b/deployable/CREDENTIAL_MIGRATION.md @@ -188,3 +188,13 @@ Windows / Python 3.12.10,平台桩、临时证书/密钥、SQLite及带Lua的F 新增11项工具回归,覆盖合法环境保持、HTTPS/WSS/数据库失败在写密钥前拒绝、只读diagnose保留全部准备文件、错误不泄露数据库凭据;工具专项40项通过。实际现场哪项原始条件不符仍以更新后的diagnose输出为准,不将历史默认值当作当前配置证明。 修订后完整本地回归:**253 passed,3 skipped,30 warnings**(49.46秒);真实PostgreSQL仍未提供,warnings类型同上,Python3.7语法与diff检查通过。 + +### 部分切换故障与router公钥路径修复 + +现场activate只输出通用Docker失败,用户随后提供状态:connector已是broker候选并healthy,router同候选持续Restarting(1)。进一步只读确认router未配置HN_CONNECTOR_PUBLIC_KEY,实际日志FileNotFoundError指向默认runtime/pki/connector.pub。这是旧router此前不消费connector公钥,而新模式开始验证路由签名后,迁移叠加漏传原信任目录公钥路径;首次import/Settings探测未实际打开公钥,没提前发现。 + +修订新prepare识别现有只读信任挂载中的公钥并显式传HN_CONNECTOR_PUBLIC_KEY,runtime_check挂载并实际load_public。新增repair-router面向已知健康候选connector的部分切换,备份叠加/plan、保留keyring/原.env/其他镜像,只补router路径并重建router。原密钥或运行配置漂移时拒绝,不自动删除/重准备。 + +新增state只读查看停止/重启容器并对本地日志只提取固定错误类(包含Docker logs的stderr);run错误报告阶段/退出码而非原始配置。activate输出阶段,明确up失败可能部分切换。rollback的目标服务可为已停止的已知原/候选容器,不再因只查询运行容器而无法识别;未知镜像仍拒绝。 + +工具专项50项通过(新增state/部分回滚/错误脱敏4项、公钥修复5项、旧部分计划CLI回滚1项)。完整本地回归262 passed、3 skipped、30 warnings(49.76秒),之后追加的回滚入口检查再次通过工具50项;Python3.7语法与diff检查通过。真实PostgreSQL仍未提供。实际修复结果仍需现场执行并核验,不能把本地Docker桩通过当作router已恢复。 diff --git a/deployable/EXISTING_ECS_BROKER_DEPLOY.md b/deployable/EXISTING_ECS_BROKER_DEPLOY.md index abfbf45..5d3cfac 100644 --- a/deployable/EXISTING_ECS_BROKER_DEPLOY.md +++ b/deployable/EXISTING_ECS_BROKER_DEPLOY.md @@ -102,6 +102,30 @@ sudo python3 deployable/scripts/broker_deploy.py activate \ ## 6. 应急容器回退 +### 已部分切换且router缺少公钥参数 + +现场已确认一种旧配置兼容遗漏:connector候选健康,router不断重启,日志报`FileNotFoundError: runtime/pki/connector.pub`;router原环境仅有`HN_CA_CERT=/trust/ca.pem`,没有`HN_CONNECTOR_PUBLIC_KEY`。不要修改公钥内容、重新生成CA或keyring,不要重复activate。 + +更新暂存仓库后可先只读查看包含停止/重启中容器的状态和固定错误类型: + +```bash +sudo python3 deployable/scripts/broker_deploy.py state \ + --plan /home/cnooc/hainuo-broker-prepared-20261104 +``` + +仅在已知候选connector运行且healthy时执行: + +```bash +sudo python3 deployable/scripts/broker_deploy.py repair-router \ + --plan /home/cnooc/hainuo-broker-prepared-20261104 +``` + +该操作先验证原配置/密钥/准备目录完整性及已知候选镜像,从router已有只读信任挂载识别实际公钥位置(该现场为`/trust/connector.pub`),备份原叠加和plan后只增加router公钥参数并同步计划校验。不会重生成任何密钥、修改原.env、重建connector或其他服务。重跑隔离配置/公钥实际读取检查后,只up router;重复执行不会覆盖备份或改密钥。 + +新prepare已包含此参数;build/activate的隔离容器现在会实际加载挂载的router公钥,不能只验证Settings字段就标记候选就绪。`state`不打印原始日志或环境,只报告原/候选镜像类别、容器状态/健康/退出码和固定错误类。已部分切换时rollback可识别停止或重启中的已知候选容器,但拒绝未知镜像;原有回滚限制仍适用。 + +### 完整回退 + 若切换失败,先停止broker客户端继续使用,按记录的每服务镜像回退: ```bash diff --git a/deployable/scripts/broker_deploy.py b/deployable/scripts/broker_deploy.py index 59a546b..f76f50f 100644 --- a/deployable/scripts/broker_deploy.py +++ b/deployable/scripts/broker_deploy.py @@ -9,12 +9,13 @@ import base64 import hashlib import json import os -from pathlib import Path +from pathlib import Path, PurePosixPath import re import secrets import shutil import subprocess import sys +import tempfile from urllib.parse import urlsplit @@ -27,7 +28,17 @@ def run(args, json_result=False, optional=False, input_text=None): if result.returncode: if optional: return None - raise DeployError("Docker/Git command failed; raw output withheld to protect configuration") + if args[:2] == ["docker","compose"]: + stage = "compose-up" if "up" in args else "compose-config" + elif args[:2] == ["docker","run"]: + stage = "candidate-runtime-check" + elif args[:3] == ["docker","image","inspect"]: + stage = "image-inspect" + else: + stage = "docker-command" + raise DeployError(stage+" failed (exit "+str(result.returncode)+"); raw output withheld; run state to inspect current services") + if args[:2] == ["docker","logs"]: + return (result.stdout+"\n"+result.stderr).strip() try: return json.loads(result.stdout) if json_result else result.stdout.strip() except ValueError: @@ -46,17 +57,32 @@ def write_json(path, value): stream.write("\n") +def replace_json(path,value): + name = None + try: + fd,name = tempfile.mkstemp(prefix='.'+path.name+'.',dir=str(path.parent)) + with os.fdopen(fd,'w',encoding='utf-8') as stream: + json.dump(value,stream,ensure_ascii=False,indent=2) + stream.write('\n');stream.flush();os.fsync(stream.fileno()) + os.replace(name,str(path)) + finally: + if name and os.path.exists(name):os.unlink(name) + + def inspect(container): template = ('{"id":{{json .Id}},"image":{{json .Image}},' - '"image_ref":{{json .Config.Image}},"labels":{{json .Config.Labels}}}') + '"image_ref":{{json .Config.Image}},"labels":{{json .Config.Labels}},' + '"state":{"status":{{json .State.Status}},"health":' + '{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},' + '"exit_code":{{.State.ExitCode}},"restart_count":{{.RestartCount}}}}') return run(["docker", "inspect", container, "--format", template], True) -def service(project, name): - values = run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project="+project, +def service(project, name, include_stopped=False): + values = run(["docker", "ps", "-aq" if include_stopped else "-q", "--filter", "label=com.docker.compose.project="+project, "--filter", "label=com.docker.compose.service="+name]).splitlines() if len(values) != 1: - raise DeployError("expected exactly one running relay service: "+name) + raise DeployError("expected exactly one "+("existing" if include_stopped else "running")+" relay service: "+name) return inspect(values[0]) @@ -125,7 +151,30 @@ def require_platform(env): raise DeployError("broker platform configuration incomplete; "+"; ".join(problems)) -def validate_overlay(base, merged, candidate): +def router_public_key(config): + router=config['services']['router'] + env=router.get('environment',{}) + value=env.get('HN_CONNECTOR_PUBLIC_KEY') or str(PurePosixPath(env.get('HN_CA_CERT','/trust/ca.pem')).parent/'connector.pub') + target=PurePosixPath(value) + if not target.is_absolute() or '..' in target.parts: + raise DeployError('invalid router connector public-key path') + for mount in router.get('volumes',[]): + if mount.get('type')!='bind' or not mount.get('read_only'):continue + base=PurePosixPath(mount['target']) + try: + relative=target.relative_to(base) + except ValueError: + continue + source=Path(mount['source']) + source=source.joinpath(*relative.parts) if relative.parts else source + if source.is_file() and not source.is_symlink(): + data=source.read_bytes() + if b'-----BEGIN PUBLIC KEY-----' in data and b'PRIVATE KEY' not in data: + return str(target),source + raise DeployError('router connector public key is not a verified read-only trust mount') + + +def validate_overlay(base, merged, candidate, router_key_required=True): old, new = base["services"], merged["services"] require_services(old) require_services(new) @@ -172,6 +221,10 @@ def validate_overlay(base, merged, candidate): mounts = [v for v in new["connector"].get("volumes", []) if v["target"] == "/run/hainuo-credentials/keyring.json"] if len(mounts) != 1 or not mounts[0].get("read_only"): raise DeployError("missing read-only credential keyring mount") + if router_key_required: + target,_=router_public_key(merged) + if new['router']['environment'].get('HN_CONNECTOR_PUBLIC_KEY')!=target: + raise DeployError('router HN_CONNECTOR_PUBLIC_KEY missing; use repair-router for an already switched plan') def preflight(args): @@ -249,6 +302,7 @@ def prepare(args): if base["services"]["connector"].get("environment",{}).get("HN_CREDENTIAL_MODE","legacy") != "legacy": raise DeployError("already broker: preserve the existing keyring and migration plan") require_platform({**base["services"]["connector"].get("environment",{}),"HN_OAUTH_ENABLED":"true"}) + public_key,_=router_public_key(base) output.mkdir(mode=0o700,parents=False) os.chmod(str(output),0o700) keyring = {"active_key_id":"initial-v1","keys":{"initial-v1":base64.b64encode(os.urandom(32)).decode("ascii")}} @@ -264,6 +318,7 @@ def prepare(args): "volumes":[{"type":"bind","source":str(output/"credential-keyring.json"), "target":"/run/hainuo-credentials/keyring.json","read_only":True}]}, "router":{"image":candidate,"environment":{"HN_CREDENTIAL_MODE":"broker", + "HN_CONNECTOR_PUBLIC_KEY":public_key, "HN_ROUTER_ADMISSION_SECRET":admission,"HN_CONNECTOR_INTERNAL_URL":"http://connector:8000", "HN_FEISHU_APP_ID":"${HN_FEISHU_APP_ID:?Configure existing application}", "HN_ENVIRONMENT":"${HN_ENVIRONMENT:?Configure existing environment}"}}}} @@ -317,7 +372,45 @@ def diagnose(root,plan): raise DeployError("review the named fields above; do not paste configuration or regenerate keys") -def check(root, plan, baseline=True): +def log_signals(text): + """Classify locally; never return arbitrary log lines, SQL, URLs or tokens.""" + known = ('ModuleNotFoundError','ImportError','FileNotFoundError','PermissionError','ValidationError', + 'CredentialError','OperationalError','ProgrammingError','UndefinedColumn','UndefinedTable', + 'InsufficientPrivilege','ReadOnlySqlTransaction','SyntaxError','IndentationError', + 'ConnectionRefusedError','AuthenticationError','ConnectionError','TimeoutError','MemoryError') + signals = [name for name in known if re.search(r'\b'+name+r'\b',text)] + if 'read-only file system' in text.lower(): + signals.append('read-only-filesystem') + if 'address already in use' in text.lower(): + signals.append('port-in-use') + return signals + + +def deployment_state(root,plan): + expected = None + if (root/"candidate.json").is_file(): + expected = json.loads((root/"candidate.json").read_text(encoding="utf-8")).get("image") + for name in ("connector","router","longconn","enrollment-signer","edge"): + try: + entry = service(plan["project"],name,include_stopped=True) + except DeployError: + print(name+": no unique existing container; inspect docker ps -a for this project") + continue + state = entry.get("state",{}) + status = state.get("status") if state.get("status") in {'created','running','paused','restarting','removing','exited','dead'} else 'unknown' + health = state.get("health") if state.get("health") in {'none','starting','healthy','unhealthy'} else 'unknown' + image = 'candidate' if expected and entry['image']==expected else ( + 'original' if entry['image']==plan['roles'][name]['image'] else 'different') + numbers = {k:v for k,v in state.items() if k in {'exit_code','restart_count'} and type(v) is int} + print(name+': image='+image+' status='+status+' health='+health+' '+json.dumps(numbers,sort_keys=True)) + if name in {'connector','router'}: + raw = run(['docker','logs','--tail','100',entry['id']],optional=True) + if raw is not None: + print(' startup signals: '+(', '.join(log_signals(raw)) or 'no recognized error class; raw logs withheld')) + print('STATE inspection only; no files, keys or containers changed. Do not retry activation before reviewing partial switch state.') + + +def check(root, plan, baseline=True, router_key_required=True): if os.name != "nt": if root.stat().st_mode & 0o077: raise DeployError("migration directory must not be accessible to other users") @@ -336,12 +429,18 @@ def check(root, plan, baseline=True): if digest(root/path) != expected: raise DeployError("prepared keyring or overlay changed; do not regenerate credential keys") for name, old in plan["roles"].items(): - current = service(plan["project"],name) + current = service(plan["project"],name,include_stopped=not baseline and name in {"connector","router"}) if (baseline or name not in {"connector","router"}) and current["image"] != old["image"]: raise DeployError("running service image changed since preparation: "+name) + if not baseline and name in {"connector","router"}: + allowed = {old['image']} + if (root/'candidate.json').is_file(): + allowed.add(json.loads((root/'candidate.json').read_text(encoding='utf-8'))['image']) + if current['image'] not in allowed: + raise DeployError('unreviewed target image; refusing rollback: '+name) base = run(compose(plan)+["config","--format","json"],True) merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True) - validate_overlay(base,merged,plan["candidate_image"]) + validate_overlay(base,merged,plan["candidate_image"],router_key_required=router_key_required) # Require the current edge's explicit private-path block, before catch-all handling. mounts = base["services"]["edge"].get("volumes",[]) caddy = next((v["source"] for v in mounts if v.get("target") == "/etc/caddy/Caddyfile" and v.get("type") == "bind"),None) @@ -353,7 +452,7 @@ def check(root, plan, baseline=True): first_handle = re.search(r"\bhandle(?:_path)?\b",text) if not deny or (first_handle and deny.start() > first_handle.start()): raise DeployError("edge private-path block is not the reviewed configuration") - print("CHECKED: existing Compose/storage/images retained; no services restarted") + print("CHECKED: Compose/storage and permitted service images verified; no services restarted") def build(root,plan): @@ -374,30 +473,75 @@ def build(root,plan): def runtime_check(root,plan): merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True) + public_key,public_source=router_public_key(merged) # Actual secrets travel via stdin only: no command-line values, env dump, # database connection or platform request. The container has no network. code = ("import json,os,sys\nfrom pathlib import Path\nfrom relay.config import Settings\n" "from relay.credential_store import CredentialKeyring\n" + "from relay.crypto import load_public\n" "try:\n data=json.load(sys.stdin)\n os.environ.update(data['environment'])\n" " s=Settings(role=data['role'])\n" " if data['role']=='connector': CredentialKeyring.from_file(s.credential_keyring_file)\n" + " if data['role']=='router': load_public(s.connector_public_key)\n" "except Exception:\n sys.exit(1)\n") for role in ("connector","router"): env = {key:str(value) for key,value in merged["services"][role]["environment"].items() if value is not None} - run(["docker","run","--rm","-i","--network","none","--read-only","--entrypoint","python", + command=["docker","run","--rm","-i","--network","none","--read-only","--entrypoint","python", "-v",str(root/"credential-keyring.json")+":/run/hainuo-credentials/keyring.json:ro", - plan["candidate_image"],"-c",code],input_text=json.dumps({"role":role,"environment":env})) + "-v",str(public_source)+":"+public_key+":ro",plan["candidate_image"],"-c",code] + print('Runtime check: '+role+' configuration and required key files',flush=True) + run(command,input_text=json.dumps({"role":role,"environment":env})) + + +def repair_router(root,plan): + # Read all known partial-switch states, but do not weaken general activate checks. + check(root,plan,baseline=False,router_key_required=False) + built=json.loads((root/'candidate.json').read_text(encoding='utf-8')) + connector=service(plan['project'],'connector',include_stopped=True) + if (connector['image']!=built['image'] or connector.get('state',{}).get('status')!='running' + or connector.get('state',{}).get('health')!='healthy'): + raise DeployError('repair-router requires the known candidate connector to be running and healthy; inspect state or rollback') + base=run(compose(plan)+['config','--format','json'],True) + target,_=router_public_key(base) + path=root/'compose.broker.json' + overlay=json.loads(path.read_text(encoding='utf-8')) + old=overlay['services']['router']['environment'].get('HN_CONNECTOR_PUBLIC_KEY') + if old not in (None,'',target): + raise DeployError('unexpected prepared router public-key path; manual review required') + if old!=target: + backup=root/'compose.broker.before-router-key.json' + plan_backup=root/'plan.before-router-key.json' + if backup.exists() or plan_backup.exists(): + raise DeployError('repair backup already exists; preserve keys and inspect the interrupted repair') + write_json(backup,overlay);write_json(plan_backup,plan) + overlay['services']['router']['environment']['HN_CONNECTOR_PUBLIC_KEY']=target + replace_json(path,overlay) + plan['prepared_hashes']['compose.broker.json']=digest(path) + replace_json(root/'plan.json',plan) + print('REPAIR: existing mounted connector public key selected; credential keys and connector preserved',flush=True) + check(root,plan,baseline=False) + runtime_check(root,plan) + image=run(['docker','image','inspect',plan['candidate_image'],'--format','{{.Id}}']) + if image!=built['image']: + raise DeployError('candidate image identity changed') + print('REPAIR: recreate router only',flush=True) + run(compose(plan,root/'compose.broker.json')+['up','-d','--no-deps','--no-build','--pull','never', + '--wait','--wait-timeout','180','router']) + print('REPAIRED: router only; verify state and real client handshake before acceptance') def activate(root,plan,ready): if not ready: raise DeployError("v2 client not confirmed ready; only prepare/check/build are allowed now") check(root,plan) + print('ACTIVATION step: verify candidate image identity',flush=True) built = json.loads((root/"candidate.json").read_text(encoding="utf-8")) image = run(["docker","image","inspect",plan["candidate_image"],"--format","{{.Id}}"]) if built != {"image":image,"tag":plan["candidate_image"]}: raise DeployError("candidate image identity changed") + print('ACTIVATION step: isolated candidate configuration/keyring check (no service switch)',flush=True) runtime_check(root,plan) + print('ACTIVATION step: recreate connector/router; failures may leave a partial switch, inspect state',flush=True) run(compose(plan,root/"compose.broker.json")+["up","-d","--no-deps","--no-build","--pull","never", "--wait","--wait-timeout","180","connector","router"]) print("ACTIVATED: connector/router only. Verify real OAuth, v2 client, unbind and task routing.") @@ -415,7 +559,7 @@ def main(): prep.add_argument("--profiles",required=True) prep.add_argument("--output",required=True) prep.add_argument("--connector-container",default="hainuo-relay-connector-1") - for name in ("diagnose","check","build","activate","rollback"): + for name in ("repair-router","state","diagnose","check","build","activate","rollback"): cmd = sub.add_parser(name) cmd.add_argument("--plan",required=True) if name == "activate": @@ -432,7 +576,11 @@ def main(): return root = Path(args.plan).resolve() plan = load_plan(root) - if args.action == "diagnose": + if args.action == "repair-router": + repair_router(root,plan) + elif args.action == "state": + deployment_state(root,plan) + elif args.action == "diagnose": diagnose(root,plan) elif args.action == "check": check(root,plan) @@ -443,7 +591,7 @@ def main(): else: if not args.stop_broker_use: raise DeployError("rollback requires stopping broker clients; old refresh credentials are not restored") - check(root,plan,baseline=False) + check(root,plan,baseline=False,router_key_required=False) run(compose(plan,root/"compose.rollback.json")+["up","-d","--no-deps","--no-build","--pull","never", "--wait","--wait-timeout","180","connector","router"]) print("ROLLED BACK containers only; preserve keyring/database. Client mode and reauthorization must be coordinated.") diff --git a/deployable/tests/test_broker_deploy.py b/deployable/tests/test_broker_deploy.py index 10273fb..71a59c6 100644 --- a/deployable/tests/test_broker_deploy.py +++ b/deployable/tests/test_broker_deploy.py @@ -38,6 +38,10 @@ def scenario(tmp_path,monkeypatch): base = {"services":{name:{"image":"old:tag","environment":{},"networks":{"backend":None},"volumes":[]} for name in [*roles,"postgres","redis"]},"volumes":{"metadata":{}},"networks":{"backend":{}}} base["services"]["connector"]["environment"] = env + trust=live/'runtime/trust';trust.mkdir(parents=True) + (trust/'connector.pub').write_text('-----BEGIN PUBLIC KEY-----\nsynthetic-public-material\n-----END PUBLIC KEY-----\n') + base['services']['router']['environment']['HN_CA_CERT']='/trust/ca.pem' + base['services']['router']['volumes']=[{'type':'bind','source':str(trust),'target':'/trust','read_only':True}] base["services"]["connector"]["volumes"] = [{"type":"bind","source":str(live/"runtime"),"target":"/data"}] base["services"]["edge"]["volumes"] = [{"type":"bind","source":str(caddy),"target":"/etc/caddy/Caddyfile"}] profiles = tmp_path/"product.json" @@ -135,10 +139,13 @@ def test_prepare_preserves_all_seven_files_runtime_identity_and_no_service_chang plan = prepared(scenario) assert len(plan["compose_files"]) == 7 assert (scenario["live"]/".env").read_bytes() == original - assert not (scenario["live"]/"runtime").exists() + assert sorted(p.relative_to(scenario['live']/'runtime').as_posix() + for p in (scenario['live']/'runtime').rglob('*') if p.is_file())==['trust/connector.pub'] + assert not (scenario['live']/'runtime/issuer').exists() overlay = json.loads((scenario["output"]/"compose.broker.json").read_text()) assert set(overlay["services"]) == {"connector","router"} assert overlay["services"]["connector"]["environment"]["HN_CREDENTIAL_MODE"] == "broker" + assert overlay['services']['router']['environment']['HN_CONNECTOR_PUBLIC_KEY']=='/trust/connector.pub' assert len(json.loads((scenario["output"]/"credential-keyring.json").read_text())["keys"]) == 1 rollback = json.loads((scenario["output"]/"compose.rollback.json").read_text()) assert rollback["services"]["connector"]["image"] != rollback["services"]["router"]["image"] @@ -317,6 +324,132 @@ def test_activation_validation_failure_never_recreates_services(scenario,monkeyp assert not any("up" in c for c in scenario["calls"]) +def test_state_sees_exited_candidate_without_showing_logs_or_secrets(scenario,monkeypatch,capsys): + plan = prepared(scenario) + deploy.write_json(scenario['output']/'candidate.json',{'image':scenario['roles']['connector']['image'],'tag':plan['candidate_image']}) + scenario['roles']['connector']['state']={'status':'exited','health':'unhealthy','exit_code':1,'restart_count':7} + original=deploy.run + def logs(args,*a,**kw): + if args[:2]==['docker','logs']: + scenario['calls'].append(args) + return 'Traceback PermissionError synthetic-private-password https://secret.invalid/private' + return original(args,*a,**kw) + monkeypatch.setattr(deploy,'run',logs) + scenario['calls'].clear() + deploy.deployment_state(scenario['output'],plan) + output=capsys.readouterr().out + assert 'image=candidate status=exited health=unhealthy' in output + assert 'PermissionError' in output and 'synthetic-private-password' not in output and 'secret.invalid' not in output + assert all('-aq' in c for c in scenario['calls'] if c[:2]==['docker','ps']) + assert not any('up' in c or 'tag' in c for c in scenario['calls']) + + +def test_rollback_checks_allow_stopped_known_candidate_but_reject_other_image(scenario): + plan=prepared(scenario) + scenario['roles']['connector']['image']='sha256:'+'a'*64 + deploy.write_json(scenario['output']/'candidate.json',{'image':'sha256:'+'a'*64,'tag':plan['candidate_image']}) + scenario['calls'].clear() + deploy.check(scenario['output'],plan,baseline=False) + assert any(c[:2]==['docker','ps'] and '-aq' in c for c in scenario['calls']) + scenario['roles']['router']['image']='sha256:'+'f'*64 + with pytest.raises(deploy.DeployError,match='unreviewed target image'): + deploy.check(scenario['output'],plan,baseline=False) + + +def partial_old_plan(scenario): + plan=prepared(scenario) + overlay_path=scenario['output']/'compose.broker.json' + overlay=json.loads(overlay_path.read_text()) + overlay['services']['router']['environment'].pop('HN_CONNECTOR_PUBLIC_KEY') + deploy.replace_json(overlay_path,overlay) + plan['prepared_hashes']['compose.broker.json']=deploy.digest(overlay_path) + deploy.replace_json(scenario['output']/'plan.json',plan) + image='sha256:'+'a'*64 + deploy.write_json(scenario['output']/'candidate.json',{'image':image,'tag':plan['candidate_image']}) + scenario['roles']['connector']['image']=image + scenario['roles']['connector']['state']={'status':'running','health':'healthy'} + scenario['roles']['router']['image']=image + scenario['roles']['router']['state']={'status':'restarting','health':'unhealthy'} + return plan + + +def test_repair_old_partial_switch_only_restarts_router_preserving_keys(scenario): + plan=partial_old_plan(scenario) + key=(scenario['output']/'credential-keyring.json').read_bytes() + original_env=(scenario['live']/'.env').read_bytes() + scenario['calls'].clear() + deploy.repair_router(scenario['output'],plan) + assert (scenario['output']/'credential-keyring.json').read_bytes()==key + assert (scenario['live']/'.env').read_bytes()==original_env + fixed=json.loads((scenario['output']/'compose.broker.json').read_text()) + assert fixed['services']['router']['environment']['HN_CONNECTOR_PUBLIC_KEY']=='/trust/connector.pub' + saved=deploy.load_plan(scenario['output']) + assert saved['prepared_hashes']['compose.broker.json']==deploy.digest(scenario['output']/'compose.broker.json') + commands=[c for c in scenario['calls'] if 'up' in c] + assert len(commands)==1 and commands[0][-1]=='router' and commands[0][-2]=='180' + assert '--no-deps' in commands[0] and commands[0][-2:]!=['connector','router'] + assert (scenario['output']/'plan.before-router-key.json').is_file() + assert (scenario['output']/'compose.broker.before-router-key.json').is_file() + + +def test_repair_retry_does_not_regenerate_key_or_overwrite_backup(scenario): + plan=partial_old_plan(scenario) + deploy.repair_router(scenario['output'],plan) + files={p.name:p.read_bytes() for p in scenario['output'].iterdir() if p.is_file()} + deploy.repair_router(scenario['output'],deploy.load_plan(scenario['output'])) + assert files=={p.name:p.read_bytes() for p in scenario['output'].iterdir() if p.is_file()} + + +def test_repair_requires_healthy_known_connector(scenario): + plan=partial_old_plan(scenario) + scenario['roles']['connector']['state']['health']='unhealthy' + before=(scenario['output']/'compose.broker.json').read_bytes() + with pytest.raises(deploy.DeployError,match='healthy'): + deploy.repair_router(scenario['output'],plan) + assert (scenario['output']/'compose.broker.json').read_bytes()==before + assert not any('up' in c for c in scenario['calls']) + + +def test_old_plan_fails_normal_check_until_explicit_repair(scenario): + plan=partial_old_plan(scenario) + with pytest.raises(deploy.DeployError,match='HN_CONNECTOR_PUBLIC_KEY'): + deploy.check(scenario['output'],plan,baseline=False) + + +def test_cli_rollback_of_old_partial_plan_does_not_require_new_router_key(scenario,monkeypatch): + partial_old_plan(scenario) + monkeypatch.setattr(deploy.sys,'argv',['broker_deploy.py','rollback','--plan',str(scenario['output']),'--stop-broker-use']) + scenario['calls'].clear() + deploy.main() + commands=[c for c in scenario['calls'] if 'up' in c] + assert len(commands)==1 and commands[0][-2:]==['connector','router'] + assert str(scenario['output']/'compose.rollback.json') in commands[0] + + +def test_missing_trust_public_key_refused_before_any_new_credentials(scenario): + trust=Path(scenario['base']['services']['router']['volumes'][0]['source']) + (trust/'connector.pub').unlink() + with pytest.raises(deploy.DeployError,match='public key'): + deploy.prepare(scenario['args']) + assert not scenario['output'].exists() + + +def test_run_failure_identifies_stage_without_echoing_command_secrets(monkeypatch): + import types + def fail(*a,**kw): + return types.SimpleNamespace(returncode=17,stdout='synthetic-access',stderr='synthetic-secret') + monkeypatch.setattr(deploy.subprocess,'run',fail) + with pytest.raises(deploy.DeployError) as error: + deploy.run(['docker','compose','--env-file','sensitive-path','up','connector']) + assert 'compose-up' in str(error.value) and '17' in str(error.value) + assert 'synthetic' not in str(error.value) and 'sensitive-path' not in str(error.value) + + +def test_log_signals_only_emit_fixed_error_classes(): + assert deploy.log_signals('PermissionError: synthetic-secret\nread-only file system')==['PermissionError','read-only-filesystem'] + assert deploy.log_signals('fake-secret https://secret.invalid')==[] + + def test_catchall_before_private_block_requires_manual_review(scenario): plan = prepared(scenario) original = scenario["caddy"].read_text()