diff --git a/deployable/CREDENTIAL_MIGRATION.md b/deployable/CREDENTIAL_MIGRATION.md index aeb2112..a966b44 100644 --- a/deployable/CREDENTIAL_MIGRATION.md +++ b/deployable/CREDENTIAL_MIGRATION.md @@ -198,3 +198,11 @@ Windows / Python 3.12.10,平台桩、临时证书/密钥、SQLite及带Lua的F 新增state只读查看停止/重启容器并对本地日志只提取固定错误类(包含Docker logs的stderr);run错误报告阶段/退出码而非原始配置。activate输出阶段,明确up失败可能部分切换。rollback的目标服务可为已停止的已知原/候选容器,不再因只查询运行容器而无法识别;未知镜像仍拒绝。 工具专项50项通过(新增state/部分回滚/错误脱敏4项、公钥修复5项、旧部分计划CLI回滚1项)。完整本地回归262 passed、3 skipped、30 warnings(49.76秒),之后追加的回滚入口检查再次通过工具50项;Python3.7语法与diff检查通过。真实PostgreSQL仍未提供。实际修复结果仍需现场执行并核验,不能把本地Docker桩通过当作router已恢复。 + +### 无Healthcheck容器导致修复前检查退出 + +现场repair-router未进入修改步骤即报docker-command失败,state能读connector/router/signer,却将longconn/edge显示为no unique。核对新版inspect模板使用`.State.Health`可选键,longconn/edge没有健康检查时Docker map模板会失败;state原先又把所有DeployError误归为容器数量问题。 + +改为`with index .State "Health"`安全读取,无字段记health=none;单独区分ServiceLookupError和容器inspect错误,命令失败阶段明确为container-inspect。新增2项回归模拟无健康检查的现有服务,覆盖state与整条部分切换repair流程以及错误分类,工具52项通过。未改配置、已生成keyring或运行容器;仍需现场更新并重跑修复核验。 + +最后完整本地回归:**265 passed,3 skipped,30 warnings**(63.22秒);真实PostgreSQL未配置,warnings类型同上,Python3.7语法及diff检查通过。 diff --git a/deployable/EXISTING_ECS_BROKER_DEPLOY.md b/deployable/EXISTING_ECS_BROKER_DEPLOY.md index 5d3cfac..240e4d8 100644 --- a/deployable/EXISTING_ECS_BROKER_DEPLOY.md +++ b/deployable/EXISTING_ECS_BROKER_DEPLOY.md @@ -124,6 +124,8 @@ sudo python3 deployable/scripts/broker_deploy.py repair-router \ 新prepare已包含此参数;build/activate的隔离容器现在会实际加载挂载的router公钥,不能只验证Settings字段就标记候选就绪。`state`不打印原始日志或环境,只报告原/候选镜像类别、容器状态/健康/退出码和固定错误类。已部分切换时rollback可识别停止或重启中的已知候选容器,但拒绝未知镜像;原有回滚限制仍适用。 +兼容修订:没有HEALTHCHECK的longconn/edge不包含State.Health,旧查询模板会因此在修复开始前失败。新模板使用可选map键读取并将其健康状态记为none;容器inspect失败与确实查不到唯一容器分别报告。更新后对原计划再次执行repair-router即可,不需重新prepare、删除备份或生成密钥。 + ### 完整回退 若切换失败,先停止broker客户端继续使用,按记录的每服务镜像回退: diff --git a/deployable/scripts/broker_deploy.py b/deployable/scripts/broker_deploy.py index f76f50f..c8dec23 100644 --- a/deployable/scripts/broker_deploy.py +++ b/deployable/scripts/broker_deploy.py @@ -23,6 +23,10 @@ class DeployError(RuntimeError): pass +class ServiceLookupError(DeployError): + pass + + def run(args, json_result=False, optional=False, input_text=None): result = subprocess.run(args, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) if result.returncode: @@ -34,6 +38,8 @@ def run(args, json_result=False, optional=False, input_text=None): stage = "candidate-runtime-check" elif args[:3] == ["docker","image","inspect"]: stage = "image-inspect" + elif args[:2] == ["docker","inspect"]: + stage = "container-inspect" else: stage = "docker-command" raise DeployError(stage+" failed (exit "+str(result.returncode)+"); raw output withheld; run state to inspect current services") @@ -73,7 +79,7 @@ def inspect(container): template = ('{"id":{{json .Id}},"image":{{json .Image}},' '"image_ref":{{json .Config.Image}},"labels":{{json .Config.Labels}},' '"state":{"status":{{json .State.Status}},"health":' - '{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},' + '{{with index .State "Health"}}{{json .Status}}{{else}}"none"{{end}},' '"exit_code":{{.State.ExitCode}},"restart_count":{{.RestartCount}}}}') return run(["docker", "inspect", container, "--format", template], True) @@ -82,7 +88,7 @@ def service(project, name, include_stopped=False): values = run(["docker", "ps", "-aq" if include_stopped else "-q", "--filter", "label=com.docker.compose.project="+project, "--filter", "label=com.docker.compose.service="+name]).splitlines() if len(values) != 1: - raise DeployError("expected exactly one "+("existing" if include_stopped else "running")+" relay service: "+name) + raise ServiceLookupError("expected exactly one "+("existing" if include_stopped else "running")+" relay service: "+name) return inspect(values[0]) @@ -393,9 +399,12 @@ def deployment_state(root,plan): for name in ("connector","router","longconn","enrollment-signer","edge"): try: entry = service(plan["project"],name,include_stopped=True) - except DeployError: + except ServiceLookupError: print(name+": no unique existing container; inspect docker ps -a for this project") continue + except DeployError: + print(name+": container inspection failed; no container state inferred") + continue state = entry.get("state",{}) status = state.get("status") if state.get("status") in {'created','running','paused','restarting','removing','exited','dead'} else 'unknown' health = state.get("health") if state.get("health") in {'none','starting','healthy','unhealthy'} else 'unknown' diff --git a/deployable/tests/test_broker_deploy.py b/deployable/tests/test_broker_deploy.py index 71a59c6..7483674 100644 --- a/deployable/tests/test_broker_deploy.py +++ b/deployable/tests/test_broker_deploy.py @@ -344,6 +344,50 @@ def test_state_sees_exited_candidate_without_showing_logs_or_secrets(scenario,mo assert not any('up' in c or 'tag' in c for c in scenario['calls']) +def test_containers_without_healthcheck_do_not_break_state_or_router_repair(scenario,monkeypatch,capsys): + plan=partial_old_plan(scenario) + original=deploy.run + inspected=[] + def optional_health(args,*a,**kw): + if args[:2]==['docker','inspect']: + template=args[args.index('--format')+1] + name=args[2].replace('container-','').replace('hainuo-relay-','').removesuffix('-1') + inspected.append(name) + if name in {'longconn','edge'}: + # Docker map templates fail on missing .State.Health; index + # returns nil and the with/else branch emits health="none". + if '.State.Health' in template: + raise deploy.DeployError('container-inspect failed; no map entry') + assert 'index .State "Health"' in template + result=copy.deepcopy(original(args,*a,**kw)) + result['state']={'status':'running','health':'none','exit_code':0,'restart_count':0} + return result + return original(args,*a,**kw) + monkeypatch.setattr(deploy,'run',optional_health) + deploy.deployment_state(scenario['output'],plan) + output=capsys.readouterr().out + assert 'longconn: image=original status=running health=none' in output + assert 'edge: image=original status=running health=none' in output + deploy.repair_router(scenario['output'],plan) + assert {'longconn','edge','connector','router'}.issubset(inspected) + commands=[c for c in scenario['calls'] if 'up' in c] + assert len(commands)==1 and commands[0][-1]=='router' + + +def test_state_does_not_misreport_inspect_failure_as_missing_container(scenario,monkeypatch,capsys): + plan=prepared(scenario) + original=deploy.run + def failure(args,*a,**kw): + if args[:2]==['docker','inspect'] and args[2]=='container-edge': + raise deploy.DeployError('container-inspect failed') + return original(args,*a,**kw) + monkeypatch.setattr(deploy,'run',failure) + deploy.deployment_state(scenario['output'],plan) + output=capsys.readouterr().out + assert 'edge: container inspection failed' in output + assert 'edge: no unique existing container' not in output + + def test_rollback_checks_allow_stopped_known_candidate_but_reject_other_image(scenario): plan=prepared(scenario) scenario['roles']['connector']['image']='sha256:'+'a'*64