fix(deploy): tolerate absent Docker health state during router recovery

Containers without a configured HEALTHCHECK omit State.Health. Direct map
field access in the new inspect template made recovery prechecks fail on
longconn/edge before the router path could be repaired. Use a guarded map
index and report health=none without weakening image/config/key checks.

Distinguish service lookup errors from inspect/template failures instead of
misreporting every query failure as no unique container. Identify the
container-inspect stage in safe command errors.

Add regressions for absent health state through state and the full partial
router repair, plus accurate inspect failure classification. Validation:
265 passed, 3 skipped (no real PostgreSQL), 30 existing warnings; tool tests
52 passed. Python 3.7 syntax and staged diff checks pass.
No remote keys/configuration or containers were changed; rerun the existing
repair-router plan after updating, do not prepare or activate again.
This commit is contained in:
ex_tianzy
2026-11-04 16:33:35 +08:00
parent 8a94d60155
commit ce965aaf58
4 changed files with 66 additions and 3 deletions
+12 -3
View File
@@ -23,6 +23,10 @@ class DeployError(RuntimeError):
pass
class ServiceLookupError(DeployError):
pass
def run(args, json_result=False, optional=False, input_text=None):
result = subprocess.run(args, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
if result.returncode:
@@ -34,6 +38,8 @@ def run(args, json_result=False, optional=False, input_text=None):
stage = "candidate-runtime-check"
elif args[:3] == ["docker","image","inspect"]:
stage = "image-inspect"
elif args[:2] == ["docker","inspect"]:
stage = "container-inspect"
else:
stage = "docker-command"
raise DeployError(stage+" failed (exit "+str(result.returncode)+"); raw output withheld; run state to inspect current services")
@@ -73,7 +79,7 @@ def inspect(container):
template = ('{"id":{{json .Id}},"image":{{json .Image}},'
'"image_ref":{{json .Config.Image}},"labels":{{json .Config.Labels}},'
'"state":{"status":{{json .State.Status}},"health":'
'{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},'
'{{with index .State "Health"}}{{json .Status}}{{else}}"none"{{end}},'
'"exit_code":{{.State.ExitCode}},"restart_count":{{.RestartCount}}}}')
return run(["docker", "inspect", container, "--format", template], True)
@@ -82,7 +88,7 @@ def service(project, name, include_stopped=False):
values = run(["docker", "ps", "-aq" if include_stopped else "-q", "--filter", "label=com.docker.compose.project="+project,
"--filter", "label=com.docker.compose.service="+name]).splitlines()
if len(values) != 1:
raise DeployError("expected exactly one "+("existing" if include_stopped else "running")+" relay service: "+name)
raise ServiceLookupError("expected exactly one "+("existing" if include_stopped else "running")+" relay service: "+name)
return inspect(values[0])
@@ -393,9 +399,12 @@ def deployment_state(root,plan):
for name in ("connector","router","longconn","enrollment-signer","edge"):
try:
entry = service(plan["project"],name,include_stopped=True)
except DeployError:
except ServiceLookupError:
print(name+": no unique existing container; inspect docker ps -a for this project")
continue
except DeployError:
print(name+": container inspection failed; no container state inferred")
continue
state = entry.get("state",{})
status = state.get("status") if state.get("status") in {'created','running','paused','restarting','removing','exited','dead'} else 'unknown'
health = state.get("health") if state.get("health") in {'none','starting','healthy','unhealthy'} else 'unknown'