fix(deploy): repair missing router public-key path after partial activation

Live evidence showed a healthy candidate connector and a restarting router:
legacy router environment omitted HN_CONNECTOR_PUBLIC_KEY, so the new routing
verifier tried runtime/pki/connector.pub instead of the existing /trust mount.

Resolve the public key from the existing read-only trust bind, set its path
in new overlays, and actually load it during isolated runtime checks. Add
repair-router for a known healthy candidate connector: preserve credential
keys and live configuration, back up overlay/plan, add only the router path,
update integrity metadata, probe, and recreate router alone with --no-deps.
Retries do not regenerate keys or overwrite repair backups.

Add read-only state including stopped/restarting containers and fixed startup
error classes, including stderr, without printing raw logs or credentials.
Identify command stage/exit code and activation progress. Permit rollback
of known stopped candidate targets and old overlays without requiring the
new router key; unknown images still fail closed.

Validation: full suite 262 passed, 3 skipped (real PostgreSQL unavailable),
30 existing warnings; final tooling suite 50 passed after the added CLI
rollback regression. Python 3.7 syntax and staged diff checks pass.
No live repair was executed by this agent; operator must run repair-router
and verify health and the real client handshake on ECS.
This commit is contained in:
ex_tianzy
2026-11-04 16:27:47 +08:00
parent 2657146334
commit 8a94d60155
4 changed files with 332 additions and 17 deletions
+164 -16
View File
@@ -9,12 +9,13 @@ import base64
import hashlib
import json
import os
from pathlib import Path
from pathlib import Path, PurePosixPath
import re
import secrets
import shutil
import subprocess
import sys
import tempfile
from urllib.parse import urlsplit
@@ -27,7 +28,17 @@ def run(args, json_result=False, optional=False, input_text=None):
if result.returncode:
if optional:
return None
raise DeployError("Docker/Git command failed; raw output withheld to protect configuration")
if args[:2] == ["docker","compose"]:
stage = "compose-up" if "up" in args else "compose-config"
elif args[:2] == ["docker","run"]:
stage = "candidate-runtime-check"
elif args[:3] == ["docker","image","inspect"]:
stage = "image-inspect"
else:
stage = "docker-command"
raise DeployError(stage+" failed (exit "+str(result.returncode)+"); raw output withheld; run state to inspect current services")
if args[:2] == ["docker","logs"]:
return (result.stdout+"\n"+result.stderr).strip()
try:
return json.loads(result.stdout) if json_result else result.stdout.strip()
except ValueError:
@@ -46,17 +57,32 @@ def write_json(path, value):
stream.write("\n")
def replace_json(path,value):
name = None
try:
fd,name = tempfile.mkstemp(prefix='.'+path.name+'.',dir=str(path.parent))
with os.fdopen(fd,'w',encoding='utf-8') as stream:
json.dump(value,stream,ensure_ascii=False,indent=2)
stream.write('\n');stream.flush();os.fsync(stream.fileno())
os.replace(name,str(path))
finally:
if name and os.path.exists(name):os.unlink(name)
def inspect(container):
template = ('{"id":{{json .Id}},"image":{{json .Image}},'
'"image_ref":{{json .Config.Image}},"labels":{{json .Config.Labels}}}')
'"image_ref":{{json .Config.Image}},"labels":{{json .Config.Labels}},'
'"state":{"status":{{json .State.Status}},"health":'
'{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},'
'"exit_code":{{.State.ExitCode}},"restart_count":{{.RestartCount}}}}')
return run(["docker", "inspect", container, "--format", template], True)
def service(project, name):
values = run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project="+project,
def service(project, name, include_stopped=False):
values = run(["docker", "ps", "-aq" if include_stopped else "-q", "--filter", "label=com.docker.compose.project="+project,
"--filter", "label=com.docker.compose.service="+name]).splitlines()
if len(values) != 1:
raise DeployError("expected exactly one running relay service: "+name)
raise DeployError("expected exactly one "+("existing" if include_stopped else "running")+" relay service: "+name)
return inspect(values[0])
@@ -125,7 +151,30 @@ def require_platform(env):
raise DeployError("broker platform configuration incomplete; "+"; ".join(problems))
def validate_overlay(base, merged, candidate):
def router_public_key(config):
router=config['services']['router']
env=router.get('environment',{})
value=env.get('HN_CONNECTOR_PUBLIC_KEY') or str(PurePosixPath(env.get('HN_CA_CERT','/trust/ca.pem')).parent/'connector.pub')
target=PurePosixPath(value)
if not target.is_absolute() or '..' in target.parts:
raise DeployError('invalid router connector public-key path')
for mount in router.get('volumes',[]):
if mount.get('type')!='bind' or not mount.get('read_only'):continue
base=PurePosixPath(mount['target'])
try:
relative=target.relative_to(base)
except ValueError:
continue
source=Path(mount['source'])
source=source.joinpath(*relative.parts) if relative.parts else source
if source.is_file() and not source.is_symlink():
data=source.read_bytes()
if b'-----BEGIN PUBLIC KEY-----' in data and b'PRIVATE KEY' not in data:
return str(target),source
raise DeployError('router connector public key is not a verified read-only trust mount')
def validate_overlay(base, merged, candidate, router_key_required=True):
old, new = base["services"], merged["services"]
require_services(old)
require_services(new)
@@ -172,6 +221,10 @@ def validate_overlay(base, merged, candidate):
mounts = [v for v in new["connector"].get("volumes", []) if v["target"] == "/run/hainuo-credentials/keyring.json"]
if len(mounts) != 1 or not mounts[0].get("read_only"):
raise DeployError("missing read-only credential keyring mount")
if router_key_required:
target,_=router_public_key(merged)
if new['router']['environment'].get('HN_CONNECTOR_PUBLIC_KEY')!=target:
raise DeployError('router HN_CONNECTOR_PUBLIC_KEY missing; use repair-router for an already switched plan')
def preflight(args):
@@ -249,6 +302,7 @@ def prepare(args):
if base["services"]["connector"].get("environment",{}).get("HN_CREDENTIAL_MODE","legacy") != "legacy":
raise DeployError("already broker: preserve the existing keyring and migration plan")
require_platform({**base["services"]["connector"].get("environment",{}),"HN_OAUTH_ENABLED":"true"})
public_key,_=router_public_key(base)
output.mkdir(mode=0o700,parents=False)
os.chmod(str(output),0o700)
keyring = {"active_key_id":"initial-v1","keys":{"initial-v1":base64.b64encode(os.urandom(32)).decode("ascii")}}
@@ -264,6 +318,7 @@ def prepare(args):
"volumes":[{"type":"bind","source":str(output/"credential-keyring.json"),
"target":"/run/hainuo-credentials/keyring.json","read_only":True}]},
"router":{"image":candidate,"environment":{"HN_CREDENTIAL_MODE":"broker",
"HN_CONNECTOR_PUBLIC_KEY":public_key,
"HN_ROUTER_ADMISSION_SECRET":admission,"HN_CONNECTOR_INTERNAL_URL":"http://connector:8000",
"HN_FEISHU_APP_ID":"${HN_FEISHU_APP_ID:?Configure existing application}",
"HN_ENVIRONMENT":"${HN_ENVIRONMENT:?Configure existing environment}"}}}}
@@ -317,7 +372,45 @@ def diagnose(root,plan):
raise DeployError("review the named fields above; do not paste configuration or regenerate keys")
def check(root, plan, baseline=True):
def log_signals(text):
"""Classify locally; never return arbitrary log lines, SQL, URLs or tokens."""
known = ('ModuleNotFoundError','ImportError','FileNotFoundError','PermissionError','ValidationError',
'CredentialError','OperationalError','ProgrammingError','UndefinedColumn','UndefinedTable',
'InsufficientPrivilege','ReadOnlySqlTransaction','SyntaxError','IndentationError',
'ConnectionRefusedError','AuthenticationError','ConnectionError','TimeoutError','MemoryError')
signals = [name for name in known if re.search(r'\b'+name+r'\b',text)]
if 'read-only file system' in text.lower():
signals.append('read-only-filesystem')
if 'address already in use' in text.lower():
signals.append('port-in-use')
return signals
def deployment_state(root,plan):
expected = None
if (root/"candidate.json").is_file():
expected = json.loads((root/"candidate.json").read_text(encoding="utf-8")).get("image")
for name in ("connector","router","longconn","enrollment-signer","edge"):
try:
entry = service(plan["project"],name,include_stopped=True)
except DeployError:
print(name+": no unique existing container; inspect docker ps -a for this project")
continue
state = entry.get("state",{})
status = state.get("status") if state.get("status") in {'created','running','paused','restarting','removing','exited','dead'} else 'unknown'
health = state.get("health") if state.get("health") in {'none','starting','healthy','unhealthy'} else 'unknown'
image = 'candidate' if expected and entry['image']==expected else (
'original' if entry['image']==plan['roles'][name]['image'] else 'different')
numbers = {k:v for k,v in state.items() if k in {'exit_code','restart_count'} and type(v) is int}
print(name+': image='+image+' status='+status+' health='+health+' '+json.dumps(numbers,sort_keys=True))
if name in {'connector','router'}:
raw = run(['docker','logs','--tail','100',entry['id']],optional=True)
if raw is not None:
print(' startup signals: '+(', '.join(log_signals(raw)) or 'no recognized error class; raw logs withheld'))
print('STATE inspection only; no files, keys or containers changed. Do not retry activation before reviewing partial switch state.')
def check(root, plan, baseline=True, router_key_required=True):
if os.name != "nt":
if root.stat().st_mode & 0o077:
raise DeployError("migration directory must not be accessible to other users")
@@ -336,12 +429,18 @@ def check(root, plan, baseline=True):
if digest(root/path) != expected:
raise DeployError("prepared keyring or overlay changed; do not regenerate credential keys")
for name, old in plan["roles"].items():
current = service(plan["project"],name)
current = service(plan["project"],name,include_stopped=not baseline and name in {"connector","router"})
if (baseline or name not in {"connector","router"}) and current["image"] != old["image"]:
raise DeployError("running service image changed since preparation: "+name)
if not baseline and name in {"connector","router"}:
allowed = {old['image']}
if (root/'candidate.json').is_file():
allowed.add(json.loads((root/'candidate.json').read_text(encoding='utf-8'))['image'])
if current['image'] not in allowed:
raise DeployError('unreviewed target image; refusing rollback: '+name)
base = run(compose(plan)+["config","--format","json"],True)
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
validate_overlay(base,merged,plan["candidate_image"])
validate_overlay(base,merged,plan["candidate_image"],router_key_required=router_key_required)
# Require the current edge's explicit private-path block, before catch-all handling.
mounts = base["services"]["edge"].get("volumes",[])
caddy = next((v["source"] for v in mounts if v.get("target") == "/etc/caddy/Caddyfile" and v.get("type") == "bind"),None)
@@ -353,7 +452,7 @@ def check(root, plan, baseline=True):
first_handle = re.search(r"\bhandle(?:_path)?\b",text)
if not deny or (first_handle and deny.start() > first_handle.start()):
raise DeployError("edge private-path block is not the reviewed configuration")
print("CHECKED: existing Compose/storage/images retained; no services restarted")
print("CHECKED: Compose/storage and permitted service images verified; no services restarted")
def build(root,plan):
@@ -374,30 +473,75 @@ def build(root,plan):
def runtime_check(root,plan):
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
public_key,public_source=router_public_key(merged)
# Actual secrets travel via stdin only: no command-line values, env dump,
# database connection or platform request. The container has no network.
code = ("import json,os,sys\nfrom pathlib import Path\nfrom relay.config import Settings\n"
"from relay.credential_store import CredentialKeyring\n"
"from relay.crypto import load_public\n"
"try:\n data=json.load(sys.stdin)\n os.environ.update(data['environment'])\n"
" s=Settings(role=data['role'])\n"
" if data['role']=='connector': CredentialKeyring.from_file(s.credential_keyring_file)\n"
" if data['role']=='router': load_public(s.connector_public_key)\n"
"except Exception:\n sys.exit(1)\n")
for role in ("connector","router"):
env = {key:str(value) for key,value in merged["services"][role]["environment"].items() if value is not None}
run(["docker","run","--rm","-i","--network","none","--read-only","--entrypoint","python",
command=["docker","run","--rm","-i","--network","none","--read-only","--entrypoint","python",
"-v",str(root/"credential-keyring.json")+":/run/hainuo-credentials/keyring.json:ro",
plan["candidate_image"],"-c",code],input_text=json.dumps({"role":role,"environment":env}))
"-v",str(public_source)+":"+public_key+":ro",plan["candidate_image"],"-c",code]
print('Runtime check: '+role+' configuration and required key files',flush=True)
run(command,input_text=json.dumps({"role":role,"environment":env}))
def repair_router(root,plan):
# Read all known partial-switch states, but do not weaken general activate checks.
check(root,plan,baseline=False,router_key_required=False)
built=json.loads((root/'candidate.json').read_text(encoding='utf-8'))
connector=service(plan['project'],'connector',include_stopped=True)
if (connector['image']!=built['image'] or connector.get('state',{}).get('status')!='running'
or connector.get('state',{}).get('health')!='healthy'):
raise DeployError('repair-router requires the known candidate connector to be running and healthy; inspect state or rollback')
base=run(compose(plan)+['config','--format','json'],True)
target,_=router_public_key(base)
path=root/'compose.broker.json'
overlay=json.loads(path.read_text(encoding='utf-8'))
old=overlay['services']['router']['environment'].get('HN_CONNECTOR_PUBLIC_KEY')
if old not in (None,'',target):
raise DeployError('unexpected prepared router public-key path; manual review required')
if old!=target:
backup=root/'compose.broker.before-router-key.json'
plan_backup=root/'plan.before-router-key.json'
if backup.exists() or plan_backup.exists():
raise DeployError('repair backup already exists; preserve keys and inspect the interrupted repair')
write_json(backup,overlay);write_json(plan_backup,plan)
overlay['services']['router']['environment']['HN_CONNECTOR_PUBLIC_KEY']=target
replace_json(path,overlay)
plan['prepared_hashes']['compose.broker.json']=digest(path)
replace_json(root/'plan.json',plan)
print('REPAIR: existing mounted connector public key selected; credential keys and connector preserved',flush=True)
check(root,plan,baseline=False)
runtime_check(root,plan)
image=run(['docker','image','inspect',plan['candidate_image'],'--format','{{.Id}}'])
if image!=built['image']:
raise DeployError('candidate image identity changed')
print('REPAIR: recreate router only',flush=True)
run(compose(plan,root/'compose.broker.json')+['up','-d','--no-deps','--no-build','--pull','never',
'--wait','--wait-timeout','180','router'])
print('REPAIRED: router only; verify state and real client handshake before acceptance')
def activate(root,plan,ready):
if not ready:
raise DeployError("v2 client not confirmed ready; only prepare/check/build are allowed now")
check(root,plan)
print('ACTIVATION step: verify candidate image identity',flush=True)
built = json.loads((root/"candidate.json").read_text(encoding="utf-8"))
image = run(["docker","image","inspect",plan["candidate_image"],"--format","{{.Id}}"])
if built != {"image":image,"tag":plan["candidate_image"]}:
raise DeployError("candidate image identity changed")
print('ACTIVATION step: isolated candidate configuration/keyring check (no service switch)',flush=True)
runtime_check(root,plan)
print('ACTIVATION step: recreate connector/router; failures may leave a partial switch, inspect state',flush=True)
run(compose(plan,root/"compose.broker.json")+["up","-d","--no-deps","--no-build","--pull","never",
"--wait","--wait-timeout","180","connector","router"])
print("ACTIVATED: connector/router only. Verify real OAuth, v2 client, unbind and task routing.")
@@ -415,7 +559,7 @@ def main():
prep.add_argument("--profiles",required=True)
prep.add_argument("--output",required=True)
prep.add_argument("--connector-container",default="hainuo-relay-connector-1")
for name in ("diagnose","check","build","activate","rollback"):
for name in ("repair-router","state","diagnose","check","build","activate","rollback"):
cmd = sub.add_parser(name)
cmd.add_argument("--plan",required=True)
if name == "activate":
@@ -432,7 +576,11 @@ def main():
return
root = Path(args.plan).resolve()
plan = load_plan(root)
if args.action == "diagnose":
if args.action == "repair-router":
repair_router(root,plan)
elif args.action == "state":
deployment_state(root,plan)
elif args.action == "diagnose":
diagnose(root,plan)
elif args.action == "check":
check(root,plan)
@@ -443,7 +591,7 @@ def main():
else:
if not args.stop_broker_use:
raise DeployError("rollback requires stopping broker clients; old refresh credentials are not restored")
check(root,plan,baseline=False)
check(root,plan,baseline=False,router_key_required=False)
run(compose(plan,root/"compose.rollback.json")+["up","-d","--no-deps","--no-build","--pull","never",
"--wait","--wait-timeout","180","connector","router"])
print("ROLLED BACK containers only; preserve keyring/database. Client mode and reauthorization must be coordinated.")