fix(deploy): repair missing router public-key path after partial activation
Live evidence showed a healthy candidate connector and a restarting router: legacy router environment omitted HN_CONNECTOR_PUBLIC_KEY, so the new routing verifier tried runtime/pki/connector.pub instead of the existing /trust mount. Resolve the public key from the existing read-only trust bind, set its path in new overlays, and actually load it during isolated runtime checks. Add repair-router for a known healthy candidate connector: preserve credential keys and live configuration, back up overlay/plan, add only the router path, update integrity metadata, probe, and recreate router alone with --no-deps. Retries do not regenerate keys or overwrite repair backups. Add read-only state including stopped/restarting containers and fixed startup error classes, including stderr, without printing raw logs or credentials. Identify command stage/exit code and activation progress. Permit rollback of known stopped candidate targets and old overlays without requiring the new router key; unknown images still fail closed. Validation: full suite 262 passed, 3 skipped (real PostgreSQL unavailable), 30 existing warnings; final tooling suite 50 passed after the added CLI rollback regression. Python 3.7 syntax and staged diff checks pass. No live repair was executed by this agent; operator must run repair-router and verify health and the real client handshake on ECS.
This commit is contained in:
@@ -9,12 +9,13 @@ import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
@@ -27,7 +28,17 @@ def run(args, json_result=False, optional=False, input_text=None):
|
||||
if result.returncode:
|
||||
if optional:
|
||||
return None
|
||||
raise DeployError("Docker/Git command failed; raw output withheld to protect configuration")
|
||||
if args[:2] == ["docker","compose"]:
|
||||
stage = "compose-up" if "up" in args else "compose-config"
|
||||
elif args[:2] == ["docker","run"]:
|
||||
stage = "candidate-runtime-check"
|
||||
elif args[:3] == ["docker","image","inspect"]:
|
||||
stage = "image-inspect"
|
||||
else:
|
||||
stage = "docker-command"
|
||||
raise DeployError(stage+" failed (exit "+str(result.returncode)+"); raw output withheld; run state to inspect current services")
|
||||
if args[:2] == ["docker","logs"]:
|
||||
return (result.stdout+"\n"+result.stderr).strip()
|
||||
try:
|
||||
return json.loads(result.stdout) if json_result else result.stdout.strip()
|
||||
except ValueError:
|
||||
@@ -46,17 +57,32 @@ def write_json(path, value):
|
||||
stream.write("\n")
|
||||
|
||||
|
||||
def replace_json(path,value):
|
||||
name = None
|
||||
try:
|
||||
fd,name = tempfile.mkstemp(prefix='.'+path.name+'.',dir=str(path.parent))
|
||||
with os.fdopen(fd,'w',encoding='utf-8') as stream:
|
||||
json.dump(value,stream,ensure_ascii=False,indent=2)
|
||||
stream.write('\n');stream.flush();os.fsync(stream.fileno())
|
||||
os.replace(name,str(path))
|
||||
finally:
|
||||
if name and os.path.exists(name):os.unlink(name)
|
||||
|
||||
|
||||
def inspect(container):
|
||||
template = ('{"id":{{json .Id}},"image":{{json .Image}},'
|
||||
'"image_ref":{{json .Config.Image}},"labels":{{json .Config.Labels}}}')
|
||||
'"image_ref":{{json .Config.Image}},"labels":{{json .Config.Labels}},'
|
||||
'"state":{"status":{{json .State.Status}},"health":'
|
||||
'{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},'
|
||||
'"exit_code":{{.State.ExitCode}},"restart_count":{{.RestartCount}}}}')
|
||||
return run(["docker", "inspect", container, "--format", template], True)
|
||||
|
||||
|
||||
def service(project, name):
|
||||
values = run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project="+project,
|
||||
def service(project, name, include_stopped=False):
|
||||
values = run(["docker", "ps", "-aq" if include_stopped else "-q", "--filter", "label=com.docker.compose.project="+project,
|
||||
"--filter", "label=com.docker.compose.service="+name]).splitlines()
|
||||
if len(values) != 1:
|
||||
raise DeployError("expected exactly one running relay service: "+name)
|
||||
raise DeployError("expected exactly one "+("existing" if include_stopped else "running")+" relay service: "+name)
|
||||
return inspect(values[0])
|
||||
|
||||
|
||||
@@ -125,7 +151,30 @@ def require_platform(env):
|
||||
raise DeployError("broker platform configuration incomplete; "+"; ".join(problems))
|
||||
|
||||
|
||||
def validate_overlay(base, merged, candidate):
|
||||
def router_public_key(config):
|
||||
router=config['services']['router']
|
||||
env=router.get('environment',{})
|
||||
value=env.get('HN_CONNECTOR_PUBLIC_KEY') or str(PurePosixPath(env.get('HN_CA_CERT','/trust/ca.pem')).parent/'connector.pub')
|
||||
target=PurePosixPath(value)
|
||||
if not target.is_absolute() or '..' in target.parts:
|
||||
raise DeployError('invalid router connector public-key path')
|
||||
for mount in router.get('volumes',[]):
|
||||
if mount.get('type')!='bind' or not mount.get('read_only'):continue
|
||||
base=PurePosixPath(mount['target'])
|
||||
try:
|
||||
relative=target.relative_to(base)
|
||||
except ValueError:
|
||||
continue
|
||||
source=Path(mount['source'])
|
||||
source=source.joinpath(*relative.parts) if relative.parts else source
|
||||
if source.is_file() and not source.is_symlink():
|
||||
data=source.read_bytes()
|
||||
if b'-----BEGIN PUBLIC KEY-----' in data and b'PRIVATE KEY' not in data:
|
||||
return str(target),source
|
||||
raise DeployError('router connector public key is not a verified read-only trust mount')
|
||||
|
||||
|
||||
def validate_overlay(base, merged, candidate, router_key_required=True):
|
||||
old, new = base["services"], merged["services"]
|
||||
require_services(old)
|
||||
require_services(new)
|
||||
@@ -172,6 +221,10 @@ def validate_overlay(base, merged, candidate):
|
||||
mounts = [v for v in new["connector"].get("volumes", []) if v["target"] == "/run/hainuo-credentials/keyring.json"]
|
||||
if len(mounts) != 1 or not mounts[0].get("read_only"):
|
||||
raise DeployError("missing read-only credential keyring mount")
|
||||
if router_key_required:
|
||||
target,_=router_public_key(merged)
|
||||
if new['router']['environment'].get('HN_CONNECTOR_PUBLIC_KEY')!=target:
|
||||
raise DeployError('router HN_CONNECTOR_PUBLIC_KEY missing; use repair-router for an already switched plan')
|
||||
|
||||
|
||||
def preflight(args):
|
||||
@@ -249,6 +302,7 @@ def prepare(args):
|
||||
if base["services"]["connector"].get("environment",{}).get("HN_CREDENTIAL_MODE","legacy") != "legacy":
|
||||
raise DeployError("already broker: preserve the existing keyring and migration plan")
|
||||
require_platform({**base["services"]["connector"].get("environment",{}),"HN_OAUTH_ENABLED":"true"})
|
||||
public_key,_=router_public_key(base)
|
||||
output.mkdir(mode=0o700,parents=False)
|
||||
os.chmod(str(output),0o700)
|
||||
keyring = {"active_key_id":"initial-v1","keys":{"initial-v1":base64.b64encode(os.urandom(32)).decode("ascii")}}
|
||||
@@ -264,6 +318,7 @@ def prepare(args):
|
||||
"volumes":[{"type":"bind","source":str(output/"credential-keyring.json"),
|
||||
"target":"/run/hainuo-credentials/keyring.json","read_only":True}]},
|
||||
"router":{"image":candidate,"environment":{"HN_CREDENTIAL_MODE":"broker",
|
||||
"HN_CONNECTOR_PUBLIC_KEY":public_key,
|
||||
"HN_ROUTER_ADMISSION_SECRET":admission,"HN_CONNECTOR_INTERNAL_URL":"http://connector:8000",
|
||||
"HN_FEISHU_APP_ID":"${HN_FEISHU_APP_ID:?Configure existing application}",
|
||||
"HN_ENVIRONMENT":"${HN_ENVIRONMENT:?Configure existing environment}"}}}}
|
||||
@@ -317,7 +372,45 @@ def diagnose(root,plan):
|
||||
raise DeployError("review the named fields above; do not paste configuration or regenerate keys")
|
||||
|
||||
|
||||
def check(root, plan, baseline=True):
|
||||
def log_signals(text):
|
||||
"""Classify locally; never return arbitrary log lines, SQL, URLs or tokens."""
|
||||
known = ('ModuleNotFoundError','ImportError','FileNotFoundError','PermissionError','ValidationError',
|
||||
'CredentialError','OperationalError','ProgrammingError','UndefinedColumn','UndefinedTable',
|
||||
'InsufficientPrivilege','ReadOnlySqlTransaction','SyntaxError','IndentationError',
|
||||
'ConnectionRefusedError','AuthenticationError','ConnectionError','TimeoutError','MemoryError')
|
||||
signals = [name for name in known if re.search(r'\b'+name+r'\b',text)]
|
||||
if 'read-only file system' in text.lower():
|
||||
signals.append('read-only-filesystem')
|
||||
if 'address already in use' in text.lower():
|
||||
signals.append('port-in-use')
|
||||
return signals
|
||||
|
||||
|
||||
def deployment_state(root,plan):
|
||||
expected = None
|
||||
if (root/"candidate.json").is_file():
|
||||
expected = json.loads((root/"candidate.json").read_text(encoding="utf-8")).get("image")
|
||||
for name in ("connector","router","longconn","enrollment-signer","edge"):
|
||||
try:
|
||||
entry = service(plan["project"],name,include_stopped=True)
|
||||
except DeployError:
|
||||
print(name+": no unique existing container; inspect docker ps -a for this project")
|
||||
continue
|
||||
state = entry.get("state",{})
|
||||
status = state.get("status") if state.get("status") in {'created','running','paused','restarting','removing','exited','dead'} else 'unknown'
|
||||
health = state.get("health") if state.get("health") in {'none','starting','healthy','unhealthy'} else 'unknown'
|
||||
image = 'candidate' if expected and entry['image']==expected else (
|
||||
'original' if entry['image']==plan['roles'][name]['image'] else 'different')
|
||||
numbers = {k:v for k,v in state.items() if k in {'exit_code','restart_count'} and type(v) is int}
|
||||
print(name+': image='+image+' status='+status+' health='+health+' '+json.dumps(numbers,sort_keys=True))
|
||||
if name in {'connector','router'}:
|
||||
raw = run(['docker','logs','--tail','100',entry['id']],optional=True)
|
||||
if raw is not None:
|
||||
print(' startup signals: '+(', '.join(log_signals(raw)) or 'no recognized error class; raw logs withheld'))
|
||||
print('STATE inspection only; no files, keys or containers changed. Do not retry activation before reviewing partial switch state.')
|
||||
|
||||
|
||||
def check(root, plan, baseline=True, router_key_required=True):
|
||||
if os.name != "nt":
|
||||
if root.stat().st_mode & 0o077:
|
||||
raise DeployError("migration directory must not be accessible to other users")
|
||||
@@ -336,12 +429,18 @@ def check(root, plan, baseline=True):
|
||||
if digest(root/path) != expected:
|
||||
raise DeployError("prepared keyring or overlay changed; do not regenerate credential keys")
|
||||
for name, old in plan["roles"].items():
|
||||
current = service(plan["project"],name)
|
||||
current = service(plan["project"],name,include_stopped=not baseline and name in {"connector","router"})
|
||||
if (baseline or name not in {"connector","router"}) and current["image"] != old["image"]:
|
||||
raise DeployError("running service image changed since preparation: "+name)
|
||||
if not baseline and name in {"connector","router"}:
|
||||
allowed = {old['image']}
|
||||
if (root/'candidate.json').is_file():
|
||||
allowed.add(json.loads((root/'candidate.json').read_text(encoding='utf-8'))['image'])
|
||||
if current['image'] not in allowed:
|
||||
raise DeployError('unreviewed target image; refusing rollback: '+name)
|
||||
base = run(compose(plan)+["config","--format","json"],True)
|
||||
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
|
||||
validate_overlay(base,merged,plan["candidate_image"])
|
||||
validate_overlay(base,merged,plan["candidate_image"],router_key_required=router_key_required)
|
||||
# Require the current edge's explicit private-path block, before catch-all handling.
|
||||
mounts = base["services"]["edge"].get("volumes",[])
|
||||
caddy = next((v["source"] for v in mounts if v.get("target") == "/etc/caddy/Caddyfile" and v.get("type") == "bind"),None)
|
||||
@@ -353,7 +452,7 @@ def check(root, plan, baseline=True):
|
||||
first_handle = re.search(r"\bhandle(?:_path)?\b",text)
|
||||
if not deny or (first_handle and deny.start() > first_handle.start()):
|
||||
raise DeployError("edge private-path block is not the reviewed configuration")
|
||||
print("CHECKED: existing Compose/storage/images retained; no services restarted")
|
||||
print("CHECKED: Compose/storage and permitted service images verified; no services restarted")
|
||||
|
||||
|
||||
def build(root,plan):
|
||||
@@ -374,30 +473,75 @@ def build(root,plan):
|
||||
|
||||
def runtime_check(root,plan):
|
||||
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
|
||||
public_key,public_source=router_public_key(merged)
|
||||
# Actual secrets travel via stdin only: no command-line values, env dump,
|
||||
# database connection or platform request. The container has no network.
|
||||
code = ("import json,os,sys\nfrom pathlib import Path\nfrom relay.config import Settings\n"
|
||||
"from relay.credential_store import CredentialKeyring\n"
|
||||
"from relay.crypto import load_public\n"
|
||||
"try:\n data=json.load(sys.stdin)\n os.environ.update(data['environment'])\n"
|
||||
" s=Settings(role=data['role'])\n"
|
||||
" if data['role']=='connector': CredentialKeyring.from_file(s.credential_keyring_file)\n"
|
||||
" if data['role']=='router': load_public(s.connector_public_key)\n"
|
||||
"except Exception:\n sys.exit(1)\n")
|
||||
for role in ("connector","router"):
|
||||
env = {key:str(value) for key,value in merged["services"][role]["environment"].items() if value is not None}
|
||||
run(["docker","run","--rm","-i","--network","none","--read-only","--entrypoint","python",
|
||||
command=["docker","run","--rm","-i","--network","none","--read-only","--entrypoint","python",
|
||||
"-v",str(root/"credential-keyring.json")+":/run/hainuo-credentials/keyring.json:ro",
|
||||
plan["candidate_image"],"-c",code],input_text=json.dumps({"role":role,"environment":env}))
|
||||
"-v",str(public_source)+":"+public_key+":ro",plan["candidate_image"],"-c",code]
|
||||
print('Runtime check: '+role+' configuration and required key files',flush=True)
|
||||
run(command,input_text=json.dumps({"role":role,"environment":env}))
|
||||
|
||||
|
||||
def repair_router(root,plan):
|
||||
# Read all known partial-switch states, but do not weaken general activate checks.
|
||||
check(root,plan,baseline=False,router_key_required=False)
|
||||
built=json.loads((root/'candidate.json').read_text(encoding='utf-8'))
|
||||
connector=service(plan['project'],'connector',include_stopped=True)
|
||||
if (connector['image']!=built['image'] or connector.get('state',{}).get('status')!='running'
|
||||
or connector.get('state',{}).get('health')!='healthy'):
|
||||
raise DeployError('repair-router requires the known candidate connector to be running and healthy; inspect state or rollback')
|
||||
base=run(compose(plan)+['config','--format','json'],True)
|
||||
target,_=router_public_key(base)
|
||||
path=root/'compose.broker.json'
|
||||
overlay=json.loads(path.read_text(encoding='utf-8'))
|
||||
old=overlay['services']['router']['environment'].get('HN_CONNECTOR_PUBLIC_KEY')
|
||||
if old not in (None,'',target):
|
||||
raise DeployError('unexpected prepared router public-key path; manual review required')
|
||||
if old!=target:
|
||||
backup=root/'compose.broker.before-router-key.json'
|
||||
plan_backup=root/'plan.before-router-key.json'
|
||||
if backup.exists() or plan_backup.exists():
|
||||
raise DeployError('repair backup already exists; preserve keys and inspect the interrupted repair')
|
||||
write_json(backup,overlay);write_json(plan_backup,plan)
|
||||
overlay['services']['router']['environment']['HN_CONNECTOR_PUBLIC_KEY']=target
|
||||
replace_json(path,overlay)
|
||||
plan['prepared_hashes']['compose.broker.json']=digest(path)
|
||||
replace_json(root/'plan.json',plan)
|
||||
print('REPAIR: existing mounted connector public key selected; credential keys and connector preserved',flush=True)
|
||||
check(root,plan,baseline=False)
|
||||
runtime_check(root,plan)
|
||||
image=run(['docker','image','inspect',plan['candidate_image'],'--format','{{.Id}}'])
|
||||
if image!=built['image']:
|
||||
raise DeployError('candidate image identity changed')
|
||||
print('REPAIR: recreate router only',flush=True)
|
||||
run(compose(plan,root/'compose.broker.json')+['up','-d','--no-deps','--no-build','--pull','never',
|
||||
'--wait','--wait-timeout','180','router'])
|
||||
print('REPAIRED: router only; verify state and real client handshake before acceptance')
|
||||
|
||||
|
||||
def activate(root,plan,ready):
|
||||
if not ready:
|
||||
raise DeployError("v2 client not confirmed ready; only prepare/check/build are allowed now")
|
||||
check(root,plan)
|
||||
print('ACTIVATION step: verify candidate image identity',flush=True)
|
||||
built = json.loads((root/"candidate.json").read_text(encoding="utf-8"))
|
||||
image = run(["docker","image","inspect",plan["candidate_image"],"--format","{{.Id}}"])
|
||||
if built != {"image":image,"tag":plan["candidate_image"]}:
|
||||
raise DeployError("candidate image identity changed")
|
||||
print('ACTIVATION step: isolated candidate configuration/keyring check (no service switch)',flush=True)
|
||||
runtime_check(root,plan)
|
||||
print('ACTIVATION step: recreate connector/router; failures may leave a partial switch, inspect state',flush=True)
|
||||
run(compose(plan,root/"compose.broker.json")+["up","-d","--no-deps","--no-build","--pull","never",
|
||||
"--wait","--wait-timeout","180","connector","router"])
|
||||
print("ACTIVATED: connector/router only. Verify real OAuth, v2 client, unbind and task routing.")
|
||||
@@ -415,7 +559,7 @@ def main():
|
||||
prep.add_argument("--profiles",required=True)
|
||||
prep.add_argument("--output",required=True)
|
||||
prep.add_argument("--connector-container",default="hainuo-relay-connector-1")
|
||||
for name in ("diagnose","check","build","activate","rollback"):
|
||||
for name in ("repair-router","state","diagnose","check","build","activate","rollback"):
|
||||
cmd = sub.add_parser(name)
|
||||
cmd.add_argument("--plan",required=True)
|
||||
if name == "activate":
|
||||
@@ -432,7 +576,11 @@ def main():
|
||||
return
|
||||
root = Path(args.plan).resolve()
|
||||
plan = load_plan(root)
|
||||
if args.action == "diagnose":
|
||||
if args.action == "repair-router":
|
||||
repair_router(root,plan)
|
||||
elif args.action == "state":
|
||||
deployment_state(root,plan)
|
||||
elif args.action == "diagnose":
|
||||
diagnose(root,plan)
|
||||
elif args.action == "check":
|
||||
check(root,plan)
|
||||
@@ -443,7 +591,7 @@ def main():
|
||||
else:
|
||||
if not args.stop_broker_use:
|
||||
raise DeployError("rollback requires stopping broker clients; old refresh credentials are not restored")
|
||||
check(root,plan,baseline=False)
|
||||
check(root,plan,baseline=False,router_key_required=False)
|
||||
run(compose(plan,root/"compose.rollback.json")+["up","-d","--no-deps","--no-build","--pull","never",
|
||||
"--wait","--wait-timeout","180","connector","router"])
|
||||
print("ROLLED BACK containers only; preserve keyring/database. Client mode and reauthorization must be coordinated.")
|
||||
|
||||
Reference in New Issue
Block a user