Containers without a configured HEALTHCHECK omit State.Health. Direct map field access in the new inspect template made recovery prechecks fail on longconn/edge before the router path could be repaired. Use a guarded map index and report health=none without weakening image/config/key checks. Distinguish service lookup errors from inspect/template failures instead of misreporting every query failure as no unique container. Identify the container-inspect stage in safe command errors. Add regressions for absent health state through state and the full partial router repair, plus accurate inspect failure classification. Validation: 265 passed, 3 skipped (no real PostgreSQL), 30 existing warnings; tool tests 52 passed. Python 3.7 syntax and staged diff checks pass. No remote keys/configuration or containers were changed; rerun the existing repair-router plan after updating, do not prepare or activate again.
615 lines
34 KiB
Python
615 lines
34 KiB
Python
#!/usr/bin/env python3
|
|
"""Existing-deployment broker migration: stdlib-only host tool, Python >=3.7.
|
|
|
|
Never initialize identities, edit the live .env, or dump resolved Compose output.
|
|
prepare/check/build do not recreate services. Activation requires a v2 client.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import secrets
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from urllib.parse import urlsplit
|
|
|
|
|
|
class DeployError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class ServiceLookupError(DeployError):
|
|
pass
|
|
|
|
|
|
def run(args, json_result=False, optional=False, input_text=None):
|
|
result = subprocess.run(args, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
|
|
if result.returncode:
|
|
if optional:
|
|
return None
|
|
if args[:2] == ["docker","compose"]:
|
|
stage = "compose-up" if "up" in args else "compose-config"
|
|
elif args[:2] == ["docker","run"]:
|
|
stage = "candidate-runtime-check"
|
|
elif args[:3] == ["docker","image","inspect"]:
|
|
stage = "image-inspect"
|
|
elif args[:2] == ["docker","inspect"]:
|
|
stage = "container-inspect"
|
|
else:
|
|
stage = "docker-command"
|
|
raise DeployError(stage+" failed (exit "+str(result.returncode)+"); raw output withheld; run state to inspect current services")
|
|
if args[:2] == ["docker","logs"]:
|
|
return (result.stdout+"\n"+result.stderr).strip()
|
|
try:
|
|
return json.loads(result.stdout) if json_result else result.stdout.strip()
|
|
except ValueError:
|
|
raise DeployError("invalid command response") from None
|
|
|
|
|
|
def digest(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def write_json(path, value):
|
|
# All outputs belong to the newly created, private migration directory.
|
|
fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
with os.fdopen(fd, "w", encoding="utf-8") as stream:
|
|
json.dump(value, stream, ensure_ascii=False, indent=2)
|
|
stream.write("\n")
|
|
|
|
|
|
def replace_json(path,value):
|
|
name = None
|
|
try:
|
|
fd,name = tempfile.mkstemp(prefix='.'+path.name+'.',dir=str(path.parent))
|
|
with os.fdopen(fd,'w',encoding='utf-8') as stream:
|
|
json.dump(value,stream,ensure_ascii=False,indent=2)
|
|
stream.write('\n');stream.flush();os.fsync(stream.fileno())
|
|
os.replace(name,str(path))
|
|
finally:
|
|
if name and os.path.exists(name):os.unlink(name)
|
|
|
|
|
|
def inspect(container):
|
|
template = ('{"id":{{json .Id}},"image":{{json .Image}},'
|
|
'"image_ref":{{json .Config.Image}},"labels":{{json .Config.Labels}},'
|
|
'"state":{"status":{{json .State.Status}},"health":'
|
|
'{{with index .State "Health"}}{{json .Status}}{{else}}"none"{{end}},'
|
|
'"exit_code":{{.State.ExitCode}},"restart_count":{{.RestartCount}}}}')
|
|
return run(["docker", "inspect", container, "--format", template], True)
|
|
|
|
|
|
def service(project, name, include_stopped=False):
|
|
values = run(["docker", "ps", "-aq" if include_stopped else "-q", "--filter", "label=com.docker.compose.project="+project,
|
|
"--filter", "label=com.docker.compose.service="+name]).splitlines()
|
|
if len(values) != 1:
|
|
raise ServiceLookupError("expected exactly one "+("existing" if include_stopped else "running")+" relay service: "+name)
|
|
return inspect(values[0])
|
|
|
|
|
|
def compose(plan, override=None):
|
|
args = ["docker", "compose", "--project-directory", plan["deployment_root"],
|
|
"--env-file", str(Path(plan["deployment_root"])/".env"), "-p", plan["project"],
|
|
"--profile", "longconn"]
|
|
for file in plan["compose_files"]:
|
|
args.extend(["-f", file])
|
|
if override:
|
|
args.extend(["-f", str(override)])
|
|
return args
|
|
|
|
|
|
def require_services(names):
|
|
expected = {"connector","router","longconn","enrollment-signer","postgres","redis","edge"}
|
|
if set(names) != expected:
|
|
# Service names only, never resolved environment/configuration contents.
|
|
raise DeployError("unexpected Compose services; missing="+json.dumps(sorted(expected-set(names)))
|
|
+"; extra="+json.dumps(sorted(set(names)-expected)))
|
|
|
|
|
|
def profiles_from(path):
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
if not isinstance(data, dict) or not data or len(data) > 32:
|
|
raise ValueError()
|
|
for name, scopes in data.items():
|
|
if (not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", name) or not isinstance(scopes, list)
|
|
or not scopes or len(scopes) > 128 or any(not isinstance(s, str)
|
|
or not re.fullmatch(r"[A-Za-z0-9_:.-]{1,256}", s) for s in scopes)):
|
|
raise ValueError()
|
|
return {name: sorted(set(scopes)) for name, scopes in data.items()}
|
|
except (OSError, ValueError, TypeError):
|
|
raise DeployError("invalid product permission profiles; use the product-approved JSON object") from None
|
|
|
|
|
|
def enabled(value):
|
|
return str(value).strip().lower() in {"true","1","yes","on"}
|
|
|
|
|
|
def platform_problems(env):
|
|
problems = []
|
|
if not str(env.get("HN_DATABASE_URL", "")).startswith(("postgresql://","postgres://")):
|
|
problems.append("HN_DATABASE_URL: PostgreSQL connection required (value withheld)")
|
|
if env.get("HN_ENVIRONMENT") not in {"production","development","test"}:
|
|
problems.append("HN_ENVIRONMENT: expected production/development/test; preserve the reviewed environment")
|
|
for name in ("HN_FEISHU_ENABLED","HN_OAUTH_ENABLED"):
|
|
if not enabled(env.get(name)):
|
|
problems.append(name+": must be enabled")
|
|
for name,scheme in (("HN_PUBLIC_URL","https"),("HN_ROUTER_URL","wss"),
|
|
("HN_FEISHU_API_BASE","https"),("HN_FEISHU_AUTHORIZE_URL","https")):
|
|
try:
|
|
value = urlsplit(str(env.get(name,"")))
|
|
valid = value.scheme == scheme and value.hostname and not value.username and not value.password
|
|
except ValueError:
|
|
valid = False
|
|
if not valid:
|
|
problems.append(name+": fixed "+scheme.upper()+" endpoint required (value withheld)")
|
|
return problems
|
|
|
|
|
|
def require_platform(env):
|
|
problems = platform_problems(env)
|
|
if problems:
|
|
raise DeployError("broker platform configuration incomplete; "+"; ".join(problems))
|
|
|
|
|
|
def router_public_key(config):
|
|
router=config['services']['router']
|
|
env=router.get('environment',{})
|
|
value=env.get('HN_CONNECTOR_PUBLIC_KEY') or str(PurePosixPath(env.get('HN_CA_CERT','/trust/ca.pem')).parent/'connector.pub')
|
|
target=PurePosixPath(value)
|
|
if not target.is_absolute() or '..' in target.parts:
|
|
raise DeployError('invalid router connector public-key path')
|
|
for mount in router.get('volumes',[]):
|
|
if mount.get('type')!='bind' or not mount.get('read_only'):continue
|
|
base=PurePosixPath(mount['target'])
|
|
try:
|
|
relative=target.relative_to(base)
|
|
except ValueError:
|
|
continue
|
|
source=Path(mount['source'])
|
|
source=source.joinpath(*relative.parts) if relative.parts else source
|
|
if source.is_file() and not source.is_symlink():
|
|
data=source.read_bytes()
|
|
if b'-----BEGIN PUBLIC KEY-----' in data and b'PRIVATE KEY' not in data:
|
|
return str(target),source
|
|
raise DeployError('router connector public key is not a verified read-only trust mount')
|
|
|
|
|
|
def validate_overlay(base, merged, candidate, router_key_required=True):
|
|
old, new = base["services"], merged["services"]
|
|
require_services(old)
|
|
require_services(new)
|
|
for name in set(old)-{"connector","router"}:
|
|
if old[name] != new[name]:
|
|
raise DeployError("overlay changes an unrelated relay service: "+name)
|
|
for name in ("connector","router"):
|
|
for field in set(old[name]) | set(new[name]):
|
|
if field not in {"environment","image","volumes"} and old[name].get(field) != new[name].get(field):
|
|
raise DeployError("overlay changes non-migration service settings")
|
|
if new[name].get("image") != candidate:
|
|
raise DeployError("candidate image mismatch")
|
|
env = new[name]["environment"]
|
|
for key, value in old[name].get("environment", {}).items():
|
|
if key not in {"HN_CREDENTIAL_MODE","HN_ROUTER_ADMISSION_SECRET","HN_CREDENTIAL_KEYRING_FILE",
|
|
"HN_CREDENTIAL_PERMISSION_PROFILES","HN_CONNECTOR_INTERNAL_URL","HN_FEISHU_APP_ID","HN_OAUTH_ENABLED"}:
|
|
if env.get(key) != value:
|
|
raise DeployError("existing environment changed: "+key)
|
|
old_volumes = {v["target"]:v for v in old[name].get("volumes", [])}
|
|
new_volumes = {v["target"]:v for v in new[name].get("volumes", [])}
|
|
if any(new_volumes.get(key) != value for key,value in old_volumes.items()):
|
|
raise DeployError("existing storage or trust mount changed")
|
|
expected_targets = set(old_volumes) | ({"/run/hainuo-credentials/keyring.json"} if name == "connector" else set())
|
|
if set(new_volumes) != expected_targets or env.get("HN_CREDENTIAL_MODE") != "broker":
|
|
raise DeployError("unexpected migration mount or credential mode")
|
|
if merged.get("volumes") != base.get("volumes") or merged.get("networks") != base.get("networks"):
|
|
raise DeployError("database volume or network definitions changed")
|
|
env = new["connector"]["environment"]
|
|
if (env.get("HN_FEISHU_APP_ID") != old["connector"]["environment"].get("HN_FEISHU_APP_ID")
|
|
or new["router"]["environment"].get("HN_FEISHU_APP_ID") != env.get("HN_FEISHU_APP_ID")
|
|
or new["router"]["environment"].get("HN_ENVIRONMENT") != env.get("HN_ENVIRONMENT")
|
|
or env.get("HN_CREDENTIAL_KEYRING_FILE") != "/run/hainuo-credentials/keyring.json"
|
|
or new["router"]["environment"].get("HN_CONNECTOR_INTERNAL_URL") != "http://connector:8000"):
|
|
raise DeployError("application/environment or credential routing mismatch")
|
|
# Old installer initializes HN_ENVIRONMENT=development. Preserve this identity
|
|
# namespace rather than silently converting it to production. All modes still
|
|
# require PostgreSQL and HTTPS/WSS for this existing-server migration.
|
|
require_platform(env)
|
|
for key in ("HN_ADMIN_SECRET","HN_INTERNAL_SECRET","HN_FEISHU_APP_SECRET"):
|
|
if env["HN_ROUTER_ADMISSION_SECRET"] == env.get(key):
|
|
raise DeployError("router admission credential must be independent")
|
|
if any(new["router"]["environment"].get(k) for k in ("HN_ADMIN_SECRET","HN_INTERNAL_SECRET","HN_FEISHU_APP_SECRET")):
|
|
raise DeployError("router has unrelated privileged credentials")
|
|
mounts = [v for v in new["connector"].get("volumes", []) if v["target"] == "/run/hainuo-credentials/keyring.json"]
|
|
if len(mounts) != 1 or not mounts[0].get("read_only"):
|
|
raise DeployError("missing read-only credential keyring mount")
|
|
if router_key_required:
|
|
target,_=router_public_key(merged)
|
|
if new['router']['environment'].get('HN_CONNECTOR_PUBLIC_KEY')!=target:
|
|
raise DeployError('router HN_CONNECTOR_PUBLIC_KEY missing; use repair-router for an already switched plan')
|
|
|
|
|
|
def preflight(args):
|
|
deployment = Path(args.deployment_root).resolve()
|
|
current = inspect(args.connector_container)
|
|
labels = current["labels"]
|
|
if Path(labels["com.docker.compose.project.working_dir"]).resolve() != deployment:
|
|
raise DeployError("deployment path differs from the running connector")
|
|
files = [Path(p) for p in labels["com.docker.compose.project.config_files"].split(",")]
|
|
if any(not p.is_file() or p.resolve().parent != deployment for p in files):
|
|
raise DeployError("missing or external Compose file; review deployment layout")
|
|
plan = {"deployment_root":str(deployment),"project":labels["com.docker.compose.project"],
|
|
"compose_files":[str(p) for p in files]}
|
|
base = run(compose(plan)+["config","--format","json"],True)
|
|
names = set(base["services"])
|
|
require_services(names)
|
|
architecture = run(["docker","image","inspect",current["image"],"--format","{{.Architecture}}"])
|
|
if architecture != "arm64":
|
|
raise DeployError("expected current ARM64 runtime")
|
|
for name in ("connector","router","longconn","enrollment-signer"):
|
|
entry = service(plan["project"],name)
|
|
print(name+": "+entry["image"])
|
|
env = base["services"]["connector"].get("environment",{})
|
|
print("Project: "+plan["project"])
|
|
print("Compose files: "+str(len(files)))
|
|
print("Compose profile included: longconn (configuration parsing; no service startup)")
|
|
print("Current credential mode: "+env.get("HN_CREDENTIAL_MODE","legacy"))
|
|
print("Preflight complete; no files, keys or services changed. Resolved settings withheld.")
|
|
|
|
|
|
def prepare(args):
|
|
if os.name != "nt" and os.geteuid() != 0:
|
|
raise DeployError("run preparation with sudo for protected key ownership")
|
|
deployment = Path(args.deployment_root).resolve()
|
|
output = Path(args.output)
|
|
source = Path(args.source_root).resolve()
|
|
profiles = profiles_from(Path(args.profiles))
|
|
if output.exists() or output.is_symlink() or not output.is_absolute():
|
|
raise DeployError("output must be a new absolute migration directory")
|
|
output = output.resolve()
|
|
if any(c in str(output) for c in ("\n","\r","$")):
|
|
raise DeployError("migration path contains unsupported Compose interpolation characters")
|
|
if output == deployment or deployment in output.parents:
|
|
raise DeployError("migration output must be outside the running deployment")
|
|
current = inspect(args.connector_container)
|
|
labels = current["labels"]
|
|
if Path(labels["com.docker.compose.project.working_dir"]).resolve() != deployment:
|
|
raise DeployError("running deployment directory does not match the supplied root")
|
|
files = [Path(p) for p in labels["com.docker.compose.project.config_files"].split(",")]
|
|
if any(not p.is_absolute() or p.resolve().parent != deployment or not p.is_file() for p in files):
|
|
raise DeployError("Compose files must belong to the existing deployment root")
|
|
if not (deployment/".env").is_file() or not (source/"relay/credential_api.py").is_file():
|
|
raise DeployError("missing live configuration or staged broker source")
|
|
project = labels["com.docker.compose.project"]
|
|
if not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", project):
|
|
raise DeployError("invalid Compose project")
|
|
roles = {name:service(project,name) for name in ("connector","router","longconn","enrollment-signer","edge")}
|
|
if any(not re.fullmatch(r"sha256:[a-f0-9]{64}",r["image"]) for r in roles.values()):
|
|
raise DeployError("invalid running image identity")
|
|
if roles["connector"]["id"] != current["id"]:
|
|
raise DeployError("connector inventory changed during preparation")
|
|
architecture = run(["docker","image","inspect",current["image"],"--format","{{.Architecture}}"])
|
|
if architecture != "arm64":
|
|
raise DeployError("this migration targets the verified ARM64 runtime only")
|
|
plan_id = secrets.token_hex(8)
|
|
candidate = "hainuo-relay:broker-candidate-"+plan_id
|
|
plan = {"format":1,"plan_id":plan_id,"deployment_root":str(deployment),"project":project,
|
|
"compose_files":[str(p) for p in files],"roles":roles,"candidate_image":candidate,
|
|
"base_image":current["image"],"architecture":architecture,
|
|
"baseline_hashes":{str(p):digest(p) for p in [deployment/".env",*files]}}
|
|
# Durable tags preserve each heterogeneous running application image.
|
|
for name in ("connector","router","longconn","enrollment-signer"):
|
|
run(["docker","tag",roles[name]["image"],"hainuo-relay:broker-rollback-"+plan_id+"-"+name])
|
|
base = run(compose(plan)+["config","--format","json"],True)
|
|
if base["services"]["connector"].get("environment",{}).get("HN_CREDENTIAL_MODE","legacy") != "legacy":
|
|
raise DeployError("already broker: preserve the existing keyring and migration plan")
|
|
require_platform({**base["services"]["connector"].get("environment",{}),"HN_OAUTH_ENABLED":"true"})
|
|
public_key,_=router_public_key(base)
|
|
output.mkdir(mode=0o700,parents=False)
|
|
os.chmod(str(output),0o700)
|
|
keyring = {"active_key_id":"initial-v1","keys":{"initial-v1":base64.b64encode(os.urandom(32)).decode("ascii")}}
|
|
write_json(output/"credential-keyring.json",keyring)
|
|
if os.name != "nt":
|
|
os.chown(str(output/"credential-keyring.json"),10001,10001)
|
|
admission = secrets.token_hex(32)
|
|
overlay = {"services":{
|
|
"connector":{"image":candidate,"environment":{"HN_CREDENTIAL_MODE":"broker",
|
|
"HN_CREDENTIAL_KEYRING_FILE":"/run/hainuo-credentials/keyring.json",
|
|
"HN_CREDENTIAL_PERMISSION_PROFILES":json.dumps(profiles,separators=(",", ":")),
|
|
"HN_ROUTER_ADMISSION_SECRET":admission,"HN_OAUTH_ENABLED":"true"},
|
|
"volumes":[{"type":"bind","source":str(output/"credential-keyring.json"),
|
|
"target":"/run/hainuo-credentials/keyring.json","read_only":True}]},
|
|
"router":{"image":candidate,"environment":{"HN_CREDENTIAL_MODE":"broker",
|
|
"HN_CONNECTOR_PUBLIC_KEY":public_key,
|
|
"HN_ROUTER_ADMISSION_SECRET":admission,"HN_CONNECTOR_INTERNAL_URL":"http://connector:8000",
|
|
"HN_FEISHU_APP_ID":"${HN_FEISHU_APP_ID:?Configure existing application}",
|
|
"HN_ENVIRONMENT":"${HN_ENVIRONMENT:?Configure existing environment}"}}}}
|
|
write_json(output/"compose.broker.json",overlay)
|
|
write_json(output/"compose.rollback.json",{"services":{name:{"image":roles[name]["image"]}
|
|
for name in ("connector","router","longconn","enrollment-signer")}})
|
|
payload = output/"build"
|
|
payload.mkdir(mode=0o700)
|
|
for path in (source/"relay").rglob("*"):
|
|
if path.is_symlink():
|
|
raise DeployError("symlink in staged source")
|
|
if path.is_file() and path.suffix in {".py",".js",".css",".html"} and "__pycache__" not in path.parts:
|
|
target = payload/path.relative_to(source)
|
|
target.parent.mkdir(parents=True,exist_ok=True)
|
|
shutil.copyfile(str(path),str(target))
|
|
os.chmod(str(target),0o644)
|
|
dockerfile = ('ARG BASE_IMAGE\nFROM ${BASE_IMAGE}\nWORKDIR /app\nUSER 0:0\n'
|
|
'COPY --chown=10001:10001 relay /app/relay\nUSER 10001:10001\n'
|
|
'RUN python -c "import relay.credential_api, relay.binding_state, relay.router"\n'
|
|
'ENTRYPOINT ["python", "-m", "relay.cli"]\n')
|
|
(payload/"Dockerfile").write_bytes(dockerfile.encode("utf-8"))
|
|
plan["payload_hashes"] = {str(p.relative_to(output)):digest(p) for p in payload.rglob("*") if p.is_file()}
|
|
plan["prepared_hashes"] = {p:digest(output/p) for p in (
|
|
"credential-keyring.json","compose.broker.json","compose.rollback.json")}
|
|
write_json(output/"plan.json",plan)
|
|
merged = run(compose(plan,output/"compose.broker.json")+["config","--format","json"],True)
|
|
validate_overlay(base,merged,candidate)
|
|
print("PREPARED: "+str(output)+" (services unchanged; private files must not be shared)")
|
|
|
|
|
|
def load_plan(root):
|
|
plan = json.loads((root/"plan.json").read_text(encoding="utf-8"))
|
|
if plan.get("format") != 1 or not re.fullmatch(r"[a-f0-9]{16}",plan.get("plan_id","")):
|
|
raise DeployError("unsupported migration plan")
|
|
return plan
|
|
|
|
|
|
def diagnose(root,plan):
|
|
config = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
|
|
env = config["services"]["connector"].get("environment",{})
|
|
value = env.get("HN_ENVIRONMENT")
|
|
print("HN_ENVIRONMENT: "+(value if value in {"production","development","test"} else "missing or invalid"))
|
|
print("HN_DATABASE_URL: "+("PostgreSQL configured (value withheld)" if str(env.get("HN_DATABASE_URL","")).startswith(("postgresql://","postgres://")) else "missing or unsupported (value withheld)"))
|
|
for name in ("HN_FEISHU_ENABLED","HN_OAUTH_ENABLED"):
|
|
print(name+": "+("enabled" if enabled(env.get(name)) else "disabled or missing"))
|
|
problems = platform_problems(env)
|
|
for problem in problems:
|
|
print("CHECK: "+problem)
|
|
print("Read-only diagnosis; existing preparation and keys preserved; no container restart.")
|
|
if problems:
|
|
raise DeployError("review the named fields above; do not paste configuration or regenerate keys")
|
|
|
|
|
|
def log_signals(text):
|
|
"""Classify locally; never return arbitrary log lines, SQL, URLs or tokens."""
|
|
known = ('ModuleNotFoundError','ImportError','FileNotFoundError','PermissionError','ValidationError',
|
|
'CredentialError','OperationalError','ProgrammingError','UndefinedColumn','UndefinedTable',
|
|
'InsufficientPrivilege','ReadOnlySqlTransaction','SyntaxError','IndentationError',
|
|
'ConnectionRefusedError','AuthenticationError','ConnectionError','TimeoutError','MemoryError')
|
|
signals = [name for name in known if re.search(r'\b'+name+r'\b',text)]
|
|
if 'read-only file system' in text.lower():
|
|
signals.append('read-only-filesystem')
|
|
if 'address already in use' in text.lower():
|
|
signals.append('port-in-use')
|
|
return signals
|
|
|
|
|
|
def deployment_state(root,plan):
|
|
expected = None
|
|
if (root/"candidate.json").is_file():
|
|
expected = json.loads((root/"candidate.json").read_text(encoding="utf-8")).get("image")
|
|
for name in ("connector","router","longconn","enrollment-signer","edge"):
|
|
try:
|
|
entry = service(plan["project"],name,include_stopped=True)
|
|
except ServiceLookupError:
|
|
print(name+": no unique existing container; inspect docker ps -a for this project")
|
|
continue
|
|
except DeployError:
|
|
print(name+": container inspection failed; no container state inferred")
|
|
continue
|
|
state = entry.get("state",{})
|
|
status = state.get("status") if state.get("status") in {'created','running','paused','restarting','removing','exited','dead'} else 'unknown'
|
|
health = state.get("health") if state.get("health") in {'none','starting','healthy','unhealthy'} else 'unknown'
|
|
image = 'candidate' if expected and entry['image']==expected else (
|
|
'original' if entry['image']==plan['roles'][name]['image'] else 'different')
|
|
numbers = {k:v for k,v in state.items() if k in {'exit_code','restart_count'} and type(v) is int}
|
|
print(name+': image='+image+' status='+status+' health='+health+' '+json.dumps(numbers,sort_keys=True))
|
|
if name in {'connector','router'}:
|
|
raw = run(['docker','logs','--tail','100',entry['id']],optional=True)
|
|
if raw is not None:
|
|
print(' startup signals: '+(', '.join(log_signals(raw)) or 'no recognized error class; raw logs withheld'))
|
|
print('STATE inspection only; no files, keys or containers changed. Do not retry activation before reviewing partial switch state.')
|
|
|
|
|
|
def check(root, plan, baseline=True, router_key_required=True):
|
|
if os.name != "nt":
|
|
if root.stat().st_mode & 0o077:
|
|
raise DeployError("migration directory must not be accessible to other users")
|
|
for name in ("credential-keyring.json","compose.broker.json","compose.rollback.json","plan.json"):
|
|
if (root/name).stat().st_mode & 0o077:
|
|
raise DeployError("migration private file permissions are too broad")
|
|
if (root/"credential-keyring.json").stat().st_uid != 10001:
|
|
raise DeployError("credential keyring owner must match service UID 10001")
|
|
for path, expected in plan["baseline_hashes"].items():
|
|
if digest(Path(path)) != expected:
|
|
raise DeployError("live Compose/configuration changed; review before proceeding")
|
|
for path, expected in plan["payload_hashes"].items():
|
|
if digest(root/path) != expected:
|
|
raise DeployError("staged source changed after preparation")
|
|
for path, expected in plan["prepared_hashes"].items():
|
|
if digest(root/path) != expected:
|
|
raise DeployError("prepared keyring or overlay changed; do not regenerate credential keys")
|
|
for name, old in plan["roles"].items():
|
|
current = service(plan["project"],name,include_stopped=not baseline and name in {"connector","router"})
|
|
if (baseline or name not in {"connector","router"}) and current["image"] != old["image"]:
|
|
raise DeployError("running service image changed since preparation: "+name)
|
|
if not baseline and name in {"connector","router"}:
|
|
allowed = {old['image']}
|
|
if (root/'candidate.json').is_file():
|
|
allowed.add(json.loads((root/'candidate.json').read_text(encoding='utf-8'))['image'])
|
|
if current['image'] not in allowed:
|
|
raise DeployError('unreviewed target image; refusing rollback: '+name)
|
|
base = run(compose(plan)+["config","--format","json"],True)
|
|
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
|
|
validate_overlay(base,merged,plan["candidate_image"],router_key_required=router_key_required)
|
|
# Require the current edge's explicit private-path block, before catch-all handling.
|
|
mounts = base["services"]["edge"].get("volumes",[])
|
|
caddy = next((v["source"] for v in mounts if v.get("target") == "/etc/caddy/Caddyfile" and v.get("type") == "bind"),None)
|
|
if not caddy:
|
|
raise DeployError("cannot verify current edge configuration")
|
|
text = Path(caddy).read_text(encoding="utf-8")
|
|
rule = re.search(r"@([A-Za-z0-9_-]+)\s+path\s+/admin/\*\s+/internal/\*\s+/ready",text)
|
|
deny = re.search(r"respond\s+@"+re.escape(rule.group(1))+r"\s+404",text) if rule else None
|
|
first_handle = re.search(r"\bhandle(?:_path)?\b",text)
|
|
if not deny or (first_handle and deny.start() > first_handle.start()):
|
|
raise DeployError("edge private-path block is not the reviewed configuration")
|
|
print("CHECKED: Compose/storage and permitted service images verified; no services restarted")
|
|
|
|
|
|
def build(root,plan):
|
|
check(root,plan)
|
|
if run(["docker","image","inspect",plan["candidate_image"]],optional=True) is not None:
|
|
raise DeployError("candidate image already exists; do not overwrite a built candidate")
|
|
base_tag = "hainuo-relay:broker-base-"+plan["plan_id"]
|
|
run(["docker","tag",plan["base_image"],base_tag])
|
|
run(["docker","build","--pull=false","--network","none","--build-arg","BASE_IMAGE="+base_tag,
|
|
"-t",plan["candidate_image"],str(root/"build")])
|
|
image = run(["docker","image","inspect",plan["candidate_image"],"--format","{{.Id}}"])
|
|
run(["docker","run","--rm","--network","none","--read-only","--entrypoint","python",
|
|
plan["candidate_image"],"-c","import relay.credential_api, relay.binding_state, relay.device_lifecycle"])
|
|
runtime_check(root,plan)
|
|
write_json(root/"candidate.json",{"image":image,"tag":plan["candidate_image"]})
|
|
print("BUILT: "+plan["candidate_image"]+" (offline runtime reuse; not activated)")
|
|
|
|
|
|
def runtime_check(root,plan):
|
|
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
|
|
public_key,public_source=router_public_key(merged)
|
|
# Actual secrets travel via stdin only: no command-line values, env dump,
|
|
# database connection or platform request. The container has no network.
|
|
code = ("import json,os,sys\nfrom pathlib import Path\nfrom relay.config import Settings\n"
|
|
"from relay.credential_store import CredentialKeyring\n"
|
|
"from relay.crypto import load_public\n"
|
|
"try:\n data=json.load(sys.stdin)\n os.environ.update(data['environment'])\n"
|
|
" s=Settings(role=data['role'])\n"
|
|
" if data['role']=='connector': CredentialKeyring.from_file(s.credential_keyring_file)\n"
|
|
" if data['role']=='router': load_public(s.connector_public_key)\n"
|
|
"except Exception:\n sys.exit(1)\n")
|
|
for role in ("connector","router"):
|
|
env = {key:str(value) for key,value in merged["services"][role]["environment"].items() if value is not None}
|
|
command=["docker","run","--rm","-i","--network","none","--read-only","--entrypoint","python",
|
|
"-v",str(root/"credential-keyring.json")+":/run/hainuo-credentials/keyring.json:ro",
|
|
"-v",str(public_source)+":"+public_key+":ro",plan["candidate_image"],"-c",code]
|
|
print('Runtime check: '+role+' configuration and required key files',flush=True)
|
|
run(command,input_text=json.dumps({"role":role,"environment":env}))
|
|
|
|
|
|
def repair_router(root,plan):
|
|
# Read all known partial-switch states, but do not weaken general activate checks.
|
|
check(root,plan,baseline=False,router_key_required=False)
|
|
built=json.loads((root/'candidate.json').read_text(encoding='utf-8'))
|
|
connector=service(plan['project'],'connector',include_stopped=True)
|
|
if (connector['image']!=built['image'] or connector.get('state',{}).get('status')!='running'
|
|
or connector.get('state',{}).get('health')!='healthy'):
|
|
raise DeployError('repair-router requires the known candidate connector to be running and healthy; inspect state or rollback')
|
|
base=run(compose(plan)+['config','--format','json'],True)
|
|
target,_=router_public_key(base)
|
|
path=root/'compose.broker.json'
|
|
overlay=json.loads(path.read_text(encoding='utf-8'))
|
|
old=overlay['services']['router']['environment'].get('HN_CONNECTOR_PUBLIC_KEY')
|
|
if old not in (None,'',target):
|
|
raise DeployError('unexpected prepared router public-key path; manual review required')
|
|
if old!=target:
|
|
backup=root/'compose.broker.before-router-key.json'
|
|
plan_backup=root/'plan.before-router-key.json'
|
|
if backup.exists() or plan_backup.exists():
|
|
raise DeployError('repair backup already exists; preserve keys and inspect the interrupted repair')
|
|
write_json(backup,overlay);write_json(plan_backup,plan)
|
|
overlay['services']['router']['environment']['HN_CONNECTOR_PUBLIC_KEY']=target
|
|
replace_json(path,overlay)
|
|
plan['prepared_hashes']['compose.broker.json']=digest(path)
|
|
replace_json(root/'plan.json',plan)
|
|
print('REPAIR: existing mounted connector public key selected; credential keys and connector preserved',flush=True)
|
|
check(root,plan,baseline=False)
|
|
runtime_check(root,plan)
|
|
image=run(['docker','image','inspect',plan['candidate_image'],'--format','{{.Id}}'])
|
|
if image!=built['image']:
|
|
raise DeployError('candidate image identity changed')
|
|
print('REPAIR: recreate router only',flush=True)
|
|
run(compose(plan,root/'compose.broker.json')+['up','-d','--no-deps','--no-build','--pull','never',
|
|
'--wait','--wait-timeout','180','router'])
|
|
print('REPAIRED: router only; verify state and real client handshake before acceptance')
|
|
|
|
|
|
def activate(root,plan,ready):
|
|
if not ready:
|
|
raise DeployError("v2 client not confirmed ready; only prepare/check/build are allowed now")
|
|
check(root,plan)
|
|
print('ACTIVATION step: verify candidate image identity',flush=True)
|
|
built = json.loads((root/"candidate.json").read_text(encoding="utf-8"))
|
|
image = run(["docker","image","inspect",plan["candidate_image"],"--format","{{.Id}}"])
|
|
if built != {"image":image,"tag":plan["candidate_image"]}:
|
|
raise DeployError("candidate image identity changed")
|
|
print('ACTIVATION step: isolated candidate configuration/keyring check (no service switch)',flush=True)
|
|
runtime_check(root,plan)
|
|
print('ACTIVATION step: recreate connector/router; failures may leave a partial switch, inspect state',flush=True)
|
|
run(compose(plan,root/"compose.broker.json")+["up","-d","--no-deps","--no-build","--pull","never",
|
|
"--wait","--wait-timeout","180","connector","router"])
|
|
print("ACTIVATED: connector/router only. Verify real OAuth, v2 client, unbind and task routing.")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
sub = parser.add_subparsers(dest="action",required=True)
|
|
flight = sub.add_parser("preflight")
|
|
flight.add_argument("--deployment-root",required=True)
|
|
flight.add_argument("--connector-container",default="hainuo-relay-connector-1")
|
|
prep = sub.add_parser("prepare")
|
|
prep.add_argument("--deployment-root",required=True)
|
|
prep.add_argument("--source-root",default=str(Path(__file__).resolve().parents[1]))
|
|
prep.add_argument("--profiles",required=True)
|
|
prep.add_argument("--output",required=True)
|
|
prep.add_argument("--connector-container",default="hainuo-relay-connector-1")
|
|
for name in ("repair-router","state","diagnose","check","build","activate","rollback"):
|
|
cmd = sub.add_parser(name)
|
|
cmd.add_argument("--plan",required=True)
|
|
if name == "activate":
|
|
cmd.add_argument("--client-v2-ready",action="store_true")
|
|
if name == "rollback":
|
|
cmd.add_argument("--stop-broker-use",action="store_true")
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.action == "preflight":
|
|
preflight(args)
|
|
return
|
|
if args.action == "prepare":
|
|
prepare(args)
|
|
return
|
|
root = Path(args.plan).resolve()
|
|
plan = load_plan(root)
|
|
if args.action == "repair-router":
|
|
repair_router(root,plan)
|
|
elif args.action == "state":
|
|
deployment_state(root,plan)
|
|
elif args.action == "diagnose":
|
|
diagnose(root,plan)
|
|
elif args.action == "check":
|
|
check(root,plan)
|
|
elif args.action == "build":
|
|
build(root,plan)
|
|
elif args.action == "activate":
|
|
activate(root,plan,args.client_v2_ready)
|
|
else:
|
|
if not args.stop_broker_use:
|
|
raise DeployError("rollback requires stopping broker clients; old refresh credentials are not restored")
|
|
check(root,plan,baseline=False,router_key_required=False)
|
|
run(compose(plan,root/"compose.rollback.json")+["up","-d","--no-deps","--no-build","--pull","never",
|
|
"--wait","--wait-timeout","180","connector","router"])
|
|
print("ROLLED BACK containers only; preserve keyring/database. Client mode and reauthorization must be coordinated.")
|
|
except (DeployError,OSError,ValueError,KeyError,TypeError):
|
|
error = sys.exc_info()[1]
|
|
print("STOP: "+(str(error) if isinstance(error,DeployError) else "invalid/missing migration input; secret details withheld"),file=sys.stderr)
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|