Files
ecs-relay-service/deployable/scripts/broker_deploy.py
T

757 lines
43 KiB
Python

#!/usr/bin/env python3
"""Existing-deployment broker migration: stdlib-only host tool, Python >=3.7.
Never initialize identities, edit the live .env, or dump resolved Compose output.
prepare/check/build do not recreate services. Activation requires a v2 client.
"""
import argparse
import base64
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import secrets
import shutil
import subprocess
import sys
import tempfile
from urllib.parse import urlsplit
class DeployError(RuntimeError):
pass
class ServiceLookupError(DeployError):
pass
def run(args, json_result=False, optional=False, input_text=None):
result = subprocess.run(args, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
if result.returncode:
if optional:
return None
if args[:2] == ["docker","compose"]:
stage = "compose-up" if "up" in args else "compose-config"
elif args[:2] == ["docker","run"]:
stage = "candidate-runtime-check"
elif args[:3] == ["docker","image","inspect"]:
stage = "image-inspect"
elif args[:2] == ["docker","inspect"]:
stage = "container-inspect"
else:
stage = "docker-command"
raise DeployError(stage+" failed (exit "+str(result.returncode)+"); raw output withheld; run state to inspect current services")
if args[:2] == ["docker","logs"]:
return (result.stdout+"\n"+result.stderr).strip()
try:
return json.loads(result.stdout) if json_result else result.stdout.strip()
except ValueError:
raise DeployError("invalid command response") from None
def digest(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def write_json(path, value):
# All outputs belong to the newly created, private migration directory.
fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, "w", encoding="utf-8") as stream:
json.dump(value, stream, ensure_ascii=False, indent=2)
stream.write("\n")
def replace_json(path,value):
name = None
try:
fd,name = tempfile.mkstemp(prefix='.'+path.name+'.',dir=str(path.parent))
with os.fdopen(fd,'w',encoding='utf-8') as stream:
json.dump(value,stream,ensure_ascii=False,indent=2)
stream.write('\n');stream.flush();os.fsync(stream.fileno())
os.replace(name,str(path))
finally:
if name and os.path.exists(name):os.unlink(name)
def inspect(container):
template = ('{"id":{{json .Id}},"image":{{json .Image}},'
'"image_ref":{{json .Config.Image}},"labels":{{json .Config.Labels}},'
'"state":{"status":{{json .State.Status}},"health":'
'{{with index .State "Health"}}{{json .Status}}{{else}}"none"{{end}},'
'"exit_code":{{.State.ExitCode}},"restart_count":{{.RestartCount}}}}')
return run(["docker", "inspect", container, "--format", template], True)
def service(project, name, include_stopped=False):
values = run(["docker", "ps", "-aq" if include_stopped else "-q", "--filter", "label=com.docker.compose.project="+project,
"--filter", "label=com.docker.compose.service="+name]).splitlines()
if len(values) != 1:
raise ServiceLookupError("expected exactly one "+("existing" if include_stopped else "running")+" relay service: "+name)
return inspect(values[0])
def compose(plan, override=None):
args = ["docker", "compose", "--project-directory", plan["deployment_root"],
"--env-file", str(Path(plan["deployment_root"])/".env"), "-p", plan["project"],
"--profile", "longconn"]
for file in plan["compose_files"]:
args.extend(["-f", file])
if override:
args.extend(["-f", str(override)])
return args
def require_services(names):
expected = {"connector","router","longconn","enrollment-signer","postgres","redis","edge"}
if set(names) != expected:
# Service names only, never resolved environment/configuration contents.
raise DeployError("unexpected Compose services; missing="+json.dumps(sorted(expected-set(names)))
+"; extra="+json.dumps(sorted(set(names)-expected)))
def profiles_from(path):
try:
data = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(data, dict) or not data or len(data) > 32:
raise ValueError()
for name, scopes in data.items():
if (not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", name) or not isinstance(scopes, list)
or not scopes or len(scopes) > 128 or any(not isinstance(s, str)
or not re.fullmatch(r"[A-Za-z0-9_:.-]{1,256}", s) for s in scopes)):
raise ValueError()
return {name: sorted(set(scopes)) for name, scopes in data.items()}
except (OSError, ValueError, TypeError):
raise DeployError("invalid product permission profiles; use the product-approved JSON object") from None
def enabled(value):
return str(value).strip().lower() in {"true","1","yes","on"}
def platform_problems(env):
problems = []
if not str(env.get("HN_DATABASE_URL", "")).startswith(("postgresql://","postgres://")):
problems.append("HN_DATABASE_URL: PostgreSQL connection required (value withheld)")
if env.get("HN_ENVIRONMENT") not in {"production","development","test"}:
problems.append("HN_ENVIRONMENT: expected production/development/test; preserve the reviewed environment")
for name in ("HN_FEISHU_ENABLED","HN_OAUTH_ENABLED"):
if not enabled(env.get(name)):
problems.append(name+": must be enabled")
for name,scheme in (("HN_PUBLIC_URL","https"),("HN_ROUTER_URL","wss"),
("HN_FEISHU_API_BASE","https"),("HN_FEISHU_AUTHORIZE_URL","https")):
try:
value = urlsplit(str(env.get(name,"")))
valid = value.scheme == scheme and value.hostname and not value.username and not value.password
except ValueError:
valid = False
if not valid:
problems.append(name+": fixed "+scheme.upper()+" endpoint required (value withheld)")
return problems
def require_platform(env):
problems = platform_problems(env)
if problems:
raise DeployError("broker platform configuration incomplete; "+"; ".join(problems))
def router_public_key(config):
router=config['services']['router']
env=router.get('environment',{})
value=env.get('HN_CONNECTOR_PUBLIC_KEY') or str(PurePosixPath(env.get('HN_CA_CERT','/trust/ca.pem')).parent/'connector.pub')
target=PurePosixPath(value)
if not target.is_absolute() or '..' in target.parts:
raise DeployError('invalid router connector public-key path')
for mount in router.get('volumes',[]):
if mount.get('type')!='bind' or not mount.get('read_only'):continue
base=PurePosixPath(mount['target'])
try:
relative=target.relative_to(base)
except ValueError:
continue
source=Path(mount['source'])
source=source.joinpath(*relative.parts) if relative.parts else source
if source.is_file() and not source.is_symlink():
data=source.read_bytes()
if b'-----BEGIN PUBLIC KEY-----' in data and b'PRIVATE KEY' not in data:
return str(target),source
raise DeployError('router connector public key is not a verified read-only trust mount')
def validate_overlay(base, merged, candidate, router_key_required=True):
old, new = base["services"], merged["services"]
require_services(old)
require_services(new)
for name in set(old)-{"connector","router"}:
if old[name] != new[name]:
raise DeployError("overlay changes an unrelated relay service: "+name)
for name in ("connector","router"):
for field in set(old[name]) | set(new[name]):
if field not in {"environment","image","volumes"} and old[name].get(field) != new[name].get(field):
raise DeployError("overlay changes non-migration service settings")
if new[name].get("image") != candidate:
raise DeployError("candidate image mismatch")
env = new[name]["environment"]
for key, value in old[name].get("environment", {}).items():
if key not in {"HN_CREDENTIAL_MODE","HN_ROUTER_ADMISSION_SECRET","HN_CREDENTIAL_KEYRING_FILE",
"HN_CREDENTIAL_PERMISSION_PROFILES","HN_CONNECTOR_INTERNAL_URL","HN_FEISHU_APP_ID","HN_OAUTH_ENABLED"}:
if env.get(key) != value:
raise DeployError("existing environment changed: "+key)
old_volumes = {v["target"]:v for v in old[name].get("volumes", [])}
new_volumes = {v["target"]:v for v in new[name].get("volumes", [])}
if any(new_volumes.get(key) != value for key,value in old_volumes.items()):
raise DeployError("existing storage or trust mount changed")
expected_targets = set(old_volumes) | ({"/run/hainuo-credentials/keyring.json"} if name == "connector" else set())
if set(new_volumes) != expected_targets or env.get("HN_CREDENTIAL_MODE") != "broker":
raise DeployError("unexpected migration mount or credential mode")
if merged.get("volumes") != base.get("volumes") or merged.get("networks") != base.get("networks"):
raise DeployError("database volume or network definitions changed")
env = new["connector"]["environment"]
if (env.get("HN_FEISHU_APP_ID") != old["connector"]["environment"].get("HN_FEISHU_APP_ID")
or new["router"]["environment"].get("HN_FEISHU_APP_ID") != env.get("HN_FEISHU_APP_ID")
or new["router"]["environment"].get("HN_ENVIRONMENT") != env.get("HN_ENVIRONMENT")
or env.get("HN_CREDENTIAL_KEYRING_FILE") != "/run/hainuo-credentials/keyring.json"
or new["router"]["environment"].get("HN_CONNECTOR_INTERNAL_URL") != "http://connector:8000"):
raise DeployError("application/environment or credential routing mismatch")
# Old installer initializes HN_ENVIRONMENT=development. Preserve this identity
# namespace rather than silently converting it to production. All modes still
# require PostgreSQL and HTTPS/WSS for this existing-server migration.
require_platform(env)
for key in ("HN_ADMIN_SECRET","HN_INTERNAL_SECRET","HN_FEISHU_APP_SECRET"):
if env["HN_ROUTER_ADMISSION_SECRET"] == env.get(key):
raise DeployError("router admission credential must be independent")
if any(new["router"]["environment"].get(k) for k in ("HN_ADMIN_SECRET","HN_INTERNAL_SECRET","HN_FEISHU_APP_SECRET")):
raise DeployError("router has unrelated privileged credentials")
mounts = [v for v in new["connector"].get("volumes", []) if v["target"] == "/run/hainuo-credentials/keyring.json"]
if len(mounts) != 1 or not mounts[0].get("read_only"):
raise DeployError("missing read-only credential keyring mount")
if router_key_required:
target,_=router_public_key(merged)
if new['router']['environment'].get('HN_CONNECTOR_PUBLIC_KEY')!=target:
raise DeployError('router HN_CONNECTOR_PUBLIC_KEY missing; use repair-router for an already switched plan')
def preflight(args):
deployment = Path(args.deployment_root).resolve()
current = inspect(args.connector_container)
labels = current["labels"]
if Path(labels["com.docker.compose.project.working_dir"]).resolve() != deployment:
raise DeployError("deployment path differs from the running connector")
files = [Path(p) for p in labels["com.docker.compose.project.config_files"].split(",")]
if any(not p.is_file() or p.resolve().parent != deployment for p in files):
raise DeployError("missing or external Compose file; review deployment layout")
plan = {"deployment_root":str(deployment),"project":labels["com.docker.compose.project"],
"compose_files":[str(p) for p in files]}
base = run(compose(plan)+["config","--format","json"],True)
names = set(base["services"])
require_services(names)
architecture = run(["docker","image","inspect",current["image"],"--format","{{.Architecture}}"])
if architecture != "arm64":
raise DeployError("expected current ARM64 runtime")
for name in ("connector","router","longconn","enrollment-signer"):
entry = service(plan["project"],name)
print(name+": "+entry["image"])
env = base["services"]["connector"].get("environment",{})
print("Project: "+plan["project"])
print("Compose files: "+str(len(files)))
print("Compose profile included: longconn (configuration parsing; no service startup)")
print("Current credential mode: "+env.get("HN_CREDENTIAL_MODE","legacy"))
print("Preflight complete; no files, keys or services changed. Resolved settings withheld.")
def prepare(args):
if os.name != "nt" and os.geteuid() != 0:
raise DeployError("run preparation with sudo for protected key ownership")
deployment = Path(args.deployment_root).resolve()
output = Path(args.output)
source = Path(args.source_root).resolve()
profiles = profiles_from(Path(args.profiles))
if output.exists() or output.is_symlink() or not output.is_absolute():
raise DeployError("output must be a new absolute migration directory")
output = output.resolve()
if any(c in str(output) for c in ("\n","\r","$")):
raise DeployError("migration path contains unsupported Compose interpolation characters")
if output == deployment or deployment in output.parents:
raise DeployError("migration output must be outside the running deployment")
current = inspect(args.connector_container)
labels = current["labels"]
if Path(labels["com.docker.compose.project.working_dir"]).resolve() != deployment:
raise DeployError("running deployment directory does not match the supplied root")
files = [Path(p) for p in labels["com.docker.compose.project.config_files"].split(",")]
if any(not p.is_absolute() or p.resolve().parent != deployment or not p.is_file() for p in files):
raise DeployError("Compose files must belong to the existing deployment root")
if not (deployment/".env").is_file() or not (source/"relay/credential_api.py").is_file():
raise DeployError("missing live configuration or staged broker source")
project = labels["com.docker.compose.project"]
if not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", project):
raise DeployError("invalid Compose project")
roles = {name:service(project,name) for name in ("connector","router","longconn","enrollment-signer","edge")}
if any(not re.fullmatch(r"sha256:[a-f0-9]{64}",r["image"]) for r in roles.values()):
raise DeployError("invalid running image identity")
if roles["connector"]["id"] != current["id"]:
raise DeployError("connector inventory changed during preparation")
architecture = run(["docker","image","inspect",current["image"],"--format","{{.Architecture}}"])
if architecture != "arm64":
raise DeployError("this migration targets the verified ARM64 runtime only")
plan_id = secrets.token_hex(8)
candidate = "hainuo-relay:broker-candidate-"+plan_id
plan = {"format":1,"plan_id":plan_id,"deployment_root":str(deployment),"project":project,
"compose_files":[str(p) for p in files],"roles":roles,"candidate_image":candidate,
"base_image":current["image"],"architecture":architecture,
"baseline_hashes":{str(p):digest(p) for p in [deployment/".env",*files]}}
# Durable tags preserve each heterogeneous running application image.
for name in ("connector","router","longconn","enrollment-signer"):
run(["docker","tag",roles[name]["image"],"hainuo-relay:broker-rollback-"+plan_id+"-"+name])
base = run(compose(plan)+["config","--format","json"],True)
if base["services"]["connector"].get("environment",{}).get("HN_CREDENTIAL_MODE","legacy") != "legacy":
raise DeployError("already broker: preserve the existing keyring and migration plan")
require_platform({**base["services"]["connector"].get("environment",{}),"HN_OAUTH_ENABLED":"true"})
public_key,_=router_public_key(base)
output.mkdir(mode=0o700,parents=False)
os.chmod(str(output),0o700)
keyring = {"active_key_id":"initial-v1","keys":{"initial-v1":base64.b64encode(os.urandom(32)).decode("ascii")}}
write_json(output/"credential-keyring.json",keyring)
if os.name != "nt":
os.chown(str(output/"credential-keyring.json"),10001,10001)
admission = secrets.token_hex(32)
overlay = {"services":{
"connector":{"image":candidate,"environment":{"HN_CREDENTIAL_MODE":"broker",
"HN_CREDENTIAL_KEYRING_FILE":"/run/hainuo-credentials/keyring.json",
"HN_CREDENTIAL_PERMISSION_PROFILES":json.dumps(profiles,separators=(",", ":")),
"HN_ROUTER_ADMISSION_SECRET":admission,"HN_OAUTH_ENABLED":"true"},
"volumes":[{"type":"bind","source":str(output/"credential-keyring.json"),
"target":"/run/hainuo-credentials/keyring.json","read_only":True}]},
"router":{"image":candidate,"environment":{"HN_CREDENTIAL_MODE":"broker",
"HN_CONNECTOR_PUBLIC_KEY":public_key,
"HN_ROUTER_ADMISSION_SECRET":admission,"HN_CONNECTOR_INTERNAL_URL":"http://connector:8000",
"HN_FEISHU_APP_ID":"${HN_FEISHU_APP_ID:?Configure existing application}",
"HN_ENVIRONMENT":"${HN_ENVIRONMENT:?Configure existing environment}"}}}}
write_json(output/"compose.broker.json",overlay)
write_json(output/"compose.rollback.json",{"services":{name:{"image":roles[name]["image"]}
for name in ("connector","router","longconn","enrollment-signer")}})
payload = output/"build"
payload.mkdir(mode=0o700)
for path in (source/"relay").rglob("*"):
if path.is_symlink():
raise DeployError("symlink in staged source")
if path.is_file() and path.suffix in {".py",".js",".css",".html"} and "__pycache__" not in path.parts:
target = payload/path.relative_to(source)
target.parent.mkdir(parents=True,exist_ok=True)
shutil.copyfile(str(path),str(target))
os.chmod(str(target),0o644)
dockerfile = ('ARG BASE_IMAGE\nFROM ${BASE_IMAGE}\nWORKDIR /app\nUSER 0:0\n'
'COPY --chown=10001:10001 relay /app/relay\nUSER 10001:10001\n'
'RUN python -c "import relay.credential_api, relay.binding_state, relay.router"\n'
'ENTRYPOINT ["python", "-m", "relay.cli"]\n')
(payload/"Dockerfile").write_bytes(dockerfile.encode("utf-8"))
plan["payload_hashes"] = {str(p.relative_to(output)):digest(p) for p in payload.rglob("*") if p.is_file()}
plan["prepared_hashes"] = {p:digest(output/p) for p in (
"credential-keyring.json","compose.broker.json","compose.rollback.json")}
write_json(output/"plan.json",plan)
merged = run(compose(plan,output/"compose.broker.json")+["config","--format","json"],True)
validate_overlay(base,merged,candidate)
print("PREPARED: "+str(output)+" (services unchanged; private files must not be shared)")
def prepare_update(args):
"""Stage an already-active broker update without generating credentials."""
root=Path(args.plan).resolve()
parent=load_plan(root)
check(root,parent,baseline=False)
built=json.loads((root/'candidate.json').read_text(encoding='utf-8'))
roles={name:service(parent['project'],name) for name in parent['roles']}
if any(roles[name]['image'] != built['image'] for name in ('connector','router')):
raise DeployError('update requires both services on the confirmed parent candidate; inspect state first')
output=Path(args.output)
source=Path(args.source_root).resolve()
deployment=Path(parent['deployment_root']).resolve()
if (not output.is_absolute() or output.exists() or output.is_symlink()
or output.resolve()==deployment or deployment in output.resolve().parents
or any(c in str(output) for c in ('\n','\r','$'))):
raise DeployError('update output must be a new absolute directory outside the live deployment')
if not (source/'relay/connector.py').is_file():
raise DeployError('missing update source')
output=output.resolve()
output.mkdir(mode=0o700)
os.chmod(output,0o700)
plan=json.loads(json.dumps(parent))
plan_id=secrets.token_hex(8)
plan.update(plan_id=plan_id,candidate_image='hainuo-relay:broker-update-'+plan_id,
roles=roles,base_image=built['image'],parent_plan=str(root),update_preserves_credentials=True)
# Copy identical server key material; never rotate/reinitialize while updating code.
shutil.copyfile(root/'credential-keyring.json',output/'credential-keyring.json')
os.chmod(output/'credential-keyring.json',0o600)
if os.name!='nt':os.chown(output/'credential-keyring.json',10001,10001)
overlay=json.loads((root/'compose.broker.json').read_text(encoding='utf-8'))
for name in ('connector','router'):overlay['services'][name]['image']=plan['candidate_image']
for mount in overlay['services']['connector']['volumes']:
if mount['target']=='/run/hainuo-credentials/keyring.json':
mount['source']=str(output/'credential-keyring.json')
write_json(output/'compose.broker.json',overlay)
rollback=json.loads(json.dumps(overlay))
for name in ('connector','router'):
rollback['services'][name]['image']=roles[name]['image']
# An update rollback must retain broker env/key mounts, never silently
# revert to the original legacy credential owner from base Compose files.
write_json(output/'compose.rollback.json',rollback)
payload=output/'build';payload.mkdir(mode=0o700)
for path in (source/'relay').rglob('*'):
if path.is_symlink():raise DeployError('symlink in update source')
if path.is_file() and path.suffix in {'.py','.js','.css','.html'} and '__pycache__' not in path.parts:
target=payload/path.relative_to(source);target.parent.mkdir(parents=True,exist_ok=True)
shutil.copyfile(path,target);os.chmod(target,0o644)
shutil.copyfile(root/'build/Dockerfile',payload/'Dockerfile')
plan['payload_hashes']={str(p.relative_to(output)):digest(p) for p in payload.rglob('*') if p.is_file()}
plan['prepared_hashes']={name:digest(output/name) for name in
('credential-keyring.json','compose.broker.json','compose.rollback.json')}
write_json(output/'plan.json',plan)
check(output,plan)
print('UPDATE PREPARED: existing credential key material and permissions preserved; no services restarted')
def load_plan(root):
plan = json.loads((root/"plan.json").read_text(encoding="utf-8"))
if plan.get("format") != 1 or not re.fullmatch(r"[a-f0-9]{16}",plan.get("plan_id","")):
raise DeployError("unsupported migration plan")
return plan
def diagnose(root,plan):
config = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
env = config["services"]["connector"].get("environment",{})
value = env.get("HN_ENVIRONMENT")
print("HN_ENVIRONMENT: "+(value if value in {"production","development","test"} else "missing or invalid"))
print("HN_DATABASE_URL: "+("PostgreSQL configured (value withheld)" if str(env.get("HN_DATABASE_URL","")).startswith(("postgresql://","postgres://")) else "missing or unsupported (value withheld)"))
for name in ("HN_FEISHU_ENABLED","HN_OAUTH_ENABLED"):
print(name+": "+("enabled" if enabled(env.get(name)) else "disabled or missing"))
problems = platform_problems(env)
for problem in problems:
print("CHECK: "+problem)
print("Read-only diagnosis; existing preparation and keys preserved; no container restart.")
if problems:
raise DeployError("review the named fields above; do not paste configuration or regenerate keys")
def log_signals(text):
"""Classify locally; never return arbitrary log lines, SQL, URLs or tokens."""
known = ('ModuleNotFoundError','ImportError','FileNotFoundError','PermissionError','ValidationError',
'CredentialError','OperationalError','ProgrammingError','UndefinedColumn','UndefinedTable',
'InsufficientPrivilege','ReadOnlySqlTransaction','SyntaxError','IndentationError',
'ConnectionRefusedError','AuthenticationError','ConnectionError','TimeoutError','MemoryError')
signals = [name for name in known if re.search(r'\b'+name+r'\b',text)]
if 'read-only file system' in text.lower():
signals.append('read-only-filesystem')
if 'address already in use' in text.lower():
signals.append('port-in-use')
return signals
def deployment_state(root,plan):
expected = None
if (root/"candidate.json").is_file():
expected = json.loads((root/"candidate.json").read_text(encoding="utf-8")).get("image")
for name in ("connector","router","longconn","enrollment-signer","edge"):
try:
entry = service(plan["project"],name,include_stopped=True)
except ServiceLookupError:
print(name+": no unique existing container; inspect docker ps -a for this project")
continue
except DeployError:
print(name+": container inspection failed; no container state inferred")
continue
state = entry.get("state",{})
status = state.get("status") if state.get("status") in {'created','running','paused','restarting','removing','exited','dead'} else 'unknown'
health = state.get("health") if state.get("health") in {'none','starting','healthy','unhealthy'} else 'unknown'
image = 'candidate' if expected and entry['image']==expected else (
'original' if entry['image']==plan['roles'][name]['image'] else 'different')
numbers = {k:v for k,v in state.items() if k in {'exit_code','restart_count'} and type(v) is int}
print(name+': image='+image+' status='+status+' health='+health+' '+json.dumps(numbers,sort_keys=True))
if name in {'connector','router'}:
raw = run(['docker','logs','--tail','100',entry['id']],optional=True)
if raw is not None:
print(' startup signals: '+(', '.join(log_signals(raw)) or 'no recognized error class; raw logs withheld'))
print('STATE inspection only; no files, keys or containers changed. Healthy candidate services need no reactivation; inspect mixed/unhealthy states before further actions.')
def check(root, plan, baseline=True, router_key_required=True):
if os.name != "nt":
if root.stat().st_mode & 0o077:
raise DeployError("migration directory must not be accessible to other users")
for name in ("credential-keyring.json","compose.broker.json","compose.rollback.json","plan.json"):
if (root/name).stat().st_mode & 0o077:
raise DeployError("migration private file permissions are too broad")
if (root/"credential-keyring.json").stat().st_uid != 10001:
raise DeployError("credential keyring owner must match service UID 10001")
for path, expected in plan["baseline_hashes"].items():
if digest(Path(path)) != expected:
raise DeployError("live Compose/configuration changed; review before proceeding")
for path, expected in plan["payload_hashes"].items():
if digest(root/path) != expected:
raise DeployError("staged source changed after preparation")
for path, expected in plan["prepared_hashes"].items():
if digest(root/path) != expected:
raise DeployError("prepared keyring or overlay changed; do not regenerate credential keys")
for name, old in plan["roles"].items():
current = service(plan["project"],name,include_stopped=not baseline and name in {"connector","router"})
if (baseline or name not in {"connector","router"}) and current["image"] != old["image"]:
raise DeployError("running service image changed since preparation: "+name+"; this plan may already be activated. Run state; do not rebuild or reactivate a healthy candidate")
if not baseline and name in {"connector","router"}:
allowed = {old['image']}
if (root/'candidate.json').is_file():
allowed.add(json.loads((root/'candidate.json').read_text(encoding='utf-8'))['image'])
if current['image'] not in allowed:
raise DeployError('unreviewed target image; selected parent plan does not match running '+name+'; use the latest successfully activated plan for a new update. No services changed')
base = run(compose(plan)+["config","--format","json"],True)
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
validate_overlay(base,merged,plan["candidate_image"],router_key_required=router_key_required)
# Require the current edge's explicit private-path block, before catch-all handling.
mounts = base["services"]["edge"].get("volumes",[])
caddy = next((v["source"] for v in mounts if v.get("target") == "/etc/caddy/Caddyfile" and v.get("type") == "bind"),None)
if not caddy:
raise DeployError("cannot verify current edge configuration")
text = Path(caddy).read_text(encoding="utf-8")
rule = re.search(r"@([A-Za-z0-9_-]+)\s+path\s+/admin/\*\s+/internal/\*\s+/ready",text)
deny = re.search(r"respond\s+@"+re.escape(rule.group(1))+r"\s+404",text) if rule else None
first_handle = re.search(r"\bhandle(?:_path)?\b",text)
if not deny or (first_handle and deny.start() > first_handle.start()):
raise DeployError("edge private-path block is not the reviewed configuration")
print("CHECKED: Compose/storage and permitted service images verified; no services restarted")
def build(root,plan):
check(root,plan)
if run(["docker","image","inspect",plan["candidate_image"]],optional=True) is not None:
raise DeployError("candidate image already exists; do not overwrite a built candidate")
base_tag = "hainuo-relay:broker-base-"+plan["plan_id"]
run(["docker","tag",plan["base_image"],base_tag])
run(["docker","build","--pull=false","--network","none","--build-arg","BASE_IMAGE="+base_tag,
"-t",plan["candidate_image"],str(root/"build")])
image = run(["docker","image","inspect",plan["candidate_image"],"--format","{{.Id}}"])
run(["docker","run","--rm","--network","none","--read-only","--entrypoint","python",
plan["candidate_image"],"-c","import relay.credential_api, relay.binding_state, relay.device_lifecycle"])
runtime_check(root,plan)
write_json(root/"candidate.json",{"image":image,"tag":plan["candidate_image"]})
print("BUILT: "+plan["candidate_image"]+" (offline runtime reuse; not activated)")
def runtime_check(root,plan):
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
public_key,public_source=router_public_key(merged)
# Actual secrets travel via stdin only: no command-line values, env dump,
# database connection or platform request. The container has no network.
code = ("import json,os,sys\nfrom pathlib import Path\nfrom relay.config import Settings\n"
"from relay.credential_store import CredentialKeyring\n"
"from relay.crypto import load_public\n"
"try:\n data=json.load(sys.stdin)\n os.environ.update(data['environment'])\n"
" s=Settings(role=data['role'])\n"
" if data['role']=='connector': CredentialKeyring.from_file(s.credential_keyring_file)\n"
" if data['role']=='router': load_public(s.connector_public_key)\n"
"except Exception:\n sys.exit(1)\n")
for role in ("connector","router"):
env = {key:str(value) for key,value in merged["services"][role]["environment"].items() if value is not None}
command=["docker","run","--rm","-i","--network","none","--read-only","--entrypoint","python",
"-v",str(root/"credential-keyring.json")+":/run/hainuo-credentials/keyring.json:ro",
"-v",str(public_source)+":"+public_key+":ro",plan["candidate_image"],"-c",code]
print('Runtime check: '+role+' configuration and required key files',flush=True)
run(command,input_text=json.dumps({"role":role,"environment":env}))
def repair_router(root,plan):
# Read all known partial-switch states, but do not weaken general activate checks.
check(root,plan,baseline=False,router_key_required=False)
built=json.loads((root/'candidate.json').read_text(encoding='utf-8'))
connector=service(plan['project'],'connector',include_stopped=True)
if (connector['image']!=built['image'] or connector.get('state',{}).get('status')!='running'
or connector.get('state',{}).get('health')!='healthy'):
raise DeployError('repair-router requires the known candidate connector to be running and healthy; inspect state or rollback')
base=run(compose(plan)+['config','--format','json'],True)
target,_=router_public_key(base)
path=root/'compose.broker.json'
overlay=json.loads(path.read_text(encoding='utf-8'))
old=overlay['services']['router']['environment'].get('HN_CONNECTOR_PUBLIC_KEY')
if old not in (None,'',target):
raise DeployError('unexpected prepared router public-key path; manual review required')
if old!=target:
backup=root/'compose.broker.before-router-key.json'
plan_backup=root/'plan.before-router-key.json'
if backup.exists() or plan_backup.exists():
raise DeployError('repair backup already exists; preserve keys and inspect the interrupted repair')
write_json(backup,overlay);write_json(plan_backup,plan)
overlay['services']['router']['environment']['HN_CONNECTOR_PUBLIC_KEY']=target
replace_json(path,overlay)
plan['prepared_hashes']['compose.broker.json']=digest(path)
replace_json(root/'plan.json',plan)
print('REPAIR: existing mounted connector public key selected; credential keys and connector preserved',flush=True)
check(root,plan,baseline=False)
runtime_check(root,plan)
image=run(['docker','image','inspect',plan['candidate_image'],'--format','{{.Id}}'])
if image!=built['image']:
raise DeployError('candidate image identity changed')
print('REPAIR: recreate router only',flush=True)
run(compose(plan,root/'compose.broker.json')+['up','-d','--no-deps','--no-build','--pull','never',
'--wait','--wait-timeout','180','router'])
print('REPAIRED: router only; verify state and real client handshake before acceptance')
def activate(root,plan,ready):
if not ready:
raise DeployError("v2 client not confirmed ready; only prepare/check/build are allowed now")
check(root,plan)
print('ACTIVATION step: verify candidate image identity',flush=True)
built = json.loads((root/"candidate.json").read_text(encoding="utf-8"))
image = run(["docker","image","inspect",plan["candidate_image"],"--format","{{.Id}}"])
if built != {"image":image,"tag":plan["candidate_image"]}:
raise DeployError("candidate image identity changed")
print('ACTIVATION step: isolated candidate configuration/keyring check (no service switch)',flush=True)
runtime_check(root,plan)
print('ACTIVATION step: recreate connector/router; failures may leave a partial switch, inspect state',flush=True)
run(compose(plan,root/"compose.broker.json")+["up","-d","--no-deps","--no-build","--pull","never",
"--wait","--wait-timeout","180","connector","router"])
print("ACTIVATED: connector/router only. Verify real OAuth, v2 client, unbind and task routing.")
def container_permission_profiles(container):
values=run(['docker','inspect',container['id'],'--format','{{json .Config.Env}}'],True)
env=dict(value.split('=',1) for value in values if '=' in value)
try:
value=json.loads(env['HN_CREDENTIAL_PERMISSION_PROFILES'])
if not isinstance(value,dict):raise ValueError()
return value
except (KeyError,ValueError,TypeError):
raise DeployError('cannot verify active connector product scopes') from None
def update_permissions(root,plan,profiles_path,apply=False):
"""Change approved scopes on the active broker, without changing images/keys."""
profiles=profiles_from(Path(profiles_path))
check(root,plan,baseline=False)
built=json.loads((root/'candidate.json').read_text(encoding='utf-8'))
current=service(plan['project'],'connector')
if current['image']!=built['image'] or current.get('state',{}).get('health')!='healthy':
raise DeployError('permission update requires the active healthy candidate connector')
overlay_path=root/'compose.broker.json'
overlay=json.loads(overlay_path.read_text(encoding='utf-8'))
old=json.loads(overlay['services']['connector']['environment']['HN_CREDENTIAL_PERMISSION_PROFILES'])
if container_permission_profiles(current)!=old:
raise DeployError('active connector scopes differ from selected plan; no changes made')
if set(old)!=set(profiles) or any(not set(scopes)<=set(profiles[name]) for name,scopes in old.items()):
raise DeployError('permission update is additive only; preserve all existing profiles and scopes')
additions={name:sorted(set(scopes)-set(old[name])) for name,scopes in profiles.items()}
print('Product scope additions: '+json.dumps(additions,sort_keys=True))
if not any(additions.values()):
print('PERMISSIONS UNCHANGED: no files or containers changed')
return
if not apply:
print('PERMISSIONS REVIEW: no files or containers changed; use --apply after platform approval/publication')
return
backup=root/('permission-backup-'+secrets.token_hex(8))
backup.mkdir(mode=0o700)
for name in ('compose.broker.json','plan.json'):
shutil.copyfile(str(root/name),str(backup/name));os.chmod(str(backup/name),0o600)
next_overlay=json.loads(json.dumps(overlay))
next_overlay['services']['connector']['environment']['HN_CREDENTIAL_PERMISSION_PROFILES']=json.dumps(profiles,separators=(',',':'))
next_plan=json.loads(json.dumps(plan))
touched=False
try:
replace_json(overlay_path,next_overlay);touched=True
next_plan['prepared_hashes']['compose.broker.json']=digest(overlay_path)
replace_json(root/'plan.json',next_plan)
check(root,next_plan,baseline=False)
# Validate what Compose will actually pass, with secrets kept in-process.
resolved=run(compose(next_plan,overlay_path)+['config','--format','json'],True)
value=resolved['services']['connector']['environment'].get('HN_CREDENTIAL_PERMISSION_PROFILES','')
if json.loads(value)!=profiles:raise DeployError('resolved product scopes do not match approved input')
print('PERMISSIONS APPLY: recreate connector only; same image, credentials and storage',flush=True)
command=compose(next_plan,overlay_path)+['up','-d','--no-deps','--no-build','--pull','never',
'--wait','--wait-timeout','180','connector']
run(command)
after=service(plan['project'],'connector')
if after['image']!=current['image'] or after.get('state',{}).get('health')!='healthy':
raise DeployError('connector did not retain its reviewed healthy image')
if container_permission_profiles(after)!=profiles:
raise DeployError('running connector did not load the approved product scopes')
except Exception:
if touched:
for name in ('compose.broker.json','plan.json'):
replace_json(root/name,json.loads((backup/name).read_text(encoding='utf-8')))
try:
run(compose(plan,overlay_path)+['up','-d','--no-deps','--no-build','--pull','never',
'--wait','--wait-timeout','180','connector'])
except Exception:
raise DeployError('permission update failed; files restored but connector rollback unconfirmed; run state') from None
raise DeployError('permission update failed; original configuration restored; no credential keys changed') from None
print('PERMISSIONS UPDATED: connector only; backup='+str(backup)+'; obtain a NEW authorization transaction next')
def main():
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest="action",required=True)
flight = sub.add_parser("preflight")
flight.add_argument("--deployment-root",required=True)
flight.add_argument("--connector-container",default="hainuo-relay-connector-1")
prep = sub.add_parser("prepare")
prep.add_argument("--deployment-root",required=True)
prep.add_argument("--source-root",default=str(Path(__file__).resolve().parents[1]))
prep.add_argument("--profiles",required=True)
prep.add_argument("--output",required=True)
prep.add_argument("--connector-container",default="hainuo-relay-connector-1")
update = sub.add_parser("prepare-update")
update.add_argument("--plan",required=True)
update.add_argument("--source-root",default=str(Path(__file__).resolve().parents[1]))
update.add_argument("--output",required=True)
permissions=sub.add_parser('update-permissions')
permissions.add_argument('--plan',required=True)
permissions.add_argument('--profiles',required=True)
permissions.add_argument('--apply',action='store_true')
for name in ("repair-router","state","diagnose","check","build","activate","rollback"):
cmd = sub.add_parser(name)
cmd.add_argument("--plan",required=True)
if name == "activate":
cmd.add_argument("--client-v2-ready",action="store_true")
if name == "rollback":
cmd.add_argument("--stop-broker-use",action="store_true")
args = parser.parse_args()
try:
if args.action == "preflight":
preflight(args)
return
if args.action == "prepare":
prepare(args)
return
if args.action == "prepare-update":
prepare_update(args)
return
root = Path(args.plan).resolve()
plan = load_plan(root)
if args.action == 'update-permissions':
update_permissions(root,plan,args.profiles,args.apply)
elif args.action == "repair-router":
repair_router(root,plan)
elif args.action == "state":
deployment_state(root,plan)
elif args.action == "diagnose":
diagnose(root,plan)
elif args.action == "check":
check(root,plan)
elif args.action == "build":
build(root,plan)
elif args.action == "activate":
activate(root,plan,args.client_v2_ready)
else:
if not args.stop_broker_use:
raise DeployError("rollback requires stopping broker clients; old refresh credentials are not restored")
check(root,plan,baseline=False,router_key_required=False)
run(compose(plan,root/"compose.rollback.json")+["up","-d","--no-deps","--no-build","--pull","never",
"--wait","--wait-timeout","180","connector","router"])
print("ROLLED BACK containers only; preserve keyring/database. Client mode and reauthorization must be coordinated.")
except (DeployError,OSError,ValueError,KeyError,TypeError):
error = sys.exc_info()[1]
print("STOP: "+(str(error) if isinstance(error,DeployError) else "invalid/missing migration input; secret details withheld"),file=sys.stderr)
raise SystemExit(1)
if __name__ == "__main__":
main()