Files
ecs-relay-service/deployable/scripts/broker_deploy.py
T
ex_tianzy da1b4230b9 feat(deploy): add guarded existing-ECS broker migration tooling
Discover the current Compose project and all override files from container
labels. Add read-only preflight plus prepare/check/build/activate/rollback
commands without editing the running .env, CA, database volumes or unrelated
services. Preserve heterogeneous application image identities for rollback.

Prepare a private keyring, independent router credential, pinned source
snapshot and additive connector/router overlay. Refuse key overwrite and
configuration/image drift. Build offline from the exact existing ARM64
runtime, smoke import and validate role settings through stdin-only inputs.
Require explicit v2 client readiness before activation and recreate only
connector/router with --no-deps. Document actual backup/image evidence,
current client limitation and token-lifecycle rollback boundaries.

Validation: 239 passed, 3 skipped (real PostgreSQL absent), 30 existing
dependency deprecation warnings. Add 26 mocked Docker policy tests; host
script parses as Python 3.7, CLI help/docs/diff checks pass. Docker policy
mocks are not actual Compose or ARM64 build validation.

No remote files, keys, credentials, containers or live configuration were
changed. Stage 5 operational acceptance remains pending actual ECS build,
real databases/platform, v2 client, key recovery and controlled rollout.
2026-11-04 12:18:43 +08:00

399 lines
22 KiB
Python

#!/usr/bin/env python3
"""Existing-deployment broker migration: stdlib-only host tool, Python >=3.7.
Never initialize identities, edit the live .env, or dump resolved Compose output.
prepare/check/build do not recreate services. Activation requires a v2 client.
"""
import argparse
import base64
import hashlib
import json
import os
from pathlib import Path
import re
import secrets
import shutil
import subprocess
import sys
class DeployError(RuntimeError):
pass
def run(args, json_result=False, optional=False, input_text=None):
result = subprocess.run(args, input=input_text, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
if result.returncode:
if optional:
return None
raise DeployError("Docker/Git command failed; raw output withheld to protect configuration")
try:
return json.loads(result.stdout) if json_result else result.stdout.strip()
except ValueError:
raise DeployError("invalid command response") from None
def digest(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def write_json(path, value):
# All outputs belong to the newly created, private migration directory.
fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, "w", encoding="utf-8") as stream:
json.dump(value, stream, ensure_ascii=False, indent=2)
stream.write("\n")
def inspect(container):
template = ('{"id":{{json .Id}},"image":{{json .Image}},'
'"image_ref":{{json .Config.Image}},"labels":{{json .Config.Labels}}}')
return run(["docker", "inspect", container, "--format", template], True)
def service(project, name):
values = run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project="+project,
"--filter", "label=com.docker.compose.service="+name]).splitlines()
if len(values) != 1:
raise DeployError("expected exactly one running relay service: "+name)
return inspect(values[0])
def compose(plan, override=None):
args = ["docker", "compose", "--project-directory", plan["deployment_root"],
"--env-file", str(Path(plan["deployment_root"])/".env"), "-p", plan["project"]]
for file in plan["compose_files"]:
args.extend(["-f", file])
if override:
args.extend(["-f", str(override)])
return args
def profiles_from(path):
try:
data = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(data, dict) or not data or len(data) > 32:
raise ValueError()
for name, scopes in data.items():
if (not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", name) or not isinstance(scopes, list)
or not scopes or len(scopes) > 128 or any(not isinstance(s, str)
or not re.fullmatch(r"[A-Za-z0-9_:.-]{1,256}", s) for s in scopes)):
raise ValueError()
return {name: sorted(set(scopes)) for name, scopes in data.items()}
except (OSError, ValueError, TypeError):
raise DeployError("invalid product permission profiles; use the product-approved JSON object") from None
def validate_overlay(base, merged, candidate):
old, new = base["services"], merged["services"]
if set(old) != set(new) or set(old) != {"connector","router","longconn","enrollment-signer","postgres","redis","edge"}:
raise DeployError("unexpected Compose services; migration requires review")
for name in set(old)-{"connector","router"}:
if old[name] != new[name]:
raise DeployError("overlay changes an unrelated relay service: "+name)
for name in ("connector","router"):
for field in set(old[name]) | set(new[name]):
if field not in {"environment","image","volumes"} and old[name].get(field) != new[name].get(field):
raise DeployError("overlay changes non-migration service settings")
if new[name].get("image") != candidate:
raise DeployError("candidate image mismatch")
env = new[name]["environment"]
for key, value in old[name].get("environment", {}).items():
if key not in {"HN_CREDENTIAL_MODE","HN_ROUTER_ADMISSION_SECRET","HN_CREDENTIAL_KEYRING_FILE",
"HN_CREDENTIAL_PERMISSION_PROFILES","HN_CONNECTOR_INTERNAL_URL","HN_FEISHU_APP_ID","HN_OAUTH_ENABLED"}:
if env.get(key) != value:
raise DeployError("existing environment changed: "+key)
old_volumes = {v["target"]:v for v in old[name].get("volumes", [])}
new_volumes = {v["target"]:v for v in new[name].get("volumes", [])}
if any(new_volumes.get(key) != value for key,value in old_volumes.items()):
raise DeployError("existing storage or trust mount changed")
expected_targets = set(old_volumes) | ({"/run/hainuo-credentials/keyring.json"} if name == "connector" else set())
if set(new_volumes) != expected_targets or env.get("HN_CREDENTIAL_MODE") != "broker":
raise DeployError("unexpected migration mount or credential mode")
if merged.get("volumes") != base.get("volumes") or merged.get("networks") != base.get("networks"):
raise DeployError("database volume or network definitions changed")
env = new["connector"]["environment"]
if (env.get("HN_FEISHU_APP_ID") != old["connector"]["environment"].get("HN_FEISHU_APP_ID")
or new["router"]["environment"].get("HN_FEISHU_APP_ID") != env.get("HN_FEISHU_APP_ID")
or new["router"]["environment"].get("HN_ENVIRONMENT") != env.get("HN_ENVIRONMENT")
or env.get("HN_CREDENTIAL_KEYRING_FILE") != "/run/hainuo-credentials/keyring.json"
or new["router"]["environment"].get("HN_CONNECTOR_INTERNAL_URL") != "http://connector:8000"):
raise DeployError("application/environment or credential routing mismatch")
if (not env.get("HN_DATABASE_URL", "").startswith(("postgresql://","postgres://"))
or env.get("HN_ENVIRONMENT") != "production"
or str(env.get("HN_FEISHU_ENABLED", "")).lower() != "true"
or str(env.get("HN_OAUTH_ENABLED", "")).lower() != "true"):
raise DeployError("broker production platform/database configuration incomplete")
for key in ("HN_ADMIN_SECRET","HN_INTERNAL_SECRET","HN_FEISHU_APP_SECRET"):
if env["HN_ROUTER_ADMISSION_SECRET"] == env.get(key):
raise DeployError("router admission credential must be independent")
if any(new["router"]["environment"].get(k) for k in ("HN_ADMIN_SECRET","HN_INTERNAL_SECRET","HN_FEISHU_APP_SECRET")):
raise DeployError("router has unrelated privileged credentials")
mounts = [v for v in new["connector"].get("volumes", []) if v["target"] == "/run/hainuo-credentials/keyring.json"]
if len(mounts) != 1 or not mounts[0].get("read_only"):
raise DeployError("missing read-only credential keyring mount")
def preflight(args):
deployment = Path(args.deployment_root).resolve()
current = inspect(args.connector_container)
labels = current["labels"]
if Path(labels["com.docker.compose.project.working_dir"]).resolve() != deployment:
raise DeployError("deployment path differs from the running connector")
files = [Path(p) for p in labels["com.docker.compose.project.config_files"].split(",")]
if any(not p.is_file() or p.resolve().parent != deployment for p in files):
raise DeployError("missing or external Compose file; review deployment layout")
plan = {"deployment_root":str(deployment),"project":labels["com.docker.compose.project"],
"compose_files":[str(p) for p in files]}
base = run(compose(plan)+["config","--format","json"],True)
names = set(base["services"])
if names != {"connector","router","longconn","enrollment-signer","postgres","redis","edge"}:
raise DeployError("unexpected Compose services")
architecture = run(["docker","image","inspect",current["image"],"--format","{{.Architecture}}"])
if architecture != "arm64":
raise DeployError("expected current ARM64 runtime")
for name in ("connector","router","longconn","enrollment-signer"):
entry = service(plan["project"],name)
print(name+": "+entry["image"])
env = base["services"]["connector"].get("environment",{})
print("Project: "+plan["project"])
print("Compose files: "+str(len(files)))
print("Current credential mode: "+env.get("HN_CREDENTIAL_MODE","legacy"))
print("Preflight complete; no files, keys or services changed. Resolved settings withheld.")
def prepare(args):
if os.name != "nt" and os.geteuid() != 0:
raise DeployError("run preparation with sudo for protected key ownership")
deployment = Path(args.deployment_root).resolve()
output = Path(args.output)
source = Path(args.source_root).resolve()
profiles = profiles_from(Path(args.profiles))
if output.exists() or output.is_symlink() or not output.is_absolute():
raise DeployError("output must be a new absolute migration directory")
output = output.resolve()
if any(c in str(output) for c in ("\n","\r","$")):
raise DeployError("migration path contains unsupported Compose interpolation characters")
if output == deployment or deployment in output.parents:
raise DeployError("migration output must be outside the running deployment")
current = inspect(args.connector_container)
labels = current["labels"]
if Path(labels["com.docker.compose.project.working_dir"]).resolve() != deployment:
raise DeployError("running deployment directory does not match the supplied root")
files = [Path(p) for p in labels["com.docker.compose.project.config_files"].split(",")]
if any(not p.is_absolute() or p.resolve().parent != deployment or not p.is_file() for p in files):
raise DeployError("Compose files must belong to the existing deployment root")
if not (deployment/".env").is_file() or not (source/"relay/credential_api.py").is_file():
raise DeployError("missing live configuration or staged broker source")
project = labels["com.docker.compose.project"]
if not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", project):
raise DeployError("invalid Compose project")
roles = {name:service(project,name) for name in ("connector","router","longconn","enrollment-signer","edge")}
if any(not re.fullmatch(r"sha256:[a-f0-9]{64}",r["image"]) for r in roles.values()):
raise DeployError("invalid running image identity")
if roles["connector"]["id"] != current["id"]:
raise DeployError("connector inventory changed during preparation")
architecture = run(["docker","image","inspect",current["image"],"--format","{{.Architecture}}"])
if architecture != "arm64":
raise DeployError("this migration targets the verified ARM64 runtime only")
plan_id = secrets.token_hex(8)
candidate = "hainuo-relay:broker-candidate-"+plan_id
plan = {"format":1,"plan_id":plan_id,"deployment_root":str(deployment),"project":project,
"compose_files":[str(p) for p in files],"roles":roles,"candidate_image":candidate,
"base_image":current["image"],"architecture":architecture,
"baseline_hashes":{str(p):digest(p) for p in [deployment/".env",*files]}}
# Durable tags preserve each heterogeneous running application image.
for name in ("connector","router","longconn","enrollment-signer"):
run(["docker","tag",roles[name]["image"],"hainuo-relay:broker-rollback-"+plan_id+"-"+name])
base = run(compose(plan)+["config","--format","json"],True)
if base["services"]["connector"].get("environment",{}).get("HN_CREDENTIAL_MODE","legacy") != "legacy":
raise DeployError("already broker: preserve the existing keyring and migration plan")
output.mkdir(mode=0o700,parents=False)
os.chmod(str(output),0o700)
keyring = {"active_key_id":"initial-v1","keys":{"initial-v1":base64.b64encode(os.urandom(32)).decode("ascii")}}
write_json(output/"credential-keyring.json",keyring)
if os.name != "nt":
os.chown(str(output/"credential-keyring.json"),10001,10001)
admission = secrets.token_hex(32)
overlay = {"services":{
"connector":{"image":candidate,"environment":{"HN_CREDENTIAL_MODE":"broker",
"HN_CREDENTIAL_KEYRING_FILE":"/run/hainuo-credentials/keyring.json",
"HN_CREDENTIAL_PERMISSION_PROFILES":json.dumps(profiles,separators=(",", ":")),
"HN_ROUTER_ADMISSION_SECRET":admission,"HN_OAUTH_ENABLED":"true"},
"volumes":[{"type":"bind","source":str(output/"credential-keyring.json"),
"target":"/run/hainuo-credentials/keyring.json","read_only":True}]},
"router":{"image":candidate,"environment":{"HN_CREDENTIAL_MODE":"broker",
"HN_ROUTER_ADMISSION_SECRET":admission,"HN_CONNECTOR_INTERNAL_URL":"http://connector:8000",
"HN_FEISHU_APP_ID":"${HN_FEISHU_APP_ID:?Configure existing application}",
"HN_ENVIRONMENT":"${HN_ENVIRONMENT:?Configure existing environment}"}}}}
write_json(output/"compose.broker.json",overlay)
write_json(output/"compose.rollback.json",{"services":{name:{"image":roles[name]["image"]}
for name in ("connector","router","longconn","enrollment-signer")}})
payload = output/"build"
payload.mkdir(mode=0o700)
for path in (source/"relay").rglob("*"):
if path.is_symlink():
raise DeployError("symlink in staged source")
if path.is_file() and path.suffix in {".py",".js",".css",".html"} and "__pycache__" not in path.parts:
target = payload/path.relative_to(source)
target.parent.mkdir(parents=True,exist_ok=True)
shutil.copyfile(str(path),str(target))
os.chmod(str(target),0o644)
dockerfile = ('ARG BASE_IMAGE\nFROM ${BASE_IMAGE}\nWORKDIR /app\nUSER 0:0\n'
'COPY --chown=10001:10001 relay /app/relay\nUSER 10001:10001\n'
'RUN python -c "import relay.credential_api, relay.binding_state, relay.router"\n'
'ENTRYPOINT ["python", "-m", "relay.cli"]\n')
(payload/"Dockerfile").write_bytes(dockerfile.encode("utf-8"))
plan["payload_hashes"] = {str(p.relative_to(output)):digest(p) for p in payload.rglob("*") if p.is_file()}
plan["prepared_hashes"] = {p:digest(output/p) for p in (
"credential-keyring.json","compose.broker.json","compose.rollback.json")}
write_json(output/"plan.json",plan)
merged = run(compose(plan,output/"compose.broker.json")+["config","--format","json"],True)
validate_overlay(base,merged,candidate)
print("PREPARED: "+str(output)+" (services unchanged; private files must not be shared)")
def load_plan(root):
plan = json.loads((root/"plan.json").read_text(encoding="utf-8"))
if plan.get("format") != 1 or not re.fullmatch(r"[a-f0-9]{16}",plan.get("plan_id","")):
raise DeployError("unsupported migration plan")
return plan
def check(root, plan, baseline=True):
if os.name != "nt":
if root.stat().st_mode & 0o077:
raise DeployError("migration directory must not be accessible to other users")
for name in ("credential-keyring.json","compose.broker.json","compose.rollback.json","plan.json"):
if (root/name).stat().st_mode & 0o077:
raise DeployError("migration private file permissions are too broad")
if (root/"credential-keyring.json").stat().st_uid != 10001:
raise DeployError("credential keyring owner must match service UID 10001")
for path, expected in plan["baseline_hashes"].items():
if digest(Path(path)) != expected:
raise DeployError("live Compose/configuration changed; review before proceeding")
for path, expected in plan["payload_hashes"].items():
if digest(root/path) != expected:
raise DeployError("staged source changed after preparation")
for path, expected in plan["prepared_hashes"].items():
if digest(root/path) != expected:
raise DeployError("prepared keyring or overlay changed; do not regenerate credential keys")
for name, old in plan["roles"].items():
current = service(plan["project"],name)
if (baseline or name not in {"connector","router"}) and current["image"] != old["image"]:
raise DeployError("running service image changed since preparation: "+name)
base = run(compose(plan)+["config","--format","json"],True)
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
validate_overlay(base,merged,plan["candidate_image"])
# Require the current edge's explicit private-path block, before catch-all handling.
mounts = base["services"]["edge"].get("volumes",[])
caddy = next((v["source"] for v in mounts if v.get("target") == "/etc/caddy/Caddyfile" and v.get("type") == "bind"),None)
if not caddy:
raise DeployError("cannot verify current edge configuration")
text = Path(caddy).read_text(encoding="utf-8")
rule = re.search(r"@([A-Za-z0-9_-]+)\s+path\s+/admin/\*\s+/internal/\*\s+/ready",text)
deny = re.search(r"respond\s+@"+re.escape(rule.group(1))+r"\s+404",text) if rule else None
first_handle = re.search(r"\bhandle(?:_path)?\b",text)
if not deny or (first_handle and deny.start() > first_handle.start()):
raise DeployError("edge private-path block is not the reviewed configuration")
print("CHECKED: existing Compose/storage/images retained; no services restarted")
def build(root,plan):
check(root,plan)
if run(["docker","image","inspect",plan["candidate_image"]],optional=True) is not None:
raise DeployError("candidate image already exists; do not overwrite a built candidate")
base_tag = "hainuo-relay:broker-base-"+plan["plan_id"]
run(["docker","tag",plan["base_image"],base_tag])
run(["docker","build","--pull=false","--network","none","--build-arg","BASE_IMAGE="+base_tag,
"-t",plan["candidate_image"],str(root/"build")])
image = run(["docker","image","inspect",plan["candidate_image"],"--format","{{.Id}}"])
run(["docker","run","--rm","--network","none","--read-only","--entrypoint","python",
plan["candidate_image"],"-c","import relay.credential_api, relay.binding_state, relay.device_lifecycle"])
runtime_check(root,plan)
write_json(root/"candidate.json",{"image":image,"tag":plan["candidate_image"]})
print("BUILT: "+plan["candidate_image"]+" (offline runtime reuse; not activated)")
def runtime_check(root,plan):
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
# Actual secrets travel via stdin only: no command-line values, env dump,
# database connection or platform request. The container has no network.
code = ("import json,os,sys\nfrom pathlib import Path\nfrom relay.config import Settings\n"
"from relay.credential_store import CredentialKeyring\n"
"try:\n data=json.load(sys.stdin)\n os.environ.update(data['environment'])\n"
" s=Settings(role=data['role'])\n"
" if data['role']=='connector': CredentialKeyring.from_file(s.credential_keyring_file)\n"
"except Exception:\n sys.exit(1)\n")
for role in ("connector","router"):
env = {key:str(value) for key,value in merged["services"][role]["environment"].items() if value is not None}
run(["docker","run","--rm","-i","--network","none","--read-only","--entrypoint","python",
"-v",str(root/"credential-keyring.json")+":/run/hainuo-credentials/keyring.json:ro",
plan["candidate_image"],"-c",code],input_text=json.dumps({"role":role,"environment":env}))
def activate(root,plan,ready):
if not ready:
raise DeployError("v2 client not confirmed ready; only prepare/check/build are allowed now")
check(root,plan)
built = json.loads((root/"candidate.json").read_text(encoding="utf-8"))
image = run(["docker","image","inspect",plan["candidate_image"],"--format","{{.Id}}"])
if built != {"image":image,"tag":plan["candidate_image"]}:
raise DeployError("candidate image identity changed")
runtime_check(root,plan)
run(compose(plan,root/"compose.broker.json")+["up","-d","--no-deps","--no-build","--pull","never",
"--wait","--wait-timeout","180","connector","router"])
print("ACTIVATED: connector/router only. Verify real OAuth, v2 client, unbind and task routing.")
def main():
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest="action",required=True)
flight = sub.add_parser("preflight")
flight.add_argument("--deployment-root",required=True)
flight.add_argument("--connector-container",default="hainuo-relay-connector-1")
prep = sub.add_parser("prepare")
prep.add_argument("--deployment-root",required=True)
prep.add_argument("--source-root",default=str(Path(__file__).resolve().parents[1]))
prep.add_argument("--profiles",required=True)
prep.add_argument("--output",required=True)
prep.add_argument("--connector-container",default="hainuo-relay-connector-1")
for name in ("check","build","activate","rollback"):
cmd = sub.add_parser(name)
cmd.add_argument("--plan",required=True)
if name == "activate":
cmd.add_argument("--client-v2-ready",action="store_true")
if name == "rollback":
cmd.add_argument("--stop-broker-use",action="store_true")
args = parser.parse_args()
try:
if args.action == "preflight":
preflight(args)
return
if args.action == "prepare":
prepare(args)
return
root = Path(args.plan).resolve()
plan = load_plan(root)
if args.action == "check":
check(root,plan)
elif args.action == "build":
build(root,plan)
elif args.action == "activate":
activate(root,plan,args.client_v2_ready)
else:
if not args.stop_broker_use:
raise DeployError("rollback requires stopping broker clients; old refresh credentials are not restored")
check(root,plan,baseline=False)
run(compose(plan,root/"compose.rollback.json")+["up","-d","--no-deps","--no-build","--pull","never",
"--wait","--wait-timeout","180","connector","router"])
print("ROLLED BACK containers only; preserve keyring/database. Client mode and reauthorization must be coordinated.")
except (DeployError,OSError,ValueError,KeyError,TypeError):
error = sys.exc_info()[1]
print("STOP: "+(str(error) if isinstance(error,DeployError) else "invalid/missing migration input; secret details withheld"),file=sys.stderr)
raise SystemExit(1)
if __name__ == "__main__":
main()