fix(deploy): repair missing router public-key path after partial activation

Live evidence showed a healthy candidate connector and a restarting router:
legacy router environment omitted HN_CONNECTOR_PUBLIC_KEY, so the new routing
verifier tried runtime/pki/connector.pub instead of the existing /trust mount.

Resolve the public key from the existing read-only trust bind, set its path
in new overlays, and actually load it during isolated runtime checks. Add
repair-router for a known healthy candidate connector: preserve credential
keys and live configuration, back up overlay/plan, add only the router path,
update integrity metadata, probe, and recreate router alone with --no-deps.
Retries do not regenerate keys or overwrite repair backups.

Add read-only state including stopped/restarting containers and fixed startup
error classes, including stderr, without printing raw logs or credentials.
Identify command stage/exit code and activation progress. Permit rollback
of known stopped candidate targets and old overlays without requiring the
new router key; unknown images still fail closed.

Validation: full suite 262 passed, 3 skipped (real PostgreSQL unavailable),
30 existing warnings; final tooling suite 50 passed after the added CLI
rollback regression. Python 3.7 syntax and staged diff checks pass.
No live repair was executed by this agent; operator must run repair-router
and verify health and the real client handshake on ECS.
This commit is contained in:
ex_tianzy
2026-11-04 16:27:47 +08:00
parent 2657146334
commit 8a94d60155
4 changed files with 332 additions and 17 deletions
+134 -1
View File
@@ -38,6 +38,10 @@ def scenario(tmp_path,monkeypatch):
base = {"services":{name:{"image":"old:tag","environment":{},"networks":{"backend":None},"volumes":[]}
for name in [*roles,"postgres","redis"]},"volumes":{"metadata":{}},"networks":{"backend":{}}}
base["services"]["connector"]["environment"] = env
trust=live/'runtime/trust';trust.mkdir(parents=True)
(trust/'connector.pub').write_text('-----BEGIN PUBLIC KEY-----\nsynthetic-public-material\n-----END PUBLIC KEY-----\n')
base['services']['router']['environment']['HN_CA_CERT']='/trust/ca.pem'
base['services']['router']['volumes']=[{'type':'bind','source':str(trust),'target':'/trust','read_only':True}]
base["services"]["connector"]["volumes"] = [{"type":"bind","source":str(live/"runtime"),"target":"/data"}]
base["services"]["edge"]["volumes"] = [{"type":"bind","source":str(caddy),"target":"/etc/caddy/Caddyfile"}]
profiles = tmp_path/"product.json"
@@ -135,10 +139,13 @@ def test_prepare_preserves_all_seven_files_runtime_identity_and_no_service_chang
plan = prepared(scenario)
assert len(plan["compose_files"]) == 7
assert (scenario["live"]/".env").read_bytes() == original
assert not (scenario["live"]/"runtime").exists()
assert sorted(p.relative_to(scenario['live']/'runtime').as_posix()
for p in (scenario['live']/'runtime').rglob('*') if p.is_file())==['trust/connector.pub']
assert not (scenario['live']/'runtime/issuer').exists()
overlay = json.loads((scenario["output"]/"compose.broker.json").read_text())
assert set(overlay["services"]) == {"connector","router"}
assert overlay["services"]["connector"]["environment"]["HN_CREDENTIAL_MODE"] == "broker"
assert overlay['services']['router']['environment']['HN_CONNECTOR_PUBLIC_KEY']=='/trust/connector.pub'
assert len(json.loads((scenario["output"]/"credential-keyring.json").read_text())["keys"]) == 1
rollback = json.loads((scenario["output"]/"compose.rollback.json").read_text())
assert rollback["services"]["connector"]["image"] != rollback["services"]["router"]["image"]
@@ -317,6 +324,132 @@ def test_activation_validation_failure_never_recreates_services(scenario,monkeyp
assert not any("up" in c for c in scenario["calls"])
def test_state_sees_exited_candidate_without_showing_logs_or_secrets(scenario,monkeypatch,capsys):
plan = prepared(scenario)
deploy.write_json(scenario['output']/'candidate.json',{'image':scenario['roles']['connector']['image'],'tag':plan['candidate_image']})
scenario['roles']['connector']['state']={'status':'exited','health':'unhealthy','exit_code':1,'restart_count':7}
original=deploy.run
def logs(args,*a,**kw):
if args[:2]==['docker','logs']:
scenario['calls'].append(args)
return 'Traceback PermissionError synthetic-private-password https://secret.invalid/private'
return original(args,*a,**kw)
monkeypatch.setattr(deploy,'run',logs)
scenario['calls'].clear()
deploy.deployment_state(scenario['output'],plan)
output=capsys.readouterr().out
assert 'image=candidate status=exited health=unhealthy' in output
assert 'PermissionError' in output and 'synthetic-private-password' not in output and 'secret.invalid' not in output
assert all('-aq' in c for c in scenario['calls'] if c[:2]==['docker','ps'])
assert not any('up' in c or 'tag' in c for c in scenario['calls'])
def test_rollback_checks_allow_stopped_known_candidate_but_reject_other_image(scenario):
plan=prepared(scenario)
scenario['roles']['connector']['image']='sha256:'+'a'*64
deploy.write_json(scenario['output']/'candidate.json',{'image':'sha256:'+'a'*64,'tag':plan['candidate_image']})
scenario['calls'].clear()
deploy.check(scenario['output'],plan,baseline=False)
assert any(c[:2]==['docker','ps'] and '-aq' in c for c in scenario['calls'])
scenario['roles']['router']['image']='sha256:'+'f'*64
with pytest.raises(deploy.DeployError,match='unreviewed target image'):
deploy.check(scenario['output'],plan,baseline=False)
def partial_old_plan(scenario):
plan=prepared(scenario)
overlay_path=scenario['output']/'compose.broker.json'
overlay=json.loads(overlay_path.read_text())
overlay['services']['router']['environment'].pop('HN_CONNECTOR_PUBLIC_KEY')
deploy.replace_json(overlay_path,overlay)
plan['prepared_hashes']['compose.broker.json']=deploy.digest(overlay_path)
deploy.replace_json(scenario['output']/'plan.json',plan)
image='sha256:'+'a'*64
deploy.write_json(scenario['output']/'candidate.json',{'image':image,'tag':plan['candidate_image']})
scenario['roles']['connector']['image']=image
scenario['roles']['connector']['state']={'status':'running','health':'healthy'}
scenario['roles']['router']['image']=image
scenario['roles']['router']['state']={'status':'restarting','health':'unhealthy'}
return plan
def test_repair_old_partial_switch_only_restarts_router_preserving_keys(scenario):
plan=partial_old_plan(scenario)
key=(scenario['output']/'credential-keyring.json').read_bytes()
original_env=(scenario['live']/'.env').read_bytes()
scenario['calls'].clear()
deploy.repair_router(scenario['output'],plan)
assert (scenario['output']/'credential-keyring.json').read_bytes()==key
assert (scenario['live']/'.env').read_bytes()==original_env
fixed=json.loads((scenario['output']/'compose.broker.json').read_text())
assert fixed['services']['router']['environment']['HN_CONNECTOR_PUBLIC_KEY']=='/trust/connector.pub'
saved=deploy.load_plan(scenario['output'])
assert saved['prepared_hashes']['compose.broker.json']==deploy.digest(scenario['output']/'compose.broker.json')
commands=[c for c in scenario['calls'] if 'up' in c]
assert len(commands)==1 and commands[0][-1]=='router' and commands[0][-2]=='180'
assert '--no-deps' in commands[0] and commands[0][-2:]!=['connector','router']
assert (scenario['output']/'plan.before-router-key.json').is_file()
assert (scenario['output']/'compose.broker.before-router-key.json').is_file()
def test_repair_retry_does_not_regenerate_key_or_overwrite_backup(scenario):
plan=partial_old_plan(scenario)
deploy.repair_router(scenario['output'],plan)
files={p.name:p.read_bytes() for p in scenario['output'].iterdir() if p.is_file()}
deploy.repair_router(scenario['output'],deploy.load_plan(scenario['output']))
assert files=={p.name:p.read_bytes() for p in scenario['output'].iterdir() if p.is_file()}
def test_repair_requires_healthy_known_connector(scenario):
plan=partial_old_plan(scenario)
scenario['roles']['connector']['state']['health']='unhealthy'
before=(scenario['output']/'compose.broker.json').read_bytes()
with pytest.raises(deploy.DeployError,match='healthy'):
deploy.repair_router(scenario['output'],plan)
assert (scenario['output']/'compose.broker.json').read_bytes()==before
assert not any('up' in c for c in scenario['calls'])
def test_old_plan_fails_normal_check_until_explicit_repair(scenario):
plan=partial_old_plan(scenario)
with pytest.raises(deploy.DeployError,match='HN_CONNECTOR_PUBLIC_KEY'):
deploy.check(scenario['output'],plan,baseline=False)
def test_cli_rollback_of_old_partial_plan_does_not_require_new_router_key(scenario,monkeypatch):
partial_old_plan(scenario)
monkeypatch.setattr(deploy.sys,'argv',['broker_deploy.py','rollback','--plan',str(scenario['output']),'--stop-broker-use'])
scenario['calls'].clear()
deploy.main()
commands=[c for c in scenario['calls'] if 'up' in c]
assert len(commands)==1 and commands[0][-2:]==['connector','router']
assert str(scenario['output']/'compose.rollback.json') in commands[0]
def test_missing_trust_public_key_refused_before_any_new_credentials(scenario):
trust=Path(scenario['base']['services']['router']['volumes'][0]['source'])
(trust/'connector.pub').unlink()
with pytest.raises(deploy.DeployError,match='public key'):
deploy.prepare(scenario['args'])
assert not scenario['output'].exists()
def test_run_failure_identifies_stage_without_echoing_command_secrets(monkeypatch):
import types
def fail(*a,**kw):
return types.SimpleNamespace(returncode=17,stdout='synthetic-access',stderr='synthetic-secret')
monkeypatch.setattr(deploy.subprocess,'run',fail)
with pytest.raises(deploy.DeployError) as error:
deploy.run(['docker','compose','--env-file','sensitive-path','up','connector'])
assert 'compose-up' in str(error.value) and '17' in str(error.value)
assert 'synthetic' not in str(error.value) and 'sensitive-path' not in str(error.value)
def test_log_signals_only_emit_fixed_error_classes():
assert deploy.log_signals('PermissionError: synthetic-secret\nread-only file system')==['PermissionError','read-only-filesystem']
assert deploy.log_signals('fake-secret https://secret.invalid')==[]
def test_catchall_before_private_block_requires_manual_review(scenario):
plan = prepared(scenario)
original = scenario["caddy"].read_text()