feat(work): stage authenticated remote approvals with scoped execution fencing

This commit is contained in:
ex_tianzy
2026-11-17 11:43:33 +08:00
parent d2881b82b0
commit c80c6a0d27
19 changed files with 1586 additions and 25 deletions
+122 -18
View File
@@ -180,14 +180,33 @@ def router_public_key(config):
raise DeployError('router connector public key is not a verified read-only trust mount')
def validate_overlay(base, merged, candidate, router_key_required=True):
def targets(plan):
return ('connector','router','longconn')if plan.get('remote_approvals_managed')else('connector','router')
def target_tag(plan,name):
return plan['longconn_candidate_image']if name=='longconn'else plan['candidate_image']
def receipt_file(root,name):
return root/('candidate-longconn.json'if name=='longconn'else'candidate.json')
def validate_overlay(base, merged, candidate, router_key_required=True, approval_plan=None):
old, new = base["services"], merged["services"]
require_services(old)
require_services(new)
for name in set(old)-{"connector","router"}:
managed=targets(approval_plan or {})
for name in set(old)-set(managed):
if old[name] != new[name]:
raise DeployError("overlay changes an unrelated relay service: "+name)
for name in ("connector","router"):
for name in managed:
if name=='longconn':
expected_env=dict(old[name].get('environment',{}))
expected_env.update((approval_plan or {})['longconn_approval_environment'])
expected=dict(old[name]);expected.update(image=approval_plan['longconn_candidate_image'],environment=expected_env)
if new[name]!=expected:raise DeployError('approval overlay changes unrelated longconn settings')
continue
for field in set(old[name]) | set(new[name]):
if field not in {"environment","image","volumes"} and old[name].get(field) != new[name].get(field):
raise DeployError("overlay changes non-migration service settings")
@@ -196,7 +215,7 @@ def validate_overlay(base, merged, candidate, router_key_required=True):
env = new[name]["environment"]
for key, value in old[name].get("environment", {}).items():
if key not in {"HN_CREDENTIAL_MODE","HN_ROUTER_ADMISSION_SECRET","HN_CREDENTIAL_KEYRING_FILE",
"HN_CREDENTIAL_PERMISSION_PROFILES","HN_CONNECTOR_INTERNAL_URL","HN_FEISHU_APP_ID","HN_OAUTH_ENABLED","HN_WORK_FILES_ENABLED","HN_WORK_TASK_CARDS_ENABLED"}:
"HN_CREDENTIAL_PERMISSION_PROFILES","HN_CONNECTOR_INTERNAL_URL","HN_FEISHU_APP_ID","HN_OAUTH_ENABLED","HN_WORK_FILES_ENABLED","HN_WORK_TASK_CARDS_ENABLED","HN_WORK_REMOTE_APPROVALS_ENABLED","HN_FEISHU_TENANT_KEY","HN_FEISHU_VERIFICATION_TOKEN"}:
if env.get(key) != value:
raise DeployError("existing environment changed: "+key)
old_volumes = {v["target"]:v for v in old[name].get("volumes", [])}
@@ -219,6 +238,14 @@ def validate_overlay(base, merged, candidate, router_key_required=True):
# namespace rather than silently converting it to production. All modes still
# require PostgreSQL and HTTPS/WSS for this existing-server migration.
require_platform(env)
approval_keys={'HN_WORK_REMOTE_APPROVALS_ENABLED','HN_FEISHU_TENANT_KEY','HN_FEISHU_VERIFICATION_TOKEN'}
for name in ('connector','router'):
before=old[name].get('environment',{});after=new[name]['environment']
if name=='connector'and approval_plan and approval_plan.get('remote_approvals_managed'):
wanted=approval_plan['connector_approval_environment']
if any(after.get(k)!=v for k,v in wanted.items()):raise DeployError('approval identity mismatch')
elif any(after.get(k)!=before.get(k)for k in approval_keys):
raise DeployError('unreviewed approval environment change')
for key in ("HN_ADMIN_SECRET","HN_INTERNAL_SECRET","HN_FEISHU_APP_SECRET"):
if env["HN_ROUTER_ADMISSION_SECRET"] == env.get(key):
raise DeployError("router admission credential must be independent")
@@ -377,6 +404,31 @@ def prepare_update(args):
raise DeployError('work files implementation missing from update source')
if getattr(args,'enable_task_cards',False)and not (source/'relay/task_cards.py').is_file():
raise DeployError('task cards implementation missing from update source')
enable_approvals=bool(getattr(args,'enable_remote_approvals',False))
if parent.get('remote_approvals_managed')and not enable_approvals:
raise DeployError('approval parent requires explicit --enable-remote-approvals and callback identity on each update')
approval_identity=None
if enable_approvals:
if any(not (source/'relay'/name).is_file()for name in ('approval_api.py','approval_worker.py','approval_callback.py')):
raise DeployError('remote approval implementation missing')
identity_path=Path(args.callback_identity)
if identity_path.is_symlink()or not identity_path.is_file()or identity_path.stat().st_size>8192:
raise DeployError('invalid private callback identity file')
if os.name!='nt'and identity_path.stat().st_mode&0o077:
raise DeployError('callback identity file must be private (chmod 600)')
approval_identity=json.loads(identity_path.read_text(encoding='utf-8'))
if (not isinstance(approval_identity,dict)or set(approval_identity)!={'tenant_key','verification_token'}
or not isinstance(approval_identity['tenant_key'],str)
or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}',approval_identity['tenant_key'])
or not isinstance(approval_identity['verification_token'],str)
or not 8<=len(approval_identity['verification_token'])<=512
or any(ord(c)<33 or ord(c)>126 for c in approval_identity['verification_token'])):
raise DeployError('invalid callback identity fields; values withheld')
live=run(compose(parent,root/'compose.broker.json')+['config','--format','json'],True)
a=live['services']['connector']['environment'];b=live['services']['longconn']['environment']
if (a.get('HN_FEISHU_APP_ID')!=b.get('HN_FEISHU_APP_ID')or a.get('HN_INTERNAL_SECRET')!=b.get('HN_INTERNAL_SECRET')
or b.get('HN_FEISHU_EVENT_SOURCE')!='longconn'):
raise DeployError('existing unique longconn does not match connector identity')
output=output.resolve()
output.mkdir(mode=0o700)
os.chmod(output,0o700)
@@ -394,6 +446,26 @@ def prepare_update(args):
if mount['target']=='/run/hainuo-credentials/keyring.json':
mount['source']=str(output/'credential-keyring.json')
rollback=json.loads(json.dumps(overlay))
if enable_approvals:
plan['remote_approvals_managed']=True
plan['longconn_candidate_image']='hainuo-relay:approval-longconn-'+plan_id
# Keep separate runtime ancestry: the connector image may lack lark-oapi.
plan['longconn_base_image']=roles['longconn']['image']
shared={'HN_WORK_REMOTE_APPROVALS_ENABLED':'true',
'HN_FEISHU_TENANT_KEY':approval_identity['tenant_key'],
'HN_FEISHU_VERIFICATION_TOKEN':approval_identity['verification_token']}
plan['connector_approval_environment']=shared
plan['longconn_approval_environment']={**shared,'HN_CREDENTIAL_MODE':'broker'}
overlay['services']['connector']['environment'].update(shared)
overlay['services']['connector']['environment']['HN_WORK_TASK_CARDS_ENABLED']='true'
plan['work_task_cards_enabled']=True
original_long=live['services']['longconn']
overlay['services']['longconn']={'image':plan['longconn_candidate_image'],
'environment':plan['longconn_approval_environment']}
defaults={'HN_WORK_REMOTE_APPROVALS_ENABLED':'false','HN_CREDENTIAL_MODE':'legacy',
'HN_FEISHU_TENANT_KEY':'','HN_FEISHU_VERIFICATION_TOKEN':''}
rollback['services']['longconn']={'image':roles['longconn']['image'],
'environment':{k:original_long.get('environment',{}).get(k,defaults[k])for k in plan['longconn_approval_environment']}}
if getattr(args,'enable_work_files',False):
overlay['services']['connector']['environment']['HN_WORK_FILES_ENABLED']='true'
plan['work_files_enabled']=True
@@ -474,14 +546,24 @@ def deployment_state(root,plan):
state = entry.get("state",{})
status = state.get("status") if state.get("status") in {'created','running','paused','restarting','removing','exited','dead'} else 'unknown'
health = state.get("health") if state.get("health") in {'none','starting','healthy','unhealthy'} else 'unknown'
image = 'candidate' if expected and entry['image']==expected else (
role_expected=expected
if name=='longconn'and plan.get('remote_approvals_managed')and receipt_file(root,name).is_file():
role_expected=json.loads(receipt_file(root,name).read_text(encoding='utf-8')).get('image')
image = 'candidate' if role_expected and entry['image']==role_expected else (
'original' if entry['image']==plan['roles'][name]['image'] else 'different')
numbers = {k:v for k,v in state.items() if k in {'exit_code','restart_count'} and type(v) is int}
print(name+': image='+image+' status='+status+' health='+health+' '+json.dumps(numbers,sort_keys=True))
if name in {'connector','router'}:
if name in targets(plan):
raw = run(['docker','logs','--tail','100',entry['id']],optional=True)
if raw is not None:
print(' startup signals: '+(', '.join(log_signals(raw)) or 'no recognized error class; raw logs withheld'))
if name=='longconn':
observed=False
for line in raw.splitlines():
try:record=json.loads(line)
except ValueError:continue
if isinstance(record,dict)and record.get('event')=='platform_connected':observed=True
print(' platform_connected observed: '+('yes'if observed else'not yet; inspect again after startup'))
print('STATE inspection only; no files, keys or containers changed. Healthy candidate services need no reactivation; inspect mixed/unhealthy states before further actions.')
@@ -504,18 +586,19 @@ def check(root, plan, baseline=True, router_key_required=True):
if digest(root/path) != expected:
raise DeployError("prepared keyring or overlay changed; do not regenerate credential keys")
for name, old in plan["roles"].items():
current = service(plan["project"],name,include_stopped=not baseline and name in {"connector","router"})
if (baseline or name not in {"connector","router"}) and current["image"] != old["image"]:
current = service(plan["project"],name,include_stopped=not baseline and name in targets(plan))
if (baseline or name not in targets(plan)) and current["image"] != old["image"]:
raise DeployError("running service image changed since preparation: "+name+"; this plan may already be activated. Run state; do not rebuild or reactivate a healthy candidate")
if not baseline and name in {"connector","router"}:
if not baseline and name in targets(plan):
allowed = {old['image']}
if (root/'candidate.json').is_file():
allowed.add(json.loads((root/'candidate.json').read_text(encoding='utf-8'))['image'])
receipt=receipt_file(root,name)
if receipt.is_file():
allowed.add(json.loads(receipt.read_text(encoding='utf-8'))['image'])
if current['image'] not in allowed:
raise DeployError('unreviewed target image; selected parent plan does not match running '+name+'; use the latest successfully activated plan for a new update. No services changed')
base = run(compose(plan)+["config","--format","json"],True)
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
validate_overlay(base,merged,plan["candidate_image"],router_key_required=router_key_required)
validate_overlay(base,merged,plan["candidate_image"],router_key_required=router_key_required,approval_plan=plan)
# Require the current edge's explicit private-path block, before catch-all handling.
mounts = base["services"]["edge"].get("volumes",[])
caddy = next((v["source"] for v in mounts if v.get("target") == "/etc/caddy/Caddyfile" and v.get("type") == "bind"),None)
@@ -541,8 +624,17 @@ def build(root,plan):
image = run(["docker","image","inspect",plan["candidate_image"],"--format","{{.Id}}"])
run(["docker","run","--rm","--network","none","--read-only","--entrypoint","python",
plan["candidate_image"],"-c","import relay.credential_api, relay.binding_state, relay.device_lifecycle"])
long_image=None
if plan.get('remote_approvals_managed'):
tag=plan['longconn_candidate_image']
if run(['docker','image','inspect',tag],optional=True)is not None:
raise DeployError('longconn candidate already exists; do not overwrite')
run(['docker','build','--pull=false','--network','none','--build-arg',
'BASE_IMAGE='+plan['longconn_base_image'],'-t',tag,str(root/'build')])
long_image=run(['docker','image','inspect',tag,'--format','{{.Id}}'])
runtime_check(root,plan)
write_json(root/"candidate.json",{"image":image,"tag":plan["candidate_image"]})
if long_image:write_json(root/'candidate-longconn.json',{'image':long_image,'tag':plan['longconn_candidate_image']})
print("BUILT: "+plan["candidate_image"]+" (offline runtime reuse; not activated)")
@@ -558,12 +650,17 @@ def runtime_check(root,plan):
" s=Settings(role=data['role'])\n"
" if data['role']=='connector': CredentialKeyring.from_file(s.credential_keyring_file)\n"
" if data['role']=='router': load_public(s.connector_public_key)\n"
" if data['role']=='connector' and s.work_remote_approvals_enabled: import relay.approval_api, relay.approval_worker\n"
" if data['role']=='longconn':\n"
" import lark_oapi as lark\n"
" from lark_oapi.event.callback.model.p2_card_action_trigger import P2CardActionTriggerResponse\n"
" lark.EventDispatcherHandler.builder('','').register_p2_card_action_trigger(lambda e:P2CardActionTriggerResponse({})).build()\n"
"except Exception:\n sys.exit(1)\n")
for role in ("connector","router"):
for role in targets(plan):
env = {key:str(value) for key,value in merged["services"][role]["environment"].items() if value is not None}
command=["docker","run","--rm","-i","--network","none","--read-only","--entrypoint","python",
"-v",str(root/"credential-keyring.json")+":/run/hainuo-credentials/keyring.json:ro",
"-v",str(public_source)+":"+public_key+":ro",plan["candidate_image"],"-c",code]
"-v",str(public_source)+":"+public_key+":ro",target_tag(plan,role),"-c",code]
print('Runtime check: '+role+' configuration and required key files',flush=True)
run(command,input_text=json.dumps({"role":role,"environment":env}))
@@ -614,12 +711,17 @@ def activate(root,plan,ready):
image = run(["docker","image","inspect",plan["candidate_image"],"--format","{{.Id}}"])
if built != {"image":image,"tag":plan["candidate_image"]}:
raise DeployError("candidate image identity changed")
if plan.get('remote_approvals_managed'):
tag=plan['longconn_candidate_image']
saved=json.loads((root/'candidate-longconn.json').read_text(encoding='utf-8'))
actual=run(['docker','image','inspect',tag,'--format','{{.Id}}'])
if saved!={'image':actual,'tag':tag}:raise DeployError('longconn candidate image identity changed')
print('ACTIVATION step: isolated candidate configuration/keyring check (no service switch)',flush=True)
runtime_check(root,plan)
print('ACTIVATION step: recreate connector/router; failures may leave a partial switch, inspect state',flush=True)
print('ACTIVATION step: recreate '+','.join(targets(plan))+'; failures may leave a partial switch, inspect state',flush=True)
run(compose(plan,root/"compose.broker.json")+["up","-d","--no-deps","--no-build","--pull","never",
"--wait","--wait-timeout","180","connector","router"])
print("ACTIVATED: connector/router only. Verify real OAuth, v2 client, unbind and task routing.")
"--wait","--wait-timeout","180",*targets(plan)])
print('ACTIVATED: '+','.join(targets(plan))+' only. Verify real platform connection and task routing.')
def container_permission_profiles(container):
@@ -720,6 +822,8 @@ def main():
update.add_argument("--output",required=True)
update.add_argument('--enable-work-files',action='store_true',help='Opt in after paired-client preparation; rollback retains the previous flag.')
update.add_argument('--enable-task-cards',action='store_true',help='Opt in to paired-client status cards; rollback retains the previous flag.')
update.add_argument('--enable-remote-approvals',action='store_true',help='Update the existing longconn too; needs private callback identity file.')
update.add_argument('--callback-identity',help='Private JSON file with tenant_key and verification_token; never paste values into command arguments.')
permissions=sub.add_parser('update-permissions')
permissions.add_argument('--plan',required=True)
permissions.add_argument('--profiles',required=True)
@@ -764,7 +868,7 @@ def main():
raise DeployError("rollback requires stopping broker clients; old refresh credentials are not restored")
check(root,plan,baseline=False,router_key_required=False)
run(compose(plan,root/"compose.rollback.json")+["up","-d","--no-deps","--no-build","--pull","never",
"--wait","--wait-timeout","180","connector","router"])
"--wait","--wait-timeout","180",*targets(plan)])
print("ROLLED BACK containers only; preserve keyring/database. Client mode and reauthorization must be coordinated.")
except (DeployError,OSError,ValueError,KeyError,TypeError):
error = sys.exc_info()[1]