diff --git a/deployable/ASYNC_BROKER_UPDATE.md b/deployable/ASYNC_BROKER_UPDATE.md new file mode 100644 index 0000000..b90b40c --- /dev/null +++ b/deployable/ASYNC_BROKER_UPDATE.md @@ -0,0 +1,28 @@ +# 已运行 broker 的异步任务更新 + +适用:现有 connector/router 均健康,且已使用 broker 凭据方案。更新不会生成新密钥或重新授权,不修改数据库卷、权限配置和其他组件。本轮为经过当前身份和任务校验的进度续期,仍有绝对上限;ECS 重启或连接丢失后的任务自动恢复不在本轮范围。 + +在当前 ECS 执行(输出目录必须尚不存在): + +```bash +cd /home/cnooc/ecs-relay-service-staging-http +git pull --ff-only +sudo python3 deployable/scripts/broker_deploy.py prepare-update \ + --plan /home/cnooc/hainuo-broker-prepared-20261104 \ + --output /home/cnooc/hainuo-broker-update-20261104-async +sudo python3 deployable/scripts/broker_deploy.py build \ + --plan /home/cnooc/hainuo-broker-update-20261104-async +sudo python3 deployable/scripts/broker_deploy.py activate \ + --plan /home/cnooc/hainuo-broker-update-20261104-async \ + --client-v2-ready +sudo python3 deployable/scripts/broker_deploy.py state \ + --plan /home/cnooc/hainuo-broker-update-20261104-async +``` + +一次执行一条,失败先看 state,不重复 activate。不运行原迁移 prepare,也不重新生成 keyring。prepare-update 沿用父计划的同一密钥内容和权限/应用身份,在独立受保护目录内保留更新计划,旧计划不被覆盖。新镜像从当前已确认镜像离线复用依赖;回滚目标为更新前的 broker 镜像。密钥副本和新旧计划目录均需保留且不得外传。 + +更新工具先验证父计划、关键配置、运行镜像和存储;部分切换会拒绝生成更新。新计划的 rollback 仍要求停止 broker 使用,防止切换期间丢失任务;不恢复数据库或旧 refresh token。回滚只退应用代码,任务执行状态必须另行核对。 + +更新先于配套客户端安装。老 v2 客户端仍可使用,未发送进度时沿用闲置超时。新客户端以任务 ID 查询原生状态,只有校验通过的加密 progress 才续期闲置等待,最终结果仍按原任务/消息发送一次。默认闲置超时 HN_REQUEST_TIMEOUT=45 秒,绝对上限 HN_TASK_TRACKING_TIMEOUT=1800 秒(允许 30–3600);进度不能无限续期。 + +验收:授权状态不变;今日日程一次调用正确时间范围;超过原 150 秒的任务最终能回原消息;双用户不串结果;执行中解绑后不再推旧结果。Docker 构建和这些现场行为未在开发机上实测,需在 ECS 执行并核对。 diff --git a/deployable/relay/binding_state.py b/deployable/relay/binding_state.py index 7cb20aa..0002d61 100644 --- a/deployable/relay/binding_state.py +++ b/deployable/relay/binding_state.py @@ -161,6 +161,14 @@ class BindingState: WHERE turn_id=? AND environment=? AND app_id=? AND status=?""", (status,time.time(),turn,self.environment,self.app_id,previous)).rowcount == 1 + def progress(self, turn, host, session, worker): + # A progress packet may keep transport waiting, never change task outcome + # or extend the absolute lifetime. Unbound/stale sessions fail closed. + task = self.task(turn) + return bool(self.task_current(task) and + (task['host_id'],task['session_id'],task['worker_id'],task['status']) == + (host,session,worker,'inflight')) + def complete(self, turn, host, session, worker): with self.db: task = self.task(turn) diff --git a/deployable/relay/config.py b/deployable/relay/config.py index a46b485..6d4f775 100644 --- a/deployable/relay/config.py +++ b/deployable/relay/config.py @@ -34,6 +34,7 @@ class Settings(BaseSettings): # A fixed allowlist, never a user supplied upstream URL. proxy_paths: str = "/open-apis/drive/v1/files,/open-apis/docx/v1/documents" request_timeout: int = 45 + task_tracking_timeout: int = 1800 max_body: int = 262144 max_pending: int = 1000 max_devices: int = 1000 @@ -49,6 +50,8 @@ class Settings(BaseSettings): @model_validator(mode="after") def validate_runtime(self): + if not 30 <= self.task_tracking_timeout <= 3600: + raise ValueError("task tracking timeout must be 30-3600 seconds") if self.credential_mode not in {"legacy", "broker"}: raise ValueError("invalid credential mode") if self.credential_mode == "broker" and self.role in {"connector","router"}: diff --git a/deployable/relay/connector.py b/deployable/relay/connector.py index 5dc04f2..2f11880 100644 --- a/deployable/relay/connector.py +++ b/deployable/relay/connector.py @@ -10,7 +10,7 @@ import sqlite3 import time import uuid from contextlib import asynccontextmanager -from dataclasses import dataclass +from dataclasses import dataclass, field from urllib.parse import unquote, urlsplit from cryptography.hazmat.primitives import hashes, padding @@ -55,6 +55,7 @@ class Pending: public: object host: str session: str + progress: asyncio.Event = field(default_factory=asyncio.Event) class Broker: @@ -76,7 +77,14 @@ class Broker: continue plain = open_envelope(data["envelope"], item.private, item.public, host_id=item.host, session_id=item.session) - if plain["turn_id"] != data["turn_id"] or plain.get("type") != "result": + if plain["turn_id"] != data["turn_id"]: + continue + if plain.get('type') == 'progress': + if self.lifecycle and self.lifecycle.progress(data['turn_id'],item.host,item.session,self.worker_id): + await self.r.expire(f"hn:inflight:{item.host}:{data['turn_id']}", self.s.request_timeout + 5) + item.progress.set() + continue + if plain.get('type') != 'result': continue result = plain.get("result", "") if not isinstance(result, str) or len(result) > self.s.max_result: @@ -88,6 +96,28 @@ class Broker: # Invalid packets never consume a pending result; do not log packet contents. continue + async def wait_tracked(self, item): + # Authenticated native status polls renew idle waiting, never the hard + # task deadline. A lost device/worker remains unknown and is not replayed. + deadline = asyncio.get_running_loop().time() + self.s.task_tracking_timeout + while True: + remaining = deadline - asyncio.get_running_loop().time() + if remaining <= 0: + raise asyncio.TimeoutError() + item.progress.clear() + progress = asyncio.create_task(item.progress.wait()) + try: + done, _ = await asyncio.wait({item.future, progress}, + timeout=min(self.s.request_timeout, remaining), return_when=asyncio.FIRST_COMPLETED) + if item.future in done: + return item.future.result() + if not done: + raise asyncio.TimeoutError() + finally: + progress.cancel() + with contextlib.suppress(asyncio.CancelledError): + await progress + async def dispatch(self, owner, text, turn, *, chat_id="", source_message_id=""): device = self.store.by_owner(owner) if not device: @@ -120,7 +150,7 @@ class Broker: if self.lifecycle: try: task = self.lifecycle.create_task(turn,owner,host,device["binding_generation"],session, - self.worker_id,time.time()+self.s.request_timeout,chat_id,source_message_id) + self.worker_id,time.time()+self.s.task_tracking_timeout,chat_id,source_message_id) except ValueError: self.store.finish(turn,"unbound") return {"status":"unbound","turn_id":turn,"result":"绑定已变化,任务未下发。"} @@ -157,7 +187,10 @@ class Broker: if task: self.lifecycle.transition(turn,"inflight","offline") return {"status": "offline", "turn_id": turn, "result": "你的电脑当前离线。"} - result = await asyncio.wait_for(future, timeout=self.s.request_timeout) + if task: + result = await self.wait_tracked(self.pending[turn]) + else: + result = await asyncio.wait_for(future, timeout=self.s.request_timeout) self.store.finish(turn, result.get("status","unknown") if task else "completed") if task and not source_message_id: self.lifecycle.transition(turn,"completed","finished") diff --git a/deployable/scripts/broker_deploy.py b/deployable/scripts/broker_deploy.py index c8dec23..e2070a2 100644 --- a/deployable/scripts/broker_deploy.py +++ b/deployable/scripts/broker_deploy.py @@ -355,6 +355,58 @@ def prepare(args): print("PREPARED: "+str(output)+" (services unchanged; private files must not be shared)") +def prepare_update(args): + """Stage an already-active broker update without generating credentials.""" + root=Path(args.plan).resolve() + parent=load_plan(root) + check(root,parent,baseline=False) + built=json.loads((root/'candidate.json').read_text(encoding='utf-8')) + roles={name:service(parent['project'],name) for name in parent['roles']} + if any(roles[name]['image'] != built['image'] for name in ('connector','router')): + raise DeployError('update requires both services on the confirmed parent candidate; inspect state first') + output=Path(args.output) + source=Path(args.source_root).resolve() + deployment=Path(parent['deployment_root']).resolve() + if (not output.is_absolute() or output.exists() or output.is_symlink() + or output.resolve()==deployment or deployment in output.resolve().parents + or any(c in str(output) for c in ('\n','\r','$'))): + raise DeployError('update output must be a new absolute directory outside the live deployment') + if not (source/'relay/connector.py').is_file(): + raise DeployError('missing update source') + output=output.resolve() + output.mkdir(mode=0o700) + os.chmod(output,0o700) + plan=json.loads(json.dumps(parent)) + plan_id=secrets.token_hex(8) + plan.update(plan_id=plan_id,candidate_image='hainuo-relay:broker-update-'+plan_id, + roles=roles,base_image=built['image'],parent_plan=str(root),update_preserves_credentials=True) + # Copy identical server key material; never rotate/reinitialize while updating code. + shutil.copyfile(root/'credential-keyring.json',output/'credential-keyring.json') + os.chmod(output/'credential-keyring.json',0o600) + if os.name!='nt':os.chown(output/'credential-keyring.json',10001,10001) + overlay=json.loads((root/'compose.broker.json').read_text(encoding='utf-8')) + for name in ('connector','router'):overlay['services'][name]['image']=plan['candidate_image'] + for mount in overlay['services']['connector']['volumes']: + if mount['target']=='/run/hainuo-credentials/keyring.json': + mount['source']=str(output/'credential-keyring.json') + write_json(output/'compose.broker.json',overlay) + write_json(output/'compose.rollback.json',{'services':{name:{'image':roles[name]['image']} + for name in ('connector','router','longconn','enrollment-signer')}}) + payload=output/'build';payload.mkdir(mode=0o700) + for path in (source/'relay').rglob('*'): + if path.is_symlink():raise DeployError('symlink in update source') + if path.is_file() and path.suffix in {'.py','.js','.css','.html'} and '__pycache__' not in path.parts: + target=payload/path.relative_to(source);target.parent.mkdir(parents=True,exist_ok=True) + shutil.copyfile(path,target);os.chmod(target,0o644) + shutil.copyfile(root/'build/Dockerfile',payload/'Dockerfile') + plan['payload_hashes']={str(p.relative_to(output)):digest(p) for p in payload.rglob('*') if p.is_file()} + plan['prepared_hashes']={name:digest(output/name) for name in + ('credential-keyring.json','compose.broker.json','compose.rollback.json')} + write_json(output/'plan.json',plan) + check(output,plan) + print('UPDATE PREPARED: existing credential key material and permissions preserved; no services restarted') + + def load_plan(root): plan = json.loads((root/"plan.json").read_text(encoding="utf-8")) if plan.get("format") != 1 or not re.fullmatch(r"[a-f0-9]{16}",plan.get("plan_id","")): @@ -568,6 +620,10 @@ def main(): prep.add_argument("--profiles",required=True) prep.add_argument("--output",required=True) prep.add_argument("--connector-container",default="hainuo-relay-connector-1") + update = sub.add_parser("prepare-update") + update.add_argument("--plan",required=True) + update.add_argument("--source-root",default=str(Path(__file__).resolve().parents[1])) + update.add_argument("--output",required=True) for name in ("repair-router","state","diagnose","check","build","activate","rollback"): cmd = sub.add_parser(name) cmd.add_argument("--plan",required=True) @@ -583,6 +639,9 @@ def main(): if args.action == "prepare": prepare(args) return + if args.action == "prepare-update": + prepare_update(args) + return root = Path(args.plan).resolve() plan = load_plan(root) if args.action == "repair-router": diff --git a/deployable/tests/test_broker_deploy.py b/deployable/tests/test_broker_deploy.py index 7483674..f47980c 100644 --- a/deployable/tests/test_broker_deploy.py +++ b/deployable/tests/test_broker_deploy.py @@ -510,3 +510,40 @@ def test_overlay_validator_rejects_database_volume_or_network_change(scenario,fi merged[field] = {"unexpected":{}} with pytest.raises(deploy.DeployError): deploy.validate_overlay(base,merged,plan["candidate_image"]) + + +def test_prepare_update_preserves_key_material_and_parent_plan_without_restart(scenario): + parent=prepared(scenario) + root=scenario['output'] + original_key=(root/'credential-keyring.json').read_bytes() + original_plan=(root/'plan.json').read_bytes() + image='sha256:'+'a'*64 + for name in ('connector','router'):scenario['roles'][name]['image']=image + deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']}) + output=root.parent/'update' + args=argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output)) + scenario['calls'].clear() + deploy.prepare_update(args) + updated=deploy.load_plan(output) + assert (output/'credential-keyring.json').read_bytes()==original_key + assert (root/'plan.json').read_bytes()==original_plan + assert updated['base_image']==image and updated['roles']['connector']['image']==image + assert updated['candidate_image']!=parent['candidate_image'] + assert updated['update_preserves_credentials'] is True + overlay=json.loads((output/'compose.broker.json').read_text()) + old=json.loads((root/'compose.broker.json').read_text()) + assert overlay['services']['connector']['environment']==old['services']['connector']['environment'] + assert not any('up' in call or 'build' in call or 'tag' in call for call in scenario['calls']) + + +def test_prepare_update_rejects_partial_switch_and_existing_output(scenario): + parent=prepared(scenario);root=scenario['output'];image='sha256:'+'a'*64 + deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']}) + scenario['roles']['connector']['image']=image + output=root.parent/'update' + args=argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output)) + with pytest.raises(deploy.DeployError,match='both services'):deploy.prepare_update(args) + assert not output.exists() + scenario['roles']['router']['image']=image + args.output=str(root) + with pytest.raises(deploy.DeployError,match='new absolute'):deploy.prepare_update(args) diff --git a/deployable/tests/test_device_lifecycle.py b/deployable/tests/test_device_lifecycle.py index cf636ca..ac18c54 100644 --- a/deployable/tests/test_device_lifecycle.py +++ b/deployable/tests/test_device_lifecycle.py @@ -477,3 +477,56 @@ async def test_admission_denies_dead_tasks_without_exposing_credentials(env): headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret}, json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"}) assert response.status_code == 403 and "token" not in response.text.lower() + + +async def test_native_progress_keeps_task_until_final_and_original_reply_once(routed): + routed['settings'].request_timeout=1 + connection=await device(routed) + await send_event(routed,event(routed,'om-progress','synthetic slow task')) + payload=await instruction(routed,connection) + ws,session,_=connection + for _ in range(4): + envelope=seal({'type':'progress','turn_id':payload['turn_id']},payload['reply_key'], + routed['keys']['a'*32],host_id='a'*32,session_id=session) + await ws.send(json.dumps({'type':'reply','turn_id':payload['turn_id'],'envelope':envelope})) + await asyncio.sleep(.4) + assert not routed['platform'].replies + assert routed['store'].lifecycle.task(payload['turn_id'])['status']=='inflight' + await reply(routed,connection,payload,'synthetic final') + await wait_for_reply(routed['platform'],1) + assert routed['platform'].replies==[('om-progress','synthetic final')] + await send_event(routed,event(routed,'om-progress','synthetic slow task')) + await asyncio.sleep(.05) + assert len(routed['platform'].replies)==1 + + +async def test_progress_wait_cannot_extend_hard_deadline_or_hide_idle_failure(): + from relay.connector import Broker,Pending + from types import SimpleNamespace + broker=object.__new__(Broker) + broker.s=SimpleNamespace(task_tracking_timeout=.12,request_timeout=.08) + item=Pending(asyncio.get_running_loop().create_future(),None,None,'host','session') + async def pulses(): + while True: + item.progress.set() + await asyncio.sleep(.02) + pulse=asyncio.create_task(pulses()) + try: + with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item) + assert not item.future.done() + finally: + pulse.cancel() + try:await pulse + except asyncio.CancelledError:pass + broker.s.task_tracking_timeout=1 + with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item) + + +async def test_unbound_or_wrong_session_progress_cannot_keep_task_alive(env): + state=env['store'].lifecycle + state.create_task('progress-bound','ou_a','a'*32,1,'session-a','worker-a',time.time()+60) + assert state.progress('progress-bound','a'*32,'session-a','worker-a') + assert not state.progress('progress-bound','a'*32,'session-b','worker-a') + assert not state.progress('progress-bound','a'*32,'session-a','worker-b') + env['store'].unbind('ou_a') + assert not state.progress('progress-bound','a'*32,'session-a','worker-a')