feat(tasks): track authenticated native progress until final reply

Renew broker idle waits only for encrypted current-task progress, keep an absolute deadline and binding/session/worker fences, and retain original-message reply deduplication. Add guarded prepare-update staging that reuses the active broker key material and application configuration. Validation: 216 passed, 2 skipped; guarded update and long-task routing covered with synthetic Docker configuration and loopback WebSocket tests. Real ECS build and acceptance remain pending.
This commit is contained in:
ex_tianzy
2026-11-04 18:12:12 +08:00
parent ce965aaf58
commit 660d7a48d2
7 changed files with 225 additions and 4 deletions
+28
View File
@@ -0,0 +1,28 @@
# 已运行 broker 的异步任务更新
适用:现有 connector/router 均健康,且已使用 broker 凭据方案。更新不会生成新密钥或重新授权,不修改数据库卷、权限配置和其他组件。本轮为经过当前身份和任务校验的进度续期,仍有绝对上限;ECS 重启或连接丢失后的任务自动恢复不在本轮范围。
在当前 ECS 执行(输出目录必须尚不存在):
```bash
cd /home/cnooc/ecs-relay-service-staging-http
git pull --ff-only
sudo python3 deployable/scripts/broker_deploy.py prepare-update \
--plan /home/cnooc/hainuo-broker-prepared-20261104 \
--output /home/cnooc/hainuo-broker-update-20261104-async
sudo python3 deployable/scripts/broker_deploy.py build \
--plan /home/cnooc/hainuo-broker-update-20261104-async
sudo python3 deployable/scripts/broker_deploy.py activate \
--plan /home/cnooc/hainuo-broker-update-20261104-async \
--client-v2-ready
sudo python3 deployable/scripts/broker_deploy.py state \
--plan /home/cnooc/hainuo-broker-update-20261104-async
```
一次执行一条,失败先看 state,不重复 activate。不运行原迁移 prepare,也不重新生成 keyring。prepare-update 沿用父计划的同一密钥内容和权限/应用身份,在独立受保护目录内保留更新计划,旧计划不被覆盖。新镜像从当前已确认镜像离线复用依赖;回滚目标为更新前的 broker 镜像。密钥副本和新旧计划目录均需保留且不得外传。
更新工具先验证父计划、关键配置、运行镜像和存储;部分切换会拒绝生成更新。新计划的 rollback 仍要求停止 broker 使用,防止切换期间丢失任务;不恢复数据库或旧 refresh token。回滚只退应用代码,任务执行状态必须另行核对。
更新先于配套客户端安装。老 v2 客户端仍可使用,未发送进度时沿用闲置超时。新客户端以任务 ID 查询原生状态,只有校验通过的加密 progress 才续期闲置等待,最终结果仍按原任务/消息发送一次。默认闲置超时 HN_REQUEST_TIMEOUT=45 秒,绝对上限 HN_TASK_TRACKING_TIMEOUT=1800 秒(允许 30–3600);进度不能无限续期。
验收:授权状态不变;今日日程一次调用正确时间范围;超过原 150 秒的任务最终能回原消息;双用户不串结果;执行中解绑后不再推旧结果。Docker 构建和这些现场行为未在开发机上实测,需在 ECS 执行并核对。
+8
View File
@@ -161,6 +161,14 @@ class BindingState:
WHERE turn_id=? AND environment=? AND app_id=? AND status=?""",
(status,time.time(),turn,self.environment,self.app_id,previous)).rowcount == 1
def progress(self, turn, host, session, worker):
# A progress packet may keep transport waiting, never change task outcome
# or extend the absolute lifetime. Unbound/stale sessions fail closed.
task = self.task(turn)
return bool(self.task_current(task) and
(task['host_id'],task['session_id'],task['worker_id'],task['status']) ==
(host,session,worker,'inflight'))
def complete(self, turn, host, session, worker):
with self.db:
task = self.task(turn)
+3
View File
@@ -34,6 +34,7 @@ class Settings(BaseSettings):
# A fixed allowlist, never a user supplied upstream URL.
proxy_paths: str = "/open-apis/drive/v1/files,/open-apis/docx/v1/documents"
request_timeout: int = 45
task_tracking_timeout: int = 1800
max_body: int = 262144
max_pending: int = 1000
max_devices: int = 1000
@@ -49,6 +50,8 @@ class Settings(BaseSettings):
@model_validator(mode="after")
def validate_runtime(self):
if not 30 <= self.task_tracking_timeout <= 3600:
raise ValueError("task tracking timeout must be 30-3600 seconds")
if self.credential_mode not in {"legacy", "broker"}:
raise ValueError("invalid credential mode")
if self.credential_mode == "broker" and self.role in {"connector","router"}:
+37 -4
View File
@@ -10,7 +10,7 @@ import sqlite3
import time
import uuid
from contextlib import asynccontextmanager
from dataclasses import dataclass
from dataclasses import dataclass, field
from urllib.parse import unquote, urlsplit
from cryptography.hazmat.primitives import hashes, padding
@@ -55,6 +55,7 @@ class Pending:
public: object
host: str
session: str
progress: asyncio.Event = field(default_factory=asyncio.Event)
class Broker:
@@ -76,7 +77,14 @@ class Broker:
continue
plain = open_envelope(data["envelope"], item.private, item.public,
host_id=item.host, session_id=item.session)
if plain["turn_id"] != data["turn_id"] or plain.get("type") != "result":
if plain["turn_id"] != data["turn_id"]:
continue
if plain.get('type') == 'progress':
if self.lifecycle and self.lifecycle.progress(data['turn_id'],item.host,item.session,self.worker_id):
await self.r.expire(f"hn:inflight:{item.host}:{data['turn_id']}", self.s.request_timeout + 5)
item.progress.set()
continue
if plain.get('type') != 'result':
continue
result = plain.get("result", "")
if not isinstance(result, str) or len(result) > self.s.max_result:
@@ -88,6 +96,28 @@ class Broker:
# Invalid packets never consume a pending result; do not log packet contents.
continue
async def wait_tracked(self, item):
# Authenticated native status polls renew idle waiting, never the hard
# task deadline. A lost device/worker remains unknown and is not replayed.
deadline = asyncio.get_running_loop().time() + self.s.task_tracking_timeout
while True:
remaining = deadline - asyncio.get_running_loop().time()
if remaining <= 0:
raise asyncio.TimeoutError()
item.progress.clear()
progress = asyncio.create_task(item.progress.wait())
try:
done, _ = await asyncio.wait({item.future, progress},
timeout=min(self.s.request_timeout, remaining), return_when=asyncio.FIRST_COMPLETED)
if item.future in done:
return item.future.result()
if not done:
raise asyncio.TimeoutError()
finally:
progress.cancel()
with contextlib.suppress(asyncio.CancelledError):
await progress
async def dispatch(self, owner, text, turn, *, chat_id="", source_message_id=""):
device = self.store.by_owner(owner)
if not device:
@@ -120,7 +150,7 @@ class Broker:
if self.lifecycle:
try:
task = self.lifecycle.create_task(turn,owner,host,device["binding_generation"],session,
self.worker_id,time.time()+self.s.request_timeout,chat_id,source_message_id)
self.worker_id,time.time()+self.s.task_tracking_timeout,chat_id,source_message_id)
except ValueError:
self.store.finish(turn,"unbound")
return {"status":"unbound","turn_id":turn,"result":"绑定已变化,任务未下发。"}
@@ -157,7 +187,10 @@ class Broker:
if task:
self.lifecycle.transition(turn,"inflight","offline")
return {"status": "offline", "turn_id": turn, "result": "你的电脑当前离线。"}
result = await asyncio.wait_for(future, timeout=self.s.request_timeout)
if task:
result = await self.wait_tracked(self.pending[turn])
else:
result = await asyncio.wait_for(future, timeout=self.s.request_timeout)
self.store.finish(turn, result.get("status","unknown") if task else "completed")
if task and not source_message_id:
self.lifecycle.transition(turn,"completed","finished")
+59
View File
@@ -355,6 +355,58 @@ def prepare(args):
print("PREPARED: "+str(output)+" (services unchanged; private files must not be shared)")
def prepare_update(args):
"""Stage an already-active broker update without generating credentials."""
root=Path(args.plan).resolve()
parent=load_plan(root)
check(root,parent,baseline=False)
built=json.loads((root/'candidate.json').read_text(encoding='utf-8'))
roles={name:service(parent['project'],name) for name in parent['roles']}
if any(roles[name]['image'] != built['image'] for name in ('connector','router')):
raise DeployError('update requires both services on the confirmed parent candidate; inspect state first')
output=Path(args.output)
source=Path(args.source_root).resolve()
deployment=Path(parent['deployment_root']).resolve()
if (not output.is_absolute() or output.exists() or output.is_symlink()
or output.resolve()==deployment or deployment in output.resolve().parents
or any(c in str(output) for c in ('\n','\r','$'))):
raise DeployError('update output must be a new absolute directory outside the live deployment')
if not (source/'relay/connector.py').is_file():
raise DeployError('missing update source')
output=output.resolve()
output.mkdir(mode=0o700)
os.chmod(output,0o700)
plan=json.loads(json.dumps(parent))
plan_id=secrets.token_hex(8)
plan.update(plan_id=plan_id,candidate_image='hainuo-relay:broker-update-'+plan_id,
roles=roles,base_image=built['image'],parent_plan=str(root),update_preserves_credentials=True)
# Copy identical server key material; never rotate/reinitialize while updating code.
shutil.copyfile(root/'credential-keyring.json',output/'credential-keyring.json')
os.chmod(output/'credential-keyring.json',0o600)
if os.name!='nt':os.chown(output/'credential-keyring.json',10001,10001)
overlay=json.loads((root/'compose.broker.json').read_text(encoding='utf-8'))
for name in ('connector','router'):overlay['services'][name]['image']=plan['candidate_image']
for mount in overlay['services']['connector']['volumes']:
if mount['target']=='/run/hainuo-credentials/keyring.json':
mount['source']=str(output/'credential-keyring.json')
write_json(output/'compose.broker.json',overlay)
write_json(output/'compose.rollback.json',{'services':{name:{'image':roles[name]['image']}
for name in ('connector','router','longconn','enrollment-signer')}})
payload=output/'build';payload.mkdir(mode=0o700)
for path in (source/'relay').rglob('*'):
if path.is_symlink():raise DeployError('symlink in update source')
if path.is_file() and path.suffix in {'.py','.js','.css','.html'} and '__pycache__' not in path.parts:
target=payload/path.relative_to(source);target.parent.mkdir(parents=True,exist_ok=True)
shutil.copyfile(path,target);os.chmod(target,0o644)
shutil.copyfile(root/'build/Dockerfile',payload/'Dockerfile')
plan['payload_hashes']={str(p.relative_to(output)):digest(p) for p in payload.rglob('*') if p.is_file()}
plan['prepared_hashes']={name:digest(output/name) for name in
('credential-keyring.json','compose.broker.json','compose.rollback.json')}
write_json(output/'plan.json',plan)
check(output,plan)
print('UPDATE PREPARED: existing credential key material and permissions preserved; no services restarted')
def load_plan(root):
plan = json.loads((root/"plan.json").read_text(encoding="utf-8"))
if plan.get("format") != 1 or not re.fullmatch(r"[a-f0-9]{16}",plan.get("plan_id","")):
@@ -568,6 +620,10 @@ def main():
prep.add_argument("--profiles",required=True)
prep.add_argument("--output",required=True)
prep.add_argument("--connector-container",default="hainuo-relay-connector-1")
update = sub.add_parser("prepare-update")
update.add_argument("--plan",required=True)
update.add_argument("--source-root",default=str(Path(__file__).resolve().parents[1]))
update.add_argument("--output",required=True)
for name in ("repair-router","state","diagnose","check","build","activate","rollback"):
cmd = sub.add_parser(name)
cmd.add_argument("--plan",required=True)
@@ -583,6 +639,9 @@ def main():
if args.action == "prepare":
prepare(args)
return
if args.action == "prepare-update":
prepare_update(args)
return
root = Path(args.plan).resolve()
plan = load_plan(root)
if args.action == "repair-router":
+37
View File
@@ -510,3 +510,40 @@ def test_overlay_validator_rejects_database_volume_or_network_change(scenario,fi
merged[field] = {"unexpected":{}}
with pytest.raises(deploy.DeployError):
deploy.validate_overlay(base,merged,plan["candidate_image"])
def test_prepare_update_preserves_key_material_and_parent_plan_without_restart(scenario):
parent=prepared(scenario)
root=scenario['output']
original_key=(root/'credential-keyring.json').read_bytes()
original_plan=(root/'plan.json').read_bytes()
image='sha256:'+'a'*64
for name in ('connector','router'):scenario['roles'][name]['image']=image
deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']})
output=root.parent/'update'
args=argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output))
scenario['calls'].clear()
deploy.prepare_update(args)
updated=deploy.load_plan(output)
assert (output/'credential-keyring.json').read_bytes()==original_key
assert (root/'plan.json').read_bytes()==original_plan
assert updated['base_image']==image and updated['roles']['connector']['image']==image
assert updated['candidate_image']!=parent['candidate_image']
assert updated['update_preserves_credentials'] is True
overlay=json.loads((output/'compose.broker.json').read_text())
old=json.loads((root/'compose.broker.json').read_text())
assert overlay['services']['connector']['environment']==old['services']['connector']['environment']
assert not any('up' in call or 'build' in call or 'tag' in call for call in scenario['calls'])
def test_prepare_update_rejects_partial_switch_and_existing_output(scenario):
parent=prepared(scenario);root=scenario['output'];image='sha256:'+'a'*64
deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']})
scenario['roles']['connector']['image']=image
output=root.parent/'update'
args=argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output))
with pytest.raises(deploy.DeployError,match='both services'):deploy.prepare_update(args)
assert not output.exists()
scenario['roles']['router']['image']=image
args.output=str(root)
with pytest.raises(deploy.DeployError,match='new absolute'):deploy.prepare_update(args)
+53
View File
@@ -477,3 +477,56 @@ async def test_admission_denies_dead_tasks_without_exposing_credentials(env):
headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret},
json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"})
assert response.status_code == 403 and "token" not in response.text.lower()
async def test_native_progress_keeps_task_until_final_and_original_reply_once(routed):
routed['settings'].request_timeout=1
connection=await device(routed)
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
payload=await instruction(routed,connection)
ws,session,_=connection
for _ in range(4):
envelope=seal({'type':'progress','turn_id':payload['turn_id']},payload['reply_key'],
routed['keys']['a'*32],host_id='a'*32,session_id=session)
await ws.send(json.dumps({'type':'reply','turn_id':payload['turn_id'],'envelope':envelope}))
await asyncio.sleep(.4)
assert not routed['platform'].replies
assert routed['store'].lifecycle.task(payload['turn_id'])['status']=='inflight'
await reply(routed,connection,payload,'synthetic final')
await wait_for_reply(routed['platform'],1)
assert routed['platform'].replies==[('om-progress','synthetic final')]
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
await asyncio.sleep(.05)
assert len(routed['platform'].replies)==1
async def test_progress_wait_cannot_extend_hard_deadline_or_hide_idle_failure():
from relay.connector import Broker,Pending
from types import SimpleNamespace
broker=object.__new__(Broker)
broker.s=SimpleNamespace(task_tracking_timeout=.12,request_timeout=.08)
item=Pending(asyncio.get_running_loop().create_future(),None,None,'host','session')
async def pulses():
while True:
item.progress.set()
await asyncio.sleep(.02)
pulse=asyncio.create_task(pulses())
try:
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
assert not item.future.done()
finally:
pulse.cancel()
try:await pulse
except asyncio.CancelledError:pass
broker.s.task_tracking_timeout=1
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
async def test_unbound_or_wrong_session_progress_cannot_keep_task_alive(env):
state=env['store'].lifecycle
state.create_task('progress-bound','ou_a','a'*32,1,'session-a','worker-a',time.time()+60)
assert state.progress('progress-bound','a'*32,'session-a','worker-a')
assert not state.progress('progress-bound','a'*32,'session-b','worker-a')
assert not state.progress('progress-bound','a'*32,'session-a','worker-b')
env['store'].unbind('ou_a')
assert not state.progress('progress-bound','a'*32,'session-a','worker-a')