feat(tasks): track authenticated native progress until final reply

Renew broker idle waits only for encrypted current-task progress, keep an absolute deadline and binding/session/worker fences, and retain original-message reply deduplication. Add guarded prepare-update staging that reuses the active broker key material and application configuration. Validation: 216 passed, 2 skipped; guarded update and long-task routing covered with synthetic Docker configuration and loopback WebSocket tests. Real ECS build and acceptance remain pending.
This commit is contained in:
ex_tianzy
2026-11-04 18:12:12 +08:00
parent ce965aaf58
commit 660d7a48d2
7 changed files with 225 additions and 4 deletions
+28
View File
@@ -0,0 +1,28 @@
# 已运行 broker 的异步任务更新
适用:现有 connector/router 均健康,且已使用 broker 凭据方案。更新不会生成新密钥或重新授权,不修改数据库卷、权限配置和其他组件。本轮为经过当前身份和任务校验的进度续期,仍有绝对上限;ECS 重启或连接丢失后的任务自动恢复不在本轮范围。
在当前 ECS 执行(输出目录必须尚不存在):
```bash
cd /home/cnooc/ecs-relay-service-staging-http
git pull --ff-only
sudo python3 deployable/scripts/broker_deploy.py prepare-update \
--plan /home/cnooc/hainuo-broker-prepared-20261104 \
--output /home/cnooc/hainuo-broker-update-20261104-async
sudo python3 deployable/scripts/broker_deploy.py build \
--plan /home/cnooc/hainuo-broker-update-20261104-async
sudo python3 deployable/scripts/broker_deploy.py activate \
--plan /home/cnooc/hainuo-broker-update-20261104-async \
--client-v2-ready
sudo python3 deployable/scripts/broker_deploy.py state \
--plan /home/cnooc/hainuo-broker-update-20261104-async
```
一次执行一条,失败先看 state,不重复 activate。不运行原迁移 prepare,也不重新生成 keyring。prepare-update 沿用父计划的同一密钥内容和权限/应用身份,在独立受保护目录内保留更新计划,旧计划不被覆盖。新镜像从当前已确认镜像离线复用依赖;回滚目标为更新前的 broker 镜像。密钥副本和新旧计划目录均需保留且不得外传。
更新工具先验证父计划、关键配置、运行镜像和存储;部分切换会拒绝生成更新。新计划的 rollback 仍要求停止 broker 使用,防止切换期间丢失任务;不恢复数据库或旧 refresh token。回滚只退应用代码,任务执行状态必须另行核对。
更新先于配套客户端安装。老 v2 客户端仍可使用,未发送进度时沿用闲置超时。新客户端以任务 ID 查询原生状态,只有校验通过的加密 progress 才续期闲置等待,最终结果仍按原任务/消息发送一次。默认闲置超时 HN_REQUEST_TIMEOUT=45 秒,绝对上限 HN_TASK_TRACKING_TIMEOUT=1800 秒(允许 30–3600);进度不能无限续期。
验收:授权状态不变;今日日程一次调用正确时间范围;超过原 150 秒的任务最终能回原消息;双用户不串结果;执行中解绑后不再推旧结果。Docker 构建和这些现场行为未在开发机上实测,需在 ECS 执行并核对。
+8
View File
@@ -161,6 +161,14 @@ class BindingState:
WHERE turn_id=? AND environment=? AND app_id=? AND status=?""", WHERE turn_id=? AND environment=? AND app_id=? AND status=?""",
(status,time.time(),turn,self.environment,self.app_id,previous)).rowcount == 1 (status,time.time(),turn,self.environment,self.app_id,previous)).rowcount == 1
def progress(self, turn, host, session, worker):
# A progress packet may keep transport waiting, never change task outcome
# or extend the absolute lifetime. Unbound/stale sessions fail closed.
task = self.task(turn)
return bool(self.task_current(task) and
(task['host_id'],task['session_id'],task['worker_id'],task['status']) ==
(host,session,worker,'inflight'))
def complete(self, turn, host, session, worker): def complete(self, turn, host, session, worker):
with self.db: with self.db:
task = self.task(turn) task = self.task(turn)
+3
View File
@@ -34,6 +34,7 @@ class Settings(BaseSettings):
# A fixed allowlist, never a user supplied upstream URL. # A fixed allowlist, never a user supplied upstream URL.
proxy_paths: str = "/open-apis/drive/v1/files,/open-apis/docx/v1/documents" proxy_paths: str = "/open-apis/drive/v1/files,/open-apis/docx/v1/documents"
request_timeout: int = 45 request_timeout: int = 45
task_tracking_timeout: int = 1800
max_body: int = 262144 max_body: int = 262144
max_pending: int = 1000 max_pending: int = 1000
max_devices: int = 1000 max_devices: int = 1000
@@ -49,6 +50,8 @@ class Settings(BaseSettings):
@model_validator(mode="after") @model_validator(mode="after")
def validate_runtime(self): def validate_runtime(self):
if not 30 <= self.task_tracking_timeout <= 3600:
raise ValueError("task tracking timeout must be 30-3600 seconds")
if self.credential_mode not in {"legacy", "broker"}: if self.credential_mode not in {"legacy", "broker"}:
raise ValueError("invalid credential mode") raise ValueError("invalid credential mode")
if self.credential_mode == "broker" and self.role in {"connector","router"}: if self.credential_mode == "broker" and self.role in {"connector","router"}:
+37 -4
View File
@@ -10,7 +10,7 @@ import sqlite3
import time import time
import uuid import uuid
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from dataclasses import dataclass from dataclasses import dataclass, field
from urllib.parse import unquote, urlsplit from urllib.parse import unquote, urlsplit
from cryptography.hazmat.primitives import hashes, padding from cryptography.hazmat.primitives import hashes, padding
@@ -55,6 +55,7 @@ class Pending:
public: object public: object
host: str host: str
session: str session: str
progress: asyncio.Event = field(default_factory=asyncio.Event)
class Broker: class Broker:
@@ -76,7 +77,14 @@ class Broker:
continue continue
plain = open_envelope(data["envelope"], item.private, item.public, plain = open_envelope(data["envelope"], item.private, item.public,
host_id=item.host, session_id=item.session) host_id=item.host, session_id=item.session)
if plain["turn_id"] != data["turn_id"] or plain.get("type") != "result": if plain["turn_id"] != data["turn_id"]:
continue
if plain.get('type') == 'progress':
if self.lifecycle and self.lifecycle.progress(data['turn_id'],item.host,item.session,self.worker_id):
await self.r.expire(f"hn:inflight:{item.host}:{data['turn_id']}", self.s.request_timeout + 5)
item.progress.set()
continue
if plain.get('type') != 'result':
continue continue
result = plain.get("result", "") result = plain.get("result", "")
if not isinstance(result, str) or len(result) > self.s.max_result: if not isinstance(result, str) or len(result) > self.s.max_result:
@@ -88,6 +96,28 @@ class Broker:
# Invalid packets never consume a pending result; do not log packet contents. # Invalid packets never consume a pending result; do not log packet contents.
continue continue
async def wait_tracked(self, item):
# Authenticated native status polls renew idle waiting, never the hard
# task deadline. A lost device/worker remains unknown and is not replayed.
deadline = asyncio.get_running_loop().time() + self.s.task_tracking_timeout
while True:
remaining = deadline - asyncio.get_running_loop().time()
if remaining <= 0:
raise asyncio.TimeoutError()
item.progress.clear()
progress = asyncio.create_task(item.progress.wait())
try:
done, _ = await asyncio.wait({item.future, progress},
timeout=min(self.s.request_timeout, remaining), return_when=asyncio.FIRST_COMPLETED)
if item.future in done:
return item.future.result()
if not done:
raise asyncio.TimeoutError()
finally:
progress.cancel()
with contextlib.suppress(asyncio.CancelledError):
await progress
async def dispatch(self, owner, text, turn, *, chat_id="", source_message_id=""): async def dispatch(self, owner, text, turn, *, chat_id="", source_message_id=""):
device = self.store.by_owner(owner) device = self.store.by_owner(owner)
if not device: if not device:
@@ -120,7 +150,7 @@ class Broker:
if self.lifecycle: if self.lifecycle:
try: try:
task = self.lifecycle.create_task(turn,owner,host,device["binding_generation"],session, task = self.lifecycle.create_task(turn,owner,host,device["binding_generation"],session,
self.worker_id,time.time()+self.s.request_timeout,chat_id,source_message_id) self.worker_id,time.time()+self.s.task_tracking_timeout,chat_id,source_message_id)
except ValueError: except ValueError:
self.store.finish(turn,"unbound") self.store.finish(turn,"unbound")
return {"status":"unbound","turn_id":turn,"result":"绑定已变化,任务未下发。"} return {"status":"unbound","turn_id":turn,"result":"绑定已变化,任务未下发。"}
@@ -157,7 +187,10 @@ class Broker:
if task: if task:
self.lifecycle.transition(turn,"inflight","offline") self.lifecycle.transition(turn,"inflight","offline")
return {"status": "offline", "turn_id": turn, "result": "你的电脑当前离线。"} return {"status": "offline", "turn_id": turn, "result": "你的电脑当前离线。"}
result = await asyncio.wait_for(future, timeout=self.s.request_timeout) if task:
result = await self.wait_tracked(self.pending[turn])
else:
result = await asyncio.wait_for(future, timeout=self.s.request_timeout)
self.store.finish(turn, result.get("status","unknown") if task else "completed") self.store.finish(turn, result.get("status","unknown") if task else "completed")
if task and not source_message_id: if task and not source_message_id:
self.lifecycle.transition(turn,"completed","finished") self.lifecycle.transition(turn,"completed","finished")
+59
View File
@@ -355,6 +355,58 @@ def prepare(args):
print("PREPARED: "+str(output)+" (services unchanged; private files must not be shared)") print("PREPARED: "+str(output)+" (services unchanged; private files must not be shared)")
def prepare_update(args):
"""Stage an already-active broker update without generating credentials."""
root=Path(args.plan).resolve()
parent=load_plan(root)
check(root,parent,baseline=False)
built=json.loads((root/'candidate.json').read_text(encoding='utf-8'))
roles={name:service(parent['project'],name) for name in parent['roles']}
if any(roles[name]['image'] != built['image'] for name in ('connector','router')):
raise DeployError('update requires both services on the confirmed parent candidate; inspect state first')
output=Path(args.output)
source=Path(args.source_root).resolve()
deployment=Path(parent['deployment_root']).resolve()
if (not output.is_absolute() or output.exists() or output.is_symlink()
or output.resolve()==deployment or deployment in output.resolve().parents
or any(c in str(output) for c in ('\n','\r','$'))):
raise DeployError('update output must be a new absolute directory outside the live deployment')
if not (source/'relay/connector.py').is_file():
raise DeployError('missing update source')
output=output.resolve()
output.mkdir(mode=0o700)
os.chmod(output,0o700)
plan=json.loads(json.dumps(parent))
plan_id=secrets.token_hex(8)
plan.update(plan_id=plan_id,candidate_image='hainuo-relay:broker-update-'+plan_id,
roles=roles,base_image=built['image'],parent_plan=str(root),update_preserves_credentials=True)
# Copy identical server key material; never rotate/reinitialize while updating code.
shutil.copyfile(root/'credential-keyring.json',output/'credential-keyring.json')
os.chmod(output/'credential-keyring.json',0o600)
if os.name!='nt':os.chown(output/'credential-keyring.json',10001,10001)
overlay=json.loads((root/'compose.broker.json').read_text(encoding='utf-8'))
for name in ('connector','router'):overlay['services'][name]['image']=plan['candidate_image']
for mount in overlay['services']['connector']['volumes']:
if mount['target']=='/run/hainuo-credentials/keyring.json':
mount['source']=str(output/'credential-keyring.json')
write_json(output/'compose.broker.json',overlay)
write_json(output/'compose.rollback.json',{'services':{name:{'image':roles[name]['image']}
for name in ('connector','router','longconn','enrollment-signer')}})
payload=output/'build';payload.mkdir(mode=0o700)
for path in (source/'relay').rglob('*'):
if path.is_symlink():raise DeployError('symlink in update source')
if path.is_file() and path.suffix in {'.py','.js','.css','.html'} and '__pycache__' not in path.parts:
target=payload/path.relative_to(source);target.parent.mkdir(parents=True,exist_ok=True)
shutil.copyfile(path,target);os.chmod(target,0o644)
shutil.copyfile(root/'build/Dockerfile',payload/'Dockerfile')
plan['payload_hashes']={str(p.relative_to(output)):digest(p) for p in payload.rglob('*') if p.is_file()}
plan['prepared_hashes']={name:digest(output/name) for name in
('credential-keyring.json','compose.broker.json','compose.rollback.json')}
write_json(output/'plan.json',plan)
check(output,plan)
print('UPDATE PREPARED: existing credential key material and permissions preserved; no services restarted')
def load_plan(root): def load_plan(root):
plan = json.loads((root/"plan.json").read_text(encoding="utf-8")) plan = json.loads((root/"plan.json").read_text(encoding="utf-8"))
if plan.get("format") != 1 or not re.fullmatch(r"[a-f0-9]{16}",plan.get("plan_id","")): if plan.get("format") != 1 or not re.fullmatch(r"[a-f0-9]{16}",plan.get("plan_id","")):
@@ -568,6 +620,10 @@ def main():
prep.add_argument("--profiles",required=True) prep.add_argument("--profiles",required=True)
prep.add_argument("--output",required=True) prep.add_argument("--output",required=True)
prep.add_argument("--connector-container",default="hainuo-relay-connector-1") prep.add_argument("--connector-container",default="hainuo-relay-connector-1")
update = sub.add_parser("prepare-update")
update.add_argument("--plan",required=True)
update.add_argument("--source-root",default=str(Path(__file__).resolve().parents[1]))
update.add_argument("--output",required=True)
for name in ("repair-router","state","diagnose","check","build","activate","rollback"): for name in ("repair-router","state","diagnose","check","build","activate","rollback"):
cmd = sub.add_parser(name) cmd = sub.add_parser(name)
cmd.add_argument("--plan",required=True) cmd.add_argument("--plan",required=True)
@@ -583,6 +639,9 @@ def main():
if args.action == "prepare": if args.action == "prepare":
prepare(args) prepare(args)
return return
if args.action == "prepare-update":
prepare_update(args)
return
root = Path(args.plan).resolve() root = Path(args.plan).resolve()
plan = load_plan(root) plan = load_plan(root)
if args.action == "repair-router": if args.action == "repair-router":
+37
View File
@@ -510,3 +510,40 @@ def test_overlay_validator_rejects_database_volume_or_network_change(scenario,fi
merged[field] = {"unexpected":{}} merged[field] = {"unexpected":{}}
with pytest.raises(deploy.DeployError): with pytest.raises(deploy.DeployError):
deploy.validate_overlay(base,merged,plan["candidate_image"]) deploy.validate_overlay(base,merged,plan["candidate_image"])
def test_prepare_update_preserves_key_material_and_parent_plan_without_restart(scenario):
parent=prepared(scenario)
root=scenario['output']
original_key=(root/'credential-keyring.json').read_bytes()
original_plan=(root/'plan.json').read_bytes()
image='sha256:'+'a'*64
for name in ('connector','router'):scenario['roles'][name]['image']=image
deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']})
output=root.parent/'update'
args=argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output))
scenario['calls'].clear()
deploy.prepare_update(args)
updated=deploy.load_plan(output)
assert (output/'credential-keyring.json').read_bytes()==original_key
assert (root/'plan.json').read_bytes()==original_plan
assert updated['base_image']==image and updated['roles']['connector']['image']==image
assert updated['candidate_image']!=parent['candidate_image']
assert updated['update_preserves_credentials'] is True
overlay=json.loads((output/'compose.broker.json').read_text())
old=json.loads((root/'compose.broker.json').read_text())
assert overlay['services']['connector']['environment']==old['services']['connector']['environment']
assert not any('up' in call or 'build' in call or 'tag' in call for call in scenario['calls'])
def test_prepare_update_rejects_partial_switch_and_existing_output(scenario):
parent=prepared(scenario);root=scenario['output'];image='sha256:'+'a'*64
deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']})
scenario['roles']['connector']['image']=image
output=root.parent/'update'
args=argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output))
with pytest.raises(deploy.DeployError,match='both services'):deploy.prepare_update(args)
assert not output.exists()
scenario['roles']['router']['image']=image
args.output=str(root)
with pytest.raises(deploy.DeployError,match='new absolute'):deploy.prepare_update(args)
+53
View File
@@ -477,3 +477,56 @@ async def test_admission_denies_dead_tasks_without_exposing_credentials(env):
headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret}, headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret},
json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"}) json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"})
assert response.status_code == 403 and "token" not in response.text.lower() assert response.status_code == 403 and "token" not in response.text.lower()
async def test_native_progress_keeps_task_until_final_and_original_reply_once(routed):
routed['settings'].request_timeout=1
connection=await device(routed)
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
payload=await instruction(routed,connection)
ws,session,_=connection
for _ in range(4):
envelope=seal({'type':'progress','turn_id':payload['turn_id']},payload['reply_key'],
routed['keys']['a'*32],host_id='a'*32,session_id=session)
await ws.send(json.dumps({'type':'reply','turn_id':payload['turn_id'],'envelope':envelope}))
await asyncio.sleep(.4)
assert not routed['platform'].replies
assert routed['store'].lifecycle.task(payload['turn_id'])['status']=='inflight'
await reply(routed,connection,payload,'synthetic final')
await wait_for_reply(routed['platform'],1)
assert routed['platform'].replies==[('om-progress','synthetic final')]
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
await asyncio.sleep(.05)
assert len(routed['platform'].replies)==1
async def test_progress_wait_cannot_extend_hard_deadline_or_hide_idle_failure():
from relay.connector import Broker,Pending
from types import SimpleNamespace
broker=object.__new__(Broker)
broker.s=SimpleNamespace(task_tracking_timeout=.12,request_timeout=.08)
item=Pending(asyncio.get_running_loop().create_future(),None,None,'host','session')
async def pulses():
while True:
item.progress.set()
await asyncio.sleep(.02)
pulse=asyncio.create_task(pulses())
try:
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
assert not item.future.done()
finally:
pulse.cancel()
try:await pulse
except asyncio.CancelledError:pass
broker.s.task_tracking_timeout=1
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
async def test_unbound_or_wrong_session_progress_cannot_keep_task_alive(env):
state=env['store'].lifecycle
state.create_task('progress-bound','ou_a','a'*32,1,'session-a','worker-a',time.time()+60)
assert state.progress('progress-bound','a'*32,'session-a','worker-a')
assert not state.progress('progress-bound','a'*32,'session-b','worker-a')
assert not state.progress('progress-bound','a'*32,'session-a','worker-b')
env['store'].unbind('ou_a')
assert not state.progress('progress-bound','a'*32,'session-a','worker-a')