From e4c3a8ad4e73c13a6be58f550b7853092c099524 Mon Sep 17 00:00:00 2001 From: ex_tianzy Date: Thu, 5 Nov 2026 10:24:55 +0800 Subject: [PATCH] feat(tasks): persist encrypted result handoff and recover received tasks Add signed device-scoped task state/result endpoints and AES-GCM result outbox. Atomically claim original-message replies, reject cross-user/stale/expired tasks, preserve unknown platform receipts without blind replay, and retain ambiguous ciphertext for at most seven days. Maintain legacy synchronous compatibility and clarify stale deployment-plan diagnostics. Validation: ECS suite 227 passed/2 skipped, later outbox guards 12 passed, deployment diagnostics 54 passed. Real ECS/PostgreSQL/platform failure acceptance remains pending. --- deployable/TASK_RECOVERY_PHASE1.md | 25 ++++ deployable/relay/binding_state.py | 2 +- deployable/relay/connector.py | 10 ++ deployable/relay/task_delivery.py | 127 ++++++++++++++++++++ deployable/scripts/broker_deploy.py | 6 +- deployable/tests/test_task_delivery.py | 159 +++++++++++++++++++++++++ 6 files changed, 325 insertions(+), 4 deletions(-) create mode 100644 deployable/TASK_RECOVERY_PHASE1.md create mode 100644 deployable/relay/task_delivery.py create mode 100644 deployable/tests/test_task_delivery.py diff --git a/deployable/TASK_RECOVERY_PHASE1.md b/deployable/TASK_RECOVERY_PHASE1.md new file mode 100644 index 0000000..c01529b --- /dev/null +++ b/deployable/TASK_RECOVERY_PHASE1.md @@ -0,0 +1,25 @@ +# 任务恢复与结果交接:第一阶段 + +已实现:客户端对已收到的任务保存当前用户 DPAPI 加密记录,内容包括精确任务身份、请求摘要、原生 run_id 和未交接结果。重连后先校验当前应用/环境/本人/设备/绑定代次,再用签名 /work/v2/tasks/state 核对服务器任务是否仍有效。已知 run_id 只 GET,不重新执行;未知受理回执不自动重发。任务必须尚在原绝对期限内。 + +客户端将结果签名上传 /work/v2/tasks/result;ECS 按原任务归属和结果摘要幂等接收,使用现有服务器 keyring 对结果作 AES-GCM 加密,AAD 包含环境、应用、本人、设备、绑定代次、任务及摘要。客户端只有收到相同摘要的持久化回执后才清理本地结果。 + +后台 outbox 先原子领取 ready 状态,再回复原 source_message_id。完成后清除密文;发送中断/回执不明标为 unknown,不自动重发。发送前 Token 预检失败可继续重试,但进入 sending 后不作盲目重试。unknown 密文最多保留7天用于受控排障,之后清除。worker 死亡检测带60秒宽限,存活 worker 的发送不被其他 worker 抢占。解绑/撤销会使旧任务失效,未发结果丢弃。 + +兼容性:只有带原消息ID的机器人任务声明 durable-http-v1,旧客户端仍用现有 WebSocket 回包;管理员同步指令维持原返回语义。需要新 ECS 与新客户端配套。schema 为新增表,不重建数据库,不轮换 keyring。 + +范围与限制:本阶段恢复的是客户端已经持久化接收的任务及已知原生 run;ECS 在下发前死亡、客户端尚未保存的任务不自动重放;原生受理请求已发送但没有拿到run_id的任务保持unknown,需人工核对。任务不能跨过期、解绑或绑定代次变化恢复。飞书是否已收消息而回执丢失仍需平台幂等能力验证,本阶段不声称端到端 exactly-once。下一阶段再做完整下发队列、会话调度与用户可见任务查询。 + +## 现场更新(开发测试通过后再执行) + +当前已运行父计划是 /home/cnooc/hainuo-broker-update-20261104-async。待安装包就绪后,先拉最新代码,再用 prepare-update 从这个父计划生成新的独立目录,例如 /home/cnooc/hainuo-broker-update-20261105-recovery。然后对新目录逐条 build、activate、state。不要复用旧的 prepared 迁移父计划,不重新生成密钥,不重做原迁移 prepare。切换后再安装配套1.078客户端。 + +## 必须现场验证 + +1. 只读测试任务执行中断网再恢复,原生任务不得再执行一遍,最终回复原消息。 +2. 任务已拿到run_id后退出/重启桥接,原生进程仍运行时能继续查询;原生也重启时如实报告interrupted而非重跑。 +3. 执行中重启ECS,原绑定和授权不变,客户端恢复上传结果,原消息只回复一次。 +4. 执行中解绑再绑定,旧任务不得推送;第二账号不能读取或提交第一账号任务。 +5. 故障注入结果上传回执丢失/平台发送回执丢失,分别验证重复交接幂等和unknown不重发。 + +开发机使用临时SQLite/FakeRedis、loopback WebSocket/HTTP、合成身份与结果,不代表真实PostgreSQL/Docker/私有化飞书现场验收。测试不读取真实Token,也不调用真实模型业务。 diff --git a/deployable/relay/binding_state.py b/deployable/relay/binding_state.py index 0002d61..2500cf0 100644 --- a/deployable/relay/binding_state.py +++ b/deployable/relay/binding_state.py @@ -88,7 +88,7 @@ class BindingState: (self.environment, self.app_id, device["host_id"], generation)) self.db.execute("""UPDATE work_tasks_v2 SET status=?,updated_at=? WHERE environment=? AND app_id=? AND host_id=? AND binding_generation<=? - AND status IN ('inflight','completed','replying')""", + AND status IN ('inflight','completed','replying','delivery_ready')""", ("revoked" if reason == "revoke" else "unbound", time.time(), self.environment, self.app_id, device["host_id"], generation)) event = uuid.uuid4().hex diff --git a/deployable/relay/connector.py b/deployable/relay/connector.py index 2f11880..8c7c275 100644 --- a/deployable/relay/connector.py +++ b/deployable/relay/connector.py @@ -161,6 +161,8 @@ class Broker: "text": text, "read_only": True, "channel": "im", "reply_key": raw_public(private.public_key())} if task: + if source_message_id: + payload['result_delivery']='durable-http-v1' payload["task_context"] = {key:task[key] for key in ("environment","app_id","owner","host_id","binding_generation", "conversation_id","chat_id","source_message_id")} payload["task_context"].update(task_id=turn,channel="work-dm") @@ -230,6 +232,8 @@ def create_app(settings=None, redis=None, platform=None): store.lifecycle = BindingState(store,s.environment,s.feishu_app_id) broker.lifecycle = store.lifecycle lifecycle = DeviceLifecycle(store.lifecycle,r,broker) + from .task_delivery import TaskDelivery + deliveries = TaskDelivery(oauth,store.lifecycle,broker,f) else: oauth = OAuth(s, r, f) oauth.store = store @@ -249,6 +253,7 @@ def create_app(settings=None, redis=None, platform=None): await r.ping() if lifecycle: await lifecycle.maintain() + await deliveries.recover_interrupted_sends() for row in store.db.execute("SELECT host_id,fingerprint,revoked FROM devices").fetchall(): await r.set("hn:device:" + row["host_id"], row["fingerprint"]) if row["revoked"]: @@ -269,6 +274,8 @@ def create_app(settings=None, redis=None, platform=None): await asyncio.sleep(5) try: await lifecycle.maintain() + await deliveries.recover_interrupted_sends() + await deliveries.flush() except Exception: # No credentials or raw exception text in background logs. pass @@ -292,6 +299,9 @@ def create_app(settings=None, redis=None, platform=None): app = FastAPI(title="Hainuo B1 connector", lifespan=lifespan, docs_url=None, redoc_url=None) app.add_middleware(BodyLimitMiddleware, max_body=s.max_body) app.include_router(oauth.router) + if lifecycle: + app.include_router(deliveries.router) + app.state.deliveries=deliveries app.include_router(notification_router(s, r, store, f, oauth)) from .web_session import WebSessions web_sessions = WebSessions(s, r, store, broker, f) diff --git a/deployable/relay/task_delivery.py b/deployable/relay/task_delivery.py new file mode 100644 index 0000000..e7d4899 --- /dev/null +++ b/deployable/relay/task_delivery.py @@ -0,0 +1,127 @@ +"""Durable, encrypted result handoff; ambiguous platform sends never replay.""" +import hashlib +import time +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import JSONResponse +from pydantic import Field +from .credential_api import DeviceRequest +from .common import no_cache +from .crypto import canonical + + +class TaskRequest(DeviceRequest): + task_id: str = Field(pattern=r'^[A-Za-z0-9_-]{1,128}$') + + +class ResultRequest(TaskRequest): + result: str = Field(max_length=16000, repr=False) + + +class TaskDelivery: + def __init__(self, auth, state, broker, platform): + self.auth,self.state,self.broker,self.platform=auth,state,broker,platform + self.db=state.db + self.keyring=auth.broker.store.keyring + with self.db: + self.db.execute("""CREATE TABLE IF NOT EXISTS work_result_outbox_v2( + task_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL, + owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL, + digest TEXT NOT NULL,key_id TEXT NOT NULL,ciphertext TEXT NOT NULL,worker_id TEXT NOT NULL, + status TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)""") + self.router=APIRouter() + self.router.add_api_route('/work/v2/tasks/state',self.status,methods=['POST']) + self.router.add_api_route('/work/v2/tasks/result',self.submit,methods=['POST']) + + def current(self, body): + self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation) + task=self.state.task(body.task_id) + if (not self.state.task_current(task) or + not task['source_message_id'] or + (task['host_id'],task['owner'],task['binding_generation']) != + (body.host_id,body.binding_owner,body.binding_generation) or + task['status'] in {'unbound','revoked','result_discarded'}): + raise HTTPException(403,'task_not_recoverable') + known=self.db.execute('SELECT task_id FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone() + if not known and task['status'] not in {'inflight','unknown'}: + raise HTTPException(403,'task_not_recoverable') + return task + + def receipt(self,task): + row=self.db.execute('SELECT status,digest FROM work_result_outbox_v2 WHERE task_id=?', + (task['turn_id'],)).fetchone() + return {'task_id':task['turn_id'],'status':row['status'] if row else task['status'], + 'result_received':bool(row),'result_digest':row['digest'] if row else None, + 'deadline':task['deadline']} + + async def status(self,body:TaskRequest,request:Request): + await self.auth.authenticate(body,request) + task=self.current(body) + pending=self.broker.pending.get(body.task_id) + if pending and pending.host==body.host_id: + pending.progress.set() + return no_cache(JSONResponse(self.receipt(task))) + + async def submit(self,body:ResultRequest,request:Request): + await self.auth.authenticate(body,request) + digest=hashlib.sha256(body.result.encode()).hexdigest() + with self.db: + self.state._owner_lock(body.binding_owner) + task=self.current(body) + old=self.db.execute('SELECT digest FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone() + if old and old['digest']!=digest: + raise HTTPException(409,'task_result_conflict') + if not old: + aad=canonical(['work-result-v2',task['environment'],task['app_id'],task['owner'], + task['host_id'],task['binding_generation'],body.task_id,digest]) + key,cipher=self.keyring.encrypt(body.result.encode(),aad) + self.db.execute('INSERT INTO work_result_outbox_v2 VALUES(?,?,?,?,?,?,?,?,?,?,?,?)', + (body.task_id,task['environment'],task['app_id'],task['owner'],task['host_id'], + task['binding_generation'],digest,key,cipher,'','ready',time.time())) + self.db.execute("UPDATE work_tasks_v2 SET status='delivery_ready',updated_at=? WHERE turn_id=?", + (time.time(),body.task_id)) + pending=self.broker.pending.get(body.task_id) + if pending and not pending.future.done(): + pending.future.set_result({'status':'delivery_ready','turn_id':body.task_id,'result':''}) + return no_cache(JSONResponse(self.receipt(task))) + + async def flush(self): + # Bounded retention for encrypted results with ambiguous platform receipt. + with self.db: + self.db.execute("UPDATE work_result_outbox_v2 SET ciphertext='' WHERE environment=? AND app_id=? AND status='unknown' AND updated_at