feat(tasks): persist encrypted result handoff and recover received tasks

Add signed device-scoped task state/result endpoints and AES-GCM result outbox. Atomically claim original-message replies, reject cross-user/stale/expired tasks, preserve unknown platform receipts without blind replay, and retain ambiguous ciphertext for at most seven days. Maintain legacy synchronous compatibility and clarify stale deployment-plan diagnostics. Validation: ECS suite 227 passed/2 skipped, later outbox guards 12 passed, deployment diagnostics 54 passed. Real ECS/PostgreSQL/platform failure acceptance remains pending.
This commit is contained in:
ex_tianzy
2026-11-05 10:24:55 +08:00
parent db116284b1
commit e4c3a8ad4e
6 changed files with 325 additions and 4 deletions
+25
View File
@@ -0,0 +1,25 @@
# 任务恢复与结果交接:第一阶段
已实现:客户端对已收到的任务保存当前用户 DPAPI 加密记录,内容包括精确任务身份、请求摘要、原生 run_id 和未交接结果。重连后先校验当前应用/环境/本人/设备/绑定代次,再用签名 /work/v2/tasks/state 核对服务器任务是否仍有效。已知 run_id 只 GET,不重新执行;未知受理回执不自动重发。任务必须尚在原绝对期限内。
客户端将结果签名上传 /work/v2/tasks/result;ECS 按原任务归属和结果摘要幂等接收,使用现有服务器 keyring 对结果作 AES-GCM 加密,AAD 包含环境、应用、本人、设备、绑定代次、任务及摘要。客户端只有收到相同摘要的持久化回执后才清理本地结果。
后台 outbox 先原子领取 ready 状态,再回复原 source_message_id。完成后清除密文;发送中断/回执不明标为 unknown,不自动重发。发送前 Token 预检失败可继续重试,但进入 sending 后不作盲目重试。unknown 密文最多保留7天用于受控排障,之后清除。worker 死亡检测带60秒宽限,存活 worker 的发送不被其他 worker 抢占。解绑/撤销会使旧任务失效,未发结果丢弃。
兼容性:只有带原消息ID的机器人任务声明 durable-http-v1,旧客户端仍用现有 WebSocket 回包;管理员同步指令维持原返回语义。需要新 ECS 与新客户端配套。schema 为新增表,不重建数据库,不轮换 keyring。
范围与限制:本阶段恢复的是客户端已经持久化接收的任务及已知原生 run;ECS 在下发前死亡、客户端尚未保存的任务不自动重放;原生受理请求已发送但没有拿到run_id的任务保持unknown,需人工核对。任务不能跨过期、解绑或绑定代次变化恢复。飞书是否已收消息而回执丢失仍需平台幂等能力验证,本阶段不声称端到端 exactly-once。下一阶段再做完整下发队列、会话调度与用户可见任务查询。
## 现场更新(开发测试通过后再执行)
当前已运行父计划是 /home/cnooc/hainuo-broker-update-20261104-async。待安装包就绪后,先拉最新代码,再用 prepare-update 从这个父计划生成新的独立目录,例如 /home/cnooc/hainuo-broker-update-20261105-recovery。然后对新目录逐条 build、activate、state。不要复用旧的 prepared 迁移父计划,不重新生成密钥,不重做原迁移 prepare。切换后再安装配套1.078客户端。
## 必须现场验证
1. 只读测试任务执行中断网再恢复,原生任务不得再执行一遍,最终回复原消息。
2. 任务已拿到run_id后退出/重启桥接,原生进程仍运行时能继续查询;原生也重启时如实报告interrupted而非重跑。
3. 执行中重启ECS,原绑定和授权不变,客户端恢复上传结果,原消息只回复一次。
4. 执行中解绑再绑定,旧任务不得推送;第二账号不能读取或提交第一账号任务。
5. 故障注入结果上传回执丢失/平台发送回执丢失,分别验证重复交接幂等和unknown不重发。
开发机使用临时SQLite/FakeRedis、loopback WebSocket/HTTP、合成身份与结果,不代表真实PostgreSQL/Docker/私有化飞书现场验收。测试不读取真实Token,也不调用真实模型业务。
+1 -1
View File
@@ -88,7 +88,7 @@ class BindingState:
(self.environment, self.app_id, device["host_id"], generation))
self.db.execute("""UPDATE work_tasks_v2 SET status=?,updated_at=?
WHERE environment=? AND app_id=? AND host_id=? AND binding_generation<=?
AND status IN ('inflight','completed','replying')""",
AND status IN ('inflight','completed','replying','delivery_ready')""",
("revoked" if reason == "revoke" else "unbound", time.time(), self.environment,
self.app_id, device["host_id"], generation))
event = uuid.uuid4().hex
+10
View File
@@ -161,6 +161,8 @@ class Broker:
"text": text, "read_only": True, "channel": "im",
"reply_key": raw_public(private.public_key())}
if task:
if source_message_id:
payload['result_delivery']='durable-http-v1'
payload["task_context"] = {key:task[key] for key in ("environment","app_id","owner","host_id","binding_generation",
"conversation_id","chat_id","source_message_id")}
payload["task_context"].update(task_id=turn,channel="work-dm")
@@ -230,6 +232,8 @@ def create_app(settings=None, redis=None, platform=None):
store.lifecycle = BindingState(store,s.environment,s.feishu_app_id)
broker.lifecycle = store.lifecycle
lifecycle = DeviceLifecycle(store.lifecycle,r,broker)
from .task_delivery import TaskDelivery
deliveries = TaskDelivery(oauth,store.lifecycle,broker,f)
else:
oauth = OAuth(s, r, f)
oauth.store = store
@@ -249,6 +253,7 @@ def create_app(settings=None, redis=None, platform=None):
await r.ping()
if lifecycle:
await lifecycle.maintain()
await deliveries.recover_interrupted_sends()
for row in store.db.execute("SELECT host_id,fingerprint,revoked FROM devices").fetchall():
await r.set("hn:device:" + row["host_id"], row["fingerprint"])
if row["revoked"]:
@@ -269,6 +274,8 @@ def create_app(settings=None, redis=None, platform=None):
await asyncio.sleep(5)
try:
await lifecycle.maintain()
await deliveries.recover_interrupted_sends()
await deliveries.flush()
except Exception:
# No credentials or raw exception text in background logs.
pass
@@ -292,6 +299,9 @@ def create_app(settings=None, redis=None, platform=None):
app = FastAPI(title="Hainuo B1 connector", lifespan=lifespan, docs_url=None, redoc_url=None)
app.add_middleware(BodyLimitMiddleware, max_body=s.max_body)
app.include_router(oauth.router)
if lifecycle:
app.include_router(deliveries.router)
app.state.deliveries=deliveries
app.include_router(notification_router(s, r, store, f, oauth))
from .web_session import WebSessions
web_sessions = WebSessions(s, r, store, broker, f)
+127
View File
@@ -0,0 +1,127 @@
"""Durable, encrypted result handoff; ambiguous platform sends never replay."""
import hashlib
import time
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import Field
from .credential_api import DeviceRequest
from .common import no_cache
from .crypto import canonical
class TaskRequest(DeviceRequest):
task_id: str = Field(pattern=r'^[A-Za-z0-9_-]{1,128}$')
class ResultRequest(TaskRequest):
result: str = Field(max_length=16000, repr=False)
class TaskDelivery:
def __init__(self, auth, state, broker, platform):
self.auth,self.state,self.broker,self.platform=auth,state,broker,platform
self.db=state.db
self.keyring=auth.broker.store.keyring
with self.db:
self.db.execute("""CREATE TABLE IF NOT EXISTS work_result_outbox_v2(
task_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,
digest TEXT NOT NULL,key_id TEXT NOT NULL,ciphertext TEXT NOT NULL,worker_id TEXT NOT NULL,
status TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)""")
self.router=APIRouter()
self.router.add_api_route('/work/v2/tasks/state',self.status,methods=['POST'])
self.router.add_api_route('/work/v2/tasks/result',self.submit,methods=['POST'])
def current(self, body):
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
task=self.state.task(body.task_id)
if (not self.state.task_current(task) or
not task['source_message_id'] or
(task['host_id'],task['owner'],task['binding_generation']) !=
(body.host_id,body.binding_owner,body.binding_generation) or
task['status'] in {'unbound','revoked','result_discarded'}):
raise HTTPException(403,'task_not_recoverable')
known=self.db.execute('SELECT task_id FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
if not known and task['status'] not in {'inflight','unknown'}:
raise HTTPException(403,'task_not_recoverable')
return task
def receipt(self,task):
row=self.db.execute('SELECT status,digest FROM work_result_outbox_v2 WHERE task_id=?',
(task['turn_id'],)).fetchone()
return {'task_id':task['turn_id'],'status':row['status'] if row else task['status'],
'result_received':bool(row),'result_digest':row['digest'] if row else None,
'deadline':task['deadline']}
async def status(self,body:TaskRequest,request:Request):
await self.auth.authenticate(body,request)
task=self.current(body)
pending=self.broker.pending.get(body.task_id)
if pending and pending.host==body.host_id:
pending.progress.set()
return no_cache(JSONResponse(self.receipt(task)))
async def submit(self,body:ResultRequest,request:Request):
await self.auth.authenticate(body,request)
digest=hashlib.sha256(body.result.encode()).hexdigest()
with self.db:
self.state._owner_lock(body.binding_owner)
task=self.current(body)
old=self.db.execute('SELECT digest FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
if old and old['digest']!=digest:
raise HTTPException(409,'task_result_conflict')
if not old:
aad=canonical(['work-result-v2',task['environment'],task['app_id'],task['owner'],
task['host_id'],task['binding_generation'],body.task_id,digest])
key,cipher=self.keyring.encrypt(body.result.encode(),aad)
self.db.execute('INSERT INTO work_result_outbox_v2 VALUES(?,?,?,?,?,?,?,?,?,?,?,?)',
(body.task_id,task['environment'],task['app_id'],task['owner'],task['host_id'],
task['binding_generation'],digest,key,cipher,'','ready',time.time()))
self.db.execute("UPDATE work_tasks_v2 SET status='delivery_ready',updated_at=? WHERE turn_id=?",
(time.time(),body.task_id))
pending=self.broker.pending.get(body.task_id)
if pending and not pending.future.done():
pending.future.set_result({'status':'delivery_ready','turn_id':body.task_id,'result':''})
return no_cache(JSONResponse(self.receipt(task)))
async def flush(self):
# Bounded retention for encrypted results with ambiguous platform receipt.
with self.db:
self.db.execute("UPDATE work_result_outbox_v2 SET ciphertext='' WHERE environment=? AND app_id=? AND status='unknown' AND updated_at<?",(self.state.environment,self.state.app_id,time.time()-7*86400))
rows=self.db.execute("""SELECT * FROM work_result_outbox_v2 WHERE environment=? AND app_id=?
AND status='ready' ORDER BY updated_at LIMIT 20""",
(self.state.environment,self.state.app_id)).fetchall()
for row in rows:
task=self.state.task(row['task_id'])
if not self.state.task_current(task):
with self.db:self.db.execute("UPDATE work_result_outbox_v2 SET status='discarded',ciphertext='',updated_at=? WHERE task_id=? AND status='ready'",(time.time(),row['task_id']))
continue
# Preflight may await token refresh; revalidate the binding afterwards.
try:await self.platform.tenant_token()
except Exception:continue
with self.db:
self.state._owner_lock(row['owner'])
task=self.state.task(row['task_id'])
if not self.state.task_current(task):continue
claimed=self.db.execute("UPDATE work_result_outbox_v2 SET status='sending',worker_id=?,updated_at=? WHERE task_id=? AND status='ready'",(self.broker.worker_id,time.time(),row['task_id'])).rowcount
if claimed!=1:continue
aad=canonical(['work-result-v2',row['environment'],row['app_id'],row['owner'],
row['host_id'],row['binding_generation'],row['task_id'],row['digest']])
outcome='unknown'
try:
result=self.keyring.decrypt(row['key_id'],row['ciphertext'],aad).decode()
if hashlib.sha256(result.encode()).hexdigest()!=row['digest']:raise ValueError()
await self.platform.reply(task['source_message_id'],result)
outcome='replied'
except Exception:pass
with self.db:
self.db.execute("UPDATE work_result_outbox_v2 SET status=?,ciphertext=?,updated_at=? WHERE task_id=? AND status='sending'",(outcome,'' if outcome=='replied' else row['ciphertext'],time.time(),row['task_id']))
self.db.execute("UPDATE work_tasks_v2 SET status=?,updated_at=? WHERE turn_id=? AND status='delivery_ready'",
('replied' if outcome=='replied' else 'reply_unknown',time.time(),row['task_id']))
async def recover_interrupted_sends(self):
rows=self.db.execute("SELECT task_id,worker_id,updated_at FROM work_result_outbox_v2 WHERE environment=? AND app_id=? AND status='sending'",(self.state.environment,self.state.app_id)).fetchall()
for row in rows:
if time.time()-row['updated_at']<60 or await self.auth.r.exists('hn:task-worker:'+row['worker_id']):continue
with self.db:
self.db.execute("UPDATE work_result_outbox_v2 SET status='unknown',updated_at=? WHERE task_id=? AND status='sending' AND worker_id=?",(time.time(),row['task_id'],row['worker_id']))
+3 -3
View File
@@ -472,7 +472,7 @@ def deployment_state(root,plan):
raw = run(['docker','logs','--tail','100',entry['id']],optional=True)
if raw is not None:
print(' startup signals: '+(', '.join(log_signals(raw)) or 'no recognized error class; raw logs withheld'))
print('STATE inspection only; no files, keys or containers changed. Do not retry activation before reviewing partial switch state.')
print('STATE inspection only; no files, keys or containers changed. Healthy candidate services need no reactivation; inspect mixed/unhealthy states before further actions.')
def check(root, plan, baseline=True, router_key_required=True):
@@ -496,13 +496,13 @@ def check(root, plan, baseline=True, router_key_required=True):
for name, old in plan["roles"].items():
current = service(plan["project"],name,include_stopped=not baseline and name in {"connector","router"})
if (baseline or name not in {"connector","router"}) and current["image"] != old["image"]:
raise DeployError("running service image changed since preparation: "+name)
raise DeployError("running service image changed since preparation: "+name+"; this plan may already be activated. Run state; do not rebuild or reactivate a healthy candidate")
if not baseline and name in {"connector","router"}:
allowed = {old['image']}
if (root/'candidate.json').is_file():
allowed.add(json.loads((root/'candidate.json').read_text(encoding='utf-8'))['image'])
if current['image'] not in allowed:
raise DeployError('unreviewed target image; refusing rollback: '+name)
raise DeployError('unreviewed target image; selected parent plan does not match running '+name+'; use the latest successfully activated plan for a new update. No services changed')
base = run(compose(plan)+["config","--format","json"],True)
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
validate_overlay(base,merged,plan["candidate_image"],router_key_required=router_key_required)
+159
View File
@@ -0,0 +1,159 @@
import asyncio
import time
import pytest
from test_credential_api import env, wire
from relay.task_delivery import TaskDelivery
def task(env,turn='task-durable',owner='ou_a',host='a'*32):
device=env['store'].device(host)
return env['store'].lifecycle.create_task(turn,owner,host,device['binding_generation'],
'old-session','old-worker',time.time()+600,'chat-synthetic','om-synthetic')
async def upload(env,turn='task-durable',text='synthetic-private-result'):
path='/work/v2/tasks/result'
return await env['client'].post(path,json=wire(env,path,task_id=turn,result=text))
async def test_result_is_encrypted_durable_and_delivered_once_after_worker_loss(env):
task(env)
api=env['app'].state.deliveries
first=await upload(env)
assert first.status_code==200 and first.json()['result_received']
row=env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchone()
assert 'synthetic-private-result' not in str(dict(row)) and row['status']=='ready'
# Recreate application delivery service: no process-local future or keys required.
recovered=TaskDelivery(env['api'],env['store'].lifecycle,env['app'].state.broker,env['platform'])
await recovered.flush()
assert env['platform'].replies==[('om-synthetic','synthetic-private-result')]
retry=await upload(env)
assert retry.json()['status']=='replied'
await recovered.flush()
assert len(env['platform'].replies)==1
assert env['store'].db.execute('SELECT ciphertext FROM work_result_outbox_v2').fetchone()['ciphertext']==''
async def test_result_conflict_and_foreign_task_are_rejected(env):
task(env)
assert (await upload(env)).status_code==200
assert (await upload(env,text='different-result')).status_code==409
task(env,'foreign-task','ou_b','b'*32)
assert (await upload(env,'foreign-task')).status_code==403
assert len(env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall())==1
async def test_unbind_before_flush_discards_without_platform_send(env):
task(env)
await upload(env)
env['store'].unbind('ou_a')
await env['app'].state.deliveries.flush()
assert not env['platform'].replies
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='discarded'
async def test_ambiguous_platform_send_never_retries(env,monkeypatch):
task(env);await upload(env)
calls=[]
async def uncertain(*args):
calls.append(args)
raise ConnectionError('synthetic receipt loss')
monkeypatch.setattr(env['platform'],'reply',uncertain)
await env['app'].state.deliveries.flush()
await env['app'].state.deliveries.flush()
assert len(calls)==1
assert (await upload(env)).json()['status']=='unknown'
async def test_live_sender_is_not_retired_by_other_worker_and_dead_sender_not_replayed(env):
task(env);await upload(env)
db=env['store'].db
with db:db.execute("UPDATE work_result_outbox_v2 SET status='sending',worker_id='sender-a',updated_at=0")
await env['redis'].set('hn:task-worker:sender-a','alive',ex=30)
api=env['app'].state.deliveries
await api.recover_interrupted_sends()
assert db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='sending'
await env['redis'].delete('hn:task-worker:sender-a')
await api.recover_interrupted_sends();await api.flush()
assert db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='unknown'
assert not env['platform'].replies
async def test_task_status_uses_signed_device_binding_and_rejects_expired_tasks(env):
task(env)
path='/work/v2/tasks/state'
r=await env['client'].post(path,json=wire(env,path,task_id='task-durable'))
assert r.status_code==200 and not r.json()['result_received']
with env['store'].db:env['store'].db.execute('UPDATE work_tasks_v2 SET deadline=?',(time.time()-1,))
assert (await env['client'].post(path,json=wire(env,path,task_id='task-durable'))).status_code==403
async def test_result_receipt_loss_retry_does_not_reexecute_or_duplicate_and_rebind_blocks(env):
task(env)
assert (await upload(env)).status_code==200
# Lost HTTP receipt: the same payload is retried against durable storage.
assert (await upload(env)).status_code==200
assert len(env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall())==1
env['store'].unbind('ou_a');env['store'].bind('a'*32,'ou_a')
assert (await upload(env)).status_code==403
await env['app'].state.deliveries.flush()
assert not env['platform'].replies
async def test_worker_restart_marks_task_unknown_but_same_binding_can_finish(env):
task(env)
await env['app'].state.lifecycle.maintain()
assert env['store'].lifecycle.task('task-durable')['status']=='unknown'
assert (await upload(env)).status_code==200
await env['app'].state.deliveries.flush()
assert env['platform'].replies==[('om-synthetic','synthetic-private-result')]
async def test_send_preflight_failure_retries_only_before_sending_and_rechecks_unbind(env,monkeypatch):
task(env);await upload(env)
api=env['app'].state.deliveries
original=env['platform'].tenant_token
async def unavailable():raise ConnectionError('synthetic preflight outage')
monkeypatch.setattr(env['platform'],'tenant_token',unavailable)
await api.flush()
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='ready'
async def invalidate():
env['store'].unbind('ou_a')
return 'fake-token'
monkeypatch.setattr(env['platform'],'tenant_token',invalidate)
await api.flush()
assert not env['platform'].replies
async def test_parallel_outbox_sweepers_claim_only_one_platform_send(env,monkeypatch):
task(env);await upload(env)
api=env['app'].state.deliveries
sends=[];entered=asyncio.Event();release=asyncio.Event()
async def slow(message,text):
sends.append((message,text));entered.set();await release.wait()
monkeypatch.setattr(env['platform'],'reply',slow)
first=asyncio.create_task(api.flush())
await entered.wait()
await api.flush()
assert len(sends)==1
release.set();await first
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='replied'
async def test_ambiguous_result_is_encrypted_retained_then_expires_without_resend(env,monkeypatch):
task(env);await upload(env)
async def fail(*args):raise ConnectionError('synthetic ambiguous')
monkeypatch.setattr(env['platform'],'reply',fail)
api=env['app'].state.deliveries;await api.flush()
row=env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchone()
assert row['status']=='unknown' and row['ciphertext'] and 'synthetic-private-result' not in str(dict(row))
with env['store'].db:env['store'].db.execute('UPDATE work_result_outbox_v2 SET updated_at=?',(time.time()-8*86400,))
await api.flush()
assert env['store'].db.execute('SELECT ciphertext FROM work_result_outbox_v2').fetchone()['ciphertext']==''
async def test_completed_legacy_reply_cannot_be_uploaded_again_via_new_protocol(env):
task(env)
with env['store'].db:env['store'].db.execute("UPDATE work_tasks_v2 SET status='replied'")
assert (await upload(env)).status_code==403
assert not env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall()