feat(tasks): persist encrypted result handoff and recover received tasks
Add signed device-scoped task state/result endpoints and AES-GCM result outbox. Atomically claim original-message replies, reject cross-user/stale/expired tasks, preserve unknown platform receipts without blind replay, and retain ambiguous ciphertext for at most seven days. Maintain legacy synchronous compatibility and clarify stale deployment-plan diagnostics. Validation: ECS suite 227 passed/2 skipped, later outbox guards 12 passed, deployment diagnostics 54 passed. Real ECS/PostgreSQL/platform failure acceptance remains pending.
This commit is contained in:
@@ -0,0 +1,25 @@
|
|||||||
|
# 任务恢复与结果交接:第一阶段
|
||||||
|
|
||||||
|
已实现:客户端对已收到的任务保存当前用户 DPAPI 加密记录,内容包括精确任务身份、请求摘要、原生 run_id 和未交接结果。重连后先校验当前应用/环境/本人/设备/绑定代次,再用签名 /work/v2/tasks/state 核对服务器任务是否仍有效。已知 run_id 只 GET,不重新执行;未知受理回执不自动重发。任务必须尚在原绝对期限内。
|
||||||
|
|
||||||
|
客户端将结果签名上传 /work/v2/tasks/result;ECS 按原任务归属和结果摘要幂等接收,使用现有服务器 keyring 对结果作 AES-GCM 加密,AAD 包含环境、应用、本人、设备、绑定代次、任务及摘要。客户端只有收到相同摘要的持久化回执后才清理本地结果。
|
||||||
|
|
||||||
|
后台 outbox 先原子领取 ready 状态,再回复原 source_message_id。完成后清除密文;发送中断/回执不明标为 unknown,不自动重发。发送前 Token 预检失败可继续重试,但进入 sending 后不作盲目重试。unknown 密文最多保留7天用于受控排障,之后清除。worker 死亡检测带60秒宽限,存活 worker 的发送不被其他 worker 抢占。解绑/撤销会使旧任务失效,未发结果丢弃。
|
||||||
|
|
||||||
|
兼容性:只有带原消息ID的机器人任务声明 durable-http-v1,旧客户端仍用现有 WebSocket 回包;管理员同步指令维持原返回语义。需要新 ECS 与新客户端配套。schema 为新增表,不重建数据库,不轮换 keyring。
|
||||||
|
|
||||||
|
范围与限制:本阶段恢复的是客户端已经持久化接收的任务及已知原生 run;ECS 在下发前死亡、客户端尚未保存的任务不自动重放;原生受理请求已发送但没有拿到run_id的任务保持unknown,需人工核对。任务不能跨过期、解绑或绑定代次变化恢复。飞书是否已收消息而回执丢失仍需平台幂等能力验证,本阶段不声称端到端 exactly-once。下一阶段再做完整下发队列、会话调度与用户可见任务查询。
|
||||||
|
|
||||||
|
## 现场更新(开发测试通过后再执行)
|
||||||
|
|
||||||
|
当前已运行父计划是 /home/cnooc/hainuo-broker-update-20261104-async。待安装包就绪后,先拉最新代码,再用 prepare-update 从这个父计划生成新的独立目录,例如 /home/cnooc/hainuo-broker-update-20261105-recovery。然后对新目录逐条 build、activate、state。不要复用旧的 prepared 迁移父计划,不重新生成密钥,不重做原迁移 prepare。切换后再安装配套1.078客户端。
|
||||||
|
|
||||||
|
## 必须现场验证
|
||||||
|
|
||||||
|
1. 只读测试任务执行中断网再恢复,原生任务不得再执行一遍,最终回复原消息。
|
||||||
|
2. 任务已拿到run_id后退出/重启桥接,原生进程仍运行时能继续查询;原生也重启时如实报告interrupted而非重跑。
|
||||||
|
3. 执行中重启ECS,原绑定和授权不变,客户端恢复上传结果,原消息只回复一次。
|
||||||
|
4. 执行中解绑再绑定,旧任务不得推送;第二账号不能读取或提交第一账号任务。
|
||||||
|
5. 故障注入结果上传回执丢失/平台发送回执丢失,分别验证重复交接幂等和unknown不重发。
|
||||||
|
|
||||||
|
开发机使用临时SQLite/FakeRedis、loopback WebSocket/HTTP、合成身份与结果,不代表真实PostgreSQL/Docker/私有化飞书现场验收。测试不读取真实Token,也不调用真实模型业务。
|
||||||
@@ -88,7 +88,7 @@ class BindingState:
|
|||||||
(self.environment, self.app_id, device["host_id"], generation))
|
(self.environment, self.app_id, device["host_id"], generation))
|
||||||
self.db.execute("""UPDATE work_tasks_v2 SET status=?,updated_at=?
|
self.db.execute("""UPDATE work_tasks_v2 SET status=?,updated_at=?
|
||||||
WHERE environment=? AND app_id=? AND host_id=? AND binding_generation<=?
|
WHERE environment=? AND app_id=? AND host_id=? AND binding_generation<=?
|
||||||
AND status IN ('inflight','completed','replying')""",
|
AND status IN ('inflight','completed','replying','delivery_ready')""",
|
||||||
("revoked" if reason == "revoke" else "unbound", time.time(), self.environment,
|
("revoked" if reason == "revoke" else "unbound", time.time(), self.environment,
|
||||||
self.app_id, device["host_id"], generation))
|
self.app_id, device["host_id"], generation))
|
||||||
event = uuid.uuid4().hex
|
event = uuid.uuid4().hex
|
||||||
|
|||||||
@@ -161,6 +161,8 @@ class Broker:
|
|||||||
"text": text, "read_only": True, "channel": "im",
|
"text": text, "read_only": True, "channel": "im",
|
||||||
"reply_key": raw_public(private.public_key())}
|
"reply_key": raw_public(private.public_key())}
|
||||||
if task:
|
if task:
|
||||||
|
if source_message_id:
|
||||||
|
payload['result_delivery']='durable-http-v1'
|
||||||
payload["task_context"] = {key:task[key] for key in ("environment","app_id","owner","host_id","binding_generation",
|
payload["task_context"] = {key:task[key] for key in ("environment","app_id","owner","host_id","binding_generation",
|
||||||
"conversation_id","chat_id","source_message_id")}
|
"conversation_id","chat_id","source_message_id")}
|
||||||
payload["task_context"].update(task_id=turn,channel="work-dm")
|
payload["task_context"].update(task_id=turn,channel="work-dm")
|
||||||
@@ -230,6 +232,8 @@ def create_app(settings=None, redis=None, platform=None):
|
|||||||
store.lifecycle = BindingState(store,s.environment,s.feishu_app_id)
|
store.lifecycle = BindingState(store,s.environment,s.feishu_app_id)
|
||||||
broker.lifecycle = store.lifecycle
|
broker.lifecycle = store.lifecycle
|
||||||
lifecycle = DeviceLifecycle(store.lifecycle,r,broker)
|
lifecycle = DeviceLifecycle(store.lifecycle,r,broker)
|
||||||
|
from .task_delivery import TaskDelivery
|
||||||
|
deliveries = TaskDelivery(oauth,store.lifecycle,broker,f)
|
||||||
else:
|
else:
|
||||||
oauth = OAuth(s, r, f)
|
oauth = OAuth(s, r, f)
|
||||||
oauth.store = store
|
oauth.store = store
|
||||||
@@ -249,6 +253,7 @@ def create_app(settings=None, redis=None, platform=None):
|
|||||||
await r.ping()
|
await r.ping()
|
||||||
if lifecycle:
|
if lifecycle:
|
||||||
await lifecycle.maintain()
|
await lifecycle.maintain()
|
||||||
|
await deliveries.recover_interrupted_sends()
|
||||||
for row in store.db.execute("SELECT host_id,fingerprint,revoked FROM devices").fetchall():
|
for row in store.db.execute("SELECT host_id,fingerprint,revoked FROM devices").fetchall():
|
||||||
await r.set("hn:device:" + row["host_id"], row["fingerprint"])
|
await r.set("hn:device:" + row["host_id"], row["fingerprint"])
|
||||||
if row["revoked"]:
|
if row["revoked"]:
|
||||||
@@ -269,6 +274,8 @@ def create_app(settings=None, redis=None, platform=None):
|
|||||||
await asyncio.sleep(5)
|
await asyncio.sleep(5)
|
||||||
try:
|
try:
|
||||||
await lifecycle.maintain()
|
await lifecycle.maintain()
|
||||||
|
await deliveries.recover_interrupted_sends()
|
||||||
|
await deliveries.flush()
|
||||||
except Exception:
|
except Exception:
|
||||||
# No credentials or raw exception text in background logs.
|
# No credentials or raw exception text in background logs.
|
||||||
pass
|
pass
|
||||||
@@ -292,6 +299,9 @@ def create_app(settings=None, redis=None, platform=None):
|
|||||||
app = FastAPI(title="Hainuo B1 connector", lifespan=lifespan, docs_url=None, redoc_url=None)
|
app = FastAPI(title="Hainuo B1 connector", lifespan=lifespan, docs_url=None, redoc_url=None)
|
||||||
app.add_middleware(BodyLimitMiddleware, max_body=s.max_body)
|
app.add_middleware(BodyLimitMiddleware, max_body=s.max_body)
|
||||||
app.include_router(oauth.router)
|
app.include_router(oauth.router)
|
||||||
|
if lifecycle:
|
||||||
|
app.include_router(deliveries.router)
|
||||||
|
app.state.deliveries=deliveries
|
||||||
app.include_router(notification_router(s, r, store, f, oauth))
|
app.include_router(notification_router(s, r, store, f, oauth))
|
||||||
from .web_session import WebSessions
|
from .web_session import WebSessions
|
||||||
web_sessions = WebSessions(s, r, store, broker, f)
|
web_sessions = WebSessions(s, r, store, broker, f)
|
||||||
|
|||||||
@@ -0,0 +1,127 @@
|
|||||||
|
"""Durable, encrypted result handoff; ambiguous platform sends never replay."""
|
||||||
|
import hashlib
|
||||||
|
import time
|
||||||
|
from fastapi import APIRouter, HTTPException, Request
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
from pydantic import Field
|
||||||
|
from .credential_api import DeviceRequest
|
||||||
|
from .common import no_cache
|
||||||
|
from .crypto import canonical
|
||||||
|
|
||||||
|
|
||||||
|
class TaskRequest(DeviceRequest):
|
||||||
|
task_id: str = Field(pattern=r'^[A-Za-z0-9_-]{1,128}$')
|
||||||
|
|
||||||
|
|
||||||
|
class ResultRequest(TaskRequest):
|
||||||
|
result: str = Field(max_length=16000, repr=False)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskDelivery:
|
||||||
|
def __init__(self, auth, state, broker, platform):
|
||||||
|
self.auth,self.state,self.broker,self.platform=auth,state,broker,platform
|
||||||
|
self.db=state.db
|
||||||
|
self.keyring=auth.broker.store.keyring
|
||||||
|
with self.db:
|
||||||
|
self.db.execute("""CREATE TABLE IF NOT EXISTS work_result_outbox_v2(
|
||||||
|
task_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
|
||||||
|
owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,
|
||||||
|
digest TEXT NOT NULL,key_id TEXT NOT NULL,ciphertext TEXT NOT NULL,worker_id TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)""")
|
||||||
|
self.router=APIRouter()
|
||||||
|
self.router.add_api_route('/work/v2/tasks/state',self.status,methods=['POST'])
|
||||||
|
self.router.add_api_route('/work/v2/tasks/result',self.submit,methods=['POST'])
|
||||||
|
|
||||||
|
def current(self, body):
|
||||||
|
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
|
||||||
|
task=self.state.task(body.task_id)
|
||||||
|
if (not self.state.task_current(task) or
|
||||||
|
not task['source_message_id'] or
|
||||||
|
(task['host_id'],task['owner'],task['binding_generation']) !=
|
||||||
|
(body.host_id,body.binding_owner,body.binding_generation) or
|
||||||
|
task['status'] in {'unbound','revoked','result_discarded'}):
|
||||||
|
raise HTTPException(403,'task_not_recoverable')
|
||||||
|
known=self.db.execute('SELECT task_id FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
|
||||||
|
if not known and task['status'] not in {'inflight','unknown'}:
|
||||||
|
raise HTTPException(403,'task_not_recoverable')
|
||||||
|
return task
|
||||||
|
|
||||||
|
def receipt(self,task):
|
||||||
|
row=self.db.execute('SELECT status,digest FROM work_result_outbox_v2 WHERE task_id=?',
|
||||||
|
(task['turn_id'],)).fetchone()
|
||||||
|
return {'task_id':task['turn_id'],'status':row['status'] if row else task['status'],
|
||||||
|
'result_received':bool(row),'result_digest':row['digest'] if row else None,
|
||||||
|
'deadline':task['deadline']}
|
||||||
|
|
||||||
|
async def status(self,body:TaskRequest,request:Request):
|
||||||
|
await self.auth.authenticate(body,request)
|
||||||
|
task=self.current(body)
|
||||||
|
pending=self.broker.pending.get(body.task_id)
|
||||||
|
if pending and pending.host==body.host_id:
|
||||||
|
pending.progress.set()
|
||||||
|
return no_cache(JSONResponse(self.receipt(task)))
|
||||||
|
|
||||||
|
async def submit(self,body:ResultRequest,request:Request):
|
||||||
|
await self.auth.authenticate(body,request)
|
||||||
|
digest=hashlib.sha256(body.result.encode()).hexdigest()
|
||||||
|
with self.db:
|
||||||
|
self.state._owner_lock(body.binding_owner)
|
||||||
|
task=self.current(body)
|
||||||
|
old=self.db.execute('SELECT digest FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
|
||||||
|
if old and old['digest']!=digest:
|
||||||
|
raise HTTPException(409,'task_result_conflict')
|
||||||
|
if not old:
|
||||||
|
aad=canonical(['work-result-v2',task['environment'],task['app_id'],task['owner'],
|
||||||
|
task['host_id'],task['binding_generation'],body.task_id,digest])
|
||||||
|
key,cipher=self.keyring.encrypt(body.result.encode(),aad)
|
||||||
|
self.db.execute('INSERT INTO work_result_outbox_v2 VALUES(?,?,?,?,?,?,?,?,?,?,?,?)',
|
||||||
|
(body.task_id,task['environment'],task['app_id'],task['owner'],task['host_id'],
|
||||||
|
task['binding_generation'],digest,key,cipher,'','ready',time.time()))
|
||||||
|
self.db.execute("UPDATE work_tasks_v2 SET status='delivery_ready',updated_at=? WHERE turn_id=?",
|
||||||
|
(time.time(),body.task_id))
|
||||||
|
pending=self.broker.pending.get(body.task_id)
|
||||||
|
if pending and not pending.future.done():
|
||||||
|
pending.future.set_result({'status':'delivery_ready','turn_id':body.task_id,'result':''})
|
||||||
|
return no_cache(JSONResponse(self.receipt(task)))
|
||||||
|
|
||||||
|
async def flush(self):
|
||||||
|
# Bounded retention for encrypted results with ambiguous platform receipt.
|
||||||
|
with self.db:
|
||||||
|
self.db.execute("UPDATE work_result_outbox_v2 SET ciphertext='' WHERE environment=? AND app_id=? AND status='unknown' AND updated_at<?",(self.state.environment,self.state.app_id,time.time()-7*86400))
|
||||||
|
rows=self.db.execute("""SELECT * FROM work_result_outbox_v2 WHERE environment=? AND app_id=?
|
||||||
|
AND status='ready' ORDER BY updated_at LIMIT 20""",
|
||||||
|
(self.state.environment,self.state.app_id)).fetchall()
|
||||||
|
for row in rows:
|
||||||
|
task=self.state.task(row['task_id'])
|
||||||
|
if not self.state.task_current(task):
|
||||||
|
with self.db:self.db.execute("UPDATE work_result_outbox_v2 SET status='discarded',ciphertext='',updated_at=? WHERE task_id=? AND status='ready'",(time.time(),row['task_id']))
|
||||||
|
continue
|
||||||
|
# Preflight may await token refresh; revalidate the binding afterwards.
|
||||||
|
try:await self.platform.tenant_token()
|
||||||
|
except Exception:continue
|
||||||
|
with self.db:
|
||||||
|
self.state._owner_lock(row['owner'])
|
||||||
|
task=self.state.task(row['task_id'])
|
||||||
|
if not self.state.task_current(task):continue
|
||||||
|
claimed=self.db.execute("UPDATE work_result_outbox_v2 SET status='sending',worker_id=?,updated_at=? WHERE task_id=? AND status='ready'",(self.broker.worker_id,time.time(),row['task_id'])).rowcount
|
||||||
|
if claimed!=1:continue
|
||||||
|
aad=canonical(['work-result-v2',row['environment'],row['app_id'],row['owner'],
|
||||||
|
row['host_id'],row['binding_generation'],row['task_id'],row['digest']])
|
||||||
|
outcome='unknown'
|
||||||
|
try:
|
||||||
|
result=self.keyring.decrypt(row['key_id'],row['ciphertext'],aad).decode()
|
||||||
|
if hashlib.sha256(result.encode()).hexdigest()!=row['digest']:raise ValueError()
|
||||||
|
await self.platform.reply(task['source_message_id'],result)
|
||||||
|
outcome='replied'
|
||||||
|
except Exception:pass
|
||||||
|
with self.db:
|
||||||
|
self.db.execute("UPDATE work_result_outbox_v2 SET status=?,ciphertext=?,updated_at=? WHERE task_id=? AND status='sending'",(outcome,'' if outcome=='replied' else row['ciphertext'],time.time(),row['task_id']))
|
||||||
|
self.db.execute("UPDATE work_tasks_v2 SET status=?,updated_at=? WHERE turn_id=? AND status='delivery_ready'",
|
||||||
|
('replied' if outcome=='replied' else 'reply_unknown',time.time(),row['task_id']))
|
||||||
|
|
||||||
|
async def recover_interrupted_sends(self):
|
||||||
|
rows=self.db.execute("SELECT task_id,worker_id,updated_at FROM work_result_outbox_v2 WHERE environment=? AND app_id=? AND status='sending'",(self.state.environment,self.state.app_id)).fetchall()
|
||||||
|
for row in rows:
|
||||||
|
if time.time()-row['updated_at']<60 or await self.auth.r.exists('hn:task-worker:'+row['worker_id']):continue
|
||||||
|
with self.db:
|
||||||
|
self.db.execute("UPDATE work_result_outbox_v2 SET status='unknown',updated_at=? WHERE task_id=? AND status='sending' AND worker_id=?",(time.time(),row['task_id'],row['worker_id']))
|
||||||
@@ -472,7 +472,7 @@ def deployment_state(root,plan):
|
|||||||
raw = run(['docker','logs','--tail','100',entry['id']],optional=True)
|
raw = run(['docker','logs','--tail','100',entry['id']],optional=True)
|
||||||
if raw is not None:
|
if raw is not None:
|
||||||
print(' startup signals: '+(', '.join(log_signals(raw)) or 'no recognized error class; raw logs withheld'))
|
print(' startup signals: '+(', '.join(log_signals(raw)) or 'no recognized error class; raw logs withheld'))
|
||||||
print('STATE inspection only; no files, keys or containers changed. Do not retry activation before reviewing partial switch state.')
|
print('STATE inspection only; no files, keys or containers changed. Healthy candidate services need no reactivation; inspect mixed/unhealthy states before further actions.')
|
||||||
|
|
||||||
|
|
||||||
def check(root, plan, baseline=True, router_key_required=True):
|
def check(root, plan, baseline=True, router_key_required=True):
|
||||||
@@ -496,13 +496,13 @@ def check(root, plan, baseline=True, router_key_required=True):
|
|||||||
for name, old in plan["roles"].items():
|
for name, old in plan["roles"].items():
|
||||||
current = service(plan["project"],name,include_stopped=not baseline and name in {"connector","router"})
|
current = service(plan["project"],name,include_stopped=not baseline and name in {"connector","router"})
|
||||||
if (baseline or name not in {"connector","router"}) and current["image"] != old["image"]:
|
if (baseline or name not in {"connector","router"}) and current["image"] != old["image"]:
|
||||||
raise DeployError("running service image changed since preparation: "+name)
|
raise DeployError("running service image changed since preparation: "+name+"; this plan may already be activated. Run state; do not rebuild or reactivate a healthy candidate")
|
||||||
if not baseline and name in {"connector","router"}:
|
if not baseline and name in {"connector","router"}:
|
||||||
allowed = {old['image']}
|
allowed = {old['image']}
|
||||||
if (root/'candidate.json').is_file():
|
if (root/'candidate.json').is_file():
|
||||||
allowed.add(json.loads((root/'candidate.json').read_text(encoding='utf-8'))['image'])
|
allowed.add(json.loads((root/'candidate.json').read_text(encoding='utf-8'))['image'])
|
||||||
if current['image'] not in allowed:
|
if current['image'] not in allowed:
|
||||||
raise DeployError('unreviewed target image; refusing rollback: '+name)
|
raise DeployError('unreviewed target image; selected parent plan does not match running '+name+'; use the latest successfully activated plan for a new update. No services changed')
|
||||||
base = run(compose(plan)+["config","--format","json"],True)
|
base = run(compose(plan)+["config","--format","json"],True)
|
||||||
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
|
merged = run(compose(plan,root/"compose.broker.json")+["config","--format","json"],True)
|
||||||
validate_overlay(base,merged,plan["candidate_image"],router_key_required=router_key_required)
|
validate_overlay(base,merged,plan["candidate_image"],router_key_required=router_key_required)
|
||||||
|
|||||||
@@ -0,0 +1,159 @@
|
|||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
import pytest
|
||||||
|
from test_credential_api import env, wire
|
||||||
|
from relay.task_delivery import TaskDelivery
|
||||||
|
|
||||||
|
|
||||||
|
def task(env,turn='task-durable',owner='ou_a',host='a'*32):
|
||||||
|
device=env['store'].device(host)
|
||||||
|
return env['store'].lifecycle.create_task(turn,owner,host,device['binding_generation'],
|
||||||
|
'old-session','old-worker',time.time()+600,'chat-synthetic','om-synthetic')
|
||||||
|
|
||||||
|
|
||||||
|
async def upload(env,turn='task-durable',text='synthetic-private-result'):
|
||||||
|
path='/work/v2/tasks/result'
|
||||||
|
return await env['client'].post(path,json=wire(env,path,task_id=turn,result=text))
|
||||||
|
|
||||||
|
|
||||||
|
async def test_result_is_encrypted_durable_and_delivered_once_after_worker_loss(env):
|
||||||
|
task(env)
|
||||||
|
api=env['app'].state.deliveries
|
||||||
|
first=await upload(env)
|
||||||
|
assert first.status_code==200 and first.json()['result_received']
|
||||||
|
row=env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchone()
|
||||||
|
assert 'synthetic-private-result' not in str(dict(row)) and row['status']=='ready'
|
||||||
|
# Recreate application delivery service: no process-local future or keys required.
|
||||||
|
recovered=TaskDelivery(env['api'],env['store'].lifecycle,env['app'].state.broker,env['platform'])
|
||||||
|
await recovered.flush()
|
||||||
|
assert env['platform'].replies==[('om-synthetic','synthetic-private-result')]
|
||||||
|
retry=await upload(env)
|
||||||
|
assert retry.json()['status']=='replied'
|
||||||
|
await recovered.flush()
|
||||||
|
assert len(env['platform'].replies)==1
|
||||||
|
assert env['store'].db.execute('SELECT ciphertext FROM work_result_outbox_v2').fetchone()['ciphertext']==''
|
||||||
|
|
||||||
|
|
||||||
|
async def test_result_conflict_and_foreign_task_are_rejected(env):
|
||||||
|
task(env)
|
||||||
|
assert (await upload(env)).status_code==200
|
||||||
|
assert (await upload(env,text='different-result')).status_code==409
|
||||||
|
task(env,'foreign-task','ou_b','b'*32)
|
||||||
|
assert (await upload(env,'foreign-task')).status_code==403
|
||||||
|
assert len(env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall())==1
|
||||||
|
|
||||||
|
|
||||||
|
async def test_unbind_before_flush_discards_without_platform_send(env):
|
||||||
|
task(env)
|
||||||
|
await upload(env)
|
||||||
|
env['store'].unbind('ou_a')
|
||||||
|
await env['app'].state.deliveries.flush()
|
||||||
|
assert not env['platform'].replies
|
||||||
|
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='discarded'
|
||||||
|
|
||||||
|
|
||||||
|
async def test_ambiguous_platform_send_never_retries(env,monkeypatch):
|
||||||
|
task(env);await upload(env)
|
||||||
|
calls=[]
|
||||||
|
async def uncertain(*args):
|
||||||
|
calls.append(args)
|
||||||
|
raise ConnectionError('synthetic receipt loss')
|
||||||
|
monkeypatch.setattr(env['platform'],'reply',uncertain)
|
||||||
|
await env['app'].state.deliveries.flush()
|
||||||
|
await env['app'].state.deliveries.flush()
|
||||||
|
assert len(calls)==1
|
||||||
|
assert (await upload(env)).json()['status']=='unknown'
|
||||||
|
|
||||||
|
|
||||||
|
async def test_live_sender_is_not_retired_by_other_worker_and_dead_sender_not_replayed(env):
|
||||||
|
task(env);await upload(env)
|
||||||
|
db=env['store'].db
|
||||||
|
with db:db.execute("UPDATE work_result_outbox_v2 SET status='sending',worker_id='sender-a',updated_at=0")
|
||||||
|
await env['redis'].set('hn:task-worker:sender-a','alive',ex=30)
|
||||||
|
api=env['app'].state.deliveries
|
||||||
|
await api.recover_interrupted_sends()
|
||||||
|
assert db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='sending'
|
||||||
|
await env['redis'].delete('hn:task-worker:sender-a')
|
||||||
|
await api.recover_interrupted_sends();await api.flush()
|
||||||
|
assert db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='unknown'
|
||||||
|
assert not env['platform'].replies
|
||||||
|
|
||||||
|
|
||||||
|
async def test_task_status_uses_signed_device_binding_and_rejects_expired_tasks(env):
|
||||||
|
task(env)
|
||||||
|
path='/work/v2/tasks/state'
|
||||||
|
r=await env['client'].post(path,json=wire(env,path,task_id='task-durable'))
|
||||||
|
assert r.status_code==200 and not r.json()['result_received']
|
||||||
|
with env['store'].db:env['store'].db.execute('UPDATE work_tasks_v2 SET deadline=?',(time.time()-1,))
|
||||||
|
assert (await env['client'].post(path,json=wire(env,path,task_id='task-durable'))).status_code==403
|
||||||
|
|
||||||
|
|
||||||
|
async def test_result_receipt_loss_retry_does_not_reexecute_or_duplicate_and_rebind_blocks(env):
|
||||||
|
task(env)
|
||||||
|
assert (await upload(env)).status_code==200
|
||||||
|
# Lost HTTP receipt: the same payload is retried against durable storage.
|
||||||
|
assert (await upload(env)).status_code==200
|
||||||
|
assert len(env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall())==1
|
||||||
|
env['store'].unbind('ou_a');env['store'].bind('a'*32,'ou_a')
|
||||||
|
assert (await upload(env)).status_code==403
|
||||||
|
await env['app'].state.deliveries.flush()
|
||||||
|
assert not env['platform'].replies
|
||||||
|
|
||||||
|
|
||||||
|
async def test_worker_restart_marks_task_unknown_but_same_binding_can_finish(env):
|
||||||
|
task(env)
|
||||||
|
await env['app'].state.lifecycle.maintain()
|
||||||
|
assert env['store'].lifecycle.task('task-durable')['status']=='unknown'
|
||||||
|
assert (await upload(env)).status_code==200
|
||||||
|
await env['app'].state.deliveries.flush()
|
||||||
|
assert env['platform'].replies==[('om-synthetic','synthetic-private-result')]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_send_preflight_failure_retries_only_before_sending_and_rechecks_unbind(env,monkeypatch):
|
||||||
|
task(env);await upload(env)
|
||||||
|
api=env['app'].state.deliveries
|
||||||
|
original=env['platform'].tenant_token
|
||||||
|
async def unavailable():raise ConnectionError('synthetic preflight outage')
|
||||||
|
monkeypatch.setattr(env['platform'],'tenant_token',unavailable)
|
||||||
|
await api.flush()
|
||||||
|
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='ready'
|
||||||
|
async def invalidate():
|
||||||
|
env['store'].unbind('ou_a')
|
||||||
|
return 'fake-token'
|
||||||
|
monkeypatch.setattr(env['platform'],'tenant_token',invalidate)
|
||||||
|
await api.flush()
|
||||||
|
assert not env['platform'].replies
|
||||||
|
|
||||||
|
|
||||||
|
async def test_parallel_outbox_sweepers_claim_only_one_platform_send(env,monkeypatch):
|
||||||
|
task(env);await upload(env)
|
||||||
|
api=env['app'].state.deliveries
|
||||||
|
sends=[];entered=asyncio.Event();release=asyncio.Event()
|
||||||
|
async def slow(message,text):
|
||||||
|
sends.append((message,text));entered.set();await release.wait()
|
||||||
|
monkeypatch.setattr(env['platform'],'reply',slow)
|
||||||
|
first=asyncio.create_task(api.flush())
|
||||||
|
await entered.wait()
|
||||||
|
await api.flush()
|
||||||
|
assert len(sends)==1
|
||||||
|
release.set();await first
|
||||||
|
assert env['store'].db.execute('SELECT status FROM work_result_outbox_v2').fetchone()['status']=='replied'
|
||||||
|
|
||||||
|
|
||||||
|
async def test_ambiguous_result_is_encrypted_retained_then_expires_without_resend(env,monkeypatch):
|
||||||
|
task(env);await upload(env)
|
||||||
|
async def fail(*args):raise ConnectionError('synthetic ambiguous')
|
||||||
|
monkeypatch.setattr(env['platform'],'reply',fail)
|
||||||
|
api=env['app'].state.deliveries;await api.flush()
|
||||||
|
row=env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchone()
|
||||||
|
assert row['status']=='unknown' and row['ciphertext'] and 'synthetic-private-result' not in str(dict(row))
|
||||||
|
with env['store'].db:env['store'].db.execute('UPDATE work_result_outbox_v2 SET updated_at=?',(time.time()-8*86400,))
|
||||||
|
await api.flush()
|
||||||
|
assert env['store'].db.execute('SELECT ciphertext FROM work_result_outbox_v2').fetchone()['ciphertext']==''
|
||||||
|
|
||||||
|
|
||||||
|
async def test_completed_legacy_reply_cannot_be_uploaded_again_via_new_protocol(env):
|
||||||
|
task(env)
|
||||||
|
with env['store'].db:env['store'].db.execute("UPDATE work_tasks_v2 SET status='replied'")
|
||||||
|
assert (await upload(env)).status_code==403
|
||||||
|
assert not env['store'].db.execute('SELECT * FROM work_result_outbox_v2').fetchall()
|
||||||
Reference in New Issue
Block a user