feat(tasks): persist encrypted result handoff and recover received tasks

Add signed device-scoped task state/result endpoints and AES-GCM result outbox. Atomically claim original-message replies, reject cross-user/stale/expired tasks, preserve unknown platform receipts without blind replay, and retain ambiguous ciphertext for at most seven days. Maintain legacy synchronous compatibility and clarify stale deployment-plan diagnostics. Validation: ECS suite 227 passed/2 skipped, later outbox guards 12 passed, deployment diagnostics 54 passed. Real ECS/PostgreSQL/platform failure acceptance remains pending.
This commit is contained in:
ex_tianzy
2026-11-05 10:24:55 +08:00
parent db116284b1
commit e4c3a8ad4e
6 changed files with 325 additions and 4 deletions
+127
View File
@@ -0,0 +1,127 @@
"""Durable, encrypted result handoff; ambiguous platform sends never replay."""
import hashlib
import time
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import Field
from .credential_api import DeviceRequest
from .common import no_cache
from .crypto import canonical
class TaskRequest(DeviceRequest):
task_id: str = Field(pattern=r'^[A-Za-z0-9_-]{1,128}$')
class ResultRequest(TaskRequest):
result: str = Field(max_length=16000, repr=False)
class TaskDelivery:
def __init__(self, auth, state, broker, platform):
self.auth,self.state,self.broker,self.platform=auth,state,broker,platform
self.db=state.db
self.keyring=auth.broker.store.keyring
with self.db:
self.db.execute("""CREATE TABLE IF NOT EXISTS work_result_outbox_v2(
task_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,
digest TEXT NOT NULL,key_id TEXT NOT NULL,ciphertext TEXT NOT NULL,worker_id TEXT NOT NULL,
status TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)""")
self.router=APIRouter()
self.router.add_api_route('/work/v2/tasks/state',self.status,methods=['POST'])
self.router.add_api_route('/work/v2/tasks/result',self.submit,methods=['POST'])
def current(self, body):
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
task=self.state.task(body.task_id)
if (not self.state.task_current(task) or
not task['source_message_id'] or
(task['host_id'],task['owner'],task['binding_generation']) !=
(body.host_id,body.binding_owner,body.binding_generation) or
task['status'] in {'unbound','revoked','result_discarded'}):
raise HTTPException(403,'task_not_recoverable')
known=self.db.execute('SELECT task_id FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
if not known and task['status'] not in {'inflight','unknown'}:
raise HTTPException(403,'task_not_recoverable')
return task
def receipt(self,task):
row=self.db.execute('SELECT status,digest FROM work_result_outbox_v2 WHERE task_id=?',
(task['turn_id'],)).fetchone()
return {'task_id':task['turn_id'],'status':row['status'] if row else task['status'],
'result_received':bool(row),'result_digest':row['digest'] if row else None,
'deadline':task['deadline']}
async def status(self,body:TaskRequest,request:Request):
await self.auth.authenticate(body,request)
task=self.current(body)
pending=self.broker.pending.get(body.task_id)
if pending and pending.host==body.host_id:
pending.progress.set()
return no_cache(JSONResponse(self.receipt(task)))
async def submit(self,body:ResultRequest,request:Request):
await self.auth.authenticate(body,request)
digest=hashlib.sha256(body.result.encode()).hexdigest()
with self.db:
self.state._owner_lock(body.binding_owner)
task=self.current(body)
old=self.db.execute('SELECT digest FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
if old and old['digest']!=digest:
raise HTTPException(409,'task_result_conflict')
if not old:
aad=canonical(['work-result-v2',task['environment'],task['app_id'],task['owner'],
task['host_id'],task['binding_generation'],body.task_id,digest])
key,cipher=self.keyring.encrypt(body.result.encode(),aad)
self.db.execute('INSERT INTO work_result_outbox_v2 VALUES(?,?,?,?,?,?,?,?,?,?,?,?)',
(body.task_id,task['environment'],task['app_id'],task['owner'],task['host_id'],
task['binding_generation'],digest,key,cipher,'','ready',time.time()))
self.db.execute("UPDATE work_tasks_v2 SET status='delivery_ready',updated_at=? WHERE turn_id=?",
(time.time(),body.task_id))
pending=self.broker.pending.get(body.task_id)
if pending and not pending.future.done():
pending.future.set_result({'status':'delivery_ready','turn_id':body.task_id,'result':''})
return no_cache(JSONResponse(self.receipt(task)))
async def flush(self):
# Bounded retention for encrypted results with ambiguous platform receipt.
with self.db:
self.db.execute("UPDATE work_result_outbox_v2 SET ciphertext='' WHERE environment=? AND app_id=? AND status='unknown' AND updated_at<?",(self.state.environment,self.state.app_id,time.time()-7*86400))
rows=self.db.execute("""SELECT * FROM work_result_outbox_v2 WHERE environment=? AND app_id=?
AND status='ready' ORDER BY updated_at LIMIT 20""",
(self.state.environment,self.state.app_id)).fetchall()
for row in rows:
task=self.state.task(row['task_id'])
if not self.state.task_current(task):
with self.db:self.db.execute("UPDATE work_result_outbox_v2 SET status='discarded',ciphertext='',updated_at=? WHERE task_id=? AND status='ready'",(time.time(),row['task_id']))
continue
# Preflight may await token refresh; revalidate the binding afterwards.
try:await self.platform.tenant_token()
except Exception:continue
with self.db:
self.state._owner_lock(row['owner'])
task=self.state.task(row['task_id'])
if not self.state.task_current(task):continue
claimed=self.db.execute("UPDATE work_result_outbox_v2 SET status='sending',worker_id=?,updated_at=? WHERE task_id=? AND status='ready'",(self.broker.worker_id,time.time(),row['task_id'])).rowcount
if claimed!=1:continue
aad=canonical(['work-result-v2',row['environment'],row['app_id'],row['owner'],
row['host_id'],row['binding_generation'],row['task_id'],row['digest']])
outcome='unknown'
try:
result=self.keyring.decrypt(row['key_id'],row['ciphertext'],aad).decode()
if hashlib.sha256(result.encode()).hexdigest()!=row['digest']:raise ValueError()
await self.platform.reply(task['source_message_id'],result)
outcome='replied'
except Exception:pass
with self.db:
self.db.execute("UPDATE work_result_outbox_v2 SET status=?,ciphertext=?,updated_at=? WHERE task_id=? AND status='sending'",(outcome,'' if outcome=='replied' else row['ciphertext'],time.time(),row['task_id']))
self.db.execute("UPDATE work_tasks_v2 SET status=?,updated_at=? WHERE turn_id=? AND status='delivery_ready'",
('replied' if outcome=='replied' else 'reply_unknown',time.time(),row['task_id']))
async def recover_interrupted_sends(self):
rows=self.db.execute("SELECT task_id,worker_id,updated_at FROM work_result_outbox_v2 WHERE environment=? AND app_id=? AND status='sending'",(self.state.environment,self.state.app_id)).fetchall()
for row in rows:
if time.time()-row['updated_at']<60 or await self.auth.r.exists('hn:task-worker:'+row['worker_id']):continue
with self.db:
self.db.execute("UPDATE work_result_outbox_v2 SET status='unknown',updated_at=? WHERE task_id=? AND status='sending' AND worker_id=?",(time.time(),row['task_id'],row['worker_id']))