feat(tasks): persist encrypted result handoff and recover received tasks
Add signed device-scoped task state/result endpoints and AES-GCM result outbox. Atomically claim original-message replies, reject cross-user/stale/expired tasks, preserve unknown platform receipts without blind replay, and retain ambiguous ciphertext for at most seven days. Maintain legacy synchronous compatibility and clarify stale deployment-plan diagnostics. Validation: ECS suite 227 passed/2 skipped, later outbox guards 12 passed, deployment diagnostics 54 passed. Real ECS/PostgreSQL/platform failure acceptance remains pending.
This commit is contained in:
@@ -0,0 +1,127 @@
|
||||
"""Durable, encrypted result handoff; ambiguous platform sends never replay."""
|
||||
import hashlib
|
||||
import time
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import Field
|
||||
from .credential_api import DeviceRequest
|
||||
from .common import no_cache
|
||||
from .crypto import canonical
|
||||
|
||||
|
||||
class TaskRequest(DeviceRequest):
|
||||
task_id: str = Field(pattern=r'^[A-Za-z0-9_-]{1,128}$')
|
||||
|
||||
|
||||
class ResultRequest(TaskRequest):
|
||||
result: str = Field(max_length=16000, repr=False)
|
||||
|
||||
|
||||
class TaskDelivery:
|
||||
def __init__(self, auth, state, broker, platform):
|
||||
self.auth,self.state,self.broker,self.platform=auth,state,broker,platform
|
||||
self.db=state.db
|
||||
self.keyring=auth.broker.store.keyring
|
||||
with self.db:
|
||||
self.db.execute("""CREATE TABLE IF NOT EXISTS work_result_outbox_v2(
|
||||
task_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
|
||||
owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,
|
||||
digest TEXT NOT NULL,key_id TEXT NOT NULL,ciphertext TEXT NOT NULL,worker_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)""")
|
||||
self.router=APIRouter()
|
||||
self.router.add_api_route('/work/v2/tasks/state',self.status,methods=['POST'])
|
||||
self.router.add_api_route('/work/v2/tasks/result',self.submit,methods=['POST'])
|
||||
|
||||
def current(self, body):
|
||||
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
|
||||
task=self.state.task(body.task_id)
|
||||
if (not self.state.task_current(task) or
|
||||
not task['source_message_id'] or
|
||||
(task['host_id'],task['owner'],task['binding_generation']) !=
|
||||
(body.host_id,body.binding_owner,body.binding_generation) or
|
||||
task['status'] in {'unbound','revoked','result_discarded'}):
|
||||
raise HTTPException(403,'task_not_recoverable')
|
||||
known=self.db.execute('SELECT task_id FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
|
||||
if not known and task['status'] not in {'inflight','unknown'}:
|
||||
raise HTTPException(403,'task_not_recoverable')
|
||||
return task
|
||||
|
||||
def receipt(self,task):
|
||||
row=self.db.execute('SELECT status,digest FROM work_result_outbox_v2 WHERE task_id=?',
|
||||
(task['turn_id'],)).fetchone()
|
||||
return {'task_id':task['turn_id'],'status':row['status'] if row else task['status'],
|
||||
'result_received':bool(row),'result_digest':row['digest'] if row else None,
|
||||
'deadline':task['deadline']}
|
||||
|
||||
async def status(self,body:TaskRequest,request:Request):
|
||||
await self.auth.authenticate(body,request)
|
||||
task=self.current(body)
|
||||
pending=self.broker.pending.get(body.task_id)
|
||||
if pending and pending.host==body.host_id:
|
||||
pending.progress.set()
|
||||
return no_cache(JSONResponse(self.receipt(task)))
|
||||
|
||||
async def submit(self,body:ResultRequest,request:Request):
|
||||
await self.auth.authenticate(body,request)
|
||||
digest=hashlib.sha256(body.result.encode()).hexdigest()
|
||||
with self.db:
|
||||
self.state._owner_lock(body.binding_owner)
|
||||
task=self.current(body)
|
||||
old=self.db.execute('SELECT digest FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
|
||||
if old and old['digest']!=digest:
|
||||
raise HTTPException(409,'task_result_conflict')
|
||||
if not old:
|
||||
aad=canonical(['work-result-v2',task['environment'],task['app_id'],task['owner'],
|
||||
task['host_id'],task['binding_generation'],body.task_id,digest])
|
||||
key,cipher=self.keyring.encrypt(body.result.encode(),aad)
|
||||
self.db.execute('INSERT INTO work_result_outbox_v2 VALUES(?,?,?,?,?,?,?,?,?,?,?,?)',
|
||||
(body.task_id,task['environment'],task['app_id'],task['owner'],task['host_id'],
|
||||
task['binding_generation'],digest,key,cipher,'','ready',time.time()))
|
||||
self.db.execute("UPDATE work_tasks_v2 SET status='delivery_ready',updated_at=? WHERE turn_id=?",
|
||||
(time.time(),body.task_id))
|
||||
pending=self.broker.pending.get(body.task_id)
|
||||
if pending and not pending.future.done():
|
||||
pending.future.set_result({'status':'delivery_ready','turn_id':body.task_id,'result':''})
|
||||
return no_cache(JSONResponse(self.receipt(task)))
|
||||
|
||||
async def flush(self):
|
||||
# Bounded retention for encrypted results with ambiguous platform receipt.
|
||||
with self.db:
|
||||
self.db.execute("UPDATE work_result_outbox_v2 SET ciphertext='' WHERE environment=? AND app_id=? AND status='unknown' AND updated_at<?",(self.state.environment,self.state.app_id,time.time()-7*86400))
|
||||
rows=self.db.execute("""SELECT * FROM work_result_outbox_v2 WHERE environment=? AND app_id=?
|
||||
AND status='ready' ORDER BY updated_at LIMIT 20""",
|
||||
(self.state.environment,self.state.app_id)).fetchall()
|
||||
for row in rows:
|
||||
task=self.state.task(row['task_id'])
|
||||
if not self.state.task_current(task):
|
||||
with self.db:self.db.execute("UPDATE work_result_outbox_v2 SET status='discarded',ciphertext='',updated_at=? WHERE task_id=? AND status='ready'",(time.time(),row['task_id']))
|
||||
continue
|
||||
# Preflight may await token refresh; revalidate the binding afterwards.
|
||||
try:await self.platform.tenant_token()
|
||||
except Exception:continue
|
||||
with self.db:
|
||||
self.state._owner_lock(row['owner'])
|
||||
task=self.state.task(row['task_id'])
|
||||
if not self.state.task_current(task):continue
|
||||
claimed=self.db.execute("UPDATE work_result_outbox_v2 SET status='sending',worker_id=?,updated_at=? WHERE task_id=? AND status='ready'",(self.broker.worker_id,time.time(),row['task_id'])).rowcount
|
||||
if claimed!=1:continue
|
||||
aad=canonical(['work-result-v2',row['environment'],row['app_id'],row['owner'],
|
||||
row['host_id'],row['binding_generation'],row['task_id'],row['digest']])
|
||||
outcome='unknown'
|
||||
try:
|
||||
result=self.keyring.decrypt(row['key_id'],row['ciphertext'],aad).decode()
|
||||
if hashlib.sha256(result.encode()).hexdigest()!=row['digest']:raise ValueError()
|
||||
await self.platform.reply(task['source_message_id'],result)
|
||||
outcome='replied'
|
||||
except Exception:pass
|
||||
with self.db:
|
||||
self.db.execute("UPDATE work_result_outbox_v2 SET status=?,ciphertext=?,updated_at=? WHERE task_id=? AND status='sending'",(outcome,'' if outcome=='replied' else row['ciphertext'],time.time(),row['task_id']))
|
||||
self.db.execute("UPDATE work_tasks_v2 SET status=?,updated_at=? WHERE turn_id=? AND status='delivery_ready'",
|
||||
('replied' if outcome=='replied' else 'reply_unknown',time.time(),row['task_id']))
|
||||
|
||||
async def recover_interrupted_sends(self):
|
||||
rows=self.db.execute("SELECT task_id,worker_id,updated_at FROM work_result_outbox_v2 WHERE environment=? AND app_id=? AND status='sending'",(self.state.environment,self.state.app_id)).fetchall()
|
||||
for row in rows:
|
||||
if time.time()-row['updated_at']<60 or await self.auth.r.exists('hn:task-worker:'+row['worker_id']):continue
|
||||
with self.db:
|
||||
self.db.execute("UPDATE work_result_outbox_v2 SET status='unknown',updated_at=? WHERE task_id=? AND status='sending' AND worker_id=?",(time.time(),row['task_id'],row['worker_id']))
|
||||
Reference in New Issue
Block a user