Files
ecs-relay-service/deployable/relay/task_delivery.py
T

172 lines
10 KiB
Python

"""Durable, encrypted result handoff; ambiguous platform sends never replay."""
import hashlib
import time
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import Field
from .credential_api import DeviceRequest
from .common import no_cache
from .crypto import canonical
class TaskRequest(DeviceRequest):
task_id: str = Field(pattern=r'^[A-Za-z0-9_-]{1,128}$')
class ResultRequest(TaskRequest):
result: str = Field(max_length=16000, repr=False)
class StateRequest(TaskRequest):
execution_state: str = Field(pattern=r'^(queued|running|waiting_for_approval|completed|failed|cancelled|interrupted|unknown)$')
status_sequence: int = Field(ge=1, le=2147483647, strict=True)
class TaskDelivery:
def __init__(self, auth, state, broker, platform):
self.auth,self.state,self.broker,self.platform=auth,state,broker,platform
self.db=state.db
self.keyring=auth.broker.store.keyring
with self.db:
self.db.execute('''CREATE TABLE IF NOT EXISTS work_execution_state_v3(
task_id TEXT PRIMARY KEY,execution_state TEXT NOT NULL,
status_sequence BIGINT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)''')
self.db.execute("""CREATE TABLE IF NOT EXISTS work_result_outbox_v2(
task_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,
digest TEXT NOT NULL,key_id TEXT NOT NULL,ciphertext TEXT NOT NULL,worker_id TEXT NOT NULL,
status TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)""")
self.router=APIRouter()
self.router.add_api_route('/work/v2/tasks/state',self.status,methods=['POST'])
self.router.add_api_route('/work/v2/tasks/progress',self.progress,methods=['POST'])
self.router.add_api_route('/work/v2/tasks/result',self.submit,methods=['POST'])
self.cards=None
if auth.s.work_task_cards_enabled:
from .task_cards import TaskCards
self.cards=TaskCards(state,platform,broker.worker_id,files_enabled=auth.s.work_files_enabled,
remote_approvals=auth.s.work_remote_approvals_enabled)
broker.task_cards=self.cards
def current(self, body):
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
task=self.state.task(body.task_id)
if (not self.state.task_current(task) or
not task['source_message_id'] or
(task['host_id'],task['owner'],task['binding_generation']) !=
(body.host_id,body.binding_owner,body.binding_generation) or
task['status'] in {'unbound','revoked','result_discarded'}):
raise HTTPException(403,'task_not_recoverable')
known=self.db.execute('SELECT task_id FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
if not known and task['status'] not in {'inflight','unknown'}:
raise HTTPException(403,'task_not_recoverable')
return task
def receipt(self,task):
row=self.db.execute('SELECT status,digest FROM work_result_outbox_v2 WHERE task_id=?',
(task['turn_id'],)).fetchone()
execution=self.db.execute('SELECT execution_state,status_sequence FROM work_execution_state_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
return {'task_id':task['turn_id'],'status':row['status'] if row else task['status'],
'progress_protocol':'work-progress-v1',
'execution_state':execution['execution_state'] if execution else 'unknown',
'status_sequence':execution['status_sequence'] if execution else 0,
'result_received':bool(row),'result_digest':row['digest'] if row else None,
'deadline':task['deadline']}
async def status(self,body:TaskRequest,request:Request):
await self.auth.authenticate(body,request)
task=self.current(body)
pending=self.broker.pending.get(body.task_id)
if pending and pending.host==body.host_id:pending.progress.set()
return no_cache(JSONResponse(self.receipt(task)))
async def progress(self,body:StateRequest,request:Request):
await self.auth.authenticate(body,request)
with self.db:
self.state._owner_lock(body.binding_owner)
task=self.current(body)
if body.execution_state is not None:
old=self.db.execute('SELECT * FROM work_execution_state_v3 WHERE task_id=?',(body.task_id,)).fetchone()
terminal={'completed','failed','cancelled','interrupted'}
if old and body.status_sequence==old['status_sequence'] and body.execution_state!=old['execution_state']:
raise HTTPException(409,'execution_state_conflict')
if not old:
self.db.execute('INSERT INTO work_execution_state_v3 VALUES(?,?,?,?)',
(body.task_id,body.execution_state,body.status_sequence,time.time()))
elif body.status_sequence>old['status_sequence']:
if old['execution_state'] in terminal and body.execution_state!=old['execution_state']:
raise HTTPException(409,'execution_already_terminal')
self.db.execute('UPDATE work_execution_state_v3 SET execution_state=?,status_sequence=?,updated_at=? WHERE task_id=?',
(body.execution_state,body.status_sequence,time.time(),body.task_id))
pending=self.broker.pending.get(body.task_id)
if self.cards:
old=self.db.execute('SELECT status_sequence FROM work_execution_state_v3 WHERE task_id=?',(body.task_id,)).fetchone()
if old and body.status_sequence>=old['status_sequence']:self.cards.clear_connection_lost(body.task_id)
if pending and pending.host==body.host_id:
pending.progress.set()
return no_cache(JSONResponse(self.receipt(task)))
async def submit(self,body:ResultRequest,request:Request):
await self.auth.authenticate(body,request)
digest=hashlib.sha256(body.result.encode()).hexdigest()
with self.db:
self.state._owner_lock(body.binding_owner)
task=self.current(body)
old=self.db.execute('SELECT digest FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
if old and old['digest']!=digest:
raise HTTPException(409,'task_result_conflict')
if not old:
aad=canonical(['work-result-v2',task['environment'],task['app_id'],task['owner'],
task['host_id'],task['binding_generation'],body.task_id,digest])
key,cipher=self.keyring.encrypt(body.result.encode(),aad)
self.db.execute('INSERT INTO work_result_outbox_v2 VALUES(?,?,?,?,?,?,?,?,?,?,?,?)',
(body.task_id,task['environment'],task['app_id'],task['owner'],task['host_id'],
task['binding_generation'],digest,key,cipher,'','ready',time.time()))
self.db.execute("UPDATE work_tasks_v2 SET status='delivery_ready',updated_at=? WHERE turn_id=?",
(time.time(),body.task_id))
pending=self.broker.pending.get(body.task_id)
if pending and not pending.future.done():
pending.future.set_result({'status':'delivery_ready','turn_id':body.task_id,'result':''})
return no_cache(JSONResponse(self.receipt(task)))
async def flush(self):
# Bounded retention for encrypted results with ambiguous platform receipt.
with self.db:
self.db.execute("UPDATE work_result_outbox_v2 SET ciphertext='' WHERE environment=? AND app_id=? AND status='unknown' AND updated_at<?",(self.state.environment,self.state.app_id,time.time()-7*86400))
rows=self.db.execute("""SELECT * FROM work_result_outbox_v2 WHERE environment=? AND app_id=?
AND status='ready' ORDER BY updated_at LIMIT 20""",
(self.state.environment,self.state.app_id)).fetchall()
for row in rows:
task=self.state.task(row['task_id'])
if not self.state.task_current(task):
with self.db:self.db.execute("UPDATE work_result_outbox_v2 SET status='discarded',ciphertext='',updated_at=? WHERE task_id=? AND status='ready'",(time.time(),row['task_id']))
continue
# Preflight may await token refresh; revalidate the binding afterwards.
try:await self.platform.tenant_token()
except Exception:continue
with self.db:
self.state._owner_lock(row['owner'])
task=self.state.task(row['task_id'])
if not self.state.task_current(task):continue
claimed=self.db.execute("UPDATE work_result_outbox_v2 SET status='sending',worker_id=?,updated_at=? WHERE task_id=? AND status='ready'",(self.broker.worker_id,time.time(),row['task_id'])).rowcount
if claimed!=1:continue
aad=canonical(['work-result-v2',row['environment'],row['app_id'],row['owner'],
row['host_id'],row['binding_generation'],row['task_id'],row['digest']])
outcome='unknown'
try:
result=self.keyring.decrypt(row['key_id'],row['ciphertext'],aad).decode()
if hashlib.sha256(result.encode()).hexdigest()!=row['digest']:raise ValueError()
await self.platform.reply(task['source_message_id'],result)
outcome='replied'
except Exception:pass
with self.db:
self.db.execute("UPDATE work_result_outbox_v2 SET status=?,ciphertext=?,updated_at=? WHERE task_id=? AND status='sending'",(outcome,'' if outcome=='replied' else row['ciphertext'],time.time(),row['task_id']))
self.db.execute("UPDATE work_tasks_v2 SET status=?,updated_at=? WHERE turn_id=? AND status='delivery_ready'",
('replied' if outcome=='replied' else 'reply_unknown',time.time(),row['task_id']))
async def recover_interrupted_sends(self):
rows=self.db.execute("SELECT task_id,worker_id,updated_at FROM work_result_outbox_v2 WHERE environment=? AND app_id=? AND status='sending'",(self.state.environment,self.state.app_id)).fetchall()
for row in rows:
if time.time()-row['updated_at']<60 or await self.auth.r.exists('hn:task-worker:'+row['worker_id']):continue
with self.db:
self.db.execute("UPDATE work_result_outbox_v2 SET status='unknown',updated_at=? WHERE task_id=? AND status='sending' AND worker_id=?",(time.time(),row['task_id'],row['worker_id']))