172 lines
10 KiB
Python
172 lines
10 KiB
Python
"""Durable, encrypted result handoff; ambiguous platform sends never replay."""
|
|
import hashlib
|
|
import time
|
|
from fastapi import APIRouter, HTTPException, Request
|
|
from fastapi.responses import JSONResponse
|
|
from pydantic import Field
|
|
from .credential_api import DeviceRequest
|
|
from .common import no_cache
|
|
from .crypto import canonical
|
|
|
|
|
|
class TaskRequest(DeviceRequest):
|
|
task_id: str = Field(pattern=r'^[A-Za-z0-9_-]{1,128}$')
|
|
|
|
|
|
class ResultRequest(TaskRequest):
|
|
result: str = Field(max_length=16000, repr=False)
|
|
|
|
|
|
class StateRequest(TaskRequest):
|
|
execution_state: str = Field(pattern=r'^(queued|running|waiting_for_approval|completed|failed|cancelled|interrupted|unknown)$')
|
|
status_sequence: int = Field(ge=1, le=2147483647, strict=True)
|
|
|
|
|
|
class TaskDelivery:
|
|
def __init__(self, auth, state, broker, platform):
|
|
self.auth,self.state,self.broker,self.platform=auth,state,broker,platform
|
|
self.db=state.db
|
|
self.keyring=auth.broker.store.keyring
|
|
with self.db:
|
|
self.db.execute('''CREATE TABLE IF NOT EXISTS work_execution_state_v3(
|
|
task_id TEXT PRIMARY KEY,execution_state TEXT NOT NULL,
|
|
status_sequence BIGINT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)''')
|
|
self.db.execute("""CREATE TABLE IF NOT EXISTS work_result_outbox_v2(
|
|
task_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
|
|
owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,
|
|
digest TEXT NOT NULL,key_id TEXT NOT NULL,ciphertext TEXT NOT NULL,worker_id TEXT NOT NULL,
|
|
status TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)""")
|
|
self.router=APIRouter()
|
|
self.router.add_api_route('/work/v2/tasks/state',self.status,methods=['POST'])
|
|
self.router.add_api_route('/work/v2/tasks/progress',self.progress,methods=['POST'])
|
|
self.router.add_api_route('/work/v2/tasks/result',self.submit,methods=['POST'])
|
|
self.cards=None
|
|
if auth.s.work_task_cards_enabled:
|
|
from .task_cards import TaskCards
|
|
self.cards=TaskCards(state,platform,broker.worker_id,files_enabled=auth.s.work_files_enabled,
|
|
remote_approvals=auth.s.work_remote_approvals_enabled)
|
|
broker.task_cards=self.cards
|
|
|
|
def current(self, body):
|
|
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
|
|
task=self.state.task(body.task_id)
|
|
if (not self.state.task_current(task) or
|
|
not task['source_message_id'] or
|
|
(task['host_id'],task['owner'],task['binding_generation']) !=
|
|
(body.host_id,body.binding_owner,body.binding_generation) or
|
|
task['status'] in {'unbound','revoked','result_discarded'}):
|
|
raise HTTPException(403,'task_not_recoverable')
|
|
known=self.db.execute('SELECT task_id FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
|
|
if not known and task['status'] not in {'inflight','unknown'}:
|
|
raise HTTPException(403,'task_not_recoverable')
|
|
return task
|
|
|
|
def receipt(self,task):
|
|
row=self.db.execute('SELECT status,digest FROM work_result_outbox_v2 WHERE task_id=?',
|
|
(task['turn_id'],)).fetchone()
|
|
execution=self.db.execute('SELECT execution_state,status_sequence FROM work_execution_state_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
|
return {'task_id':task['turn_id'],'status':row['status'] if row else task['status'],
|
|
'progress_protocol':'work-progress-v1',
|
|
'execution_state':execution['execution_state'] if execution else 'unknown',
|
|
'status_sequence':execution['status_sequence'] if execution else 0,
|
|
'result_received':bool(row),'result_digest':row['digest'] if row else None,
|
|
'deadline':task['deadline']}
|
|
|
|
async def status(self,body:TaskRequest,request:Request):
|
|
await self.auth.authenticate(body,request)
|
|
task=self.current(body)
|
|
pending=self.broker.pending.get(body.task_id)
|
|
if pending and pending.host==body.host_id:pending.progress.set()
|
|
return no_cache(JSONResponse(self.receipt(task)))
|
|
|
|
async def progress(self,body:StateRequest,request:Request):
|
|
await self.auth.authenticate(body,request)
|
|
with self.db:
|
|
self.state._owner_lock(body.binding_owner)
|
|
task=self.current(body)
|
|
if body.execution_state is not None:
|
|
old=self.db.execute('SELECT * FROM work_execution_state_v3 WHERE task_id=?',(body.task_id,)).fetchone()
|
|
terminal={'completed','failed','cancelled','interrupted'}
|
|
if old and body.status_sequence==old['status_sequence'] and body.execution_state!=old['execution_state']:
|
|
raise HTTPException(409,'execution_state_conflict')
|
|
if not old:
|
|
self.db.execute('INSERT INTO work_execution_state_v3 VALUES(?,?,?,?)',
|
|
(body.task_id,body.execution_state,body.status_sequence,time.time()))
|
|
elif body.status_sequence>old['status_sequence']:
|
|
if old['execution_state'] in terminal and body.execution_state!=old['execution_state']:
|
|
raise HTTPException(409,'execution_already_terminal')
|
|
self.db.execute('UPDATE work_execution_state_v3 SET execution_state=?,status_sequence=?,updated_at=? WHERE task_id=?',
|
|
(body.execution_state,body.status_sequence,time.time(),body.task_id))
|
|
pending=self.broker.pending.get(body.task_id)
|
|
if self.cards:
|
|
old=self.db.execute('SELECT status_sequence FROM work_execution_state_v3 WHERE task_id=?',(body.task_id,)).fetchone()
|
|
if old and body.status_sequence>=old['status_sequence']:self.cards.clear_connection_lost(body.task_id)
|
|
if pending and pending.host==body.host_id:
|
|
pending.progress.set()
|
|
return no_cache(JSONResponse(self.receipt(task)))
|
|
|
|
async def submit(self,body:ResultRequest,request:Request):
|
|
await self.auth.authenticate(body,request)
|
|
digest=hashlib.sha256(body.result.encode()).hexdigest()
|
|
with self.db:
|
|
self.state._owner_lock(body.binding_owner)
|
|
task=self.current(body)
|
|
old=self.db.execute('SELECT digest FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
|
|
if old and old['digest']!=digest:
|
|
raise HTTPException(409,'task_result_conflict')
|
|
if not old:
|
|
aad=canonical(['work-result-v2',task['environment'],task['app_id'],task['owner'],
|
|
task['host_id'],task['binding_generation'],body.task_id,digest])
|
|
key,cipher=self.keyring.encrypt(body.result.encode(),aad)
|
|
self.db.execute('INSERT INTO work_result_outbox_v2 VALUES(?,?,?,?,?,?,?,?,?,?,?,?)',
|
|
(body.task_id,task['environment'],task['app_id'],task['owner'],task['host_id'],
|
|
task['binding_generation'],digest,key,cipher,'','ready',time.time()))
|
|
self.db.execute("UPDATE work_tasks_v2 SET status='delivery_ready',updated_at=? WHERE turn_id=?",
|
|
(time.time(),body.task_id))
|
|
pending=self.broker.pending.get(body.task_id)
|
|
if pending and not pending.future.done():
|
|
pending.future.set_result({'status':'delivery_ready','turn_id':body.task_id,'result':''})
|
|
return no_cache(JSONResponse(self.receipt(task)))
|
|
|
|
async def flush(self):
|
|
# Bounded retention for encrypted results with ambiguous platform receipt.
|
|
with self.db:
|
|
self.db.execute("UPDATE work_result_outbox_v2 SET ciphertext='' WHERE environment=? AND app_id=? AND status='unknown' AND updated_at<?",(self.state.environment,self.state.app_id,time.time()-7*86400))
|
|
rows=self.db.execute("""SELECT * FROM work_result_outbox_v2 WHERE environment=? AND app_id=?
|
|
AND status='ready' ORDER BY updated_at LIMIT 20""",
|
|
(self.state.environment,self.state.app_id)).fetchall()
|
|
for row in rows:
|
|
task=self.state.task(row['task_id'])
|
|
if not self.state.task_current(task):
|
|
with self.db:self.db.execute("UPDATE work_result_outbox_v2 SET status='discarded',ciphertext='',updated_at=? WHERE task_id=? AND status='ready'",(time.time(),row['task_id']))
|
|
continue
|
|
# Preflight may await token refresh; revalidate the binding afterwards.
|
|
try:await self.platform.tenant_token()
|
|
except Exception:continue
|
|
with self.db:
|
|
self.state._owner_lock(row['owner'])
|
|
task=self.state.task(row['task_id'])
|
|
if not self.state.task_current(task):continue
|
|
claimed=self.db.execute("UPDATE work_result_outbox_v2 SET status='sending',worker_id=?,updated_at=? WHERE task_id=? AND status='ready'",(self.broker.worker_id,time.time(),row['task_id'])).rowcount
|
|
if claimed!=1:continue
|
|
aad=canonical(['work-result-v2',row['environment'],row['app_id'],row['owner'],
|
|
row['host_id'],row['binding_generation'],row['task_id'],row['digest']])
|
|
outcome='unknown'
|
|
try:
|
|
result=self.keyring.decrypt(row['key_id'],row['ciphertext'],aad).decode()
|
|
if hashlib.sha256(result.encode()).hexdigest()!=row['digest']:raise ValueError()
|
|
await self.platform.reply(task['source_message_id'],result)
|
|
outcome='replied'
|
|
except Exception:pass
|
|
with self.db:
|
|
self.db.execute("UPDATE work_result_outbox_v2 SET status=?,ciphertext=?,updated_at=? WHERE task_id=? AND status='sending'",(outcome,'' if outcome=='replied' else row['ciphertext'],time.time(),row['task_id']))
|
|
self.db.execute("UPDATE work_tasks_v2 SET status=?,updated_at=? WHERE turn_id=? AND status='delivery_ready'",
|
|
('replied' if outcome=='replied' else 'reply_unknown',time.time(),row['task_id']))
|
|
|
|
async def recover_interrupted_sends(self):
|
|
rows=self.db.execute("SELECT task_id,worker_id,updated_at FROM work_result_outbox_v2 WHERE environment=? AND app_id=? AND status='sending'",(self.state.environment,self.state.app_id)).fetchall()
|
|
for row in rows:
|
|
if time.time()-row['updated_at']<60 or await self.auth.r.exists('hn:task-worker:'+row['worker_id']):continue
|
|
with self.db:
|
|
self.db.execute("UPDATE work_result_outbox_v2 SET status='unknown',updated_at=? WHERE task_id=? AND status='sending' AND worker_id=?",(time.time(),row['task_id'],row['worker_id']))
|