feat(tasks): persist encrypted result handoff and recover received tasks
Add signed device-scoped task state/result endpoints and AES-GCM result outbox. Atomically claim original-message replies, reject cross-user/stale/expired tasks, preserve unknown platform receipts without blind replay, and retain ambiguous ciphertext for at most seven days. Maintain legacy synchronous compatibility and clarify stale deployment-plan diagnostics. Validation: ECS suite 227 passed/2 skipped, later outbox guards 12 passed, deployment diagnostics 54 passed. Real ECS/PostgreSQL/platform failure acceptance remains pending.
This commit is contained in:
@@ -88,7 +88,7 @@ class BindingState:
|
||||
(self.environment, self.app_id, device["host_id"], generation))
|
||||
self.db.execute("""UPDATE work_tasks_v2 SET status=?,updated_at=?
|
||||
WHERE environment=? AND app_id=? AND host_id=? AND binding_generation<=?
|
||||
AND status IN ('inflight','completed','replying')""",
|
||||
AND status IN ('inflight','completed','replying','delivery_ready')""",
|
||||
("revoked" if reason == "revoke" else "unbound", time.time(), self.environment,
|
||||
self.app_id, device["host_id"], generation))
|
||||
event = uuid.uuid4().hex
|
||||
|
||||
@@ -161,6 +161,8 @@ class Broker:
|
||||
"text": text, "read_only": True, "channel": "im",
|
||||
"reply_key": raw_public(private.public_key())}
|
||||
if task:
|
||||
if source_message_id:
|
||||
payload['result_delivery']='durable-http-v1'
|
||||
payload["task_context"] = {key:task[key] for key in ("environment","app_id","owner","host_id","binding_generation",
|
||||
"conversation_id","chat_id","source_message_id")}
|
||||
payload["task_context"].update(task_id=turn,channel="work-dm")
|
||||
@@ -230,6 +232,8 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
store.lifecycle = BindingState(store,s.environment,s.feishu_app_id)
|
||||
broker.lifecycle = store.lifecycle
|
||||
lifecycle = DeviceLifecycle(store.lifecycle,r,broker)
|
||||
from .task_delivery import TaskDelivery
|
||||
deliveries = TaskDelivery(oauth,store.lifecycle,broker,f)
|
||||
else:
|
||||
oauth = OAuth(s, r, f)
|
||||
oauth.store = store
|
||||
@@ -249,6 +253,7 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
await r.ping()
|
||||
if lifecycle:
|
||||
await lifecycle.maintain()
|
||||
await deliveries.recover_interrupted_sends()
|
||||
for row in store.db.execute("SELECT host_id,fingerprint,revoked FROM devices").fetchall():
|
||||
await r.set("hn:device:" + row["host_id"], row["fingerprint"])
|
||||
if row["revoked"]:
|
||||
@@ -269,6 +274,8 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
await asyncio.sleep(5)
|
||||
try:
|
||||
await lifecycle.maintain()
|
||||
await deliveries.recover_interrupted_sends()
|
||||
await deliveries.flush()
|
||||
except Exception:
|
||||
# No credentials or raw exception text in background logs.
|
||||
pass
|
||||
@@ -292,6 +299,9 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
app = FastAPI(title="Hainuo B1 connector", lifespan=lifespan, docs_url=None, redoc_url=None)
|
||||
app.add_middleware(BodyLimitMiddleware, max_body=s.max_body)
|
||||
app.include_router(oauth.router)
|
||||
if lifecycle:
|
||||
app.include_router(deliveries.router)
|
||||
app.state.deliveries=deliveries
|
||||
app.include_router(notification_router(s, r, store, f, oauth))
|
||||
from .web_session import WebSessions
|
||||
web_sessions = WebSessions(s, r, store, broker, f)
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Durable, encrypted result handoff; ambiguous platform sends never replay."""
|
||||
import hashlib
|
||||
import time
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import Field
|
||||
from .credential_api import DeviceRequest
|
||||
from .common import no_cache
|
||||
from .crypto import canonical
|
||||
|
||||
|
||||
class TaskRequest(DeviceRequest):
|
||||
task_id: str = Field(pattern=r'^[A-Za-z0-9_-]{1,128}$')
|
||||
|
||||
|
||||
class ResultRequest(TaskRequest):
|
||||
result: str = Field(max_length=16000, repr=False)
|
||||
|
||||
|
||||
class TaskDelivery:
|
||||
def __init__(self, auth, state, broker, platform):
|
||||
self.auth,self.state,self.broker,self.platform=auth,state,broker,platform
|
||||
self.db=state.db
|
||||
self.keyring=auth.broker.store.keyring
|
||||
with self.db:
|
||||
self.db.execute("""CREATE TABLE IF NOT EXISTS work_result_outbox_v2(
|
||||
task_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
|
||||
owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,
|
||||
digest TEXT NOT NULL,key_id TEXT NOT NULL,ciphertext TEXT NOT NULL,worker_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)""")
|
||||
self.router=APIRouter()
|
||||
self.router.add_api_route('/work/v2/tasks/state',self.status,methods=['POST'])
|
||||
self.router.add_api_route('/work/v2/tasks/result',self.submit,methods=['POST'])
|
||||
|
||||
def current(self, body):
|
||||
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
|
||||
task=self.state.task(body.task_id)
|
||||
if (not self.state.task_current(task) or
|
||||
not task['source_message_id'] or
|
||||
(task['host_id'],task['owner'],task['binding_generation']) !=
|
||||
(body.host_id,body.binding_owner,body.binding_generation) or
|
||||
task['status'] in {'unbound','revoked','result_discarded'}):
|
||||
raise HTTPException(403,'task_not_recoverable')
|
||||
known=self.db.execute('SELECT task_id FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
|
||||
if not known and task['status'] not in {'inflight','unknown'}:
|
||||
raise HTTPException(403,'task_not_recoverable')
|
||||
return task
|
||||
|
||||
def receipt(self,task):
|
||||
row=self.db.execute('SELECT status,digest FROM work_result_outbox_v2 WHERE task_id=?',
|
||||
(task['turn_id'],)).fetchone()
|
||||
return {'task_id':task['turn_id'],'status':row['status'] if row else task['status'],
|
||||
'result_received':bool(row),'result_digest':row['digest'] if row else None,
|
||||
'deadline':task['deadline']}
|
||||
|
||||
async def status(self,body:TaskRequest,request:Request):
|
||||
await self.auth.authenticate(body,request)
|
||||
task=self.current(body)
|
||||
pending=self.broker.pending.get(body.task_id)
|
||||
if pending and pending.host==body.host_id:
|
||||
pending.progress.set()
|
||||
return no_cache(JSONResponse(self.receipt(task)))
|
||||
|
||||
async def submit(self,body:ResultRequest,request:Request):
|
||||
await self.auth.authenticate(body,request)
|
||||
digest=hashlib.sha256(body.result.encode()).hexdigest()
|
||||
with self.db:
|
||||
self.state._owner_lock(body.binding_owner)
|
||||
task=self.current(body)
|
||||
old=self.db.execute('SELECT digest FROM work_result_outbox_v2 WHERE task_id=?',(body.task_id,)).fetchone()
|
||||
if old and old['digest']!=digest:
|
||||
raise HTTPException(409,'task_result_conflict')
|
||||
if not old:
|
||||
aad=canonical(['work-result-v2',task['environment'],task['app_id'],task['owner'],
|
||||
task['host_id'],task['binding_generation'],body.task_id,digest])
|
||||
key,cipher=self.keyring.encrypt(body.result.encode(),aad)
|
||||
self.db.execute('INSERT INTO work_result_outbox_v2 VALUES(?,?,?,?,?,?,?,?,?,?,?,?)',
|
||||
(body.task_id,task['environment'],task['app_id'],task['owner'],task['host_id'],
|
||||
task['binding_generation'],digest,key,cipher,'','ready',time.time()))
|
||||
self.db.execute("UPDATE work_tasks_v2 SET status='delivery_ready',updated_at=? WHERE turn_id=?",
|
||||
(time.time(),body.task_id))
|
||||
pending=self.broker.pending.get(body.task_id)
|
||||
if pending and not pending.future.done():
|
||||
pending.future.set_result({'status':'delivery_ready','turn_id':body.task_id,'result':''})
|
||||
return no_cache(JSONResponse(self.receipt(task)))
|
||||
|
||||
async def flush(self):
|
||||
# Bounded retention for encrypted results with ambiguous platform receipt.
|
||||
with self.db:
|
||||
self.db.execute("UPDATE work_result_outbox_v2 SET ciphertext='' WHERE environment=? AND app_id=? AND status='unknown' AND updated_at<?",(self.state.environment,self.state.app_id,time.time()-7*86400))
|
||||
rows=self.db.execute("""SELECT * FROM work_result_outbox_v2 WHERE environment=? AND app_id=?
|
||||
AND status='ready' ORDER BY updated_at LIMIT 20""",
|
||||
(self.state.environment,self.state.app_id)).fetchall()
|
||||
for row in rows:
|
||||
task=self.state.task(row['task_id'])
|
||||
if not self.state.task_current(task):
|
||||
with self.db:self.db.execute("UPDATE work_result_outbox_v2 SET status='discarded',ciphertext='',updated_at=? WHERE task_id=? AND status='ready'",(time.time(),row['task_id']))
|
||||
continue
|
||||
# Preflight may await token refresh; revalidate the binding afterwards.
|
||||
try:await self.platform.tenant_token()
|
||||
except Exception:continue
|
||||
with self.db:
|
||||
self.state._owner_lock(row['owner'])
|
||||
task=self.state.task(row['task_id'])
|
||||
if not self.state.task_current(task):continue
|
||||
claimed=self.db.execute("UPDATE work_result_outbox_v2 SET status='sending',worker_id=?,updated_at=? WHERE task_id=? AND status='ready'",(self.broker.worker_id,time.time(),row['task_id'])).rowcount
|
||||
if claimed!=1:continue
|
||||
aad=canonical(['work-result-v2',row['environment'],row['app_id'],row['owner'],
|
||||
row['host_id'],row['binding_generation'],row['task_id'],row['digest']])
|
||||
outcome='unknown'
|
||||
try:
|
||||
result=self.keyring.decrypt(row['key_id'],row['ciphertext'],aad).decode()
|
||||
if hashlib.sha256(result.encode()).hexdigest()!=row['digest']:raise ValueError()
|
||||
await self.platform.reply(task['source_message_id'],result)
|
||||
outcome='replied'
|
||||
except Exception:pass
|
||||
with self.db:
|
||||
self.db.execute("UPDATE work_result_outbox_v2 SET status=?,ciphertext=?,updated_at=? WHERE task_id=? AND status='sending'",(outcome,'' if outcome=='replied' else row['ciphertext'],time.time(),row['task_id']))
|
||||
self.db.execute("UPDATE work_tasks_v2 SET status=?,updated_at=? WHERE turn_id=? AND status='delivery_ready'",
|
||||
('replied' if outcome=='replied' else 'reply_unknown',time.time(),row['task_id']))
|
||||
|
||||
async def recover_interrupted_sends(self):
|
||||
rows=self.db.execute("SELECT task_id,worker_id,updated_at FROM work_result_outbox_v2 WHERE environment=? AND app_id=? AND status='sending'",(self.state.environment,self.state.app_id)).fetchall()
|
||||
for row in rows:
|
||||
if time.time()-row['updated_at']<60 or await self.auth.r.exists('hn:task-worker:'+row['worker_id']):continue
|
||||
with self.db:
|
||||
self.db.execute("UPDATE work_result_outbox_v2 SET status='unknown',updated_at=? WHERE task_id=? AND status='sending' AND worker_id=?",(time.time(),row['task_id'],row['worker_id']))
|
||||
Reference in New Issue
Block a user