feat(work): add signed task progress and durable status cards
This commit is contained in:
@@ -41,6 +41,7 @@ class Settings(BaseSettings):
|
||||
max_connections: int = 1000
|
||||
max_result: int = 16000
|
||||
work_files_enabled: bool = False
|
||||
work_task_cards_enabled: bool = False
|
||||
work_file_max_bytes: int = 10*1024*1024
|
||||
work_file_batch_limit: int = 20
|
||||
work_file_retention: int = 86400
|
||||
@@ -56,6 +57,7 @@ class Settings(BaseSettings):
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_runtime(self):
|
||||
if self.work_task_cards_enabled and self.credential_mode!='broker':raise ValueError('task cards require broker mode')
|
||||
if not 1024<=self.work_file_max_bytes<=100*1024*1024 or not 1<=self.work_file_batch_limit<=50 or not 300<=self.work_file_retention<=7*86400:
|
||||
raise ValueError('invalid work file deployment limits')
|
||||
if self.work_files_enabled and self.credential_mode!='broker':raise ValueError('work files require broker mode')
|
||||
|
||||
@@ -298,10 +298,15 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
# No credentials or raw exception text in background logs.
|
||||
pass
|
||||
lifecycle_task = asyncio.create_task(maintenance()) if lifecycle else None
|
||||
async def card_maintenance():
|
||||
while True:
|
||||
await asyncio.sleep(5)
|
||||
with contextlib.suppress(Exception):await deliveries.cards.flush()
|
||||
cards_task=asyncio.create_task(card_maintenance()) if lifecycle and deliveries.cards else None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
work = [reply_task,sweep_task,*background] + ([lifecycle_task] if lifecycle_task else [])
|
||||
work = [reply_task,sweep_task,*background] + ([lifecycle_task] if lifecycle_task else []) + ([cards_task] if cards_task else [])
|
||||
for task in work:
|
||||
task.cancel()
|
||||
await asyncio.gather(*work, return_exceptions=True)
|
||||
|
||||
@@ -122,6 +122,22 @@ class Feishu:
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json={"msg_type": "text", "content": json.dumps({"text": text}, ensure_ascii=False)})
|
||||
|
||||
async def reply_task_card(self,message_id,card,delivery_id,*,token):
|
||||
import re
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,128}',message_id):raise PlatformError('invalid reply reference')
|
||||
data=await self._json('POST',f'/open-apis/im/v1/messages/{message_id}/reply',
|
||||
headers={'Authorization':'Bearer '+token},json={'msg_type':'interactive',
|
||||
'content':json.dumps(card,ensure_ascii=False),'uuid':delivery_id})
|
||||
result=data.get('data',{}).get('message_id')
|
||||
if not isinstance(result,str)or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}',result):raise PlatformError('card receipt invalid')
|
||||
return result
|
||||
|
||||
async def update_task_card(self,message_id,card,*,token):
|
||||
import re
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,128}',message_id):raise PlatformError('invalid card reference')
|
||||
await self._json('PATCH',f'/open-apis/im/v1/messages/{message_id}',
|
||||
headers={'Authorization':'Bearer '+token},json={'content':json.dumps(card,ensure_ascii=False)})
|
||||
|
||||
async def download_message_file(self,message_id,file_key,max_bytes):
|
||||
import re
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,128}',message_id)or not re.fullmatch(r'[A-Za-z0-9_-]{1,256}',file_key):raise PlatformError('invalid file reference')
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Durable private task status cards. No task body, credentials or approval buttons."""
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
|
||||
|
||||
EXECUTION={'queued':'排队中','running':'执行中','waiting_for_approval':'等待本人确认',
|
||||
'completed':'执行完成','failed':'执行失败','cancelled':'已取消','interrupted':'已中断','unknown':'状态待核对'}
|
||||
DELIVERY={None:'尚未接收结果','ready':'已接收结果,等待投递','sending':'正在投递结果',
|
||||
'replied':'结果已发送到当前会话','unknown':'结果投递待核对','discarded':'结果已停止交付'}
|
||||
|
||||
|
||||
def card_content(task_id,execution,delivery):
|
||||
execution=execution if execution in EXECUTION else 'unknown'
|
||||
color='green' if execution=='completed' else 'red' if execution=='failed' else 'orange' if execution in {'waiting_for_approval','unknown'} else 'blue'
|
||||
elements=[{'tag':'div','text':{'tag':'plain_text','content':'任务状态:'+EXECUTION[execution]}},
|
||||
{'tag':'div','text':{'tag':'plain_text','content':'结果交付:'+DELIVERY.get(delivery,'状态待核对')}},
|
||||
{'tag':'note','elements':[{'tag':'plain_text','content':'任务编号:'+task_id}]}]
|
||||
if execution=='waiting_for_approval':
|
||||
elements.insert(1,{'tag':'div','text':{'tag':'plain_text','content':'请在客户端查看并确认具体操作。本卡片暂不提供远程批准。'}})
|
||||
if execution=='completed' and delivery!='replied':
|
||||
elements.insert(1,{'tag':'div','text':{'tag':'plain_text','content':'任务已执行完成,结果交付请看下方状态。'}})
|
||||
return {'config':{'wide_screen_mode':True,'enable_forward':False},
|
||||
'header':{'template':color,'title':{'tag':'plain_text','content':'海诺任务进度'}},'elements':elements}
|
||||
|
||||
|
||||
class TaskCards:
|
||||
def __init__(self,state,platform,worker_id,clock=time.time):
|
||||
self.state,self.db,self.platform,self.worker_id,self.clock=state,state.db,platform,worker_id,clock
|
||||
with self.db:
|
||||
self.db.execute('''CREATE TABLE IF NOT EXISTS work_task_cards_v3(
|
||||
task_id TEXT PRIMARY KEY,message_id TEXT NOT NULL,digest TEXT NOT NULL,
|
||||
status TEXT NOT NULL,worker_id TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)''')
|
||||
|
||||
def valid(self,task):
|
||||
device=self.state.store.device(task['host_id']) if task else None
|
||||
return bool(device and not device['revoked'] and device['owner']==task['owner']
|
||||
and device['binding_generation']==task['binding_generation'] and task['source_message_id'] and task['chat_id'])
|
||||
|
||||
def snapshot(self,task):
|
||||
row=self.db.execute('SELECT execution_state FROM work_execution_state_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
||||
delivery=self.db.execute('SELECT status FROM work_result_outbox_v2 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
||||
execution=row['execution_state'] if row else 'unknown'
|
||||
if task['status']=='unknown' and execution not in {'completed','failed','cancelled','interrupted'}:execution='unknown'
|
||||
if task['deadline']<=self.clock() and execution not in {'completed','failed','cancelled','interrupted'}:execution='unknown'
|
||||
return card_content(task['turn_id'],execution,delivery['status'] if delivery else None)
|
||||
|
||||
async def flush(self):
|
||||
# Bound one maintenance pass; business result delivery remains independent.
|
||||
rows=self.db.execute('''SELECT * FROM work_tasks_v2 WHERE environment=? AND app_id=?
|
||||
AND source_message_id<>'' AND chat_id<>'' AND updated_at>? ORDER BY updated_at DESC LIMIT 100''',
|
||||
(self.state.environment,self.state.app_id,self.clock()-86400)).fetchall()
|
||||
sent=0
|
||||
for raw in rows:
|
||||
if sent>=10:break
|
||||
task=dict(raw)
|
||||
if not self.valid(task):continue
|
||||
card=self.snapshot(task)
|
||||
digest=hashlib.sha256(json.dumps(card,sort_keys=True,ensure_ascii=True).encode()).hexdigest()
|
||||
old=self.db.execute('SELECT * FROM work_task_cards_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
||||
if old and (old['status']=='unknown' or old['digest']==digest or self.clock()-old['updated_at']<5):continue
|
||||
if not old and task['deadline']<=self.clock():continue
|
||||
try:token=await self.platform.tenant_token()
|
||||
except Exception:continue
|
||||
with self.db:
|
||||
self.state._owner_lock(task['owner'])
|
||||
task=self.state.task(task['turn_id'])
|
||||
if not self.valid(task):continue
|
||||
old=self.db.execute('SELECT * FROM work_task_cards_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
||||
if old and old['status']=='sending':
|
||||
# A lost creation receipt cannot safely become another card.
|
||||
if self.clock()-old['updated_at']>=60:
|
||||
self.db.execute("UPDATE work_task_cards_v3 SET status=? WHERE task_id=?",
|
||||
('ready' if old['message_id'] else 'unknown',task['turn_id']))
|
||||
continue
|
||||
if old and (old['status']=='unknown' or old['digest']==digest or self.clock()-old['updated_at']<5):continue
|
||||
# Recompute after token/lock awaits so an older observation cannot overwrite new status.
|
||||
card=self.snapshot(task)
|
||||
digest=hashlib.sha256(json.dumps(card,sort_keys=True,ensure_ascii=True).encode()).hexdigest()
|
||||
message_id=old['message_id'] if old else ''
|
||||
if not old:
|
||||
self.db.execute('INSERT INTO work_task_cards_v3 VALUES(?,?,?,?,?,?)',
|
||||
(task['turn_id'],'','','sending',self.worker_id,self.clock()))
|
||||
else:
|
||||
self.db.execute("UPDATE work_task_cards_v3 SET status='sending',worker_id=?,updated_at=? WHERE task_id=?",
|
||||
(self.worker_id,self.clock(),task['turn_id']))
|
||||
try:
|
||||
sent+=1
|
||||
if message_id:await self.platform.update_task_card(message_id,card,token=token)
|
||||
else:
|
||||
delivery_id=hashlib.sha256(('task-card-v3\0'+self.state.environment+'\0'+self.state.app_id+'\0'+task['turn_id']).encode()).hexdigest()[:32]
|
||||
message_id=await self.platform.reply_task_card(task['source_message_id'],card,delivery_id,token=token)
|
||||
status='ready';saved_digest=digest
|
||||
except Exception:
|
||||
status='ready' if message_id else 'unknown';saved_digest=old['digest'] if old else ''
|
||||
with self.db:
|
||||
self.db.execute("UPDATE work_task_cards_v3 SET message_id=?,digest=?,status=?,updated_at=? WHERE task_id=? AND status='sending' AND worker_id=?",
|
||||
(message_id,saved_digest,status,self.clock(),task['turn_id'],self.worker_id))
|
||||
@@ -17,12 +17,20 @@ class ResultRequest(TaskRequest):
|
||||
result: str = Field(max_length=16000, repr=False)
|
||||
|
||||
|
||||
class StateRequest(TaskRequest):
|
||||
execution_state: str = Field(pattern=r'^(queued|running|waiting_for_approval|completed|failed|cancelled|interrupted|unknown)$')
|
||||
status_sequence: int = Field(ge=1, le=2147483647, strict=True)
|
||||
|
||||
|
||||
class TaskDelivery:
|
||||
def __init__(self, auth, state, broker, platform):
|
||||
self.auth,self.state,self.broker,self.platform=auth,state,broker,platform
|
||||
self.db=state.db
|
||||
self.keyring=auth.broker.store.keyring
|
||||
with self.db:
|
||||
self.db.execute('''CREATE TABLE IF NOT EXISTS work_execution_state_v3(
|
||||
task_id TEXT PRIMARY KEY,execution_state TEXT NOT NULL,
|
||||
status_sequence BIGINT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)''')
|
||||
self.db.execute("""CREATE TABLE IF NOT EXISTS work_result_outbox_v2(
|
||||
task_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
|
||||
owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,
|
||||
@@ -30,7 +38,12 @@ class TaskDelivery:
|
||||
status TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)""")
|
||||
self.router=APIRouter()
|
||||
self.router.add_api_route('/work/v2/tasks/state',self.status,methods=['POST'])
|
||||
self.router.add_api_route('/work/v2/tasks/progress',self.progress,methods=['POST'])
|
||||
self.router.add_api_route('/work/v2/tasks/result',self.submit,methods=['POST'])
|
||||
self.cards=None
|
||||
if auth.s.work_task_cards_enabled:
|
||||
from .task_cards import TaskCards
|
||||
self.cards=TaskCards(state,platform,broker.worker_id)
|
||||
|
||||
def current(self, body):
|
||||
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
|
||||
@@ -49,7 +62,10 @@ class TaskDelivery:
|
||||
def receipt(self,task):
|
||||
row=self.db.execute('SELECT status,digest FROM work_result_outbox_v2 WHERE task_id=?',
|
||||
(task['turn_id'],)).fetchone()
|
||||
execution=self.db.execute('SELECT execution_state,status_sequence FROM work_execution_state_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
||||
return {'task_id':task['turn_id'],'status':row['status'] if row else task['status'],
|
||||
'execution_state':execution['execution_state'] if execution else 'unknown',
|
||||
'status_sequence':execution['status_sequence'] if execution else 0,
|
||||
'result_received':bool(row),'result_digest':row['digest'] if row else None,
|
||||
'deadline':task['deadline']}
|
||||
|
||||
@@ -57,6 +73,28 @@ class TaskDelivery:
|
||||
await self.auth.authenticate(body,request)
|
||||
task=self.current(body)
|
||||
pending=self.broker.pending.get(body.task_id)
|
||||
if pending and pending.host==body.host_id:pending.progress.set()
|
||||
return no_cache(JSONResponse(self.receipt(task)))
|
||||
|
||||
async def progress(self,body:StateRequest,request:Request):
|
||||
await self.auth.authenticate(body,request)
|
||||
with self.db:
|
||||
self.state._owner_lock(body.binding_owner)
|
||||
task=self.current(body)
|
||||
if body.execution_state is not None:
|
||||
old=self.db.execute('SELECT * FROM work_execution_state_v3 WHERE task_id=?',(body.task_id,)).fetchone()
|
||||
terminal={'completed','failed','cancelled','interrupted'}
|
||||
if old and body.status_sequence==old['status_sequence'] and body.execution_state!=old['execution_state']:
|
||||
raise HTTPException(409,'execution_state_conflict')
|
||||
if not old:
|
||||
self.db.execute('INSERT INTO work_execution_state_v3 VALUES(?,?,?,?)',
|
||||
(body.task_id,body.execution_state,body.status_sequence,time.time()))
|
||||
elif body.status_sequence>old['status_sequence']:
|
||||
if old['execution_state'] in terminal and body.execution_state!=old['execution_state']:
|
||||
raise HTTPException(409,'execution_already_terminal')
|
||||
self.db.execute('UPDATE work_execution_state_v3 SET execution_state=?,status_sequence=?,updated_at=? WHERE task_id=?',
|
||||
(body.execution_state,body.status_sequence,time.time(),body.task_id))
|
||||
pending=self.broker.pending.get(body.task_id)
|
||||
if pending and pending.host==body.host_id:
|
||||
pending.progress.set()
|
||||
return no_cache(JSONResponse(self.receipt(task)))
|
||||
|
||||
Reference in New Issue
Block a user