fix(work): show client idle timeout as red result-unknown card
This commit is contained in:
@@ -58,6 +58,10 @@ class Pending:
|
||||
progress: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
|
||||
|
||||
class ClientIdleTimeout(asyncio.TimeoutError):
|
||||
"""No authenticated task progress arrived within the idle timeout."""
|
||||
|
||||
|
||||
class Broker:
|
||||
def __init__(self, settings, redis, store):
|
||||
self.s, self.r, self.store = settings, redis, store
|
||||
@@ -66,6 +70,7 @@ class Broker:
|
||||
self.worker_id = uuid.uuid4().hex
|
||||
self.lifecycle = None
|
||||
self.files = None
|
||||
self.task_cards = None
|
||||
|
||||
async def replies(self, pub):
|
||||
async for message in pub.listen():
|
||||
@@ -113,7 +118,8 @@ class Broker:
|
||||
if item.future in done:
|
||||
return item.future.result()
|
||||
if not done:
|
||||
raise asyncio.TimeoutError()
|
||||
if remaining<=self.s.request_timeout:raise asyncio.TimeoutError()
|
||||
raise ClientIdleTimeout()
|
||||
finally:
|
||||
progress.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
@@ -212,10 +218,12 @@ class Broker:
|
||||
self.lifecycle.transition(turn,"completed","finished")
|
||||
self.store.audit(owner, "instruction", host, result.get("status","unknown") if task else "completed")
|
||||
return result
|
||||
except asyncio.TimeoutError:
|
||||
except asyncio.TimeoutError as error:
|
||||
self.store.finish(turn, "unknown")
|
||||
if task:
|
||||
self.lifecycle.transition(turn,"inflight","unknown")
|
||||
if isinstance(error,ClientIdleTimeout) and self.task_cards:
|
||||
self.task_cards.mark_connection_lost(turn)
|
||||
return {"status": "unknown", "turn_id": turn,
|
||||
"result": "等待结果超时,执行状态未知;不要直接重发有副作用的任务。"}
|
||||
finally:
|
||||
|
||||
@@ -10,10 +10,13 @@ DELIVERY={None:'尚未接收结果','ready':'已接收结果,等待投递','se
|
||||
'replied':'结果已发送到当前会话','unknown':'结果投递待核对','discarded':'结果已停止交付'}
|
||||
|
||||
|
||||
def card_content(task_id,execution,delivery,files=None):
|
||||
def card_content(task_id,execution,delivery,files=None,connection_lost=False):
|
||||
execution=execution if execution in EXECUTION else 'unknown'
|
||||
color='green' if execution=='completed' else 'red' if execution in {'failed','interrupted'} else 'orange' if execution in {'waiting_for_approval','unknown'} else 'blue'
|
||||
elements=[{'tag':'div','text':{'tag':'plain_text','content':'任务状态:'+EXECUTION[execution]}},
|
||||
label=EXECUTION[execution]
|
||||
if connection_lost and execution not in {'completed','failed','cancelled','interrupted'}:
|
||||
color='red';label='客户端失联,任务结果待核对'
|
||||
elements=[{'tag':'div','text':{'tag':'plain_text','content':'任务状态:'+label}},
|
||||
{'tag':'div','text':{'tag':'plain_text','content':'文字回复:'+DELIVERY.get(delivery,'状态待核对')}},
|
||||
{'tag':'note','elements':[{'tag':'plain_text','content':'任务编号:'+task_id}]}]
|
||||
if files is not None:
|
||||
@@ -38,10 +41,19 @@ class TaskCards:
|
||||
self.state,self.db,self.platform,self.worker_id,self.clock=state,state.db,platform,worker_id,clock
|
||||
self.files_enabled=files_enabled
|
||||
with self.db:
|
||||
self.db.execute('''CREATE TABLE IF NOT EXISTS work_task_connection_v3(
|
||||
task_id TEXT PRIMARY KEY,reason TEXT NOT NULL)''')
|
||||
self.db.execute('''CREATE TABLE IF NOT EXISTS work_task_cards_v3(
|
||||
task_id TEXT PRIMARY KEY,message_id TEXT NOT NULL,digest TEXT NOT NULL,
|
||||
status TEXT NOT NULL,worker_id TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)''')
|
||||
|
||||
def mark_connection_lost(self,task_id):
|
||||
with self.db:
|
||||
self.db.execute("INSERT INTO work_task_connection_v3 VALUES(?,'client_idle_timeout') ON CONFLICT DO NOTHING",(task_id,))
|
||||
|
||||
def clear_connection_lost(self,task_id):
|
||||
with self.db:self.db.execute('DELETE FROM work_task_connection_v3 WHERE task_id=?',(task_id,))
|
||||
|
||||
def valid(self,task):
|
||||
device=self.state.store.device(task['host_id']) if task else None
|
||||
return bool(device and not device['revoked'] and device['owner']==task['owner']
|
||||
@@ -57,7 +69,8 @@ class TaskCards:
|
||||
if self.files_enabled:
|
||||
files=[r['status']for r in self.db.execute("SELECT status FROM work_files_v1 WHERE task_id=? AND environment=? AND app_id=? AND direction='output'",
|
||||
(task['turn_id'],self.state.environment,self.state.app_id)).fetchall()]
|
||||
return card_content(task['turn_id'],execution,delivery['status'] if delivery else None,files)
|
||||
lost=self.db.execute('SELECT reason FROM work_task_connection_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
||||
return card_content(task['turn_id'],execution,delivery['status'] if delivery else None,files,connection_lost=bool(lost))
|
||||
|
||||
async def flush(self):
|
||||
# Bound one maintenance pass; business result delivery remains independent.
|
||||
|
||||
@@ -44,6 +44,7 @@ class TaskDelivery:
|
||||
if auth.s.work_task_cards_enabled:
|
||||
from .task_cards import TaskCards
|
||||
self.cards=TaskCards(state,platform,broker.worker_id,files_enabled=auth.s.work_files_enabled)
|
||||
broker.task_cards=self.cards
|
||||
|
||||
def current(self, body):
|
||||
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
|
||||
@@ -96,6 +97,9 @@ class TaskDelivery:
|
||||
self.db.execute('UPDATE work_execution_state_v3 SET execution_state=?,status_sequence=?,updated_at=? WHERE task_id=?',
|
||||
(body.execution_state,body.status_sequence,time.time(),body.task_id))
|
||||
pending=self.broker.pending.get(body.task_id)
|
||||
if self.cards:
|
||||
old=self.db.execute('SELECT status_sequence FROM work_execution_state_v3 WHERE task_id=?',(body.task_id,)).fetchone()
|
||||
if old and body.status_sequence>=old['status_sequence']:self.cards.clear_connection_lost(body.task_id)
|
||||
if pending and pending.host==body.host_id:
|
||||
pending.progress.set()
|
||||
return no_cache(JSONResponse(self.receipt(task)))
|
||||
|
||||
Reference in New Issue
Block a user