fix(work): show client idle timeout as red result-unknown card

This commit is contained in:
ex_tianzy
2026-11-12 16:47:23 +08:00
parent 43e7b97c0f
commit b497f8604e
5 changed files with 68 additions and 5 deletions
+2
View File
@@ -17,3 +17,5 @@
ECS卡片、签名状态、凭据和部署套件:125 passed,1 skipped(已有可选测试)。客户端任务账本/原生运行/契约:27 passed。新增联合源码测试:1 passed,真实客户端账本及轮询代码通过签名HTTP向ECS上报四次状态,创建一次卡片、更新三次,原生任务仅受理一次。平台响应和原生Agent使用合成适配;没有发送真实Work消息或操作业务数据,不能代替私有化平台真人验收。
已覆盖:乱序状态、终态回退、身份/解绑、5秒合并、创建回执丢失、更新失败恢复、跨worker占用、创建进程中断、过期任务、卡片故障不阻断文字结果、候选开关及回滚保留父环境。尚需上线验证平台卡片格式兼容、更新权限/限流及真实跨端显示。
断连超时补充:等待可信任务进度超过request_timeout(默认45秒)时记录独立的client_idle_timeout原因;卡片在后续5秒维护周期变红,显示“客户端失联,任务结果待核对”。硬性任务期限、一般unknown仍单独处理,不冒充客户端断连;已知执行终态不被失联覆盖。原因持久保存,后续经设备签名验证的当前序号进度可清除;不重新下发或执行任务。实际耗时受部署参数、卡片限流及平台可用性影响,45—50秒不是SLA。
+10 -2
View File
@@ -58,6 +58,10 @@ class Pending:
progress: asyncio.Event = field(default_factory=asyncio.Event)
class ClientIdleTimeout(asyncio.TimeoutError):
"""No authenticated task progress arrived within the idle timeout."""
class Broker:
def __init__(self, settings, redis, store):
self.s, self.r, self.store = settings, redis, store
@@ -66,6 +70,7 @@ class Broker:
self.worker_id = uuid.uuid4().hex
self.lifecycle = None
self.files = None
self.task_cards = None
async def replies(self, pub):
async for message in pub.listen():
@@ -113,7 +118,8 @@ class Broker:
if item.future in done:
return item.future.result()
if not done:
raise asyncio.TimeoutError()
if remaining<=self.s.request_timeout:raise asyncio.TimeoutError()
raise ClientIdleTimeout()
finally:
progress.cancel()
with contextlib.suppress(asyncio.CancelledError):
@@ -212,10 +218,12 @@ class Broker:
self.lifecycle.transition(turn,"completed","finished")
self.store.audit(owner, "instruction", host, result.get("status","unknown") if task else "completed")
return result
except asyncio.TimeoutError:
except asyncio.TimeoutError as error:
self.store.finish(turn, "unknown")
if task:
self.lifecycle.transition(turn,"inflight","unknown")
if isinstance(error,ClientIdleTimeout) and self.task_cards:
self.task_cards.mark_connection_lost(turn)
return {"status": "unknown", "turn_id": turn,
"result": "等待结果超时,执行状态未知;不要直接重发有副作用的任务。"}
finally:
+16 -3
View File
@@ -10,10 +10,13 @@ DELIVERY={None:'尚未接收结果','ready':'已接收结果,等待投递','se
'replied':'结果已发送到当前会话','unknown':'结果投递待核对','discarded':'结果已停止交付'}
def card_content(task_id,execution,delivery,files=None):
def card_content(task_id,execution,delivery,files=None,connection_lost=False):
execution=execution if execution in EXECUTION else 'unknown'
color='green' if execution=='completed' else 'red' if execution in {'failed','interrupted'} else 'orange' if execution in {'waiting_for_approval','unknown'} else 'blue'
elements=[{'tag':'div','text':{'tag':'plain_text','content':'任务状态:'+EXECUTION[execution]}},
label=EXECUTION[execution]
if connection_lost and execution not in {'completed','failed','cancelled','interrupted'}:
color='red';label='客户端失联,任务结果待核对'
elements=[{'tag':'div','text':{'tag':'plain_text','content':'任务状态:'+label}},
{'tag':'div','text':{'tag':'plain_text','content':'文字回复:'+DELIVERY.get(delivery,'状态待核对')}},
{'tag':'note','elements':[{'tag':'plain_text','content':'任务编号:'+task_id}]}]
if files is not None:
@@ -38,10 +41,19 @@ class TaskCards:
self.state,self.db,self.platform,self.worker_id,self.clock=state,state.db,platform,worker_id,clock
self.files_enabled=files_enabled
with self.db:
self.db.execute('''CREATE TABLE IF NOT EXISTS work_task_connection_v3(
task_id TEXT PRIMARY KEY,reason TEXT NOT NULL)''')
self.db.execute('''CREATE TABLE IF NOT EXISTS work_task_cards_v3(
task_id TEXT PRIMARY KEY,message_id TEXT NOT NULL,digest TEXT NOT NULL,
status TEXT NOT NULL,worker_id TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)''')
def mark_connection_lost(self,task_id):
with self.db:
self.db.execute("INSERT INTO work_task_connection_v3 VALUES(?,'client_idle_timeout') ON CONFLICT DO NOTHING",(task_id,))
def clear_connection_lost(self,task_id):
with self.db:self.db.execute('DELETE FROM work_task_connection_v3 WHERE task_id=?',(task_id,))
def valid(self,task):
device=self.state.store.device(task['host_id']) if task else None
return bool(device and not device['revoked'] and device['owner']==task['owner']
@@ -57,7 +69,8 @@ class TaskCards:
if self.files_enabled:
files=[r['status']for r in self.db.execute("SELECT status FROM work_files_v1 WHERE task_id=? AND environment=? AND app_id=? AND direction='output'",
(task['turn_id'],self.state.environment,self.state.app_id)).fetchall()]
return card_content(task['turn_id'],execution,delivery['status'] if delivery else None,files)
lost=self.db.execute('SELECT reason FROM work_task_connection_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
return card_content(task['turn_id'],execution,delivery['status'] if delivery else None,files,connection_lost=bool(lost))
async def flush(self):
# Bound one maintenance pass; business result delivery remains independent.
+4
View File
@@ -44,6 +44,7 @@ class TaskDelivery:
if auth.s.work_task_cards_enabled:
from .task_cards import TaskCards
self.cards=TaskCards(state,platform,broker.worker_id,files_enabled=auth.s.work_files_enabled)
broker.task_cards=self.cards
def current(self, body):
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
@@ -96,6 +97,9 @@ class TaskDelivery:
self.db.execute('UPDATE work_execution_state_v3 SET execution_state=?,status_sequence=?,updated_at=? WHERE task_id=?',
(body.execution_state,body.status_sequence,time.time(),body.task_id))
pending=self.broker.pending.get(body.task_id)
if self.cards:
old=self.db.execute('SELECT status_sequence FROM work_execution_state_v3 WHERE task_id=?',(body.task_id,)).fetchone()
if old and body.status_sequence>=old['status_sequence']:self.cards.clear_connection_lost(body.task_id)
if pending and pending.host==body.host_id:
pending.progress.set()
return no_cache(JSONResponse(self.receipt(task)))
+36
View File
@@ -153,6 +153,42 @@ async def test_timeout_projects_unknown_without_claiming_failure(env,monkeypatch
assert '状态待核对' in text and '执行失败' not in text
async def test_lost_client_turns_red_persists_and_recovers_on_authenticated_progress(env,monkeypatch):
task(env);cards,calls,clock=setup_cards(env,monkeypatch)
api=env['app'].state.deliveries;api.cards=cards
await report(env,'running',1);await cards.flush()
cards.mark_connection_lost('task-durable')
with env['store'].db:env['store'].db.execute("UPDATE work_tasks_v2 SET status='unknown'")
clock[0]+=6;await cards.flush()
card=calls[-1][2]
assert card['header']['template']=='red'
text=json.dumps(card,ensure_ascii=False)
assert '客户端失联,任务结果待核对' in text and '执行失败' not in text
reopened=TaskCards(env['store'].lifecycle,env['platform'],'worker-new',clock=lambda:clock[0])
assert reopened.snapshot(env['store'].lifecycle.task('task-durable'))['header']['template']=='red'
await report(env,'completed',2);clock[0]+=6;await cards.flush()
assert calls[-1][2]['header']['template']=='green'
assert not env['store'].db.execute('SELECT * FROM work_task_connection_v3').fetchall()
async def test_idle_timeout_is_distinct_from_hard_tracking_limit():
from types import SimpleNamespace
from relay.connector import Broker,Pending,ClientIdleTimeout
broker=object.__new__(Broker)
broker.s=SimpleNamespace(request_timeout=.01,task_tracking_timeout=1)
item=Pending(asyncio.get_running_loop().create_future(),None,None,'synthetic','synthetic')
with pytest.raises(ClientIdleTimeout):await broker.wait_tracked(item)
broker.s.task_tracking_timeout=0
with pytest.raises(asyncio.TimeoutError)as error:await broker.wait_tracked(item)
assert not isinstance(error.value,ClientIdleTimeout)
@pytest.mark.parametrize('state',['completed','failed','cancelled','interrupted'])
def test_lost_connection_never_overwrites_known_terminal_state(state):
card=card_content('synthetic',state,None,connection_lost=True)
assert '客户端失联' not in json.dumps(card,ensure_ascii=False)
async def test_platform_card_wire_format_and_receipt_validation(env):
seen=[]
def handler(request):