diff --git a/deployable/REMOTE_INTERACTION_C.md b/deployable/REMOTE_INTERACTION_C.md index 500f40f..2c31397 100644 --- a/deployable/REMOTE_INTERACTION_C.md +++ b/deployable/REMOTE_INTERACTION_C.md @@ -17,3 +17,5 @@ ECS卡片、签名状态、凭据和部署套件:125 passed,1 skipped(已有可选测试)。客户端任务账本/原生运行/契约:27 passed。新增联合源码测试:1 passed,真实客户端账本及轮询代码通过签名HTTP向ECS上报四次状态,创建一次卡片、更新三次,原生任务仅受理一次。平台响应和原生Agent使用合成适配;没有发送真实Work消息或操作业务数据,不能代替私有化平台真人验收。 已覆盖:乱序状态、终态回退、身份/解绑、5秒合并、创建回执丢失、更新失败恢复、跨worker占用、创建进程中断、过期任务、卡片故障不阻断文字结果、候选开关及回滚保留父环境。尚需上线验证平台卡片格式兼容、更新权限/限流及真实跨端显示。 + +断连超时补充:等待可信任务进度超过request_timeout(默认45秒)时记录独立的client_idle_timeout原因;卡片在后续5秒维护周期变红,显示“客户端失联,任务结果待核对”。硬性任务期限、一般unknown仍单独处理,不冒充客户端断连;已知执行终态不被失联覆盖。原因持久保存,后续经设备签名验证的当前序号进度可清除;不重新下发或执行任务。实际耗时受部署参数、卡片限流及平台可用性影响,45—50秒不是SLA。 diff --git a/deployable/relay/connector.py b/deployable/relay/connector.py index 3ece602..edc7c65 100644 --- a/deployable/relay/connector.py +++ b/deployable/relay/connector.py @@ -58,6 +58,10 @@ class Pending: progress: asyncio.Event = field(default_factory=asyncio.Event) +class ClientIdleTimeout(asyncio.TimeoutError): + """No authenticated task progress arrived within the idle timeout.""" + + class Broker: def __init__(self, settings, redis, store): self.s, self.r, self.store = settings, redis, store @@ -66,6 +70,7 @@ class Broker: self.worker_id = uuid.uuid4().hex self.lifecycle = None self.files = None + self.task_cards = None async def replies(self, pub): async for message in pub.listen(): @@ -113,7 +118,8 @@ class Broker: if item.future in done: return item.future.result() if not done: - raise asyncio.TimeoutError() + if remaining<=self.s.request_timeout:raise asyncio.TimeoutError() + raise ClientIdleTimeout() finally: progress.cancel() with contextlib.suppress(asyncio.CancelledError): @@ -212,10 +218,12 @@ class Broker: self.lifecycle.transition(turn,"completed","finished") self.store.audit(owner, "instruction", host, result.get("status","unknown") if task else "completed") return result - except asyncio.TimeoutError: + except asyncio.TimeoutError as error: self.store.finish(turn, "unknown") if task: self.lifecycle.transition(turn,"inflight","unknown") + if isinstance(error,ClientIdleTimeout) and self.task_cards: + self.task_cards.mark_connection_lost(turn) return {"status": "unknown", "turn_id": turn, "result": "等待结果超时,执行状态未知;不要直接重发有副作用的任务。"} finally: diff --git a/deployable/relay/task_cards.py b/deployable/relay/task_cards.py index 3dcb726..9108bf1 100644 --- a/deployable/relay/task_cards.py +++ b/deployable/relay/task_cards.py @@ -10,10 +10,13 @@ DELIVERY={None:'尚未接收结果','ready':'已接收结果,等待投递','se 'replied':'结果已发送到当前会话','unknown':'结果投递待核对','discarded':'结果已停止交付'} -def card_content(task_id,execution,delivery,files=None): +def card_content(task_id,execution,delivery,files=None,connection_lost=False): execution=execution if execution in EXECUTION else 'unknown' color='green' if execution=='completed' else 'red' if execution in {'failed','interrupted'} else 'orange' if execution in {'waiting_for_approval','unknown'} else 'blue' - elements=[{'tag':'div','text':{'tag':'plain_text','content':'任务状态:'+EXECUTION[execution]}}, + label=EXECUTION[execution] + if connection_lost and execution not in {'completed','failed','cancelled','interrupted'}: + color='red';label='客户端失联,任务结果待核对' + elements=[{'tag':'div','text':{'tag':'plain_text','content':'任务状态:'+label}}, {'tag':'div','text':{'tag':'plain_text','content':'文字回复:'+DELIVERY.get(delivery,'状态待核对')}}, {'tag':'note','elements':[{'tag':'plain_text','content':'任务编号:'+task_id}]}] if files is not None: @@ -38,10 +41,19 @@ class TaskCards: self.state,self.db,self.platform,self.worker_id,self.clock=state,state.db,platform,worker_id,clock self.files_enabled=files_enabled with self.db: + self.db.execute('''CREATE TABLE IF NOT EXISTS work_task_connection_v3( + task_id TEXT PRIMARY KEY,reason TEXT NOT NULL)''') self.db.execute('''CREATE TABLE IF NOT EXISTS work_task_cards_v3( task_id TEXT PRIMARY KEY,message_id TEXT NOT NULL,digest TEXT NOT NULL, status TEXT NOT NULL,worker_id TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)''') + def mark_connection_lost(self,task_id): + with self.db: + self.db.execute("INSERT INTO work_task_connection_v3 VALUES(?,'client_idle_timeout') ON CONFLICT DO NOTHING",(task_id,)) + + def clear_connection_lost(self,task_id): + with self.db:self.db.execute('DELETE FROM work_task_connection_v3 WHERE task_id=?',(task_id,)) + def valid(self,task): device=self.state.store.device(task['host_id']) if task else None return bool(device and not device['revoked'] and device['owner']==task['owner'] @@ -57,7 +69,8 @@ class TaskCards: if self.files_enabled: files=[r['status']for r in self.db.execute("SELECT status FROM work_files_v1 WHERE task_id=? AND environment=? AND app_id=? AND direction='output'", (task['turn_id'],self.state.environment,self.state.app_id)).fetchall()] - return card_content(task['turn_id'],execution,delivery['status'] if delivery else None,files) + lost=self.db.execute('SELECT reason FROM work_task_connection_v3 WHERE task_id=?',(task['turn_id'],)).fetchone() + return card_content(task['turn_id'],execution,delivery['status'] if delivery else None,files,connection_lost=bool(lost)) async def flush(self): # Bound one maintenance pass; business result delivery remains independent. diff --git a/deployable/relay/task_delivery.py b/deployable/relay/task_delivery.py index 2a4cf65..f684794 100644 --- a/deployable/relay/task_delivery.py +++ b/deployable/relay/task_delivery.py @@ -44,6 +44,7 @@ class TaskDelivery: if auth.s.work_task_cards_enabled: from .task_cards import TaskCards self.cards=TaskCards(state,platform,broker.worker_id,files_enabled=auth.s.work_files_enabled) + broker.task_cards=self.cards def current(self, body): self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation) @@ -96,6 +97,9 @@ class TaskDelivery: self.db.execute('UPDATE work_execution_state_v3 SET execution_state=?,status_sequence=?,updated_at=? WHERE task_id=?', (body.execution_state,body.status_sequence,time.time(),body.task_id)) pending=self.broker.pending.get(body.task_id) + if self.cards: + old=self.db.execute('SELECT status_sequence FROM work_execution_state_v3 WHERE task_id=?',(body.task_id,)).fetchone() + if old and body.status_sequence>=old['status_sequence']:self.cards.clear_connection_lost(body.task_id) if pending and pending.host==body.host_id: pending.progress.set() return no_cache(JSONResponse(self.receipt(task))) diff --git a/deployable/tests/test_task_cards.py b/deployable/tests/test_task_cards.py index 42c96bd..85afeba 100644 --- a/deployable/tests/test_task_cards.py +++ b/deployable/tests/test_task_cards.py @@ -153,6 +153,42 @@ async def test_timeout_projects_unknown_without_claiming_failure(env,monkeypatch assert '状态待核对' in text and '执行失败' not in text +async def test_lost_client_turns_red_persists_and_recovers_on_authenticated_progress(env,monkeypatch): + task(env);cards,calls,clock=setup_cards(env,monkeypatch) + api=env['app'].state.deliveries;api.cards=cards + await report(env,'running',1);await cards.flush() + cards.mark_connection_lost('task-durable') + with env['store'].db:env['store'].db.execute("UPDATE work_tasks_v2 SET status='unknown'") + clock[0]+=6;await cards.flush() + card=calls[-1][2] + assert card['header']['template']=='red' + text=json.dumps(card,ensure_ascii=False) + assert '客户端失联,任务结果待核对' in text and '执行失败' not in text + reopened=TaskCards(env['store'].lifecycle,env['platform'],'worker-new',clock=lambda:clock[0]) + assert reopened.snapshot(env['store'].lifecycle.task('task-durable'))['header']['template']=='red' + await report(env,'completed',2);clock[0]+=6;await cards.flush() + assert calls[-1][2]['header']['template']=='green' + assert not env['store'].db.execute('SELECT * FROM work_task_connection_v3').fetchall() + + +async def test_idle_timeout_is_distinct_from_hard_tracking_limit(): + from types import SimpleNamespace + from relay.connector import Broker,Pending,ClientIdleTimeout + broker=object.__new__(Broker) + broker.s=SimpleNamespace(request_timeout=.01,task_tracking_timeout=1) + item=Pending(asyncio.get_running_loop().create_future(),None,None,'synthetic','synthetic') + with pytest.raises(ClientIdleTimeout):await broker.wait_tracked(item) + broker.s.task_tracking_timeout=0 + with pytest.raises(asyncio.TimeoutError)as error:await broker.wait_tracked(item) + assert not isinstance(error.value,ClientIdleTimeout) + + +@pytest.mark.parametrize('state',['completed','failed','cancelled','interrupted']) +def test_lost_connection_never_overwrites_known_terminal_state(state): + card=card_content('synthetic',state,None,connection_lost=True) + assert '客户端失联' not in json.dumps(card,ensure_ascii=False) + + async def test_platform_card_wire_format_and_receipt_validation(env): seen=[] def handler(request):