Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d2881b82b0 | ||
|
|
b497f8604e | ||
|
|
43e7b97c0f | ||
|
|
a33ce02fa6 | ||
|
|
26bb8de927 | ||
|
|
abe2550dd0 | ||
|
|
2e0d5987c0 |
@@ -0,0 +1,21 @@
|
||||
# C 阶段服务端进度
|
||||
|
||||
2026-11-12,基线 2e0d598,分支 feat/remote-interaction-C-20261112。
|
||||
|
||||
首批实现:新增设备签名的 POST /work/v2/tasks/progress,持久记录状态及递增序号;旧 state 接口保持签名兼容;本人、设备、当前绑定、任务期限校验沿用既有授权。旧序号不覆盖新状态,重复状态不重复执行,同序号冲突返回 409,终态不回退。执行完成、结果接收、平台投递分别判断。
|
||||
|
||||
已实现状态卡片:原私聊消息下创建一张 interactive 卡片,PATCH 更新同一消息;5秒合并更新,每次扫描最多发送10次请求,单次扫描限100条最近一天内任务。卡片只含任务编号、执行状态与交付状态,不含正文和工具参数,不含远程批准按钮。任务编号与源消息从服务端账本取得。创建回执未知停止重建;已知消息ID的绝对状态更新失败可在后续周期更新到最新状态。卡片维护独立于结果投递任务,失败保留既有文字/附件交付。
|
||||
|
||||
尚未实现:流式正文、可信按钮回调与远程审批、持久下发队列和取消、完整故障恢复及多设备会话管理。当前状态表只存任务标识、枚举、序号和服务端时间,不存正文或凭据。
|
||||
|
||||
客户端配套分支同名,必须先升级 ECS 再升级客户端。当前未部署、未更改长连接订阅,也未新增平台权限;真实私有化平台卡片兼容性待后续探测和验收。
|
||||
|
||||
默认关闭卡片;测试部署使用 prepare-update --enable-task-cards,将 HN_WORK_TASK_CARDS_ENABLED=true 仅加到候选connector。回滚使用父计划原环境,绑定、密钥与存储不重建。此开关不代表已验证私有化平台支持 interactive 回复和消息 PATCH。旧客户端仍可用原 state 接口,但不能凭结果投递推断执行成功,缺失执行状态时卡片显示待核对。
|
||||
|
||||
## 验证记录
|
||||
|
||||
ECS卡片、签名状态、凭据和部署套件:125 passed,1 skipped(已有可选测试)。客户端任务账本/原生运行/契约:27 passed。新增联合源码测试:1 passed,真实客户端账本及轮询代码通过签名HTTP向ECS上报四次状态,创建一次卡片、更新三次,原生任务仅受理一次。平台响应和原生Agent使用合成适配;没有发送真实Work消息或操作业务数据,不能代替私有化平台真人验收。
|
||||
|
||||
已覆盖:乱序状态、终态回退、身份/解绑、5秒合并、创建回执丢失、更新失败恢复、跨worker占用、创建进程中断、过期任务、卡片故障不阻断文字结果、候选开关及回滚保留父环境。尚需上线验证平台卡片格式兼容、更新权限/限流及真实跨端显示。
|
||||
|
||||
断连超时补充:等待可信任务进度超过request_timeout(默认45秒)时记录独立的client_idle_timeout原因;卡片在后续5秒维护周期变红,显示“客户端失联,任务结果待核对”。硬性任务期限、一般unknown仍单独处理,不冒充客户端断连;已知执行终态不被失联覆盖。原因持久保存,后续经设备签名验证的当前序号进度可清除;不重新下发或执行任务。实际耗时受部署参数、卡片限流及平台可用性影响,45—50秒不是SLA。
|
||||
@@ -41,6 +41,7 @@ class Settings(BaseSettings):
|
||||
max_connections: int = 1000
|
||||
max_result: int = 16000
|
||||
work_files_enabled: bool = False
|
||||
work_task_cards_enabled: bool = False
|
||||
work_file_max_bytes: int = 10*1024*1024
|
||||
work_file_batch_limit: int = 20
|
||||
work_file_retention: int = 86400
|
||||
@@ -56,6 +57,7 @@ class Settings(BaseSettings):
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_runtime(self):
|
||||
if self.work_task_cards_enabled and self.credential_mode!='broker':raise ValueError('task cards require broker mode')
|
||||
if not 1024<=self.work_file_max_bytes<=100*1024*1024 or not 1<=self.work_file_batch_limit<=50 or not 300<=self.work_file_retention<=7*86400:
|
||||
raise ValueError('invalid work file deployment limits')
|
||||
if self.work_files_enabled and self.credential_mode!='broker':raise ValueError('work files require broker mode')
|
||||
|
||||
@@ -58,6 +58,10 @@ class Pending:
|
||||
progress: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
|
||||
|
||||
class ClientIdleTimeout(asyncio.TimeoutError):
|
||||
"""No authenticated task progress arrived within the idle timeout."""
|
||||
|
||||
|
||||
class Broker:
|
||||
def __init__(self, settings, redis, store):
|
||||
self.s, self.r, self.store = settings, redis, store
|
||||
@@ -66,6 +70,7 @@ class Broker:
|
||||
self.worker_id = uuid.uuid4().hex
|
||||
self.lifecycle = None
|
||||
self.files = None
|
||||
self.task_cards = None
|
||||
|
||||
async def replies(self, pub):
|
||||
async for message in pub.listen():
|
||||
@@ -113,7 +118,8 @@ class Broker:
|
||||
if item.future in done:
|
||||
return item.future.result()
|
||||
if not done:
|
||||
raise asyncio.TimeoutError()
|
||||
if remaining<=self.s.request_timeout:raise asyncio.TimeoutError()
|
||||
raise ClientIdleTimeout()
|
||||
finally:
|
||||
progress.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
@@ -212,10 +218,12 @@ class Broker:
|
||||
self.lifecycle.transition(turn,"completed","finished")
|
||||
self.store.audit(owner, "instruction", host, result.get("status","unknown") if task else "completed")
|
||||
return result
|
||||
except asyncio.TimeoutError:
|
||||
except asyncio.TimeoutError as error:
|
||||
self.store.finish(turn, "unknown")
|
||||
if task:
|
||||
self.lifecycle.transition(turn,"inflight","unknown")
|
||||
if isinstance(error,ClientIdleTimeout) and self.task_cards:
|
||||
self.task_cards.mark_connection_lost(turn)
|
||||
return {"status": "unknown", "turn_id": turn,
|
||||
"result": "等待结果超时,执行状态未知;不要直接重发有副作用的任务。"}
|
||||
finally:
|
||||
@@ -298,10 +306,15 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
# No credentials or raw exception text in background logs.
|
||||
pass
|
||||
lifecycle_task = asyncio.create_task(maintenance()) if lifecycle else None
|
||||
async def card_maintenance():
|
||||
while True:
|
||||
await asyncio.sleep(5)
|
||||
with contextlib.suppress(Exception):await deliveries.cards.flush()
|
||||
cards_task=asyncio.create_task(card_maintenance()) if lifecycle and deliveries.cards else None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
work = [reply_task,sweep_task,*background] + ([lifecycle_task] if lifecycle_task else [])
|
||||
work = [reply_task,sweep_task,*background] + ([lifecycle_task] if lifecycle_task else []) + ([cards_task] if cards_task else [])
|
||||
for task in work:
|
||||
task.cancel()
|
||||
await asyncio.gather(*work, return_exceptions=True)
|
||||
|
||||
@@ -122,6 +122,22 @@ class Feishu:
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json={"msg_type": "text", "content": json.dumps({"text": text}, ensure_ascii=False)})
|
||||
|
||||
async def reply_task_card(self,message_id,card,delivery_id,*,token):
|
||||
import re
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,128}',message_id):raise PlatformError('invalid reply reference')
|
||||
data=await self._json('POST',f'/open-apis/im/v1/messages/{message_id}/reply',
|
||||
headers={'Authorization':'Bearer '+token},json={'msg_type':'interactive',
|
||||
'content':json.dumps(card,ensure_ascii=False),'uuid':delivery_id})
|
||||
result=data.get('data',{}).get('message_id')
|
||||
if not isinstance(result,str)or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}',result):raise PlatformError('card receipt invalid')
|
||||
return result
|
||||
|
||||
async def update_task_card(self,message_id,card,*,token):
|
||||
import re
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,128}',message_id):raise PlatformError('invalid card reference')
|
||||
await self._json('PATCH',f'/open-apis/im/v1/messages/{message_id}',
|
||||
headers={'Authorization':'Bearer '+token},json={'content':json.dumps(card,ensure_ascii=False)})
|
||||
|
||||
async def download_message_file(self,message_id,file_key,max_bytes):
|
||||
import re
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,128}',message_id)or not re.fullmatch(r'[A-Za-z0-9_-]{1,256}',file_key):raise PlatformError('invalid file reference')
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Durable private task status cards. No task body, credentials or approval buttons."""
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
|
||||
|
||||
EXECUTION={'queued':'排队中','running':'执行中','waiting_for_approval':'等待本人确认',
|
||||
'completed':'执行完成','failed':'执行失败','cancelled':'已取消','interrupted':'已中断','unknown':'状态待核对'}
|
||||
DELIVERY={None:'尚未接收结果','ready':'已接收结果,等待投递','sending':'正在投递结果',
|
||||
'replied':'结果已发送到当前会话','unknown':'结果投递待核对','discarded':'结果已停止交付'}
|
||||
|
||||
|
||||
def card_content(task_id,execution,delivery,files=None,connection_lost=False):
|
||||
execution=execution if execution in EXECUTION else 'unknown'
|
||||
color='green' if execution=='completed' else 'red' if execution in {'failed','interrupted'} else 'orange' if execution in {'waiting_for_approval','unknown'} else 'blue'
|
||||
label=EXECUTION[execution]
|
||||
if connection_lost and execution not in {'completed','failed','cancelled','interrupted'}:
|
||||
color='red';label='客户端失联,任务结果待核对'
|
||||
elements=[{'tag':'div','text':{'tag':'plain_text','content':'任务状态:'+label}},
|
||||
{'tag':'div','text':{'tag':'plain_text','content':'文字回复:'+DELIVERY.get(delivery,'状态待核对')}},
|
||||
{'tag':'note','elements':[{'tag':'plain_text','content':'任务编号:'+task_id}]}]
|
||||
if execution=='waiting_for_approval':
|
||||
elements.insert(1,{'tag':'div','text':{'tag':'plain_text','content':'请在客户端查看并确认具体操作。本卡片暂不提供远程批准。'}})
|
||||
if execution=='completed' and delivery!='replied':
|
||||
elements.insert(1,{'tag':'div','text':{'tag':'plain_text','content':'任务已执行完成,结果交付请看下方状态。'}})
|
||||
return {'config':{'wide_screen_mode':True,'enable_forward':False},
|
||||
'header':{'template':color,'title':{'tag':'plain_text','content':'海诺任务进度'}},'elements':elements}
|
||||
|
||||
|
||||
class TaskCards:
|
||||
def __init__(self,state,platform,worker_id,clock=time.time,files_enabled=False):
|
||||
self.state,self.db,self.platform,self.worker_id,self.clock=state,state.db,platform,worker_id,clock
|
||||
self.files_enabled=files_enabled
|
||||
with self.db:
|
||||
self.db.execute('''CREATE TABLE IF NOT EXISTS work_task_connection_v3(
|
||||
task_id TEXT PRIMARY KEY,reason TEXT NOT NULL)''')
|
||||
self.db.execute('''CREATE TABLE IF NOT EXISTS work_task_cards_v3(
|
||||
task_id TEXT PRIMARY KEY,message_id TEXT NOT NULL,digest TEXT NOT NULL,
|
||||
status TEXT NOT NULL,worker_id TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)''')
|
||||
|
||||
def mark_connection_lost(self,task_id):
|
||||
with self.db:
|
||||
self.db.execute("INSERT INTO work_task_connection_v3 VALUES(?,'client_idle_timeout') ON CONFLICT DO NOTHING",(task_id,))
|
||||
|
||||
def clear_connection_lost(self,task_id):
|
||||
with self.db:self.db.execute('DELETE FROM work_task_connection_v3 WHERE task_id=?',(task_id,))
|
||||
|
||||
def valid(self,task):
|
||||
device=self.state.store.device(task['host_id']) if task else None
|
||||
return bool(device and not device['revoked'] and device['owner']==task['owner']
|
||||
and device['binding_generation']==task['binding_generation'] and task['source_message_id'] and task['chat_id'])
|
||||
|
||||
def snapshot(self,task):
|
||||
row=self.db.execute('SELECT execution_state FROM work_execution_state_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
||||
delivery=self.db.execute('SELECT status FROM work_result_outbox_v2 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
||||
execution=row['execution_state'] if row else 'unknown'
|
||||
if task['status']=='unknown' and execution not in {'completed','failed','cancelled','interrupted'}:execution='unknown'
|
||||
if task['deadline']<=self.clock() and execution not in {'completed','failed','cancelled','interrupted'}:execution='unknown'
|
||||
lost=self.db.execute('SELECT reason FROM work_task_connection_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
||||
return card_content(task['turn_id'],execution,delivery['status'] if delivery else None,connection_lost=bool(lost))
|
||||
|
||||
async def flush(self):
|
||||
# Bound one maintenance pass; business result delivery remains independent.
|
||||
rows=self.db.execute('''SELECT * FROM work_tasks_v2 WHERE environment=? AND app_id=?
|
||||
AND source_message_id<>'' AND chat_id<>'' AND updated_at>? ORDER BY updated_at DESC LIMIT 100''',
|
||||
(self.state.environment,self.state.app_id,self.clock()-86400)).fetchall()
|
||||
sent=0
|
||||
for raw in rows:
|
||||
if sent>=10:break
|
||||
task=dict(raw)
|
||||
if not self.valid(task):continue
|
||||
card=self.snapshot(task)
|
||||
digest=hashlib.sha256(json.dumps(card,sort_keys=True,ensure_ascii=True).encode()).hexdigest()
|
||||
old=self.db.execute('SELECT * FROM work_task_cards_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
||||
if old and (old['status']=='unknown' or old['digest']==digest or self.clock()-old['updated_at']<5):continue
|
||||
if not old and task['deadline']<=self.clock():continue
|
||||
try:token=await self.platform.tenant_token()
|
||||
except Exception:continue
|
||||
with self.db:
|
||||
self.state._owner_lock(task['owner'])
|
||||
task=self.state.task(task['turn_id'])
|
||||
if not self.valid(task):continue
|
||||
old=self.db.execute('SELECT * FROM work_task_cards_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
||||
if old and old['status']=='sending':
|
||||
# A lost creation receipt cannot safely become another card.
|
||||
if self.clock()-old['updated_at']>=60:
|
||||
self.db.execute("UPDATE work_task_cards_v3 SET status=? WHERE task_id=?",
|
||||
('ready' if old['message_id'] else 'unknown',task['turn_id']))
|
||||
continue
|
||||
if old and (old['status']=='unknown' or old['digest']==digest or self.clock()-old['updated_at']<5):continue
|
||||
# Recompute after token/lock awaits so an older observation cannot overwrite new status.
|
||||
card=self.snapshot(task)
|
||||
digest=hashlib.sha256(json.dumps(card,sort_keys=True,ensure_ascii=True).encode()).hexdigest()
|
||||
message_id=old['message_id'] if old else ''
|
||||
if not old:
|
||||
self.db.execute('INSERT INTO work_task_cards_v3 VALUES(?,?,?,?,?,?)',
|
||||
(task['turn_id'],'','','sending',self.worker_id,self.clock()))
|
||||
else:
|
||||
self.db.execute("UPDATE work_task_cards_v3 SET status='sending',worker_id=?,updated_at=? WHERE task_id=?",
|
||||
(self.worker_id,self.clock(),task['turn_id']))
|
||||
try:
|
||||
sent+=1
|
||||
if message_id:await self.platform.update_task_card(message_id,card,token=token)
|
||||
else:
|
||||
delivery_id=hashlib.sha256(('task-card-v3\0'+self.state.environment+'\0'+self.state.app_id+'\0'+task['turn_id']).encode()).hexdigest()[:32]
|
||||
message_id=await self.platform.reply_task_card(task['source_message_id'],card,delivery_id,token=token)
|
||||
status='ready';saved_digest=digest
|
||||
except Exception:
|
||||
status='ready' if message_id else 'unknown';saved_digest=old['digest'] if old else ''
|
||||
with self.db:
|
||||
self.db.execute("UPDATE work_task_cards_v3 SET message_id=?,digest=?,status=?,updated_at=? WHERE task_id=? AND status='sending' AND worker_id=?",
|
||||
(message_id,saved_digest,status,self.clock(),task['turn_id'],self.worker_id))
|
||||
@@ -17,12 +17,20 @@ class ResultRequest(TaskRequest):
|
||||
result: str = Field(max_length=16000, repr=False)
|
||||
|
||||
|
||||
class StateRequest(TaskRequest):
|
||||
execution_state: str = Field(pattern=r'^(queued|running|waiting_for_approval|completed|failed|cancelled|interrupted|unknown)$')
|
||||
status_sequence: int = Field(ge=1, le=2147483647, strict=True)
|
||||
|
||||
|
||||
class TaskDelivery:
|
||||
def __init__(self, auth, state, broker, platform):
|
||||
self.auth,self.state,self.broker,self.platform=auth,state,broker,platform
|
||||
self.db=state.db
|
||||
self.keyring=auth.broker.store.keyring
|
||||
with self.db:
|
||||
self.db.execute('''CREATE TABLE IF NOT EXISTS work_execution_state_v3(
|
||||
task_id TEXT PRIMARY KEY,execution_state TEXT NOT NULL,
|
||||
status_sequence BIGINT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)''')
|
||||
self.db.execute("""CREATE TABLE IF NOT EXISTS work_result_outbox_v2(
|
||||
task_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
|
||||
owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,
|
||||
@@ -30,7 +38,13 @@ class TaskDelivery:
|
||||
status TEXT NOT NULL,updated_at DOUBLE PRECISION NOT NULL)""")
|
||||
self.router=APIRouter()
|
||||
self.router.add_api_route('/work/v2/tasks/state',self.status,methods=['POST'])
|
||||
self.router.add_api_route('/work/v2/tasks/progress',self.progress,methods=['POST'])
|
||||
self.router.add_api_route('/work/v2/tasks/result',self.submit,methods=['POST'])
|
||||
self.cards=None
|
||||
if auth.s.work_task_cards_enabled:
|
||||
from .task_cards import TaskCards
|
||||
self.cards=TaskCards(state,platform,broker.worker_id,files_enabled=auth.s.work_files_enabled)
|
||||
broker.task_cards=self.cards
|
||||
|
||||
def current(self, body):
|
||||
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
|
||||
@@ -49,7 +63,11 @@ class TaskDelivery:
|
||||
def receipt(self,task):
|
||||
row=self.db.execute('SELECT status,digest FROM work_result_outbox_v2 WHERE task_id=?',
|
||||
(task['turn_id'],)).fetchone()
|
||||
execution=self.db.execute('SELECT execution_state,status_sequence FROM work_execution_state_v3 WHERE task_id=?',(task['turn_id'],)).fetchone()
|
||||
return {'task_id':task['turn_id'],'status':row['status'] if row else task['status'],
|
||||
'progress_protocol':'work-progress-v1',
|
||||
'execution_state':execution['execution_state'] if execution else 'unknown',
|
||||
'status_sequence':execution['status_sequence'] if execution else 0,
|
||||
'result_received':bool(row),'result_digest':row['digest'] if row else None,
|
||||
'deadline':task['deadline']}
|
||||
|
||||
@@ -57,6 +75,31 @@ class TaskDelivery:
|
||||
await self.auth.authenticate(body,request)
|
||||
task=self.current(body)
|
||||
pending=self.broker.pending.get(body.task_id)
|
||||
if pending and pending.host==body.host_id:pending.progress.set()
|
||||
return no_cache(JSONResponse(self.receipt(task)))
|
||||
|
||||
async def progress(self,body:StateRequest,request:Request):
|
||||
await self.auth.authenticate(body,request)
|
||||
with self.db:
|
||||
self.state._owner_lock(body.binding_owner)
|
||||
task=self.current(body)
|
||||
if body.execution_state is not None:
|
||||
old=self.db.execute('SELECT * FROM work_execution_state_v3 WHERE task_id=?',(body.task_id,)).fetchone()
|
||||
terminal={'completed','failed','cancelled','interrupted'}
|
||||
if old and body.status_sequence==old['status_sequence'] and body.execution_state!=old['execution_state']:
|
||||
raise HTTPException(409,'execution_state_conflict')
|
||||
if not old:
|
||||
self.db.execute('INSERT INTO work_execution_state_v3 VALUES(?,?,?,?)',
|
||||
(body.task_id,body.execution_state,body.status_sequence,time.time()))
|
||||
elif body.status_sequence>old['status_sequence']:
|
||||
if old['execution_state'] in terminal and body.execution_state!=old['execution_state']:
|
||||
raise HTTPException(409,'execution_already_terminal')
|
||||
self.db.execute('UPDATE work_execution_state_v3 SET execution_state=?,status_sequence=?,updated_at=? WHERE task_id=?',
|
||||
(body.execution_state,body.status_sequence,time.time(),body.task_id))
|
||||
pending=self.broker.pending.get(body.task_id)
|
||||
if self.cards:
|
||||
old=self.db.execute('SELECT status_sequence FROM work_execution_state_v3 WHERE task_id=?',(body.task_id,)).fetchone()
|
||||
if old and body.status_sequence>=old['status_sequence']:self.cards.clear_connection_lost(body.task_id)
|
||||
if pending and pending.host==body.host_id:
|
||||
pending.progress.set()
|
||||
return no_cache(JSONResponse(self.receipt(task)))
|
||||
|
||||
@@ -196,7 +196,7 @@ def validate_overlay(base, merged, candidate, router_key_required=True):
|
||||
env = new[name]["environment"]
|
||||
for key, value in old[name].get("environment", {}).items():
|
||||
if key not in {"HN_CREDENTIAL_MODE","HN_ROUTER_ADMISSION_SECRET","HN_CREDENTIAL_KEYRING_FILE",
|
||||
"HN_CREDENTIAL_PERMISSION_PROFILES","HN_CONNECTOR_INTERNAL_URL","HN_FEISHU_APP_ID","HN_OAUTH_ENABLED","HN_WORK_FILES_ENABLED"}:
|
||||
"HN_CREDENTIAL_PERMISSION_PROFILES","HN_CONNECTOR_INTERNAL_URL","HN_FEISHU_APP_ID","HN_OAUTH_ENABLED","HN_WORK_FILES_ENABLED","HN_WORK_TASK_CARDS_ENABLED"}:
|
||||
if env.get(key) != value:
|
||||
raise DeployError("existing environment changed: "+key)
|
||||
old_volumes = {v["target"]:v for v in old[name].get("volumes", [])}
|
||||
@@ -375,6 +375,8 @@ def prepare_update(args):
|
||||
raise DeployError('missing update source')
|
||||
if getattr(args,'enable_work_files',False)and not (source/'relay/work_files.py').is_file():
|
||||
raise DeployError('work files implementation missing from update source')
|
||||
if getattr(args,'enable_task_cards',False)and not (source/'relay/task_cards.py').is_file():
|
||||
raise DeployError('task cards implementation missing from update source')
|
||||
output=output.resolve()
|
||||
output.mkdir(mode=0o700)
|
||||
os.chmod(output,0o700)
|
||||
@@ -395,6 +397,9 @@ def prepare_update(args):
|
||||
if getattr(args,'enable_work_files',False):
|
||||
overlay['services']['connector']['environment']['HN_WORK_FILES_ENABLED']='true'
|
||||
plan['work_files_enabled']=True
|
||||
if getattr(args,'enable_task_cards',False):
|
||||
overlay['services']['connector']['environment']['HN_WORK_TASK_CARDS_ENABLED']='true'
|
||||
plan['work_task_cards_enabled']=True
|
||||
write_json(output/'compose.broker.json',overlay)
|
||||
for name in ('connector','router'):
|
||||
rollback['services'][name]['image']=roles[name]['image']
|
||||
@@ -714,6 +719,7 @@ def main():
|
||||
update.add_argument("--source-root",default=str(Path(__file__).resolve().parents[1]))
|
||||
update.add_argument("--output",required=True)
|
||||
update.add_argument('--enable-work-files',action='store_true',help='Opt in after paired-client preparation; rollback retains the previous flag.')
|
||||
update.add_argument('--enable-task-cards',action='store_true',help='Opt in to paired-client status cards; rollback retains the previous flag.')
|
||||
permissions=sub.add_parser('update-permissions')
|
||||
permissions.add_argument('--plan',required=True)
|
||||
permissions.add_argument('--profiles',required=True)
|
||||
|
||||
@@ -658,3 +658,17 @@ def test_prepare_update_file_feature_is_explicit_and_rollback_restores_parent_fl
|
||||
assert current['services']['connector']['environment']['HN_WORK_FILES_ENABLED']=='true'
|
||||
assert rollback['services']['connector']['environment']==old['services']['connector']['environment']
|
||||
assert (output/'credential-keyring.json').read_bytes()==(root/'credential-keyring.json').read_bytes()
|
||||
|
||||
|
||||
def test_prepare_update_cards_are_opt_in_and_rollback_keeps_parent_environment(scenario):
|
||||
parent=prepared(scenario);root=scenario['output'];image='sha256:'+'a'*64
|
||||
for name in ('connector','router'):scenario['roles'][name]['image']=image
|
||||
deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']})
|
||||
output=root.parent/'update-cards'
|
||||
deploy.prepare_update(argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output),enable_task_cards=True))
|
||||
current=json.loads((output/'compose.broker.json').read_text())
|
||||
rollback=json.loads((output/'compose.rollback.json').read_text())
|
||||
old=json.loads((root/'compose.broker.json').read_text())
|
||||
assert current['services']['connector']['environment']['HN_WORK_TASK_CARDS_ENABLED']=='true'
|
||||
assert rollback['services']['connector']['environment']==old['services']['connector']['environment']
|
||||
assert (output/'credential-keyring.json').read_bytes()==(root/'credential-keyring.json').read_bytes()
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Joint source test: encrypted PC journal -> signed ECS progress -> card projection."""
|
||||
import importlib
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import types
|
||||
import httpx
|
||||
import pytest
|
||||
from test_credential_api import env,wire
|
||||
from test_task_delivery import task
|
||||
from test_task_cards import setup_cards
|
||||
|
||||
|
||||
async def test_native_states_reach_signed_server_and_existing_card(env,tmp_path,monkeypatch):
|
||||
client=Path(__file__).resolve().parents[3]/'hainuo-build-remote-C-20261112/integrations/work-relay/service/relay'
|
||||
if not client.is_dir():pytest.skip('paired client source not present')
|
||||
package=types.ModuleType('joint_pc_relay');package.__path__=[str(client)]
|
||||
monkeypatch.setitem(sys.modules,'joint_pc_relay',package)
|
||||
journal_module=importlib.import_module('joint_pc_relay.task_journal')
|
||||
native=importlib.import_module('joint_pc_relay.hainuo1065')
|
||||
monkeypatch.setattr(journal_module,'protect',lambda data:b'test:'+data)
|
||||
monkeypatch.setattr(journal_module,'unprotect',lambda data:data[5:])
|
||||
clock=[0.0]
|
||||
# Patch only the native module clock, not shared time.monotonic used by event loop.
|
||||
monkeypatch.setattr(native,'time',types.SimpleNamespace(monotonic=lambda:clock[0],time=__import__('time').time))
|
||||
async def no_wait(_):clock[0]+=1
|
||||
monkeypatch.setattr(native,'asyncio',types.SimpleNamespace(sleep=no_wait))
|
||||
context={'task_id':'task-durable','conversation_id':'synthetic-conversation'}
|
||||
ledger=journal_module.TaskJournal(tmp_path);ledger.begin(context,'synthetic')
|
||||
task(env);cards,calls,card_clock=setup_cards(env,monkeypatch)
|
||||
states=iter(['running','waiting_for_approval','running','completed']);reported=[];posts=[]
|
||||
run_id='run_'+'a'*32
|
||||
def handler(request):
|
||||
if request.method=='POST':posts.append(request);return httpx.Response(202,json={'run_id':run_id})
|
||||
return httpx.Response(200,json={'run_id':run_id,'status':next(states),'output':'synthetic result'})
|
||||
config=tmp_path/'native.json';config.write_text(__import__('json').dumps({'url':'http://127.0.0.1:8643','api_key':'fake-'*10,'mode':'text-only'}))
|
||||
adapter=native.Hainuo1065Agent(config,transport=httpx.MockTransport(handler));adapter.model='synthetic'
|
||||
async def progress():
|
||||
saved=ledger.get(context);path='/work/v2/tasks/progress'
|
||||
response=await env['client'].post(path,json=wire(env,path,task_id=context['task_id'],
|
||||
execution_state=saved['execution_state'],status_sequence=saved['status_sequence']))
|
||||
assert response.status_code==200,response.text
|
||||
reported.append(response.json()['execution_state'])
|
||||
card_clock[0]+=6;await cards.flush()
|
||||
try:
|
||||
assert await adapter._execute_run('synthetic',{'X-Hermes-Session-Id':'synthetic-session'},progress,journal=ledger,context=context)=='synthetic result'
|
||||
assert reported==['running','waiting_for_approval','running','completed']
|
||||
assert ledger.get(context)['status_sequence']==4
|
||||
assert len(posts)==1
|
||||
assert [call[0] for call in calls]==['create','update','update','update']
|
||||
assert len(env['platform'].replies)==0 # Execution completion is not result delivery.
|
||||
finally:await adapter.close()
|
||||
@@ -0,0 +1,207 @@
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
import httpx
|
||||
import pytest
|
||||
from test_credential_api import env,wire
|
||||
from test_task_delivery import task,upload
|
||||
from relay.task_cards import TaskCards,card_content
|
||||
from relay.platform import Feishu,PlatformError
|
||||
|
||||
|
||||
def setup_cards(env,monkeypatch):
|
||||
calls=[];clock=[time.time()]
|
||||
async def create(message,card,delivery_id,*,token):
|
||||
calls.append(('create',message,card,delivery_id));return 'om_card'
|
||||
async def update(message,card,*,token):calls.append(('update',message,card))
|
||||
monkeypatch.setattr(env['platform'],'reply_task_card',create,raising=False)
|
||||
monkeypatch.setattr(env['platform'],'update_task_card',update,raising=False)
|
||||
return TaskCards(env['store'].lifecycle,env['platform'],'worker-test',clock=lambda:clock[0]),calls,clock
|
||||
|
||||
|
||||
async def report(env,state,sequence):
|
||||
path='/work/v2/tasks/progress'
|
||||
response=await env['client'].post(path,json=wire(env,path,task_id='task-durable',execution_state=state,status_sequence=sequence))
|
||||
assert response.status_code==200,response.text
|
||||
|
||||
|
||||
async def test_card_created_once_updates_in_order_and_delivery_separate(env,monkeypatch):
|
||||
task(env);cards,calls,clock=setup_cards(env,monkeypatch)
|
||||
await report(env,'running',1);await cards.flush();await cards.flush()
|
||||
assert len(calls)==1 and calls[0][0]=='create' and calls[0][1]=='om-synthetic'
|
||||
await report(env,'waiting_for_approval',2)
|
||||
await cards.flush();assert len(calls)==1 # Update throttle.
|
||||
clock[0]+=6;await cards.flush()
|
||||
assert calls[-1][0]=='update' and '等待本人确认' in json.dumps(calls[-1],ensure_ascii=False)
|
||||
assert 'button' not in json.dumps(calls[-1])
|
||||
await report(env,'completed',3);clock[0]+=6;await cards.flush()
|
||||
text=json.dumps(calls[-1],ensure_ascii=False)
|
||||
assert '执行完成' in text and '尚未接收结果' in text and '结果已发送' not in text
|
||||
await upload(env);await env['app'].state.deliveries.flush()
|
||||
clock[0]+=6;await cards.flush()
|
||||
assert '结果已发送到当前会话' in json.dumps(calls[-1],ensure_ascii=False)
|
||||
assert len([c for c in calls if c[0]=='create'])==1
|
||||
|
||||
|
||||
async def test_unknown_create_receipt_is_durable_and_never_recreated(env,monkeypatch):
|
||||
task(env);cards,calls,clock=setup_cards(env,monkeypatch)
|
||||
async def lost(*args,**kwargs):calls.append(('lost',));raise TimeoutError()
|
||||
monkeypatch.setattr(env['platform'],'reply_task_card',lost)
|
||||
await report(env,'running',1);await cards.flush()
|
||||
clock[0]+=65
|
||||
reopened=TaskCards(env['store'].lifecycle,env['platform'],'new-worker',clock=lambda:clock[0])
|
||||
await report(env,'completed',2);await reopened.flush()
|
||||
assert len(calls)==1
|
||||
assert env['store'].db.execute('SELECT status FROM work_task_cards_v3').fetchone()['status']=='unknown'
|
||||
await upload(env);await env['app'].state.deliveries.flush()
|
||||
assert len(env['platform'].replies)==1 # Card failure never suppresses result text.
|
||||
|
||||
|
||||
async def test_failed_update_retries_same_card_with_newest_state(env,monkeypatch):
|
||||
task(env);cards,calls,clock=setup_cards(env,monkeypatch)
|
||||
await report(env,'running',1);await cards.flush()
|
||||
original=env['platform'].update_task_card
|
||||
async def lost(*args,**kwargs):calls.append(('lost-update',args[0]));raise TimeoutError()
|
||||
monkeypatch.setattr(env['platform'],'update_task_card',lost)
|
||||
await report(env,'waiting_for_approval',2);clock[0]+=6;await cards.flush()
|
||||
monkeypatch.setattr(env['platform'],'update_task_card',original)
|
||||
await report(env,'failed',3);clock[0]+=6;await cards.flush()
|
||||
assert calls[-1][0:2]==('update','om_card')
|
||||
assert '执行失败' in json.dumps(calls[-1],ensure_ascii=False)
|
||||
assert len([c for c in calls if c[0]=='create'])==1
|
||||
|
||||
|
||||
async def test_unbind_during_token_preflight_prevents_card_send(env,monkeypatch):
|
||||
task(env);cards,calls,clock=setup_cards(env,monkeypatch)
|
||||
async def token():env['store'].unbind('ou_a');return 'fake'
|
||||
monkeypatch.setattr(env['platform'],'tenant_token',token)
|
||||
await cards.flush();assert not calls
|
||||
|
||||
|
||||
async def test_two_workers_and_stale_snapshot_cannot_send_old_state(env,monkeypatch):
|
||||
task(env);cards,calls,clock=setup_cards(env,monkeypatch)
|
||||
await report(env,'running',1)
|
||||
paused=asyncio.Event();resume=asyncio.Event()
|
||||
original=env['platform'].reply_task_card
|
||||
async def create(*args,**kwargs):paused.set();await resume.wait();return await original(*args,**kwargs)
|
||||
monkeypatch.setattr(env['platform'],'reply_task_card',create)
|
||||
first=asyncio.create_task(cards.flush());await paused.wait()
|
||||
other=TaskCards(env['store'].lifecycle,env['platform'],'worker-other',clock=lambda:clock[0])
|
||||
await other.flush();assert not calls
|
||||
resume.set();await first
|
||||
assert len(calls)==1
|
||||
await report(env,'waiting_for_approval',2)
|
||||
async def token():
|
||||
# A new authenticated status arrives while card worker awaits preflight.
|
||||
await report(env,'completed',3);return 'fake'
|
||||
monkeypatch.setattr(env['platform'],'tenant_token',token)
|
||||
clock[0]+=6;await other.flush()
|
||||
assert '执行完成' in json.dumps(calls[-1],ensure_ascii=False)
|
||||
|
||||
|
||||
async def test_stale_creation_claim_is_unknown_but_stale_update_can_recover(env,monkeypatch):
|
||||
task(env);cards,calls,clock=setup_cards(env,monkeypatch)
|
||||
db=env['store'].db
|
||||
with db:db.execute('INSERT INTO work_task_cards_v3 VALUES(?,?,?,?,?,?)',('task-durable','','','sending','dead',clock[0]-61))
|
||||
await cards.flush();await cards.flush();assert not calls
|
||||
assert db.execute('SELECT status FROM work_task_cards_v3').fetchone()['status']=='unknown'
|
||||
with db:db.execute("UPDATE work_task_cards_v3 SET message_id='om_card',status='sending',updated_at=?",(clock[0]-61,))
|
||||
await cards.flush();await cards.flush()
|
||||
assert calls[0][0:2]==('update','om_card')
|
||||
|
||||
|
||||
def test_card_projection_never_exposes_task_body_or_markdown_controls():
|
||||
data=card_content('synthetic-task','waiting_for_approval',None)
|
||||
text=json.dumps(data,ensure_ascii=False)
|
||||
assert 'button' not in text and 'markdown' not in text
|
||||
assert data['config']['enable_forward'] is False
|
||||
|
||||
|
||||
def test_text_delivery_never_claims_attachments_delivered():
|
||||
text=json.dumps(card_content('synthetic','completed','replied',[]),ensure_ascii=False)
|
||||
assert '文字回复:' in text and '附件交付' not in text and '未收到登记' not in text
|
||||
mixed=json.dumps(card_content('synthetic','completed','replied',['sent','ready','unknown']),ensure_ascii=False)
|
||||
assert '附件交付' not in mixed and '待投递1件' not in mixed
|
||||
|
||||
|
||||
@pytest.mark.parametrize('state,color',[
|
||||
('running','blue'),('waiting_for_approval','orange'),('unknown','orange'),
|
||||
('completed','green'),('failed','red'),('interrupted','red')])
|
||||
def test_execution_color_is_independent_of_text_delivery(state,color):
|
||||
card=card_content('synthetic',state,'replied',[])
|
||||
assert card['header']['template']==color
|
||||
if state=='interrupted':
|
||||
assert '已中断' in json.dumps(card,ensure_ascii=False)
|
||||
|
||||
|
||||
async def test_card_uses_task_bound_source_and_skips_foreign_and_expired(env,monkeypatch):
|
||||
task(env);cards,calls,clock=setup_cards(env,monkeypatch)
|
||||
with env['store'].db:
|
||||
env['store'].db.execute('UPDATE work_tasks_v2 SET deadline=?',(clock[0]-1,))
|
||||
await cards.flush();assert not calls
|
||||
with env['store'].db:
|
||||
env['store'].db.execute("UPDATE work_tasks_v2 SET deadline=?,binding_generation=999",(clock[0]+300,))
|
||||
await cards.flush();assert not calls
|
||||
|
||||
|
||||
async def test_timeout_projects_unknown_without_claiming_failure(env,monkeypatch):
|
||||
task(env);cards,calls,clock=setup_cards(env,monkeypatch)
|
||||
await report(env,'running',1);await cards.flush()
|
||||
with env['store'].db:env['store'].db.execute("UPDATE work_tasks_v2 SET status='unknown'")
|
||||
clock[0]+=6;await cards.flush()
|
||||
text=json.dumps(calls[-1],ensure_ascii=False)
|
||||
assert '状态待核对' in text and '执行失败' not in text
|
||||
|
||||
|
||||
async def test_lost_client_turns_red_persists_and_recovers_on_authenticated_progress(env,monkeypatch):
|
||||
task(env);cards,calls,clock=setup_cards(env,monkeypatch)
|
||||
api=env['app'].state.deliveries;api.cards=cards
|
||||
await report(env,'running',1);await cards.flush()
|
||||
cards.mark_connection_lost('task-durable')
|
||||
with env['store'].db:env['store'].db.execute("UPDATE work_tasks_v2 SET status='unknown'")
|
||||
clock[0]+=6;await cards.flush()
|
||||
card=calls[-1][2]
|
||||
assert card['header']['template']=='red'
|
||||
text=json.dumps(card,ensure_ascii=False)
|
||||
assert '客户端失联,任务结果待核对' in text and '执行失败' not in text
|
||||
reopened=TaskCards(env['store'].lifecycle,env['platform'],'worker-new',clock=lambda:clock[0])
|
||||
assert reopened.snapshot(env['store'].lifecycle.task('task-durable'))['header']['template']=='red'
|
||||
await report(env,'completed',2);clock[0]+=6;await cards.flush()
|
||||
assert calls[-1][2]['header']['template']=='green'
|
||||
assert not env['store'].db.execute('SELECT * FROM work_task_connection_v3').fetchall()
|
||||
|
||||
|
||||
async def test_idle_timeout_is_distinct_from_hard_tracking_limit():
|
||||
from types import SimpleNamespace
|
||||
from relay.connector import Broker,Pending,ClientIdleTimeout
|
||||
broker=object.__new__(Broker)
|
||||
broker.s=SimpleNamespace(request_timeout=.01,task_tracking_timeout=1)
|
||||
item=Pending(asyncio.get_running_loop().create_future(),None,None,'synthetic','synthetic')
|
||||
with pytest.raises(ClientIdleTimeout):await broker.wait_tracked(item)
|
||||
broker.s.task_tracking_timeout=0
|
||||
with pytest.raises(asyncio.TimeoutError)as error:await broker.wait_tracked(item)
|
||||
assert not isinstance(error.value,ClientIdleTimeout)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('state',['completed','failed','cancelled','interrupted'])
|
||||
def test_lost_connection_never_overwrites_known_terminal_state(state):
|
||||
card=card_content('synthetic',state,None,connection_lost=True)
|
||||
assert '客户端失联' not in json.dumps(card,ensure_ascii=False)
|
||||
|
||||
|
||||
async def test_platform_card_wire_format_and_receipt_validation(env):
|
||||
seen=[]
|
||||
def handler(request):
|
||||
seen.append(request)
|
||||
return httpx.Response(200,json={'code':0,'data':{'message_id':'om_receipt'}})
|
||||
platform=Feishu(env['settings'],transport=httpx.MockTransport(handler))
|
||||
card=card_content('synthetic','running',None)
|
||||
try:
|
||||
assert await platform.reply_task_card('om_source',card,'a'*32,token='fake')=='om_receipt'
|
||||
await platform.update_task_card('om_receipt',card,token='fake')
|
||||
body=json.loads(seen[0].content)
|
||||
assert body['msg_type']=='interactive' and json.loads(body['content'])==card
|
||||
assert seen[1].method=='PATCH' and seen[1].url.path.endswith('/om_receipt')
|
||||
with pytest.raises(PlatformError):await platform.reply_task_card('../other',card,'a'*32,token='fake')
|
||||
assert len(seen)==2
|
||||
finally:await platform.close()
|
||||
@@ -16,6 +16,40 @@ async def upload(env,turn='task-durable',text='synthetic-private-result'):
|
||||
return await env['client'].post(path,json=wire(env,path,task_id=turn,result=text))
|
||||
|
||||
|
||||
async def test_execution_status_order_replay_terminal_and_delivery_are_separate(env):
|
||||
task(env)
|
||||
path='/work/v2/tasks/progress'
|
||||
async def report(state,sequence,turn='task-durable'):
|
||||
return await env['client'].post(path,json=wire(env,path,task_id=turn,
|
||||
execution_state=state,status_sequence=sequence))
|
||||
assert (await report('running',1)).json()['execution_state']=='running'
|
||||
waiting=await report('waiting_for_approval',2)
|
||||
assert waiting.json()['status_sequence']==2
|
||||
assert (await report('running',1)).json()['execution_state']=='waiting_for_approval'
|
||||
assert (await report('waiting_for_approval',2)).status_code==200
|
||||
assert (await report('running',2)).status_code==409
|
||||
completed=await report('completed',3)
|
||||
assert completed.json()['execution_state']=='completed'
|
||||
assert not completed.json()['result_received']
|
||||
assert (await report('running',4)).status_code==409
|
||||
await upload(env)
|
||||
assert (await report('completed',3)).json()['result_received']
|
||||
|
||||
|
||||
async def test_execution_state_auth_binding_and_parameter_validation(env):
|
||||
task(env)
|
||||
path='/work/v2/tasks/progress'
|
||||
missing=await env['client'].post(path,json=wire(env,path,task_id='task-durable',execution_state='running'))
|
||||
assert missing.status_code==422
|
||||
task(env,'foreign-task','ou_b','b'*32)
|
||||
foreign=await env['client'].post(path,json=wire(env,path,task_id='foreign-task',execution_state='running',status_sequence=1))
|
||||
assert foreign.status_code==403
|
||||
env['store'].unbind('ou_a')
|
||||
revoked=await env['client'].post(path,json=wire(env,path,task_id='task-durable',execution_state='running',status_sequence=1))
|
||||
assert revoked.status_code==403
|
||||
assert not env['store'].db.execute('SELECT * FROM work_execution_state_v3').fetchall()
|
||||
|
||||
|
||||
async def test_result_is_encrypted_durable_and_delivered_once_after_worker_loss(env):
|
||||
task(env)
|
||||
api=env['app'].state.deliveries
|
||||
|
||||
Reference in New Issue
Block a user