feat(tasks): track authenticated native progress until final reply
Renew broker idle waits only for encrypted current-task progress, keep an absolute deadline and binding/session/worker fences, and retain original-message reply deduplication. Add guarded prepare-update staging that reuses the active broker key material and application configuration. Validation: 216 passed, 2 skipped; guarded update and long-task routing covered with synthetic Docker configuration and loopback WebSocket tests. Real ECS build and acceptance remain pending.
This commit is contained in:
@@ -0,0 +1,28 @@
|
|||||||
|
# 已运行 broker 的异步任务更新
|
||||||
|
|
||||||
|
适用:现有 connector/router 均健康,且已使用 broker 凭据方案。更新不会生成新密钥或重新授权,不修改数据库卷、权限配置和其他组件。本轮为经过当前身份和任务校验的进度续期,仍有绝对上限;ECS 重启或连接丢失后的任务自动恢复不在本轮范围。
|
||||||
|
|
||||||
|
在当前 ECS 执行(输出目录必须尚不存在):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /home/cnooc/ecs-relay-service-staging-http
|
||||||
|
git pull --ff-only
|
||||||
|
sudo python3 deployable/scripts/broker_deploy.py prepare-update \
|
||||||
|
--plan /home/cnooc/hainuo-broker-prepared-20261104 \
|
||||||
|
--output /home/cnooc/hainuo-broker-update-20261104-async
|
||||||
|
sudo python3 deployable/scripts/broker_deploy.py build \
|
||||||
|
--plan /home/cnooc/hainuo-broker-update-20261104-async
|
||||||
|
sudo python3 deployable/scripts/broker_deploy.py activate \
|
||||||
|
--plan /home/cnooc/hainuo-broker-update-20261104-async \
|
||||||
|
--client-v2-ready
|
||||||
|
sudo python3 deployable/scripts/broker_deploy.py state \
|
||||||
|
--plan /home/cnooc/hainuo-broker-update-20261104-async
|
||||||
|
```
|
||||||
|
|
||||||
|
一次执行一条,失败先看 state,不重复 activate。不运行原迁移 prepare,也不重新生成 keyring。prepare-update 沿用父计划的同一密钥内容和权限/应用身份,在独立受保护目录内保留更新计划,旧计划不被覆盖。新镜像从当前已确认镜像离线复用依赖;回滚目标为更新前的 broker 镜像。密钥副本和新旧计划目录均需保留且不得外传。
|
||||||
|
|
||||||
|
更新工具先验证父计划、关键配置、运行镜像和存储;部分切换会拒绝生成更新。新计划的 rollback 仍要求停止 broker 使用,防止切换期间丢失任务;不恢复数据库或旧 refresh token。回滚只退应用代码,任务执行状态必须另行核对。
|
||||||
|
|
||||||
|
更新先于配套客户端安装。老 v2 客户端仍可使用,未发送进度时沿用闲置超时。新客户端以任务 ID 查询原生状态,只有校验通过的加密 progress 才续期闲置等待,最终结果仍按原任务/消息发送一次。默认闲置超时 HN_REQUEST_TIMEOUT=45 秒,绝对上限 HN_TASK_TRACKING_TIMEOUT=1800 秒(允许 30–3600);进度不能无限续期。
|
||||||
|
|
||||||
|
验收:授权状态不变;今日日程一次调用正确时间范围;超过原 150 秒的任务最终能回原消息;双用户不串结果;执行中解绑后不再推旧结果。Docker 构建和这些现场行为未在开发机上实测,需在 ECS 执行并核对。
|
||||||
@@ -161,6 +161,14 @@ class BindingState:
|
|||||||
WHERE turn_id=? AND environment=? AND app_id=? AND status=?""",
|
WHERE turn_id=? AND environment=? AND app_id=? AND status=?""",
|
||||||
(status,time.time(),turn,self.environment,self.app_id,previous)).rowcount == 1
|
(status,time.time(),turn,self.environment,self.app_id,previous)).rowcount == 1
|
||||||
|
|
||||||
|
def progress(self, turn, host, session, worker):
|
||||||
|
# A progress packet may keep transport waiting, never change task outcome
|
||||||
|
# or extend the absolute lifetime. Unbound/stale sessions fail closed.
|
||||||
|
task = self.task(turn)
|
||||||
|
return bool(self.task_current(task) and
|
||||||
|
(task['host_id'],task['session_id'],task['worker_id'],task['status']) ==
|
||||||
|
(host,session,worker,'inflight'))
|
||||||
|
|
||||||
def complete(self, turn, host, session, worker):
|
def complete(self, turn, host, session, worker):
|
||||||
with self.db:
|
with self.db:
|
||||||
task = self.task(turn)
|
task = self.task(turn)
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ class Settings(BaseSettings):
|
|||||||
# A fixed allowlist, never a user supplied upstream URL.
|
# A fixed allowlist, never a user supplied upstream URL.
|
||||||
proxy_paths: str = "/open-apis/drive/v1/files,/open-apis/docx/v1/documents"
|
proxy_paths: str = "/open-apis/drive/v1/files,/open-apis/docx/v1/documents"
|
||||||
request_timeout: int = 45
|
request_timeout: int = 45
|
||||||
|
task_tracking_timeout: int = 1800
|
||||||
max_body: int = 262144
|
max_body: int = 262144
|
||||||
max_pending: int = 1000
|
max_pending: int = 1000
|
||||||
max_devices: int = 1000
|
max_devices: int = 1000
|
||||||
@@ -49,6 +50,8 @@ class Settings(BaseSettings):
|
|||||||
|
|
||||||
@model_validator(mode="after")
|
@model_validator(mode="after")
|
||||||
def validate_runtime(self):
|
def validate_runtime(self):
|
||||||
|
if not 30 <= self.task_tracking_timeout <= 3600:
|
||||||
|
raise ValueError("task tracking timeout must be 30-3600 seconds")
|
||||||
if self.credential_mode not in {"legacy", "broker"}:
|
if self.credential_mode not in {"legacy", "broker"}:
|
||||||
raise ValueError("invalid credential mode")
|
raise ValueError("invalid credential mode")
|
||||||
if self.credential_mode == "broker" and self.role in {"connector","router"}:
|
if self.credential_mode == "broker" and self.role in {"connector","router"}:
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import sqlite3
|
|||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, field
|
||||||
from urllib.parse import unquote, urlsplit
|
from urllib.parse import unquote, urlsplit
|
||||||
|
|
||||||
from cryptography.hazmat.primitives import hashes, padding
|
from cryptography.hazmat.primitives import hashes, padding
|
||||||
@@ -55,6 +55,7 @@ class Pending:
|
|||||||
public: object
|
public: object
|
||||||
host: str
|
host: str
|
||||||
session: str
|
session: str
|
||||||
|
progress: asyncio.Event = field(default_factory=asyncio.Event)
|
||||||
|
|
||||||
|
|
||||||
class Broker:
|
class Broker:
|
||||||
@@ -76,7 +77,14 @@ class Broker:
|
|||||||
continue
|
continue
|
||||||
plain = open_envelope(data["envelope"], item.private, item.public,
|
plain = open_envelope(data["envelope"], item.private, item.public,
|
||||||
host_id=item.host, session_id=item.session)
|
host_id=item.host, session_id=item.session)
|
||||||
if plain["turn_id"] != data["turn_id"] or plain.get("type") != "result":
|
if plain["turn_id"] != data["turn_id"]:
|
||||||
|
continue
|
||||||
|
if plain.get('type') == 'progress':
|
||||||
|
if self.lifecycle and self.lifecycle.progress(data['turn_id'],item.host,item.session,self.worker_id):
|
||||||
|
await self.r.expire(f"hn:inflight:{item.host}:{data['turn_id']}", self.s.request_timeout + 5)
|
||||||
|
item.progress.set()
|
||||||
|
continue
|
||||||
|
if plain.get('type') != 'result':
|
||||||
continue
|
continue
|
||||||
result = plain.get("result", "")
|
result = plain.get("result", "")
|
||||||
if not isinstance(result, str) or len(result) > self.s.max_result:
|
if not isinstance(result, str) or len(result) > self.s.max_result:
|
||||||
@@ -88,6 +96,28 @@ class Broker:
|
|||||||
# Invalid packets never consume a pending result; do not log packet contents.
|
# Invalid packets never consume a pending result; do not log packet contents.
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
async def wait_tracked(self, item):
|
||||||
|
# Authenticated native status polls renew idle waiting, never the hard
|
||||||
|
# task deadline. A lost device/worker remains unknown and is not replayed.
|
||||||
|
deadline = asyncio.get_running_loop().time() + self.s.task_tracking_timeout
|
||||||
|
while True:
|
||||||
|
remaining = deadline - asyncio.get_running_loop().time()
|
||||||
|
if remaining <= 0:
|
||||||
|
raise asyncio.TimeoutError()
|
||||||
|
item.progress.clear()
|
||||||
|
progress = asyncio.create_task(item.progress.wait())
|
||||||
|
try:
|
||||||
|
done, _ = await asyncio.wait({item.future, progress},
|
||||||
|
timeout=min(self.s.request_timeout, remaining), return_when=asyncio.FIRST_COMPLETED)
|
||||||
|
if item.future in done:
|
||||||
|
return item.future.result()
|
||||||
|
if not done:
|
||||||
|
raise asyncio.TimeoutError()
|
||||||
|
finally:
|
||||||
|
progress.cancel()
|
||||||
|
with contextlib.suppress(asyncio.CancelledError):
|
||||||
|
await progress
|
||||||
|
|
||||||
async def dispatch(self, owner, text, turn, *, chat_id="", source_message_id=""):
|
async def dispatch(self, owner, text, turn, *, chat_id="", source_message_id=""):
|
||||||
device = self.store.by_owner(owner)
|
device = self.store.by_owner(owner)
|
||||||
if not device:
|
if not device:
|
||||||
@@ -120,7 +150,7 @@ class Broker:
|
|||||||
if self.lifecycle:
|
if self.lifecycle:
|
||||||
try:
|
try:
|
||||||
task = self.lifecycle.create_task(turn,owner,host,device["binding_generation"],session,
|
task = self.lifecycle.create_task(turn,owner,host,device["binding_generation"],session,
|
||||||
self.worker_id,time.time()+self.s.request_timeout,chat_id,source_message_id)
|
self.worker_id,time.time()+self.s.task_tracking_timeout,chat_id,source_message_id)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
self.store.finish(turn,"unbound")
|
self.store.finish(turn,"unbound")
|
||||||
return {"status":"unbound","turn_id":turn,"result":"绑定已变化,任务未下发。"}
|
return {"status":"unbound","turn_id":turn,"result":"绑定已变化,任务未下发。"}
|
||||||
@@ -157,7 +187,10 @@ class Broker:
|
|||||||
if task:
|
if task:
|
||||||
self.lifecycle.transition(turn,"inflight","offline")
|
self.lifecycle.transition(turn,"inflight","offline")
|
||||||
return {"status": "offline", "turn_id": turn, "result": "你的电脑当前离线。"}
|
return {"status": "offline", "turn_id": turn, "result": "你的电脑当前离线。"}
|
||||||
result = await asyncio.wait_for(future, timeout=self.s.request_timeout)
|
if task:
|
||||||
|
result = await self.wait_tracked(self.pending[turn])
|
||||||
|
else:
|
||||||
|
result = await asyncio.wait_for(future, timeout=self.s.request_timeout)
|
||||||
self.store.finish(turn, result.get("status","unknown") if task else "completed")
|
self.store.finish(turn, result.get("status","unknown") if task else "completed")
|
||||||
if task and not source_message_id:
|
if task and not source_message_id:
|
||||||
self.lifecycle.transition(turn,"completed","finished")
|
self.lifecycle.transition(turn,"completed","finished")
|
||||||
|
|||||||
@@ -355,6 +355,58 @@ def prepare(args):
|
|||||||
print("PREPARED: "+str(output)+" (services unchanged; private files must not be shared)")
|
print("PREPARED: "+str(output)+" (services unchanged; private files must not be shared)")
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_update(args):
|
||||||
|
"""Stage an already-active broker update without generating credentials."""
|
||||||
|
root=Path(args.plan).resolve()
|
||||||
|
parent=load_plan(root)
|
||||||
|
check(root,parent,baseline=False)
|
||||||
|
built=json.loads((root/'candidate.json').read_text(encoding='utf-8'))
|
||||||
|
roles={name:service(parent['project'],name) for name in parent['roles']}
|
||||||
|
if any(roles[name]['image'] != built['image'] for name in ('connector','router')):
|
||||||
|
raise DeployError('update requires both services on the confirmed parent candidate; inspect state first')
|
||||||
|
output=Path(args.output)
|
||||||
|
source=Path(args.source_root).resolve()
|
||||||
|
deployment=Path(parent['deployment_root']).resolve()
|
||||||
|
if (not output.is_absolute() or output.exists() or output.is_symlink()
|
||||||
|
or output.resolve()==deployment or deployment in output.resolve().parents
|
||||||
|
or any(c in str(output) for c in ('\n','\r','$'))):
|
||||||
|
raise DeployError('update output must be a new absolute directory outside the live deployment')
|
||||||
|
if not (source/'relay/connector.py').is_file():
|
||||||
|
raise DeployError('missing update source')
|
||||||
|
output=output.resolve()
|
||||||
|
output.mkdir(mode=0o700)
|
||||||
|
os.chmod(output,0o700)
|
||||||
|
plan=json.loads(json.dumps(parent))
|
||||||
|
plan_id=secrets.token_hex(8)
|
||||||
|
plan.update(plan_id=plan_id,candidate_image='hainuo-relay:broker-update-'+plan_id,
|
||||||
|
roles=roles,base_image=built['image'],parent_plan=str(root),update_preserves_credentials=True)
|
||||||
|
# Copy identical server key material; never rotate/reinitialize while updating code.
|
||||||
|
shutil.copyfile(root/'credential-keyring.json',output/'credential-keyring.json')
|
||||||
|
os.chmod(output/'credential-keyring.json',0o600)
|
||||||
|
if os.name!='nt':os.chown(output/'credential-keyring.json',10001,10001)
|
||||||
|
overlay=json.loads((root/'compose.broker.json').read_text(encoding='utf-8'))
|
||||||
|
for name in ('connector','router'):overlay['services'][name]['image']=plan['candidate_image']
|
||||||
|
for mount in overlay['services']['connector']['volumes']:
|
||||||
|
if mount['target']=='/run/hainuo-credentials/keyring.json':
|
||||||
|
mount['source']=str(output/'credential-keyring.json')
|
||||||
|
write_json(output/'compose.broker.json',overlay)
|
||||||
|
write_json(output/'compose.rollback.json',{'services':{name:{'image':roles[name]['image']}
|
||||||
|
for name in ('connector','router','longconn','enrollment-signer')}})
|
||||||
|
payload=output/'build';payload.mkdir(mode=0o700)
|
||||||
|
for path in (source/'relay').rglob('*'):
|
||||||
|
if path.is_symlink():raise DeployError('symlink in update source')
|
||||||
|
if path.is_file() and path.suffix in {'.py','.js','.css','.html'} and '__pycache__' not in path.parts:
|
||||||
|
target=payload/path.relative_to(source);target.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
shutil.copyfile(path,target);os.chmod(target,0o644)
|
||||||
|
shutil.copyfile(root/'build/Dockerfile',payload/'Dockerfile')
|
||||||
|
plan['payload_hashes']={str(p.relative_to(output)):digest(p) for p in payload.rglob('*') if p.is_file()}
|
||||||
|
plan['prepared_hashes']={name:digest(output/name) for name in
|
||||||
|
('credential-keyring.json','compose.broker.json','compose.rollback.json')}
|
||||||
|
write_json(output/'plan.json',plan)
|
||||||
|
check(output,plan)
|
||||||
|
print('UPDATE PREPARED: existing credential key material and permissions preserved; no services restarted')
|
||||||
|
|
||||||
|
|
||||||
def load_plan(root):
|
def load_plan(root):
|
||||||
plan = json.loads((root/"plan.json").read_text(encoding="utf-8"))
|
plan = json.loads((root/"plan.json").read_text(encoding="utf-8"))
|
||||||
if plan.get("format") != 1 or not re.fullmatch(r"[a-f0-9]{16}",plan.get("plan_id","")):
|
if plan.get("format") != 1 or not re.fullmatch(r"[a-f0-9]{16}",plan.get("plan_id","")):
|
||||||
@@ -568,6 +620,10 @@ def main():
|
|||||||
prep.add_argument("--profiles",required=True)
|
prep.add_argument("--profiles",required=True)
|
||||||
prep.add_argument("--output",required=True)
|
prep.add_argument("--output",required=True)
|
||||||
prep.add_argument("--connector-container",default="hainuo-relay-connector-1")
|
prep.add_argument("--connector-container",default="hainuo-relay-connector-1")
|
||||||
|
update = sub.add_parser("prepare-update")
|
||||||
|
update.add_argument("--plan",required=True)
|
||||||
|
update.add_argument("--source-root",default=str(Path(__file__).resolve().parents[1]))
|
||||||
|
update.add_argument("--output",required=True)
|
||||||
for name in ("repair-router","state","diagnose","check","build","activate","rollback"):
|
for name in ("repair-router","state","diagnose","check","build","activate","rollback"):
|
||||||
cmd = sub.add_parser(name)
|
cmd = sub.add_parser(name)
|
||||||
cmd.add_argument("--plan",required=True)
|
cmd.add_argument("--plan",required=True)
|
||||||
@@ -583,6 +639,9 @@ def main():
|
|||||||
if args.action == "prepare":
|
if args.action == "prepare":
|
||||||
prepare(args)
|
prepare(args)
|
||||||
return
|
return
|
||||||
|
if args.action == "prepare-update":
|
||||||
|
prepare_update(args)
|
||||||
|
return
|
||||||
root = Path(args.plan).resolve()
|
root = Path(args.plan).resolve()
|
||||||
plan = load_plan(root)
|
plan = load_plan(root)
|
||||||
if args.action == "repair-router":
|
if args.action == "repair-router":
|
||||||
|
|||||||
@@ -510,3 +510,40 @@ def test_overlay_validator_rejects_database_volume_or_network_change(scenario,fi
|
|||||||
merged[field] = {"unexpected":{}}
|
merged[field] = {"unexpected":{}}
|
||||||
with pytest.raises(deploy.DeployError):
|
with pytest.raises(deploy.DeployError):
|
||||||
deploy.validate_overlay(base,merged,plan["candidate_image"])
|
deploy.validate_overlay(base,merged,plan["candidate_image"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_prepare_update_preserves_key_material_and_parent_plan_without_restart(scenario):
|
||||||
|
parent=prepared(scenario)
|
||||||
|
root=scenario['output']
|
||||||
|
original_key=(root/'credential-keyring.json').read_bytes()
|
||||||
|
original_plan=(root/'plan.json').read_bytes()
|
||||||
|
image='sha256:'+'a'*64
|
||||||
|
for name in ('connector','router'):scenario['roles'][name]['image']=image
|
||||||
|
deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']})
|
||||||
|
output=root.parent/'update'
|
||||||
|
args=argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output))
|
||||||
|
scenario['calls'].clear()
|
||||||
|
deploy.prepare_update(args)
|
||||||
|
updated=deploy.load_plan(output)
|
||||||
|
assert (output/'credential-keyring.json').read_bytes()==original_key
|
||||||
|
assert (root/'plan.json').read_bytes()==original_plan
|
||||||
|
assert updated['base_image']==image and updated['roles']['connector']['image']==image
|
||||||
|
assert updated['candidate_image']!=parent['candidate_image']
|
||||||
|
assert updated['update_preserves_credentials'] is True
|
||||||
|
overlay=json.loads((output/'compose.broker.json').read_text())
|
||||||
|
old=json.loads((root/'compose.broker.json').read_text())
|
||||||
|
assert overlay['services']['connector']['environment']==old['services']['connector']['environment']
|
||||||
|
assert not any('up' in call or 'build' in call or 'tag' in call for call in scenario['calls'])
|
||||||
|
|
||||||
|
|
||||||
|
def test_prepare_update_rejects_partial_switch_and_existing_output(scenario):
|
||||||
|
parent=prepared(scenario);root=scenario['output'];image='sha256:'+'a'*64
|
||||||
|
deploy.write_json(root/'candidate.json',{'image':image,'tag':parent['candidate_image']})
|
||||||
|
scenario['roles']['connector']['image']=image
|
||||||
|
output=root.parent/'update'
|
||||||
|
args=argparse.Namespace(plan=str(root),source_root=str(ROOT),output=str(output))
|
||||||
|
with pytest.raises(deploy.DeployError,match='both services'):deploy.prepare_update(args)
|
||||||
|
assert not output.exists()
|
||||||
|
scenario['roles']['router']['image']=image
|
||||||
|
args.output=str(root)
|
||||||
|
with pytest.raises(deploy.DeployError,match='new absolute'):deploy.prepare_update(args)
|
||||||
|
|||||||
@@ -477,3 +477,56 @@ async def test_admission_denies_dead_tasks_without_exposing_credentials(env):
|
|||||||
headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret},
|
headers={"X-Router-Admission-Secret":env["settings"].router_admission_secret},
|
||||||
json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"})
|
json={"nonce":secrets.token_hex(16),"turn_id":"task-check","session_id":"s"})
|
||||||
assert response.status_code == 403 and "token" not in response.text.lower()
|
assert response.status_code == 403 and "token" not in response.text.lower()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_native_progress_keeps_task_until_final_and_original_reply_once(routed):
|
||||||
|
routed['settings'].request_timeout=1
|
||||||
|
connection=await device(routed)
|
||||||
|
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
|
||||||
|
payload=await instruction(routed,connection)
|
||||||
|
ws,session,_=connection
|
||||||
|
for _ in range(4):
|
||||||
|
envelope=seal({'type':'progress','turn_id':payload['turn_id']},payload['reply_key'],
|
||||||
|
routed['keys']['a'*32],host_id='a'*32,session_id=session)
|
||||||
|
await ws.send(json.dumps({'type':'reply','turn_id':payload['turn_id'],'envelope':envelope}))
|
||||||
|
await asyncio.sleep(.4)
|
||||||
|
assert not routed['platform'].replies
|
||||||
|
assert routed['store'].lifecycle.task(payload['turn_id'])['status']=='inflight'
|
||||||
|
await reply(routed,connection,payload,'synthetic final')
|
||||||
|
await wait_for_reply(routed['platform'],1)
|
||||||
|
assert routed['platform'].replies==[('om-progress','synthetic final')]
|
||||||
|
await send_event(routed,event(routed,'om-progress','synthetic slow task'))
|
||||||
|
await asyncio.sleep(.05)
|
||||||
|
assert len(routed['platform'].replies)==1
|
||||||
|
|
||||||
|
|
||||||
|
async def test_progress_wait_cannot_extend_hard_deadline_or_hide_idle_failure():
|
||||||
|
from relay.connector import Broker,Pending
|
||||||
|
from types import SimpleNamespace
|
||||||
|
broker=object.__new__(Broker)
|
||||||
|
broker.s=SimpleNamespace(task_tracking_timeout=.12,request_timeout=.08)
|
||||||
|
item=Pending(asyncio.get_running_loop().create_future(),None,None,'host','session')
|
||||||
|
async def pulses():
|
||||||
|
while True:
|
||||||
|
item.progress.set()
|
||||||
|
await asyncio.sleep(.02)
|
||||||
|
pulse=asyncio.create_task(pulses())
|
||||||
|
try:
|
||||||
|
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
|
||||||
|
assert not item.future.done()
|
||||||
|
finally:
|
||||||
|
pulse.cancel()
|
||||||
|
try:await pulse
|
||||||
|
except asyncio.CancelledError:pass
|
||||||
|
broker.s.task_tracking_timeout=1
|
||||||
|
with pytest.raises(asyncio.TimeoutError):await broker.wait_tracked(item)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_unbound_or_wrong_session_progress_cannot_keep_task_alive(env):
|
||||||
|
state=env['store'].lifecycle
|
||||||
|
state.create_task('progress-bound','ou_a','a'*32,1,'session-a','worker-a',time.time()+60)
|
||||||
|
assert state.progress('progress-bound','a'*32,'session-a','worker-a')
|
||||||
|
assert not state.progress('progress-bound','a'*32,'session-b','worker-a')
|
||||||
|
assert not state.progress('progress-bound','a'*32,'session-a','worker-b')
|
||||||
|
env['store'].unbind('ou_a')
|
||||||
|
assert not state.progress('progress-bound','a'*32,'session-a','worker-a')
|
||||||
|
|||||||
Reference in New Issue
Block a user