feat(tasks): track authenticated native progress until final reply

Renew broker idle waits only for encrypted current-task progress, keep an absolute deadline and binding/session/worker fences, and retain original-message reply deduplication. Add guarded prepare-update staging that reuses the active broker key material and application configuration. Validation: 216 passed, 2 skipped; guarded update and long-task routing covered with synthetic Docker configuration and loopback WebSocket tests. Real ECS build and acceptance remain pending.
This commit is contained in:
ex_tianzy
2026-11-04 18:12:12 +08:00
parent ce965aaf58
commit 660d7a48d2
7 changed files with 225 additions and 4 deletions
+8
View File
@@ -161,6 +161,14 @@ class BindingState:
WHERE turn_id=? AND environment=? AND app_id=? AND status=?""",
(status,time.time(),turn,self.environment,self.app_id,previous)).rowcount == 1
def progress(self, turn, host, session, worker):
# A progress packet may keep transport waiting, never change task outcome
# or extend the absolute lifetime. Unbound/stale sessions fail closed.
task = self.task(turn)
return bool(self.task_current(task) and
(task['host_id'],task['session_id'],task['worker_id'],task['status']) ==
(host,session,worker,'inflight'))
def complete(self, turn, host, session, worker):
with self.db:
task = self.task(turn)
+3
View File
@@ -34,6 +34,7 @@ class Settings(BaseSettings):
# A fixed allowlist, never a user supplied upstream URL.
proxy_paths: str = "/open-apis/drive/v1/files,/open-apis/docx/v1/documents"
request_timeout: int = 45
task_tracking_timeout: int = 1800
max_body: int = 262144
max_pending: int = 1000
max_devices: int = 1000
@@ -49,6 +50,8 @@ class Settings(BaseSettings):
@model_validator(mode="after")
def validate_runtime(self):
if not 30 <= self.task_tracking_timeout <= 3600:
raise ValueError("task tracking timeout must be 30-3600 seconds")
if self.credential_mode not in {"legacy", "broker"}:
raise ValueError("invalid credential mode")
if self.credential_mode == "broker" and self.role in {"connector","router"}:
+37 -4
View File
@@ -10,7 +10,7 @@ import sqlite3
import time
import uuid
from contextlib import asynccontextmanager
from dataclasses import dataclass
from dataclasses import dataclass, field
from urllib.parse import unquote, urlsplit
from cryptography.hazmat.primitives import hashes, padding
@@ -55,6 +55,7 @@ class Pending:
public: object
host: str
session: str
progress: asyncio.Event = field(default_factory=asyncio.Event)
class Broker:
@@ -76,7 +77,14 @@ class Broker:
continue
plain = open_envelope(data["envelope"], item.private, item.public,
host_id=item.host, session_id=item.session)
if plain["turn_id"] != data["turn_id"] or plain.get("type") != "result":
if plain["turn_id"] != data["turn_id"]:
continue
if plain.get('type') == 'progress':
if self.lifecycle and self.lifecycle.progress(data['turn_id'],item.host,item.session,self.worker_id):
await self.r.expire(f"hn:inflight:{item.host}:{data['turn_id']}", self.s.request_timeout + 5)
item.progress.set()
continue
if plain.get('type') != 'result':
continue
result = plain.get("result", "")
if not isinstance(result, str) or len(result) > self.s.max_result:
@@ -88,6 +96,28 @@ class Broker:
# Invalid packets never consume a pending result; do not log packet contents.
continue
async def wait_tracked(self, item):
# Authenticated native status polls renew idle waiting, never the hard
# task deadline. A lost device/worker remains unknown and is not replayed.
deadline = asyncio.get_running_loop().time() + self.s.task_tracking_timeout
while True:
remaining = deadline - asyncio.get_running_loop().time()
if remaining <= 0:
raise asyncio.TimeoutError()
item.progress.clear()
progress = asyncio.create_task(item.progress.wait())
try:
done, _ = await asyncio.wait({item.future, progress},
timeout=min(self.s.request_timeout, remaining), return_when=asyncio.FIRST_COMPLETED)
if item.future in done:
return item.future.result()
if not done:
raise asyncio.TimeoutError()
finally:
progress.cancel()
with contextlib.suppress(asyncio.CancelledError):
await progress
async def dispatch(self, owner, text, turn, *, chat_id="", source_message_id=""):
device = self.store.by_owner(owner)
if not device:
@@ -120,7 +150,7 @@ class Broker:
if self.lifecycle:
try:
task = self.lifecycle.create_task(turn,owner,host,device["binding_generation"],session,
self.worker_id,time.time()+self.s.request_timeout,chat_id,source_message_id)
self.worker_id,time.time()+self.s.task_tracking_timeout,chat_id,source_message_id)
except ValueError:
self.store.finish(turn,"unbound")
return {"status":"unbound","turn_id":turn,"result":"绑定已变化,任务未下发。"}
@@ -157,7 +187,10 @@ class Broker:
if task:
self.lifecycle.transition(turn,"inflight","offline")
return {"status": "offline", "turn_id": turn, "result": "你的电脑当前离线。"}
result = await asyncio.wait_for(future, timeout=self.s.request_timeout)
if task:
result = await self.wait_tracked(self.pending[turn])
else:
result = await asyncio.wait_for(future, timeout=self.s.request_timeout)
self.store.finish(turn, result.get("status","unknown") if task else "completed")
if task and not source_message_id:
self.lifecycle.transition(turn,"completed","finished")