feat(tasks): track authenticated native progress until final reply
Renew broker idle waits only for encrypted current-task progress, keep an absolute deadline and binding/session/worker fences, and retain original-message reply deduplication. Add guarded prepare-update staging that reuses the active broker key material and application configuration. Validation: 216 passed, 2 skipped; guarded update and long-task routing covered with synthetic Docker configuration and loopback WebSocket tests. Real ECS build and acceptance remain pending.
This commit is contained in:
@@ -10,7 +10,7 @@ import sqlite3
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from cryptography.hazmat.primitives import hashes, padding
|
||||
@@ -55,6 +55,7 @@ class Pending:
|
||||
public: object
|
||||
host: str
|
||||
session: str
|
||||
progress: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
|
||||
|
||||
class Broker:
|
||||
@@ -76,7 +77,14 @@ class Broker:
|
||||
continue
|
||||
plain = open_envelope(data["envelope"], item.private, item.public,
|
||||
host_id=item.host, session_id=item.session)
|
||||
if plain["turn_id"] != data["turn_id"] or plain.get("type") != "result":
|
||||
if plain["turn_id"] != data["turn_id"]:
|
||||
continue
|
||||
if plain.get('type') == 'progress':
|
||||
if self.lifecycle and self.lifecycle.progress(data['turn_id'],item.host,item.session,self.worker_id):
|
||||
await self.r.expire(f"hn:inflight:{item.host}:{data['turn_id']}", self.s.request_timeout + 5)
|
||||
item.progress.set()
|
||||
continue
|
||||
if plain.get('type') != 'result':
|
||||
continue
|
||||
result = plain.get("result", "")
|
||||
if not isinstance(result, str) or len(result) > self.s.max_result:
|
||||
@@ -88,6 +96,28 @@ class Broker:
|
||||
# Invalid packets never consume a pending result; do not log packet contents.
|
||||
continue
|
||||
|
||||
async def wait_tracked(self, item):
|
||||
# Authenticated native status polls renew idle waiting, never the hard
|
||||
# task deadline. A lost device/worker remains unknown and is not replayed.
|
||||
deadline = asyncio.get_running_loop().time() + self.s.task_tracking_timeout
|
||||
while True:
|
||||
remaining = deadline - asyncio.get_running_loop().time()
|
||||
if remaining <= 0:
|
||||
raise asyncio.TimeoutError()
|
||||
item.progress.clear()
|
||||
progress = asyncio.create_task(item.progress.wait())
|
||||
try:
|
||||
done, _ = await asyncio.wait({item.future, progress},
|
||||
timeout=min(self.s.request_timeout, remaining), return_when=asyncio.FIRST_COMPLETED)
|
||||
if item.future in done:
|
||||
return item.future.result()
|
||||
if not done:
|
||||
raise asyncio.TimeoutError()
|
||||
finally:
|
||||
progress.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await progress
|
||||
|
||||
async def dispatch(self, owner, text, turn, *, chat_id="", source_message_id=""):
|
||||
device = self.store.by_owner(owner)
|
||||
if not device:
|
||||
@@ -120,7 +150,7 @@ class Broker:
|
||||
if self.lifecycle:
|
||||
try:
|
||||
task = self.lifecycle.create_task(turn,owner,host,device["binding_generation"],session,
|
||||
self.worker_id,time.time()+self.s.request_timeout,chat_id,source_message_id)
|
||||
self.worker_id,time.time()+self.s.task_tracking_timeout,chat_id,source_message_id)
|
||||
except ValueError:
|
||||
self.store.finish(turn,"unbound")
|
||||
return {"status":"unbound","turn_id":turn,"result":"绑定已变化,任务未下发。"}
|
||||
@@ -157,7 +187,10 @@ class Broker:
|
||||
if task:
|
||||
self.lifecycle.transition(turn,"inflight","offline")
|
||||
return {"status": "offline", "turn_id": turn, "result": "你的电脑当前离线。"}
|
||||
result = await asyncio.wait_for(future, timeout=self.s.request_timeout)
|
||||
if task:
|
||||
result = await self.wait_tracked(self.pending[turn])
|
||||
else:
|
||||
result = await asyncio.wait_for(future, timeout=self.s.request_timeout)
|
||||
self.store.finish(turn, result.get("status","unknown") if task else "completed")
|
||||
if task and not source_message_id:
|
||||
self.lifecycle.transition(turn,"completed","finished")
|
||||
|
||||
Reference in New Issue
Block a user