feat(work): add task-scoped attachment intake and durable file delivery
This commit is contained in:
@@ -23,19 +23,31 @@ async def rate_limit(redis, bucket: str, limit=60, seconds=60):
|
||||
|
||||
|
||||
class BodyLimitMiddleware:
|
||||
def __init__(self, app, max_body=262144):
|
||||
def __init__(self, app, max_body=262144, path_limits=None):
|
||||
self.app, self.max_body = app, max_body
|
||||
self.path_limits=path_limits or {}
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] != "http":
|
||||
return await self.app(scope, receive, send)
|
||||
if scope.get('path')in self.path_limits:
|
||||
# Authenticate upload metadata before buffering its binary body.
|
||||
class FileBodyTooLarge(Exception):pass
|
||||
size=0
|
||||
async def bounded():
|
||||
nonlocal size
|
||||
message=await receive();size+=len(message.get('body',b''))
|
||||
if size>self.path_limits[scope['path']]:raise FileBodyTooLarge()
|
||||
return message
|
||||
try:return await self.app(scope,bounded,send)
|
||||
except FileBodyTooLarge:return await JSONResponse({'detail':'body too large'},status_code=413)(scope,receive,send)
|
||||
messages, size = [], 0
|
||||
while True:
|
||||
message = await receive()
|
||||
if message["type"] == "http.disconnect":
|
||||
return
|
||||
size += len(message.get("body", b""))
|
||||
if size > self.max_body:
|
||||
if size > self.path_limits.get(scope.get('path'),self.max_body):
|
||||
return await JSONResponse({"detail": "body too large"}, status_code=413)(scope, receive, send)
|
||||
messages.append(message)
|
||||
if not message.get("more_body", False):
|
||||
|
||||
@@ -40,6 +40,12 @@ class Settings(BaseSettings):
|
||||
max_devices: int = 1000
|
||||
max_connections: int = 1000
|
||||
max_result: int = 16000
|
||||
work_files_enabled: bool = False
|
||||
work_file_max_bytes: int = 10*1024*1024
|
||||
work_file_batch_limit: int = 20
|
||||
work_file_retention: int = 86400
|
||||
work_file_owner_quota_bytes: int = 512*1024*1024
|
||||
work_file_global_quota_bytes: int = 2*1024*1024*1024
|
||||
allow_insecure_local: bool = False
|
||||
enrollment_signer_url: str = ""
|
||||
credential_mode: str = "legacy"
|
||||
@@ -50,6 +56,11 @@ class Settings(BaseSettings):
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_runtime(self):
|
||||
if not 1024<=self.work_file_max_bytes<=100*1024*1024 or not 1<=self.work_file_batch_limit<=50 or not 300<=self.work_file_retention<=7*86400:
|
||||
raise ValueError('invalid work file deployment limits')
|
||||
if self.work_files_enabled and self.credential_mode!='broker':raise ValueError('work files require broker mode')
|
||||
if not self.work_file_max_bytes<=self.work_file_owner_quota_bytes<=self.work_file_global_quota_bytes:
|
||||
raise ValueError('invalid work file storage quotas')
|
||||
if not 30 <= self.task_tracking_timeout <= 3600:
|
||||
raise ValueError("task tracking timeout must be 30-3600 seconds")
|
||||
if self.credential_mode not in {"legacy", "broker"}:
|
||||
|
||||
@@ -65,6 +65,7 @@ class Broker:
|
||||
self.pending: dict[str, Pending] = {}
|
||||
self.worker_id = uuid.uuid4().hex
|
||||
self.lifecycle = None
|
||||
self.files = None
|
||||
|
||||
async def replies(self, pub):
|
||||
async for message in pub.listen():
|
||||
@@ -144,6 +145,11 @@ class Broker:
|
||||
except Exception:
|
||||
self.store.audit(owner, "dispatch", host, "identity_rejected")
|
||||
raise HTTPException(409, "device identity rejected") from None
|
||||
file_capable=bool(self.files and proof.get('work_files')=='work-files-v1' and chat_id and source_message_id)
|
||||
if self.files and self.files.pending_count(owner,host,device['binding_generation'],chat_id,True):
|
||||
return {'status':'materials_receiving','turn_id':turn,'result':'材料仍在接收,尚未执行任务。请等附件接收回执后再发送任务要求。'}
|
||||
if self.files and self.files.pending_count(owner,host,device['binding_generation'],chat_id) and not file_capable:
|
||||
return {'status':'unavailable','turn_id':turn,'result':'材料已保留,但当前客户端尚未支持附件处理;请升级配套客户端后再发任务要求。'}
|
||||
if not self.store.begin_turn(turn, owner, host):
|
||||
return {"status": "duplicate", "turn_id": turn, "result": "该请求已接收,不会重复执行。"}
|
||||
task = None
|
||||
@@ -154,6 +160,12 @@ class Broker:
|
||||
except ValueError:
|
||||
self.store.finish(turn,"unbound")
|
||||
return {"status":"unbound","turn_id":turn,"result":"绑定已变化,任务未下发。"}
|
||||
if task and file_capable:
|
||||
try:self.files.claim(task)
|
||||
except ValueError:
|
||||
self.store.finish(turn,'not_dispatched')
|
||||
self.lifecycle.transition(turn,'inflight','completed')
|
||||
return {'status':'not_dispatched','turn_id':turn,'result':'材料接收或设备绑定状态发生变化,尚未执行任务;请等附件回执后核对。'}
|
||||
private = x25519.X25519PrivateKey.generate()
|
||||
future = asyncio.get_running_loop().create_future()
|
||||
self.pending[turn] = Pending(future, private, public, host, session)
|
||||
@@ -161,6 +173,8 @@ class Broker:
|
||||
"text": text, "read_only": True, "channel": "im",
|
||||
"reply_key": raw_public(private.public_key())}
|
||||
if task:
|
||||
if file_capable:
|
||||
payload['file_transfer']='work-files-v1'
|
||||
if source_message_id:
|
||||
payload['result_delivery']='durable-http-v1'
|
||||
payload["task_context"] = {key:task[key] for key in ("environment","app_id","owner","host_id","binding_generation",
|
||||
@@ -234,6 +248,9 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
lifecycle = DeviceLifecycle(store.lifecycle,r,broker)
|
||||
from .task_delivery import TaskDelivery
|
||||
deliveries = TaskDelivery(oauth,store.lifecycle,broker,f)
|
||||
if s.work_files_enabled:
|
||||
from .work_files import WorkFiles
|
||||
broker.files=WorkFiles(s,oauth,store.lifecycle,f)
|
||||
else:
|
||||
oauth = OAuth(s, r, f)
|
||||
oauth.store = store
|
||||
@@ -275,6 +292,7 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
try:
|
||||
await lifecycle.maintain()
|
||||
await deliveries.recover_interrupted_sends()
|
||||
if broker.files:await broker.files.flush()
|
||||
await deliveries.flush()
|
||||
except Exception:
|
||||
# No credentials or raw exception text in background logs.
|
||||
@@ -297,11 +315,15 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
await r.aclose()
|
||||
|
||||
app = FastAPI(title="Hainuo B1 connector", lifespan=lifespan, docs_url=None, redoc_url=None)
|
||||
app.add_middleware(BodyLimitMiddleware, max_body=s.max_body)
|
||||
app.add_middleware(BodyLimitMiddleware, max_body=s.max_body,
|
||||
path_limits={'/work/v2/files/upload':s.work_file_max_bytes}if broker.files else {})
|
||||
app.include_router(oauth.router)
|
||||
if lifecycle:
|
||||
app.include_router(deliveries.router)
|
||||
app.state.deliveries=deliveries
|
||||
if broker.files:
|
||||
app.include_router(broker.files.router)
|
||||
app.state.files=broker.files
|
||||
app.include_router(notification_router(s, r, store, f, oauth))
|
||||
from .web_session import WebSessions
|
||||
web_sessions = WebSessions(s, r, store, broker, f)
|
||||
@@ -528,7 +550,8 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
message, sender = event.get("message", {}), event.get("sender", {})
|
||||
owner = sender.get("sender_id", {}).get("open_id")
|
||||
message_id = message.get("message_id", "")
|
||||
if (message.get("chat_type") != "p2p" or message.get("message_type") != "text"
|
||||
kind=message.get('message_type')
|
||||
if (message.get("chat_type") != "p2p" or kind not in ({'text','file'}if broker.files else {'text'})
|
||||
or sender.get("sender_type") != "user" or not isinstance(owner,str)
|
||||
or not 0 < len(owner) <= 128 or any(ord(c) < 32 for c in owner)
|
||||
or not re.fullmatch(r"[A-Za-z0-9_-]{1,128}", message_id)):
|
||||
@@ -539,13 +562,23 @@ def create_app(settings=None, redis=None, platform=None):
|
||||
or any(ord(c) < 32 for c in message["chat_id"])):
|
||||
return
|
||||
try:
|
||||
text = json.loads(message["content"])["text"].strip()
|
||||
if not text or len(text) > 8000:
|
||||
content=json.loads(message['content'])
|
||||
text = content['text'].strip() if kind=='text' else ''
|
||||
if kind=='text' and (not text or len(text) > 8000):
|
||||
return
|
||||
# Persistent receipt BEFORE any side effect. Restarts do not execute a delivery twice.
|
||||
event_turn = "event_" + hashlib.sha256(message_id.encode()).hexdigest()
|
||||
if not store.begin_turn(event_turn, owner, "event"):
|
||||
return
|
||||
if kind=='file':
|
||||
try:
|
||||
await broker.files.receive(owner,message['chat_id'],message_id,content)
|
||||
result='材料已接收。可以继续发送其他材料,最后发出本次任务要求;暂未执行模型任务。'
|
||||
except Exception:
|
||||
result='这件附件未接收成功,请核对文件格式、大小、资源权限或设备绑定;其他已接收材料仍保留。'
|
||||
try:await f.reply(message_id,result)
|
||||
finally:store.finish(event_turn,'attachment_processed')
|
||||
return
|
||||
if text == "/绑定":
|
||||
result = await make_binding_code(owner)
|
||||
elif text == "/解绑":
|
||||
|
||||
@@ -122,6 +122,39 @@ class Feishu:
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json={"msg_type": "text", "content": json.dumps({"text": text}, ensure_ascii=False)})
|
||||
|
||||
async def download_message_file(self,message_id,file_key,max_bytes):
|
||||
import re
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,128}',message_id)or not re.fullmatch(r'[A-Za-z0-9_-]{1,256}',file_key):raise PlatformError('invalid file reference')
|
||||
token=await self.tenant_token()
|
||||
try:
|
||||
async with self.http.stream('GET',f'/open-apis/im/v1/messages/{message_id}/resources/{file_key}',
|
||||
params={'type':'file'},headers={'Authorization':'Bearer '+token},timeout=60)as response:
|
||||
if response.status_code!=200 or response.headers.get('content-type','').lower().startswith(('application/json','text/html')):raise PlatformError('message file download rejected')
|
||||
raw=bytearray()
|
||||
async for chunk in response.aiter_bytes():
|
||||
raw.extend(chunk)
|
||||
if len(raw)>max_bytes:raise PlatformError('message file size limit')
|
||||
if not raw:raise PlatformError('empty message file')
|
||||
return bytes(raw)
|
||||
except httpx.HTTPError:raise PlatformError('message file download failed')from None
|
||||
|
||||
async def upload_file(self,name,content,mime,*,token):
|
||||
data=await self._json('POST','/open-apis/im/v1/files',headers={'Authorization':'Bearer '+token},
|
||||
data={'file_type':'stream','file_name':name},files={'file':(name,content,mime)},timeout=60)
|
||||
key=data.get('data',{}).get('file_key')
|
||||
import re
|
||||
if not isinstance(key,str)or not re.fullmatch(r'[A-Za-z0-9_-]{1,256}',key):raise PlatformError('file upload receipt invalid')
|
||||
return key
|
||||
|
||||
async def reply_file(self,message_id,file_key,delivery_id,*,token):
|
||||
import re
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,128}',message_id):raise PlatformError('invalid reply reference')
|
||||
data=await self._json('POST',f'/open-apis/im/v1/messages/{message_id}/reply',
|
||||
headers={'Authorization':'Bearer '+token},json={'msg_type':'file','content':json.dumps({'file_key':file_key}), 'uuid':delivery_id})
|
||||
message=data.get('data',{}).get('message_id')
|
||||
if not isinstance(message,str)or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}',message):raise PlatformError('file message receipt invalid')
|
||||
return message
|
||||
|
||||
async def send(self, owner, msg_type, content, delivery_id, *, token=None):
|
||||
token = token or await self.tenant_token()
|
||||
data = await self._json('POST', '/open-apis/im/v1/messages',
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Connector-owned persistent metadata. Never stores message/file/token bodies."""
|
||||
"""Core routing metadata; adjunct services store payloads only as authenticated ciphertext."""
|
||||
import sqlite3
|
||||
import time
|
||||
import psycopg
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Opt-in, device/task-bound encrypted materials and artifact outbox."""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from pathlib import PurePosixPath
|
||||
|
||||
from fastapi import APIRouter,HTTPException,Request
|
||||
from fastapi.responses import JSONResponse,Response
|
||||
from pydantic import Field
|
||||
|
||||
from .common import no_cache
|
||||
from .crypto import canonical
|
||||
from .task_delivery import TaskRequest
|
||||
|
||||
FORMATS={'.docx':'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'.xlsx':'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
'.pptx':'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||
'.pdf':'application/pdf','.txt':'text/plain','.csv':'text/csv','.md':'text/markdown'}
|
||||
|
||||
|
||||
def file_name(value):
|
||||
if (not isinstance(value,str)or not 1<=len(value)<=200 or value!=PurePosixPath(value).name
|
||||
or any(c in value for c in '\\/:\r\n\x00')or any(ord(c)<32 for c in value)
|
||||
or value.endswith((' ','.'))or PurePosixPath(value).suffix.lower()not in FORMATS):
|
||||
raise ValueError('unsupported_file_name_or_format')
|
||||
if value.split('.')[0].upper()in {'CON','PRN','AUX','NUL',*('COM'+str(i)for i in range(1,10)),*('LPT'+str(i)for i in range(1,10))}:raise ValueError('unsupported_file_name_or_format')
|
||||
return value
|
||||
|
||||
|
||||
class FileRequest(TaskRequest):
|
||||
file_id:str=Field(pattern=r'^[A-Za-z0-9_-]{8,100}$')
|
||||
|
||||
|
||||
class UploadRequest(TaskRequest):
|
||||
artifact_id:str=Field(pattern=r'^[A-Za-z0-9_-]{8,100}$')
|
||||
name:str=Field(min_length=1,max_length=200)
|
||||
sha256:str=Field(pattern=r'^[a-f0-9]{64}$')
|
||||
size:int=Field(strict=True,ge=1)
|
||||
|
||||
|
||||
class WorkFiles:
|
||||
def __init__(self,settings,auth,state,platform):
|
||||
self.s,self.auth,self.state,self.platform=settings,auth,state,platform
|
||||
self.db=state.db;self.keyring=auth.broker.store.keyring
|
||||
self.environment,self.app_id=state.environment,state.app_id
|
||||
with self.db:
|
||||
self.db.execute('CREATE TABLE IF NOT EXISTS work_file_quota_v1(environment TEXT NOT NULL,app_id TEXT NOT NULL,epoch BIGINT NOT NULL,PRIMARY KEY(environment,app_id))')
|
||||
self.db.execute('''CREATE TABLE IF NOT EXISTS work_files_v1(
|
||||
file_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
|
||||
owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,
|
||||
chat_id TEXT NOT NULL,task_id TEXT NOT NULL,source_message_id TEXT NOT NULL,
|
||||
direction TEXT NOT NULL,name TEXT NOT NULL,size BIGINT NOT NULL,sha256 TEXT NOT NULL,
|
||||
key_id TEXT NOT NULL,ciphertext TEXT NOT NULL,status TEXT NOT NULL,
|
||||
platform_key TEXT NOT NULL,platform_message TEXT NOT NULL,
|
||||
updated DOUBLE PRECISION NOT NULL,expires DOUBLE PRECISION NOT NULL)''')
|
||||
self.router=APIRouter()
|
||||
self.router.add_api_route('/work/v2/files/list',self.list_files,methods=['POST'])
|
||||
self.router.add_api_route('/work/v2/files/download',self.download,methods=['POST'])
|
||||
self.router.add_api_route('/work/v2/files/upload',self.upload,methods=['POST'])
|
||||
self.router.add_api_route('/work/v2/files/state',self.file_state,methods=['POST'])
|
||||
|
||||
def _aad(self,row):
|
||||
return canonical(['work-files-v1',*[row[k]for k in ('environment','app_id','owner','host_id','binding_generation',
|
||||
'chat_id','source_message_id','direction','file_id','sha256','size','name')]])
|
||||
|
||||
def _id(self,kind,*parts):return kind+'_'+hashlib.sha256(canonical([self.environment,self.app_id,*parts])).hexdigest()
|
||||
|
||||
def reserve(self,owner,size):
|
||||
# Called inside the owner's transaction. Serialize global quota checks
|
||||
# across PostgreSQL workers as well as the SQLite test implementation.
|
||||
scope=(self.environment,self.app_id)
|
||||
self.db.execute('INSERT INTO work_file_quota_v1 VALUES(?,?,0) ON CONFLICT DO NOTHING',scope)
|
||||
self.db.execute('UPDATE work_file_quota_v1 SET epoch=epoch+0 WHERE environment=? AND app_id=?',scope)
|
||||
total=self.db.execute("SELECT COALESCE(SUM(size),0) AS n FROM work_files_v1 WHERE environment=? AND app_id=? AND ciphertext!='' AND expires>?",(*scope,time.time())).fetchone()['n']
|
||||
own=self.db.execute("SELECT COALESCE(SUM(size),0) AS n FROM work_files_v1 WHERE environment=? AND app_id=? AND owner=? AND ciphertext!='' AND expires>?",(*scope,owner,time.time())).fetchone()['n']
|
||||
if total+size>self.s.work_file_global_quota_bytes or own+size>self.s.work_file_owner_quota_bytes:raise HTTPException(413,'file_storage_quota')
|
||||
|
||||
def _insert(self,*,file_id,owner,host,generation,chat,task,message,direction,name,raw,status,reserved_size=None):
|
||||
row=dict(file_id=file_id,environment=self.environment,app_id=self.app_id,owner=owner,
|
||||
host_id=host,binding_generation=generation,chat_id=chat,task_id=task,source_message_id=message,
|
||||
direction=direction,name=name,size=len(raw)if reserved_size is None else reserved_size,sha256=hashlib.sha256(raw).hexdigest())
|
||||
key,cipher=self.keyring.encrypt(raw,self._aad(row))
|
||||
self.db.execute('INSERT INTO work_files_v1 VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT DO NOTHING',
|
||||
(*[row[k]for k in ('file_id','environment','app_id','owner','host_id','binding_generation','chat_id','task_id',
|
||||
'source_message_id','direction','name','size','sha256')],key,cipher,status,'','',time.time(),time.time()+self.s.work_file_retention))
|
||||
return dict(self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=?',(file_id,)).fetchone())
|
||||
|
||||
async def receive(self,owner,chat,message,content):
|
||||
"""Only called after platform-event validation and persistent event receipt."""
|
||||
device=self.state.store.by_owner(owner)
|
||||
if not device:raise ValueError('bound_device_required')
|
||||
name=file_name(content.get('file_name'));key=content.get('file_key')
|
||||
if not isinstance(key,str)or not re.fullmatch(r'[A-Za-z0-9_-]{1,256}',key):raise ValueError('invalid_platform_file_reference')
|
||||
identity=(owner,device['host_id'],device['binding_generation'])
|
||||
fid=self._id('in',*identity,message)
|
||||
with self.db:
|
||||
self.state._owner_lock(owner)
|
||||
self.auth.require_binding(device['host_id'],owner,device['binding_generation'])
|
||||
if self.pending_count(owner,device['host_id'],device['binding_generation'],chat)>=self.s.work_file_batch_limit:raise ValueError('pending_material_count_limit')
|
||||
old=self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=?',(fid,)).fetchone()
|
||||
if old:return self.metadata(dict(old))
|
||||
self.reserve(owner,self.s.work_file_max_bytes)
|
||||
self._insert(file_id=fid,owner=owner,host=device['host_id'],generation=device['binding_generation'],
|
||||
chat=chat,task='',message=message,direction='input',name=name,raw=b'',status='receiving',reserved_size=self.s.work_file_max_bytes)
|
||||
try:
|
||||
raw=await self.platform.download_message_file(message,key,self.s.work_file_max_bytes)
|
||||
if not raw or len(raw)>self.s.work_file_max_bytes:raise ValueError('file_size_limit')
|
||||
with self.db:
|
||||
self.state._owner_lock(owner)
|
||||
self.auth.require_binding(device['host_id'],owner,device['binding_generation'])
|
||||
row=dict(self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=?',(fid,)).fetchone())
|
||||
if row['status']!='receiving':raise ValueError('material_state_changed')
|
||||
row.update(size=len(raw),sha256=hashlib.sha256(raw).hexdigest())
|
||||
key_id,cipher=self.keyring.encrypt(raw,self._aad(row))
|
||||
self.db.execute("UPDATE work_files_v1 SET size=?,sha256=?,key_id=?,ciphertext=?,status='staged',updated=? WHERE file_id=? AND status='receiving'",(len(raw),row['sha256'],key_id,cipher,time.time(),fid))
|
||||
row['status']='staged'
|
||||
return self.metadata(row)
|
||||
except Exception:
|
||||
with self.db:self.db.execute("UPDATE work_files_v1 SET status='failed',size=0,ciphertext='',updated=? WHERE file_id=? AND status='receiving'",(time.time(),fid))
|
||||
raise
|
||||
|
||||
def pending_count(self,owner,host,generation,chat,receiving_only=False):
|
||||
statuses=('receiving',)if receiving_only else('staged','receiving','failed')
|
||||
placeholders=','.join('?'for _ in statuses)
|
||||
return self.db.execute("SELECT COUNT(*) AS n FROM work_files_v1 WHERE environment=? AND app_id=? AND owner=? AND host_id=? AND binding_generation=? AND chat_id=? AND status IN ("+placeholders+") AND expires>?",(self.environment,self.app_id,owner,host,generation,chat,*statuses,time.time())).fetchone()['n']
|
||||
|
||||
def claim(self,task):
|
||||
with self.db:
|
||||
self.state._owner_lock(task['owner'])
|
||||
if not self.state.task_current(task):raise ValueError('stale_task')
|
||||
if self.pending_count(task['owner'],task['host_id'],task['binding_generation'],task['chat_id'],True):raise ValueError('materials_still_receiving')
|
||||
self.db.execute("UPDATE work_files_v1 SET task_id=?,status=CASE WHEN status='failed' THEN 'failed_assigned' ELSE 'assigned' END,updated=? WHERE environment=? AND app_id=? AND owner=? AND host_id=? AND binding_generation=? AND chat_id=? AND direction='input' AND status IN ('staged','failed') AND expires>?",(task['turn_id'],time.time(),self.environment,self.app_id,task['owner'],task['host_id'],task['binding_generation'],task['chat_id'],time.time()))
|
||||
|
||||
def current(self,body):
|
||||
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
|
||||
task=self.state.task(body.task_id)
|
||||
if (not self.state.task_current(task)or not task['source_message_id']or
|
||||
(task['owner'],task['host_id'],task['binding_generation'])!=(body.binding_owner,body.host_id,body.binding_generation)
|
||||
or task['status']in {'unbound','revoked','result_discarded','offline'}):raise HTTPException(403,'task_not_recoverable')
|
||||
return task
|
||||
|
||||
def metadata(self,row):
|
||||
return {k:row[k]for k in ('file_id','direction','name','size','sha256','status')}
|
||||
|
||||
async def list_files(self,body:TaskRequest,request:Request):
|
||||
await self.auth.authenticate(body,request);self.current(body)
|
||||
rows=self.db.execute("SELECT * FROM work_files_v1 WHERE environment=? AND app_id=? AND task_id=? AND expires>? ORDER BY file_id",(self.environment,self.app_id,body.task_id,time.time())).fetchall()
|
||||
return no_cache(JSONResponse({'protocol':'work-files-v1','files':[self.metadata(dict(r))for r in rows],
|
||||
'max_bytes':self.s.work_file_max_bytes,'batch_limit':self.s.work_file_batch_limit}))
|
||||
|
||||
def selected(self,body):
|
||||
task=self.current(body)
|
||||
row=self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=? AND task_id=? AND environment=? AND app_id=?',
|
||||
(body.file_id,body.task_id,self.environment,self.app_id)).fetchone()
|
||||
if not row or row['expires']<=time.time():raise HTTPException(404,'file_not_available')
|
||||
if (row['owner'],row['host_id'],row['binding_generation'])!=(task['owner'],task['host_id'],task['binding_generation']):raise HTTPException(403,'file_not_available')
|
||||
return dict(row)
|
||||
|
||||
async def download(self,body:FileRequest,request:Request):
|
||||
await self.auth.authenticate(body,request);row=self.selected(body)
|
||||
if row['direction']!='input' or row['status']!='assigned':raise HTTPException(403,'input_file_required')
|
||||
raw=self.keyring.decrypt(row['key_id'],row['ciphertext'],self._aad(row))
|
||||
if len(raw)!=row['size']or hashlib.sha256(raw).hexdigest()!=row['sha256']:raise HTTPException(409,'file_integrity_failed')
|
||||
self.current(body)
|
||||
return no_cache(Response(raw,media_type='application/octet-stream',headers={'X-Hainuo-File-SHA256':row['sha256']}))
|
||||
|
||||
async def upload(self,request:Request):
|
||||
encoded=request.headers.get('X-Hainuo-File-Request','')
|
||||
if not encoded or len(encoded)>24000:raise HTTPException(422,'file_metadata_required')
|
||||
try:body=UploadRequest.model_validate(json.loads(base64.b64decode(encoded,altchars=b'-_',validate=True)))
|
||||
except Exception:raise HTTPException(422,'invalid_file_metadata')from None
|
||||
await self.auth.authenticate(body,request);task=self.current(body)
|
||||
try:name=file_name(body.name)
|
||||
except ValueError:raise HTTPException(422,'unsupported_file_name_or_format')from None
|
||||
if body.size>self.s.work_file_max_bytes:raise HTTPException(413,'file_size_limit')
|
||||
raw=await request.body()
|
||||
if len(raw)!=body.size or hashlib.sha256(raw).hexdigest()!=body.sha256:raise HTTPException(409,'file_integrity_failed')
|
||||
fid=self._id('out',task['turn_id'],body.artifact_id)
|
||||
with self.db:
|
||||
self.state._owner_lock(task['owner']);self.current(body)
|
||||
old=self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=?',(fid,)).fetchone()
|
||||
if old and (old['expires']<=time.time()or old['status']in {'expired','discarded'}):raise HTTPException(410,'file_expired')
|
||||
if not old:
|
||||
count=self.db.execute("SELECT COUNT(*) AS n FROM work_files_v1 WHERE task_id=? AND direction='output'",(body.task_id,)).fetchone()['n']
|
||||
if count>=self.s.work_file_batch_limit:raise HTTPException(413,'artifact_count_limit')
|
||||
self.reserve(task['owner'],len(raw))
|
||||
row=self._insert(file_id=fid,owner=task['owner'],host=task['host_id'],generation=task['binding_generation'],
|
||||
chat=task['chat_id'],task=task['turn_id'],message=task['source_message_id'],direction='output',name=name,raw=raw,status='ready')
|
||||
if (row['sha256'],row['size'],row['name'])!=(body.sha256,body.size,name):raise HTTPException(409,'artifact_conflict')
|
||||
return no_cache(JSONResponse({'received':True,**self.metadata(row)}))
|
||||
|
||||
async def file_state(self,body:FileRequest,request:Request):
|
||||
await self.auth.authenticate(body,request)
|
||||
return no_cache(JSONResponse(self.metadata(self.selected(body))))
|
||||
|
||||
async def flush(self):
|
||||
now=time.time()
|
||||
with self.db:
|
||||
self.db.execute("UPDATE work_files_v1 SET status='expired',ciphertext='',platform_key='' WHERE environment=? AND app_id=? AND expires<? AND ciphertext!=''",(self.environment,self.app_id,now))
|
||||
self.db.execute("UPDATE work_files_v1 SET status='failed',size=0,ciphertext='',updated=? WHERE environment=? AND app_id=? AND status='receiving' AND updated<?",(now,self.environment,self.app_id,now-120))
|
||||
# Interrupted mutating platform calls remain ambiguous, never replayed.
|
||||
self.db.execute("UPDATE work_files_v1 SET status='unknown',updated=? WHERE environment=? AND app_id=? AND status IN ('uploading','sending') AND updated<?",(now,self.environment,self.app_id,now-120))
|
||||
rows=self.db.execute("SELECT * FROM work_files_v1 WHERE environment=? AND app_id=? AND direction='output' AND status IN ('ready','uploaded') AND expires>? ORDER BY updated LIMIT 10",(self.environment,self.app_id,now)).fetchall()
|
||||
for raw_row in rows:
|
||||
row=dict(raw_row);task=self.state.task(row['task_id'])
|
||||
if not self.state.task_current(task):
|
||||
with self.db:self.db.execute("UPDATE work_files_v1 SET status='discarded',ciphertext='',platform_key='' WHERE file_id=?",(row['file_id'],))
|
||||
continue
|
||||
try:token=await self.platform.tenant_token()
|
||||
except Exception:continue
|
||||
with self.db:
|
||||
self.state._owner_lock(row['owner'])
|
||||
if not self.state.task_current(self.state.task(row['task_id'])):continue
|
||||
working='uploading'if row['status']=='ready'else'sending'
|
||||
changed=self.db.execute('UPDATE work_files_v1 SET status=?,updated=? WHERE file_id=? AND status=?',(working,time.time(),row['file_id'],row['status'])).rowcount
|
||||
if changed!=1:continue
|
||||
try:
|
||||
if working=='uploading':
|
||||
content=self.keyring.decrypt(row['key_id'],row['ciphertext'],self._aad(row))
|
||||
if hashlib.sha256(content).hexdigest()!=row['sha256']:raise ValueError()
|
||||
key=await self.platform.upload_file(row['name'],content,FORMATS[PurePosixPath(row['name']).suffix.lower()],token=token)
|
||||
with self.db:self.db.execute("UPDATE work_files_v1 SET status='uploaded',platform_key=?,updated=? WHERE file_id=? AND status='uploading'",(key,time.time(),row['file_id']))
|
||||
else:
|
||||
response=await self.platform.reply_file(task['source_message_id'],row['platform_key'],hashlib.sha256(row['file_id'].encode()).hexdigest()[:32],token=token)
|
||||
with self.db:self.db.execute("UPDATE work_files_v1 SET status='sent',platform_message=?,ciphertext='',updated=? WHERE file_id=? AND status='sending'",(response,time.time(),row['file_id']))
|
||||
except Exception:
|
||||
with self.db:self.db.execute("UPDATE work_files_v1 SET status='unknown',updated=? WHERE file_id=? AND status=?",(time.time(),row['file_id'],working))
|
||||
Reference in New Issue
Block a user