"""Opt-in, device/task-bound encrypted materials and artifact outbox.""" import base64 import hashlib import json import re import time from pathlib import PurePosixPath from fastapi import APIRouter,HTTPException,Request from fastapi.responses import JSONResponse,Response from pydantic import Field from .common import no_cache from .crypto import canonical from .task_delivery import TaskRequest FORMATS={'.docx':'application/vnd.openxmlformats-officedocument.wordprocessingml.document', '.xlsx':'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', '.pptx':'application/vnd.openxmlformats-officedocument.presentationml.presentation', '.pdf':'application/pdf','.txt':'text/plain','.csv':'text/csv','.md':'text/markdown'} def file_name(value): if (not isinstance(value,str)or not 1<=len(value)<=200 or value!=PurePosixPath(value).name or any(c in value for c in '\\/:\r\n\x00')or any(ord(c)<32 for c in value) or value.endswith((' ','.'))or PurePosixPath(value).suffix.lower()not in FORMATS): raise ValueError('unsupported_file_name_or_format') if value.split('.')[0].upper()in {'CON','PRN','AUX','NUL',*('COM'+str(i)for i in range(1,10)),*('LPT'+str(i)for i in range(1,10))}:raise ValueError('unsupported_file_name_or_format') return value class FileRequest(TaskRequest): file_id:str=Field(pattern=r'^[A-Za-z0-9_-]{8,100}$') class UploadRequest(TaskRequest): artifact_id:str=Field(pattern=r'^[A-Za-z0-9_-]{8,100}$') name:str=Field(min_length=1,max_length=200) sha256:str=Field(pattern=r'^[a-f0-9]{64}$') size:int=Field(strict=True,ge=1) class WorkFiles: def __init__(self,settings,auth,state,platform): self.s,self.auth,self.state,self.platform=settings,auth,state,platform self.db=state.db;self.keyring=auth.broker.store.keyring self.environment,self.app_id=state.environment,state.app_id with self.db: self.db.execute('CREATE TABLE IF NOT EXISTS work_file_quota_v1(environment TEXT NOT NULL,app_id TEXT NOT NULL,epoch BIGINT NOT NULL,PRIMARY KEY(environment,app_id))') self.db.execute('''CREATE TABLE IF NOT EXISTS work_files_v1( file_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL, owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL, chat_id TEXT NOT NULL,task_id TEXT NOT NULL,source_message_id TEXT NOT NULL, direction TEXT NOT NULL,name TEXT NOT NULL,size BIGINT NOT NULL,sha256 TEXT NOT NULL, key_id TEXT NOT NULL,ciphertext TEXT NOT NULL,status TEXT NOT NULL, platform_key TEXT NOT NULL,platform_message TEXT NOT NULL, updated DOUBLE PRECISION NOT NULL,expires DOUBLE PRECISION NOT NULL)''') self.router=APIRouter() self.router.add_api_route('/work/v2/files/list',self.list_files,methods=['POST']) self.router.add_api_route('/work/v2/files/download',self.download,methods=['POST']) self.router.add_api_route('/work/v2/files/upload',self.upload,methods=['POST']) self.router.add_api_route('/work/v2/files/state',self.file_state,methods=['POST']) def _aad(self,row): return canonical(['work-files-v1',*[row[k]for k in ('environment','app_id','owner','host_id','binding_generation', 'chat_id','source_message_id','direction','file_id','sha256','size','name')]]) def _id(self,kind,*parts):return kind+'_'+hashlib.sha256(canonical([self.environment,self.app_id,*parts])).hexdigest() def reserve(self,owner,size): # Called inside the owner's transaction. Serialize global quota checks # across PostgreSQL workers as well as the SQLite test implementation. scope=(self.environment,self.app_id) self.db.execute('INSERT INTO work_file_quota_v1 VALUES(?,?,0) ON CONFLICT DO NOTHING',scope) self.db.execute('UPDATE work_file_quota_v1 SET epoch=epoch+0 WHERE environment=? AND app_id=?',scope) total=self.db.execute("SELECT COALESCE(SUM(size),0) AS n FROM work_files_v1 WHERE environment=? AND app_id=? AND ciphertext!='' AND expires>?",(*scope,time.time())).fetchone()['n'] own=self.db.execute("SELECT COALESCE(SUM(size),0) AS n FROM work_files_v1 WHERE environment=? AND app_id=? AND owner=? AND ciphertext!='' AND expires>?",(*scope,owner,time.time())).fetchone()['n'] if total+size>self.s.work_file_global_quota_bytes or own+size>self.s.work_file_owner_quota_bytes:raise HTTPException(413,'file_storage_quota') def _insert(self,*,file_id,owner,host,generation,chat,task,message,direction,name,raw,status,reserved_size=None): row=dict(file_id=file_id,environment=self.environment,app_id=self.app_id,owner=owner, host_id=host,binding_generation=generation,chat_id=chat,task_id=task,source_message_id=message, direction=direction,name=name,size=len(raw)if reserved_size is None else reserved_size,sha256=hashlib.sha256(raw).hexdigest()) key,cipher=self.keyring.encrypt(raw,self._aad(row)) self.db.execute('INSERT INTO work_files_v1 VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT DO NOTHING', (*[row[k]for k in ('file_id','environment','app_id','owner','host_id','binding_generation','chat_id','task_id', 'source_message_id','direction','name','size','sha256')],key,cipher,status,'','',time.time(),time.time()+self.s.work_file_retention)) return dict(self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=?',(file_id,)).fetchone()) async def receive(self,owner,chat,message,content): """Only called after platform-event validation and persistent event receipt.""" device=self.state.store.by_owner(owner) if not device:raise ValueError('bound_device_required') name=file_name(content.get('file_name'));key=content.get('file_key') if not isinstance(key,str)or not re.fullmatch(r'[A-Za-z0-9_-]{1,256}',key):raise ValueError('invalid_platform_file_reference') identity=(owner,device['host_id'],device['binding_generation']) fid=self._id('in',*identity,message) with self.db: self.state._owner_lock(owner) self.auth.require_binding(device['host_id'],owner,device['binding_generation']) if self.pending_count(owner,device['host_id'],device['binding_generation'],chat)>=self.s.work_file_batch_limit:raise ValueError('pending_material_count_limit') old=self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=?',(fid,)).fetchone() if old:return self.metadata(dict(old)) self.reserve(owner,self.s.work_file_max_bytes) self._insert(file_id=fid,owner=owner,host=device['host_id'],generation=device['binding_generation'], chat=chat,task='',message=message,direction='input',name=name,raw=b'',status='receiving',reserved_size=self.s.work_file_max_bytes) try: raw=await self.platform.download_message_file(message,key,self.s.work_file_max_bytes) if not raw or len(raw)>self.s.work_file_max_bytes:raise ValueError('file_size_limit') with self.db: self.state._owner_lock(owner) self.auth.require_binding(device['host_id'],owner,device['binding_generation']) row=dict(self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=?',(fid,)).fetchone()) if row['status']!='receiving':raise ValueError('material_state_changed') row.update(size=len(raw),sha256=hashlib.sha256(raw).hexdigest()) key_id,cipher=self.keyring.encrypt(raw,self._aad(row)) self.db.execute("UPDATE work_files_v1 SET size=?,sha256=?,key_id=?,ciphertext=?,status='staged',updated=? WHERE file_id=? AND status='receiving'",(len(raw),row['sha256'],key_id,cipher,time.time(),fid)) row['status']='staged' return self.metadata(row) except Exception: with self.db:self.db.execute("UPDATE work_files_v1 SET status='failed',size=0,ciphertext='',updated=? WHERE file_id=? AND status='receiving'",(time.time(),fid)) raise def pending_count(self,owner,host,generation,chat,receiving_only=False): statuses=('receiving',)if receiving_only else('staged','receiving','failed') placeholders=','.join('?'for _ in statuses) return self.db.execute("SELECT COUNT(*) AS n FROM work_files_v1 WHERE environment=? AND app_id=? AND owner=? AND host_id=? AND binding_generation=? AND chat_id=? AND status IN ("+placeholders+") AND expires>?",(self.environment,self.app_id,owner,host,generation,chat,*statuses,time.time())).fetchone()['n'] def claim(self,task): with self.db: self.state._owner_lock(task['owner']) if not self.state.task_current(task):raise ValueError('stale_task') if self.pending_count(task['owner'],task['host_id'],task['binding_generation'],task['chat_id'],True):raise ValueError('materials_still_receiving') self.db.execute("UPDATE work_files_v1 SET task_id=?,status=CASE WHEN status='failed' THEN 'failed_assigned' ELSE 'assigned' END,updated=? WHERE environment=? AND app_id=? AND owner=? AND host_id=? AND binding_generation=? AND chat_id=? AND direction='input' AND status IN ('staged','failed') AND expires>?",(task['turn_id'],time.time(),self.environment,self.app_id,task['owner'],task['host_id'],task['binding_generation'],task['chat_id'],time.time())) def current(self,body): self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation) task=self.state.task(body.task_id) if (not self.state.task_current(task)or not task['source_message_id']or (task['owner'],task['host_id'],task['binding_generation'])!=(body.binding_owner,body.host_id,body.binding_generation) or task['status']in {'unbound','revoked','result_discarded','offline'}):raise HTTPException(403,'task_not_recoverable') return task def metadata(self,row): return {k:row[k]for k in ('file_id','direction','name','size','sha256','status')} async def list_files(self,body:TaskRequest,request:Request): await self.auth.authenticate(body,request);self.current(body) rows=self.db.execute("SELECT * FROM work_files_v1 WHERE environment=? AND app_id=? AND task_id=? AND expires>? ORDER BY file_id",(self.environment,self.app_id,body.task_id,time.time())).fetchall() return no_cache(JSONResponse({'protocol':'work-files-v1','files':[self.metadata(dict(r))for r in rows], 'max_bytes':self.s.work_file_max_bytes,'batch_limit':self.s.work_file_batch_limit})) def selected(self,body): task=self.current(body) row=self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=? AND task_id=? AND environment=? AND app_id=?', (body.file_id,body.task_id,self.environment,self.app_id)).fetchone() if not row or row['expires']<=time.time():raise HTTPException(404,'file_not_available') if (row['owner'],row['host_id'],row['binding_generation'])!=(task['owner'],task['host_id'],task['binding_generation']):raise HTTPException(403,'file_not_available') return dict(row) async def download(self,body:FileRequest,request:Request): await self.auth.authenticate(body,request);row=self.selected(body) if row['direction']!='input' or row['status']!='assigned':raise HTTPException(403,'input_file_required') raw=self.keyring.decrypt(row['key_id'],row['ciphertext'],self._aad(row)) if len(raw)!=row['size']or hashlib.sha256(raw).hexdigest()!=row['sha256']:raise HTTPException(409,'file_integrity_failed') self.current(body) return no_cache(Response(raw,media_type='application/octet-stream',headers={'X-Hainuo-File-SHA256':row['sha256']})) async def upload(self,request:Request): encoded=request.headers.get('X-Hainuo-File-Request','') if not encoded or len(encoded)>24000:raise HTTPException(422,'file_metadata_required') try:body=UploadRequest.model_validate(json.loads(base64.b64decode(encoded,altchars=b'-_',validate=True))) except Exception:raise HTTPException(422,'invalid_file_metadata')from None await self.auth.authenticate(body,request);task=self.current(body) try:name=file_name(body.name) except ValueError:raise HTTPException(422,'unsupported_file_name_or_format')from None if body.size>self.s.work_file_max_bytes:raise HTTPException(413,'file_size_limit') raw=await request.body() if len(raw)!=body.size or hashlib.sha256(raw).hexdigest()!=body.sha256:raise HTTPException(409,'file_integrity_failed') fid=self._id('out',task['turn_id'],body.artifact_id) with self.db: self.state._owner_lock(task['owner']);self.current(body) old=self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=?',(fid,)).fetchone() if old and (old['expires']<=time.time()or old['status']in {'expired','discarded'}):raise HTTPException(410,'file_expired') if not old: count=self.db.execute("SELECT COUNT(*) AS n FROM work_files_v1 WHERE task_id=? AND direction='output'",(body.task_id,)).fetchone()['n'] if count>=self.s.work_file_batch_limit:raise HTTPException(413,'artifact_count_limit') self.reserve(task['owner'],len(raw)) row=self._insert(file_id=fid,owner=task['owner'],host=task['host_id'],generation=task['binding_generation'], chat=task['chat_id'],task=task['turn_id'],message=task['source_message_id'],direction='output',name=name,raw=raw,status='ready') if (row['sha256'],row['size'],row['name'])!=(body.sha256,body.size,name):raise HTTPException(409,'artifact_conflict') return no_cache(JSONResponse({'received':True,**self.metadata(row)})) async def file_state(self,body:FileRequest,request:Request): await self.auth.authenticate(body,request) return no_cache(JSONResponse(self.metadata(self.selected(body)))) async def flush(self): now=time.time() with self.db: self.db.execute("UPDATE work_files_v1 SET status='expired',ciphertext='',platform_key='' WHERE environment=? AND app_id=? AND expires? ORDER BY updated LIMIT 10",(self.environment,self.app_id,now)).fetchall() for raw_row in rows: row=dict(raw_row);task=self.state.task(row['task_id']) if not self.state.task_current(task): with self.db:self.db.execute("UPDATE work_files_v1 SET status='discarded',ciphertext='',platform_key='' WHERE file_id=?",(row['file_id'],)) continue try:token=await self.platform.tenant_token() except Exception:continue with self.db: self.state._owner_lock(row['owner']) if not self.state.task_current(self.state.task(row['task_id'])):continue working='uploading'if row['status']=='ready'else'sending' changed=self.db.execute('UPDATE work_files_v1 SET status=?,updated=? WHERE file_id=? AND status=?',(working,time.time(),row['file_id'],row['status'])).rowcount if changed!=1:continue try: if working=='uploading': content=self.keyring.decrypt(row['key_id'],row['ciphertext'],self._aad(row)) if hashlib.sha256(content).hexdigest()!=row['sha256']:raise ValueError() key=await self.platform.upload_file(row['name'],content,FORMATS[PurePosixPath(row['name']).suffix.lower()],token=token) with self.db:self.db.execute("UPDATE work_files_v1 SET status='uploaded',platform_key=?,updated=? WHERE file_id=? AND status='uploading'",(key,time.time(),row['file_id'])) else: response=await self.platform.reply_file(task['source_message_id'],row['platform_key'],hashlib.sha256(row['file_id'].encode()).hexdigest()[:32],token=token) with self.db:self.db.execute("UPDATE work_files_v1 SET status='sent',platform_message=?,ciphertext='',updated=? WHERE file_id=? AND status='sending'",(response,time.time(),row['file_id'])) except Exception: with self.db:self.db.execute("UPDATE work_files_v1 SET status='unknown',updated=? WHERE file_id=? AND status=?",(time.time(),row['file_id'],working))