230 lines
17 KiB
Python
230 lines
17 KiB
Python
"""Opt-in, device/task-bound encrypted materials and artifact outbox."""
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import time
|
|
from pathlib import PurePosixPath
|
|
|
|
from fastapi import APIRouter,HTTPException,Request
|
|
from fastapi.responses import JSONResponse,Response
|
|
from pydantic import Field
|
|
|
|
from .common import no_cache
|
|
from .crypto import canonical
|
|
from .task_delivery import TaskRequest
|
|
|
|
FORMATS={'.docx':'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
'.xlsx':'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
'.pptx':'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
'.pdf':'application/pdf','.txt':'text/plain','.csv':'text/csv','.md':'text/markdown'}
|
|
|
|
|
|
def file_name(value):
|
|
if (not isinstance(value,str)or not 1<=len(value)<=200 or value!=PurePosixPath(value).name
|
|
or any(c in value for c in '\\/:\r\n\x00')or any(ord(c)<32 for c in value)
|
|
or value.endswith((' ','.'))or PurePosixPath(value).suffix.lower()not in FORMATS):
|
|
raise ValueError('unsupported_file_name_or_format')
|
|
if value.split('.')[0].upper()in {'CON','PRN','AUX','NUL',*('COM'+str(i)for i in range(1,10)),*('LPT'+str(i)for i in range(1,10))}:raise ValueError('unsupported_file_name_or_format')
|
|
return value
|
|
|
|
|
|
class FileRequest(TaskRequest):
|
|
file_id:str=Field(pattern=r'^[A-Za-z0-9_-]{8,100}$')
|
|
|
|
|
|
class UploadRequest(TaskRequest):
|
|
artifact_id:str=Field(pattern=r'^[A-Za-z0-9_-]{8,100}$')
|
|
name:str=Field(min_length=1,max_length=200)
|
|
sha256:str=Field(pattern=r'^[a-f0-9]{64}$')
|
|
size:int=Field(strict=True,ge=1)
|
|
|
|
|
|
class WorkFiles:
|
|
def __init__(self,settings,auth,state,platform):
|
|
self.s,self.auth,self.state,self.platform=settings,auth,state,platform
|
|
self.db=state.db;self.keyring=auth.broker.store.keyring
|
|
self.environment,self.app_id=state.environment,state.app_id
|
|
with self.db:
|
|
self.db.execute('CREATE TABLE IF NOT EXISTS work_file_quota_v1(environment TEXT NOT NULL,app_id TEXT NOT NULL,epoch BIGINT NOT NULL,PRIMARY KEY(environment,app_id))')
|
|
self.db.execute('''CREATE TABLE IF NOT EXISTS work_files_v1(
|
|
file_id TEXT PRIMARY KEY,environment TEXT NOT NULL,app_id TEXT NOT NULL,
|
|
owner TEXT NOT NULL,host_id TEXT NOT NULL,binding_generation BIGINT NOT NULL,
|
|
chat_id TEXT NOT NULL,task_id TEXT NOT NULL,source_message_id TEXT NOT NULL,
|
|
direction TEXT NOT NULL,name TEXT NOT NULL,size BIGINT NOT NULL,sha256 TEXT NOT NULL,
|
|
key_id TEXT NOT NULL,ciphertext TEXT NOT NULL,status TEXT NOT NULL,
|
|
platform_key TEXT NOT NULL,platform_message TEXT NOT NULL,
|
|
updated DOUBLE PRECISION NOT NULL,expires DOUBLE PRECISION NOT NULL)''')
|
|
self.router=APIRouter()
|
|
self.router.add_api_route('/work/v2/files/list',self.list_files,methods=['POST'])
|
|
self.router.add_api_route('/work/v2/files/download',self.download,methods=['POST'])
|
|
self.router.add_api_route('/work/v2/files/upload',self.upload,methods=['POST'])
|
|
self.router.add_api_route('/work/v2/files/state',self.file_state,methods=['POST'])
|
|
|
|
def _aad(self,row):
|
|
return canonical(['work-files-v1',*[row[k]for k in ('environment','app_id','owner','host_id','binding_generation',
|
|
'chat_id','source_message_id','direction','file_id','sha256','size','name')]])
|
|
|
|
def _id(self,kind,*parts):return kind+'_'+hashlib.sha256(canonical([self.environment,self.app_id,*parts])).hexdigest()
|
|
|
|
def reserve(self,owner,size):
|
|
# Called inside the owner's transaction. Serialize global quota checks
|
|
# across PostgreSQL workers as well as the SQLite test implementation.
|
|
scope=(self.environment,self.app_id)
|
|
self.db.execute('INSERT INTO work_file_quota_v1 VALUES(?,?,0) ON CONFLICT DO NOTHING',scope)
|
|
self.db.execute('UPDATE work_file_quota_v1 SET epoch=epoch+0 WHERE environment=? AND app_id=?',scope)
|
|
total=self.db.execute("SELECT COALESCE(SUM(size),0) AS n FROM work_files_v1 WHERE environment=? AND app_id=? AND ciphertext!='' AND expires>?",(*scope,time.time())).fetchone()['n']
|
|
own=self.db.execute("SELECT COALESCE(SUM(size),0) AS n FROM work_files_v1 WHERE environment=? AND app_id=? AND owner=? AND ciphertext!='' AND expires>?",(*scope,owner,time.time())).fetchone()['n']
|
|
if total+size>self.s.work_file_global_quota_bytes or own+size>self.s.work_file_owner_quota_bytes:raise HTTPException(413,'file_storage_quota')
|
|
|
|
def _insert(self,*,file_id,owner,host,generation,chat,task,message,direction,name,raw,status,reserved_size=None):
|
|
row=dict(file_id=file_id,environment=self.environment,app_id=self.app_id,owner=owner,
|
|
host_id=host,binding_generation=generation,chat_id=chat,task_id=task,source_message_id=message,
|
|
direction=direction,name=name,size=len(raw)if reserved_size is None else reserved_size,sha256=hashlib.sha256(raw).hexdigest())
|
|
key,cipher=self.keyring.encrypt(raw,self._aad(row))
|
|
self.db.execute('INSERT INTO work_files_v1 VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT DO NOTHING',
|
|
(*[row[k]for k in ('file_id','environment','app_id','owner','host_id','binding_generation','chat_id','task_id',
|
|
'source_message_id','direction','name','size','sha256')],key,cipher,status,'','',time.time(),time.time()+self.s.work_file_retention))
|
|
return dict(self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=?',(file_id,)).fetchone())
|
|
|
|
async def receive(self,owner,chat,message,content):
|
|
"""Only called after platform-event validation and persistent event receipt."""
|
|
device=self.state.store.by_owner(owner)
|
|
if not device:raise ValueError('bound_device_required')
|
|
name=file_name(content.get('file_name'));key=content.get('file_key')
|
|
if not isinstance(key,str)or not re.fullmatch(r'[A-Za-z0-9_-]{1,256}',key):raise ValueError('invalid_platform_file_reference')
|
|
identity=(owner,device['host_id'],device['binding_generation'])
|
|
fid=self._id('in',*identity,message)
|
|
with self.db:
|
|
self.state._owner_lock(owner)
|
|
self.auth.require_binding(device['host_id'],owner,device['binding_generation'])
|
|
if self.pending_count(owner,device['host_id'],device['binding_generation'],chat)>=self.s.work_file_batch_limit:raise ValueError('pending_material_count_limit')
|
|
old=self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=?',(fid,)).fetchone()
|
|
if old:return self.metadata(dict(old))
|
|
self.reserve(owner,self.s.work_file_max_bytes)
|
|
self._insert(file_id=fid,owner=owner,host=device['host_id'],generation=device['binding_generation'],
|
|
chat=chat,task='',message=message,direction='input',name=name,raw=b'',status='receiving',reserved_size=self.s.work_file_max_bytes)
|
|
try:
|
|
raw=await self.platform.download_message_file(message,key,self.s.work_file_max_bytes)
|
|
if not raw or len(raw)>self.s.work_file_max_bytes:raise ValueError('file_size_limit')
|
|
with self.db:
|
|
self.state._owner_lock(owner)
|
|
self.auth.require_binding(device['host_id'],owner,device['binding_generation'])
|
|
row=dict(self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=?',(fid,)).fetchone())
|
|
if row['status']!='receiving':raise ValueError('material_state_changed')
|
|
row.update(size=len(raw),sha256=hashlib.sha256(raw).hexdigest())
|
|
key_id,cipher=self.keyring.encrypt(raw,self._aad(row))
|
|
self.db.execute("UPDATE work_files_v1 SET size=?,sha256=?,key_id=?,ciphertext=?,status='staged',updated=? WHERE file_id=? AND status='receiving'",(len(raw),row['sha256'],key_id,cipher,time.time(),fid))
|
|
row['status']='staged'
|
|
return self.metadata(row)
|
|
except Exception:
|
|
with self.db:self.db.execute("UPDATE work_files_v1 SET status='failed',size=0,ciphertext='',updated=? WHERE file_id=? AND status='receiving'",(time.time(),fid))
|
|
raise
|
|
|
|
def pending_count(self,owner,host,generation,chat,receiving_only=False):
|
|
statuses=('receiving',)if receiving_only else('staged','receiving','failed')
|
|
placeholders=','.join('?'for _ in statuses)
|
|
return self.db.execute("SELECT COUNT(*) AS n FROM work_files_v1 WHERE environment=? AND app_id=? AND owner=? AND host_id=? AND binding_generation=? AND chat_id=? AND status IN ("+placeholders+") AND expires>?",(self.environment,self.app_id,owner,host,generation,chat,*statuses,time.time())).fetchone()['n']
|
|
|
|
def claim(self,task):
|
|
with self.db:
|
|
self.state._owner_lock(task['owner'])
|
|
if not self.state.task_current(task):raise ValueError('stale_task')
|
|
if self.pending_count(task['owner'],task['host_id'],task['binding_generation'],task['chat_id'],True):raise ValueError('materials_still_receiving')
|
|
self.db.execute("UPDATE work_files_v1 SET task_id=?,status=CASE WHEN status='failed' THEN 'failed_assigned' ELSE 'assigned' END,updated=? WHERE environment=? AND app_id=? AND owner=? AND host_id=? AND binding_generation=? AND chat_id=? AND direction='input' AND status IN ('staged','failed') AND expires>?",(task['turn_id'],time.time(),self.environment,self.app_id,task['owner'],task['host_id'],task['binding_generation'],task['chat_id'],time.time()))
|
|
|
|
def current(self,body):
|
|
self.auth.require_binding(body.host_id,body.binding_owner,body.binding_generation)
|
|
task=self.state.task(body.task_id)
|
|
if (not self.state.task_current(task)or not task['source_message_id']or
|
|
(task['owner'],task['host_id'],task['binding_generation'])!=(body.binding_owner,body.host_id,body.binding_generation)
|
|
or task['status']in {'unbound','revoked','result_discarded','offline'}):raise HTTPException(403,'task_not_recoverable')
|
|
return task
|
|
|
|
def metadata(self,row):
|
|
return {k:row[k]for k in ('file_id','direction','name','size','sha256','status')}
|
|
|
|
async def list_files(self,body:TaskRequest,request:Request):
|
|
await self.auth.authenticate(body,request);self.current(body)
|
|
rows=self.db.execute("SELECT * FROM work_files_v1 WHERE environment=? AND app_id=? AND task_id=? AND expires>? ORDER BY file_id",(self.environment,self.app_id,body.task_id,time.time())).fetchall()
|
|
return no_cache(JSONResponse({'protocol':'work-files-v1','files':[self.metadata(dict(r))for r in rows],
|
|
'max_bytes':self.s.work_file_max_bytes,'batch_limit':self.s.work_file_batch_limit}))
|
|
|
|
def selected(self,body):
|
|
task=self.current(body)
|
|
row=self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=? AND task_id=? AND environment=? AND app_id=?',
|
|
(body.file_id,body.task_id,self.environment,self.app_id)).fetchone()
|
|
if not row or row['expires']<=time.time():raise HTTPException(404,'file_not_available')
|
|
if (row['owner'],row['host_id'],row['binding_generation'])!=(task['owner'],task['host_id'],task['binding_generation']):raise HTTPException(403,'file_not_available')
|
|
return dict(row)
|
|
|
|
async def download(self,body:FileRequest,request:Request):
|
|
await self.auth.authenticate(body,request);row=self.selected(body)
|
|
if row['direction']!='input' or row['status']!='assigned':raise HTTPException(403,'input_file_required')
|
|
raw=self.keyring.decrypt(row['key_id'],row['ciphertext'],self._aad(row))
|
|
if len(raw)!=row['size']or hashlib.sha256(raw).hexdigest()!=row['sha256']:raise HTTPException(409,'file_integrity_failed')
|
|
self.current(body)
|
|
return no_cache(Response(raw,media_type='application/octet-stream',headers={'X-Hainuo-File-SHA256':row['sha256']}))
|
|
|
|
async def upload(self,request:Request):
|
|
encoded=request.headers.get('X-Hainuo-File-Request','')
|
|
if not encoded or len(encoded)>24000:raise HTTPException(422,'file_metadata_required')
|
|
try:body=UploadRequest.model_validate(json.loads(base64.b64decode(encoded,altchars=b'-_',validate=True)))
|
|
except Exception:raise HTTPException(422,'invalid_file_metadata')from None
|
|
await self.auth.authenticate(body,request);task=self.current(body)
|
|
try:name=file_name(body.name)
|
|
except ValueError:raise HTTPException(422,'unsupported_file_name_or_format')from None
|
|
if body.size>self.s.work_file_max_bytes:raise HTTPException(413,'file_size_limit')
|
|
raw=await request.body()
|
|
if len(raw)!=body.size or hashlib.sha256(raw).hexdigest()!=body.sha256:raise HTTPException(409,'file_integrity_failed')
|
|
fid=self._id('out',task['turn_id'],body.artifact_id)
|
|
with self.db:
|
|
self.state._owner_lock(task['owner']);self.current(body)
|
|
old=self.db.execute('SELECT * FROM work_files_v1 WHERE file_id=?',(fid,)).fetchone()
|
|
if old and (old['expires']<=time.time()or old['status']in {'expired','discarded'}):raise HTTPException(410,'file_expired')
|
|
if not old:
|
|
count=self.db.execute("SELECT COUNT(*) AS n FROM work_files_v1 WHERE task_id=? AND direction='output'",(body.task_id,)).fetchone()['n']
|
|
if count>=self.s.work_file_batch_limit:raise HTTPException(413,'artifact_count_limit')
|
|
self.reserve(task['owner'],len(raw))
|
|
row=self._insert(file_id=fid,owner=task['owner'],host=task['host_id'],generation=task['binding_generation'],
|
|
chat=task['chat_id'],task=task['turn_id'],message=task['source_message_id'],direction='output',name=name,raw=raw,status='ready')
|
|
if (row['sha256'],row['size'],row['name'])!=(body.sha256,body.size,name):raise HTTPException(409,'artifact_conflict')
|
|
return no_cache(JSONResponse({'received':True,**self.metadata(row)}))
|
|
|
|
async def file_state(self,body:FileRequest,request:Request):
|
|
await self.auth.authenticate(body,request)
|
|
return no_cache(JSONResponse(self.metadata(self.selected(body))))
|
|
|
|
async def flush(self):
|
|
now=time.time()
|
|
with self.db:
|
|
self.db.execute("UPDATE work_files_v1 SET status='expired',ciphertext='',platform_key='' WHERE environment=? AND app_id=? AND expires<? AND ciphertext!=''",(self.environment,self.app_id,now))
|
|
self.db.execute("UPDATE work_files_v1 SET status='failed',size=0,ciphertext='',updated=? WHERE environment=? AND app_id=? AND status='receiving' AND updated<?",(now,self.environment,self.app_id,now-120))
|
|
# Interrupted mutating platform calls remain ambiguous, never replayed.
|
|
self.db.execute("UPDATE work_files_v1 SET status='unknown',updated=? WHERE environment=? AND app_id=? AND status IN ('uploading','sending') AND updated<?",(now,self.environment,self.app_id,now-120))
|
|
rows=self.db.execute("SELECT * FROM work_files_v1 WHERE environment=? AND app_id=? AND direction='output' AND status IN ('ready','uploaded') AND expires>? ORDER BY updated LIMIT 10",(self.environment,self.app_id,now)).fetchall()
|
|
for raw_row in rows:
|
|
row=dict(raw_row);task=self.state.task(row['task_id'])
|
|
if not self.state.task_current(task):
|
|
with self.db:self.db.execute("UPDATE work_files_v1 SET status='discarded',ciphertext='',platform_key='' WHERE file_id=?",(row['file_id'],))
|
|
continue
|
|
try:token=await self.platform.tenant_token()
|
|
except Exception:continue
|
|
with self.db:
|
|
self.state._owner_lock(row['owner'])
|
|
if not self.state.task_current(self.state.task(row['task_id'])):continue
|
|
working='uploading'if row['status']=='ready'else'sending'
|
|
changed=self.db.execute('UPDATE work_files_v1 SET status=?,updated=? WHERE file_id=? AND status=?',(working,time.time(),row['file_id'],row['status'])).rowcount
|
|
if changed!=1:continue
|
|
try:
|
|
if working=='uploading':
|
|
content=self.keyring.decrypt(row['key_id'],row['ciphertext'],self._aad(row))
|
|
if hashlib.sha256(content).hexdigest()!=row['sha256']:raise ValueError()
|
|
key=await self.platform.upload_file(row['name'],content,FORMATS[PurePosixPath(row['name']).suffix.lower()],token=token)
|
|
with self.db:self.db.execute("UPDATE work_files_v1 SET status='uploaded',platform_key=?,updated=? WHERE file_id=? AND status='uploading'",(key,time.time(),row['file_id']))
|
|
else:
|
|
response=await self.platform.reply_file(task['source_message_id'],row['platform_key'],hashlib.sha256(row['file_id'].encode()).hexdigest()[:32],token=token)
|
|
with self.db:self.db.execute("UPDATE work_files_v1 SET status='sent',platform_message=?,ciphertext='',updated=? WHERE file_id=? AND status='sending'",(response,time.time(),row['file_id']))
|
|
except Exception:
|
|
with self.db:self.db.execute("UPDATE work_files_v1 SET status='unknown',updated=? WHERE file_id=? AND status=?",(time.time(),row['file_id'],working))
|