69 خطوط
3.2 KiB
Python
69 خطوط
3.2 KiB
Python
"""Private local/S3 storage. All object keys are application-generated."""
|
|
import os
|
|
import re
|
|
import secrets
|
|
from functools import lru_cache
|
|
from pathlib import Path
|
|
from .config import settings
|
|
|
|
KEY = re.compile(r'^[a-f0-9]{48}/(?:input\.(?:SchDoc|PcbDoc)|(?:old|new)\.(?:SchDoc|PcbDoc)|comparison\.json|pcb-comparison\.json|attempt-[12]/(?:data\.json|drawing\.svg|old\.svg|new\.svg))$')
|
|
|
|
class Storage:
|
|
def __init__(self):
|
|
self.config = settings()
|
|
if self.config.storage_backend == 's3':
|
|
import boto3
|
|
from botocore.config import Config
|
|
self.s3 = boto3.client('s3', endpoint_url=self.config.s3_endpoint, region_name=self.config.s3_region,
|
|
config=Config(signature_version='s3v4',
|
|
s3={'addressing_style': self.config.s3_addressing_style},
|
|
connect_timeout=self.config.s3_connect_timeout,
|
|
read_timeout=self.config.s3_read_timeout,
|
|
retries={'mode': 'standard', 'total_max_attempts': self.config.s3_max_attempts}))
|
|
else: self.s3 = None
|
|
|
|
def object_key(self, key):
|
|
"""Prefix exists only at the storage boundary, never in API IDs or DB keys."""
|
|
if not KEY.fullmatch(key): raise ValueError('Invalid storage key')
|
|
return f'{self.config.s3_prefix}/{key}' if self.config.s3_prefix else key
|
|
|
|
def path(self, key):
|
|
if not KEY.fullmatch(key): raise ValueError('Invalid storage key')
|
|
p = self.config.storage_dir / 'objects' / key
|
|
if not p.resolve().is_relative_to(self.config.storage_dir / 'objects'):
|
|
raise ValueError('Invalid storage path')
|
|
return p
|
|
|
|
def put(self, key, data, content_type='application/octet-stream'):
|
|
remote_key = self.object_key(key)
|
|
if self.s3 is not None:
|
|
options = {}
|
|
if self.config.s3_encryption: options['ServerSideEncryption'] = self.config.s3_encryption
|
|
if self.config.s3_kms_key_id: options['SSEKMSKeyId'] = self.config.s3_kms_key_id
|
|
self.s3.put_object(Bucket=self.config.s3_bucket, Key=remote_key, Body=data, ContentType=content_type, **options)
|
|
else:
|
|
p = self.path(key)
|
|
p.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
tmp = p.with_name(p.name + '.' + secrets.token_hex(8) + '.tmp')
|
|
try:
|
|
with open(tmp, 'xb') as f:
|
|
os.chmod(tmp, 0o600)
|
|
f.write(data)
|
|
os.replace(tmp, p)
|
|
finally: tmp.unlink(missing_ok=True)
|
|
|
|
def get(self, key):
|
|
remote_key = self.object_key(key)
|
|
if self.s3 is not None:
|
|
with self.s3.get_object(Bucket=self.config.s3_bucket, Key=remote_key)['Body'] as body:
|
|
return body.read()
|
|
return self.path(key).read_bytes()
|
|
|
|
def delete(self, key):
|
|
remote_key = self.object_key(key)
|
|
if self.s3 is not None: self.s3.delete_object(Bucket=self.config.s3_bucket, Key=remote_key)
|
|
else: self.path(key).unlink(missing_ok=True)
|
|
|
|
@lru_cache
|
|
def storage(): return Storage()
|