اضافه شدن قابلیت حافظه S3 به پروژه

backend/app/config.py

@@ -3,6 +3,7 @@ import os
 from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
+from urllib.parse import urlsplit
 
 @dataclass(frozen=True)
 class Settings:
@@ -18,10 +19,30 @@ class Settings:
     s3_bucket: str
     s3_endpoint: str | None
     s3_region: str
+    storage_backend: str
+    s3_prefix: str
+    s3_addressing_style: str
+    s3_connect_timeout: int
+    s3_read_timeout: int
+    s3_max_attempts: int
+    s3_encryption: str
+    s3_kms_key_id: str
 
 @lru_cache
 def settings():
     root = Path(os.getenv('STORAGE_DIR', './data')).resolve()
+    bucket = os.getenv('S3_BUCKET', '').strip()
+    backend = os.getenv('STORAGE_BACKEND', 'auto').strip().lower()
+    if backend == 'auto': backend = 's3' if bucket else 'local'
+    if backend not in ('local', 's3'):
+        raise RuntimeError('STORAGE_BACKEND must be auto, local or s3')
+    if backend == 's3' and not bucket:
+        raise RuntimeError('S3_BUCKET is required when STORAGE_BACKEND=s3')
+    prefix = os.getenv('S3_PREFIX', '').strip('/')
+    if (len(prefix.encode('utf-8')) > 512 or '\\' in prefix or
+        any(ord(c) < 32 or ord(c) == 127 for c in prefix) or
+        (prefix and any(part in ('', '.', '..') for part in prefix.split('/')))):
+        raise RuntimeError('S3_PREFIX must be a relative object prefix without empty, dot or traversal segments (max 512 UTF-8 bytes)')
     config = Settings(
         environment=os.getenv('APP_ENV', 'development'),
         database_url=os.getenv('DATABASE_URL', f'sqlite:///{root}/jobs.db'),
@@ -32,9 +53,17 @@ def settings():
         parse_timeout=int(os.getenv('PARSE_TIMEOUT_SECONDS', '60')),
         link_ttl=int(os.getenv('LINK_TTL_SECONDS', '604800')),
         max_pending=int(os.getenv('MAX_PENDING_JOBS', '100')),
-        s3_bucket=os.getenv('S3_BUCKET', ''),
+        s3_bucket=bucket,
         s3_endpoint=os.getenv('S3_ENDPOINT_URL') or None,
-        s3_region=os.getenv('AWS_DEFAULT_REGION', 'us-east-1'),
+        s3_region=os.getenv('S3_REGION') or os.getenv('AWS_DEFAULT_REGION') or 'us-east-1',
+        storage_backend=backend,
+        s3_prefix=prefix,
+        s3_addressing_style=os.getenv('S3_ADDRESSING_STYLE', 'auto').strip().lower(),
+        s3_connect_timeout=int(os.getenv('S3_CONNECT_TIMEOUT', '5')),
+        s3_read_timeout=int(os.getenv('S3_READ_TIMEOUT', '60')),
+        s3_max_attempts=int(os.getenv('S3_MAX_ATTEMPTS', '3')),
+        s3_encryption=os.getenv('S3_SERVER_SIDE_ENCRYPTION', '').strip(),
+        s3_kms_key_id=os.getenv('S3_KMS_KEY_ID', '').strip(),
     )
     if config.environment == 'production' and len(config.api_key) < 32:
         raise RuntimeError('Production requires API_KEY with at least 32 characters')
@@ -42,5 +71,18 @@ def settings():
         raise RuntimeError('Production requires an HTTPS PUBLIC_BASE_URL')
     if min(config.max_upload_bytes, config.parse_timeout, config.link_ttl, config.max_pending) <= 0:
         raise RuntimeError('Upload, parse, expiry and queue limits must be positive')
-    root.mkdir(parents=True, exist_ok=True, mode=0o700)
+    if config.s3_addressing_style not in ('auto', 'path', 'virtual'):
+        raise RuntimeError('S3_ADDRESSING_STYLE must be auto, path or virtual')
+    if min(config.s3_connect_timeout, config.s3_read_timeout, config.s3_max_attempts) <= 0:
+        raise RuntimeError('S3 timeouts and max attempts must be positive')
+    if config.s3_encryption not in ('', 'AES256', 'aws:kms'):
+        raise RuntimeError('S3_SERVER_SIDE_ENCRYPTION must be empty, AES256 or aws:kms')
+    if config.s3_kms_key_id and config.s3_encryption != 'aws:kms':
+        raise RuntimeError('S3_KMS_KEY_ID requires S3_SERVER_SIDE_ENCRYPTION=aws:kms')
+    if config.s3_endpoint:
+        url = urlsplit(config.s3_endpoint)
+        if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.query or url.fragment:
+            raise RuntimeError('S3_ENDPOINT_URL must be an HTTP(S) service endpoint without credentials, query or fragment')
+    if backend == 'local' or config.database_url.startswith('sqlite:'):
+        root.mkdir(parents=True, exist_ok=True, mode=0o700)
     return config

backend/app/storage.py

@@ -11,11 +11,22 @@ KEY = re.compile(r'^[a-f0-9]{48}/(?:input\.(?:SchDoc|PcbDoc)|(?:old|new)\.(?:Sch
 class Storage:
     def __init__(self):
         self.config = settings()
-        if self.config.s3_bucket:
+        if self.config.storage_backend == 's3':
             import boto3
-            self.s3 = boto3.client('s3', endpoint_url=self.config.s3_endpoint, region_name=self.config.s3_region)
+            from botocore.config import Config
+            self.s3 = boto3.client('s3', endpoint_url=self.config.s3_endpoint, region_name=self.config.s3_region,
+                                  config=Config(signature_version='s3v4',
+                                                s3={'addressing_style': self.config.s3_addressing_style},
+                                                connect_timeout=self.config.s3_connect_timeout,
+                                                read_timeout=self.config.s3_read_timeout,
+                                                retries={'mode': 'standard', 'total_max_attempts': self.config.s3_max_attempts}))
         else: self.s3 = None
 
+    def object_key(self, key):
+        """Prefix exists only at the storage boundary, never in API IDs or DB keys."""
+        if not KEY.fullmatch(key): raise ValueError('Invalid storage key')
+        return f'{self.config.s3_prefix}/{key}' if self.config.s3_prefix else key
+
     def path(self, key):
         if not KEY.fullmatch(key): raise ValueError('Invalid storage key')
         p = self.config.storage_dir / 'objects' / key
@@ -24,10 +35,14 @@ class Storage:
         return p
 
     def put(self, key, data, content_type='application/octet-stream'):
-        p = self.path(key)  # validate on both backends
-        if self.s3:
-            self.s3.put_object(Bucket=self.config.s3_bucket, Key=key, Body=data, ContentType=content_type)
+        remote_key = self.object_key(key)
+        if self.s3 is not None:
+            options = {}
+            if self.config.s3_encryption: options['ServerSideEncryption'] = self.config.s3_encryption
+            if self.config.s3_kms_key_id: options['SSEKMSKeyId'] = self.config.s3_kms_key_id
+            self.s3.put_object(Bucket=self.config.s3_bucket, Key=remote_key, Body=data, ContentType=content_type, **options)
         else:
+            p = self.path(key)
             p.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
             tmp = p.with_name(p.name + '.' + secrets.token_hex(8) + '.tmp')
             try:
@@ -38,16 +53,16 @@ class Storage:
             finally: tmp.unlink(missing_ok=True)
 
     def get(self, key):
-        p = self.path(key)
-        if self.s3:
-            with self.s3.get_object(Bucket=self.config.s3_bucket, Key=key)['Body'] as body:
+        remote_key = self.object_key(key)
+        if self.s3 is not None:
+            with self.s3.get_object(Bucket=self.config.s3_bucket, Key=remote_key)['Body'] as body:
                 return body.read()
-        return p.read_bytes()
+        return self.path(key).read_bytes()
 
     def delete(self, key):
-        p = self.path(key)
-        if self.s3: self.s3.delete_object(Bucket=self.config.s3_bucket, Key=key)
-        else: p.unlink(missing_ok=True)
+        remote_key = self.object_key(key)
+        if self.s3 is not None: self.s3.delete_object(Bucket=self.config.s3_bucket, Key=remote_key)
+        else: self.path(key).unlink(missing_ok=True)
 
 @lru_cache
 def storage(): return Storage()