اضافه شدن قابلیت حافظه S3 به پروژه

backend/app/config.py

@@ -3,6 +3,7 @@ import os
 from dataclasses import dataclass
 from functools import lru_cache
 from pathlib import Path
+from urllib.parse import urlsplit
 
 @dataclass(frozen=True)
 class Settings:
@@ -18,10 +19,30 @@ class Settings:
     s3_bucket: str
     s3_endpoint: str | None
     s3_region: str
+    storage_backend: str
+    s3_prefix: str
+    s3_addressing_style: str
+    s3_connect_timeout: int
+    s3_read_timeout: int
+    s3_max_attempts: int
+    s3_encryption: str
+    s3_kms_key_id: str
 
 @lru_cache
 def settings():
     root = Path(os.getenv('STORAGE_DIR', './data')).resolve()
+    bucket = os.getenv('S3_BUCKET', '').strip()
+    backend = os.getenv('STORAGE_BACKEND', 'auto').strip().lower()
+    if backend == 'auto': backend = 's3' if bucket else 'local'
+    if backend not in ('local', 's3'):
+        raise RuntimeError('STORAGE_BACKEND must be auto, local or s3')
+    if backend == 's3' and not bucket:
+        raise RuntimeError('S3_BUCKET is required when STORAGE_BACKEND=s3')
+    prefix = os.getenv('S3_PREFIX', '').strip('/')
+    if (len(prefix.encode('utf-8')) > 512 or '\\' in prefix or
+        any(ord(c) < 32 or ord(c) == 127 for c in prefix) or
+        (prefix and any(part in ('', '.', '..') for part in prefix.split('/')))):
+        raise RuntimeError('S3_PREFIX must be a relative object prefix without empty, dot or traversal segments (max 512 UTF-8 bytes)')
     config = Settings(
         environment=os.getenv('APP_ENV', 'development'),
         database_url=os.getenv('DATABASE_URL', f'sqlite:///{root}/jobs.db'),
@@ -32,9 +53,17 @@ def settings():
         parse_timeout=int(os.getenv('PARSE_TIMEOUT_SECONDS', '60')),
         link_ttl=int(os.getenv('LINK_TTL_SECONDS', '604800')),
         max_pending=int(os.getenv('MAX_PENDING_JOBS', '100')),
-        s3_bucket=os.getenv('S3_BUCKET', ''),
+        s3_bucket=bucket,
         s3_endpoint=os.getenv('S3_ENDPOINT_URL') or None,
-        s3_region=os.getenv('AWS_DEFAULT_REGION', 'us-east-1'),
+        s3_region=os.getenv('S3_REGION') or os.getenv('AWS_DEFAULT_REGION') or 'us-east-1',
+        storage_backend=backend,
+        s3_prefix=prefix,
+        s3_addressing_style=os.getenv('S3_ADDRESSING_STYLE', 'auto').strip().lower(),
+        s3_connect_timeout=int(os.getenv('S3_CONNECT_TIMEOUT', '5')),
+        s3_read_timeout=int(os.getenv('S3_READ_TIMEOUT', '60')),
+        s3_max_attempts=int(os.getenv('S3_MAX_ATTEMPTS', '3')),
+        s3_encryption=os.getenv('S3_SERVER_SIDE_ENCRYPTION', '').strip(),
+        s3_kms_key_id=os.getenv('S3_KMS_KEY_ID', '').strip(),
     )
     if config.environment == 'production' and len(config.api_key) < 32:
         raise RuntimeError('Production requires API_KEY with at least 32 characters')
@@ -42,5 +71,18 @@ def settings():
         raise RuntimeError('Production requires an HTTPS PUBLIC_BASE_URL')
     if min(config.max_upload_bytes, config.parse_timeout, config.link_ttl, config.max_pending) <= 0:
         raise RuntimeError('Upload, parse, expiry and queue limits must be positive')
-    root.mkdir(parents=True, exist_ok=True, mode=0o700)
+    if config.s3_addressing_style not in ('auto', 'path', 'virtual'):
+        raise RuntimeError('S3_ADDRESSING_STYLE must be auto, path or virtual')
+    if min(config.s3_connect_timeout, config.s3_read_timeout, config.s3_max_attempts) <= 0:
+        raise RuntimeError('S3 timeouts and max attempts must be positive')
+    if config.s3_encryption not in ('', 'AES256', 'aws:kms'):
+        raise RuntimeError('S3_SERVER_SIDE_ENCRYPTION must be empty, AES256 or aws:kms')
+    if config.s3_kms_key_id and config.s3_encryption != 'aws:kms':
+        raise RuntimeError('S3_KMS_KEY_ID requires S3_SERVER_SIDE_ENCRYPTION=aws:kms')
+    if config.s3_endpoint:
+        url = urlsplit(config.s3_endpoint)
+        if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.query or url.fragment:
+            raise RuntimeError('S3_ENDPOINT_URL must be an HTTP(S) service endpoint without credentials, query or fragment')
+    if backend == 'local' or config.database_url.startswith('sqlite:'):
+        root.mkdir(parents=True, exist_ok=True, mode=0o700)
     return config

backend/app/storage.py

@@ -11,11 +11,22 @@ KEY = re.compile(r'^[a-f0-9]{48}/(?:input\.(?:SchDoc|PcbDoc)|(?:old|new)\.(?:Sch
 class Storage:
     def __init__(self):
         self.config = settings()
-        if self.config.s3_bucket:
+        if self.config.storage_backend == 's3':
             import boto3
-            self.s3 = boto3.client('s3', endpoint_url=self.config.s3_endpoint, region_name=self.config.s3_region)
+            from botocore.config import Config
+            self.s3 = boto3.client('s3', endpoint_url=self.config.s3_endpoint, region_name=self.config.s3_region,
+                                  config=Config(signature_version='s3v4',
+                                                s3={'addressing_style': self.config.s3_addressing_style},
+                                                connect_timeout=self.config.s3_connect_timeout,
+                                                read_timeout=self.config.s3_read_timeout,
+                                                retries={'mode': 'standard', 'total_max_attempts': self.config.s3_max_attempts}))
         else: self.s3 = None
 
+    def object_key(self, key):
+        """Prefix exists only at the storage boundary, never in API IDs or DB keys."""
+        if not KEY.fullmatch(key): raise ValueError('Invalid storage key')
+        return f'{self.config.s3_prefix}/{key}' if self.config.s3_prefix else key
+
     def path(self, key):
         if not KEY.fullmatch(key): raise ValueError('Invalid storage key')
         p = self.config.storage_dir / 'objects' / key
@@ -24,10 +35,14 @@ class Storage:
         return p
 
     def put(self, key, data, content_type='application/octet-stream'):
-        p = self.path(key)  # validate on both backends
-        if self.s3:
-            self.s3.put_object(Bucket=self.config.s3_bucket, Key=key, Body=data, ContentType=content_type)
+        remote_key = self.object_key(key)
+        if self.s3 is not None:
+            options = {}
+            if self.config.s3_encryption: options['ServerSideEncryption'] = self.config.s3_encryption
+            if self.config.s3_kms_key_id: options['SSEKMSKeyId'] = self.config.s3_kms_key_id
+            self.s3.put_object(Bucket=self.config.s3_bucket, Key=remote_key, Body=data, ContentType=content_type, **options)
         else:
+            p = self.path(key)
             p.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
             tmp = p.with_name(p.name + '.' + secrets.token_hex(8) + '.tmp')
             try:
@@ -38,16 +53,16 @@ class Storage:
             finally: tmp.unlink(missing_ok=True)
 
     def get(self, key):
-        p = self.path(key)
-        if self.s3:
-            with self.s3.get_object(Bucket=self.config.s3_bucket, Key=key)['Body'] as body:
+        remote_key = self.object_key(key)
+        if self.s3 is not None:
+            with self.s3.get_object(Bucket=self.config.s3_bucket, Key=remote_key)['Body'] as body:
                 return body.read()
-        return p.read_bytes()
+        return self.path(key).read_bytes()
 
     def delete(self, key):
-        p = self.path(key)
-        if self.s3: self.s3.delete_object(Bucket=self.config.s3_bucket, Key=key)
-        else: p.unlink(missing_ok=True)
+        remote_key = self.object_key(key)
+        if self.s3 is not None: self.s3.delete_object(Bucket=self.config.s3_bucket, Key=remote_key)
+        else: self.path(key).unlink(missing_ok=True)
 
 @lru_cache
 def storage(): return Storage()

backend/tests/test_platform.py

@@ -15,6 +15,7 @@ KEY = 'test-api-key-' + 'x'*40
 
 @pytest.fixture
 def client(tmp_path, monkeypatch):
+    monkeypatch.setenv('STORAGE_BACKEND', 'local')
     monkeypatch.setenv('STORAGE_DIR', str(tmp_path))
     monkeypatch.setenv('DATABASE_URL', f'sqlite:///{tmp_path}/test.db')
     monkeypatch.setenv('API_KEY', KEY)

backend/tests/test_storage_s3.py

@@ -1,25 +1,174 @@
-"""Protocol-level S3 contract check with botocore's validation stub (no live S3)."""
+"""S3 protocol/configuration and real API/worker flows with a fake S3 transport.
+
+No request is sent to a live provider and no real credential is required.
+"""
 import io
-import pytest
+import importlib.util
+from pathlib import Path
+import time
 import boto3
+from botocore.exceptions import ClientError, EndpointConnectionError
 from botocore.response import StreamingBody
 from botocore.stub import Stubber
-from app.storage import Storage
+import pytest
+from sqlalchemy import update
+from app.config import settings
+from app.storage import Storage, storage
+from test_platform import client, KEY as API_KEY
+
+NAMES=['input.SchDoc','input.PcbDoc','old.SchDoc','new.SchDoc','old.PcbDoc','new.PcbDoc',
+       'comparison.json','pcb-comparison.json','attempt-1/data.json','attempt-1/drawing.svg',
+       'attempt-1/old.svg','attempt-2/new.svg']
 
 
-@pytest.mark.parametrize('name', ['input.SchDoc', 'input.PcbDoc', 'old.PcbDoc', 'new.PcbDoc', 'pcb-comparison.json', 'attempt-1/old.svg'])
-def test_s3_private_round_trip_contract(tmp_path,monkeypatch,name):
-    from app.config import settings
-    from app.storage import storage
-    monkeypatch.setenv('STORAGE_DIR',str(tmp_path)); monkeypatch.setenv('S3_BUCKET','test-private')
-    settings.cache_clear(); storage.cache_clear()
+@pytest.fixture
+def s3_config(tmp_path,monkeypatch):
+    # Explicit fake credentials/client prevent the SDK's metadata discovery network calls.
+    for name in ['STORAGE_BACKEND','S3_BUCKET','S3_PREFIX','S3_REGION','S3_ENDPOINT_URL','S3_ADDRESSING_STYLE',
+                 'S3_CONNECT_TIMEOUT','S3_READ_TIMEOUT','S3_MAX_ATTEMPTS','S3_SERVER_SIDE_ENCRYPTION','S3_KMS_KEY_ID']:
+        monkeypatch.delenv(name,raising=False)
+    monkeypatch.setenv('STORAGE_DIR',str(tmp_path))
+    monkeypatch.setenv('DATABASE_URL',f'sqlite:///{tmp_path}/storage.db')
+    monkeypatch.setenv('APP_ENV','development')
+    monkeypatch.setenv('STORAGE_BACKEND','s3');monkeypatch.setenv('S3_BUCKET','test-private')
     client=boto3.client('s3',region_name='us-east-1',aws_access_key_id='test',aws_secret_access_key='test')
-    monkeypatch.setattr(boto3,'client',lambda *a,**kw:client)
-    key='a'*48+'/'+name; payload=b'actual bytes'
+    captures=[]
+    def make_client(*args,**kwargs):captures.append((args,kwargs));return client
+    monkeypatch.setattr(boto3,'client',make_client)
+    settings.cache_clear();storage.cache_clear()
+    yield client,captures,tmp_path
+    settings.cache_clear();storage.cache_clear()
+
+
+@pytest.mark.parametrize('name',NAMES)
+@pytest.mark.parametrize('prefix',['','altium/production','/altium/production/'])
+def test_s3_private_round_trip_contract(s3_config,monkeypatch,name,prefix):
+    client,_,root=s3_config
+    monkeypatch.setenv('S3_PREFIX',prefix);settings.cache_clear()
+    key='a'*48+'/'+name;remote=prefix.strip('/')+'/'+key if prefix else key;payload=b'actual bytes'
     with Stubber(client) as stub:
-        stub.add_response('put_object',{}, {'Bucket':'test-private','Key':key,'Body':payload,'ContentType':'application/octet-stream'})
-        stub.add_response('get_object',{'Body':StreamingBody(io.BytesIO(payload),len(payload))},{'Bucket':'test-private','Key':key})
-        stub.add_response('delete_object',{}, {'Bucket':'test-private','Key':key})
+        stub.add_response('put_object',{},dict(Bucket='test-private',Key=remote,Body=payload,ContentType='application/octet-stream'))
+        stub.add_response('get_object',{'Body':StreamingBody(io.BytesIO(payload),len(payload))},dict(Bucket='test-private',Key=remote))
+        stub.add_response('delete_object',{},dict(Bucket='test-private',Key=remote))
         service=Storage();service.put(key,payload);assert service.get(key)==payload;service.delete(key)
         stub.assert_no_pending_responses()
-    settings.cache_clear(); storage.cache_clear()
+    assert not (root/'objects').exists()
+
+
+@pytest.mark.parametrize('mode,kms,options',[
+    ('','',{}),('AES256','',{'ServerSideEncryption':'AES256'}),
+    ('aws:kms','alias/altium',{'ServerSideEncryption':'aws:kms','SSEKMSKeyId':'alias/altium'}),
+])
+def test_encryption_config_and_sdk_transport(s3_config,monkeypatch,mode,kms,options):
+    client,captures,_=s3_config
+    for key,value in {'S3_ENDPOINT_URL':'https://objects.example.test','S3_REGION':'test-region',
+                      'S3_ADDRESSING_STYLE':'path','S3_CONNECT_TIMEOUT':'7','S3_READ_TIMEOUT':'31','S3_MAX_ATTEMPTS':'4',
+                      'S3_SERVER_SIDE_ENCRYPTION':mode,'S3_KMS_KEY_ID':kms}.items():monkeypatch.setenv(key,value)
+    settings.cache_clear();key='a'*48+'/input.PcbDoc'
+    with Stubber(client) as stub:
+        stub.add_response('put_object',{},dict(Bucket='test-private',Key=key,Body=b'pcb',ContentType='application/octet-stream',**options))
+        Storage().put(key,b'pcb');stub.assert_no_pending_responses()
+    args,kwargs=captures[0];assert args==('s3',)
+    assert kwargs['endpoint_url']=='https://objects.example.test' and kwargs['region_name']=='test-region'
+    config=kwargs['config'];assert config.signature_version=='s3v4' and config.s3['addressing_style']=='path'
+    assert config.connect_timeout==7 and config.read_timeout==31 and config.retries['total_max_attempts']==4
+    assert not {'aws_access_key_id','aws_secret_access_key'} & kwargs.keys()
+
+
+@pytest.mark.parametrize('key,value',[
+    ('STORAGE_BACKEND','unknown'),('S3_BUCKET',''),('S3_PREFIX','../escape'),('S3_PREFIX','nested/../escape'),
+    ('S3_PREFIX','a//b'),('S3_PREFIX','bad\\prefix'),('S3_PREFIX','bad\nname'),('S3_PREFIX','x'*513),
+    ('S3_ADDRESSING_STYLE','invalid'),('S3_CONNECT_TIMEOUT','0'),('S3_READ_TIMEOUT','-1'),('S3_MAX_ATTEMPTS','0'),
+    ('S3_SERVER_SIDE_ENCRYPTION','public'),('S3_KMS_KEY_ID','alias/missing-encryption-mode'),
+    ('S3_ENDPOINT_URL','file:///tmp/data'),('S3_ENDPOINT_URL','https://user:secret@example.test'),
+    ('S3_ENDPOINT_URL','https://example.test/?token=secret'),
+])
+def test_invalid_s3_configuration_fails_closed(s3_config,monkeypatch,key,value):
+    monkeypatch.setenv(key,value);settings.cache_clear()
+    with pytest.raises(RuntimeError):settings()
+
+
+def test_auto_backend_compatibility_and_no_local_directory_with_postgres(s3_config,monkeypatch):
+    _,_,root=s3_config
+    monkeypatch.delenv('STORAGE_BACKEND');settings.cache_clear()
+    assert settings().storage_backend=='s3'
+    monkeypatch.setenv('S3_BUCKET','');settings.cache_clear()
+    assert settings().storage_backend=='local'
+    monkeypatch.setenv('STORAGE_BACKEND','s3');monkeypatch.setenv('S3_BUCKET','test-private')
+    monkeypatch.setenv('DATABASE_URL','postgresql+psycopg://unused:unused@localhost/unused')
+    monkeypatch.setenv('STORAGE_DIR',str(root/'unused-local-dir'));settings.cache_clear()
+    assert settings().storage_backend=='s3' and not (root/'unused-local-dir').exists()
+
+
+def test_s3_errors_never_fall_back_to_disk(s3_config,monkeypatch):
+    client,_,root=s3_config
+    key='a'*48+'/input.SchDoc'
+    with Stubber(client) as stub:
+        stub.add_client_error('put_object',service_error_code='AccessDenied',http_status_code=403)
+        stub.add_client_error('get_object',service_error_code='AccessDenied',http_status_code=403)
+        stub.add_client_error('delete_object',service_error_code='AccessDenied',http_status_code=403)
+        service=Storage()
+        for operation in (lambda:service.put(key,b'x'),lambda:service.get(key),lambda:service.delete(key)):
+            with pytest.raises(ClientError):operation()
+    assert not (root/'objects').exists()
+    for key in ['../input.SchDoc','a'*48+'/input.SchDoc/other','foreign/'+'a'*48+'/input.PcbDoc']:
+        with pytest.raises(ValueError):service.object_key(key)
+
+
+class MemoryS3:
+    """Transport fake only: all file validation/parsing/rendering/worker logic is real."""
+    def __init__(self):self.objects={};self.content_types={}
+    def put_object(self,Bucket,Key,Body,ContentType,**kwargs):
+        self.objects[(Bucket,Key)]=bytes(Body);self.content_types[(Bucket,Key)]=ContentType
+        assert 'ACL' not in kwargs
+    def get_object(self,Bucket,Key):
+        if (Bucket,Key) not in self.objects:
+            raise ClientError({'Error':{'Code':'NoSuchKey'},'ResponseMetadata':{'HTTPStatusCode':404}},'GetObject')
+        data=self.objects[(Bucket,Key)];return {'Body':StreamingBody(io.BytesIO(data),len(data))}
+    def delete_object(self,Bucket,Key):self.objects.pop((Bucket,Key),None)
+
+
+@pytest.mark.parametrize('kind',['schematic_render','schematic_comparison','pcb_render','pcb_comparison'])
+def test_api_worker_read_and_expiry_cleanup_use_only_prefixed_s3(client,monkeypatch,ascii_schematic,kind):
+    from app.db import Job,claim,sessions
+    from app.worker import process
+    from app.cleanup import cleanup
+    from pcb_fixture import pcb
+    fake=MemoryS3();monkeypatch.setattr(boto3,'client',lambda *a,**kw:fake)
+    monkeypatch.setenv('STORAGE_BACKEND','s3');monkeypatch.setenv('S3_BUCKET','test-private');monkeypatch.setenv('S3_PREFIX','altium/test')
+    settings.cache_clear();storage.cache_clear()
+    pcb_kind=kind.startswith('pcb');comparison=kind.endswith('comparison')
+    suffix='PcbDoc' if pcb_kind else 'SchDoc';raw=pcb() if pcb_kind else ascii_schematic
+    route=('/api/v1/pcbs/comparisons' if comparison else '/api/v1/pcbs/render') if pcb_kind else ('/api/v1/comparisons' if comparison else '/api/v1/schematics/render')
+    files={f'{side}_file':(side+'.'+suffix,raw) for side in ('old','new')} if comparison else {'file':('input.'+suffix,raw)}
+    response=client.post(route,headers={'X-API-Key':API_KEY},files=files)
+    assert response.status_code==202,response.text
+    created=response.json();identifier=created['id'];headers={'X-Viewer-Token':created['viewer_token']}
+    fake.objects[('test-private','other-prefix/keep')]=b'other service data'
+    process(claim());status=client.get(created['status_url'],headers=headers).json()
+    assert status['status']=='completed',status
+    base=created['status_url'].removesuffix('/status')
+    assert client.get(base+('/svg/old' if comparison else '/svg'),headers=headers).status_code==200
+    assert client.get(created['status_url']).status_code==404
+    if comparison:assert status['total_changed']==0
+    else:assert client.get(base+'/data',headers=headers).status_code==200
+    assert all(key.startswith('altium/test/'+identifier+'/') for bucket,key in fake.objects if key!='other-prefix/keep')
+    assert not (settings().storage_dir/'objects').exists()
+    with sessions().begin() as db:db.execute(update(Job).where(Job.id==identifier).values(expires_at=time.time()-1))
+    assert cleanup()==1
+    assert fake.objects=={('test-private','other-prefix/keep'):b'other service data'}
+
+
+def test_migration_copies_verifies_skips_identical_and_refuses_overwrite(s3_config,monkeypatch,tmp_path):
+    fake=MemoryS3();monkeypatch.setattr(boto3,'client',lambda *a,**kw:fake)
+    monkeypatch.setenv('S3_PREFIX','altium/test');settings.cache_clear()
+    script=Path(__file__).resolve().parents[2]/'scripts/migrate-to-s3.py'
+    spec=importlib.util.spec_from_file_location('migration',script);module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
+    key='a'*48+'/input.PcbDoc';source=tmp_path/'source';file=source/'objects'/key
+    file.parent.mkdir(parents=True);file.write_bytes(b'original')
+    assert module.migrate(source)=={'copied_and_verified':1,'already_identical':0}
+    assert file.read_bytes()==b'original'
+    assert module.migrate(source)=={'copied_and_verified':0,'already_identical':1}
+    fake.objects[('test-private','altium/test/'+key)]=b'different'
+    with pytest.raises(ValueError,match='refusing to overwrite'):module.migrate(source)
+    assert file.read_bytes()==b'original'