first commit
Verify / test (push) Failing after 2m54s

This commit is contained in:
2026-11-05 17:31:17 +03:30
کامیت 56c5ee23d3
63فایلهای تغییر یافته به همراه2749 افزوده شده و 0 حذف شده
+27
مشاهده پرونده
@@ -0,0 +1,27 @@
"""Delete expired uploads/artifacts after any active lease has elapsed."""
import logging
import time
from sqlalchemy import select, delete, or_
from .db import Job, sessions, engine
from .storage import storage
log=logging.getLogger(__name__)
def cleanup(batch=100):
now=time.time(); removed=0
with sessions().begin() as db:
q=select(Job).where(Job.expires_at<=now, or_(Job.status!='processing', Job.lease_until<now)).limit(batch)
if engine().dialect.name=='postgresql': q=q.with_for_update(skip_locked=True)
for job in db.scalars(q):
try:
storage().delete(job.input_key)
for attempt in (1,2):
for name in ('data.json','drawing.svg'):
storage().delete(f'{job.id}/attempt-{attempt}/{name}')
db.delete(job); removed+=1
except Exception: log.exception('Expired artifact cleanup failed for %s',job.id)
return removed
if __name__=='__main__':
from .db import initialize
initialize(); print(f'Removed {cleanup()} expired schematics')
+46
مشاهده پرونده
@@ -0,0 +1,46 @@
"""Environment configuration; production fails closed without upload authentication."""
import os
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
@dataclass(frozen=True)
class Settings:
environment: str
database_url: str
storage_dir: Path
api_key: str
public_url: str
max_upload_bytes: int
parse_timeout: int
link_ttl: int
max_pending: int
s3_bucket: str
s3_endpoint: str | None
s3_region: str
@lru_cache
def settings():
root = Path(os.getenv('STORAGE_DIR', './data')).resolve()
config = Settings(
environment=os.getenv('APP_ENV', 'development'),
database_url=os.getenv('DATABASE_URL', f'sqlite:///{root}/jobs.db'),
storage_dir=root,
api_key=os.getenv('API_KEY', ''),
public_url=os.getenv('PUBLIC_BASE_URL', 'http://127.0.0.1:8000').rstrip('/'),
max_upload_bytes=int(os.getenv('MAX_UPLOAD_BYTES', str(32 * 1024 * 1024))),
parse_timeout=int(os.getenv('PARSE_TIMEOUT_SECONDS', '60')),
link_ttl=int(os.getenv('LINK_TTL_SECONDS', '604800')),
max_pending=int(os.getenv('MAX_PENDING_JOBS', '100')),
s3_bucket=os.getenv('S3_BUCKET', ''),
s3_endpoint=os.getenv('S3_ENDPOINT_URL') or None,
s3_region=os.getenv('AWS_DEFAULT_REGION', 'us-east-1'),
)
if config.environment == 'production' and len(config.api_key) < 32:
raise RuntimeError('Production requires API_KEY with at least 32 characters')
if config.environment == 'production' and not config.public_url.startswith('https://'):
raise RuntimeError('Production requires an HTTPS PUBLIC_BASE_URL')
if min(config.max_upload_bytes, config.parse_timeout, config.link_ttl, config.max_pending) <= 0:
raise RuntimeError('Upload, parse, expiry and queue limits must be positive')
root.mkdir(parents=True, exist_ok=True, mode=0o700)
return config
+70
مشاهده پرونده
@@ -0,0 +1,70 @@
"""Durable PostgreSQL queue (SQLite for local development).
Claims use row locks/skip-locked on PostgreSQL and atomic compare-and-swap on
both engines. The database is the authoritative queue; there is no dual write
to Redis. Versioned artifact keys fence out expired worker leases.
"""
import time
from functools import lru_cache
from sqlalchemy import create_engine, String, Text, Float, Integer, select, update, func, text
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, sessionmaker
from .config import settings
class Base(DeclarativeBase): pass
class Job(Base):
__tablename__ = 'schematics'
id: Mapped[str] = mapped_column(String(48), primary_key=True)
filename: Mapped[str] = mapped_column(String(255))
status: Mapped[str] = mapped_column(String(20), index=True)
access: Mapped[str] = mapped_column(String(16))
token_hash: Mapped[str] = mapped_column(String(64))
created_at: Mapped[float] = mapped_column(Float)
updated_at: Mapped[float] = mapped_column(Float)
expires_at: Mapped[float] = mapped_column(Float, index=True)
lease_until: Mapped[float | None] = mapped_column(Float, nullable=True)
attempt: Mapped[int] = mapped_column(Integer, default=0)
input_key: Mapped[str] = mapped_column(String(200))
output_prefix: Mapped[str | None] = mapped_column(String(200), nullable=True)
error_code: Mapped[str | None] = mapped_column(String(60), nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
summary: Mapped[str | None] = mapped_column(Text, nullable=True)
@lru_cache
def engine():
url = settings().database_url
kw = {'connect_args': {'check_same_thread': False, 'timeout': 30}} if url.startswith('sqlite:') else {'pool_pre_ping': True}
return create_engine(url, **kw)
@lru_cache
def sessions(): return sessionmaker(engine(), expire_on_commit=False)
def initialize():
# Initial schema only. Subsequent schema changes must use explicit migrations.
if engine().dialect.name == 'postgresql':
with engine().begin() as conn:
conn.execute(text('SELECT pg_advisory_xact_lock(84721308)'))
Base.metadata.create_all(conn)
else:
import fcntl
with open(settings().storage_dir / '.schema.lock', 'a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
Base.metadata.create_all(engine())
def claim():
now = time.time()
with sessions().begin() as db:
# Expired jobs are never revived. A crashed worker can be retried once.
eligible = ((Job.status == 'queued') | ((Job.status == 'processing') & (Job.lease_until < now))) & (Job.expires_at > now)
q = select(Job).where(eligible).order_by(Job.created_at).limit(1)
if engine().dialect.name == 'postgresql': q = q.with_for_update(skip_locked=True)
job = db.scalar(q)
if job is None: return None
attempt = job.attempt + 1
values = dict(status='processing', attempt=attempt, updated_at=now,
lease_until=now + settings().parse_timeout + 60)
if attempt > 2:
values.update(status='failed', error_code='worker_lost', error_message='Worker lease expired repeatedly')
changed = db.execute(update(Job).where(Job.id == job.id, Job.attempt == job.attempt, eligible).values(**values))
if not changed.rowcount or attempt > 2: return None
return {'id': job.id, 'filename': job.filename, 'input_key': job.input_key, 'attempt': attempt}
+15
مشاهده پرونده
@@ -0,0 +1,15 @@
"""Document parser boundary. Future PCB support adds an adapter, not SchDoc branches."""
from typing import Protocol
from .models.schematic import Document
from .parser.schdoc_parser import parse_schdoc
from .parser.record_decoder import ParseError
class DocumentParser(Protocol):
def __call__(self, data: bytes, filename: str, identifier: str) -> Document: ...
PARSERS: dict[str, DocumentParser] = {'schematic': parse_schdoc}
def parse_document(data: bytes, filename: str, identifier: str, document_type='schematic') -> Document:
parser=PARSERS.get(document_type)
if parser is None: raise ParseError('Unsupported document type')
return parser(data,filename,identifier)
+31
مشاهده پرونده
@@ -0,0 +1,31 @@
"""Opt-in, administrator-provided licensed webfonts. Never load paths from SchDoc."""
import json
import os
from pathlib import Path
def fonts():
directory=os.getenv('FONT_DIR')
if not directory: return {}
root=Path(directory).resolve()
manifest=root/'fonts.json'
values=json.loads(manifest.read_text())
result={}
for family,variants in values.items():
if isinstance(variants,str): variants=[{'file':variants}]
for index,variant in enumerate(variants):
file=(root/variant['file']).resolve()
if not file.is_relative_to(root) or not file.is_file() or file.suffix.lower() not in ('.ttf','.otf','.woff','.woff2'):
raise ValueError('Invalid configured font path')
weight=str(variant.get('weight','normal')); style=variant.get('style','normal')
if weight not in {'normal','bold','100','200','300','400','500','600','700','800','900'} or style not in {'normal','italic','oblique'}:
raise ValueError('Invalid configured font variant')
result[f'{len(result)}{file.suffix.lower()}']={'family':family,'file':file,'weight':weight,'style':style}
return result
def stylesheet():
rules=[]
for name,font in fonts().items():
# JSON strings safely quote arbitrary family names inside CSS strings.
family=json.dumps(font['family'],ensure_ascii=True).replace('<','\\3c ')
rules.append(f"@font-face{{font-family:{family};src:url('/fonts/{name}');font-weight:{font['weight']};font-style:{font['style']};font-display:block}}")
return '\n'.join(rules)
+217
مشاهده پرونده
@@ -0,0 +1,217 @@
import hashlib
import hmac
import json
import logging
import re
import secrets
import time
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Annotated, Literal
from fastapi import FastAPI, File, Form, Header, HTTPException, Request, UploadFile
from fastapi.exceptions import RequestValidationError
from fastapi.responses import FileResponse, JSONResponse, Response
from sqlalchemy import func, select, text
from sqlalchemy.exc import SQLAlchemyError
from starlette.concurrency import run_in_threadpool
from .config import settings
from .db import Job, initialize, sessions
from .parser.record_decoder import ParseError, detect_format
from .storage import storage
from .fonts import fonts, stylesheet
from .models.api import UploadResponse, RenderStatus, ErrorEnvelope
from .models.schematic import Document
FRONTEND = Path(__file__).resolve().parents[2] / 'frontend'
ID = re.compile(r'^[a-f0-9]{48}$')
def digest(token): return hashlib.sha256(token.encode()).hexdigest()
def error(code, message, status=400):
raise HTTPException(status, {'code': code, 'message': message})
def authenticated(key):
expected = settings().api_key
return bool(expected and key and hmac.compare_digest(expected, key))
def require_api(key):
if settings().api_key and not authenticated(key):
error('unauthorized', 'Provide a valid X-API-Key header', 401)
@asynccontextmanager
async def lifespan(app):
initialize()
fonts() # Validate configured font allowlist before accepting requests.
yield
app = FastAPI(docs_url=None, redoc_url=None, responses={code: {'model': ErrorEnvelope} for code in (401,404,409,410,413,415,422,503)}, title='Native Altium Schematic Rendering API', version='1.0.0', lifespan=lifespan,
description='Upload native SchDoc, poll durable jobs, retrieve versioned geometry and SVG. Partial compatibility is explicitly reported.')
# Count the complete request body before Starlette spools multipart uploads.
# Content-Length alone is insufficient for chunked requests.
class UploadLimit:
def __init__(self, app): self.app = app
async def __call__(self, scope, receive, send):
if scope['type'] != 'http': return await self.app(scope, receive, send)
limit = settings().max_upload_bytes + 1024 * 1024
received = 0
async def bounded_receive():
nonlocal received
message = await receive()
received += len(message.get('body', b''))
if received > limit:
raise HTTPException(413, {'code': 'upload_too_large', 'message': 'Multipart body exceeds configured size limit'})
return message
await self.app(scope, bounded_receive, send)
app.add_middleware(UploadLimit)
@app.middleware('http')
async def security(request, call_next):
# Reject unauthorized POST bodies before the multipart parser reads them.
if request.method == 'POST' and request.url.path == '/api/v1/schematics/render':
if settings().api_key and not authenticated(request.headers.get('x-api-key')):
return JSONResponse({'success': False, 'error': {'code': 'unauthorized', 'message': 'Provide a valid X-API-Key header'}}, 401)
response = await call_next(request)
response.headers.update({
'X-Content-Type-Options': 'nosniff', 'Referrer-Policy': 'no-referrer',
'Content-Security-Policy': "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; img-src 'self' data:; font-src 'self'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'",
'Cache-Control': 'no-store', 'Permissions-Policy': 'camera=(), microphone=(), geolocation=()',
})
return response
@app.exception_handler(HTTPException)
async def http_error(request, exc):
detail = exc.detail if isinstance(exc.detail, dict) else {'code': 'http_error', 'message': str(exc.detail)}
return JSONResponse({'success': False, 'error': detail}, status_code=exc.status_code, headers=exc.headers)
@app.exception_handler(RequestValidationError)
async def validation_error(request, exc):
return JSONResponse({'success': False, 'error': {'code': 'invalid_request', 'message': 'Invalid or missing request fields',
'fields': ['.'.join(map(str, e['loc'])) for e in exc.errors()]}}, status_code=422)
@app.get('/healthz', include_in_schema=False)
def health():
with sessions()() as db: db.execute(text('SELECT 1'))
return {'status': 'ok'}
def authorize(identifier, key, token):
if not ID.fullmatch(identifier): error('not_found', 'Schematic not found', 404)
with sessions()() as db: job = db.get(Job, identifier)
if job is None: error('not_found', 'Schematic not found', 404)
# Private existence and expiry are not disclosed to unauthenticated callers.
if job.access == 'private' and not (authenticated(key) or (token and hmac.compare_digest(job.token_hash, digest(token)))):
error('not_found', 'Schematic not found', 404)
if job.expires_at <= time.time(): error('link_expired', 'This schematic link has expired', 410)
return job
def public_status(job):
result = {'success': job.status != 'failed', 'id': job.id, 'status': job.status,
'viewer_url': settings().public_url + '/view/' + job.id,
'expires_at': job.expires_at, 'access': job.access}
if job.summary: result.update(json.loads(job.summary))
if job.error_code: result['error'] = {'code': job.error_code, 'message': job.error_message}
return result
@app.post('/api/v1/schematics/render', status_code=202, response_model=UploadResponse, response_model_exclude_none=True,
responses={202: {'description': 'Queued; poll status_url'}, 401: {'description': 'API key required'},
413: {'description': 'Upload too large'}, 422: {'description': 'Invalid file or fields'},
503: {'description': 'Queue or storage unavailable'}})
async def upload(file: Annotated[UploadFile, File(description='Native binary OLE or ASCII .SchDoc')],
access: Annotated[Literal['private', 'unlisted'], Form()] = 'private',
expires_in: Annotated[int | None, Form(gt=0, description='Link lifetime in seconds, up to configured maximum')] = None,
x_api_key: Annotated[str | None, Header()] = None):
require_api(x_api_key)
name = (file.filename or '').replace('\\', '/').rsplit('/', 1)[-1]
name = ''.join(c for c in name if c.isprintable())[:255]
if not name.lower().endswith('.schdoc'): error('unsupported_file_type', 'Upload a .SchDoc file', 415)
ttl = expires_in or settings().link_ttl
if ttl > settings().link_ttl: error('invalid_expiry', 'expires_in exceeds the configured link lifetime', 422)
body = bytearray()
try:
while chunk := await file.read(1024 * 1024):
body.extend(chunk)
if len(body) > settings().max_upload_bytes: error('upload_too_large', 'Schematic exceeds configured upload size', 413)
finally: await file.close()
try: detect_format(bytes(body))
except ParseError as exc: error('invalid_schematic', str(exc), 422)
identifier = secrets.token_hex(24)
token = secrets.token_urlsafe(32)
now = time.time()
job = Job(id=identifier, filename=name, status='queued', access=access, token_hash=digest(token),
created_at=now, updated_at=now, expires_at=now+ttl, attempt=0, input_key=f'{identifier}/input.SchDoc')
def persist():
with sessions().begin() as db:
# Serialize admission checks across API replicas on PostgreSQL.
if db.bind.dialect.name == 'postgresql': db.execute(text('SELECT pg_advisory_xact_lock(84721309)'))
pending = db.scalar(select(func.count()).select_from(Job).where(Job.status.in_(['queued','processing']), Job.expires_at > now))
if pending >= settings().max_pending: error('queue_full', 'Rendering queue is full; try again later', 503)
try: storage().put(job.input_key, bytes(body))
except Exception: error('storage_unavailable', 'Upload storage unavailable', 503)
try:
db.add(job)
db.flush()
except Exception:
storage().delete(job.input_key)
raise
await run_in_threadpool(persist)
result = public_status(job)
if access == 'private':
# Fragments are never sent to servers/referrers; viewer exchanges this only in a header.
result['viewer_url'] += '#token=' + token
result['viewer_token'] = token
result['status_url'] = settings().public_url + f'/api/v1/schematics/{identifier}/status'
return JSONResponse(result, status_code=202, headers={'Location': result['status_url'], 'Retry-After': '1'})
@app.get('/api/v1/schematics/{identifier}/status', response_model=RenderStatus, response_model_exclude_none=True)
def status(identifier: str, x_api_key: Annotated[str | None, Header()] = None,
x_viewer_token: Annotated[str | None, Header()] = None):
return public_status(authorize(identifier, x_api_key, x_viewer_token))
def artifact(identifier, kind, key, token):
job = authorize(identifier, key, token)
if job.status == 'failed': error(job.error_code, job.error_message, 422)
if job.status != 'completed': error('not_ready', 'Rendering is still in progress', 409)
try: data = storage().get(job.output_prefix + '/' + kind)
except Exception: error('storage_unavailable', 'Rendering artifact unavailable', 503)
return Response(data, media_type='application/json' if kind.endswith('.json') else 'image/svg+xml')
@app.get('/api/v1/schematics/{identifier}/data', response_model=Document)
def data(identifier: str, x_api_key: Annotated[str | None, Header()] = None,
x_viewer_token: Annotated[str | None, Header()] = None):
return artifact(identifier, 'data.json', x_api_key, x_viewer_token)
@app.get('/api/v1/schematics/{identifier}/svg', response_class=Response, responses={200:{'content':{'image/svg+xml':{'schema':{'type':'string'}}}}})
def svg(identifier: str, x_api_key: Annotated[str | None, Header()] = None,
x_viewer_token: Annotated[str | None, Header()] = None):
return artifact(identifier, 'drawing.svg', x_api_key, x_viewer_token)
@app.get('/view/{identifier}', include_in_schema=False)
def viewer(identifier: str):
if not ID.fullmatch(identifier): error('not_found', 'Schematic not found', 404)
return FileResponse(FRONTEND / 'viewer.html')
@app.get('/', include_in_schema=False)
def index(): return FileResponse(FRONTEND / 'index.html')
@app.get('/assets/{name}', include_in_schema=False)
def asset(name: str):
if name not in {'viewer.js','viewer.css','upload.js','upload.css','camera.js'}: error('not_found', 'Asset not found', 404)
return FileResponse(FRONTEND / name)
@app.get('/docs', include_in_schema=False)
def api_docs(): return FileResponse(FRONTEND / 'docs.html')
@app.get('/fonts.css', include_in_schema=False)
def font_css(): return Response(stylesheet(),media_type='text/css')
@app.get('/fonts/{name}', include_in_schema=False)
def font_file(name: str):
font=fonts().get(name)
if font is None: error('not_found','Font not found',404)
return FileResponse(font['file'])
@app.exception_handler(SQLAlchemyError)
async def database_error(request, exc):
logging.getLogger(__name__).error('Database request failed', exc_info=exc)
return JSONResponse({'success':False,'error':{'code':'database_unavailable','message':'Database service unavailable'}},status_code=503,headers={'Cache-Control':'no-store'})
+30
مشاهده پرونده
@@ -0,0 +1,30 @@
from typing import Literal
from pydantic import BaseModel, Field
from .schematic import Diagnostic
class ApiError(BaseModel):
code: str
message: str
class ErrorEnvelope(BaseModel):
success: Literal[False] = False
error: ApiError
class RenderStatus(BaseModel):
success: bool
id: str = Field(description='Unpredictable 192-bit identifier, encoded as 48 hex characters')
status: Literal['queued','processing','completed','failed']
viewer_url: str = Field(description='Base viewer URL; only initial upload includes private token fragment')
expires_at: float = Field(description='Unix timestamp, UTC seconds')
access: Literal['private','unlisted']
schema_version: str | None = None
compatibility: Literal['partial','supported_subset'] | None = None
record_count: int | None = None
component_count: int | None = None
primitive_count: int | None = None
diagnostics: list[Diagnostic] | None = None
error: ApiError | None = None
class UploadResponse(RenderStatus):
status_url: str
viewer_token: str | None = Field(default=None,description='Read capability returned once for a private upload')
@@ -0,0 +1,156 @@
"""Versioned, JSON-native contract. Coordinates are 10 mil, X right, Y down."""
from typing import Literal
from pydantic import BaseModel, Field
class Point(BaseModel):
x: float
y: float
class Bounds(BaseModel):
x: float
y: float
width: float
height: float
class Diagnostic(BaseModel):
severity: Literal["info", "warning", "error"]
code: str
message: str
record_id: str | None = None
record_type: int | None = None
count: int = 1
class Primitive(BaseModel):
id: str
kind: Literal["path", "text", "image"]
bounds: Bounds
clip: Bounds | None = None
path: str = ""
text: str = ""
image_data: str = ""
keep_aspect: bool = True
position: Point = Field(default_factory=lambda: Point(x=0, y=0))
rotation: float = 0
mirrored: bool = False
stroke: str = "#000000"
fill: str = "none"
width: float = 0.4
dash: list[float] = Field(default_factory=list)
font_family: str = "Arial"
font_size: float = 10
bold: bool = False
italic: bool = False
underline: bool = False
anchor: Literal["start", "middle", "end"] = "start"
baseline: Literal["alphabetic", "middle", "hanging"] = "alphabetic"
layer_id: str | None = None
net_id: str | None = None
role: str = ""
fill_rule: Literal["nonzero", "evenodd"] = "nonzero"
object_id: str
component_id: str | None = None
class SchematicObject(BaseModel):
id: str
kind: str
record_type: int
source_stream: str
source_index: int
owner_id: str | None = None
component_id: str | None = None
visible: bool = True
layer_id: str | None = None
net_id: str | None = None
position: Point
bounds: Bounds
properties: dict[str, str]
primitive_ids: list[str] = Field(default_factory=list)
connection_points: list[Point] = Field(default_factory=list)
class Pin(BaseModel):
id: str
name: str
number: str
electrical_type: str
position: Point
connection: Point
length: float
rotation: float
visible: bool
active: bool
net_id: str | None = None
class Component(BaseModel):
id: str
symbol_id: str
designator: str = ""
name: str = ""
value: str = ""
comment: str = ""
description: str = ""
library_reference: str = ""
source_library: str = ""
position: Point
rotation: float = 0
mirrored: bool = False
current_part: int = 1
bounds: Bounds
pins: list[Pin] = Field(default_factory=list)
parameters: dict[str, str] = Field(default_factory=dict)
primitive_ids: list[str] = Field(default_factory=list)
object_ids: list[str] = Field(default_factory=list)
class Symbol(BaseModel):
id: str
component_id: str
primitive_ids: list[str]
class Sheet(BaseModel):
id: str = "sheet-0"
name: str
background: str = "#fffcf8"
bounds: Bounds
properties: dict[str, str]
class BoardLayer(BaseModel):
id: str
name: str
color: str
category: str
visible: bool
order: int
object_count: int = 0
class Net(BaseModel):
id: str
name: str
properties: dict[str, str] = Field(default_factory=dict)
class Document(BaseModel):
schema_version: str = "1.0"
document_type: Literal["schematic", "pcb"] = "schematic"
layers: list[BoardLayer] = Field(default_factory=list)
nets: list[Net] = Field(default_factory=list)
id: str
filename: str
metadata: dict
sheet: Sheet
bounds: Bounds
components: list[Component]
symbols: list[Symbol]
objects: list[SchematicObject]
primitives: list[Primitive]
diagnostics: list[Diagnostic]
+34
مشاهده پرونده
@@ -0,0 +1,34 @@
"""Parser subprocess: no network/storage credentials are inherited by the worker."""
import json
import resource
import sys
from pathlib import Path
# Bound CPU, virtual memory, output size and file descriptors before parsing.
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
resource.setrlimit(resource.RLIMIT_CPU, (60, 60))
resource.setrlimit(resource.RLIMIT_FSIZE, (128 * 1024**2, 128 * 1024**2))
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
from .documents import parse_document
from .parser.record_decoder import ParseError
from .render import svg_document
def run():
source, destination, filename, identifier = sys.argv[1:]
dest = Path(destination)
try:
doc = parse_document(Path(source).read_bytes(), filename, identifier)
(dest / 'data.json').write_text(doc.model_dump_json(), encoding='utf-8')
(dest / 'drawing.svg').write_bytes(svg_document(doc))
result = {'ok': True, 'summary': {'schema_version': doc.schema_version,
'compatibility': doc.metadata['compatibility'],
'record_count': doc.metadata['record_count'], 'primitive_count': len(doc.primitives),
'component_count': len(doc.components),
'diagnostics': [d.model_dump() for d in doc.diagnostics]}}
except (ParseError, ValueError) as exc:
result = {'ok': False, 'code': 'invalid_schematic', 'message': str(exc)[:500]}
except Exception:
result = {'ok': False, 'code': 'parser_error', 'message': 'The parser could not process this schematic'}
(dest / 'result.json').write_text(json.dumps(result), encoding='utf-8')
if __name__ == '__main__': run()
@@ -0,0 +1,48 @@
import math
from app.models.schematic import Point, Bounds
MAX_COORDINATE = 10_000_000
def number(p, key, default=0):
v = float(p.get(key, default))
if not math.isfinite(v) or abs(v) > MAX_COORDINATE:
raise ValueError(f'Invalid or excessive numeric property {key}')
return v
def integer(p, key, default=0):
return int(number(p, key, default))
def flag(p, key):
return p.get(key, '').upper() in ('T', 'TRUE', '1')
def coordinate(p, key):
# Stored base is 10 mil; fractional field is 1/10000 mil.
return number(p, key) + number(p, key + '_FRAC') / 100000
def point(p, prefix='LOCATION'):
return Point(x=coordinate(p, prefix+'.X'), y=-coordinate(p, prefix+'.Y'))
def transform_point(p: Point, origin: Point, rotation: float, mirrored=False):
"""For local generated markers only. SchDoc child geometry is already placed."""
x = -p.x if mirrored else p.x
a = math.radians(-rotation)
return Point(x=origin.x+x*math.cos(a)-p.y*math.sin(a),
y=origin.y+x*math.sin(a)+p.y*math.cos(a))
def bounds(points, padding=0):
if not points:
return Bounds(x=0,y=0,width=0,height=0)
xs,ys = [p.x for p in points],[p.y for p in points]
return Bounds(x=min(xs)-padding,y=min(ys)-padding,
width=max(xs)-min(xs)+2*padding,height=max(ys)-min(ys)+2*padding)
def union(boxes):
return bounds([p for b in boxes for p in
(Point(x=b.x,y=b.y),Point(x=b.x+b.width,y=b.y+b.height))])
def color(value):
n = int(value) & 0xffffff
return f'#{n&255:02x}{n>>8&255:02x}{n>>16&255:02x}'
def poly_path(points, closed=False):
return ' '.join(('M' if i==0 else 'L')+f'{p.x:.15g},{p.y:.15g}' for i,p in enumerate(points)) + (' Z' if closed else '')
@@ -0,0 +1,58 @@
"""Decode only bounded, embedded raster assets. Never follow document filenames."""
import base64
import io
import struct
import warnings
import zlib
import olefile
from PIL import Image
from .record_decoder import binary_records, ParseError
MAX_ASSET = 16 * 1024 * 1024
MAX_PIXELS = 16_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
def asset_name(name): return name.replace('\\', '/').rsplit('/', 1)[-1].casefold()
def read_images(data):
if not data.startswith(olefile.MAGIC): return {}, []
assets, errors = {}, []
try:
with olefile.OleFileIO(io.BytesIO(data)) as ole:
if not ole.exists('Storage'): return assets, errors
raw = ole.openstream('Storage').read()
total = 0
for record in binary_records(raw, 'Storage'):
if not record.binary: continue
payload = base64.b64decode(record.properties['BINARY_BASE64'])
if len(payload) < 6 or payload[0] != 0xd0:
errors.append('Unsupported embedded storage framing'); continue
n = payload[1]
if len(payload) < n+6: raise ParseError('Truncated embedded storage asset')
name = asset_name(payload[2:2+n].decode('cp1252', errors='replace'))
count = struct.unpack_from('<I', payload, 2+n)[0]
if count != len(payload)-6-n: raise ParseError('Invalid embedded compressed size')
decoder = zlib.decompressobj()
raster = decoder.decompress(payload[6+n:], MAX_ASSET+1)
if len(raster) > MAX_ASSET or not decoder.eof or decoder.unused_data:
raise ParseError('Embedded raster exceeds decoding limit or has invalid compression')
total += len(raster)
if total > 32*1024*1024: raise ParseError('Total embedded raster size limit exceeded')
with warnings.catch_warnings():
warnings.simplefilter('error', Image.DecompressionBombWarning)
with Image.open(io.BytesIO(raster)) as source:
if source.width*source.height > MAX_PIXELS: raise ParseError('Embedded raster pixel limit exceeded')
if source.format not in {'PNG','BMP','JPEG','TIFF','GIF'}:
errors.append('Unsupported embedded raster format'); continue
if getattr(source, 'n_frames', 1) != 1:
errors.append('Multi-frame raster skipped'); continue
image = source.convert('RGBA')
output = io.BytesIO(); image.save(output, format='PNG')
png = output.getvalue()
if len(png) > MAX_ASSET: raise ParseError('Converted raster exceeds asset limit')
if name in assets:
errors.append('Ambiguous embedded filename skipped'); assets[name] = None
else: assets[name] = 'data:image/png;base64,' + base64.b64encode(png).decode('ascii')
except Exception as exc:
errors.append('Embedded storage not completely decoded: ' + str(exc)[:160])
return assets, errors
@@ -0,0 +1,28 @@
import io
import struct
import olefile
from .record_decoder import ParseError, binary_records, ascii_records, detect_format
MAX_STREAM_BYTES = 32 * 1024 * 1024
MAX_TOTAL_BYTES = 64 * 1024 * 1024
def read_document(data):
fmt=detect_format(data)
if fmt=='ascii':
return fmt,[],ascii_records(data),[]
try:
with olefile.OleFileIO(io.BytesIO(data),raise_defects=olefile.DEFECT_INCORRECT) as ole:
entries=ole.listdir(streams=True,storages=False)
if len(entries)>256: raise ParseError('Too many OLE streams')
streams=[{'name':'/'.join(p),'size':ole.get_size(p)} for p in entries]
if any(s['size']>MAX_STREAM_BYTES for s in streams) or sum(s['size'] for s in streams)>MAX_TOTAL_BYTES:
raise ParseError('OLE stream size limit exceeded')
if not ole.exists('FileHeader') or ole.get_type('FileHeader')!=2:
raise ParseError('OLE container has no FileHeader stream')
main=binary_records(ole.openstream('FileHeader').read(),'FileHeader')
extra=[]
if ole.exists('Additional') and ole.get_type('Additional')==2:
extra=binary_records(ole.openstream('Additional').read(),'Additional')
return fmt,streams,main,extra
except (OSError,IOError,ValueError,IndexError,struct.error) as exc:
raise ParseError(f'Invalid OLE schematic: {exc}') from exc
@@ -0,0 +1,200 @@
"""Decode actual symbol geometry and document-defined electrical marker styles."""
import math
import textwrap
from app.models.schematic import Primitive, Point
from .geometry import number, integer, coordinate, point, flag, bounds, color, poly_path, transform_point
KINDS={1:'component',2:'pin',3:'ieee_symbol',4:'label',5:'bezier',6:'polyline',7:'polygon',8:'ellipse',9:'pie',10:'round_rectangle',11:'elliptical_arc',12:'arc',13:'line',14:'rectangle',15:'sheet_symbol',16:'sheet_entry',17:'power_port',18:'port',22:'no_erc',25:'net_label',26:'bus',27:'wire',28:'text_frame',29:'junction',30:'image',31:'sheet',32:'sheet_name',33:'sheet_filename',34:'designator',37:'bus_entry',39:'template',41:'parameter',43:'parameter_set',44:'implementation_list',45:'implementation',46:'pin_map',47:'pin_map',48:'implementation_parameter',209:'text_frame',225:'dashed_rectangle'}
SUPPORTED=set(KINDS)-{3,30,43}
def decode(record, component_id, sheet, warn, records):
p=record.properties
t=integer(p,'RECORD')
pos=point(p)
output=[]
stroke=color(p.get('COLOR','8388608' if t in (25,26,27) else '128'))
fill=color(p.get('AREACOLOR','16777215')) if flag(p,'ISSOLID') and not flag(p,'TRANSPARENT') else 'none'
width=(0.4,1,2,4)[min(3,max(0,integer(p,'LINEWIDTH')))]
dash=[4,2] if integer(p,'LINESTYLE') or t==225 else []
if integer(p,'LINESTYLE'): warn('dash_metrics','Non-solid line dash spacing requires Altium reference validation',record)
def path(points=None,d=None,closed=False,fill_=None,stroke_=None,width_=None,box=None):
output.append(Primitive(id=f'{record.id}:p{len(output)}',kind='path',
object_id=record.id,component_id=component_id,path=d if d is not None else poly_path(points or [],closed),
bounds=box or bounds(points or [],width+1),stroke=stroke_ or stroke,fill=fill if fill_ is None else fill_,
width=width_ or width,dash=dash))
def text(value,at=pos,angle=None,font=None,anchor=None,baseline=None,frame=None,wrap_width=None):
if not value: return
fid=font or integer(p,'FONTID',integer(sheet,'SYSTEMFONT',1))
# Browser em size approximation: Altium font size is not a CSS point size.
# This yields glyph cap heights close to the KiCad importer's size/2 stroke text.
size=max(1,min(300,number(sheet,f'SIZE{fid}',10)))*0.9
just=integer(p,'JUSTIFICATION')
a=anchor or ('start','middle','end')[just%3]
base=baseline or ('alphabetic','middle','hanging')[min(2,max(0,just//3))]
rotation=-90*integer(p,'ORIENTATION') if angle is None else angle
raw=value.replace('~1','\n')
# Altium backslash suffix marks an overbar on the preceding character.
clean=raw.replace('\\','')
if raw.startswith('='):
key=raw[1:].upper()
resolved=p.get('_RESOLVED_TEXT')
if resolved is not None: clean=resolved
else: warn('unresolved_text',f'Unresolved parameter reference {raw}',record)
lines=clean.splitlines() or ['']
if wrap_width is not None:
columns=max(1,int(wrap_width/(size*0.55)))
lines=[line for paragraph in lines for line in (textwrap.wrap(paragraph,width=columns,break_long_words=True,break_on_hyphens=False,replace_whitespace=False) or [''])]
for line_i,line in enumerate(lines):
loc=Point(x=at.x,y=at.y+line_i*size*1.2)
w=max(size*.6,len(line)*size*.8)
x={'start':0,'middle':-w/2,'end':-w}[a]
y={'alphabetic':-size,'middle':-size/2,'hanging':0}[base]
corners=[transform_point(Point(x=cx,y=cy),loc,-rotation,False) for cx,cy in [(x,y),(x+w,y),(x+w,y+size*1.3),(x,y+size*1.3)]]
box=bounds(corners,2)
if frame:
x0,y0=max(box.x,frame.x),max(box.y,frame.y)
x1,y1=min(box.x+box.width,frame.x+frame.width),min(box.y+box.height,frame.y+frame.height)
if x1<=x0 or y1<=y0: continue
from app.models.schematic import Bounds
box=Bounds(x=x0,y=y0,width=x1-x0,height=y1-y0)
output.append(Primitive(id=f'{record.id}:p{len(output)}',kind='text',object_id=record.id,
component_id=component_id,text=line,position=loc,rotation=rotation,mirrored=False,
bounds=box,clip=frame,stroke=color(p.get('TEXTCOLOR',p.get('COLOR','128'))),fill='none',
font_size=size,font_family=sheet.get(f'FONTNAME{fid}','Arial'),bold=flag(sheet,f'BOLD{fid}'),
italic=flag(sheet,f'ITALIC{fid}'),underline=flag(sheet,f'UNDERLINE{fid}'),anchor=a,baseline=base))
if '\\' in raw:
warn('text_overbar','Overbar text is displayed without its per-character overline',record)
def local(points,origin=pos,rotation=None,closed=False,fill_=None):
pts=[transform_point(Point(x=x,y=y),origin,90*integer(p,'ORIENTATION') if rotation is None else rotation) for x,y in points]
path(pts,closed=closed,fill_=fill_)
def ellipse(cx,cy,rx,ry,solid='none'):
if rx<=0 or ry<=0: return
d=f'M {cx-rx:.15g},{cy:.15g} A {rx:.15g},{ry:.15g} 0 1 0 {cx+rx:.15g},{cy:.15g} A {rx:.15g},{ry:.15g} 0 1 0 {cx-rx:.15g},{cy:.15g} Z'
path(d=d,fill_=solid,box=bounds([Point(x=cx-rx,y=cy-ry),Point(x=cx+rx,y=cy+ry)],width))
if t in (5,6,7,26,27):
n=integer(p,'LOCATIONCOUNT')
extra=integer(p,'EXTRALOCATIONCOUNT')
if n<0 or extra<0 or n+extra>20000: raise ValueError('Invalid point count')
pts=[Point(x=coordinate(p,f'X{i}'),y=-coordinate(p,f'Y{i}')) for i in range(1,n+1)]
pts += [Point(x=coordinate(p,f'EX{i}'),y=-coordinate(p,f'EY{i}')) for i in range(n+1,n+extra+1)]
if t==5:
if len(pts)<4 or (len(pts)-1)%3: warn('bezier_points','Non-cubic Bezier point sequence is unsupported',record)
else:
d=f'M{pts[0].x:.15g},{pts[0].y:.15g} '
for i in range(1,len(pts),3): d+='C'+' '.join(f'{q.x:.15g},{q.y:.15g}' for q in pts[i:i+3])+' '
path(pts,d=d,fill_='none')
elif len(pts)>1: path(pts,closed=t==7,fill_=fill if t==7 else 'none')
if integer(p,'STARTLINESHAPE') or integer(p,'ENDLINESHAPE'):
warn('line_decoration','Line end decorations are not rendered',record)
elif t in (13,37): path([pos,point(p,'CORNER')],fill_='none')
elif t in (10,14,15,28,209,225):
corner=point(p,'CORNER')
if t==15: corner=Point(x=pos.x+coordinate(p,'XSIZE'),y=pos.y+coordinate(p,'YSIZE'))
x,y=min(pos.x,corner.x),min(pos.y,corner.y); w,h=abs(pos.x-corner.x),abs(pos.y-corner.y)
pts=[Point(x=x,y=y),Point(x=x+w,y=y),Point(x=x+w,y=y+h),Point(x=x,y=y+h)]
if t==10:
rx=min(w/2,abs(coordinate(p,'CORNERXRADIUS'))); ry=min(h/2,abs(coordinate(p,'CORNERYRADIUS')))
if rx and ry:
d=f'M{x+rx},{y} H{x+w-rx} A{rx},{ry} 0 0 1 {x+w},{y+ry} V{y+h-ry} A{rx},{ry} 0 0 1 {x+w-rx},{y+h} H{x+rx} A{rx},{ry} 0 0 1 {x},{y+h-ry} V{y+ry} A{rx},{ry} 0 0 1 {x+rx},{y} Z'
path(pts,d=d)
else: path(pts,closed=True)
elif t not in (28,209) or flag(p,'SHOWBORDER') or fill!='none': path(pts,closed=True,fill_=fill,stroke_=stroke if t not in (28,209) or flag(p,'SHOWBORDER') else fill)
if t in (28,209):
frame=bounds(pts) if flag(p,'CLIPTORECT') else None
text(p.get('TEXT',''),Point(x=x+2,y=y+2),angle=0,baseline='hanging',frame=frame,wrap_width=max(1,w-4) if flag(p,'WORDWRAP') else None)
if flag(p,'WORDWRAP'): warn('text_frame_metrics','Text frame wrapping uses estimated font advances; clipping is preserved',record)
if flag(p,'COLLAPSED'): warn('collapsed_note','Collapsed note is rendered as a clipped text frame',record)
elif t in (8,9,11,12):
rx=abs(coordinate(p,'RADIUS')); ry=abs(coordinate(p,'SECONDARYRADIUS')) or rx
start=number(p,'STARTANGLE'); end=number(p,'ENDANGLE',360)
if t==8 or abs(end-start)>=359.999 or start==end:
ellipse(pos.x,pos.y,rx,ry,fill if t in (8,9) else 'none')
elif rx and ry:
def pt(deg):
a=math.radians(deg); a=math.atan2(rx*math.sin(a),ry*math.cos(a))
return Point(x=pos.x+rx*math.cos(a),y=pos.y-ry*math.sin(a))
a,b=pt(start),pt(end); sweep=(end-start)%360
d=f'M{a.x:.15g},{a.y:.15g} A{rx:.15g},{ry:.15g} 0 {1 if sweep>180 else 0} 0 {b.x:.15g},{b.y:.15g}'
if t==9: d+=f' L{pos.x},{pos.y} Z'
path(d=d,fill_=fill if t==9 else 'none',box=bounds([Point(x=pos.x-rx,y=pos.y-ry),Point(x=pos.x+rx,y=pos.y+ry)],width))
elif t==2:
bits=integer(p,'PINCONGLOMERATE'); rot=(bits&3)*90
length=coordinate(p,'PINLENGTH')
end=transform_point(Point(x=length,y=0),pos,rot)
path([pos,end],fill_='none')
# Name extends into body, number lies over the pin. Keep labels upright.
horizontal=(bits&3)%2==0
direction=1 if (bits&3) in (0,3) else -1
if bits&8:
at=Point(x=pos.x-(4*direction if horizontal else 0),y=pos.y+(0 if horizontal else -4*direction))
text(p.get('NAME',''),at,angle=0 if horizontal else -90,font=integer(p,'NAME_CUSTOMFONTID',1),anchor='end' if direction>0 else 'start',baseline='middle')
if bits&16:
at=Point(x=(pos.x+end.x)/2-(0 if horizontal else 3),y=(pos.y+end.y)/2-(3 if horizontal else 0))
text(p.get('DESIGNATOR',''),at,angle=0 if horizontal else -90,font=integer(p,'DESIGNATOR_CUSTOMFONTID',1),anchor='middle')
if integer(p,'SYMBOL_OUTEREDGE')==1: ellipse(pos.x,pos.y,2,2)
elif integer(p,'SYMBOL_OUTEREDGE'): warn('pin_decoration','Unsupported outer-edge pin symbol',record)
if integer(p,'SYMBOL_INNEREDGE')==3:
local([(0,-3),(-4,0),(0,3)],rotation=rot,fill_='none')
elif integer(p,'SYMBOL_INNEREDGE'): warn('pin_decoration','Unsupported inner-edge pin symbol',record)
if integer(p,'SYMBOL_OUTER') or integer(p,'SYMBOL_INNER'): warn('pin_decoration','Additional pin decorations are not rendered',record)
if any(k in p for k in ('NAME_CUSTOMPOSITION_MARGIN','DESIGNATOR_CUSTOMPOSITION_MARGIN')) or integer(p,'PINNAME_POSITIONCONGLOMERATE') or integer(p,'PINDESIGNATOR_POSITIONCONGLOMERATE'):
warn('pin_text_position','Custom pin text positions use standard inferred offsets',record)
elif t in (4,25,32,33,34,41):
value=p.get('TEXT','')
if t==34 and component_id:
parent=records.get((record.stream,int(component_id.rsplit(':',1)[-1])))
if parent and integer(parent.properties,'PARTCOUNT')>2:
unit=integer(parent.properties,'CURRENTPARTID',1)
if 1<=unit<=26: value+=chr(64+unit)
else: warn('unit_suffix','Unit suffix above Z is unsupported',record)
if t==41 and flag(p,'SHOWNAME'): value=p.get('NAME','')+': '+value
# Parameter labels use upright text with opposite anchors at orientations 2/3.
if t in (34,41) and integer(p,'ORIENTATION') in (2,3):
text(value,angle=0 if integer(p,'ORIENTATION')==2 else -90,anchor='end',baseline='hanging')
elif t==4 and integer(p,'ORIENTATION')==2:
just=max(0,min(8,integer(p,'JUSTIFICATION')))
text(value,angle=0,anchor=('end','middle','start')[just%3],baseline=('hanging','middle','alphabetic')[just//3])
else: text(value)
elif t==29: ellipse(pos.x,pos.y,1.5,1.5,stroke)
elif t==22:
path([Point(x=pos.x-2,y=pos.y-2),Point(x=pos.x+2,y=pos.y+2)],fill_='none')
path([Point(x=pos.x-2,y=pos.y+2),Point(x=pos.x+2,y=pos.y-2)],fill_='none')
elif t==17:
style=integer(p,'STYLE'); rot=90*integer(p,'ORIENTATION')
if flag(p,'ISCROSSSHEETCONNECTOR'):
local([(8,-3),(4,0),(8,3)],fill_='none');local([(4,-3),(0,0),(4,3)],fill_='none')
elif style in (0,1,2,4):
local([(0,0),(10,0)],fill_='none')
if style==0:
c=transform_point(Point(x=10,y=0),pos,rot);ellipse(c.x,c.y,2.5,2.5)
elif style==1: local([(6,-3),(10,0),(6,3)],fill_='none')
elif style==2: local([(10,-7),(10,7)],fill_='none')
elif style==4:
for dx,span in [(10,7),(13,4),(16,1)]: local([(dx,-span),(dx,span)],fill_='none')
else: warn('power_style',f'Unsupported power port style {style}; marker omitted rather than substituted',record)
if flag(p,'SHOWNETNAME'):
offset = 20 if style==4 else 14 if flag(p,'ISCROSSSHEETCONNECTOR') else 12
at=transform_point(Point(x=offset,y=0),pos,rot)
orient=integer(p,'ORIENTATION')
text(p.get('TEXT',''),at,angle=0,anchor=('start','middle','end','middle')[orient%4],baseline=('middle','alphabetic','middle','hanging')[orient%4])
elif t==18:
style=integer(p,'STYLE'); w=coordinate(p,'WIDTH'); h=coordinate(p,'HEIGHT') or 10
# Style 0..3 horizontal and 4..7 vertical; bit0 left/top, bit1 right/bottom.
tip=min(h/2,w/3); left=bool(style&1); right=bool(style&2)
pts=[(0,0),(tip if left else 0,-h/2),(w-tip if right else w,-h/2),(w,0),(w-tip if right else w,h/2),(tip if left else 0,h/2)]
local(pts,rotation=90 if style>=4 else 0,closed=True,fill_=color(p.get('AREACOLOR','8454143')))
at=transform_point(Point(x=w/2,y=0),pos,90 if style>=4 else 0)
text(p.get('NAME',''),at,angle=-90 if style>=4 else 0,anchor='middle',baseline='middle')
if p.get('HARNESSTYPE'): warn('harness','Harness semantics are not resolved',record)
elif t==16:
parent=records.get((record.stream,integer(p,'OWNERINDEX',-1)))
if parent and integer(parent.properties,'RECORD')==15:
q=parent.properties; origin=point(q);side=integer(p,'SIDE');distance=number(p,'DISTANCEFROMTOP')*10+number(p,'DISTANCEFROMTOP_FRAC1')/100000
at=Point(x=origin.x+(coordinate(q,'XSIZE') if side==1 else distance if side>=2 else 0),y=origin.y+(coordinate(q,'YSIZE') if side==3 else distance if side<2 else 0))
text(p.get('NAME',''),at,anchor='end' if side==1 else 'start',baseline='middle')
path([Point(x=at.x-2,y=at.y),Point(x=at.x+2,y=at.y)],fill_='none')
warn('sheet_entry_shape','Sheet entry location and name are rendered; entry arrow style is unsupported',record)
else: warn('orphan_sheet_entry','Sheet entry owner not found',record)
return output
@@ -0,0 +1,78 @@
"""Bounds-checked property record reader; keeps physical indexes including binary records."""
import re
import base64
import struct
from dataclasses import dataclass
MAX_RECORDS = 100_000
MAX_RECORD_BYTES = 4 * 1024 * 1024
class ParseError(ValueError):
pass
@dataclass
class Record:
index: int
stream: str
properties: dict[str,str]
binary: bool = False
@property
def id(self):
return f'{self.stream}:{self.index}'
def properties(payload: bytes):
result = {}
# UTF8 companion fields take precedence irrespective of source order.
utf8 = {}
for item in payload.rstrip(b'\x00\r\n').split(b'|'):
if b'=' not in item:
continue
k,v = item.split(b'=',1)
key = k.decode('ascii',errors='replace').upper()
if key.startswith('%UTF8%'):
utf8[key[6:]] = v.decode('utf-8',errors='replace')
else:
result[key] = v.decode('cp1252',errors='replace')
result.update(utf8)
return result
def binary_records(data: bytes, stream: str):
offset=0
records=[]
while offset < len(data):
if len(records)>MAX_RECORDS:
raise ParseError('Record count limit exceeded')
if len(data)-offset<4:
if not any(data[offset:]): break
raise ParseError(f'{stream}: truncated length at byte {offset}')
word=struct.unpack_from('<I',data,offset)[0]
length, kind=word&0xffffff, word>>24
if not 0 < length <= MAX_RECORD_BYTES or length>len(data)-offset-4:
raise ParseError(f'{stream}: invalid record length {length} at byte {offset}')
payload=data[offset+4:offset+4+length]
# Header is outside OWNERINDEX numbering; first object is index zero.
records.append(Record(len(records)-1,stream,properties(payload) if kind==0 else {'BINARY_KIND':str(kind),'BINARY_BASE64':base64.b64encode(payload).decode('ascii')},kind!=0))
offset+=4+length
return records
def ascii_records(data: bytes):
lines=data.lstrip(b'\xef\xbb\xbf').splitlines()
records=[]
for line in lines:
if not line.strip(): continue
if len(line)>MAX_RECORD_BYTES: raise ParseError('Text record size limit exceeded')
if len(records)>MAX_RECORDS: raise ParseError('Record count limit exceeded')
p=properties(line)
if not p or not ('HEADER' in p or 'RECORD' in p):
raise ParseError('Malformed ASCII schematic record')
records.append(Record(len(records)-1,'FileHeader',p))
return records
def detect_format(data):
if data.startswith(b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1'): return 'ole'
if re.match(br'\|HEADER=',data.lstrip(b'\xef\xbb\xbf \r\n'),re.I): return 'ascii'
raise ParseError('Not a supported SchDoc: expected OLE container or |HEADER= ASCII schematic')
@@ -0,0 +1,183 @@
import hashlib
import re
from collections import Counter
from app.models.schematic import (Document, Sheet, Component, Symbol, Pin, SchematicObject, Diagnostic, Point)
from .ole_reader import read_document
from .images import read_images, asset_name
from app.models.schematic import Primitive
from .record_decoder import ParseError
from .geometry import point, flag, integer, coordinate, number, bounds, union, transform_point, color
from .primitives import decode, KINDS, SUPPORTED
ELECTRICAL=['Input','Bidirectional','Output','Open collector','Passive','Tri-state','Open emitter','Power']
def parse_schdoc(data: bytes, filename: str, document_id: str) -> Document:
fmt,streams,main,extra=read_document(data)
if not main or 'HEADER' not in main[0].properties:
raise ParseError('Missing schematic version header')
header=main[0].properties
version=header['HEADER']
if not re.fullmatch(r'Protel for Windows - Schematic Capture (?:Binary|Ascii|ASCII) File Version 5\.0',version):
raise ParseError(f'Unsupported schematic version: {version[:150]}')
records=[r for r in main+extra if r.index>=0]
if len(records)>100000: raise ParseError('Combined record count limit exceeded')
lookup={(r.stream,r.index):r for r in records}
diagnostics=[]
grouped={}
def warn(code,message,r=None,severity='warning'):
key=(code,r.properties.get('RECORD') if r else None,message if code not in ('unsupported_record','binary_record') else '')
if key in grouped:
grouped[key].count+=1
else:
try: record_type=integer(r.properties,'RECORD') if r and 'RECORD' in r.properties else None
except ValueError: record_type=None
d=Diagnostic(severity=severity,code=code,message=message,
record_id=r.id if r else None,record_type=record_type)
grouped[key]=d;diagnostics.append(d)
images, image_errors = read_images(data)
for message in image_errors: warn('embedded_storage', message)
sheet_records=[r for r in records if r.properties.get('RECORD')=='31']
if not sheet_records: raise ParseError('No schematic sheet record found')
sheet_props=sheet_records[0].properties
if len(sheet_records)>1: warn('multiple_sheets','Multiple sheet records found; only the first defines page properties')
for s in streams:
if s['name'] not in ('FileHeader','Additional','Storage'):
warn('unhandled_stream',f"Uninterpreted OLE stream: {s['name']} ({s['size']} bytes)")
if extra: warn('additional_indexing','Additional records use stream-local owner indexes; cross-stream ownership is not inferred',severity='info')
components={}
for r in records:
if r.properties.get('RECORD')=='1':
p=r.properties
try:
components[r.id]=Component(id=r.id,symbol_id=f'{r.id}:symbol',name=p.get('DESIGNITEMID',p.get('LIBREFERENCE','')),
description=p.get('COMPONENTDESCRIPTION',''),library_reference=p.get('LIBREFERENCE',''),source_library=p.get('SOURCELIBRARYNAME',''),
position=point(p),rotation=90*integer(p,'ORIENTATION'),mirrored=flag(p,'ISMIRRORED'),current_part=integer(p,'CURRENTPARTID',1),bounds=bounds([point(p)],5))
except ValueError as exc: raise ParseError(f'Invalid component {r.id}: {exc}') from exc
def owner_chain(r):
chain=[];seen={r.id};current=r
while 'OWNERINDEX' in current.properties:
index=integer(current.properties,'OWNERINDEX',-1)
parent=lookup.get((current.stream,index))
if not parent: break
if parent.id in seen: raise ValueError('Cyclic ownership')
seen.add(parent.id);chain.append(parent);current=parent
if len(chain)>64: raise ValueError('Excessive ownership depth')
return chain
# Collect component/sheet parameter values before resolving text expressions.
parameter_maps={}
for r in records:
if r.properties.get('RECORD') in ('34','41'):
p=r.properties
try: chain=owner_chain(r)
except ValueError as exc: warn('ownership',str(exc),r);continue
c=next((components[q.id] for q in chain if q.id in components),None)
key=p.get('NAME','Designator' if p.get('RECORD')=='34' else '')
value=p.get('TEXT','')
owner=c.id if c else 'sheet-0'
parameter_maps.setdefault(owner,{})[key.upper()]=value
if c:
c.parameters[key]=value
if p.get('RECORD')=='34': c.designator=value
if key.upper()=='COMMENT': c.comment=value
if key.upper()=='VALUE': c.value=value
for c in components.values():
c.value=c.value or c.comment
objects=[];primitives=[]
counts=Counter()
for r in records:
p=r.properties
try:
t=integer(p,'RECORD',-1);counts[str(t)]+=1
chain=owner_chain(r)
c=next((components[q.id] for q in [r]+chain if q.id in components),None)
owner=lookup.get((r.stream,integer(p,'OWNERINDEX',-1)))
active=True
if c and r.id!=c.id:
cp=lookup[(r.stream,int(c.id.split(':')[-1]))].properties
for q in [r]+chain:
if q.id==c.id: break
part=integer(q.properties,'OWNERPARTID',-1)
mode=integer(q.properties,'OWNERPARTDISPLAYMODE',0)
if part not in (-1,0,c.current_part) or mode!=integer(cp,'DISPLAYMODE',0): active=False
visible=active and not flag(p,'ISHIDDEN')
if t==41 and any(q.properties.get('RECORD')=='48' for q in chain): visible=False
if t==2 and integer(p,'PINCONGLOMERATE')&4: visible=False
if p.get('TEXT','').startswith('='):
key=p['TEXT'][1:].upper()
value=parameter_maps.get(c.id if c else 'sheet-0',{}).get(key,parameter_maps.get('sheet-0',{}).get(key))
if value is not None: p['_RESOLVED_TEXT']=value
obj=SchematicObject(id=r.id,kind=KINDS.get(t,'unsupported'),record_type=t,source_stream=r.stream,source_index=r.index,
owner_id=owner.id if owner else None,component_id=c.id if c else None,visible=visible,position=point(p),bounds=bounds([point(p)]),properties=p)
if r.binary:
warn('binary_record','Binary schematic record retained by index but payload is unsupported',r)
elif t == 30 and visible:
image = images.get(asset_name(p.get('FILENAME', '')))
if image:
box = bounds([point(p), point(p, 'CORNER')])
shape = Primitive(id=r.id+':image', kind='image', bounds=box, object_id=r.id, component_id=c.id if c else None, image_data=image, keep_aspect=flag(p, 'KEEPASPECT'))
primitives.append(shape); obj.primitive_ids=[shape.id]; obj.bounds=box
else: warn('missing_image', 'Embedded raster unavailable; external files are never fetched', r)
elif t not in SUPPORTED:
warn('unsupported_record',f'Unsupported schematic record type {t}; properties preserved',r)
elif visible:
shapes=decode(r,c.id if c else None,sheet_props,warn,lookup)
primitives.extend(shapes);obj.primitive_ids=[s.id for s in shapes]
if shapes: obj.bounds=union([s.bounds for s in shapes])
if t==2:
bits=integer(p,'PINCONGLOMERATE');length=coordinate(p,'PINLENGTH')
end=transform_point(Point(x=length,y=0),obj.position,(bits&3)*90)
obj.connection_points=[end]
pin=Pin(id=r.id,name=p.get('NAME',''),number=p.get('DESIGNATOR',''),electrical_type=ELECTRICAL[max(0,min(7,integer(p,'ELECTRICAL')))],
position=obj.position,connection=end,length=length,rotation=(bits&3)*90,visible=visible,active=active)
if c: c.pins.append(pin)
else: warn('orphan_pin','Pin has no component owner',r)
elif t in (26,27):
n=integer(p,'LOCATIONCOUNT')
if not 0<=n<=20000: raise ValueError('Invalid electrical point count')
obj.connection_points=[Point(x=coordinate(p,f'X{i}'),y=-coordinate(p,f'Y{i}')) for i in range(1,n+1)]
elif t in (17,18,25,29,37): obj.connection_points=[obj.position]
if c:
c.object_ids.append(obj.id);c.primitive_ids.extend(obj.primitive_ids)
objects.append(obj)
except (ValueError,OverflowError,KeyError) as exc:
warn('malformed_record',f'Record not rendered: {str(exc)[:180]}',r,severity='error')
objects.append(SchematicObject(id=r.id,kind='malformed',record_type=-1,source_stream=r.stream,source_index=r.index,
visible=False,position=Point(x=0,y=0),bounds=bounds([]),properties=p))
if len(primitives)>250000: raise ParseError('Drawable count limit exceeded')
if not primitives: raise ParseError('No supported visible geometry found in this schematic')
by_component={}
for shape in primitives:
if shape.component_id: by_component.setdefault(shape.component_id,[]).append(shape.bounds)
for c in components.values():
if c.id in by_component: c.bounds=union(by_component[c.id])
else: warn('empty_symbol',f'Component {c.designator or c.id} has no visible supported symbol geometry')
# Standard sheet presets in 10-mil units. Custom page dimensions win only when enabled.
sizes={0:(1150,760),1:(1550,1110),2:(2230,1570),3:(3150,2230),4:(4460,3150),5:(950,750),6:(1500,950),7:(2000,1500),8:(3200,2000),9:(4200,3200),10:(1100,850),11:(1400,850),12:(1700,1100),13:(990,790),14:(1540,990),15:(2060,1560),16:(3260,2060),17:(4280,3280)}
style=integer(sheet_props,'SHEETSTYLE')
if style not in sizes and not flag(sheet_props,'USECUSTOMSHEET'): raise ParseError('Unsupported sheet dimensions')
w,h=sizes.get(style,(1150,760))
if flag(sheet_props,'USECUSTOMSHEET'):
w,h=coordinate(sheet_props,'CUSTOMX'),coordinate(sheet_props,'CUSTOMY')
if integer(sheet_props,'WORKSPACEORIENTATION') and w>h: w,h=h,w
if w <= 0 or h <= 0: raise ParseError('Sheet dimensions must be positive')
page=bounds([Point(x=0,y=-h),Point(x=w,y=0)])
content=union([s.bounds for s in primitives])
if flag(sheet_props,'BORDERON') or flag(sheet_props,'TITLEBLOCKON'):
warn('page_decoration','Automatic Altium page border, zones and title block are not reconstructed. Explicit template geometry is rendered.')
try:
if 'WEIGHT' in header and int(header['WEIGHT'])!=len(main)-1:
warn('record_count_mismatch',f"Header declares {header['WEIGHT']} records; found {len(main)-1} in FileHeader")
except ValueError: warn('record_count_mismatch','Header record count is not numeric')
warn('connectivity_scope','Connection anchors and wire vertices are extracted. Net names are not a compiled netlist; bus expansion, harnesses and cross-sheet connectivity are not resolved.',severity='info')
warn('font_metrics','Exact original fonts are required on the client. Font size conversion and glyph metrics have not been validated against Altium.',severity='warning')
warn('pin_text_metrics','Default pin text offsets and port text anchors require validation against a trusted Altium export.',severity='warning')
return Document(id=document_id,filename=filename,
metadata={'format':fmt,'header':version,'minor_version':header.get('MINORVERSION'),
'sha256':hashlib.sha256(data).hexdigest(),'input_bytes':len(data),'streams':streams,
'record_counts':dict(counts),'record_count':len(records),'component_count':len(components),
'primitive_count':len(primitives),'units':'10mil','coordinates':'X right, Y down; Y inverted once from Altium',
'compatibility':'partial' if any(d.severity in ('warning','error') for d in diagnostics) else 'supported_subset',
'header_properties':header},
sheet=Sheet(name=filename,background=color(sheet_props.get('AREACOLOR','16317695')),properties=sheet_props,bounds=page),bounds=content,
components=list(components.values()),symbols=[Symbol(id=c.symbol_id,component_id=c.id,primitive_ids=c.primitive_ids) for c in components.values()],
objects=objects,primitives=primitives,diagnostics=diagnostics)
+43
مشاهده پرونده
@@ -0,0 +1,43 @@
"""Safe SVG serializer: only application-owned SVG tags and attributes are emitted."""
import re
from xml.etree.ElementTree import Element, SubElement, tostring
from .models.schematic import Document
# XML 1.0 cannot represent these controls. They never become markup or CSS.
CONTROL = re.compile('[\x00-\x08\x0b\x0c\x0e-\x1f\ud800-\udfff]')
def clean(value): return CONTROL.sub('', str(value))
def node(parent, tag, **attrs):
return SubElement(parent, tag, {k.replace('_', '-'): clean(v) for k, v in attrs.items()})
def svg_document(doc: Document) -> bytes:
b = doc.sheet.bounds
root = Element('svg', {'xmlns': 'http://www.w3.org/2000/svg', 'viewBox': f'{b.x} {b.y} {b.width} {b.height}',
'preserveAspectRatio': 'xMidYMid meet', 'role': 'img', 'aria-label': 'Schematic'})
node(root, 'rect', x=b.x, y=b.y, width=b.width, height=b.height, fill=doc.sheet.background)
defs = node(root, 'defs')
for i, p in enumerate(doc.primitives):
target = root
if p.clip:
clip = node(defs, 'clipPath', id=f'clip-{i}', clipPathUnits='userSpaceOnUse')
c = p.clip
node(clip, 'rect', x=c.x, y=c.y, width=c.width, height=c.height)
target = node(root, 'g', clip_path=f'url(#clip-{i})')
if p.kind == 'path':
n = node(target, 'path', d=p.path, stroke=p.stroke, fill=p.fill,
stroke_width=p.width, stroke_linecap='round', stroke_linejoin='round', fill_rule=p.fill_rule)
if p.dash: n.set('stroke-dasharray', ' '.join(map(str, p.dash)))
elif p.kind == 'image':
# Populated only by the trusted bounded raster decoder; never a source URL.
if p.image_data.startswith('data:image/png;base64,'):
node(target, 'image', x=p.bounds.x, y=p.bounds.y, width=p.bounds.width, height=p.bounds.height,
href=p.image_data, preserveAspectRatio='xMidYMid meet' if p.keep_aspect else 'none')
else:
transform = f'translate({p.position.x} {p.position.y}) rotate({p.rotation})'
if p.mirrored: transform += ' scale(-1 1)'
n = node(target, 'text', transform=transform, fill=p.stroke, font_family=p.font_family,
font_size=p.font_size, font_weight='bold' if p.bold else 'normal',
font_style='italic' if p.italic else 'normal', text_anchor=p.anchor,
dominant_baseline=p.baseline, text_decoration='underline' if p.underline else 'none')
n.set('{http://www.w3.org/XML/1998/namespace}space', 'preserve')
n.text = clean(p.text)
return tostring(root, encoding='utf-8', xml_declaration=True)
+53
مشاهده پرونده
@@ -0,0 +1,53 @@
"""Private local/S3 storage. All object keys are application-generated."""
import os
import re
import secrets
from functools import lru_cache
from pathlib import Path
from .config import settings
KEY = re.compile(r'^[a-f0-9]{48}/(?:input\.SchDoc|attempt-[12]/(?:data\.json|drawing\.svg))$')
class Storage:
def __init__(self):
self.config = settings()
if self.config.s3_bucket:
import boto3
self.s3 = boto3.client('s3', endpoint_url=self.config.s3_endpoint, region_name=self.config.s3_region)
else: self.s3 = None
def path(self, key):
if not KEY.fullmatch(key): raise ValueError('Invalid storage key')
p = self.config.storage_dir / 'objects' / key
if not p.resolve().is_relative_to(self.config.storage_dir / 'objects'):
raise ValueError('Invalid storage path')
return p
def put(self, key, data, content_type='application/octet-stream'):
p = self.path(key) # validate on both backends
if self.s3:
self.s3.put_object(Bucket=self.config.s3_bucket, Key=key, Body=data, ContentType=content_type)
else:
p.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
tmp = p.with_name(p.name + '.' + secrets.token_hex(8) + '.tmp')
try:
with open(tmp, 'xb') as f:
os.chmod(tmp, 0o600)
f.write(data)
os.replace(tmp, p)
finally: tmp.unlink(missing_ok=True)
def get(self, key):
p = self.path(key)
if self.s3:
with self.s3.get_object(Bucket=self.config.s3_bucket, Key=key)['Body'] as body:
return body.read()
return p.read_bytes()
def delete(self, key):
p = self.path(key)
if self.s3: self.s3.delete_object(Bucket=self.config.s3_bucket, Key=key)
else: p.unlink(missing_ok=True)
@lru_cache
def storage(): return Storage()
+68
مشاهده پرونده
@@ -0,0 +1,68 @@
"""Run `python -m app.worker`; add replicas to scale parsing independently of API."""
import json
import logging
import os
from pathlib import Path
import signal
import subprocess
import sys
import tempfile
import time
from sqlalchemy import update
from .config import settings
from .cleanup import cleanup
from .db import Job, claim, initialize, sessions
from .storage import storage
log = logging.getLogger(__name__)
stopping = False
def process(job):
config = settings()
prefix = f"{job['id']}/attempt-{job['attempt']}"
try:
with tempfile.TemporaryDirectory(prefix='schdoc-') as directory:
root = Path(directory)
source = root / 'input.SchDoc'
source.write_bytes(storage().get(job['input_key']))
env = {k: v for k, v in os.environ.items() if k in ('PATH', 'LANG', 'LC_ALL', 'PYTHONPATH')}
# Drain no untrusted output into logs. Kill/wait is handled by subprocess.run.
subprocess.run([sys.executable, '-m', 'app.parse_task', str(source), str(root), job['filename'], job['id']],
timeout=config.parse_timeout, check=True, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
result = json.loads((root / 'result.json').read_text())
if result['ok']:
storage().put(prefix + '/data.json', (root / 'data.json').read_bytes(), 'application/json')
storage().put(prefix + '/drawing.svg', (root / 'drawing.svg').read_bytes(), 'image/svg+xml')
values = dict(status='completed', output_prefix=prefix, summary=json.dumps(result['summary']))
else:
values = dict(status='failed', error_code=result['code'], error_message=result['message'])
except subprocess.TimeoutExpired:
values = dict(status='failed', error_code='parse_timeout', error_message='Schematic exceeded the parsing time limit')
except subprocess.CalledProcessError:
values = dict(status='failed', error_code='parse_resource_limit', error_message='Parser exited before completion (resource limit or invalid input)')
except Exception:
log.exception('Worker infrastructure failure for %s', job['id'])
values = dict(status='failed', error_code='processing_error', error_message='Storage or processing service unavailable')
with sessions().begin() as db:
# An expired worker may finish late. Its outputs must never replace a newer attempt.
db.execute(update(Job).where(Job.id == job['id'], Job.attempt == job['attempt'], Job.status == 'processing')
.values(**values, lease_until=None, updated_at=time.time()))
def main():
global stopping
logging.basicConfig(level=logging.INFO)
def stop(*_):
global stopping
stopping = True
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
initialize()
last_cleanup=0
while not stopping:
if time.monotonic()-last_cleanup>60:
cleanup(); last_cleanup=time.monotonic()
job = claim()
if job: process(job)
else: time.sleep(0.5)
if __name__ == '__main__': main()