@@ -0,0 +1,11 @@
|
||||
.git
|
||||
.venv
|
||||
.env
|
||||
data
|
||||
.runtime
|
||||
**/__pycache__
|
||||
.pytest_cache
|
||||
validation-output
|
||||
**/*.SchDoc
|
||||
**/*.PcbDoc
|
||||
node_modules
|
||||
@@ -0,0 +1,16 @@
|
||||
# Generate separate values with: openssl rand -hex 32
|
||||
API_KEY=replace-with-a-long-random-api-key-at-least-32-characters
|
||||
POSTGRES_PASSWORD=replace-with-a-random-hex-database-password
|
||||
APP_ENV=development
|
||||
PUBLIC_BASE_URL=http://127.0.0.1:8000
|
||||
MAX_UPLOAD_BYTES=33554432
|
||||
PARSE_TIMEOUT_SECONDS=60
|
||||
LINK_TTL_SECONDS=604800
|
||||
MAX_PENDING_JOBS=100
|
||||
# Production: APP_ENV=production and PUBLIC_BASE_URL=https://your-domain.example
|
||||
# Optional private S3-compatible storage, same settings for API and workers:
|
||||
# S3_BUCKET=private-schematics
|
||||
# S3_ENDPOINT_URL=https://s3.example.com
|
||||
# AWS_ACCESS_KEY_ID=
|
||||
# AWS_SECRET_ACCESS_KEY=
|
||||
# AWS_DEFAULT_REGION=us-east-1
|
||||
@@ -0,0 +1,27 @@
|
||||
name: Verify
|
||||
on: [push, pull_request]
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with: {python-version: '3.12'}
|
||||
- uses: actions/setup-node@v4
|
||||
with: {node-version: '24'}
|
||||
- run: sudo apt-get update && sudo apt-get install -y libcairo2
|
||||
- run: python -m venv .venv && .venv/bin/pip install -r backend/requirements-dev.txt
|
||||
- run: .venv/bin/python scripts/fetch-fixture.py
|
||||
- run: ./scripts/check.sh
|
||||
- run: docker build -t altium-render-engine:local .
|
||||
- run: cp .env.example .env
|
||||
- run: docker compose -p altium-ci up -d --no-build
|
||||
- run: curl --retry 30 --retry-connrefused --retry-delay 1 --fail http://127.0.0.1:8000/healthz
|
||||
- run: .venv/bin/python scripts/smoke.py .runtime/fixtures/Channel.SchDoc
|
||||
env:
|
||||
API_KEY: replace-with-a-long-random-api-key-at-least-32-characters
|
||||
- run: .venv/bin/python scripts/concurrency-smoke.py .runtime/fixtures/Channel.SchDoc
|
||||
env:
|
||||
API_KEY: replace-with-a-long-random-api-key-at-least-32-characters
|
||||
- if: always()
|
||||
run: docker compose -p altium-ci down
|
||||
@@ -0,0 +1,11 @@
|
||||
.venv/
|
||||
__pycache__/
|
||||
.pytest_cache/
|
||||
*.pyc
|
||||
.env
|
||||
data/
|
||||
.runtime/
|
||||
validation-output/
|
||||
node_modules/
|
||||
*.SchDoc
|
||||
*.PcbDoc
|
||||
@@ -0,0 +1,6 @@
|
||||
{$DOMAIN} {
|
||||
request_body {
|
||||
max_size 33MB
|
||||
}
|
||||
reverse_proxy api:8000
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
FROM python:3.12-slim
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/backend
|
||||
WORKDIR /app
|
||||
RUN groupadd --gid 10001 renderer && useradd --uid 10001 --gid renderer --create-home renderer
|
||||
COPY backend/requirements.lock /app/backend/requirements.lock
|
||||
ARG PIP_INDEX_URL=https://pypi.org/simple
|
||||
RUN pip install --no-cache-dir -r backend/requirements.lock
|
||||
COPY --chown=renderer:renderer backend/app /app/backend/app
|
||||
COPY --chown=renderer:renderer frontend /app/frontend
|
||||
RUN mkdir /data && chown renderer:renderer /data
|
||||
USER renderer
|
||||
EXPOSE 8000
|
||||
HEALTHCHECK --interval=15s --timeout=5s CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz')"
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
# Native Altium schematic rendering service
|
||||
|
||||
A working REST upload → durable worker → native SchDoc parser → versioned geometry/SVG → fullscreen viewer pipeline. The viewer consumes uploaded schematic records, including placed component symbol primitives; no sample circuit or substitute component library is used.
|
||||
|
||||
**Fidelity status: partial compatibility, not pixel-perfect or Altium-certified.** Parsing and rasterization have been exercised with real binary schematics. A trusted export from Altium is required before any claim of visual equivalence. Unsupported objects and text/font limitations are recorded in every result. See [compatibility](docs/COMPATIBILITY.md) and [validation](docs/VALIDATION.md).
|
||||
|
||||
## Run with Docker
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# Replace API_KEY and POSTGRES_PASSWORD with independent random values.
|
||||
# openssl rand -hex 32
|
||||
# A private local .env may already exist if this workspace was initialized.
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
Open `http://127.0.0.1:8000` for the optional upload form, or use the REST API. The actual viewer is `/view/{id}` and contains only the drawing after it loads. Local deployment binds the API explicitly to loopback. The API and worker run as UID 10001 with read-only root filesystems, dropped capabilities, memory limits, and private persistent storage. PostgreSQL has no published port.
|
||||
|
||||
For a production domain, set `DOMAIN`, `APP_ENV=production` and `PUBLIC_BASE_URL=https://your-domain.example` in `.env`, configure DNS to the server, and run:
|
||||
|
||||
```bash
|
||||
docker compose -f compose.yaml -f compose.production.yaml up -d --build
|
||||
```
|
||||
|
||||
The production overlay adds Caddy for HTTPS. Production startup rejects short API keys and non-HTTPS public URLs. Change the Caddy request-size limit when changing the upload limit. Use a URL-safe database password (random hex is suitable) because it is interpolated into a database URL. Back up the PostgreSQL and schematic volumes together. The initial schema is created under an advisory lock; subsequent schema changes require a reviewed migration, not automatic destructive schema synchronization.
|
||||
|
||||
To scale CPU-bound parsing:
|
||||
|
||||
```bash
|
||||
docker compose up -d --scale worker=4
|
||||
```
|
||||
|
||||
PostgreSQL row locks and compare-and-swap claims prevent duplicate active claims; leases recover crashes, and attempt-specific artifacts prevent a stale worker overwriting a newer result. PostgreSQL is also the durable queue, avoiding a database/Redis enqueue consistency gap. SQLite is available for local development only.
|
||||
|
||||
## Run locally
|
||||
|
||||
Python 3.12 and Node 24 (tests only) are used. Cairo is required only by the optional raster validation tool.
|
||||
|
||||
```bash
|
||||
python3 -m venv .venv
|
||||
.venv/bin/pip install -r backend/requirements-dev.txt
|
||||
./scripts/dev.sh
|
||||
```
|
||||
|
||||
Without a `DATABASE_URL`, the app uses SQLite in `data/`. `scripts/dev.sh` starts an API and worker, sourcing `.env` if present. Upload authentication is disabled only when no `API_KEY` is configured in development. Do not expose that mode publicly.
|
||||
|
||||
## API example
|
||||
|
||||
```bash
|
||||
curl -X POST http://127.0.0.1:8000/api/v1/schematics/render \
|
||||
-H "X-API-Key: $API_KEY" \
|
||||
-F 'file=@/path/to/design.SchDoc' \
|
||||
-F 'access=private' \
|
||||
-F 'expires_in=3600'
|
||||
```
|
||||
|
||||
Returns HTTP **202**:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"id": "a15bd6b4e28487335c8f907d8f402b70fa92e1c31c943f20",
|
||||
"status": "queued",
|
||||
"viewer_url": "http://127.0.0.1:8000/view/a15bd6b4e28487335c8f907d8f402b70fa92e1c31c943f20#token=READ_TOKEN",
|
||||
"viewer_token": "READ_TOKEN",
|
||||
"status_url": "http://127.0.0.1:8000/api/v1/schematics/a15bd6b4e28487335c8f907d8f402b70fa92e1c31c943f20/status",
|
||||
"access": "private",
|
||||
"expires_at": 1793876400
|
||||
}
|
||||
```
|
||||
|
||||
Poll `status_url` with `X-Viewer-Token: READ_TOKEN` or `X-API-Key`. States are `queued`, `processing`, `completed`, `failed`. An accepted upload can later fail structure/version validation; 202 never asserts that a file rendered. The viewer URL may be opened immediately and polls until ready. Completed status includes `compatibility`, record/primitive/component counts, and diagnostics. `completed` means the supported rendering pipeline finished, not full format coverage.
|
||||
|
||||
| Endpoint | Purpose |
|
||||
| --- | --- |
|
||||
| `POST /api/v1/schematics/render` | Multipart upload with `file`, optional `access`, `expires_in` |
|
||||
| `GET /api/v1/schematics/{id}/status` | State, expiry, diagnostics, failure details |
|
||||
| `GET /api/v1/schematics/{id}/data` | Versioned normalized document, original properties, owners, primitives |
|
||||
| `GET /api/v1/schematics/{id}/svg` | Safe vector rendering |
|
||||
| `GET /view/{id}` | Full-viewport, schematic-only browser viewer |
|
||||
| `GET /docs` | Self-contained API reference |
|
||||
| `GET /openapi.json` | Machine-readable OpenAPI schema |
|
||||
| `GET /healthz` | API/database liveness |
|
||||
|
||||
Error responses use `{"success": false, "error": {"code": "…", "message": "…"}}`. Missing/inaccessible private schematics return 404; expired authorized links return 410; unfinished artifacts 409; invalid files 422/415; excessive size 413; full queue or unavailable storage 503. See [API details](docs/API.md).
|
||||
|
||||
### Access and retention
|
||||
|
||||
`private` is the default. It creates a separate 256-bit read capability whose SHA-256 hash is stored in the database. The URL fragment is removed from the address bar after loading, retained in this tab's session storage, and sent only in `X-Viewer-Token`. Keep the original `viewer_url` if sharing/reopening in another browser. Status endpoints never return the token again. The API key grants service-wide access and must be kept server-side in external integrations.
|
||||
|
||||
`unlisted` is an explicit alternative: anyone with its 192-bit random identifier can read the result until expiry. There is no searchable gallery. Every access is checked on the server. Link lifetime is configurable and the worker removes expired database rows and artifacts in batches. After physical cleanup the endpoint returns 404. An already downloaded document cannot be revoked from a recipient's device. The reference viewer clears itself at expiry.
|
||||
|
||||
## Storage and configuration
|
||||
|
||||
Local files are private, written atomically with generated keys. Client filenames never determine storage paths. For private S3-compatible storage set the same `S3_BUCKET`, `S3_ENDPOINT_URL`, AWS credentials and region on API and workers; no bucket ACL or public URL is created. Prefer workload credentials/IAM and configure bucket encryption/lifecycle at the provider. On a multi-host deployment use S3; a Docker named volume is shared only on one host. Storage/database orphan reconciliation after a machine failure is an operational maintenance task; normal completion, retries and expiry cleanup are implemented.
|
||||
|
||||
| Environment | Default |
|
||||
| --- | --- |
|
||||
| `DATABASE_URL` | SQLite under `STORAGE_DIR`; Compose uses PostgreSQL |
|
||||
| `STORAGE_DIR` | `./data` (`/data` in Compose) |
|
||||
| `API_KEY` | Empty in local development; required in production |
|
||||
| `PUBLIC_BASE_URL` | `http://127.0.0.1:8000` |
|
||||
| `MAX_UPLOAD_BYTES` | 33,554,432 (32 MiB) |
|
||||
| `PARSE_TIMEOUT_SECONDS` | 60 wall-clock seconds |
|
||||
| `LINK_TTL_SECONDS` | 604,800 (7 days), maximum accepted `expires_in` |
|
||||
| `MAX_PENDING_JOBS` | 100 active/unexpired jobs |
|
||||
| `FONT_DIR` | Unset; optional administrator-managed webfont directory |
|
||||
|
||||
Malformed inputs are bounded by upload, OLE stream, record, point, ownership depth, raster expansion and subprocess resource limits. The parser never evaluates expressions, follows external image paths, runs Altium automation, or emits file-provided HTML/SVG/scripts. SVG markup is constructed using fixed tags and XML-escaped attributes/text; page CSP disables external resources and framing. No permissive CORS configuration is supplied.
|
||||
|
||||
The subprocess has a 768 MiB address-space limit, 60-second CPU ceiling and 128 MiB output-file ceiling, independently of the configurable wall-clock timeout. Resource limits contain malformed inputs; they are not a sandbox against an exploitable native image decoder. Use current patched dependencies and isolate the service from sensitive infrastructure. The example proxy does not add tenant accounts, per-client rate limiting or WAF rules; configure these to fit the deployment.
|
||||
|
||||
### Original fonts
|
||||
|
||||
Exact glyph shapes require the original fonts and appropriate licenses for web distribution. Set `FONT_DIR` to an administrator-controlled directory and create `fonts.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"Arial": [
|
||||
{"file": "arial.woff2", "weight": "normal", "style": "normal"},
|
||||
{"file": "arial-bold.woff2", "weight": "bold", "style": "normal"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Mount that directory read-only on the API container. Only allowlisted local font files are served. Fonts are never extracted as executable resources from uploaded files. Browser fallback fonts are used when unavailable; the API always reports that font metrics need validation. SVG exports reference family names; they do not bundle proprietary fonts. Cairo validation requires the matching fonts installed on the validation host as well.
|
||||
|
||||
## Tests and fidelity validation
|
||||
|
||||
```bash
|
||||
.venv/bin/python scripts/fetch-fixture.py
|
||||
./scripts/check.sh
|
||||
SCHDOC_SAMPLE='/path/to/a-real.SchDoc' ./scripts/check.sh
|
||||
PYTHONPATH=backend .venv/bin/python scripts/corpus.py /path/to/real-schematics
|
||||
.venv/bin/python scripts/validate.py /path/to/design.SchDoc \
|
||||
--reference /path/to/trusted-altium-page.png
|
||||
# With the API and worker running, test the real HTTP pipeline:
|
||||
API_KEY="$API_KEY" .venv/bin/python scripts/smoke.py /path/to/design.SchDoc
|
||||
```
|
||||
|
||||
The fixture downloader checksum-pins an actual upstream KiCad SchDoc. Unit-test circuits are explicitly synthetic and are never application demo data. Proprietary local designs are not redistributed or placed in the Docker image. The visual comparison tool emits SVG, PNG, heatmap, overlay and numerical differences, with nonzero exit on regression. It does not align/warp/crop a reference to conceal layout differences. Read [validation](docs/VALIDATION.md) for the release gate and current evidence.
|
||||
|
||||
## Architecture and PCB extension
|
||||
|
||||
See [architecture](docs/ARCHITECTURE.md). `app/documents.py` is the parser registry, the schema preserves source records separately from vector primitives, `app/render.py` is the SVG serializer, and workers/storage are independent of schematic record decoding. A future PCB adapter can introduce layer-aware primitives and an appropriate viewer through these boundaries; no PCB parser or PCB rendering is shipped here. `.PcbDoc` uploads are rejected.
|
||||
@@ -0,0 +1,9 @@
|
||||
# Third-party notices
|
||||
|
||||
This project does not embed Altium software, an Altium SDK, proprietary symbol libraries or proprietary fonts. It is an independent native-format subset implementation. The initial schematic parser/model modules originated in the user's local `work/altium-render` project; the source checkout was left unchanged.
|
||||
|
||||
Runtime packages are installed with upstream package metadata/license files: FastAPI (MIT), Uvicorn (BSD), python-multipart (Apache-2.0), olefile (BSD), SQLAlchemy (MIT), psycopg (LGPL with linking exceptions; see distribution terms), boto3/botocore (Apache-2.0), Pillow (HPND and bundled-component licenses), and their transitive dependencies. `backend/requirements.lock` pins runtime distributions; `requirements-dev.txt` adds test/validation tools. CairoSVG is LGPL; NumPy is BSD. Consult installed distribution licenses for precise terms.
|
||||
|
||||
Format research is linked from `docs/COMPATIBILITY.md`. No GPL KiCad importer code is incorporated. The checksum-pinned KiCad QA fixture is downloaded to an ignored directory and excluded from Docker/source distribution; consult upstream repository and fixture licensing before redistributing it. The user's local designs are not included in project source or the image.
|
||||
|
||||
Administrator-supplied fonts must have appropriate web-serving licenses. Embedded document images and symbols retain their owners' rights; uploading does not grant redistribution rights.
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Delete expired uploads/artifacts after any active lease has elapsed."""
|
||||
import logging
|
||||
import time
|
||||
from sqlalchemy import select, delete, or_
|
||||
from .db import Job, sessions, engine
|
||||
from .storage import storage
|
||||
|
||||
log=logging.getLogger(__name__)
|
||||
|
||||
def cleanup(batch=100):
|
||||
now=time.time(); removed=0
|
||||
with sessions().begin() as db:
|
||||
q=select(Job).where(Job.expires_at<=now, or_(Job.status!='processing', Job.lease_until<now)).limit(batch)
|
||||
if engine().dialect.name=='postgresql': q=q.with_for_update(skip_locked=True)
|
||||
for job in db.scalars(q):
|
||||
try:
|
||||
storage().delete(job.input_key)
|
||||
for attempt in (1,2):
|
||||
for name in ('data.json','drawing.svg'):
|
||||
storage().delete(f'{job.id}/attempt-{attempt}/{name}')
|
||||
db.delete(job); removed+=1
|
||||
except Exception: log.exception('Expired artifact cleanup failed for %s',job.id)
|
||||
return removed
|
||||
|
||||
if __name__=='__main__':
|
||||
from .db import initialize
|
||||
initialize(); print(f'Removed {cleanup()} expired schematics')
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Environment configuration; production fails closed without upload authentication."""
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
environment: str
|
||||
database_url: str
|
||||
storage_dir: Path
|
||||
api_key: str
|
||||
public_url: str
|
||||
max_upload_bytes: int
|
||||
parse_timeout: int
|
||||
link_ttl: int
|
||||
max_pending: int
|
||||
s3_bucket: str
|
||||
s3_endpoint: str | None
|
||||
s3_region: str
|
||||
|
||||
@lru_cache
|
||||
def settings():
|
||||
root = Path(os.getenv('STORAGE_DIR', './data')).resolve()
|
||||
config = Settings(
|
||||
environment=os.getenv('APP_ENV', 'development'),
|
||||
database_url=os.getenv('DATABASE_URL', f'sqlite:///{root}/jobs.db'),
|
||||
storage_dir=root,
|
||||
api_key=os.getenv('API_KEY', ''),
|
||||
public_url=os.getenv('PUBLIC_BASE_URL', 'http://127.0.0.1:8000').rstrip('/'),
|
||||
max_upload_bytes=int(os.getenv('MAX_UPLOAD_BYTES', str(32 * 1024 * 1024))),
|
||||
parse_timeout=int(os.getenv('PARSE_TIMEOUT_SECONDS', '60')),
|
||||
link_ttl=int(os.getenv('LINK_TTL_SECONDS', '604800')),
|
||||
max_pending=int(os.getenv('MAX_PENDING_JOBS', '100')),
|
||||
s3_bucket=os.getenv('S3_BUCKET', ''),
|
||||
s3_endpoint=os.getenv('S3_ENDPOINT_URL') or None,
|
||||
s3_region=os.getenv('AWS_DEFAULT_REGION', 'us-east-1'),
|
||||
)
|
||||
if config.environment == 'production' and len(config.api_key) < 32:
|
||||
raise RuntimeError('Production requires API_KEY with at least 32 characters')
|
||||
if config.environment == 'production' and not config.public_url.startswith('https://'):
|
||||
raise RuntimeError('Production requires an HTTPS PUBLIC_BASE_URL')
|
||||
if min(config.max_upload_bytes, config.parse_timeout, config.link_ttl, config.max_pending) <= 0:
|
||||
raise RuntimeError('Upload, parse, expiry and queue limits must be positive')
|
||||
root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
return config
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Durable PostgreSQL queue (SQLite for local development).
|
||||
|
||||
Claims use row locks/skip-locked on PostgreSQL and atomic compare-and-swap on
|
||||
both engines. The database is the authoritative queue; there is no dual write
|
||||
to Redis. Versioned artifact keys fence out expired worker leases.
|
||||
"""
|
||||
import time
|
||||
from functools import lru_cache
|
||||
from sqlalchemy import create_engine, String, Text, Float, Integer, select, update, func, text
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, sessionmaker
|
||||
from .config import settings
|
||||
|
||||
class Base(DeclarativeBase): pass
|
||||
|
||||
class Job(Base):
|
||||
__tablename__ = 'schematics'
|
||||
id: Mapped[str] = mapped_column(String(48), primary_key=True)
|
||||
filename: Mapped[str] = mapped_column(String(255))
|
||||
status: Mapped[str] = mapped_column(String(20), index=True)
|
||||
access: Mapped[str] = mapped_column(String(16))
|
||||
token_hash: Mapped[str] = mapped_column(String(64))
|
||||
created_at: Mapped[float] = mapped_column(Float)
|
||||
updated_at: Mapped[float] = mapped_column(Float)
|
||||
expires_at: Mapped[float] = mapped_column(Float, index=True)
|
||||
lease_until: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
attempt: Mapped[int] = mapped_column(Integer, default=0)
|
||||
input_key: Mapped[str] = mapped_column(String(200))
|
||||
output_prefix: Mapped[str | None] = mapped_column(String(200), nullable=True)
|
||||
error_code: Mapped[str | None] = mapped_column(String(60), nullable=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
summary: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
@lru_cache
|
||||
def engine():
|
||||
url = settings().database_url
|
||||
kw = {'connect_args': {'check_same_thread': False, 'timeout': 30}} if url.startswith('sqlite:') else {'pool_pre_ping': True}
|
||||
return create_engine(url, **kw)
|
||||
|
||||
@lru_cache
|
||||
def sessions(): return sessionmaker(engine(), expire_on_commit=False)
|
||||
|
||||
def initialize():
|
||||
# Initial schema only. Subsequent schema changes must use explicit migrations.
|
||||
if engine().dialect.name == 'postgresql':
|
||||
with engine().begin() as conn:
|
||||
conn.execute(text('SELECT pg_advisory_xact_lock(84721308)'))
|
||||
Base.metadata.create_all(conn)
|
||||
else:
|
||||
import fcntl
|
||||
with open(settings().storage_dir / '.schema.lock', 'a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
Base.metadata.create_all(engine())
|
||||
|
||||
def claim():
|
||||
now = time.time()
|
||||
with sessions().begin() as db:
|
||||
# Expired jobs are never revived. A crashed worker can be retried once.
|
||||
eligible = ((Job.status == 'queued') | ((Job.status == 'processing') & (Job.lease_until < now))) & (Job.expires_at > now)
|
||||
q = select(Job).where(eligible).order_by(Job.created_at).limit(1)
|
||||
if engine().dialect.name == 'postgresql': q = q.with_for_update(skip_locked=True)
|
||||
job = db.scalar(q)
|
||||
if job is None: return None
|
||||
attempt = job.attempt + 1
|
||||
values = dict(status='processing', attempt=attempt, updated_at=now,
|
||||
lease_until=now + settings().parse_timeout + 60)
|
||||
if attempt > 2:
|
||||
values.update(status='failed', error_code='worker_lost', error_message='Worker lease expired repeatedly')
|
||||
changed = db.execute(update(Job).where(Job.id == job.id, Job.attempt == job.attempt, eligible).values(**values))
|
||||
if not changed.rowcount or attempt > 2: return None
|
||||
return {'id': job.id, 'filename': job.filename, 'input_key': job.input_key, 'attempt': attempt}
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Document parser boundary. Future PCB support adds an adapter, not SchDoc branches."""
|
||||
from typing import Protocol
|
||||
from .models.schematic import Document
|
||||
from .parser.schdoc_parser import parse_schdoc
|
||||
from .parser.record_decoder import ParseError
|
||||
|
||||
class DocumentParser(Protocol):
|
||||
def __call__(self, data: bytes, filename: str, identifier: str) -> Document: ...
|
||||
|
||||
PARSERS: dict[str, DocumentParser] = {'schematic': parse_schdoc}
|
||||
|
||||
def parse_document(data: bytes, filename: str, identifier: str, document_type='schematic') -> Document:
|
||||
parser=PARSERS.get(document_type)
|
||||
if parser is None: raise ParseError('Unsupported document type')
|
||||
return parser(data,filename,identifier)
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Opt-in, administrator-provided licensed webfonts. Never load paths from SchDoc."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
def fonts():
|
||||
directory=os.getenv('FONT_DIR')
|
||||
if not directory: return {}
|
||||
root=Path(directory).resolve()
|
||||
manifest=root/'fonts.json'
|
||||
values=json.loads(manifest.read_text())
|
||||
result={}
|
||||
for family,variants in values.items():
|
||||
if isinstance(variants,str): variants=[{'file':variants}]
|
||||
for index,variant in enumerate(variants):
|
||||
file=(root/variant['file']).resolve()
|
||||
if not file.is_relative_to(root) or not file.is_file() or file.suffix.lower() not in ('.ttf','.otf','.woff','.woff2'):
|
||||
raise ValueError('Invalid configured font path')
|
||||
weight=str(variant.get('weight','normal')); style=variant.get('style','normal')
|
||||
if weight not in {'normal','bold','100','200','300','400','500','600','700','800','900'} or style not in {'normal','italic','oblique'}:
|
||||
raise ValueError('Invalid configured font variant')
|
||||
result[f'{len(result)}{file.suffix.lower()}']={'family':family,'file':file,'weight':weight,'style':style}
|
||||
return result
|
||||
|
||||
def stylesheet():
|
||||
rules=[]
|
||||
for name,font in fonts().items():
|
||||
# JSON strings safely quote arbitrary family names inside CSS strings.
|
||||
family=json.dumps(font['family'],ensure_ascii=True).replace('<','\\3c ')
|
||||
rules.append(f"@font-face{{font-family:{family};src:url('/fonts/{name}');font-weight:{font['weight']};font-style:{font['style']};font-display:block}}")
|
||||
return '\n'.join(rules)
|
||||
@@ -0,0 +1,217 @@
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import secrets
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Literal
|
||||
from fastapi import FastAPI, File, Form, Header, HTTPException, Request, UploadFile
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import FileResponse, JSONResponse, Response
|
||||
from sqlalchemy import func, select, text
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from .config import settings
|
||||
from .db import Job, initialize, sessions
|
||||
from .parser.record_decoder import ParseError, detect_format
|
||||
from .storage import storage
|
||||
from .fonts import fonts, stylesheet
|
||||
from .models.api import UploadResponse, RenderStatus, ErrorEnvelope
|
||||
from .models.schematic import Document
|
||||
|
||||
FRONTEND = Path(__file__).resolve().parents[2] / 'frontend'
|
||||
ID = re.compile(r'^[a-f0-9]{48}$')
|
||||
|
||||
def digest(token): return hashlib.sha256(token.encode()).hexdigest()
|
||||
|
||||
def error(code, message, status=400):
|
||||
raise HTTPException(status, {'code': code, 'message': message})
|
||||
|
||||
def authenticated(key):
|
||||
expected = settings().api_key
|
||||
return bool(expected and key and hmac.compare_digest(expected, key))
|
||||
|
||||
def require_api(key):
|
||||
if settings().api_key and not authenticated(key):
|
||||
error('unauthorized', 'Provide a valid X-API-Key header', 401)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
initialize()
|
||||
fonts() # Validate configured font allowlist before accepting requests.
|
||||
yield
|
||||
|
||||
app = FastAPI(docs_url=None, redoc_url=None, responses={code: {'model': ErrorEnvelope} for code in (401,404,409,410,413,415,422,503)}, title='Native Altium Schematic Rendering API', version='1.0.0', lifespan=lifespan,
|
||||
description='Upload native SchDoc, poll durable jobs, retrieve versioned geometry and SVG. Partial compatibility is explicitly reported.')
|
||||
|
||||
# Count the complete request body before Starlette spools multipart uploads.
|
||||
# Content-Length alone is insufficient for chunked requests.
|
||||
class UploadLimit:
|
||||
def __init__(self, app): self.app = app
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope['type'] != 'http': return await self.app(scope, receive, send)
|
||||
limit = settings().max_upload_bytes + 1024 * 1024
|
||||
received = 0
|
||||
async def bounded_receive():
|
||||
nonlocal received
|
||||
message = await receive()
|
||||
received += len(message.get('body', b''))
|
||||
if received > limit:
|
||||
raise HTTPException(413, {'code': 'upload_too_large', 'message': 'Multipart body exceeds configured size limit'})
|
||||
return message
|
||||
await self.app(scope, bounded_receive, send)
|
||||
|
||||
app.add_middleware(UploadLimit)
|
||||
|
||||
@app.middleware('http')
|
||||
async def security(request, call_next):
|
||||
# Reject unauthorized POST bodies before the multipart parser reads them.
|
||||
if request.method == 'POST' and request.url.path == '/api/v1/schematics/render':
|
||||
if settings().api_key and not authenticated(request.headers.get('x-api-key')):
|
||||
return JSONResponse({'success': False, 'error': {'code': 'unauthorized', 'message': 'Provide a valid X-API-Key header'}}, 401)
|
||||
response = await call_next(request)
|
||||
response.headers.update({
|
||||
'X-Content-Type-Options': 'nosniff', 'Referrer-Policy': 'no-referrer',
|
||||
'Content-Security-Policy': "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; img-src 'self' data:; font-src 'self'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'",
|
||||
'Cache-Control': 'no-store', 'Permissions-Policy': 'camera=(), microphone=(), geolocation=()',
|
||||
})
|
||||
return response
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request, exc):
|
||||
detail = exc.detail if isinstance(exc.detail, dict) else {'code': 'http_error', 'message': str(exc.detail)}
|
||||
return JSONResponse({'success': False, 'error': detail}, status_code=exc.status_code, headers=exc.headers)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request, exc):
|
||||
return JSONResponse({'success': False, 'error': {'code': 'invalid_request', 'message': 'Invalid or missing request fields',
|
||||
'fields': ['.'.join(map(str, e['loc'])) for e in exc.errors()]}}, status_code=422)
|
||||
|
||||
@app.get('/healthz', include_in_schema=False)
|
||||
def health():
|
||||
with sessions()() as db: db.execute(text('SELECT 1'))
|
||||
return {'status': 'ok'}
|
||||
|
||||
def authorize(identifier, key, token):
|
||||
if not ID.fullmatch(identifier): error('not_found', 'Schematic not found', 404)
|
||||
with sessions()() as db: job = db.get(Job, identifier)
|
||||
if job is None: error('not_found', 'Schematic not found', 404)
|
||||
# Private existence and expiry are not disclosed to unauthenticated callers.
|
||||
if job.access == 'private' and not (authenticated(key) or (token and hmac.compare_digest(job.token_hash, digest(token)))):
|
||||
error('not_found', 'Schematic not found', 404)
|
||||
if job.expires_at <= time.time(): error('link_expired', 'This schematic link has expired', 410)
|
||||
return job
|
||||
|
||||
def public_status(job):
|
||||
result = {'success': job.status != 'failed', 'id': job.id, 'status': job.status,
|
||||
'viewer_url': settings().public_url + '/view/' + job.id,
|
||||
'expires_at': job.expires_at, 'access': job.access}
|
||||
if job.summary: result.update(json.loads(job.summary))
|
||||
if job.error_code: result['error'] = {'code': job.error_code, 'message': job.error_message}
|
||||
return result
|
||||
|
||||
@app.post('/api/v1/schematics/render', status_code=202, response_model=UploadResponse, response_model_exclude_none=True,
|
||||
responses={202: {'description': 'Queued; poll status_url'}, 401: {'description': 'API key required'},
|
||||
413: {'description': 'Upload too large'}, 422: {'description': 'Invalid file or fields'},
|
||||
503: {'description': 'Queue or storage unavailable'}})
|
||||
async def upload(file: Annotated[UploadFile, File(description='Native binary OLE or ASCII .SchDoc')],
|
||||
access: Annotated[Literal['private', 'unlisted'], Form()] = 'private',
|
||||
expires_in: Annotated[int | None, Form(gt=0, description='Link lifetime in seconds, up to configured maximum')] = None,
|
||||
x_api_key: Annotated[str | None, Header()] = None):
|
||||
require_api(x_api_key)
|
||||
name = (file.filename or '').replace('\\', '/').rsplit('/', 1)[-1]
|
||||
name = ''.join(c for c in name if c.isprintable())[:255]
|
||||
if not name.lower().endswith('.schdoc'): error('unsupported_file_type', 'Upload a .SchDoc file', 415)
|
||||
ttl = expires_in or settings().link_ttl
|
||||
if ttl > settings().link_ttl: error('invalid_expiry', 'expires_in exceeds the configured link lifetime', 422)
|
||||
body = bytearray()
|
||||
try:
|
||||
while chunk := await file.read(1024 * 1024):
|
||||
body.extend(chunk)
|
||||
if len(body) > settings().max_upload_bytes: error('upload_too_large', 'Schematic exceeds configured upload size', 413)
|
||||
finally: await file.close()
|
||||
try: detect_format(bytes(body))
|
||||
except ParseError as exc: error('invalid_schematic', str(exc), 422)
|
||||
identifier = secrets.token_hex(24)
|
||||
token = secrets.token_urlsafe(32)
|
||||
now = time.time()
|
||||
job = Job(id=identifier, filename=name, status='queued', access=access, token_hash=digest(token),
|
||||
created_at=now, updated_at=now, expires_at=now+ttl, attempt=0, input_key=f'{identifier}/input.SchDoc')
|
||||
def persist():
|
||||
with sessions().begin() as db:
|
||||
# Serialize admission checks across API replicas on PostgreSQL.
|
||||
if db.bind.dialect.name == 'postgresql': db.execute(text('SELECT pg_advisory_xact_lock(84721309)'))
|
||||
pending = db.scalar(select(func.count()).select_from(Job).where(Job.status.in_(['queued','processing']), Job.expires_at > now))
|
||||
if pending >= settings().max_pending: error('queue_full', 'Rendering queue is full; try again later', 503)
|
||||
try: storage().put(job.input_key, bytes(body))
|
||||
except Exception: error('storage_unavailable', 'Upload storage unavailable', 503)
|
||||
try:
|
||||
db.add(job)
|
||||
db.flush()
|
||||
except Exception:
|
||||
storage().delete(job.input_key)
|
||||
raise
|
||||
await run_in_threadpool(persist)
|
||||
result = public_status(job)
|
||||
if access == 'private':
|
||||
# Fragments are never sent to servers/referrers; viewer exchanges this only in a header.
|
||||
result['viewer_url'] += '#token=' + token
|
||||
result['viewer_token'] = token
|
||||
result['status_url'] = settings().public_url + f'/api/v1/schematics/{identifier}/status'
|
||||
return JSONResponse(result, status_code=202, headers={'Location': result['status_url'], 'Retry-After': '1'})
|
||||
|
||||
@app.get('/api/v1/schematics/{identifier}/status', response_model=RenderStatus, response_model_exclude_none=True)
|
||||
def status(identifier: str, x_api_key: Annotated[str | None, Header()] = None,
|
||||
x_viewer_token: Annotated[str | None, Header()] = None):
|
||||
return public_status(authorize(identifier, x_api_key, x_viewer_token))
|
||||
|
||||
def artifact(identifier, kind, key, token):
|
||||
job = authorize(identifier, key, token)
|
||||
if job.status == 'failed': error(job.error_code, job.error_message, 422)
|
||||
if job.status != 'completed': error('not_ready', 'Rendering is still in progress', 409)
|
||||
try: data = storage().get(job.output_prefix + '/' + kind)
|
||||
except Exception: error('storage_unavailable', 'Rendering artifact unavailable', 503)
|
||||
return Response(data, media_type='application/json' if kind.endswith('.json') else 'image/svg+xml')
|
||||
|
||||
@app.get('/api/v1/schematics/{identifier}/data', response_model=Document)
|
||||
def data(identifier: str, x_api_key: Annotated[str | None, Header()] = None,
|
||||
x_viewer_token: Annotated[str | None, Header()] = None):
|
||||
return artifact(identifier, 'data.json', x_api_key, x_viewer_token)
|
||||
|
||||
@app.get('/api/v1/schematics/{identifier}/svg', response_class=Response, responses={200:{'content':{'image/svg+xml':{'schema':{'type':'string'}}}}})
|
||||
def svg(identifier: str, x_api_key: Annotated[str | None, Header()] = None,
|
||||
x_viewer_token: Annotated[str | None, Header()] = None):
|
||||
return artifact(identifier, 'drawing.svg', x_api_key, x_viewer_token)
|
||||
|
||||
@app.get('/view/{identifier}', include_in_schema=False)
|
||||
def viewer(identifier: str):
|
||||
if not ID.fullmatch(identifier): error('not_found', 'Schematic not found', 404)
|
||||
return FileResponse(FRONTEND / 'viewer.html')
|
||||
|
||||
@app.get('/', include_in_schema=False)
|
||||
def index(): return FileResponse(FRONTEND / 'index.html')
|
||||
|
||||
@app.get('/assets/{name}', include_in_schema=False)
|
||||
def asset(name: str):
|
||||
if name not in {'viewer.js','viewer.css','upload.js','upload.css','camera.js'}: error('not_found', 'Asset not found', 404)
|
||||
return FileResponse(FRONTEND / name)
|
||||
|
||||
@app.get('/docs', include_in_schema=False)
|
||||
def api_docs(): return FileResponse(FRONTEND / 'docs.html')
|
||||
|
||||
@app.get('/fonts.css', include_in_schema=False)
|
||||
def font_css(): return Response(stylesheet(),media_type='text/css')
|
||||
|
||||
@app.get('/fonts/{name}', include_in_schema=False)
|
||||
def font_file(name: str):
|
||||
font=fonts().get(name)
|
||||
if font is None: error('not_found','Font not found',404)
|
||||
return FileResponse(font['file'])
|
||||
|
||||
@app.exception_handler(SQLAlchemyError)
|
||||
async def database_error(request, exc):
|
||||
logging.getLogger(__name__).error('Database request failed', exc_info=exc)
|
||||
return JSONResponse({'success':False,'error':{'code':'database_unavailable','message':'Database service unavailable'}},status_code=503,headers={'Cache-Control':'no-store'})
|
||||
@@ -0,0 +1,30 @@
|
||||
from typing import Literal
|
||||
from pydantic import BaseModel, Field
|
||||
from .schematic import Diagnostic
|
||||
|
||||
class ApiError(BaseModel):
|
||||
code: str
|
||||
message: str
|
||||
|
||||
class ErrorEnvelope(BaseModel):
|
||||
success: Literal[False] = False
|
||||
error: ApiError
|
||||
|
||||
class RenderStatus(BaseModel):
|
||||
success: bool
|
||||
id: str = Field(description='Unpredictable 192-bit identifier, encoded as 48 hex characters')
|
||||
status: Literal['queued','processing','completed','failed']
|
||||
viewer_url: str = Field(description='Base viewer URL; only initial upload includes private token fragment')
|
||||
expires_at: float = Field(description='Unix timestamp, UTC seconds')
|
||||
access: Literal['private','unlisted']
|
||||
schema_version: str | None = None
|
||||
compatibility: Literal['partial','supported_subset'] | None = None
|
||||
record_count: int | None = None
|
||||
component_count: int | None = None
|
||||
primitive_count: int | None = None
|
||||
diagnostics: list[Diagnostic] | None = None
|
||||
error: ApiError | None = None
|
||||
|
||||
class UploadResponse(RenderStatus):
|
||||
status_url: str
|
||||
viewer_token: str | None = Field(default=None,description='Read capability returned once for a private upload')
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Versioned, JSON-native contract. Coordinates are 10 mil, X right, Y down."""
|
||||
|
||||
from typing import Literal
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class Point(BaseModel):
|
||||
x: float
|
||||
y: float
|
||||
|
||||
|
||||
class Bounds(BaseModel):
|
||||
x: float
|
||||
y: float
|
||||
width: float
|
||||
height: float
|
||||
|
||||
|
||||
class Diagnostic(BaseModel):
|
||||
severity: Literal["info", "warning", "error"]
|
||||
code: str
|
||||
message: str
|
||||
record_id: str | None = None
|
||||
record_type: int | None = None
|
||||
count: int = 1
|
||||
|
||||
|
||||
class Primitive(BaseModel):
|
||||
id: str
|
||||
kind: Literal["path", "text", "image"]
|
||||
bounds: Bounds
|
||||
clip: Bounds | None = None
|
||||
path: str = ""
|
||||
text: str = ""
|
||||
image_data: str = ""
|
||||
keep_aspect: bool = True
|
||||
position: Point = Field(default_factory=lambda: Point(x=0, y=0))
|
||||
rotation: float = 0
|
||||
mirrored: bool = False
|
||||
stroke: str = "#000000"
|
||||
fill: str = "none"
|
||||
width: float = 0.4
|
||||
dash: list[float] = Field(default_factory=list)
|
||||
font_family: str = "Arial"
|
||||
font_size: float = 10
|
||||
bold: bool = False
|
||||
italic: bool = False
|
||||
underline: bool = False
|
||||
anchor: Literal["start", "middle", "end"] = "start"
|
||||
baseline: Literal["alphabetic", "middle", "hanging"] = "alphabetic"
|
||||
layer_id: str | None = None
|
||||
net_id: str | None = None
|
||||
role: str = ""
|
||||
fill_rule: Literal["nonzero", "evenodd"] = "nonzero"
|
||||
object_id: str
|
||||
component_id: str | None = None
|
||||
|
||||
|
||||
class SchematicObject(BaseModel):
|
||||
id: str
|
||||
kind: str
|
||||
record_type: int
|
||||
source_stream: str
|
||||
source_index: int
|
||||
owner_id: str | None = None
|
||||
component_id: str | None = None
|
||||
visible: bool = True
|
||||
layer_id: str | None = None
|
||||
net_id: str | None = None
|
||||
position: Point
|
||||
bounds: Bounds
|
||||
properties: dict[str, str]
|
||||
primitive_ids: list[str] = Field(default_factory=list)
|
||||
connection_points: list[Point] = Field(default_factory=list)
|
||||
|
||||
|
||||
class Pin(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
number: str
|
||||
electrical_type: str
|
||||
position: Point
|
||||
connection: Point
|
||||
length: float
|
||||
rotation: float
|
||||
visible: bool
|
||||
active: bool
|
||||
net_id: str | None = None
|
||||
|
||||
|
||||
class Component(BaseModel):
|
||||
id: str
|
||||
symbol_id: str
|
||||
designator: str = ""
|
||||
name: str = ""
|
||||
value: str = ""
|
||||
comment: str = ""
|
||||
description: str = ""
|
||||
library_reference: str = ""
|
||||
source_library: str = ""
|
||||
position: Point
|
||||
rotation: float = 0
|
||||
mirrored: bool = False
|
||||
current_part: int = 1
|
||||
bounds: Bounds
|
||||
pins: list[Pin] = Field(default_factory=list)
|
||||
parameters: dict[str, str] = Field(default_factory=dict)
|
||||
primitive_ids: list[str] = Field(default_factory=list)
|
||||
object_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class Symbol(BaseModel):
|
||||
id: str
|
||||
component_id: str
|
||||
primitive_ids: list[str]
|
||||
|
||||
|
||||
class Sheet(BaseModel):
|
||||
id: str = "sheet-0"
|
||||
name: str
|
||||
background: str = "#fffcf8"
|
||||
bounds: Bounds
|
||||
properties: dict[str, str]
|
||||
|
||||
|
||||
class BoardLayer(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
color: str
|
||||
category: str
|
||||
visible: bool
|
||||
order: int
|
||||
object_count: int = 0
|
||||
|
||||
|
||||
class Net(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
properties: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class Document(BaseModel):
|
||||
schema_version: str = "1.0"
|
||||
document_type: Literal["schematic", "pcb"] = "schematic"
|
||||
layers: list[BoardLayer] = Field(default_factory=list)
|
||||
nets: list[Net] = Field(default_factory=list)
|
||||
id: str
|
||||
filename: str
|
||||
metadata: dict
|
||||
sheet: Sheet
|
||||
bounds: Bounds
|
||||
components: list[Component]
|
||||
symbols: list[Symbol]
|
||||
objects: list[SchematicObject]
|
||||
primitives: list[Primitive]
|
||||
diagnostics: list[Diagnostic]
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Parser subprocess: no network/storage credentials are inherited by the worker."""
|
||||
import json
|
||||
import resource
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Bound CPU, virtual memory, output size and file descriptors before parsing.
|
||||
resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (60, 60))
|
||||
resource.setrlimit(resource.RLIMIT_FSIZE, (128 * 1024**2, 128 * 1024**2))
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
|
||||
from .documents import parse_document
|
||||
from .parser.record_decoder import ParseError
|
||||
from .render import svg_document
|
||||
|
||||
def run():
|
||||
source, destination, filename, identifier = sys.argv[1:]
|
||||
dest = Path(destination)
|
||||
try:
|
||||
doc = parse_document(Path(source).read_bytes(), filename, identifier)
|
||||
(dest / 'data.json').write_text(doc.model_dump_json(), encoding='utf-8')
|
||||
(dest / 'drawing.svg').write_bytes(svg_document(doc))
|
||||
result = {'ok': True, 'summary': {'schema_version': doc.schema_version,
|
||||
'compatibility': doc.metadata['compatibility'],
|
||||
'record_count': doc.metadata['record_count'], 'primitive_count': len(doc.primitives),
|
||||
'component_count': len(doc.components),
|
||||
'diagnostics': [d.model_dump() for d in doc.diagnostics]}}
|
||||
except (ParseError, ValueError) as exc:
|
||||
result = {'ok': False, 'code': 'invalid_schematic', 'message': str(exc)[:500]}
|
||||
except Exception:
|
||||
result = {'ok': False, 'code': 'parser_error', 'message': 'The parser could not process this schematic'}
|
||||
(dest / 'result.json').write_text(json.dumps(result), encoding='utf-8')
|
||||
|
||||
if __name__ == '__main__': run()
|
||||
@@ -0,0 +1,48 @@
|
||||
import math
|
||||
from app.models.schematic import Point, Bounds
|
||||
|
||||
MAX_COORDINATE = 10_000_000
|
||||
|
||||
def number(p, key, default=0):
|
||||
v = float(p.get(key, default))
|
||||
if not math.isfinite(v) or abs(v) > MAX_COORDINATE:
|
||||
raise ValueError(f'Invalid or excessive numeric property {key}')
|
||||
return v
|
||||
|
||||
def integer(p, key, default=0):
|
||||
return int(number(p, key, default))
|
||||
|
||||
def flag(p, key):
|
||||
return p.get(key, '').upper() in ('T', 'TRUE', '1')
|
||||
|
||||
def coordinate(p, key):
|
||||
# Stored base is 10 mil; fractional field is 1/10000 mil.
|
||||
return number(p, key) + number(p, key + '_FRAC') / 100000
|
||||
|
||||
def point(p, prefix='LOCATION'):
|
||||
return Point(x=coordinate(p, prefix+'.X'), y=-coordinate(p, prefix+'.Y'))
|
||||
|
||||
def transform_point(p: Point, origin: Point, rotation: float, mirrored=False):
|
||||
"""For local generated markers only. SchDoc child geometry is already placed."""
|
||||
x = -p.x if mirrored else p.x
|
||||
a = math.radians(-rotation)
|
||||
return Point(x=origin.x+x*math.cos(a)-p.y*math.sin(a),
|
||||
y=origin.y+x*math.sin(a)+p.y*math.cos(a))
|
||||
|
||||
def bounds(points, padding=0):
|
||||
if not points:
|
||||
return Bounds(x=0,y=0,width=0,height=0)
|
||||
xs,ys = [p.x for p in points],[p.y for p in points]
|
||||
return Bounds(x=min(xs)-padding,y=min(ys)-padding,
|
||||
width=max(xs)-min(xs)+2*padding,height=max(ys)-min(ys)+2*padding)
|
||||
|
||||
def union(boxes):
|
||||
return bounds([p for b in boxes for p in
|
||||
(Point(x=b.x,y=b.y),Point(x=b.x+b.width,y=b.y+b.height))])
|
||||
|
||||
def color(value):
|
||||
n = int(value) & 0xffffff
|
||||
return f'#{n&255:02x}{n>>8&255:02x}{n>>16&255:02x}'
|
||||
|
||||
def poly_path(points, closed=False):
|
||||
return ' '.join(('M' if i==0 else 'L')+f'{p.x:.15g},{p.y:.15g}' for i,p in enumerate(points)) + (' Z' if closed else '')
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Decode only bounded, embedded raster assets. Never follow document filenames."""
|
||||
import base64
|
||||
import io
|
||||
import struct
|
||||
import warnings
|
||||
import zlib
|
||||
import olefile
|
||||
from PIL import Image
|
||||
from .record_decoder import binary_records, ParseError
|
||||
|
||||
MAX_ASSET = 16 * 1024 * 1024
|
||||
MAX_PIXELS = 16_000_000
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
|
||||
def asset_name(name): return name.replace('\\', '/').rsplit('/', 1)[-1].casefold()
|
||||
|
||||
def read_images(data):
|
||||
if not data.startswith(olefile.MAGIC): return {}, []
|
||||
assets, errors = {}, []
|
||||
try:
|
||||
with olefile.OleFileIO(io.BytesIO(data)) as ole:
|
||||
if not ole.exists('Storage'): return assets, errors
|
||||
raw = ole.openstream('Storage').read()
|
||||
total = 0
|
||||
for record in binary_records(raw, 'Storage'):
|
||||
if not record.binary: continue
|
||||
payload = base64.b64decode(record.properties['BINARY_BASE64'])
|
||||
if len(payload) < 6 or payload[0] != 0xd0:
|
||||
errors.append('Unsupported embedded storage framing'); continue
|
||||
n = payload[1]
|
||||
if len(payload) < n+6: raise ParseError('Truncated embedded storage asset')
|
||||
name = asset_name(payload[2:2+n].decode('cp1252', errors='replace'))
|
||||
count = struct.unpack_from('<I', payload, 2+n)[0]
|
||||
if count != len(payload)-6-n: raise ParseError('Invalid embedded compressed size')
|
||||
decoder = zlib.decompressobj()
|
||||
raster = decoder.decompress(payload[6+n:], MAX_ASSET+1)
|
||||
if len(raster) > MAX_ASSET or not decoder.eof or decoder.unused_data:
|
||||
raise ParseError('Embedded raster exceeds decoding limit or has invalid compression')
|
||||
total += len(raster)
|
||||
if total > 32*1024*1024: raise ParseError('Total embedded raster size limit exceeded')
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
with Image.open(io.BytesIO(raster)) as source:
|
||||
if source.width*source.height > MAX_PIXELS: raise ParseError('Embedded raster pixel limit exceeded')
|
||||
if source.format not in {'PNG','BMP','JPEG','TIFF','GIF'}:
|
||||
errors.append('Unsupported embedded raster format'); continue
|
||||
if getattr(source, 'n_frames', 1) != 1:
|
||||
errors.append('Multi-frame raster skipped'); continue
|
||||
image = source.convert('RGBA')
|
||||
output = io.BytesIO(); image.save(output, format='PNG')
|
||||
png = output.getvalue()
|
||||
if len(png) > MAX_ASSET: raise ParseError('Converted raster exceeds asset limit')
|
||||
if name in assets:
|
||||
errors.append('Ambiguous embedded filename skipped'); assets[name] = None
|
||||
else: assets[name] = 'data:image/png;base64,' + base64.b64encode(png).decode('ascii')
|
||||
except Exception as exc:
|
||||
errors.append('Embedded storage not completely decoded: ' + str(exc)[:160])
|
||||
return assets, errors
|
||||
@@ -0,0 +1,28 @@
|
||||
import io
|
||||
import struct
|
||||
import olefile
|
||||
from .record_decoder import ParseError, binary_records, ascii_records, detect_format
|
||||
|
||||
MAX_STREAM_BYTES = 32 * 1024 * 1024
|
||||
MAX_TOTAL_BYTES = 64 * 1024 * 1024
|
||||
|
||||
def read_document(data):
|
||||
fmt=detect_format(data)
|
||||
if fmt=='ascii':
|
||||
return fmt,[],ascii_records(data),[]
|
||||
try:
|
||||
with olefile.OleFileIO(io.BytesIO(data),raise_defects=olefile.DEFECT_INCORRECT) as ole:
|
||||
entries=ole.listdir(streams=True,storages=False)
|
||||
if len(entries)>256: raise ParseError('Too many OLE streams')
|
||||
streams=[{'name':'/'.join(p),'size':ole.get_size(p)} for p in entries]
|
||||
if any(s['size']>MAX_STREAM_BYTES for s in streams) or sum(s['size'] for s in streams)>MAX_TOTAL_BYTES:
|
||||
raise ParseError('OLE stream size limit exceeded')
|
||||
if not ole.exists('FileHeader') or ole.get_type('FileHeader')!=2:
|
||||
raise ParseError('OLE container has no FileHeader stream')
|
||||
main=binary_records(ole.openstream('FileHeader').read(),'FileHeader')
|
||||
extra=[]
|
||||
if ole.exists('Additional') and ole.get_type('Additional')==2:
|
||||
extra=binary_records(ole.openstream('Additional').read(),'Additional')
|
||||
return fmt,streams,main,extra
|
||||
except (OSError,IOError,ValueError,IndexError,struct.error) as exc:
|
||||
raise ParseError(f'Invalid OLE schematic: {exc}') from exc
|
||||
@@ -0,0 +1,200 @@
|
||||
"""Decode actual symbol geometry and document-defined electrical marker styles."""
|
||||
import math
|
||||
import textwrap
|
||||
from app.models.schematic import Primitive, Point
|
||||
from .geometry import number, integer, coordinate, point, flag, bounds, color, poly_path, transform_point
|
||||
|
||||
KINDS={1:'component',2:'pin',3:'ieee_symbol',4:'label',5:'bezier',6:'polyline',7:'polygon',8:'ellipse',9:'pie',10:'round_rectangle',11:'elliptical_arc',12:'arc',13:'line',14:'rectangle',15:'sheet_symbol',16:'sheet_entry',17:'power_port',18:'port',22:'no_erc',25:'net_label',26:'bus',27:'wire',28:'text_frame',29:'junction',30:'image',31:'sheet',32:'sheet_name',33:'sheet_filename',34:'designator',37:'bus_entry',39:'template',41:'parameter',43:'parameter_set',44:'implementation_list',45:'implementation',46:'pin_map',47:'pin_map',48:'implementation_parameter',209:'text_frame',225:'dashed_rectangle'}
|
||||
SUPPORTED=set(KINDS)-{3,30,43}
|
||||
|
||||
|
||||
def decode(record, component_id, sheet, warn, records):
|
||||
p=record.properties
|
||||
t=integer(p,'RECORD')
|
||||
pos=point(p)
|
||||
output=[]
|
||||
stroke=color(p.get('COLOR','8388608' if t in (25,26,27) else '128'))
|
||||
fill=color(p.get('AREACOLOR','16777215')) if flag(p,'ISSOLID') and not flag(p,'TRANSPARENT') else 'none'
|
||||
width=(0.4,1,2,4)[min(3,max(0,integer(p,'LINEWIDTH')))]
|
||||
dash=[4,2] if integer(p,'LINESTYLE') or t==225 else []
|
||||
if integer(p,'LINESTYLE'): warn('dash_metrics','Non-solid line dash spacing requires Altium reference validation',record)
|
||||
def path(points=None,d=None,closed=False,fill_=None,stroke_=None,width_=None,box=None):
|
||||
output.append(Primitive(id=f'{record.id}:p{len(output)}',kind='path',
|
||||
object_id=record.id,component_id=component_id,path=d if d is not None else poly_path(points or [],closed),
|
||||
bounds=box or bounds(points or [],width+1),stroke=stroke_ or stroke,fill=fill if fill_ is None else fill_,
|
||||
width=width_ or width,dash=dash))
|
||||
def text(value,at=pos,angle=None,font=None,anchor=None,baseline=None,frame=None,wrap_width=None):
|
||||
if not value: return
|
||||
fid=font or integer(p,'FONTID',integer(sheet,'SYSTEMFONT',1))
|
||||
# Browser em size approximation: Altium font size is not a CSS point size.
|
||||
# This yields glyph cap heights close to the KiCad importer's size/2 stroke text.
|
||||
size=max(1,min(300,number(sheet,f'SIZE{fid}',10)))*0.9
|
||||
just=integer(p,'JUSTIFICATION')
|
||||
a=anchor or ('start','middle','end')[just%3]
|
||||
base=baseline or ('alphabetic','middle','hanging')[min(2,max(0,just//3))]
|
||||
rotation=-90*integer(p,'ORIENTATION') if angle is None else angle
|
||||
raw=value.replace('~1','\n')
|
||||
# Altium backslash suffix marks an overbar on the preceding character.
|
||||
clean=raw.replace('\\','')
|
||||
if raw.startswith('='):
|
||||
key=raw[1:].upper()
|
||||
resolved=p.get('_RESOLVED_TEXT')
|
||||
if resolved is not None: clean=resolved
|
||||
else: warn('unresolved_text',f'Unresolved parameter reference {raw}',record)
|
||||
lines=clean.splitlines() or ['']
|
||||
if wrap_width is not None:
|
||||
columns=max(1,int(wrap_width/(size*0.55)))
|
||||
lines=[line for paragraph in lines for line in (textwrap.wrap(paragraph,width=columns,break_long_words=True,break_on_hyphens=False,replace_whitespace=False) or [''])]
|
||||
for line_i,line in enumerate(lines):
|
||||
loc=Point(x=at.x,y=at.y+line_i*size*1.2)
|
||||
w=max(size*.6,len(line)*size*.8)
|
||||
x={'start':0,'middle':-w/2,'end':-w}[a]
|
||||
y={'alphabetic':-size,'middle':-size/2,'hanging':0}[base]
|
||||
corners=[transform_point(Point(x=cx,y=cy),loc,-rotation,False) for cx,cy in [(x,y),(x+w,y),(x+w,y+size*1.3),(x,y+size*1.3)]]
|
||||
box=bounds(corners,2)
|
||||
if frame:
|
||||
x0,y0=max(box.x,frame.x),max(box.y,frame.y)
|
||||
x1,y1=min(box.x+box.width,frame.x+frame.width),min(box.y+box.height,frame.y+frame.height)
|
||||
if x1<=x0 or y1<=y0: continue
|
||||
from app.models.schematic import Bounds
|
||||
box=Bounds(x=x0,y=y0,width=x1-x0,height=y1-y0)
|
||||
output.append(Primitive(id=f'{record.id}:p{len(output)}',kind='text',object_id=record.id,
|
||||
component_id=component_id,text=line,position=loc,rotation=rotation,mirrored=False,
|
||||
bounds=box,clip=frame,stroke=color(p.get('TEXTCOLOR',p.get('COLOR','128'))),fill='none',
|
||||
font_size=size,font_family=sheet.get(f'FONTNAME{fid}','Arial'),bold=flag(sheet,f'BOLD{fid}'),
|
||||
italic=flag(sheet,f'ITALIC{fid}'),underline=flag(sheet,f'UNDERLINE{fid}'),anchor=a,baseline=base))
|
||||
if '\\' in raw:
|
||||
warn('text_overbar','Overbar text is displayed without its per-character overline',record)
|
||||
def local(points,origin=pos,rotation=None,closed=False,fill_=None):
|
||||
pts=[transform_point(Point(x=x,y=y),origin,90*integer(p,'ORIENTATION') if rotation is None else rotation) for x,y in points]
|
||||
path(pts,closed=closed,fill_=fill_)
|
||||
def ellipse(cx,cy,rx,ry,solid='none'):
|
||||
if rx<=0 or ry<=0: return
|
||||
d=f'M {cx-rx:.15g},{cy:.15g} A {rx:.15g},{ry:.15g} 0 1 0 {cx+rx:.15g},{cy:.15g} A {rx:.15g},{ry:.15g} 0 1 0 {cx-rx:.15g},{cy:.15g} Z'
|
||||
path(d=d,fill_=solid,box=bounds([Point(x=cx-rx,y=cy-ry),Point(x=cx+rx,y=cy+ry)],width))
|
||||
|
||||
if t in (5,6,7,26,27):
|
||||
n=integer(p,'LOCATIONCOUNT')
|
||||
extra=integer(p,'EXTRALOCATIONCOUNT')
|
||||
if n<0 or extra<0 or n+extra>20000: raise ValueError('Invalid point count')
|
||||
pts=[Point(x=coordinate(p,f'X{i}'),y=-coordinate(p,f'Y{i}')) for i in range(1,n+1)]
|
||||
pts += [Point(x=coordinate(p,f'EX{i}'),y=-coordinate(p,f'EY{i}')) for i in range(n+1,n+extra+1)]
|
||||
if t==5:
|
||||
if len(pts)<4 or (len(pts)-1)%3: warn('bezier_points','Non-cubic Bezier point sequence is unsupported',record)
|
||||
else:
|
||||
d=f'M{pts[0].x:.15g},{pts[0].y:.15g} '
|
||||
for i in range(1,len(pts),3): d+='C'+' '.join(f'{q.x:.15g},{q.y:.15g}' for q in pts[i:i+3])+' '
|
||||
path(pts,d=d,fill_='none')
|
||||
elif len(pts)>1: path(pts,closed=t==7,fill_=fill if t==7 else 'none')
|
||||
if integer(p,'STARTLINESHAPE') or integer(p,'ENDLINESHAPE'):
|
||||
warn('line_decoration','Line end decorations are not rendered',record)
|
||||
elif t in (13,37): path([pos,point(p,'CORNER')],fill_='none')
|
||||
elif t in (10,14,15,28,209,225):
|
||||
corner=point(p,'CORNER')
|
||||
if t==15: corner=Point(x=pos.x+coordinate(p,'XSIZE'),y=pos.y+coordinate(p,'YSIZE'))
|
||||
x,y=min(pos.x,corner.x),min(pos.y,corner.y); w,h=abs(pos.x-corner.x),abs(pos.y-corner.y)
|
||||
pts=[Point(x=x,y=y),Point(x=x+w,y=y),Point(x=x+w,y=y+h),Point(x=x,y=y+h)]
|
||||
if t==10:
|
||||
rx=min(w/2,abs(coordinate(p,'CORNERXRADIUS'))); ry=min(h/2,abs(coordinate(p,'CORNERYRADIUS')))
|
||||
if rx and ry:
|
||||
d=f'M{x+rx},{y} H{x+w-rx} A{rx},{ry} 0 0 1 {x+w},{y+ry} V{y+h-ry} A{rx},{ry} 0 0 1 {x+w-rx},{y+h} H{x+rx} A{rx},{ry} 0 0 1 {x},{y+h-ry} V{y+ry} A{rx},{ry} 0 0 1 {x+rx},{y} Z'
|
||||
path(pts,d=d)
|
||||
else: path(pts,closed=True)
|
||||
elif t not in (28,209) or flag(p,'SHOWBORDER') or fill!='none': path(pts,closed=True,fill_=fill,stroke_=stroke if t not in (28,209) or flag(p,'SHOWBORDER') else fill)
|
||||
if t in (28,209):
|
||||
frame=bounds(pts) if flag(p,'CLIPTORECT') else None
|
||||
text(p.get('TEXT',''),Point(x=x+2,y=y+2),angle=0,baseline='hanging',frame=frame,wrap_width=max(1,w-4) if flag(p,'WORDWRAP') else None)
|
||||
if flag(p,'WORDWRAP'): warn('text_frame_metrics','Text frame wrapping uses estimated font advances; clipping is preserved',record)
|
||||
if flag(p,'COLLAPSED'): warn('collapsed_note','Collapsed note is rendered as a clipped text frame',record)
|
||||
elif t in (8,9,11,12):
|
||||
rx=abs(coordinate(p,'RADIUS')); ry=abs(coordinate(p,'SECONDARYRADIUS')) or rx
|
||||
start=number(p,'STARTANGLE'); end=number(p,'ENDANGLE',360)
|
||||
if t==8 or abs(end-start)>=359.999 or start==end:
|
||||
ellipse(pos.x,pos.y,rx,ry,fill if t in (8,9) else 'none')
|
||||
elif rx and ry:
|
||||
def pt(deg):
|
||||
a=math.radians(deg); a=math.atan2(rx*math.sin(a),ry*math.cos(a))
|
||||
return Point(x=pos.x+rx*math.cos(a),y=pos.y-ry*math.sin(a))
|
||||
a,b=pt(start),pt(end); sweep=(end-start)%360
|
||||
d=f'M{a.x:.15g},{a.y:.15g} A{rx:.15g},{ry:.15g} 0 {1 if sweep>180 else 0} 0 {b.x:.15g},{b.y:.15g}'
|
||||
if t==9: d+=f' L{pos.x},{pos.y} Z'
|
||||
path(d=d,fill_=fill if t==9 else 'none',box=bounds([Point(x=pos.x-rx,y=pos.y-ry),Point(x=pos.x+rx,y=pos.y+ry)],width))
|
||||
elif t==2:
|
||||
bits=integer(p,'PINCONGLOMERATE'); rot=(bits&3)*90
|
||||
length=coordinate(p,'PINLENGTH')
|
||||
end=transform_point(Point(x=length,y=0),pos,rot)
|
||||
path([pos,end],fill_='none')
|
||||
# Name extends into body, number lies over the pin. Keep labels upright.
|
||||
horizontal=(bits&3)%2==0
|
||||
direction=1 if (bits&3) in (0,3) else -1
|
||||
if bits&8:
|
||||
at=Point(x=pos.x-(4*direction if horizontal else 0),y=pos.y+(0 if horizontal else -4*direction))
|
||||
text(p.get('NAME',''),at,angle=0 if horizontal else -90,font=integer(p,'NAME_CUSTOMFONTID',1),anchor='end' if direction>0 else 'start',baseline='middle')
|
||||
if bits&16:
|
||||
at=Point(x=(pos.x+end.x)/2-(0 if horizontal else 3),y=(pos.y+end.y)/2-(3 if horizontal else 0))
|
||||
text(p.get('DESIGNATOR',''),at,angle=0 if horizontal else -90,font=integer(p,'DESIGNATOR_CUSTOMFONTID',1),anchor='middle')
|
||||
if integer(p,'SYMBOL_OUTEREDGE')==1: ellipse(pos.x,pos.y,2,2)
|
||||
elif integer(p,'SYMBOL_OUTEREDGE'): warn('pin_decoration','Unsupported outer-edge pin symbol',record)
|
||||
if integer(p,'SYMBOL_INNEREDGE')==3:
|
||||
local([(0,-3),(-4,0),(0,3)],rotation=rot,fill_='none')
|
||||
elif integer(p,'SYMBOL_INNEREDGE'): warn('pin_decoration','Unsupported inner-edge pin symbol',record)
|
||||
if integer(p,'SYMBOL_OUTER') or integer(p,'SYMBOL_INNER'): warn('pin_decoration','Additional pin decorations are not rendered',record)
|
||||
if any(k in p for k in ('NAME_CUSTOMPOSITION_MARGIN','DESIGNATOR_CUSTOMPOSITION_MARGIN')) or integer(p,'PINNAME_POSITIONCONGLOMERATE') or integer(p,'PINDESIGNATOR_POSITIONCONGLOMERATE'):
|
||||
warn('pin_text_position','Custom pin text positions use standard inferred offsets',record)
|
||||
elif t in (4,25,32,33,34,41):
|
||||
value=p.get('TEXT','')
|
||||
if t==34 and component_id:
|
||||
parent=records.get((record.stream,int(component_id.rsplit(':',1)[-1])))
|
||||
if parent and integer(parent.properties,'PARTCOUNT')>2:
|
||||
unit=integer(parent.properties,'CURRENTPARTID',1)
|
||||
if 1<=unit<=26: value+=chr(64+unit)
|
||||
else: warn('unit_suffix','Unit suffix above Z is unsupported',record)
|
||||
if t==41 and flag(p,'SHOWNAME'): value=p.get('NAME','')+': '+value
|
||||
# Parameter labels use upright text with opposite anchors at orientations 2/3.
|
||||
if t in (34,41) and integer(p,'ORIENTATION') in (2,3):
|
||||
text(value,angle=0 if integer(p,'ORIENTATION')==2 else -90,anchor='end',baseline='hanging')
|
||||
elif t==4 and integer(p,'ORIENTATION')==2:
|
||||
just=max(0,min(8,integer(p,'JUSTIFICATION')))
|
||||
text(value,angle=0,anchor=('end','middle','start')[just%3],baseline=('hanging','middle','alphabetic')[just//3])
|
||||
else: text(value)
|
||||
elif t==29: ellipse(pos.x,pos.y,1.5,1.5,stroke)
|
||||
elif t==22:
|
||||
path([Point(x=pos.x-2,y=pos.y-2),Point(x=pos.x+2,y=pos.y+2)],fill_='none')
|
||||
path([Point(x=pos.x-2,y=pos.y+2),Point(x=pos.x+2,y=pos.y-2)],fill_='none')
|
||||
elif t==17:
|
||||
style=integer(p,'STYLE'); rot=90*integer(p,'ORIENTATION')
|
||||
if flag(p,'ISCROSSSHEETCONNECTOR'):
|
||||
local([(8,-3),(4,0),(8,3)],fill_='none');local([(4,-3),(0,0),(4,3)],fill_='none')
|
||||
elif style in (0,1,2,4):
|
||||
local([(0,0),(10,0)],fill_='none')
|
||||
if style==0:
|
||||
c=transform_point(Point(x=10,y=0),pos,rot);ellipse(c.x,c.y,2.5,2.5)
|
||||
elif style==1: local([(6,-3),(10,0),(6,3)],fill_='none')
|
||||
elif style==2: local([(10,-7),(10,7)],fill_='none')
|
||||
elif style==4:
|
||||
for dx,span in [(10,7),(13,4),(16,1)]: local([(dx,-span),(dx,span)],fill_='none')
|
||||
else: warn('power_style',f'Unsupported power port style {style}; marker omitted rather than substituted',record)
|
||||
if flag(p,'SHOWNETNAME'):
|
||||
offset = 20 if style==4 else 14 if flag(p,'ISCROSSSHEETCONNECTOR') else 12
|
||||
at=transform_point(Point(x=offset,y=0),pos,rot)
|
||||
orient=integer(p,'ORIENTATION')
|
||||
text(p.get('TEXT',''),at,angle=0,anchor=('start','middle','end','middle')[orient%4],baseline=('middle','alphabetic','middle','hanging')[orient%4])
|
||||
elif t==18:
|
||||
style=integer(p,'STYLE'); w=coordinate(p,'WIDTH'); h=coordinate(p,'HEIGHT') or 10
|
||||
# Style 0..3 horizontal and 4..7 vertical; bit0 left/top, bit1 right/bottom.
|
||||
tip=min(h/2,w/3); left=bool(style&1); right=bool(style&2)
|
||||
pts=[(0,0),(tip if left else 0,-h/2),(w-tip if right else w,-h/2),(w,0),(w-tip if right else w,h/2),(tip if left else 0,h/2)]
|
||||
local(pts,rotation=90 if style>=4 else 0,closed=True,fill_=color(p.get('AREACOLOR','8454143')))
|
||||
at=transform_point(Point(x=w/2,y=0),pos,90 if style>=4 else 0)
|
||||
text(p.get('NAME',''),at,angle=-90 if style>=4 else 0,anchor='middle',baseline='middle')
|
||||
if p.get('HARNESSTYPE'): warn('harness','Harness semantics are not resolved',record)
|
||||
elif t==16:
|
||||
parent=records.get((record.stream,integer(p,'OWNERINDEX',-1)))
|
||||
if parent and integer(parent.properties,'RECORD')==15:
|
||||
q=parent.properties; origin=point(q);side=integer(p,'SIDE');distance=number(p,'DISTANCEFROMTOP')*10+number(p,'DISTANCEFROMTOP_FRAC1')/100000
|
||||
at=Point(x=origin.x+(coordinate(q,'XSIZE') if side==1 else distance if side>=2 else 0),y=origin.y+(coordinate(q,'YSIZE') if side==3 else distance if side<2 else 0))
|
||||
text(p.get('NAME',''),at,anchor='end' if side==1 else 'start',baseline='middle')
|
||||
path([Point(x=at.x-2,y=at.y),Point(x=at.x+2,y=at.y)],fill_='none')
|
||||
warn('sheet_entry_shape','Sheet entry location and name are rendered; entry arrow style is unsupported',record)
|
||||
else: warn('orphan_sheet_entry','Sheet entry owner not found',record)
|
||||
return output
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Bounds-checked property record reader; keeps physical indexes including binary records."""
|
||||
import re
|
||||
import base64
|
||||
import struct
|
||||
from dataclasses import dataclass
|
||||
|
||||
MAX_RECORDS = 100_000
|
||||
MAX_RECORD_BYTES = 4 * 1024 * 1024
|
||||
|
||||
class ParseError(ValueError):
|
||||
pass
|
||||
|
||||
@dataclass
|
||||
class Record:
|
||||
index: int
|
||||
stream: str
|
||||
properties: dict[str,str]
|
||||
binary: bool = False
|
||||
@property
|
||||
def id(self):
|
||||
return f'{self.stream}:{self.index}'
|
||||
|
||||
|
||||
def properties(payload: bytes):
|
||||
result = {}
|
||||
# UTF8 companion fields take precedence irrespective of source order.
|
||||
utf8 = {}
|
||||
for item in payload.rstrip(b'\x00\r\n').split(b'|'):
|
||||
if b'=' not in item:
|
||||
continue
|
||||
k,v = item.split(b'=',1)
|
||||
key = k.decode('ascii',errors='replace').upper()
|
||||
if key.startswith('%UTF8%'):
|
||||
utf8[key[6:]] = v.decode('utf-8',errors='replace')
|
||||
else:
|
||||
result[key] = v.decode('cp1252',errors='replace')
|
||||
result.update(utf8)
|
||||
return result
|
||||
|
||||
|
||||
def binary_records(data: bytes, stream: str):
|
||||
offset=0
|
||||
records=[]
|
||||
while offset < len(data):
|
||||
if len(records)>MAX_RECORDS:
|
||||
raise ParseError('Record count limit exceeded')
|
||||
if len(data)-offset<4:
|
||||
if not any(data[offset:]): break
|
||||
raise ParseError(f'{stream}: truncated length at byte {offset}')
|
||||
word=struct.unpack_from('<I',data,offset)[0]
|
||||
length, kind=word&0xffffff, word>>24
|
||||
if not 0 < length <= MAX_RECORD_BYTES or length>len(data)-offset-4:
|
||||
raise ParseError(f'{stream}: invalid record length {length} at byte {offset}')
|
||||
payload=data[offset+4:offset+4+length]
|
||||
# Header is outside OWNERINDEX numbering; first object is index zero.
|
||||
records.append(Record(len(records)-1,stream,properties(payload) if kind==0 else {'BINARY_KIND':str(kind),'BINARY_BASE64':base64.b64encode(payload).decode('ascii')},kind!=0))
|
||||
offset+=4+length
|
||||
return records
|
||||
|
||||
|
||||
def ascii_records(data: bytes):
|
||||
lines=data.lstrip(b'\xef\xbb\xbf').splitlines()
|
||||
records=[]
|
||||
for line in lines:
|
||||
if not line.strip(): continue
|
||||
if len(line)>MAX_RECORD_BYTES: raise ParseError('Text record size limit exceeded')
|
||||
if len(records)>MAX_RECORDS: raise ParseError('Record count limit exceeded')
|
||||
p=properties(line)
|
||||
if not p or not ('HEADER' in p or 'RECORD' in p):
|
||||
raise ParseError('Malformed ASCII schematic record')
|
||||
records.append(Record(len(records)-1,'FileHeader',p))
|
||||
return records
|
||||
|
||||
|
||||
def detect_format(data):
|
||||
if data.startswith(b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1'): return 'ole'
|
||||
if re.match(br'\|HEADER=',data.lstrip(b'\xef\xbb\xbf \r\n'),re.I): return 'ascii'
|
||||
raise ParseError('Not a supported SchDoc: expected OLE container or |HEADER= ASCII schematic')
|
||||
@@ -0,0 +1,183 @@
|
||||
import hashlib
|
||||
import re
|
||||
from collections import Counter
|
||||
from app.models.schematic import (Document, Sheet, Component, Symbol, Pin, SchematicObject, Diagnostic, Point)
|
||||
from .ole_reader import read_document
|
||||
from .images import read_images, asset_name
|
||||
from app.models.schematic import Primitive
|
||||
from .record_decoder import ParseError
|
||||
from .geometry import point, flag, integer, coordinate, number, bounds, union, transform_point, color
|
||||
from .primitives import decode, KINDS, SUPPORTED
|
||||
|
||||
ELECTRICAL=['Input','Bidirectional','Output','Open collector','Passive','Tri-state','Open emitter','Power']
|
||||
|
||||
def parse_schdoc(data: bytes, filename: str, document_id: str) -> Document:
|
||||
fmt,streams,main,extra=read_document(data)
|
||||
if not main or 'HEADER' not in main[0].properties:
|
||||
raise ParseError('Missing schematic version header')
|
||||
header=main[0].properties
|
||||
version=header['HEADER']
|
||||
if not re.fullmatch(r'Protel for Windows - Schematic Capture (?:Binary|Ascii|ASCII) File Version 5\.0',version):
|
||||
raise ParseError(f'Unsupported schematic version: {version[:150]}')
|
||||
records=[r for r in main+extra if r.index>=0]
|
||||
if len(records)>100000: raise ParseError('Combined record count limit exceeded')
|
||||
lookup={(r.stream,r.index):r for r in records}
|
||||
diagnostics=[]
|
||||
grouped={}
|
||||
def warn(code,message,r=None,severity='warning'):
|
||||
key=(code,r.properties.get('RECORD') if r else None,message if code not in ('unsupported_record','binary_record') else '')
|
||||
if key in grouped:
|
||||
grouped[key].count+=1
|
||||
else:
|
||||
try: record_type=integer(r.properties,'RECORD') if r and 'RECORD' in r.properties else None
|
||||
except ValueError: record_type=None
|
||||
d=Diagnostic(severity=severity,code=code,message=message,
|
||||
record_id=r.id if r else None,record_type=record_type)
|
||||
grouped[key]=d;diagnostics.append(d)
|
||||
images, image_errors = read_images(data)
|
||||
for message in image_errors: warn('embedded_storage', message)
|
||||
sheet_records=[r for r in records if r.properties.get('RECORD')=='31']
|
||||
if not sheet_records: raise ParseError('No schematic sheet record found')
|
||||
sheet_props=sheet_records[0].properties
|
||||
if len(sheet_records)>1: warn('multiple_sheets','Multiple sheet records found; only the first defines page properties')
|
||||
for s in streams:
|
||||
if s['name'] not in ('FileHeader','Additional','Storage'):
|
||||
warn('unhandled_stream',f"Uninterpreted OLE stream: {s['name']} ({s['size']} bytes)")
|
||||
if extra: warn('additional_indexing','Additional records use stream-local owner indexes; cross-stream ownership is not inferred',severity='info')
|
||||
components={}
|
||||
for r in records:
|
||||
if r.properties.get('RECORD')=='1':
|
||||
p=r.properties
|
||||
try:
|
||||
components[r.id]=Component(id=r.id,symbol_id=f'{r.id}:symbol',name=p.get('DESIGNITEMID',p.get('LIBREFERENCE','')),
|
||||
description=p.get('COMPONENTDESCRIPTION',''),library_reference=p.get('LIBREFERENCE',''),source_library=p.get('SOURCELIBRARYNAME',''),
|
||||
position=point(p),rotation=90*integer(p,'ORIENTATION'),mirrored=flag(p,'ISMIRRORED'),current_part=integer(p,'CURRENTPARTID',1),bounds=bounds([point(p)],5))
|
||||
except ValueError as exc: raise ParseError(f'Invalid component {r.id}: {exc}') from exc
|
||||
def owner_chain(r):
|
||||
chain=[];seen={r.id};current=r
|
||||
while 'OWNERINDEX' in current.properties:
|
||||
index=integer(current.properties,'OWNERINDEX',-1)
|
||||
parent=lookup.get((current.stream,index))
|
||||
if not parent: break
|
||||
if parent.id in seen: raise ValueError('Cyclic ownership')
|
||||
seen.add(parent.id);chain.append(parent);current=parent
|
||||
if len(chain)>64: raise ValueError('Excessive ownership depth')
|
||||
return chain
|
||||
# Collect component/sheet parameter values before resolving text expressions.
|
||||
parameter_maps={}
|
||||
for r in records:
|
||||
if r.properties.get('RECORD') in ('34','41'):
|
||||
p=r.properties
|
||||
try: chain=owner_chain(r)
|
||||
except ValueError as exc: warn('ownership',str(exc),r);continue
|
||||
c=next((components[q.id] for q in chain if q.id in components),None)
|
||||
key=p.get('NAME','Designator' if p.get('RECORD')=='34' else '')
|
||||
value=p.get('TEXT','')
|
||||
owner=c.id if c else 'sheet-0'
|
||||
parameter_maps.setdefault(owner,{})[key.upper()]=value
|
||||
if c:
|
||||
c.parameters[key]=value
|
||||
if p.get('RECORD')=='34': c.designator=value
|
||||
if key.upper()=='COMMENT': c.comment=value
|
||||
if key.upper()=='VALUE': c.value=value
|
||||
for c in components.values():
|
||||
c.value=c.value or c.comment
|
||||
objects=[];primitives=[]
|
||||
counts=Counter()
|
||||
for r in records:
|
||||
p=r.properties
|
||||
try:
|
||||
t=integer(p,'RECORD',-1);counts[str(t)]+=1
|
||||
chain=owner_chain(r)
|
||||
c=next((components[q.id] for q in [r]+chain if q.id in components),None)
|
||||
owner=lookup.get((r.stream,integer(p,'OWNERINDEX',-1)))
|
||||
active=True
|
||||
if c and r.id!=c.id:
|
||||
cp=lookup[(r.stream,int(c.id.split(':')[-1]))].properties
|
||||
for q in [r]+chain:
|
||||
if q.id==c.id: break
|
||||
part=integer(q.properties,'OWNERPARTID',-1)
|
||||
mode=integer(q.properties,'OWNERPARTDISPLAYMODE',0)
|
||||
if part not in (-1,0,c.current_part) or mode!=integer(cp,'DISPLAYMODE',0): active=False
|
||||
visible=active and not flag(p,'ISHIDDEN')
|
||||
if t==41 and any(q.properties.get('RECORD')=='48' for q in chain): visible=False
|
||||
if t==2 and integer(p,'PINCONGLOMERATE')&4: visible=False
|
||||
if p.get('TEXT','').startswith('='):
|
||||
key=p['TEXT'][1:].upper()
|
||||
value=parameter_maps.get(c.id if c else 'sheet-0',{}).get(key,parameter_maps.get('sheet-0',{}).get(key))
|
||||
if value is not None: p['_RESOLVED_TEXT']=value
|
||||
obj=SchematicObject(id=r.id,kind=KINDS.get(t,'unsupported'),record_type=t,source_stream=r.stream,source_index=r.index,
|
||||
owner_id=owner.id if owner else None,component_id=c.id if c else None,visible=visible,position=point(p),bounds=bounds([point(p)]),properties=p)
|
||||
if r.binary:
|
||||
warn('binary_record','Binary schematic record retained by index but payload is unsupported',r)
|
||||
elif t == 30 and visible:
|
||||
image = images.get(asset_name(p.get('FILENAME', '')))
|
||||
if image:
|
||||
box = bounds([point(p), point(p, 'CORNER')])
|
||||
shape = Primitive(id=r.id+':image', kind='image', bounds=box, object_id=r.id, component_id=c.id if c else None, image_data=image, keep_aspect=flag(p, 'KEEPASPECT'))
|
||||
primitives.append(shape); obj.primitive_ids=[shape.id]; obj.bounds=box
|
||||
else: warn('missing_image', 'Embedded raster unavailable; external files are never fetched', r)
|
||||
elif t not in SUPPORTED:
|
||||
warn('unsupported_record',f'Unsupported schematic record type {t}; properties preserved',r)
|
||||
elif visible:
|
||||
shapes=decode(r,c.id if c else None,sheet_props,warn,lookup)
|
||||
primitives.extend(shapes);obj.primitive_ids=[s.id for s in shapes]
|
||||
if shapes: obj.bounds=union([s.bounds for s in shapes])
|
||||
if t==2:
|
||||
bits=integer(p,'PINCONGLOMERATE');length=coordinate(p,'PINLENGTH')
|
||||
end=transform_point(Point(x=length,y=0),obj.position,(bits&3)*90)
|
||||
obj.connection_points=[end]
|
||||
pin=Pin(id=r.id,name=p.get('NAME',''),number=p.get('DESIGNATOR',''),electrical_type=ELECTRICAL[max(0,min(7,integer(p,'ELECTRICAL')))],
|
||||
position=obj.position,connection=end,length=length,rotation=(bits&3)*90,visible=visible,active=active)
|
||||
if c: c.pins.append(pin)
|
||||
else: warn('orphan_pin','Pin has no component owner',r)
|
||||
elif t in (26,27):
|
||||
n=integer(p,'LOCATIONCOUNT')
|
||||
if not 0<=n<=20000: raise ValueError('Invalid electrical point count')
|
||||
obj.connection_points=[Point(x=coordinate(p,f'X{i}'),y=-coordinate(p,f'Y{i}')) for i in range(1,n+1)]
|
||||
elif t in (17,18,25,29,37): obj.connection_points=[obj.position]
|
||||
if c:
|
||||
c.object_ids.append(obj.id);c.primitive_ids.extend(obj.primitive_ids)
|
||||
objects.append(obj)
|
||||
except (ValueError,OverflowError,KeyError) as exc:
|
||||
warn('malformed_record',f'Record not rendered: {str(exc)[:180]}',r,severity='error')
|
||||
objects.append(SchematicObject(id=r.id,kind='malformed',record_type=-1,source_stream=r.stream,source_index=r.index,
|
||||
visible=False,position=Point(x=0,y=0),bounds=bounds([]),properties=p))
|
||||
if len(primitives)>250000: raise ParseError('Drawable count limit exceeded')
|
||||
if not primitives: raise ParseError('No supported visible geometry found in this schematic')
|
||||
by_component={}
|
||||
for shape in primitives:
|
||||
if shape.component_id: by_component.setdefault(shape.component_id,[]).append(shape.bounds)
|
||||
for c in components.values():
|
||||
if c.id in by_component: c.bounds=union(by_component[c.id])
|
||||
else: warn('empty_symbol',f'Component {c.designator or c.id} has no visible supported symbol geometry')
|
||||
# Standard sheet presets in 10-mil units. Custom page dimensions win only when enabled.
|
||||
sizes={0:(1150,760),1:(1550,1110),2:(2230,1570),3:(3150,2230),4:(4460,3150),5:(950,750),6:(1500,950),7:(2000,1500),8:(3200,2000),9:(4200,3200),10:(1100,850),11:(1400,850),12:(1700,1100),13:(990,790),14:(1540,990),15:(2060,1560),16:(3260,2060),17:(4280,3280)}
|
||||
style=integer(sheet_props,'SHEETSTYLE')
|
||||
if style not in sizes and not flag(sheet_props,'USECUSTOMSHEET'): raise ParseError('Unsupported sheet dimensions')
|
||||
w,h=sizes.get(style,(1150,760))
|
||||
if flag(sheet_props,'USECUSTOMSHEET'):
|
||||
w,h=coordinate(sheet_props,'CUSTOMX'),coordinate(sheet_props,'CUSTOMY')
|
||||
if integer(sheet_props,'WORKSPACEORIENTATION') and w>h: w,h=h,w
|
||||
if w <= 0 or h <= 0: raise ParseError('Sheet dimensions must be positive')
|
||||
page=bounds([Point(x=0,y=-h),Point(x=w,y=0)])
|
||||
content=union([s.bounds for s in primitives])
|
||||
if flag(sheet_props,'BORDERON') or flag(sheet_props,'TITLEBLOCKON'):
|
||||
warn('page_decoration','Automatic Altium page border, zones and title block are not reconstructed. Explicit template geometry is rendered.')
|
||||
try:
|
||||
if 'WEIGHT' in header and int(header['WEIGHT'])!=len(main)-1:
|
||||
warn('record_count_mismatch',f"Header declares {header['WEIGHT']} records; found {len(main)-1} in FileHeader")
|
||||
except ValueError: warn('record_count_mismatch','Header record count is not numeric')
|
||||
warn('connectivity_scope','Connection anchors and wire vertices are extracted. Net names are not a compiled netlist; bus expansion, harnesses and cross-sheet connectivity are not resolved.',severity='info')
|
||||
warn('font_metrics','Exact original fonts are required on the client. Font size conversion and glyph metrics have not been validated against Altium.',severity='warning')
|
||||
warn('pin_text_metrics','Default pin text offsets and port text anchors require validation against a trusted Altium export.',severity='warning')
|
||||
return Document(id=document_id,filename=filename,
|
||||
metadata={'format':fmt,'header':version,'minor_version':header.get('MINORVERSION'),
|
||||
'sha256':hashlib.sha256(data).hexdigest(),'input_bytes':len(data),'streams':streams,
|
||||
'record_counts':dict(counts),'record_count':len(records),'component_count':len(components),
|
||||
'primitive_count':len(primitives),'units':'10mil','coordinates':'X right, Y down; Y inverted once from Altium',
|
||||
'compatibility':'partial' if any(d.severity in ('warning','error') for d in diagnostics) else 'supported_subset',
|
||||
'header_properties':header},
|
||||
sheet=Sheet(name=filename,background=color(sheet_props.get('AREACOLOR','16317695')),properties=sheet_props,bounds=page),bounds=content,
|
||||
components=list(components.values()),symbols=[Symbol(id=c.symbol_id,component_id=c.id,primitive_ids=c.primitive_ids) for c in components.values()],
|
||||
objects=objects,primitives=primitives,diagnostics=diagnostics)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Safe SVG serializer: only application-owned SVG tags and attributes are emitted."""
|
||||
import re
|
||||
from xml.etree.ElementTree import Element, SubElement, tostring
|
||||
from .models.schematic import Document
|
||||
|
||||
# XML 1.0 cannot represent these controls. They never become markup or CSS.
|
||||
CONTROL = re.compile('[\x00-\x08\x0b\x0c\x0e-\x1f\ud800-\udfff]')
|
||||
def clean(value): return CONTROL.sub('', str(value))
|
||||
def node(parent, tag, **attrs):
|
||||
return SubElement(parent, tag, {k.replace('_', '-'): clean(v) for k, v in attrs.items()})
|
||||
|
||||
def svg_document(doc: Document) -> bytes:
|
||||
b = doc.sheet.bounds
|
||||
root = Element('svg', {'xmlns': 'http://www.w3.org/2000/svg', 'viewBox': f'{b.x} {b.y} {b.width} {b.height}',
|
||||
'preserveAspectRatio': 'xMidYMid meet', 'role': 'img', 'aria-label': 'Schematic'})
|
||||
node(root, 'rect', x=b.x, y=b.y, width=b.width, height=b.height, fill=doc.sheet.background)
|
||||
defs = node(root, 'defs')
|
||||
for i, p in enumerate(doc.primitives):
|
||||
target = root
|
||||
if p.clip:
|
||||
clip = node(defs, 'clipPath', id=f'clip-{i}', clipPathUnits='userSpaceOnUse')
|
||||
c = p.clip
|
||||
node(clip, 'rect', x=c.x, y=c.y, width=c.width, height=c.height)
|
||||
target = node(root, 'g', clip_path=f'url(#clip-{i})')
|
||||
if p.kind == 'path':
|
||||
n = node(target, 'path', d=p.path, stroke=p.stroke, fill=p.fill,
|
||||
stroke_width=p.width, stroke_linecap='round', stroke_linejoin='round', fill_rule=p.fill_rule)
|
||||
if p.dash: n.set('stroke-dasharray', ' '.join(map(str, p.dash)))
|
||||
elif p.kind == 'image':
|
||||
# Populated only by the trusted bounded raster decoder; never a source URL.
|
||||
if p.image_data.startswith('data:image/png;base64,'):
|
||||
node(target, 'image', x=p.bounds.x, y=p.bounds.y, width=p.bounds.width, height=p.bounds.height,
|
||||
href=p.image_data, preserveAspectRatio='xMidYMid meet' if p.keep_aspect else 'none')
|
||||
else:
|
||||
transform = f'translate({p.position.x} {p.position.y}) rotate({p.rotation})'
|
||||
if p.mirrored: transform += ' scale(-1 1)'
|
||||
n = node(target, 'text', transform=transform, fill=p.stroke, font_family=p.font_family,
|
||||
font_size=p.font_size, font_weight='bold' if p.bold else 'normal',
|
||||
font_style='italic' if p.italic else 'normal', text_anchor=p.anchor,
|
||||
dominant_baseline=p.baseline, text_decoration='underline' if p.underline else 'none')
|
||||
n.set('{http://www.w3.org/XML/1998/namespace}space', 'preserve')
|
||||
n.text = clean(p.text)
|
||||
return tostring(root, encoding='utf-8', xml_declaration=True)
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Private local/S3 storage. All object keys are application-generated."""
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from .config import settings
|
||||
|
||||
KEY = re.compile(r'^[a-f0-9]{48}/(?:input\.SchDoc|attempt-[12]/(?:data\.json|drawing\.svg))$')
|
||||
|
||||
class Storage:
|
||||
def __init__(self):
|
||||
self.config = settings()
|
||||
if self.config.s3_bucket:
|
||||
import boto3
|
||||
self.s3 = boto3.client('s3', endpoint_url=self.config.s3_endpoint, region_name=self.config.s3_region)
|
||||
else: self.s3 = None
|
||||
|
||||
def path(self, key):
|
||||
if not KEY.fullmatch(key): raise ValueError('Invalid storage key')
|
||||
p = self.config.storage_dir / 'objects' / key
|
||||
if not p.resolve().is_relative_to(self.config.storage_dir / 'objects'):
|
||||
raise ValueError('Invalid storage path')
|
||||
return p
|
||||
|
||||
def put(self, key, data, content_type='application/octet-stream'):
|
||||
p = self.path(key) # validate on both backends
|
||||
if self.s3:
|
||||
self.s3.put_object(Bucket=self.config.s3_bucket, Key=key, Body=data, ContentType=content_type)
|
||||
else:
|
||||
p.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
tmp = p.with_name(p.name + '.' + secrets.token_hex(8) + '.tmp')
|
||||
try:
|
||||
with open(tmp, 'xb') as f:
|
||||
os.chmod(tmp, 0o600)
|
||||
f.write(data)
|
||||
os.replace(tmp, p)
|
||||
finally: tmp.unlink(missing_ok=True)
|
||||
|
||||
def get(self, key):
|
||||
p = self.path(key)
|
||||
if self.s3:
|
||||
with self.s3.get_object(Bucket=self.config.s3_bucket, Key=key)['Body'] as body:
|
||||
return body.read()
|
||||
return p.read_bytes()
|
||||
|
||||
def delete(self, key):
|
||||
p = self.path(key)
|
||||
if self.s3: self.s3.delete_object(Bucket=self.config.s3_bucket, Key=key)
|
||||
else: p.unlink(missing_ok=True)
|
||||
|
||||
@lru_cache
|
||||
def storage(): return Storage()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Run `python -m app.worker`; add replicas to scale parsing independently of API."""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from sqlalchemy import update
|
||||
from .config import settings
|
||||
from .cleanup import cleanup
|
||||
from .db import Job, claim, initialize, sessions
|
||||
from .storage import storage
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
stopping = False
|
||||
|
||||
def process(job):
|
||||
config = settings()
|
||||
prefix = f"{job['id']}/attempt-{job['attempt']}"
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix='schdoc-') as directory:
|
||||
root = Path(directory)
|
||||
source = root / 'input.SchDoc'
|
||||
source.write_bytes(storage().get(job['input_key']))
|
||||
env = {k: v for k, v in os.environ.items() if k in ('PATH', 'LANG', 'LC_ALL', 'PYTHONPATH')}
|
||||
# Drain no untrusted output into logs. Kill/wait is handled by subprocess.run.
|
||||
subprocess.run([sys.executable, '-m', 'app.parse_task', str(source), str(root), job['filename'], job['id']],
|
||||
timeout=config.parse_timeout, check=True, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
result = json.loads((root / 'result.json').read_text())
|
||||
if result['ok']:
|
||||
storage().put(prefix + '/data.json', (root / 'data.json').read_bytes(), 'application/json')
|
||||
storage().put(prefix + '/drawing.svg', (root / 'drawing.svg').read_bytes(), 'image/svg+xml')
|
||||
values = dict(status='completed', output_prefix=prefix, summary=json.dumps(result['summary']))
|
||||
else:
|
||||
values = dict(status='failed', error_code=result['code'], error_message=result['message'])
|
||||
except subprocess.TimeoutExpired:
|
||||
values = dict(status='failed', error_code='parse_timeout', error_message='Schematic exceeded the parsing time limit')
|
||||
except subprocess.CalledProcessError:
|
||||
values = dict(status='failed', error_code='parse_resource_limit', error_message='Parser exited before completion (resource limit or invalid input)')
|
||||
except Exception:
|
||||
log.exception('Worker infrastructure failure for %s', job['id'])
|
||||
values = dict(status='failed', error_code='processing_error', error_message='Storage or processing service unavailable')
|
||||
with sessions().begin() as db:
|
||||
# An expired worker may finish late. Its outputs must never replace a newer attempt.
|
||||
db.execute(update(Job).where(Job.id == job['id'], Job.attempt == job['attempt'], Job.status == 'processing')
|
||||
.values(**values, lease_until=None, updated_at=time.time()))
|
||||
|
||||
def main():
|
||||
global stopping
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
def stop(*_):
|
||||
global stopping
|
||||
stopping = True
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
initialize()
|
||||
last_cleanup=0
|
||||
while not stopping:
|
||||
if time.monotonic()-last_cleanup>60:
|
||||
cleanup(); last_cleanup=time.monotonic()
|
||||
job = claim()
|
||||
if job: process(job)
|
||||
else: time.sleep(0.5)
|
||||
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,5 @@
|
||||
-r requirements.txt
|
||||
httpx==0.28.1
|
||||
pytest==8.4.2
|
||||
cairosvg==2.8.2
|
||||
numpy==2.3.4
|
||||
@@ -0,0 +1,27 @@
|
||||
annotated-doc==0.0.5
|
||||
annotated-types==0.8.0
|
||||
anyio==4.16.0
|
||||
boto3==1.40.70
|
||||
botocore==1.40.76
|
||||
click==8.5.0
|
||||
fastapi==0.121.1
|
||||
greenlet==3.5.6
|
||||
h11==0.16.0
|
||||
idna==3.20
|
||||
jmespath==1.1.0
|
||||
olefile==0.47
|
||||
pillow==12.0.0
|
||||
psycopg==3.2.12
|
||||
psycopg-binary==3.2.12
|
||||
pydantic==2.14.2
|
||||
pydantic-core==2.50.2
|
||||
python-dateutil==2.9.0.post0
|
||||
python-multipart==0.0.22
|
||||
s3transfer==0.14.0
|
||||
six==1.17.0
|
||||
sqlalchemy==2.0.44
|
||||
starlette==0.49.3
|
||||
typing-extensions==4.16.0
|
||||
typing-inspection==0.4.5
|
||||
urllib3==2.8.0
|
||||
uvicorn==0.38.0
|
||||
@@ -0,0 +1,8 @@
|
||||
fastapi==0.121.1
|
||||
uvicorn==0.38.0
|
||||
python-multipart==0.0.22
|
||||
olefile==0.47
|
||||
SQLAlchemy==2.0.44
|
||||
psycopg[binary]==3.2.12
|
||||
boto3==1.40.70
|
||||
Pillow==12.0.0
|
||||
@@ -0,0 +1,20 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
@pytest.fixture
|
||||
def ascii_schematic():
|
||||
# Purpose-built unit-test records, not an application demo or a real Altium export.
|
||||
return b'''|HEADER=Protel for Windows - Schematic Capture Ascii File Version 5.0|Weight=9
|
||||
|RECORD=31|FontIdCount=1|Size1=10|FontName1=Arial
|
||||
|RECORD=1|LibReference=R_Symbol|Location.X=100|Location.Y=200|CurrentPartId=1|PartCount=2|Orientation=1|IsMirrored=T
|
||||
|RECORD=13|OwnerIndex=1|OwnerPartId=1|Location.X=100|Location.Y=200|Corner.X=100|Corner.Y=220|Color=128
|
||||
|RECORD=2|OwnerIndex=1|OwnerPartId=1|Location.X=100|Location.Y=220|PinLength=10|PinConglomerate=25|Name=A|Designator=1|Electrical=4
|
||||
|RECORD=34|OwnerIndex=1|OwnerPartId=-1|Text=R1|Location.X=105|Location.Y=205
|
||||
|RECORD=41|OwnerIndex=1|OwnerPartId=-1|Name=Comment|Text=4K7|Location.X=110|Location.Y=205
|
||||
|RECORD=41|OwnerIndex=1|OwnerPartId=-1|Name=Manufacturer|Text=Test|IsHidden=T
|
||||
|RECORD=27|OwnerPartId=-1|LocationCount=2|X1=100|Y1=230|X2=150|Y2=230
|
||||
|RECORD=25|Text=VCC|Location.X=150|Location.Y=230
|
||||
'''
|
||||
@@ -0,0 +1,12 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
from app.fonts import fonts,stylesheet
|
||||
|
||||
|
||||
def test_font_manifest_only_allows_explicit_local_paths(tmp_path,monkeypatch):
|
||||
monkeypatch.setenv('FONT_DIR',str(tmp_path)); (tmp_path/'test.woff2').write_bytes(b'font fixture')
|
||||
(tmp_path/'fonts.json').write_text(json.dumps({'Arial':[{'file':'test.woff2','weight':'bold','style':'italic'}]}))
|
||||
assert len(fonts())==1 and "font-weight:bold" in stylesheet()
|
||||
(tmp_path/'fonts.json').write_text(json.dumps({'Arial':'../outside.woff2'}))
|
||||
with pytest.raises(ValueError):fonts()
|
||||
@@ -0,0 +1,43 @@
|
||||
import base64
|
||||
import io
|
||||
import struct
|
||||
import zlib
|
||||
from PIL import Image
|
||||
from app.parser.images import read_images
|
||||
from app.render import svg_document
|
||||
from app.models.schematic import Primitive, Bounds
|
||||
from app.parser.schdoc_parser import parse_schdoc
|
||||
|
||||
|
||||
def storage(payload):
|
||||
header=b'|HEADER=Icon storage\0'
|
||||
return struct.pack('<I',len(header))+header+struct.pack('<I',(1<<24)|len(payload))+payload
|
||||
|
||||
class Ole:
|
||||
def __init__(self,data): self.data=data
|
||||
def __enter__(self): return self
|
||||
def __exit__(self,*args): pass
|
||||
def exists(self,name): return name=='Storage'
|
||||
def openstream(self,name): return io.BytesIO(self.data)
|
||||
|
||||
|
||||
def test_embedded_raster_is_reencoded_and_never_fetches_path(monkeypatch, ascii_schematic):
|
||||
buffer=io.BytesIO(); Image.new('RGB',(2,3),(255,0,0)).save(buffer,format='BMP')
|
||||
compressed=zlib.compress(buffer.getvalue()); filename=b'C:\\Project\\image.bmp'
|
||||
payload=b'\xd0'+bytes([len(filename)])+filename+struct.pack('<I',len(compressed))+compressed
|
||||
monkeypatch.setattr('app.parser.images.olefile.OleFileIO',lambda _:Ole(storage(payload)))
|
||||
assets,errors=read_images(b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1')
|
||||
assert not errors and set(assets)=={'image.bmp'}
|
||||
png=base64.b64decode(assets['image.bmp'].split(',')[1]); assert png.startswith(b'\x89PNG')
|
||||
doc=parse_schdoc(ascii_schematic,'x.SchDoc','x')
|
||||
doc.primitives.append(Primitive(id='image',kind='image',bounds=Bounds(x=0,y=-30,width=20,height=30),object_id='image',image_data=assets['image.bmp']))
|
||||
svg=svg_document(doc)
|
||||
assert b'data:image/png;base64,' in svg and b'C:\\Project' not in svg
|
||||
|
||||
|
||||
def test_bounded_zlib_expansion(monkeypatch):
|
||||
compressed=zlib.compress(b'0'*(16*1024*1024+1)); filename=b'bomb.bmp'
|
||||
payload=b'\xd0'+bytes([len(filename)])+filename+struct.pack('<I',len(compressed))+compressed
|
||||
monkeypatch.setattr('app.parser.images.olefile.OleFileIO',lambda _:Ole(storage(payload)))
|
||||
assets,errors=read_images(b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1')
|
||||
assert not assets and any('limit' in e for e in errors)
|
||||
@@ -0,0 +1,133 @@
|
||||
import math
|
||||
import struct
|
||||
import pytest
|
||||
from app.parser.geometry import point,coordinate,transform_point,color
|
||||
from app.models.schematic import Point
|
||||
from app.parser.record_decoder import detect_format,binary_records,properties,ParseError
|
||||
from app.parser.schdoc_parser import parse_schdoc
|
||||
from app.parser.ole_reader import read_document
|
||||
|
||||
@pytest.mark.parametrize('data,expected',[(b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1','ole'),(b'|HEADER=x','ascii'),(b'\xef\xbb\xbf|HEADER=x','ascii')])
|
||||
def test_detect(data,expected): assert detect_format(data)==expected
|
||||
|
||||
@pytest.mark.parametrize('data',[b'',b'<html/>',b'PK\x03\x04',b'\0'*1024])
|
||||
def test_unknown_format(data):
|
||||
with pytest.raises(ParseError): detect_format(data)
|
||||
|
||||
def test_length_prefix_and_indexing():
|
||||
payloads=[b'|HEADER=Example\0',b'|RECORD=31\0',b'|RECORD=27|X1=42\0']
|
||||
stream=b''.join(struct.pack('<I',len(p))+p for p in payloads)
|
||||
records=binary_records(stream,'FileHeader')
|
||||
assert [r.index for r in records]==[-1,0,1]
|
||||
assert records[2].properties['X1']=='42'
|
||||
|
||||
@pytest.mark.parametrize('data',[b'\x05',struct.pack('<I',100)+b'x',struct.pack('<I',0),struct.pack('<I',0x7fffff)+b'x'])
|
||||
def test_truncated_or_excessive_record(data):
|
||||
with pytest.raises(ParseError): binary_records(data,'FileHeader')
|
||||
|
||||
def test_binary_record_does_not_shift_owners():
|
||||
p=b'|HEADER=x\0';s=struct.pack('<I',len(p))+p+struct.pack('<I',0x1000002)+b'aa'
|
||||
r=binary_records(s,'FileHeader');assert r[1].binary and r[1].index==0
|
||||
|
||||
def test_utf8_precedence_and_case():
|
||||
p=properties('|%UTF8%Comment=مقاومت Ω|Comment=legacy|Text=a=b'.encode())
|
||||
assert p['COMMENT']=='مقاومت Ω' and p['TEXT']=='a=b'
|
||||
|
||||
def test_fraction_and_axis():
|
||||
p={'LOCATION.X':'12','LOCATION.X_FRAC':'50000','LOCATION.Y':'-3','LOCATION.Y_FRAC':'25000'}
|
||||
assert coordinate(p,'LOCATION.X')==12.5
|
||||
assert point(p)==Point(x=12.5,y=2.75)
|
||||
assert color('8388608')=='#000080'
|
||||
|
||||
@pytest.mark.parametrize('rotation,expected',[(0,(12,20)),(90,(10,18)),(180,(8,20)),(270,(10,22))])
|
||||
def test_rotation(rotation,expected):
|
||||
p=transform_point(Point(x=2,y=0),Point(x=10,y=20),rotation)
|
||||
assert p.x==pytest.approx(expected[0]) and p.y==pytest.approx(expected[1])
|
||||
|
||||
def test_mirror_then_rotate():
|
||||
p=transform_point(Point(x=2,y=1),Point(x=10,y=20),90,True)
|
||||
assert p==Point(x=11,y=22)
|
||||
|
||||
def test_component_extraction_and_absolute_geometry(ascii_schematic):
|
||||
d=parse_schdoc(ascii_schematic,'test.SchDoc','test')
|
||||
c=d.components[0]
|
||||
assert (c.designator,c.value,c.comment,c.library_reference)==('R1','4K7','4K7','R_Symbol')
|
||||
assert c.rotation==90 and c.mirrored and c.parameters['Manufacturer']=='Test'
|
||||
assert c.pins[0].number=='1' and c.pins[0].name=='A'
|
||||
assert c.pins[0].connection==Point(x=100,y=-230)
|
||||
# Placed child geometry must not be rotated/mirrored again.
|
||||
line=next(p for p in d.primitives if p.object_id=='FileHeader:2')
|
||||
assert line.path=='M100,-200 L100,-220'
|
||||
assert line.component_id==c.id
|
||||
assert len({x.id for x in d.objects})==len(d.objects)
|
||||
assert set(c.primitive_ids)==set(d.symbols[0].primitive_ids)
|
||||
assert not any(p.text=='Test' for p in d.primitives)
|
||||
|
||||
def test_wire_path_and_connection_vertices(ascii_schematic):
|
||||
d=parse_schdoc(ascii_schematic,'test.SchDoc','test')
|
||||
o=next(o for o in d.objects if o.kind=='wire')
|
||||
p=next(p for p in d.primitives if p.object_id==o.id)
|
||||
assert p.path=='M100,-230 L150,-230'
|
||||
assert o.connection_points==[Point(x=100,y=-230),Point(x=150,y=-230)]
|
||||
|
||||
def test_part_visibility(ascii_schematic):
|
||||
data=ascii_schematic+b'|RECORD=13|OwnerIndex=1|OwnerPartId=2|Location.X=2|Corner.X=3\n'
|
||||
d=parse_schdoc(data,'test.SchDoc','test')
|
||||
assert not d.objects[-1].visible and not d.objects[-1].primitive_ids
|
||||
|
||||
def test_unsupported_preserved(ascii_schematic):
|
||||
d=parse_schdoc(ascii_schematic+b'|RECORD=999|UnknownProperty=important\n','test.SchDoc','test')
|
||||
assert d.objects[-1].properties['UNKNOWNPROPERTY']=='important'
|
||||
assert any(x.code=='unsupported_record' and x.record_type==999 for x in d.diagnostics)
|
||||
|
||||
def test_bad_numeric_geometry_reports_partial(ascii_schematic):
|
||||
d=parse_schdoc(ascii_schematic+b'|RECORD=13|Location.X=nan\n','test.SchDoc','test')
|
||||
assert any(x.code=='malformed_record' and x.severity=='error' for x in d.diagnostics)
|
||||
assert d.objects[-1].properties['LOCATION.X']=='nan'
|
||||
|
||||
def test_unknown_version(ascii_schematic):
|
||||
with pytest.raises(ParseError,match='Unsupported schematic version'):
|
||||
parse_schdoc(ascii_schematic.replace(b'Version 5.0',b'Version 8.0'),'test.SchDoc','test')
|
||||
|
||||
def test_ole_malformed_sector_table():
|
||||
with pytest.raises(ParseError): read_document(b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1'+b'\0'*504)
|
||||
|
||||
def test_owner_cycle(ascii_schematic):
|
||||
d=parse_schdoc(ascii_schematic+b'|RECORD=13|OwnerIndex=9\n','test.SchDoc','test')
|
||||
assert any(x.code=='malformed_record' for x in d.diagnostics)
|
||||
|
||||
@pytest.mark.parametrize('typ,props',[
|
||||
(5,'|LocationCount=4|X1=0|Y1=0|X2=5|Y2=9|X3=10|Y3=9|X4=15|Y4=0'),
|
||||
(7,'|LocationCount=3|X1=0|Y1=0|X2=5|Y2=9|X3=10|Y3=0|IsSolid=T'),
|
||||
(8,'|Radius=5|SecondaryRadius=10'),(9,'|Radius=5|StartAngle=30|EndAngle=180'),
|
||||
(10,'|Corner.X=20|Corner.Y=10|CornerXRadius=2|CornerYRadius=3'),
|
||||
(11,'|Radius=10|SecondaryRadius=5|StartAngle=45|EndAngle=200'),
|
||||
(12,'|Radius=5'),(14,'|Corner.X=10|Corner.Y=20'),
|
||||
(15,'|XSize=100|YSize=60'),(18,'|Name=DATA|Width=50|Style=3'),
|
||||
(22,''),(26,'|LocationCount=2|X1=0|Y1=0|X2=20|Y2=30'),(29,''),
|
||||
(37,'|Corner.X=10|Corner.Y=20'),(225,'|Corner.X=100|Corner.Y=50')])
|
||||
def test_supported_shapes(ascii_schematic,typ,props):
|
||||
d=parse_schdoc(ascii_schematic+f'|RECORD={typ}{props}\n'.encode(),'test.SchDoc','test')
|
||||
obj=d.objects[-1]
|
||||
assert obj.primitive_ids
|
||||
assert not any(x.code=='malformed_record' for x in d.diagnostics)
|
||||
assert all(math.isfinite(p.bounds.width) for p in d.primitives)
|
||||
|
||||
def test_clipped_text_does_not_expand_fit_bounds(ascii_schematic):
|
||||
line='|RECORD=28|Location.X=10|Location.Y=10|Corner.X=60|Corner.Y=40|FontID=1|WordWrap=T|ClipToRect=T|Text='+('Long note text '*100)+'\n'
|
||||
d=parse_schdoc(ascii_schematic+line.encode(),'test.SchDoc','test')
|
||||
obj=d.objects[-1]
|
||||
assert obj.bounds.x>=10 and obj.bounds.x+obj.bounds.width<=60
|
||||
assert obj.bounds.y>=-40 and obj.bounds.y+obj.bounds.height<=-10
|
||||
assert any(p.clip for p in d.primitives if p.object_id==obj.id)
|
||||
|
||||
def test_invalid_record_type_is_preserved(ascii_schematic):
|
||||
d=parse_schdoc(ascii_schematic+b'|RECORD=abc|OriginalField=value\n','test.SchDoc','test')
|
||||
assert d.objects[-1].properties['ORIGINALFIELD']=='value'
|
||||
assert any(w.code=='malformed_record' for w in d.diagnostics)
|
||||
|
||||
def test_implementation_parameter_is_not_drawn(ascii_schematic):
|
||||
data=ascii_schematic+b'|RECORD=48|OwnerIndex=1\n|RECORD=41|OwnerIndex=9|Name=ModelParam|Text=must-not-render\n'
|
||||
d=parse_schdoc(data,'test.SchDoc','test')
|
||||
assert not d.objects[-1].visible
|
||||
assert not any(p.text=='must-not-render' for p in d.primitives)
|
||||
@@ -0,0 +1,165 @@
|
||||
import json
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import select, update
|
||||
from app.config import settings
|
||||
from app.db import Job, claim, engine, sessions
|
||||
from app.main import app
|
||||
from app.storage import storage
|
||||
from app.worker import process
|
||||
|
||||
KEY = 'test-api-key-' + 'x'*40
|
||||
|
||||
@pytest.fixture
|
||||
def client(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv('STORAGE_DIR', str(tmp_path))
|
||||
monkeypatch.setenv('DATABASE_URL', f'sqlite:///{tmp_path}/test.db')
|
||||
monkeypatch.setenv('API_KEY', KEY)
|
||||
monkeypatch.setenv('PUBLIC_BASE_URL', 'http://testserver')
|
||||
monkeypatch.setenv('MAX_UPLOAD_BYTES', '1048576')
|
||||
settings.cache_clear(); storage.cache_clear(); sessions.cache_clear(); engine.cache_clear()
|
||||
with TestClient(app) as client: yield client
|
||||
engine().dispose()
|
||||
settings.cache_clear(); storage.cache_clear(); sessions.cache_clear(); engine.cache_clear()
|
||||
|
||||
|
||||
def upload(client, data, **fields):
|
||||
return client.post('/api/v1/schematics/render', files={'file': ('sample.SchDoc', data, 'application/octet-stream')},
|
||||
data=fields, headers={'X-API-Key': KEY})
|
||||
|
||||
|
||||
def test_private_upload_worker_viewer_and_expiry(client, ascii_schematic):
|
||||
response = upload(client, ascii_schematic)
|
||||
assert response.status_code == 202, response.text
|
||||
body = response.json(); id = body['id']; token = {'X-Viewer-Token': body['viewer_token']}
|
||||
assert len(id) == 48 and '#token=' in body['viewer_url']
|
||||
base = f'/api/v1/schematics/{id}'
|
||||
assert client.get(base+'/status').status_code == 404
|
||||
assert client.get(base+'/status',headers=token).json()['status'] == 'queued'
|
||||
assert client.get(base+'/data',headers=token).status_code == 409
|
||||
job = claim(); assert job['id'] == id
|
||||
process(job)
|
||||
response = client.get(base+'/status',headers=token)
|
||||
assert response.json()['status'] == 'completed', response.text
|
||||
assert response.json()['compatibility'] == 'partial'
|
||||
data = client.get(base+'/data',headers=token).json()
|
||||
assert data['components'][0]['designator'] == 'R1'
|
||||
drawing = client.get(base+'/svg',headers=token)
|
||||
assert drawing.status_code == 200 and '<svg' in drawing.text and 'R1' in drawing.text
|
||||
view = client.get('/view/'+id)
|
||||
assert view.status_code == 200 and 'viewer.js' in view.text
|
||||
assert 'no-referrer' == view.headers['referrer-policy']
|
||||
with sessions().begin() as db: db.execute(update(Job).where(Job.id==id).values(expires_at=time.time()-1))
|
||||
assert client.get(base+'/data',headers=token).status_code == 410
|
||||
assert client.get(base+'/status').status_code == 404
|
||||
|
||||
|
||||
def test_unlisted_and_api_key(client, ascii_schematic):
|
||||
assert client.post('/api/v1/schematics/render').status_code == 401
|
||||
assert client.post('/api/v1/schematics/render',headers={'X-API-Key':'bad'}).status_code == 401
|
||||
body = upload(client,ascii_schematic,access='unlisted').json()
|
||||
assert 'viewer_token' not in body
|
||||
assert client.get(f"/api/v1/schematics/{body['id']}/status").status_code == 200
|
||||
|
||||
|
||||
def test_upload_validation(client):
|
||||
assert upload(client,b'not a schematic').status_code == 422
|
||||
response=client.post('/api/v1/schematics/render',files={'file':('bad.PcbDoc',b'a')},headers={'X-API-Key':KEY})
|
||||
assert response.status_code == 415
|
||||
assert upload(client,b'x'*(1048576+1)).status_code == 413
|
||||
assert upload(client,b'|HEADER=x',expires_in=999999999).status_code == 422
|
||||
assert upload(client,b'|HEADER=x',access='public').status_code == 422
|
||||
|
||||
|
||||
def test_multipart_limit_before_spooling(client):
|
||||
r = client.post('/api/v1/schematics/render',content=b'x'*(3*1048576),
|
||||
headers={'X-API-Key':KEY,'Content-Type':'multipart/form-data; boundary=foo'})
|
||||
assert r.status_code == 413, r.text
|
||||
|
||||
|
||||
def test_corruption_is_durable_failure(client):
|
||||
body = upload(client,b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1'+b'\0'*504).json()
|
||||
process(claim())
|
||||
r = client.get(f"/api/v1/schematics/{body['id']}/status",headers={'X-API-Key':KEY})
|
||||
assert r.json()['status'] == 'failed'
|
||||
assert r.json()['error']['code'] == 'invalid_schematic'
|
||||
|
||||
|
||||
def test_queue_capacity(client, ascii_schematic, monkeypatch):
|
||||
monkeypatch.setenv('MAX_PENDING_JOBS','1'); settings.cache_clear()
|
||||
assert upload(client,ascii_schematic).status_code == 202
|
||||
assert upload(client,ascii_schematic).status_code == 503
|
||||
|
||||
|
||||
def test_atomic_claim_and_stale_worker_fencing(client, ascii_schematic):
|
||||
body=upload(client,ascii_schematic).json()
|
||||
with ThreadPoolExecutor(max_workers=4) as pool: jobs=list(pool.map(lambda _:claim(),range(4)))
|
||||
found=[job for job in jobs if job]
|
||||
assert len(found)==1
|
||||
old=found[0]
|
||||
with sessions().begin() as db: db.execute(update(Job).where(Job.id==body['id']).values(lease_until=time.time()-1))
|
||||
new=claim(); assert new['attempt']==2
|
||||
process(new); process(old)
|
||||
with sessions()() as db:
|
||||
job=db.get(Job,body['id'])
|
||||
assert job.status=='completed' and job.output_prefix.endswith('attempt-2')
|
||||
|
||||
|
||||
def test_storage_path_traversal(client):
|
||||
for key in ['../etc/passwd','a'*48+'/../input.SchDoc','/tmp/evil','a'*48+'/input.SchDoc/evil']:
|
||||
with pytest.raises(ValueError): storage().get(key)
|
||||
|
||||
|
||||
def test_api_docs_and_static(client):
|
||||
assert client.get('/openapi.json').json()['info']['version']=='1.0.0'
|
||||
assert client.get('/').status_code==200
|
||||
assert client.get('/assets/viewer.js').status_code==200
|
||||
assert client.get('/assets/not-allowed.js').status_code==404
|
||||
assert client.get('/view/nope').status_code==404
|
||||
|
||||
|
||||
def test_production_fails_closed(client, monkeypatch):
|
||||
monkeypatch.setenv('APP_ENV','production'); monkeypatch.setenv('API_KEY',''); settings.cache_clear()
|
||||
with pytest.raises(RuntimeError,match='API_KEY'): settings()
|
||||
monkeypatch.setenv('API_KEY',KEY); settings.cache_clear()
|
||||
with pytest.raises(RuntimeError,match='HTTPS'): settings()
|
||||
|
||||
|
||||
def test_expired_storage_cleanup(client, ascii_schematic):
|
||||
from app.cleanup import cleanup
|
||||
body=upload(client,ascii_schematic).json(); process(claim())
|
||||
identifier=body['id']
|
||||
with sessions().begin() as db: db.execute(update(Job).where(Job.id==identifier).values(expires_at=time.time()-1))
|
||||
assert cleanup()==1
|
||||
with sessions()() as db: assert db.get(Job,identifier) is None
|
||||
with pytest.raises(FileNotFoundError): storage().get(identifier+'/input.SchDoc')
|
||||
|
||||
|
||||
def test_expired_worker_is_not_reclaimed(client, ascii_schematic):
|
||||
body=upload(client,ascii_schematic).json()
|
||||
with sessions().begin() as db: db.execute(update(Job).where(Job.id==body['id']).values(expires_at=time.time()-1))
|
||||
assert claim() is None
|
||||
|
||||
|
||||
def test_repeated_worker_loss_fails_job(client, ascii_schematic):
|
||||
body=upload(client,ascii_schematic).json(); assert claim()['attempt']==1
|
||||
for attempt in (2,3):
|
||||
with sessions().begin() as db: db.execute(update(Job).where(Job.id==body['id']).values(lease_until=time.time()-1))
|
||||
result=claim()
|
||||
if attempt==2: assert result['attempt']==2
|
||||
else: assert result is None
|
||||
with sessions()() as db: assert db.get(Job,body['id']).error_code=='worker_lost'
|
||||
|
||||
|
||||
def test_parser_wall_timeout_is_reported(client,ascii_schematic,monkeypatch):
|
||||
import subprocess
|
||||
from app import worker
|
||||
body=upload(client,ascii_schematic).json(); job=claim()
|
||||
def timeout(*args,**kw): raise subprocess.TimeoutExpired('parser',60)
|
||||
monkeypatch.setattr(worker.subprocess,'run',timeout)
|
||||
process(job)
|
||||
status=client.get(f"/api/v1/schematics/{body['id']}/status",headers={'X-API-Key':KEY}).json()
|
||||
assert status['status']=='failed' and status['error']['code']=='parse_timeout'
|
||||
@@ -0,0 +1,67 @@
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
from xml.etree.ElementTree import fromstring
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from app.parser.schdoc_parser import parse_schdoc
|
||||
from app.render import svg_document
|
||||
|
||||
|
||||
def test_svg_escapes_untrusted_content(ascii_schematic):
|
||||
malicious=b'|RECORD=4|Location.X=1|Text=<script>alert("x")</script>&\n'
|
||||
doc=parse_schdoc(ascii_schematic+malicious,'x.SchDoc','x')
|
||||
svg=svg_document(doc); tree=fromstring(svg)
|
||||
assert not list(tree.iter('{http://www.w3.org/2000/svg}script'))
|
||||
assert b'<script>' in svg
|
||||
assert b'viewBox="0.0 -760.0 1150.0 760.0"' in svg
|
||||
|
||||
|
||||
def test_custom_page_and_rotation(ascii_schematic):
|
||||
raw=ascii_schematic.replace(b'FontIdCount=1',b'UseCustomSheet=T|CustomX=500|CustomY=900|FontIdCount=1')
|
||||
doc=parse_schdoc(raw,'x.SchDoc','x')
|
||||
assert doc.sheet.bounds.width==500 and doc.sheet.bounds.height==900
|
||||
assert b'rotate(-90.0)' in svg_document(doc)
|
||||
|
||||
|
||||
def test_no_substituted_unsupported_power_symbol(ascii_schematic):
|
||||
raw=ascii_schematic+b'|RECORD=17|Style=6|Location.X=55|Text=EARTH|ShowNetName=T\n'
|
||||
doc=parse_schdoc(raw,'x.SchDoc','x')
|
||||
assert any(d.code=='power_style' for d in doc.diagnostics)
|
||||
assert all(p.kind=='text' for p in doc.primitives if p.object_id==doc.objects[-1].id)
|
||||
|
||||
|
||||
FIXTURE=Path(__file__).resolve().parents[2]/'.runtime/fixtures/Channel.SchDoc'
|
||||
|
||||
@pytest.mark.parametrize('path',[FIXTURE, Path(os.getenv('SCHDOC_SAMPLE', '/nonexistent'))])
|
||||
def test_real_binary_fixture(path):
|
||||
if not path.is_file(): pytest.skip('Fetch the native fixture or set SCHDOC_SAMPLE')
|
||||
doc=parse_schdoc(path.read_bytes(),path.name,'real')
|
||||
assert doc.metadata['format']=='ole'
|
||||
assert len(doc.components)>0 and len(doc.primitives)>5
|
||||
assert all(o.properties for o in doc.objects)
|
||||
import cairosvg
|
||||
png=cairosvg.svg2png(bytestring=svg_document(doc),output_width=1600)
|
||||
with Image.open(io.BytesIO(png)) as image:
|
||||
assert image.width==1600 and image.height>0
|
||||
|
||||
|
||||
def test_mirrored_parameter_placement_keeps_glyphs_readable(ascii_schematic):
|
||||
raw=ascii_schematic+b'|RECORD=41|Text=DDR3[7A]|IsMirrored=T|Justification=5|Location.X=1077|Location.Y=781\n'
|
||||
doc=parse_schdoc(raw,'x.SchDoc','x')
|
||||
text=next(p for p in doc.primitives if p.text=='DDR3[7A]')
|
||||
assert not text.mirrored and text.anchor=='end' and text.position.x==1077
|
||||
|
||||
|
||||
def test_svg_keeps_fractional_native_coordinates(ascii_schematic):
|
||||
raw=ascii_schematic+b'|RECORD=13|Location.X=1000|Location.X_FRAC=12345|Location.Y=500|Location.Y_FRAC=67890|Corner.X=1001|Corner.Y=500\n'
|
||||
doc=parse_schdoc(raw,'x.SchDoc','x')
|
||||
primitive=next(p for p in doc.primitives if p.object_id==doc.objects[-1].id)
|
||||
assert primitive.path=='M1000.12345,-500.6789 L1001,-500'
|
||||
|
||||
|
||||
def test_rounded_rectangle_preserves_elliptical_corner_radii(ascii_schematic):
|
||||
raw=ascii_schematic+b'|RECORD=10|Corner.X=20|Corner.Y=10|CornerXRadius=2|CornerYRadius=3\n'
|
||||
doc=parse_schdoc(raw,'x.SchDoc','x')
|
||||
primitive=next(p for p in doc.primitives if p.object_id==doc.objects[-1].id)
|
||||
assert primitive.path.count('A2.0,3.0')==4 and ' Q' not in primitive.path
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Protocol-level S3 contract check with botocore's validation stub (no live S3)."""
|
||||
import io
|
||||
import boto3
|
||||
from botocore.response import StreamingBody
|
||||
from botocore.stub import Stubber
|
||||
from app.storage import Storage
|
||||
|
||||
|
||||
def test_s3_private_round_trip_contract(tmp_path,monkeypatch):
|
||||
from app.config import settings
|
||||
from app.storage import storage
|
||||
monkeypatch.setenv('STORAGE_DIR',str(tmp_path)); monkeypatch.setenv('S3_BUCKET','test-private')
|
||||
settings.cache_clear(); storage.cache_clear()
|
||||
client=boto3.client('s3',region_name='us-east-1',aws_access_key_id='test',aws_secret_access_key='test')
|
||||
monkeypatch.setattr(boto3,'client',lambda *a,**kw:client)
|
||||
key='a'*48+'/input.SchDoc'; payload=b'actual bytes'
|
||||
with Stubber(client) as stub:
|
||||
stub.add_response('put_object',{}, {'Bucket':'test-private','Key':key,'Body':payload,'ContentType':'application/octet-stream'})
|
||||
stub.add_response('get_object',{'Body':StreamingBody(io.BytesIO(payload),len(payload))},{'Bucket':'test-private','Key':key})
|
||||
stub.add_response('delete_object',{}, {'Bucket':'test-private','Key':key})
|
||||
service=Storage();service.put(key,payload);assert service.get(key)==payload;service.delete(key)
|
||||
stub.assert_no_pending_responses()
|
||||
settings.cache_clear(); storage.cache_clear()
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Validate the regression harness; these synthetic images are not Altium references."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[2]
|
||||
|
||||
def test_visual_harness_detects_pixel_changes(tmp_path,ascii_schematic):
|
||||
source=tmp_path/'unit.SchDoc';source.write_bytes(ascii_schematic)
|
||||
first=tmp_path/'initial'
|
||||
command=[sys.executable,str(ROOT/'scripts/validate.py'),str(source)]
|
||||
run=subprocess.run(command+['--width','400','--output',str(first)],capture_output=True,text=True)
|
||||
assert run.returncode==0,run.stderr
|
||||
reference=first/'rendered.png'
|
||||
equal=tmp_path/'equal'
|
||||
run=subprocess.run(command+['--reference',str(reference),'--output',str(equal),'--max-mae','0','--max-changed-fraction','0'],capture_output=True,text=True)
|
||||
assert run.returncode==0,run.stderr
|
||||
report=json.loads((equal/'report.json').read_text())
|
||||
assert report['mae']==0 and report['regression_passed'] and not report['fidelity_verified']
|
||||
image=Image.open(reference).convert('RGB');ImageDraw.Draw(image).rectangle((0,0,80,80),fill='black')
|
||||
altered=tmp_path/'changed.png';image.save(altered)
|
||||
changed=tmp_path/'different'
|
||||
run=subprocess.run(command+['--reference',str(altered),'--output',str(changed)],capture_output=True,text=True)
|
||||
assert run.returncode==1,run.stderr
|
||||
assert not json.loads((changed/'report.json').read_text())['regression_passed']
|
||||
assert (changed/'difference.png').exists() and (changed/'overlay.png').exists()
|
||||
@@ -0,0 +1,24 @@
|
||||
services:
|
||||
api:
|
||||
environment:
|
||||
APP_ENV: production
|
||||
PUBLIC_BASE_URL: https://${DOMAIN:?Set DOMAIN in .env}
|
||||
worker:
|
||||
environment:
|
||||
APP_ENV: production
|
||||
PUBLIC_BASE_URL: https://${DOMAIN:?Set DOMAIN in .env}
|
||||
proxy:
|
||||
image: caddy:2-alpine
|
||||
restart: unless-stopped
|
||||
ports: ["80:80", "443:443"]
|
||||
environment:
|
||||
DOMAIN: ${DOMAIN:?Set DOMAIN in .env}
|
||||
volumes:
|
||||
- ./Caddyfile:/etc/caddy/Caddyfile:ro
|
||||
- caddy_data:/data
|
||||
- caddy_config:/config
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
volumes:
|
||||
caddy_data:
|
||||
caddy_config:
|
||||
@@ -0,0 +1,42 @@
|
||||
x-service: &service
|
||||
build: .
|
||||
image: altium-render-engine:local
|
||||
init: true
|
||||
restart: unless-stopped
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
tmpfs: ["/tmp:size=256m,mode=1777"]
|
||||
mem_limit: 1g
|
||||
pids_limit: 128
|
||||
env_file: .env
|
||||
environment:
|
||||
DATABASE_URL: postgresql+psycopg://altium:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@db:5432/altium
|
||||
STORAGE_DIR: /data
|
||||
volumes: [schematics:/data]
|
||||
depends_on:
|
||||
db: {condition: service_healthy}
|
||||
services:
|
||||
api:
|
||||
<<: *service
|
||||
ports: ["127.0.0.1:8000:8000"]
|
||||
worker:
|
||||
<<: *service
|
||||
command: [python, -m, app.worker]
|
||||
healthcheck: {disable: true}
|
||||
stop_grace_period: 75s
|
||||
db:
|
||||
image: postgres:16-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: altium
|
||||
POSTGRES_DB: altium
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||
volumes: [postgres:/var/lib/postgresql/data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, "pg_isready -U altium -d altium"]
|
||||
interval: 5s
|
||||
retries: 20
|
||||
volumes:
|
||||
postgres:
|
||||
schematics:
|
||||
@@ -0,0 +1,43 @@
|
||||
# REST API contract
|
||||
|
||||
All routes are same-origin by default. Authentication for upload is `X-API-Key`. Private reads accept the same service key or `X-Viewer-Token`, returned only at creation. The service uses one integration key, not multi-tenant user authentication. Use a gateway for organization-specific account roles, key rotation, quotas and rate limits.
|
||||
|
||||
`POST /api/v1/schematics/render` accepts exactly one multipart file field `file`, with `.SchDoc` suffix (case insensitive). The client-provided content type is not trusted. The request body is capped before multipart spooling, and extracted file bytes are separately capped. Initial format identification accepts OLE signature or an ASCII SchDoc header; complete structural/version validation runs asynchronously in an isolated worker.
|
||||
|
||||
Optional fields:
|
||||
|
||||
- `access`: `private` (default) or `unlisted`.
|
||||
- `expires_in`: positive integer seconds, maximum `LINK_TTL_SECONDS`. Expiry starts at upload acceptance, including queue time.
|
||||
|
||||
HTTP 202 includes `id`, `status`, `viewer_url`, `status_url`, `expires_at` (Unix seconds), `access`, and a `viewer_token` for private uploads. `Location` points to status and `Retry-After: 1` provides a polling interval. Do not automatically retry a POST after a transport failure: each accepted request creates a new resource. Idempotency keys are not implemented.
|
||||
|
||||
`GET .../{id}/status` returns HTTP 200 for existing, accessible, unexpired jobs, including failed jobs with `success: false` and a stable `error.code`. Failed artifacts return HTTP 422. Status does not disclose the original filename, source bytes or read token. The returned base `viewer_url` omits the original token: retain the initial URL or append your retained read token as a fragment.
|
||||
|
||||
Completed responses add `schema_version`, `compatibility` (`partial` / `supported_subset`), `record_count`, `component_count`, `primitive_count`, and `diagnostics` (severity, code, message, record id/type, count). Current font metric warnings mean typical documents are explicitly partial. Warnings must be checked by applications where accurate reproduction matters.
|
||||
|
||||
`GET .../{id}/data` returns the document schema from `backend/app/models/schematic.py`:
|
||||
|
||||
- `schema_version`, `document_type`, `id`, `filename`, and `metadata` (source SHA-256, format/header, streams/counts).
|
||||
- `sheet` (original properties, page bounds/background), `bounds` (content bounds).
|
||||
- `components` / `symbols` / `objects` with ownership IDs, placement, raw properties, active visibility, pin anchors and primitive references.
|
||||
- `primitives` in source paint order (paths, text, sanitized embedded rasters).
|
||||
- `diagnostics` with compatibility failures/warnings. Raw properties are data, never executable DOM content.
|
||||
|
||||
Coordinates use 10 mil units, X to the right and Y downward (inverted once at parse time). The page begins at `(0, -height)` and ends at `(width, 0)`. Page dimensions are independent of content bounds. Consumers should not rotate/mirror placed component children again.
|
||||
|
||||
`GET .../{id}/svg` uses the same document primitives and page dimensions as the viewer; it has no interaction code. The interactive `/view/{id}` loads it and changes only its camera viewBox. Wheel zoom retains the point beneath the cursor; pointer events support drag/pinch. Double-click or Home resets fit; +/− and arrows provide keyboard control. Browser zoom, exact font availability and gesture behavior need actual browser validation.
|
||||
|
||||
Error code examples:
|
||||
|
||||
| HTTP | Code |
|
||||
| --- | --- |
|
||||
| 401 | unauthorized |
|
||||
| 404 | not_found (also private permission denied) |
|
||||
| 409 | not_ready |
|
||||
| 410 | link_expired (until retention cleanup) |
|
||||
| 413 | upload_too_large |
|
||||
| 415 | unsupported_file_type |
|
||||
| 422 | invalid_request / invalid_expiry / invalid_schematic / parse_timeout / parse_resource_limit |
|
||||
| 503 | queue_full / storage_unavailable / database_unavailable |
|
||||
|
||||
A worker crash is retried after its lease expires, at most twice total. A parse failure is terminal for that upload. No automatic retry of side-effecting API requests is performed. Data and SVG reads are immutable after a successful attempt; access/expiry remain server-authoritative on every read. HTTP caches are disabled to protect private drawings.
|
||||
@@ -0,0 +1,34 @@
|
||||
# Architecture
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
C[External application] -->|authenticated multipart| A[FastAPI]
|
||||
A -->|generated private key| S[Local or S3 object storage]
|
||||
A -->|durable queued row| D[(PostgreSQL)]
|
||||
W[Independent worker replicas] -->|skip-locked lease| D
|
||||
W -->|bounded input| S
|
||||
W --> P[Resource-limited parser subprocess]
|
||||
P --> N[Native OLE / ASCII decoder]
|
||||
N --> G[Source records + versioned vector primitives]
|
||||
G --> V[SVG serializer]
|
||||
W -->|attempt-scoped JSON and SVG| S
|
||||
W -->|conditional completion| D
|
||||
B[Fullscreen browser] -->|read capability / expiry check| A
|
||||
A -->|JSON / vector artifact| B
|
||||
```
|
||||
|
||||
The API persists the source before admitting a queued database row. Only committed rows are claimable. An advisory lock serializes PostgreSQL admission limits. A failure before a database commit may leave a private orphan file; it cannot be accessed through any API without a row. Production operators should audit/reconcile orphans and configure S3 lifecycle policies.
|
||||
|
||||
The queue is in PostgreSQL, using `FOR UPDATE SKIP LOCKED` and a conditional update on attempt/status. SQLite uses the conditional update for development. A lease lasts the parser timeout plus 60 seconds for process startup/artifact storage. A crashed claim is retried once. Every attempt writes to a different key prefix and updates completion only if its attempt is still current. The physical source is immutable.
|
||||
|
||||
API and worker can scale independently. Worker parsing is serial per worker container; increase replicas for concurrency. Parser subprocesses inherit no API key, database URL or storage credentials, receive files in an ephemeral directory, and run under CPU/address-space/file-size/FD limits. The decoder never fetches document URLs or loads dynamic code. This resource boundary is defense in depth; hostile-file deployments may additionally isolate parsers with a hardened runtime/network namespace.
|
||||
|
||||
Local object paths are application-generated and validated. Files are mode 0600 in directories created as 0700. Compose uses the same persistent volume and UID for API/worker. For multiple hosts replace local storage with an S3-compatible private bucket. Database metadata and objects must be backed up consistently. Credentials are environment settings, not committed files or source-derived values.
|
||||
|
||||
The normalized document keeps original property maps, owner relationships and active parts separately from drawables. This is necessary for diagnostics and future parser improvements: the SVG is an output, not the only stored interpretation. Source paint order is preserved. Geometry is not laid out by a graph library.
|
||||
|
||||
`app/documents.py` registers format adapters. To support PCB later, add a PCB adapter and PCB-specific data/layer types, reuse queue/storage/capability APIs, and provide layer-aware renderer/viewer behavior. The current endpoint intentionally accepts only SchDoc. Existing optional layer/net schema fields reserve extension points; no saved PCB data is fabricated, and no PCB functionality is implemented.
|
||||
|
||||
The viewer is dependency-free JavaScript with a pure tested camera module. It manipulates one SVG viewBox, schedules updates with requestAnimationFrame, and delegates vector rendering to the browser. There is no raster redraw loop or symbol substitution. Large native records are processed off the API process. Browser SVG performance at very large primitive counts has not been benchmarked; viewport culling or a hybrid renderer may be warranted after profiling, while preserving the same primitive model.
|
||||
|
||||
Initial schema creation is serialized on startup. This is schema version 1; database evolution must use explicit reviewed migrations. Healthz checks API/database availability; it does not prove a worker is making progress. Monitor queued age, active leases, failures, storage capacity and replica health in production.
|
||||
@@ -0,0 +1,55 @@
|
||||
# Compatibility and fidelity boundaries
|
||||
|
||||
This service is an independent, read-only native-format implementation. There is no official Altium SDK or Altium executable in its runtime. The current output is explicitly **partial**, with structured diagnostics. It must not be described as pixel-perfect, fully compatible with every Altium version, or certified for engineering signoff.
|
||||
|
||||
## File parsing
|
||||
|
||||
- OLE Compound File Binary containers with a `FileHeader` stream and a recognized format 5.0 header; also native ASCII records for that envelope.
|
||||
- `Storage` is decoded only for bounded, supported embedded raster assets. `Additional` records preserve stream-local indexes. Ambiguous cross-stream ownership is reported instead of inferred.
|
||||
- Little-endian length/kind record framing, original physical record indexes, source properties, UTF-8 companion properties, legacy Windows-1252 decoding.
|
||||
- Unknown binary payloads are preserved as base64, with diagnostics, rather than misinterpreted as text.
|
||||
- Major unknown headers, missing sheets, broken CFB/record lengths and files without supported geometry fail. Minor header revisions are reported, not treated as certified Altium application releases.
|
||||
- `.PcbDoc`, libraries without an instantiated schematic, encrypted/protected variants and unknown document envelopes are not supported.
|
||||
|
||||
## Geometry
|
||||
|
||||
Original component symbol primitives are extracted from the SchDoc itself. The renderer does not replace resistors, ICs or other components with generic icons. Ownership chains, active units/display modes, placements and fractional coordinates are retained. Placed children are not rotated a second time. Component rotation/mirroring remains in JSON; readable parameter glyphs do not inherit a placement mirror.
|
||||
|
||||
| Record IDs | Rendering behavior |
|
||||
| --- | --- |
|
||||
| 1, 2 | Component ownership, part selection; original pin segments, names/numbers; common bubble/clock decorations |
|
||||
| 4, 25, 32–34, 41 | Native labels, designators, parameter text and resolved stored parameter references |
|
||||
| 5–14 | Cubic Bezier sequences, polylines, polygons, ellipses, pie sectors, rounded rectangles, elliptical/circular arcs, lines, rectangles |
|
||||
| 15, 16 | Original sheet-symbol box and entry coordinates/names; entry arrow styles are incomplete and reported |
|
||||
| 17, 18 | Common power and off-sheet port styles using native style fields; unknown power markers omitted with diagnostics |
|
||||
| 22, 26, 27, 29, 37 | No-connect marks, bus/wire vertices, junction dots and bus entry lines |
|
||||
| 28, 209 | Text boxes, newlines, clipping, estimated wrapping (warning) |
|
||||
| 30 | Embedded BMP/PNG/JPEG/TIFF/GIF raster re-encoded as PNG after bounded decoding; multi-frame assets skipped |
|
||||
| 31 | Page background, 18 standard page presets, custom sizes and portrait orientation |
|
||||
| 39, 44–48 | Original template/model metadata retained; no library/resource execution |
|
||||
| 225 | Dashed annotation rectangle from observed records; not an official format guarantee |
|
||||
| Other | Preserved original properties and explicit unsupported-object diagnostics |
|
||||
|
||||
Source paint order is preserved. Explicit saved template lines/text render like other primitives. No schematic layout algorithm is used, and crossings do not fabricate junctions.
|
||||
|
||||
## Known visual gaps
|
||||
|
||||
1. **Automatic page decorations**: Altium-generated borders, zones and title blocks are not reconstructed. Explicit saved template primitives do render. Their omission is reported when enabled.
|
||||
2. **Typography**: Original font files and suitable web distribution licenses are required for exact glyph shapes. The optional administrator font allowlist supports regular/bold/italic variants. Browser fallback changes metrics. Font em conversion (currently 0.9 × stored size), default pin label offsets, baseline mapping and estimated wrapping require calibration against Altium reference exports. Font and pin metrics are always warned about.
|
||||
3. **Special text**: Per-character overbars, some custom text positioning, advanced compiled expressions, locale code pages other than Windows-1252/UTF-8 and unusual glyph encodings remain incomplete. Stored parameter references can resolve against extracted values; dates/times and missing data are not fabricated.
|
||||
4. **Decorations/styles**: IEEE symbols, uncommon pin decorations, line-end arrow decorations, some power marker styles, sheet-entry arrows and harness objects are not fully rendered. Unsupported power styles are omitted instead of replaced by another ground/power symbol. Common generated marker proportions and port text need reference validation. Non-solid line styles currently use a common dash sequence and are not certified to match every Altium dash pattern.
|
||||
5. **Images**: Only embedded, recognized raster files are decoded. Source filenames are treated solely as matching keys, never filesystem paths or network resources. Missing assets, multi-frame files, corrupt compression, pixel/decompression limits and ambiguous names produce warnings. No vector image/EMF/WMF conversion is provided.
|
||||
6. **Electrical scope**: Wire geometry, junctions and connection anchors are extracted; no Altium compiled netlist is created. Cross-sheet connectivity, repeated channels, bus expansion and harness semantics are not resolved. Hierarchical drawing objects on a single uploaded sheet are retained; child files are not fetched.
|
||||
7. **Layer order/performance**: Source primitive order is retained, but all proprietary editor-specific draw-order exceptions have not been characterized. Very large browser SVG scenes and mobile gestures require profiling on target devices.
|
||||
|
||||
An omitted object can be important. Applications must inspect `diagnostics`, not just `status: completed`. Original source bytes and raw properties are kept to support future decoder improvements.
|
||||
|
||||
## Proprietary dependencies and licensing
|
||||
|
||||
No proprietary component library is needed when a SchDoc contains instantiated symbol primitives. A document missing required saved geometry cannot be faithfully reconstructed without its original libraries or a licensed Altium workflow. This service does not invent that geometry. Exact fonts, external templates and absent graphics must be supplied by their owners under suitable licenses. Universal reference rendering may require licensed Altium Designer or an officially supported Altium rendering/export service; no such service is configured or impersonated here.
|
||||
|
||||
## Format research
|
||||
|
||||
Record/coordinate facts were checked against the [KiCad developer format reference](https://dev-docs.kicad.org/en/import-formats/altium/index.html) and [python-altium format notes](https://github.com/vadmium/python-altium/blob/master/format.md). The [Altium drawing-object documentation](https://www.altium.com/documentation/altium-designer/schematic/drawing-objects) and [connectivity documentation](https://www.altium.com/documentation/altium-designer/schematic/creating-circuit-connectivity) describe native graphic objects and power-port variations. These references do not constitute an official binary-format guarantee.
|
||||
|
||||
The initial independently written schematic decoding modules were reused from the user's local `/home/nearology/work/altium-render` checkout and extended here; that checkout was not modified. No GPL importer implementation has been copied into this project.
|
||||
@@ -0,0 +1,39 @@
|
||||
# Rendering validation and release gate
|
||||
|
||||
Passing a parser test is not proof that a schematic matches Altium. The service separates format acceptance, supported-subset decoding, SVG generation, interactive behavior and trusted-reference fidelity.
|
||||
|
||||
## Tests
|
||||
|
||||
`./scripts/check.sh` runs bounds/framing/version/ownership/coordinate/part visibility tests, native-file integration when fixtures are present, safe SVG serialization, embedded raster expansion controls, upload/authentication/expiry/status/error flows, subprocess worker execution, queue capacity/claims/lease recovery, stale-worker fencing, expired artifact cleanup, and a botocore-validated S3 contract. Node's built-in runner checks sheet fit, camera invariants, point-under-cursor zoom and scale limits without frontend dependencies.
|
||||
|
||||
`python scripts/fetch-fixture.py` downloads an actual native OLE fixture from KiCad QA and enforces SHA-256 `e7dc42a4e983e6b716f6883eb0863fe47164d941636e11b1f2b5ec5d0143dca4`. It remains ignored and is not redistributed in Docker/source. `SCHDOC_SAMPLE` enables an additional actual design. Missing native files cause explicit skipped tests; they are not replaced by synthetic circuits. CI fetches the real fixture before tests.
|
||||
|
||||
`python scripts/corpus.py DIRECTORY` audits every SchDoc in a directory and writes source hashes, record counts, SVG sizes, failures and diagnostics. This detects format regressions across an actual design corpus without claiming visual equivalence.
|
||||
|
||||
## Trusted-reference procedure
|
||||
|
||||
1. Select representative designs containing rotated/mirrored and multi-part components; fractional placements; all pin orientations/decorations; buses/net labels; hierarchical ports; curves/fills; text/font variants; images; standard/custom/portrait sheets; dense and large pages. Include actual supported Altium versions and retain source SHA-256 values.
|
||||
2. In licensed Altium Designer, open each source and export a full-sheet reference PDF or PNG. Preserve original colors, dimensions and clipping. Record the exact Altium version, document settings, font versions, export options, page dimensions and export hash. Do not use a screenshot with toolbars or an arbitrary zoom/crop.
|
||||
3. Install licensed matching fonts on the validation host and configure equivalent webfonts for the browser. Rasterize PDF references with `pdftoppm` when necessary, retaining DPI/page size. No image-recognition step replaces parsing.
|
||||
4. Run `.venv/bin/python scripts/validate.py design.SchDoc --reference reference.png --output validation-output/design`. The render uses exactly the reference pixel dimensions and rejects incompatible aspect ratios. It does not fit to content or align away positional differences.
|
||||
5. Review `rendered.png`, `drawing.svg`, `difference.png`, `overlay.png` and `report.json`. The tool reports mean absolute RGB error (0–255) and the fraction of pixels whose maximum channel difference exceeds a tolerance. Default thresholds are MAE ≤ 2, changed fraction ≤ 1%, channel tolerance 12; tune and freeze them against accepted references, not failing candidates. It exits nonzero when thresholds fail. White/background-heavy images can hide local errors in global metrics, so manual/local-region review is mandatory.
|
||||
6. Compare native object anchors, pin endpoints, component symbol bounds, page dimensions, stroke widths, colors, rotations and textual baselines against measured reference values. Assert these coordinates independently in unit tests. Visually inspect actual browser renders, not only Cairo output, across target browser/OS/font combinations.
|
||||
7. Check initial sheet fit, drag, wheel, touch pinch, resize/aspect ratio, long-lived expiry, load/error states and a dense-file performance budget in connected desktop/mobile browsers. Loaded viewers must contain no toolbar, information panel, header or footer.
|
||||
8. Accept a format/version only after geometry checks, trusted visual comparisons and browser checks pass. Keep known failures as explicit compatibility restrictions. Pixel similarity alone does not prove electrical equivalence or universal pixel-perfect compatibility.
|
||||
|
||||
Even a passing comparison records `fidelity_verified: false`: automated image thresholds are a regression signal, not an automatic Altium-certification flag. Human review and the chosen supported-version corpus define a release's acceptance scope. Regenerate references only through a reviewed Altium export workflow.
|
||||
|
||||
## Evidence from this implementation session (2026-11-05)
|
||||
|
||||
- Final test run: 73 Python tests and 3 camera tests passed, including the native fixtures and a regression-harness test that detects altered pixels.
|
||||
- 13 actual local LTE 6 SchDoc files parsed and serialized to SVG without fatal errors; results and per-file diagnostics are in ignored `.runtime/corpus.json`. Counts range from 81 to 5,342 records. This is format coverage evidence, not visual acceptance.
|
||||
- The CPU sheet parsed 1,942 records and 25 components into 1,002 drawables; source SHA-256 `d6a73f11a4239eaf7e8f280ed35b82e9fd497a4680a60027cc93f266c5c006ae`.
|
||||
- The public Channel fixture parsed 99 records and 2 components into 34 drawables.
|
||||
- Both native sheets were rasterized with CairoSVG; the CPU PNG was visually inspected. Inspection found parameter glyph mirroring, which was corrected and regression-tested.
|
||||
- Eight simultaneous actual HTTP uploads to PostgreSQL produced eight unique IDs and all completed; this also passed with two worker replicas.
|
||||
- A built, non-root, read-only Compose API/worker with actual PostgreSQL completed the CPU upload/status/data/SVG flow over HTTP; private reads without a token returned 404. The published socket was confirmed at `127.0.0.1:8000`.
|
||||
- S3 request/response contracts are tested with botocore stubs; a live S3 provider has not been exercised.
|
||||
- No connected browser was available, so actual wheel/drag/touch behavior and browser font appearance remain unverified.
|
||||
- No trusted Altium export was supplied or available in this session. No reference comparison, visual equivalence, Altium release certification or pixel-perfect claim has been made.
|
||||
|
||||
The implementation is operationally testable; its known rendering gaps must be resolved and reference-tested before a production deployment requiring faithful Altium reproduction.
|
||||
@@ -0,0 +1,20 @@
|
||||
/** SVG viewBox camera. All pointer math is in viewport pixels / document units. */
|
||||
export class Camera {
|
||||
constructor(bounds, width, height) { this.bounds = bounds; this.resize(width, height); this.fit(); }
|
||||
resize(width, height) { this.width = Math.max(1, width); this.height = Math.max(1, height); }
|
||||
fit() {
|
||||
const b = this.bounds;
|
||||
this.scale = Math.min(this.width / b.width, this.height / b.height);
|
||||
this.fitScale = this.scale;
|
||||
this.x = (this.width - b.width * this.scale) / 2 - b.x * this.scale;
|
||||
this.y = (this.height - b.height * this.scale) / 2 - b.y * this.scale;
|
||||
}
|
||||
world(x, y) { return { x: (x - this.x) / this.scale, y: (y - this.y) / this.scale }; }
|
||||
zoom(factor, x, y) {
|
||||
const p = this.world(x, y);
|
||||
this.scale = Math.min(this.fitScale * 200, Math.max(this.fitScale * .05, this.scale * factor));
|
||||
this.x = x - p.x * this.scale; this.y = y - p.y * this.scale;
|
||||
}
|
||||
pan(x, y) { this.x += x; this.y += y; }
|
||||
get viewBox() { return `${-this.x / this.scale} ${-this.y / this.scale} ${this.width / this.scale} ${this.height / this.scale}`; }
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { Camera } from './camera.js';
|
||||
const close = (a,b) => assert.ok(Math.abs(a-b)<1e-8,`${a} != ${b}`);
|
||||
test('fit preserves sheet aspect ratio without padding',()=> {
|
||||
const c = new Camera({x:0,y:-760,width:1150,height:760},1920,1080);
|
||||
close(c.scale,1080/760); close(c.y,1080);
|
||||
const center = c.world(960,540); close(center.x,575); close(center.y,-380);
|
||||
});
|
||||
test('zoom retains exact world coordinate under cursor',()=> {
|
||||
const c = new Camera({x:0,y:-760,width:1150,height:760},1000,800);
|
||||
const p = c.world(121,534); c.zoom(3.2,121,534); const q = c.world(121,534);
|
||||
close(p.x,q.x); close(p.y,q.y); c.pan(12,-8); const r=c.world(133,526); close(r.x,p.x); close(r.y,p.y);
|
||||
});
|
||||
test('zoom is bounded for hostile wheel deltas',()=> {
|
||||
const c = new Camera({x:0,y:0,width:100,height:100},1000,800);
|
||||
c.zoom(1e30,0,0); close(c.scale,c.fitScale*200);
|
||||
c.zoom(1e-30,0,0); close(c.scale,c.fitScale*.05);
|
||||
});
|
||||
@@ -0,0 +1 @@
|
||||
<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Schematic API reference</title><link rel="stylesheet" href="/assets/upload.css"></head><body><main><p class="eyebrow">API V1</p><h1>Schematic rendering API</h1><p>Authenticate uploads with <code>X-API-Key</code>. Native binary OLE and ASCII SchDoc format 5.0 are accepted. Files are parsed asynchronously; HTTP 202 confirms admission, not rendering fidelity.</p><h2>Upload</h2><p><code>POST /api/v1/schematics/render</code></p><p>Multipart fields: <code>file</code> (required), <code>access</code> (private or unlisted; defaults to private), <code>expires_in</code> (optional seconds, bounded by server policy).</p><h2>Poll</h2><p><code>GET /api/v1/schematics/{id}/status</code></p><p>States: queued, processing, completed, failed. Private reads require <code>X-Viewer-Token</code> from the upload response, or the service API key. Completed responses include compatibility and diagnostics.</p><h2>Retrieve</h2><p><code>GET /api/v1/schematics/{id}/data</code> returns the versioned structured document. <code>GET /api/v1/schematics/{id}/svg</code> returns the generated vector drawing.</p><h2>View</h2><p>Open the exact <code>viewer_url</code> from the upload response. The private token is in its fragment. It is stored for this browser tab and sent only as a request header. The viewer has no panels or toolbar.</p><h2>Errors</h2><p>JSON errors include <code>success: false</code> and <code>error.code</code> / <code>error.message</code>. Common HTTP codes: 401 invalid service key; 404 missing or inaccessible schematic; 409 still processing; 410 expired; 413 too large; 415 wrong extension; 422 invalid schematic; 503 storage unavailable or queue full.</p><p><a href="/openapi.json">Download OpenAPI schema</a> · <a href="/">Upload a schematic</a></p></main></body></html>
|
||||
@@ -0,0 +1,2 @@
|
||||
<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Altium schematic upload</title><link rel="stylesheet" href="/assets/upload.css"><script type="module" src="/assets/upload.js"></script></head><body><main><p class="eyebrow">NATIVE ALTIUM · SCHDOC</p><h1>Open your schematic.</h1><p>Upload a native schematic to create a private, expiring viewer link.</p><form id="upload"><label for="key">API key</label><input id="key" type="password" autocomplete="off" placeholder="Configured service key"><label for="file">Schematic file</label><input id="file" type="file" accept=".SchDoc" required><button type="submit">Render schematic <span aria-hidden="true">↗</span></button></form><output id="result" role="status"></output><p class="note">Compatibility diagnostics are returned by the API. Review them before relying on visual fidelity.</p><a href="/docs">API reference</a></main></body></html>
|
||||
@@ -0,0 +1,15 @@
|
||||
:root { font-family: system-ui,sans-serif; color: #182f31; background: #f5f6f3; color-scheme: light; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; min-height: 100dvh; display: grid; place-items: center; padding: 28px; }
|
||||
main { width: min(100%,500px); }
|
||||
.eyebrow { color: #51706d; letter-spacing: .16em; font-size: 11px; font-weight: 700; }
|
||||
h1 { font-size: clamp(30px,6vw,42px); letter-spacing: -.04em; margin: 16px 0; }
|
||||
p { font-size: 15px; line-height: 1.6; color: #526462; }
|
||||
form { margin: 28px 0; display: grid; gap: 10px; }
|
||||
label { font-size: 13px; margin-top: 10px; }
|
||||
input { padding: 13px; border: 1px solid #ccd4ce; background: white; font: inherit; border-radius: 5px; width: 100%; }
|
||||
button { display: flex; justify-content: space-between; margin-top: 12px; padding: 16px; background: #244d49; color: white; border: none; border-radius: 5px; font: inherit; cursor: pointer; }
|
||||
button:disabled { opacity: .5; cursor: wait; }
|
||||
output { display: block; font-size: 14px; overflow-wrap: anywhere; line-height: 1.6; }
|
||||
a { color: #244d49; font-size: 13px; }
|
||||
.note { margin-top: 28px; font-size: 12px; }
|
||||
@@ -0,0 +1,16 @@
|
||||
const form = document.querySelector('#upload'), result = document.querySelector('#result');
|
||||
form.addEventListener('submit',async e => {
|
||||
e.preventDefault();
|
||||
const button = form.querySelector('button'); button.disabled = true; result.textContent = 'Uploading…';
|
||||
const headers = {}, key = document.querySelector('#key').value;
|
||||
if (key) headers['X-API-Key'] = key;
|
||||
const data = new FormData(); data.append('file',document.querySelector('#file').files[0]);
|
||||
try {
|
||||
const response = await fetch('/api/v1/schematics/render',{ method:'POST',headers,body:data });
|
||||
const body = await response.json();
|
||||
if (!response.ok) throw new Error(body.error?.message || 'Upload failed');
|
||||
result.textContent = 'Queued. ';
|
||||
const link = document.createElement('a'); link.href = body.viewer_url; link.textContent = 'Open schematic'; result.append(link);
|
||||
} catch (e) { result.textContent = e.message; }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
:root { color-scheme: light; font-family: Arial,sans-serif; }
|
||||
* { box-sizing: border-box; }
|
||||
html, body, #viewport { width: 100%; height: 100%; margin: 0; overflow: hidden; }
|
||||
#viewport { touch-action: none; user-select: none; cursor: grab; outline: none; background: #fff; }
|
||||
#viewport:active { cursor: grabbing; }
|
||||
#viewport > svg { display: block; width: 100%; height: 100%; overflow: hidden; }
|
||||
#state { position: fixed; left: 50%; top: 50%; transform: translate(-50%,-50%); color: #333; font-size: 14px; text-align: center; }
|
||||
#state:empty { display: none; }
|
||||
@@ -0,0 +1,2 @@
|
||||
<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1"><title>Schematic</title><link rel="stylesheet" href="/fonts.css"><link rel="stylesheet" href="/assets/viewer.css"><script type="module" src="/assets/viewer.js"></script></head><body><main id="viewport" aria-label="Schematic viewer" tabindex="0"></main><output id="state" role="status"></output></body></html>
|
||||
@@ -0,0 +1,96 @@
|
||||
import { Camera } from './camera.js';
|
||||
const viewport = document.querySelector('#viewport');
|
||||
const state = document.querySelector('#state');
|
||||
const id = location.pathname.split('/').pop();
|
||||
const fragment = new URLSearchParams(location.hash.slice(1));
|
||||
const token = fragment.get('token') || sessionStorage.getItem(`schematic:${id}`);
|
||||
if (token) sessionStorage.setItem(`schematic:${id}`, token);
|
||||
if (location.hash) history.replaceState(null, '', location.pathname);
|
||||
const headers = token ? { 'X-Viewer-Token': token } : {};
|
||||
const controller = new AbortController();
|
||||
addEventListener('pagehide', () => controller.abort(), { once: true });
|
||||
const base = `/api/v1/schematics/${id}`;
|
||||
async function request(path, raw = false) {
|
||||
const r = await fetch(base + path, { headers, signal: controller.signal, cache: 'no-store' });
|
||||
if (!r.ok) { const body = await r.json(); throw new Error(body.error?.message || 'Schematic unavailable'); }
|
||||
return raw ? r.text() : r.json();
|
||||
}
|
||||
async function start() {
|
||||
state.textContent = 'Rendering…';
|
||||
let result;
|
||||
for (;;) {
|
||||
result = await request('/status');
|
||||
if (result.status === 'completed') break;
|
||||
if (result.status === 'failed') throw new Error(result.error?.message || 'Rendering failed');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
const text = await request('/svg', true);
|
||||
const parsed = new DOMParser().parseFromString(text, 'image/svg+xml');
|
||||
if (parsed.querySelector('parsererror') || parsed.documentElement.localName !== 'svg') throw new Error('Invalid rendering artifact');
|
||||
const svg = document.importNode(parsed.documentElement, true);
|
||||
const b = svg.viewBox.baseVal;
|
||||
const camera = new Camera({ x: b.x, y: b.y, width: b.width, height: b.height }, viewport.clientWidth, viewport.clientHeight);
|
||||
const background = svg.querySelector('rect')?.getAttribute('fill') || '#ffffff';
|
||||
// A rect spanning the camera is the original sheet color, not an added grid/UI.
|
||||
const backdrop = svg.querySelector('rect');
|
||||
let frame = 0;
|
||||
function paint() {
|
||||
if (frame) return;
|
||||
frame = requestAnimationFrame(() => {
|
||||
frame = 0;
|
||||
svg.setAttribute('viewBox', camera.viewBox);
|
||||
const [x,y,w,h] = camera.viewBox.split(' ').map(Number);
|
||||
if (backdrop) for (const [key,value] of Object.entries({ x,y,width:w,height:h })) backdrop.setAttribute(key,value);
|
||||
});
|
||||
}
|
||||
viewport.append(svg);
|
||||
await document.fonts.ready;
|
||||
viewport.style.backgroundColor = background;
|
||||
state.textContent = '';
|
||||
paint();
|
||||
viewport.addEventListener('wheel', e => {
|
||||
e.preventDefault();
|
||||
const delta = e.deltaY * (e.deltaMode === 1 ? 16 : e.deltaMode === 2 ? viewport.clientHeight : 1);
|
||||
camera.zoom(Math.exp(-Math.max(-400, Math.min(400, delta)) * .002), e.clientX, e.clientY); paint();
|
||||
}, { passive: false });
|
||||
const pointers = new Map();
|
||||
const metrics = () => {
|
||||
const p = [...pointers.values()];
|
||||
const x = p.reduce((s,p) => s+p.x,0)/p.length, y = p.reduce((s,p) => s+p.y,0)/p.length;
|
||||
return { x, y, distance: p.length > 1 ? Math.hypot(p[0].x-p[1].x,p[0].y-p[1].y) : 0 };
|
||||
};
|
||||
viewport.addEventListener('pointerdown', e => {
|
||||
if (e.button !== 0) return;
|
||||
viewport.setPointerCapture(e.pointerId); pointers.set(e.pointerId, { x:e.clientX,y:e.clientY });
|
||||
});
|
||||
viewport.addEventListener('pointermove', e => {
|
||||
if (!pointers.has(e.pointerId)) return;
|
||||
const before = metrics(); pointers.set(e.pointerId,{ x:e.clientX,y:e.clientY }); const after = metrics();
|
||||
if (before.distance > 0 && after.distance > 0) camera.zoom(after.distance/before.distance,before.x,before.y);
|
||||
camera.pan(after.x-before.x,after.y-before.y); paint();
|
||||
});
|
||||
for (const type of ['pointerup','pointercancel','lostpointercapture']) viewport.addEventListener(type,e => pointers.delete(e.pointerId));
|
||||
viewport.addEventListener('dblclick', () => { camera.fit(); paint(); });
|
||||
viewport.addEventListener('keydown', e => {
|
||||
if (e.key === 'Home' || e.key === '0') camera.fit();
|
||||
else if (e.key === '+' || e.key === '=') camera.zoom(1.25,camera.width/2,camera.height/2);
|
||||
else if (e.key === '-') camera.zoom(.8,camera.width/2,camera.height/2);
|
||||
else if (e.key.startsWith('Arrow')) camera.pan(e.key==='ArrowLeft'?40:e.key==='ArrowRight'?-40:0,e.key==='ArrowUp'?40:e.key==='ArrowDown'?-40:0);
|
||||
else return;
|
||||
e.preventDefault(); paint();
|
||||
});
|
||||
new ResizeObserver(() => {
|
||||
const center = camera.world(camera.width/2,camera.height/2);
|
||||
camera.resize(viewport.clientWidth,viewport.clientHeight);
|
||||
camera.fitScale = Math.min(camera.width/camera.bounds.width,camera.height/camera.bounds.height);
|
||||
camera.x = camera.width/2-center.x*camera.scale; camera.y = camera.height/2-center.y*camera.scale; paint();
|
||||
}).observe(viewport);
|
||||
// Enforce configured expiry even when a viewer remains open.
|
||||
function expire() {
|
||||
const remaining = result.expires_at*1000-Date.now();
|
||||
if (remaining <= 0) { viewport.replaceChildren(); state.textContent = 'This schematic link has expired'; }
|
||||
else setTimeout(expire,Math.min(2147483647,remaining));
|
||||
}
|
||||
expire();
|
||||
}
|
||||
start().catch(e => { if (e.name !== 'AbortError') state.textContent = e.message; });
|
||||
@@ -0,0 +1 @@
|
||||
{"name":"native-altium-viewer","private":true,"type":"module","scripts":{"test":"node --test frontend/*.test.js"}}
|
||||
Executable
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
PYTHONPATH=backend .venv/bin/python -m pytest backend/tests -q
|
||||
npm test
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Concurrent actual HTTP uploads; run only against a disposable QA deployment."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
import httpx
|
||||
p=argparse.ArgumentParser();p.add_argument('file',type=Path);p.add_argument('--url',default='http://127.0.0.1:8000');p.add_argument('--count',type=int,default=8);a=p.parse_args()
|
||||
raw=a.file.read_bytes();key=os.environ['API_KEY']
|
||||
|
||||
def one(_):
|
||||
with httpx.Client(base_url=a.url,timeout=30) as client:
|
||||
response=client.post('/api/v1/schematics/render',files={'file':(a.file.name,raw)},headers={'X-API-Key':key})
|
||||
assert response.status_code==202,response.text
|
||||
return response.json()
|
||||
with ThreadPoolExecutor(max_workers=a.count) as pool: jobs=list(pool.map(one,range(a.count)))
|
||||
assert len({job['id'] for job in jobs})==a.count
|
||||
with httpx.Client(base_url=a.url,timeout=30) as client:
|
||||
for job in jobs:
|
||||
deadline=time.monotonic()+90
|
||||
while time.monotonic()<deadline:
|
||||
body=client.get(f"/api/v1/schematics/{job['id']}/status",headers={'X-Viewer-Token':job['viewer_token']}).json()
|
||||
if body['status'] in ('completed','failed'):break
|
||||
time.sleep(.2)
|
||||
assert body['status']=='completed',body
|
||||
print(json.dumps({'concurrent_uploads':a.count,'unique_ids':a.count,'completed':a.count}))
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit a directory of real inputs without copying design files into source."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'backend'))
|
||||
from app.parser.schdoc_parser import parse_schdoc
|
||||
from app.render import svg_document
|
||||
p=argparse.ArgumentParser();p.add_argument('directory',type=Path);p.add_argument('--output',type=Path,default=Path('validation-output/corpus.json'));a=p.parse_args()
|
||||
results=[]
|
||||
for path in sorted(a.directory.glob('*')):
|
||||
if path.suffix.lower()!='.schdoc': continue
|
||||
raw=path.read_bytes()
|
||||
result={'filename':path.name,'sha256':hashlib.sha256(raw).hexdigest()}
|
||||
try:
|
||||
doc=parse_schdoc(raw,path.name,'audit'); svg=svg_document(doc)
|
||||
result.update(status='parsed',record_count=doc.metadata['record_count'],primitive_count=len(doc.primitives),component_count=len(doc.components),svg_bytes=len(svg),diagnostics=[d.model_dump() for d in doc.diagnostics])
|
||||
except Exception as e: result.update(status='failed',error=str(e))
|
||||
results.append(result)
|
||||
a.output.parent.mkdir(parents=True,exist_ok=True);a.output.write_text(json.dumps(results,indent=2))
|
||||
print(json.dumps({'files':len(results),'parsed':sum(r['status']=='parsed' for r in results),'failed':sum(r['status']=='failed' for r in results),'output':str(a.output)}))
|
||||
sys.exit(1 if not results or any(r['status']=='failed' for r in results) else 0)
|
||||
Executable
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
export PYTHONPATH="$PWD/backend"
|
||||
if [[ -f .env ]]; then set -a; source .env; set +a; fi
|
||||
.venv/bin/python -m app.worker &
|
||||
worker_pid=$!
|
||||
trap 'kill "$worker_pid" 2>/dev/null || true' EXIT INT TERM
|
||||
.venv/bin/python -m uvicorn app.main:app --host 127.0.0.1 --port 8000 --no-access-log
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fetch a real upstream native SchDoc; checksum-pin it and never redistribute."""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from urllib.request import urlopen
|
||||
URL='https://raw.githubusercontent.com/KiCad/kicad-source-mirror/master/qa/data/eeschema/plugins/altium/issue24861/Channel.SchDoc'
|
||||
SHA='e7dc42a4e983e6b716f6883eb0863fe47164d941636e11b1f2b5ec5d0143dca4'
|
||||
root=Path(__file__).resolve().parents[1]
|
||||
target=root/'.runtime/fixtures/Channel.SchDoc'
|
||||
if not target.exists() or hashlib.sha256(target.read_bytes()).hexdigest()!=SHA:
|
||||
with urlopen(URL,timeout=30) as response: data=response.read(4*1024*1024)
|
||||
if hashlib.sha256(data).hexdigest()!=SHA: raise SystemExit('Upstream fixture changed; review before updating expected hash')
|
||||
target.parent.mkdir(parents=True,exist_ok=True); target.write_bytes(data)
|
||||
print(target)
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise the actual HTTP API and worker using an actual native file."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
import httpx
|
||||
|
||||
p=argparse.ArgumentParser()
|
||||
p.add_argument('file',type=Path)
|
||||
p.add_argument('--url',default='http://127.0.0.1:8000')
|
||||
a=p.parse_args()
|
||||
key=os.getenv('API_KEY','')
|
||||
with httpx.Client(base_url=a.url,timeout=30) as client:
|
||||
response=client.post('/api/v1/schematics/render',files={'file':(a.file.name,a.file.read_bytes())},headers={'X-API-Key':key})
|
||||
assert response.status_code==202,response.text
|
||||
body=response.json(); id=body['id']; headers={'X-Viewer-Token':body['viewer_token']}
|
||||
base=f'/api/v1/schematics/{id}'
|
||||
assert client.get(base+'/data').status_code==404
|
||||
deadline=time.monotonic()+90
|
||||
while time.monotonic()<deadline:
|
||||
status=client.get(base+'/status',headers=headers).json()
|
||||
if status['status'] in ('completed','failed'): break
|
||||
time.sleep(.5)
|
||||
assert status['status']=='completed',status
|
||||
data=client.get(base+'/data',headers=headers)
|
||||
assert data.status_code==200 and data.json()['metadata']['format']=='ole'
|
||||
svg=client.get(base+'/svg',headers=headers)
|
||||
assert svg.status_code==200 and '<svg' in svg.text
|
||||
assert client.get('/view/'+id).status_code==200
|
||||
assert client.get('/docs').status_code==200
|
||||
print(json.dumps({'id':id,'status':status['status'],'compatibility':status['compatibility'],
|
||||
'records':status['record_count'],'primitives':status['primitive_count'],
|
||||
'private_access_enforced':True,'svg_bytes':len(svg.content)},indent=2))
|
||||
Executable
+61
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render actual SchDoc and compare to a trusted Altium page PNG, without alignment.
|
||||
|
||||
Images must represent the same sheet viewport. No registration, recoloring or
|
||||
cropping hides coordinate differences. Altium PDF exports can be rasterized to
|
||||
PNG with pdftoppm first. This tool does not manufacture reference images.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'backend'))
|
||||
import cairosvg
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from app.parser.schdoc_parser import parse_schdoc
|
||||
from app.render import svg_document
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('schematic',type=Path)
|
||||
p.add_argument('--reference',type=Path)
|
||||
p.add_argument('--output',type=Path,default=Path('validation-output'))
|
||||
p.add_argument('--width',type=int,default=2000)
|
||||
p.add_argument('--max-mae',type=float,default=2.0,help='Mean absolute RGB error threshold, 0..255')
|
||||
p.add_argument('--max-changed-fraction',type=float,default=.01)
|
||||
p.add_argument('--pixel-tolerance',type=int,default=12)
|
||||
args=p.parse_args(); args.output.mkdir(parents=True,exist_ok=True)
|
||||
raw=args.schematic.read_bytes(); doc=parse_schdoc(raw,args.schematic.name,'validation')
|
||||
svg=svg_document(doc); (args.output/'drawing.svg').write_bytes(svg)
|
||||
width=args.width; height=max(1,round(width*doc.sheet.bounds.height/doc.sheet.bounds.width))
|
||||
if args.reference:
|
||||
with Image.open(args.reference) as ref: width,height=ref.size
|
||||
expected_ratio=doc.sheet.bounds.width/doc.sheet.bounds.height
|
||||
if abs(width/height/expected_ratio-1)>.005:
|
||||
p.error('Reference aspect ratio differs from original sheet; export the same viewport')
|
||||
png=cairosvg.svg2png(bytestring=svg,output_width=width,output_height=height)
|
||||
(args.output/'rendered.png').write_bytes(png)
|
||||
report={'source_sha256':hashlib.sha256(raw).hexdigest(),'record_count':doc.metadata['record_count'],
|
||||
'component_count':len(doc.components),'primitive_count':len(doc.primitives),
|
||||
'diagnostics':[d.model_dump() for d in doc.diagnostics],
|
||||
'reference_compared':bool(args.reference),'fidelity_verified':False}
|
||||
passed=True
|
||||
if args.reference:
|
||||
reference=Image.open(args.reference).convert('RGB'); actual=Image.open(args.output/'rendered.png').convert('RGB')
|
||||
a=np.array(actual,dtype=np.int16); b=np.array(reference,dtype=np.int16); delta=np.abs(a-b)
|
||||
mae=float(delta.mean()); changed=float((delta.max(axis=2)>args.pixel_tolerance).mean())
|
||||
heat=np.zeros_like(a,dtype=np.uint8); heat[:,:,0]=np.clip(delta.max(axis=2)*4,0,255).astype(np.uint8)
|
||||
Image.fromarray(heat).save(args.output/'difference.png')
|
||||
Image.blend(reference,actual,.5).save(args.output/'overlay.png')
|
||||
passed=mae<=args.max_mae and changed<=args.max_changed_fraction
|
||||
report.update(reference_sha256=hashlib.sha256(args.reference.read_bytes()).hexdigest(),mae=mae,
|
||||
changed_pixel_fraction=changed,thresholds={'max_mae':args.max_mae,'max_changed_fraction':args.max_changed_fraction,'pixel_tolerance':args.pixel_tolerance},
|
||||
regression_passed=passed)
|
||||
(args.output/'report.json').write_text(json.dumps(report,indent=2))
|
||||
print(json.dumps({k:v for k,v in report.items() if k!='diagnostics'},indent=2))
|
||||
return 0 if passed else 1
|
||||
|
||||
if __name__=='__main__': sys.exit(main())
|
||||
مرجع در شماره جدید
مسدود کردن کاربر