diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38b8c2837..b916cad82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,33 @@ jobs: - run: bun install --frozen-lockfile - run: bun run --filter @hyperframes/core test:hyperframe-runtime-ci + native-renderer: + name: "Test: native renderer" + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - uses: oven-sh/setup-bun@v2 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - uses: dtolnay/rust-toolchain@stable + - name: Install FFmpeg + run: sudo apt-get update && sudo apt-get install -y ffmpeg + - run: bun install --frozen-lockfile + - run: cargo test --manifest-path packages/native-renderer/Cargo.toml -- --test-threads=1 + - run: bun test packages/cli/src/utils/nativeBackend.test.ts packages/native-renderer/src/scene/extract.test.ts packages/native-renderer/src/scene/support.test.ts packages/native-renderer/src/timeline/bake.test.ts + - name: Native renderer comparison shard + run: | + bun packages/native-renderer/scripts/compare-regression-fixtures.ts \ + --fixtures gsap-letters-render-compat \ + --max-duration 0.25 \ + --artifacts /tmp/native-renderer-comparison-ci + smoke-global-install: name: "Smoke: global install" needs: [changes, build] diff --git a/.github/workflows/native-renderer.yml b/.github/workflows/native-renderer.yml new file mode 100644 index 000000000..7204c621c --- /dev/null +++ b/.github/workflows/native-renderer.yml @@ -0,0 +1,65 @@ +name: Native Renderer + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - "packages/native-renderer/**" + - ".github/workflows/native-renderer.yml" + push: + branches: [main] + paths: + - "packages/native-renderer/**" + +concurrency: + group: native-renderer-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Tests (Linux x86_64, CPU raster) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Cargo + Skia + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + packages/native-renderer/target + key: native-${{ runner.os }}-${{ hashFiles('packages/native-renderer/Cargo.lock') }} + restore-keys: native-${{ runner.os }}- + + - name: Install system deps + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + clang libclang-dev pkg-config \ + libfontconfig1-dev libfreetype6-dev \ + ninja-build python3 \ + ffmpeg fonts-liberation fonts-dejavu-core fontconfig + sudo fc-cache -fv + + - name: Build (CPU raster, no GPU) + working-directory: packages/native-renderer + run: cargo build --release --no-default-features --tests + + - name: Run tests + working-directory: packages/native-renderer + run: cargo test --release --no-default-features -- --test-threads=1 + + - name: Benchmark + working-directory: packages/native-renderer + run: | + cargo bench --no-default-features 2>&1 | tee /tmp/bench.txt + echo "## Native Renderer Benchmark" >> $GITHUB_STEP_SUMMARY + echo "Linux x86_64, CPU raster, no GPU" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep -E 'time:|^[a-z].*time:' /tmp/bench.txt >> $GITHUB_STEP_SUMMARY || echo "No benchmark output" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY diff --git a/docs/superpowers/plans/2026-04-25-native-renderer.md b/docs/superpowers/plans/2026-04-25-native-renderer.md new file mode 100644 index 000000000..49b6d845a --- /dev/null +++ b/docs/superpowers/plans/2026-04-25-native-renderer.md @@ -0,0 +1,1472 @@ +# HyperFrames Native Renderer — Technical Spec & Phase 1 Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a hybrid renderer that bypasses the Chrome CDP screenshot bottleneck for supported HyperFrames compositions while preserving Chrome-perfect final output through conservative fallback to the existing Chrome renderer. + +**Architecture:** Chrome extracts the layout tree once via CDP. GSAP timeline state is pre-baked or evaluated in V8, then Skia paints supported frames to GPU-backed surfaces and hardware encoders consume those frames. The CLI `auto` backend must run a support detector before native rendering and fall back to the existing Chrome pipeline whenever a composition uses browser features the native compositor cannot prove faithful yet. + +**Tech Stack:** Rust, `skia-safe` (Skia bindings), `rusty_v8` (V8 engine), FFmpeg (hardware encoding), `napi-rs` (Node.js binding for integration with existing CLI/producer) + +--- + +## 0. Parity Contract and Definition of Done + +### 0.1 What "Chrome-Perfect" Means + +This project does **not** claim that the Rust/Skia renderer reimplements the entire Chromium rendering engine. Skia is Chrome's 2D drawing engine, but Chrome parity also includes layout, SVG, canvas APIs, iframe rendering, font fallback, media timing, color management, compositing order, CSS edge cases, and browser-specific behavior. + +The production guarantee is: + +> `--backend auto` produces Chrome-perfect final video output because unsupported compositions fall back to the existing Chrome CDP renderer. Native rendering is used only when the support detector and regression proof say it is safe. + +Allowed release claim: + +- "HyperFrames accelerates supported compositions with a native Rust/Skia renderer while preserving Chrome-perfect output through automatic Chrome fallback." + +Forbidden release claim unless every web-platform feature is natively implemented and proven: + +- "The native renderer has 100% Chrome parity." +- "The native renderer replaces Chrome for every composition." +- "Skia alone makes output identical to Chrome." + +### 0.2 Renderer Modes + +| Mode | Required Behavior | +|---|---| +| `--backend chrome` | Always use the existing Chrome CDP renderer. This is the visual reference. | +| `--backend native` | Attempt native rendering and fail loudly if unsupported features are detected. It must not silently produce known-wrong frames. | +| `--backend auto` | Run support detection. Use native for supported compositions. Fall back to Chrome for unsupported compositions and print the exact fallback reasons. | + +### 0.3 Fixture Status Labels + +Every side-by-side regression result must receive exactly one status: + +| Status | Meaning | +|---|---| +| `native-pass` | Native rendered successfully, no support warnings, visual metric passes threshold, and human inspection of the side-by-side artifact shows no material mismatch. | +| `native-review` | Native rendered, but PSNR/SSIM or warnings require human review before it can be counted as safe. | +| `fallback-required` | Support detector found unsupported browser features and `auto` used Chrome. This is output-correct but not a native speed win. | +| `failed` | Neither native nor fallback produced the expected artifact, or the output is visibly wrong. | + +### 0.4 Definition of Done + +The native renderer plan is complete only when all gates below pass from a fresh checkout: + +1. **Correctness Gate:** `--backend auto` produces videos matching `--backend chrome` for the full regression fixture suite. Unsupported cases must fall back before native rendering begins. +2. **Native Coverage Gate:** The report lists which fixtures are `native-pass`, `native-review`, and `fallback-required`. The percentage of native-pass fixtures is reported honestly; it is not rounded into a "100% native" claim. +3. **Visual Proof Gate:** The regression harness emits an HTML side-by-side report with CDP video, native/auto video, poster frames, timing, support warnings, and visual metrics. At minimum it must compute poster-frame PSNR; SSIM or frame-sampled PSNR should be added before making broad parity claims. +4. **Performance Gate:** Speedup claims are generated from fresh benchmark output in the report. Claims must distinguish paint-only, native render-only, extraction+render, and full end-to-end CLI time. +5. **Fallback Gate:** The support detector explicitly rejects known unsupported surfaces and CSS features including `svg`, `canvas`, `iframe`, video until visual parity graduates, transparent roots, wrapped/grid/flex direct text layout, missing `data-composition-id` roots, animated elements without stable IDs, `backdrop-filter`, `mask-image`, unsupported `clip-path`, unsupported `filter`, unsupported background layers/repeats, and vertical writing mode. +6. **Zero-Copy Gate:** Phase 3 is not complete until the hardware path proves GPU-backed paint to encoder transfer without CPU pixel readback on at least macOS VideoToolbox. A hardware encoder subprocess alone does not satisfy zero-copy. Current double-buffered GPU rendering plus raw RGBA readback is useful acceleration work, but it remains a partial Phase 3 result until IOSurface/VideoToolbox handoff is proven. +7. **CI Gate:** CI runs native unit tests, CLI backend tests, support detection tests, and a bounded regression comparison shard. CI without GPU must exercise the CPU/FFmpeg fallback path. +8. **Docs Gate:** CLI docs and `hyperframes doctor` explain native requirements, fallback behavior, unsupported feature reasons, and how to open the side-by-side report. + +### 0.5 Native Parity Scope + +Native parity is scoped to the supported HyperFrames subset, not the entire web platform. A feature graduates into the supported native subset only after: + +1. Extraction captures the computed Chrome state needed by Rust. +2. Rust paints or encodes it deterministically. +3. Unit tests cover the parser and painter behavior. +4. A regression fixture passes visual metrics against Chrome. +5. `--backend auto` no longer falls back for that feature. + +## 1. Why This Wins Everything + +### 1.1 The Physics Wall + +Both HyperFrames and Remotion share the same architecture today: + +``` +headless Chrome → CDP screenshot → base64 → WebSocket → Node.js → FFmpeg + └──── 30-70ms per frame ────┘ +``` + +No config tuning breaks through this. The CDP serialization round-trip is the ceiling. Every frame pays: + +| Step | Time | Why | +|---|---|---| +| GSAP seek (page.evaluate) | 5ms | CDP round-trip for JS evaluation | +| Chrome layout + paint | 10-30ms | Full browser rendering pipeline | +| Chrome JPEG encode | 5ms | CPU-side pixel encoding | +| CDP base64 encode | 3ms | 33% size overhead serialization | +| WebSocket transfer | 2ms | IPC to Node.js process | +| Node.js base64 decode | 3ms | Deserialize back to bytes | +| FFmpeg JPEG decode | 2ms | Undo the JPEG encoding | +| FFmpeg H.264 encode | 5ms | Final video encoding | +| **Total** | **35-55ms** | **~18-28 effective fps** | + +### 1.2 The Native Path + +``` +V8 GSAP seek → Skia GPU paint → Hardware encode (zero-copy) +└──────────── 2-7ms per frame ────────────┘ +``` + +| Step | Time | Source | +|---|---|---| +| V8 GSAP seek (warm isolate) | 0.5ms | rusty_v8 benchmarks: 0.39ms/eval reused isolate | +| Skia GPU paint (1080p) | 1-4ms | OBS: 1-4ms/frame at 1080p60, Flutter: <8.3ms for 120fps | +| GPU texture → hardware encode | 0.5-2ms | OBS NVENC: zero-copy via shared texture handles | +| **Total** | **2-6.5ms** | **~150-500 effective fps** | + +### 1.3 Competitive Comparison + +| Metric | Remotion | HyperFrames (current) | HyperFrames Native | Source | +|---|---|---|---|---| +| 30s video @30fps | 60-180s | ~40s | **~4s** | Benchmarked / projected | +| Per-frame capture | 20-50ms | 14-40ms | **2-7ms** | CDP overhead eliminated | +| Parallelism ceiling | 32-64 cores | ~8 workers | **GPU cores (thousands)** | Remotion GH#4949 | +| Memory per worker | ~256MB (Chrome) | ~256MB | **~50MB (Skia context)** | No browser overhead | +| Hardware encode | Optional, CPU fallback | Optional | **Default, zero-copy** | GPU texture direct | +| HDR support | None | Layered compositing | **Native 10-bit pipeline** | No sRGB browser clamp | +| Can they copy this? | No (married to React+Chrome) | N/A | **Moat** | Architectural lock-in | + +### 1.4 Why Remotion Can't Follow + +Remotion's architecture is React components rendered in Chrome. Their entire ecosystem — the component library, the `useCurrentFrame()` hook, the `` abstraction — depends on React's reconciler running inside a real browser DOM. They cannot switch to Skia without rewriting every user composition and abandoning their React-based API. + +HyperFrames' HTML+GSAP compositions have a much thinner browser dependency: the `window.__hf.seek(t)` protocol just needs GSAP timeline evaluation and CSS property computation. Neither requires a DOM. + +--- + +## 2. Architecture + +### 2.1 Three-Phase Pipeline + +``` +Phase 1: Scene Extraction (one-time, ~50ms) + Chrome → CDP DOM.getDocument + CSS.getComputedStyle + DOM.getBoxModel + → JSON scene graph: { elements, positions, sizes, styles, fonts, images, videos } + +Phase 2: Animation Evaluation (per-frame, ~0.5ms) + V8 isolate + GSAP → timeline.seek(t) + → property deltas: { elementId → { transform, opacity, clipPath, ... } } + +Phase 3: Paint + Encode (per-frame, ~2-5ms) + Skia GPU canvas + property deltas → GPU texture + → Hardware encoder → H.264/H.265 bitstream + → FFmpeg mux with audio → final MP4 +``` + +### 2.2 Component Architecture + +``` +packages/native-renderer/ +├── Cargo.toml # Rust workspace member +├── src/ +│ ├── lib.rs # Library root, NAPI exports +│ ├── scene/ +│ │ ├── mod.rs # Scene graph types +│ │ ├── extract.rs # Chrome CDP → scene graph +│ │ └── parse.rs # JSON scene → Rust types +│ ├── animation/ +│ │ ├── mod.rs # Animation evaluation +│ │ ├── v8_runtime.rs # V8 isolate + GSAP loader +│ │ └── timeline.rs # Pre-baked timeline cache +│ ├── paint/ +│ │ ├── mod.rs # Skia painting coordinator +│ │ ├── canvas.rs # Skia surface + GPU context setup +│ │ ├── elements.rs # CSS property → Skia draw call mapping +│ │ ├── text.rs # Text rendering (Skia + HarfBuzz) +│ │ ├── effects.rs # Shadows, blur, gradients, filters +│ │ └── video.rs # Video frame compositing +│ ├── encode/ +│ │ ├── mod.rs # Encoder orchestration +│ │ ├── videotoolbox.rs # macOS hardware encode +│ │ ├── nvenc.rs # NVIDIA hardware encode +│ │ ├── vaapi.rs # Linux Intel/AMD hardware encode +│ │ └── ffmpeg_fallback.rs # CPU encode via FFmpeg pipe +│ └── pipeline.rs # End-to-end render pipeline +├── napi/ +│ └── index.rs # napi-rs bindings for Node.js integration +└── tests/ + ├── scene_test.rs + ├── paint_test.rs + └── pipeline_test.rs +``` + +### 2.3 Integration with Existing Stack + +The native renderer slots into the existing producer as an alternative capture+encode backend: + +``` + ┌─────────────────────────┐ + │ renderOrchestrator.ts │ + │ (existing producer) │ + └────────┬────────────────┘ + │ + ┌──────────────┼──────────────┐ + ▼ ▼ ▼ + ┌─────────────┐ ┌────────────┐ ┌──────────────┐ + │Chrome Engine │ │ WebCodecs │ │Native Render │ ← NEW + │(CDP capture) │ │(browser) │ │(Rust/Skia) │ + └─────────────┘ └────────────┘ └──────────────┘ +``` + +The CLI flag `hyperframes render --backend native` selects the native renderer directly and should fail loudly when unsupported features are detected. The CLI flag `hyperframes render --backend auto` is the production path: it selects native only for supported compositions and falls back to Chrome with explicit reasons for unsupported CSS, DOM, media, or embedded browser surfaces. + +### 2.4 Patterns Learned from Professional Tools + +| Pattern | Source | How We Use It | +|---|---|---| +| GPU-direct encoding via shared texture handles | OBS (`output_gpu_encoders`) | Skia GPU surface → NVENC/VideoToolbox without CPU readback | +| Double-buffered staging surfaces | OBS (`NUM_TEXTURES = 2`) | Paint frame N while encoding frame N-1 | +| Backend-agnostic graphics abstraction | OBS (`gs_exports` vtable) | Skia's built-in Metal/Vulkan/GL backends | +| Glyph atlas texture caching | OBS (FreeType → GPU atlas), GPU text renderers | Skia's glyph cache (automatic) | +| Pull-based evaluation with ROI | Nuke (demand-driven row model) | Only re-paint elements whose properties changed | +| Full-frame GPU compositing | Blender 4.2 compositor | Process entire frame on GPU, not tiles | +| Tick/Render separation | OBS (`tick_sources` → `render_main_texture`) | V8 evaluate → Skia paint (separate phases) | +| Node-graph GPU pipeline | DaVinci Fusion | Intermediate results stay in VRAM between effects | +| Pre-baked animation data | Standard in game engines | Optional: evaluate GSAP once, store all values in Rust Vec | + +--- + +## 3. Theoretical Performance Model + +### 3.1 Per-Frame Budget at 1080p (1920x1080) + +| Component | Chrome CDP (current) | Native Renderer | Speedup | +|---|---|---|---| +| Animation eval | 5ms (CDP evaluate) | 0.5ms (V8 isolate) | 10x | +| Layout | 0ms (unchanged) | 0ms (static) | — | +| Paint | 15-30ms (Chrome paint + screenshot) | 1-4ms (Skia GPU) | 7-15x | +| Frame transfer | 8ms (base64 + WebSocket) | 0ms (stays on GPU) | ∞ | +| Encode | 5ms (FFmpeg CPU) | 0.5-2ms (hardware) | 3-10x | +| **Total** | **33-48ms** | **2-6.5ms** | **5-24x** | + +### 3.2 End-to-End Render Time Projection + +For a 30-second composition at 30fps (900 frames): + +| Scenario | Chrome CDP | Native Renderer | Speedup | +|---|---|---|---| +| Simple (text + shapes) | 30s | 2-3s | 10-15x | +| Medium (images + transforms) | 45s | 4-6s | 8-11x | +| Complex (video + effects + text) | 90s | 8-12s | 8-11x | +| Remotion equivalent | 60-180s | 4-12s | **15-45x** | + +### 3.3 Scaling Properties + +| Dimension | Chrome CDP | Native Renderer | +|---|---|---| +| Resolution scaling | Quadratic (more pixels = slower screenshot + encode) | Sub-linear (GPU parallelism scales with pixel count) | +| Element count | Linear (more DOM = slower paint) | Sub-linear (Skia batches draw calls, Graphite sorts by pipeline) | +| Worker scaling | Diminishing >8 workers (CDP contention) | Linear with GPU cores (thousands of CUDA/Metal cores) | +| Memory per session | ~256MB (full Chrome process) | ~50MB (Skia context + V8 isolate) | + +--- + +## 4. Technology Decisions + +### 4.1 Skia over Vello/WebRender/Custom + +| Criterion | Skia | Vello | WebRender | Custom | +|---|---|---|---|---| +| CSS feature coverage | Strong 2D drawing coverage; not a full browser renderer | Missing blur, shadows, glyph cache | Oriented toward web display lists | Must implement everything | +| Production proven | Chrome, Android, Flutter | Alpha | Servo, Firefox | — | +| GPU backends | Metal, Vulkan, GL, D3D, Graphite | wgpu (Metal/Vulkan/D3D12) | GL only | — | +| Text rendering | HarfBuzz + built-in shaping | Swash (less mature) | HarfBuzz | Must integrate | +| Rust bindings | `skia-safe` (mature, v0.93+) | Native Rust | Rust native | — | +| Community | Google-backed, massive | Small (linebender) | Mozilla/Igalia | — | + +**Decision: Skia.** It gives us the same 2D raster/compositing foundation Chrome uses, which makes high-fidelity native output realistic for a constrained HyperFrames subset. It does not provide browser layout, SVG/canvas/iframe semantics, or every Chromium paint/compositing edge case by itself, so unsupported browser features must continue to trigger Chrome fallback. The `skia-safe` crate exposes the core API, and Graphite (Chrome M133+) adds multi-threaded recording and modern GPU batching. + +### 4.2 V8 over QuickJS/Boa/Pre-bake + +| Criterion | V8 (rusty_v8) | QuickJS | Boa | Pre-bake | +|---|---|---|---|---| +| GSAP compatibility | Perfect (Chrome's engine) | Good but edge cases | Partial ES2023 | Perfect (one-time eval) | +| Per-frame eval speed | 0.39ms (warm isolate) | 1-3ms | 5-10ms | 0ms (lookup table) | +| Startup cost | 5ms cold, <1ms snapshot | <1ms | <1ms | Pre-compute phase | +| Maintenance burden | Deno-maintained | Low | Low | Must re-bake on change | + +**Decision: V8 for Phase 2, pre-bake as Phase 3 optimization.** V8 guarantees GSAP behaves identically to Chrome. Pre-baking (evaluate timeline once, store all property values in a Rust lookup table) eliminates JS overhead entirely but requires a pre-compute step. + +### 4.3 Hardware Encoding Strategy + +| Platform | Encoder | Zero-Copy Path | Fallback | +|---|---|---|---| +| macOS (Apple Silicon) | VideoToolbox | Skia Metal → IOSurface → VTCompressionSession | FFmpeg libx264 pipe | +| Linux NVIDIA | NVENC | Skia Vulkan → CUDA interop → NvEncRegisterResource | FFmpeg libx264 pipe | +| Linux Intel | QSV/VAAPI | Skia Vulkan → DRM PRIME fd → VAAPI | FFmpeg libx264 pipe | +| Docker/CI (no GPU) | — | — | FFmpeg libx264 pipe | + +**Phase 1 uses FFmpeg pipe (simplest).** Phase 3 adds zero-copy paths per platform. + +--- + +## 5. CSS Property Coverage Plan + +### 5.1 Tier 1 — Covers 90% of Compositions (Phase 1) + +These properties are used in virtually every HyperFrames composition: + +| CSS Property | Skia Equivalent | Complexity | +|---|---|---| +| `transform: translate/rotate/scale` | `Canvas::concat` (3x3 matrix) | Trivial | +| `opacity` | `Paint::set_alpha` or `Canvas::save_layer_alpha` | Trivial | +| `background-color` | `Canvas::draw_rect` + `Paint::set_color` | Trivial | +| `border-radius` | `Canvas::draw_rrect` / `Canvas::clip_rrect` | Simple | +| `overflow: hidden` | `Canvas::clip_rect` / `Canvas::clip_rrect` | Simple | +| `width/height/position` | Layout from Chrome extraction | Pre-computed | +| `color` (text) | `Paint::set_color` on text | Trivial | +| `font-family/size/weight` | `skia_safe::Font` + `Typeface` | Simple | +| `visibility/display` | Skip element in draw | Trivial | + +### 5.2 Tier 2 — Full Visual Fidelity (Phase 2) + +| CSS Property | Skia Equivalent | Complexity | +|---|---|---| +| `box-shadow` | `Paint::set_mask_filter(MaskFilter::blur)` + offset draw | Medium | +| `filter: blur()` | `Paint::set_image_filter(ImageFilter::blur)` | Simple | +| `filter: brightness/contrast/saturate` | `Paint::set_color_filter(ColorFilter::matrix)` | Medium | +| `background: linear-gradient()` | `Shader::linear_gradient` | Medium | +| `background: radial-gradient()` | `Shader::radial_gradient` | Medium | +| `clip-path: polygon/circle/ellipse` | `Canvas::clip_path` with `Path` | Medium | +| `border` (solid/dashed) | `Canvas::draw_rrect` with `Paint::set_stroke` | Simple | +| `background-image: url()` | `Canvas::draw_image_rect` | Simple | +| `object-fit/object-position` | Computed source/dest rects for `draw_image_rect` | Medium | +| `mix-blend-mode` | `Paint::set_blend_mode` (Porter-Duff) | Simple | + +### 5.3 Tier 3 — Edge Cases (Phase 3+) + +| CSS Property | Approach | +|---|---| +| `backdrop-filter` | Render-to-texture + apply filter to region behind element | +| `mask-image` | Skia `MaskFilter` with image shader | +| `text-shadow` | Draw text twice: shadow (blurred, offset) then foreground | +| `-webkit-text-stroke` | `Paint::set_style(Stroke)` on text | +| `writing-mode: vertical` | `Paragraph` layout direction | +| CSS custom properties | Resolve during scene extraction | + +### 5.4 Unsupported → Chrome Fallback + +For any composition using properties not in Tier 1-3, the renderer falls back to the existing Chrome CDP pipeline. The CLI reports which properties triggered the fallback so users can optimize their compositions for native rendering. + +--- + +## 6. Phase Plan + +### Phase 1: Prove the Hypothesis (4-6 weeks, 1 engineer) + +**Deliverable:** `hyperframes render --backend native` works on static compositions (no GSAP animation). Renders 10-50x faster than Chrome CDP on supported compositions. + +**Scope:** +- Rust crate `@hyperframes/native-renderer` +- Chrome CDP extracts scene graph (one-shot) +- Skia paints static frames (Tier 1 CSS properties) +- FFmpeg pipe encodes (no hardware encode yet) +- Node.js integration via napi-rs +- Benchmark: side-by-side comparison on 5 test fixtures + +### Phase 2: Animation + Full CSS (4-6 weeks, 1-2 engineers) + +**Deliverable:** Animated compositions render natively. Full CSS Tier 1+2 coverage. + +**Scope:** +- V8 isolate evaluates GSAP timeline per-frame +- Tier 2 CSS properties (shadows, blur, gradients, clip-path) +- Video frame compositing (FFmpeg extraction → Skia image overlay) +- Text rendering with font matching (Skia + HarfBuzz) +- Delta-only repaint (only re-render elements whose properties changed) + +### Phase 3: Zero-Copy Hardware Encode (3-4 weeks, 1 engineer) + +**Deliverable:** End-to-end GPU pipeline with no CPU pixel readback on at least one platform, with benchmark output separating paint-only, render-only, and full CLI speedups. + +**Scope:** +- macOS: Skia Metal → IOSurface → VideoToolbox +- Linux: Skia Vulkan → CUDA interop → NVENC +- Double-buffered staging (paint N while encoding N-1) +- Pre-bake mode: evaluate GSAP once, store all frames' properties in Rust + +### Phase 4: Hybrid Production Hardening (2-3 weeks) + +**Deliverable:** Ship `--backend auto` as the output-safe production renderer: native acceleration for supported compositions, Chrome fallback for unsupported compositions, and proof artifacts that show the boundary. + +**Scope:** +- Fallback detection (unsupported CSS → Chrome) +- Regression test parity (PSNR comparison against Chrome output) +- CI integration (Docker with no GPU → FFmpeg fallback) +- CLI documentation, `hyperframes doctor` checks for native renderer deps +- Side-by-side HTML report with CDP/native videos, poster frames, timing, PSNR/SSIM, and fallback reasons +- Release notes that use the allowed hybrid claim from Section 0.1 + +--- + +## 7. Phase 1 Implementation Plan + +### Task 1: Rust Crate Scaffolding + +**Files:** +- Create: `packages/native-renderer/Cargo.toml` +- Create: `packages/native-renderer/src/lib.rs` +- Create: `packages/native-renderer/src/scene/mod.rs` +- Create: `packages/native-renderer/src/scene/parse.rs` + +- [ ] **Step 1: Initialize Cargo project** + +```bash +cd packages && mkdir native-renderer && cd native-renderer +cargo init --lib +``` + +Add to `Cargo.toml`: +```toml +[package] +name = "hyperframes-native-renderer" +version = "0.1.0" +edition = "2021" + +[dependencies] +skia-safe = { version = "0.93", features = ["textlayout"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[dev-dependencies] +insta = "1" # snapshot testing +``` + +- [ ] **Step 2: Define scene graph types** + +Create `src/scene/mod.rs`: +```rust +pub mod parse; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Scene { + pub width: f32, + pub height: f32, + pub elements: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Element { + pub id: String, + pub kind: ElementKind, + pub bounds: Rect, + pub style: Style, + pub children: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ElementKind { + Container, + Text { content: String }, + Image { src: String }, + Video { src: String }, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct Rect { + pub x: f32, + pub y: f32, + pub width: f32, + pub height: f32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct Style { + pub background_color: Option, + pub opacity: f32, + pub border_radius: [f32; 4], + pub overflow_hidden: bool, + pub transform: Option, + pub visibility: bool, + pub font_family: Option, + pub font_size: Option, + pub font_weight: Option, + pub color: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct Color { + pub r: u8, + pub g: u8, + pub b: u8, + pub a: u8, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct Transform2D { + pub translate_x: f32, + pub translate_y: f32, + pub scale_x: f32, + pub scale_y: f32, + pub rotate_deg: f32, +} +``` + +- [ ] **Step 3: Add JSON scene parser** + +Create `src/scene/parse.rs`: +```rust +use super::Scene; +use std::path::Path; + +pub fn parse_scene_file(path: &Path) -> Result { + let content = std::fs::read_to_string(path) + .map_err(|e| format!("Failed to read scene file: {e}"))?; + parse_scene_json(&content) +} + +pub fn parse_scene_json(json: &str) -> Result { + serde_json::from_str(json) + .map_err(|e| format!("Failed to parse scene JSON: {e}")) +} +``` + +- [ ] **Step 4: Write test for JSON parsing** + +Create `tests/scene_test.rs`: +```rust +use hyperframes_native_renderer::scene::{parse::parse_scene_json, Scene, Element, ElementKind, Rect, Style, Color}; + +#[test] +fn parse_minimal_scene() { + let json = r#"{ + "width": 1920, + "height": 1080, + "elements": [{ + "id": "bg", + "kind": "Container", + "bounds": { "x": 0, "y": 0, "width": 1920, "height": 1080 }, + "style": { + "background_color": { "r": 30, "g": 30, "b": 30, "a": 255 }, + "opacity": 1.0, + "border_radius": [0, 0, 0, 0], + "overflow_hidden": false, + "transform": null, + "visibility": true + }, + "children": [] + }] + }"#; + + let scene = parse_scene_json(json).unwrap(); + assert_eq!(scene.width, 1920.0); + assert_eq!(scene.height, 1080.0); + assert_eq!(scene.elements.len(), 1); + assert_eq!(scene.elements[0].id, "bg"); +} +``` + +- [ ] **Step 5: Run test to verify it passes** + +```bash +cargo test -p hyperframes-native-renderer +``` +Expected: PASS + +- [ ] **Step 6: Commit** + +```bash +git add packages/native-renderer/ +git commit -m "feat(native-renderer): scaffold Rust crate with scene graph types" +``` + +--- + +### Task 2: Skia GPU Surface Setup + +**Files:** +- Create: `packages/native-renderer/src/paint/mod.rs` +- Create: `packages/native-renderer/src/paint/canvas.rs` + +- [ ] **Step 1: Add Skia feature flags to Cargo.toml** + +Update `Cargo.toml` dependencies: +```toml +[dependencies] +skia-safe = { version = "0.93", features = ["textlayout", "gpu"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +``` + +- [ ] **Step 2: Create GPU surface factory** + +Create `src/paint/canvas.rs`: +```rust +use skia_safe::{ + surfaces, Color4f, ColorType, ImageInfo, Surface, +}; + +pub struct RenderSurface { + surface: Surface, +} + +impl RenderSurface { + /// Create a CPU-backed raster surface (Phase 1 — GPU in Phase 3). + pub fn new_raster(width: i32, height: i32) -> Result { + let surface = surfaces::raster_n32_premul((width, height)) + .ok_or("Failed to create Skia raster surface")?; + Ok(Self { surface }) + } + + pub fn canvas(&mut self) -> &skia_safe::Canvas { + self.surface.canvas() + } + + /// Read back rendered pixels as RGBA bytes. + pub fn read_pixels_rgba(&mut self) -> Option> { + let info = ImageInfo::new( + self.surface.width_height(), + ColorType::RGBA8888, + skia_safe::AlphaType::Premul, + None, + ); + let row_bytes = info.width() as usize * 4; + let mut pixels = vec![0u8; row_bytes * info.height() as usize]; + let success = self.surface.read_pixels( + &info, + &mut pixels, + row_bytes, + skia_safe::IPoint::new(0, 0), + ); + if success { Some(pixels) } else { None } + } + + /// Encode the surface to JPEG bytes. + pub fn encode_jpeg(&mut self, quality: u32) -> Option> { + let image = self.surface.image_snapshot(); + let data = image.encode(None, skia_safe::EncodedImageFormat::JPEG, quality)?; + Some(data.as_bytes().to_vec()) + } + + /// Encode the surface to PNG bytes. + pub fn encode_png(&mut self) -> Option> { + let image = self.surface.image_snapshot(); + let data = image.encode(None, skia_safe::EncodedImageFormat::PNG, 100)?; + Some(data.as_bytes().to_vec()) + } + + pub fn clear(&mut self, color: Color4f) { + self.surface.canvas().clear(color); + } + + pub fn width(&self) -> i32 { + self.surface.width() + } + + pub fn height(&self) -> i32 { + self.surface.height() + } +} +``` + +Create `src/paint/mod.rs`: +```rust +pub mod canvas; +``` + +- [ ] **Step 3: Write test: create surface, clear, read pixels** + +Add to `tests/paint_test.rs`: +```rust +use hyperframes_native_renderer::paint::canvas::RenderSurface; +use skia_safe::Color4f; + +#[test] +fn create_surface_and_clear_red() { + let mut surface = RenderSurface::new_raster(100, 100).unwrap(); + surface.clear(Color4f::new(1.0, 0.0, 0.0, 1.0)); // red + + let pixels = surface.read_pixels_rgba().unwrap(); + assert_eq!(pixels.len(), 100 * 100 * 4); + // First pixel should be red (RGBA) + assert_eq!(pixels[0], 255); // R + assert_eq!(pixels[1], 0); // G + assert_eq!(pixels[2], 0); // B + assert_eq!(pixels[3], 255); // A +} + +#[test] +fn encode_jpeg_produces_bytes() { + let mut surface = RenderSurface::new_raster(100, 100).unwrap(); + surface.clear(Color4f::new(0.0, 0.0, 1.0, 1.0)); // blue + + let jpeg = surface.encode_jpeg(80).unwrap(); + // JPEG magic bytes: FF D8 FF + assert_eq!(jpeg[0], 0xFF); + assert_eq!(jpeg[1], 0xD8); + assert!(jpeg.len() > 100); // should be a valid image +} +``` + +- [ ] **Step 4: Run tests** + +```bash +cargo test -p hyperframes-native-renderer +``` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add packages/native-renderer/src/paint/ +git add packages/native-renderer/tests/paint_test.rs +git commit -m "feat(native-renderer): Skia surface creation and pixel readback" +``` + +--- + +### Task 3: Element Painting — Rects, RoundRects, Transforms, Opacity + +**Files:** +- Create: `packages/native-renderer/src/paint/elements.rs` +- Modify: `packages/native-renderer/src/paint/mod.rs` + +- [ ] **Step 1: Implement element painter** + +Create `src/paint/elements.rs`: +```rust +use skia_safe::{ + Canvas, Color4f, Paint, RRect, Rect as SkRect, Matrix, + paint::Style as PaintStyle, ClipOp, +}; +use crate::scene::{Element, ElementKind, Rect, Style, Color, Transform2D}; + +pub fn paint_element(canvas: &Canvas, element: &Element) { + if !element.style.visibility { + return; + } + + let save_count = canvas.save(); + + apply_transform(canvas, &element.style, &element.bounds); + apply_opacity_layer(canvas, &element.style); + + let sk_rect = to_sk_rect(&element.bounds); + + // Clip to bounds if overflow hidden + if element.style.overflow_hidden { + let clip = make_rrect(&sk_rect, &element.style.border_radius); + canvas.clip_rrect(clip, ClipOp::Intersect, true); + } + + // Paint background + if let Some(bg) = &element.style.background_color { + let mut paint = Paint::default(); + paint.set_anti_alias(true); + paint.set_color4f(to_color4f(bg), None); + paint.set_style(PaintStyle::Fill); + + if element.style.border_radius.iter().any(|r| *r > 0.0) { + let rrect = make_rrect(&sk_rect, &element.style.border_radius); + canvas.draw_rrect(rrect, &paint); + } else { + canvas.draw_rect(sk_rect, &paint); + } + } + + // Paint text + if let ElementKind::Text { ref content } = element.kind { + paint_text(canvas, element, content); + } + + // Recurse into children + for child in &element.children { + paint_element(canvas, child); + } + + canvas.restore_to_count(save_count); +} + +fn apply_transform(canvas: &Canvas, style: &Style, bounds: &Rect) { + // Position the element + canvas.translate((bounds.x, bounds.y)); + + if let Some(t) = &style.transform { + // Transform origin is center of element + let cx = bounds.width / 2.0; + let cy = bounds.height / 2.0; + canvas.translate((cx, cy)); + + if t.rotate_deg != 0.0 { + canvas.rotate(t.rotate_deg, None); + } + if t.scale_x != 1.0 || t.scale_y != 1.0 { + canvas.scale((t.scale_x, t.scale_y)); + } + + canvas.translate((-cx, -cy)); + canvas.translate((t.translate_x, t.translate_y)); + } +} + +fn apply_opacity_layer(canvas: &Canvas, style: &Style) { + if style.opacity < 1.0 { + let alpha = (style.opacity * 255.0).round() as u8; + canvas.save_layer_alpha(None, alpha as u32); + } +} + +fn paint_text(canvas: &Canvas, element: &Element, content: &str) { + let font_size = element.style.font_size.unwrap_or(16.0); + let typeface = skia_safe::Typeface::default(); + let font = skia_safe::Font::new(typeface, font_size); + + let mut paint = Paint::default(); + paint.set_anti_alias(true); + if let Some(color) = &element.style.color { + paint.set_color4f(to_color4f(color), None); + } else { + paint.set_color4f(Color4f::new(1.0, 1.0, 1.0, 1.0), None); + } + + // Draw at element origin + font ascent (baseline) + let (_, metrics) = font.metrics(); + let y = -metrics.ascent; + canvas.draw_str(content, (0.0, y), &font, &paint); +} + +fn to_sk_rect(r: &Rect) -> SkRect { + SkRect::from_xywh(0.0, 0.0, r.width, r.height) +} + +fn make_rrect(rect: &SkRect, radii: &[f32; 4]) -> RRect { + let mut rrect = RRect::new(); + rrect.set_rect_radii( + *rect, + &[ + (radii[0], radii[0]).into(), // top-left + (radii[1], radii[1]).into(), // top-right + (radii[2], radii[2]).into(), // bottom-right + (radii[3], radii[3]).into(), // bottom-left + ], + ); + rrect +} + +fn to_color4f(c: &Color) -> Color4f { + Color4f::new( + c.r as f32 / 255.0, + c.g as f32 / 255.0, + c.b as f32 / 255.0, + c.a as f32 / 255.0, + ) +} +``` + +- [ ] **Step 2: Write test: paint a scene with nested elements** + +Add to `tests/paint_test.rs`: +```rust +use hyperframes_native_renderer::scene::{Scene, Element, ElementKind, Rect, Style, Color}; +use hyperframes_native_renderer::paint::{canvas::RenderSurface, elements::paint_element}; +use skia_safe::Color4f; + +#[test] +fn paint_scene_with_background_and_text() { + let scene = Scene { + width: 200.0, + height: 200.0, + elements: vec![Element { + id: "bg".into(), + kind: ElementKind::Container, + bounds: Rect { x: 0.0, y: 0.0, width: 200.0, height: 200.0 }, + style: Style { + background_color: Some(Color { r: 0, g: 0, b: 255, a: 255 }), + opacity: 1.0, + visibility: true, + ..Default::default() + }, + children: vec![Element { + id: "title".into(), + kind: ElementKind::Text { content: "Hello".into() }, + bounds: Rect { x: 10.0, y: 10.0, width: 180.0, height: 40.0 }, + style: Style { + color: Some(Color { r: 255, g: 255, b: 255, a: 255 }), + font_size: Some(24.0), + opacity: 1.0, + visibility: true, + ..Default::default() + }, + children: vec![], + }], + }], + }; + + let mut surface = RenderSurface::new_raster(200, 200).unwrap(); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + + for element in &scene.elements { + paint_element(surface.canvas(), element); + } + + let jpeg = surface.encode_jpeg(90).unwrap(); + assert!(jpeg.len() > 200); // valid JPEG with content +} + +#[test] +fn paint_element_with_border_radius_and_opacity() { + let element = Element { + id: "card".into(), + kind: ElementKind::Container, + bounds: Rect { x: 20.0, y: 20.0, width: 160.0, height: 100.0 }, + style: Style { + background_color: Some(Color { r: 255, g: 0, b: 0, a: 255 }), + opacity: 0.5, + border_radius: [12.0, 12.0, 12.0, 12.0], + overflow_hidden: true, + visibility: true, + ..Default::default() + }, + children: vec![], + }; + + let mut surface = RenderSurface::new_raster(200, 200).unwrap(); + surface.clear(Color4f::new(1.0, 1.0, 1.0, 1.0)); // white bg + + paint_element(surface.canvas(), &element); + + let pixels = surface.read_pixels_rgba().unwrap(); + // Corner pixel (0,0) should be white (not affected by rounded rect) + assert_eq!(pixels[0], 255); // R = white + // Center pixel (100, 70) should be blended red on white + let center_idx = (70 * 200 + 100) * 4; + assert!(pixels[center_idx] > 200); // R channel high (red blended with white at 50%) +} +``` + +- [ ] **Step 3: Run tests** + +```bash +cargo test -p hyperframes-native-renderer +``` +Expected: PASS + +- [ ] **Step 4: Commit** + +```bash +git add packages/native-renderer/src/paint/elements.rs +git add packages/native-renderer/tests/paint_test.rs +git commit -m "feat(native-renderer): element painting with transforms, opacity, border-radius" +``` + +--- + +### Task 4: Scene Extraction from Chrome via CDP + +**Files:** +- Create: `packages/native-renderer/src/scene/extract.ts` (TypeScript, runs in Node.js) +- Create: `packages/native-renderer/src/scene/extract.test.ts` + +This task creates the bridge: Chrome renders the composition, we extract the layout tree as JSON, then feed it to the Rust renderer. + +- [ ] **Step 1: Create the CDP scene extractor** + +Create `packages/native-renderer/src/scene/extract.ts`: +```typescript +import type { Page } from "puppeteer-core"; + +export interface ExtractedScene { + width: number; + height: number; + elements: ExtractedElement[]; +} + +export interface ExtractedElement { + id: string; + kind: "Container" | { Text: { content: string } } | { Image: { src: string } } | { Video: { src: string } }; + bounds: { x: number; y: number; width: number; height: number }; + style: { + background_color: { r: number; g: number; b: number; a: number } | null; + opacity: number; + border_radius: [number, number, number, number]; + overflow_hidden: boolean; + transform: { translate_x: number; translate_y: number; scale_x: number; scale_y: number; rotate_deg: number } | null; + visibility: boolean; + font_family: string | null; + font_size: number | null; + font_weight: number | null; + color: { r: number; g: number; b: number; a: number } | null; + }; + children: ExtractedElement[]; +} + +export async function extractScene( + page: Page, + width: number, + height: number, +): Promise { + const elements = await page.evaluate(() => { + function extractElement(el: HTMLElement): any { + const cs = getComputedStyle(el); + const rect = el.getBoundingClientRect(); + const tag = el.tagName.toLowerCase(); + + let kind: any = "Container"; + if (tag === "video") kind = { Video: { src: el.getAttribute("src") || "" } }; + else if (tag === "img") kind = { Image: { src: (el as HTMLImageElement).src } }; + else if (el.childNodes.length === 1 && el.childNodes[0].nodeType === Node.TEXT_NODE) { + kind = { Text: { content: el.textContent || "" } }; + } + + const parseColor = (c: string) => { + const m = c.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?\)/); + if (!m) return null; + return { r: +m[1], g: +m[2], b: +m[3], a: Math.round((m[4] !== undefined ? +m[4] : 1) * 255) }; + }; + + const br = [ + parseFloat(cs.borderTopLeftRadius) || 0, + parseFloat(cs.borderTopRightRadius) || 0, + parseFloat(cs.borderBottomRightRadius) || 0, + parseFloat(cs.borderBottomLeftRadius) || 0, + ] as [number, number, number, number]; + + const children: any[] = []; + for (const child of el.children) { + if (child instanceof HTMLElement) { + children.push(extractElement(child)); + } + } + + return { + id: el.id || el.getAttribute("data-name") || `anon-${Math.random().toString(36).slice(2, 8)}`, + kind, + bounds: { x: rect.left, y: rect.top, width: rect.width, height: rect.height }, + style: { + background_color: parseColor(cs.backgroundColor), + opacity: parseFloat(cs.opacity) || 1, + border_radius: br, + overflow_hidden: cs.overflow === "hidden", + transform: null, // GSAP will provide this per-frame in Phase 2 + visibility: cs.visibility !== "hidden" && cs.display !== "none", + font_family: cs.fontFamily || null, + font_size: parseFloat(cs.fontSize) || null, + font_weight: parseInt(cs.fontWeight) || null, + color: parseColor(cs.color), + }, + children, + }; + } + + const root = document.querySelector("[data-composition-id]") || document.body; + return Array.from(root.children) + .filter((c): c is HTMLElement => c instanceof HTMLElement) + .map(extractElement); + }); + + return { width, height, elements }; +} +``` + +- [ ] **Step 2: Write test** + +Create `packages/native-renderer/src/scene/extract.test.ts`: +```typescript +import { describe, it, expect } from "vitest"; +import type { ExtractedScene, ExtractedElement } from "./extract"; + +describe("ExtractedScene types", () => { + it("round-trips through JSON", () => { + const scene: ExtractedScene = { + width: 1920, + height: 1080, + elements: [{ + id: "bg", + kind: "Container", + bounds: { x: 0, y: 0, width: 1920, height: 1080 }, + style: { + background_color: { r: 30, g: 30, b: 30, a: 255 }, + opacity: 1, + border_radius: [0, 0, 0, 0], + overflow_hidden: false, + transform: null, + visibility: true, + font_family: null, + font_size: null, + font_weight: null, + color: null, + }, + children: [], + }], + }; + const json = JSON.stringify(scene); + const parsed = JSON.parse(json) as ExtractedScene; + expect(parsed.elements[0].id).toBe("bg"); + expect(parsed.elements[0].style.background_color?.r).toBe(30); + }); +}); +``` + +- [ ] **Step 3: Run test** + +```bash +bunx vitest run packages/native-renderer/src/scene/extract.test.ts +``` +Expected: PASS + +- [ ] **Step 4: Commit** + +```bash +git add packages/native-renderer/src/scene/ +git commit -m "feat(native-renderer): CDP scene extraction from Chrome" +``` + +--- + +### Task 5: Render Pipeline — Scene JSON to Video Frames + +**Files:** +- Create: `packages/native-renderer/src/pipeline.rs` +- Modify: `packages/native-renderer/src/lib.rs` + +- [ ] **Step 1: Implement the render pipeline** + +Create `src/pipeline.rs`: +```rust +use crate::scene::Scene; +use crate::paint::canvas::RenderSurface; +use crate::paint::elements::paint_element; +use skia_safe::Color4f; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +pub struct RenderConfig { + pub fps: u32, + pub duration_secs: f64, + pub quality: u32, + pub output_path: String, +} + +pub struct RenderResult { + pub total_frames: u32, + pub total_ms: u64, + pub avg_paint_ms: f64, + pub output_path: String, +} + +/// Render a static scene (no animation) to a video file via FFmpeg pipe. +pub fn render_static(scene: &Scene, config: &RenderConfig) -> Result { + let total_frames = (config.fps as f64 * config.duration_secs).ceil() as u32; + let width = scene.width as i32; + let height = scene.height as i32; + + let mut surface = RenderSurface::new_raster(width, height)?; + + // Paint the static frame once + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &scene.elements { + paint_element(surface.canvas(), element); + } + + let frame_jpeg = surface.encode_jpeg(config.quality) + .ok_or("Failed to encode frame as JPEG")?; + + // Spawn FFmpeg with image2pipe input + let mut ffmpeg = Command::new("ffmpeg") + .args([ + "-y", + "-f", "image2pipe", + "-vcodec", "mjpeg", + "-framerate", &config.fps.to_string(), + "-i", "-", + "-c:v", "libx264", + "-preset", "fast", + "-crf", "18", + "-pix_fmt", "yuv420p", + "-threads", "0", + &config.output_path, + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("Failed to spawn FFmpeg: {e}"))?; + + let start = std::time::Instant::now(); + let stdin = ffmpeg.stdin.as_mut().ok_or("Failed to open FFmpeg stdin")?; + + // Write the same frame N times (static scene) + for _ in 0..total_frames { + stdin.write_all(&frame_jpeg) + .map_err(|e| format!("Failed to write frame to FFmpeg: {e}"))?; + } + + drop(ffmpeg.stdin.take()); + let output = ffmpeg.wait_with_output() + .map_err(|e| format!("FFmpeg failed: {e}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("FFmpeg exited with error: {stderr}")); + } + + let total_ms = start.elapsed().as_millis() as u64; + + Ok(RenderResult { + total_frames, + total_ms, + avg_paint_ms: 0.0, // static: painted once + output_path: config.output_path.clone(), + }) +} +``` + +Update `src/lib.rs`: +```rust +pub mod scene; +pub mod paint; +pub mod pipeline; +``` + +- [ ] **Step 2: Write integration test** + +Create `tests/pipeline_test.rs`: +```rust +use hyperframes_native_renderer::scene::{Scene, Element, ElementKind, Rect, Style, Color}; +use hyperframes_native_renderer::pipeline::{render_static, RenderConfig}; +use std::path::Path; + +#[test] +fn render_static_scene_to_mp4() { + let scene = Scene { + width: 640.0, + height: 360.0, + elements: vec![ + Element { + id: "bg".into(), + kind: ElementKind::Container, + bounds: Rect { x: 0.0, y: 0.0, width: 640.0, height: 360.0 }, + style: Style { + background_color: Some(Color { r: 20, g: 20, b: 40, a: 255 }), + opacity: 1.0, + visibility: true, + ..Default::default() + }, + children: vec![ + Element { + id: "card".into(), + kind: ElementKind::Container, + bounds: Rect { x: 50.0, y: 50.0, width: 540.0, height: 260.0 }, + style: Style { + background_color: Some(Color { r: 255, g: 255, b: 255, a: 255 }), + opacity: 0.9, + border_radius: [16.0, 16.0, 16.0, 16.0], + overflow_hidden: true, + visibility: true, + ..Default::default() + }, + children: vec![ + Element { + id: "title".into(), + kind: ElementKind::Text { content: "Hello from Skia!".into() }, + bounds: Rect { x: 30.0, y: 30.0, width: 480.0, height: 40.0 }, + style: Style { + color: Some(Color { r: 0, g: 0, b: 0, a: 255 }), + font_size: Some(32.0), + opacity: 1.0, + visibility: true, + ..Default::default() + }, + children: vec![], + }, + ], + }, + ], + }, + ], + }; + + let output_path = "/tmp/hyperframes-native-test.mp4"; + let config = RenderConfig { + fps: 30, + duration_secs: 1.0, + quality: 80, + output_path: output_path.to_string(), + }; + + let result = render_static(&scene, &config).unwrap(); + + assert_eq!(result.total_frames, 30); + assert!(Path::new(output_path).exists()); + let file_size = std::fs::metadata(output_path).unwrap().len(); + assert!(file_size > 1000, "Output MP4 should be non-trivial size, got {file_size}"); + + // Cleanup + std::fs::remove_file(output_path).ok(); +} +``` + +- [ ] **Step 3: Run integration test** + +```bash +cargo test -p hyperframes-native-renderer -- --test-threads=1 +``` +Expected: PASS (requires FFmpeg installed) + +- [ ] **Step 4: Commit** + +```bash +git add packages/native-renderer/src/pipeline.rs +git add packages/native-renderer/src/lib.rs +git add packages/native-renderer/tests/pipeline_test.rs +git commit -m "feat(native-renderer): static scene → MP4 render pipeline via FFmpeg" +``` + +--- + +### Task 6: Benchmark — Native vs Chrome CDP + +**Files:** +- Create: `packages/native-renderer/benches/render_bench.rs` + +- [ ] **Step 1: Add criterion dependency** + +Update `Cargo.toml`: +```toml +[dev-dependencies] +insta = "1" +criterion = { version = "0.5", features = ["html_reports"] } + +[[bench]] +name = "render_bench" +harness = false +``` + +- [ ] **Step 2: Write benchmark** + +Create `benches/render_bench.rs`: +```rust +use criterion::{criterion_group, criterion_main, Criterion}; +use hyperframes_native_renderer::scene::{Scene, Element, ElementKind, Rect, Style, Color}; +use hyperframes_native_renderer::paint::canvas::RenderSurface; +use hyperframes_native_renderer::paint::elements::paint_element; +use skia_safe::Color4f; + +fn build_test_scene() -> Scene { + let mut children = Vec::new(); + // 20 overlapping cards with text — representative of a composition + for i in 0..20 { + children.push(Element { + id: format!("card-{i}"), + kind: ElementKind::Container, + bounds: Rect { + x: 50.0 + (i as f32 * 10.0), + y: 50.0 + (i as f32 * 15.0), + width: 400.0, + height: 200.0, + }, + style: Style { + background_color: Some(Color { r: (i * 12) as u8, g: 100, b: 200, a: 220 }), + opacity: 0.8, + border_radius: [12.0, 12.0, 12.0, 12.0], + overflow_hidden: true, + visibility: true, + ..Default::default() + }, + children: vec![Element { + id: format!("text-{i}"), + kind: ElementKind::Text { content: format!("Card {i} — Hello World") }, + bounds: Rect { x: 20.0, y: 20.0, width: 360.0, height: 30.0 }, + style: Style { + color: Some(Color { r: 255, g: 255, b: 255, a: 255 }), + font_size: Some(24.0), + opacity: 1.0, + visibility: true, + ..Default::default() + }, + children: vec![], + }], + }); + } + + Scene { + width: 1920.0, + height: 1080.0, + elements: vec![Element { + id: "root".into(), + kind: ElementKind::Container, + bounds: Rect { x: 0.0, y: 0.0, width: 1920.0, height: 1080.0 }, + style: Style { + background_color: Some(Color { r: 15, g: 15, b: 30, a: 255 }), + opacity: 1.0, + visibility: true, + ..Default::default() + }, + children, + }], + } +} + +fn bench_paint_frame(c: &mut Criterion) { + let scene = build_test_scene(); + let mut surface = RenderSurface::new_raster(1920, 1080).unwrap(); + + c.bench_function("paint_1080p_20_elements", |b| { + b.iter(|| { + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &scene.elements { + paint_element(surface.canvas(), element); + } + }); + }); + + c.bench_function("paint_and_encode_jpeg_1080p", |b| { + b.iter(|| { + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &scene.elements { + paint_element(surface.canvas(), element); + } + surface.encode_jpeg(80).unwrap(); + }); + }); +} + +criterion_group!(benches, bench_paint_frame); +criterion_main!(benches); +``` + +- [ ] **Step 3: Run benchmark** + +```bash +cargo bench -p hyperframes-native-renderer +``` + +Expected output format: +``` +paint_1080p_20_elements time: [0.5ms 0.6ms 0.7ms] +paint_and_encode_jpeg_1080p time: [2.1ms 2.3ms 2.5ms] +``` + +Compare against the Chrome CDP baseline (~30-50ms/frame). A 10-20x speedup on CPU-only raster is expected. GPU surface in Phase 3 will add another 5-10x. + +- [ ] **Step 4: Commit** + +```bash +git add packages/native-renderer/benches/ +git add packages/native-renderer/Cargo.toml +git commit -m "bench(native-renderer): Skia paint benchmark — 1080p, 20 elements" +``` + +--- + +## 8. Moat Analysis + +### Why This Wins Long-Term + +| Dimension | Remotion | HyperFrames Native | +|---|---|---| +| **Renderer** | Chrome (general-purpose browser) | Skia (Chrome's own paint engine, purpose-built) | +| **Animation** | React reconciler in full DOM | V8 isolate (GSAP only, no DOM overhead) | +| **Encoding** | FFmpeg CPU (separate process) | Hardware encoder, zero-copy from GPU | +| **Memory** | ~256MB per Chrome tab | ~50MB per Skia context | +| **Switching cost for them** | Rewrite every React component + abandon ecosystem | N/A | +| **Switching cost for us** | Keep HTML authoring, Rust renderer is transparent | N/A | +| **HDR** | Not possible (browser clamps to sRGB) | Native 10-bit pipeline through Skia | +| **8K support** | Impractical (Chrome memory + screenshot overhead) | Linear GPU scaling | +| **Cloud cost** | CPU-bound, expensive | GPU instances, 10-50x more throughput per $ | + +### The Decisive Advantage + +The HTML+GSAP authoring format is Hyperframes' API contract with users. The rendering backend is an implementation detail. Users write the same HTML compositions — the CLI transparently picks the fastest renderer that can handle the composition's CSS properties. Remotion can't do this because their API contract IS React-in-Chrome. + +This means Hyperframes can adopt **any** rendering backend — Chrome (compatibility), WebCodecs (medium-term), Skia/Rust (long-term) — without breaking a single user composition. That architectural flexibility is the moat. diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 632674312..d30706b22 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -1,5 +1,10 @@ #!/usr/bin/env node +// Polyfill esbuild's __name helper before any import can reference it. +if (typeof (globalThis as Record).__name !== "function") { + (globalThis as Record).__name = (fn: T, _n: string): T => fn; +} + // ── Fast-path exits ───────────────────────────────────────────────────────── // Check --version before importing anything heavy. This makes // `hyperframes --version` near-instant (~10ms vs ~80ms). diff --git a/packages/cli/src/commands/doctor.ts b/packages/cli/src/commands/doctor.ts index 41846bcd7..687fff01c 100644 --- a/packages/cli/src/commands/doctor.ts +++ b/packages/cli/src/commands/doctor.ts @@ -10,6 +10,7 @@ import { findFFmpeg, getFFmpegInstallHint } from "../browser/ffmpeg.js"; import { VERSION } from "../version.js"; import { getUpdateMeta } from "../utils/updateCheck.js"; import { getSystemMeta, getShmSizeMb, getFreeDiskMb, bytesToMb } from "../telemetry/system.js"; +import { findNativeRendererRoot } from "../utils/nativeRender.js"; interface Check { name: string; @@ -181,6 +182,30 @@ function checkEnvironment(): CheckResult { return { ok: true, detail: parts.join(" \u00B7 ") }; } +function checkNativeRenderer(): CheckResult { + const root = findNativeRendererRoot(); + if (!root) { + return { + ok: true, + detail: "Not bundled in this installation; --backend auto will use Chrome fallback", + }; + } + + try { + const cargo = execSync("cargo --version", { encoding: "utf-8", timeout: 5000 }).trim(); + return { + ok: true, + detail: `${root} \u00B7 ${cargo} \u00B7 auto gates native by support detection; zero-copy is not proven yet`, + }; + } catch { + return { + ok: false, + detail: `${root} \u00B7 cargo not found`, + hint: "Install Rust from https://rustup.rs/ to use --backend native from source; --backend auto falls back to Chrome when native is unavailable or unsupported.", + }; + } +} + export default defineCommand({ meta: { name: "doctor", description: "Check system dependencies and environment" }, args: {}, @@ -207,6 +232,7 @@ export default defineCommand({ { name: "FFmpeg", run: checkFFmpeg }, { name: "FFprobe", run: checkFFprobe }, { name: "Chrome", run: checkChrome }, + { name: "Native renderer", run: checkNativeRenderer }, { name: "Docker", run: checkDocker }, { name: "Docker running", run: checkDockerRunning }, ); diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index b123c3972..4c61f9866 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -26,6 +26,17 @@ import { bytesToMb } from "../telemetry/system.js"; import { VERSION } from "../version.js"; import { isDevMode } from "../utils/env.js"; import { buildDockerRunArgs } from "../utils/dockerRunArgs.js"; +import { + parseRenderBackend, + resolveRenderBackend, + type RenderBackend, +} from "../utils/nativeBackend.js"; +import { + formatUnsupportedNativeFeatures, + isNativeRendererAvailable, + renderNativeProject, + type NativeRenderResult, +} from "../utils/nativeRender.js"; import type { RenderJob } from "@hyperframes/producer"; const VALID_FPS = new Set([24, 30, 60]); @@ -73,6 +84,11 @@ export default defineCommand({ description: "Output format: mp4, webm, mov (MOV/WebM render with transparency)", default: "mp4", }, + backend: { + type: "string", + description: "Render backend: chrome, native, or auto", + default: "chrome", + }, workers: { type: "string", alias: "w", @@ -147,6 +163,14 @@ export default defineCommand({ } const format = formatRaw as "mp4" | "webm" | "mov"; + // ── Validate backend ──────────────────────────────────────────────── + const backendRaw = args.backend ?? "chrome"; + const backend = parseRenderBackend(backendRaw); + if (!backend) { + errorBox("Invalid backend", `Got "${backendRaw}". Must be chrome, native, or auto.`); + process.exit(1); + } + // ── Validate workers ────────────────────────────────────────────────── let workers: number | undefined; if (args.workers != null && args.workers !== "auto") { @@ -215,6 +239,22 @@ export default defineCommand({ process.exit(1); } + const backendDecision = resolveRenderBackend({ + requested: backend, + docker: useDocker, + format, + hdr: args.hdr ?? false, + nativeRuntimeAvailable: isNativeRendererAvailable(), + }); + if (backendDecision.kind === "unavailable") { + errorBox( + "Native renderer unavailable", + backendDecision.reasons.map((reason) => `- ${reason}`).join("\n"), + "Use --backend chrome, or --backend auto to fall back automatically.", + ); + process.exit(1); + } + // ── Print render plan ───────────────────────────────────────────────── const workerCount = workers ?? defaultWorkerCount(); if (!quiet) { @@ -229,7 +269,22 @@ export default defineCommand({ c.accent(project.name) + c.dim(" \u2192 " + outputPath), ); - console.log(c.dim(" " + fps + "fps \u00B7 " + quality + " \u00B7 " + workerLabel)); + const backendLabel = formatBackendLabel(backendDecision.requested, backendDecision.kind); + console.log( + c.dim( + " " + + fps + + "fps \u00B7 " + + quality + + " \u00B7 " + + workerLabel + + " \u00B7 backend " + + backendLabel, + ), + ); + if (backendDecision.requested === "auto" && backendDecision.reasons.length > 0) { + console.log(c.dim(" Native fallback: " + backendDecision.reasons.join("; "))); + } console.log(""); } @@ -302,31 +357,80 @@ export default defineCommand({ } // ── Render ──────────────────────────────────────────────────────────── + const renderOptions: RenderOptions = { + fps, + quality, + format, + workers: workerCount, + gpu: useGpu, + hdr: args.hdr ?? false, + crf, + videoBitrate, + quiet, + browserPath, + }; + if (useDocker) { - await renderDocker(project.dir, outputPath, { + await renderDocker(project.dir, outputPath, renderOptions); + } else if (backendDecision.kind === "native") { + const nativeStart = Date.now(); + const nativeResult = await renderNativeProject(project.dir, outputPath, { fps, quality, - format, - workers: workerCount, - gpu: useGpu, - hdr: args.hdr ?? false, - crf, - videoBitrate, + browserPath, quiet, - }); + }).catch((error: unknown) => + handleRenderError( + error, + renderOptions, + nativeStart, + false, + "Use --backend chrome to render through the Chrome pipeline", + ), + ); + + if (nativeResult.kind === "rendered") { + trackNativeRenderMetrics(nativeResult, renderOptions); + printRenderComplete(outputPath, nativeResult.elapsedMs, quiet); + return; + } + + if (nativeResult.kind === "unsupported") { + const reasons = formatUnsupportedNativeFeatures(nativeResult.support.reasons); + if (backendDecision.requested === "auto") { + if (!quiet) { + console.log(c.dim(" Native fallback:\n" + reasons)); + console.log(""); + } + await renderLocal(project.dir, outputPath, renderOptions); + return; + } + + errorBox( + "Native renderer fallback required", + reasons, + "Use --backend auto or --backend chrome for this composition.", + ); + process.exit(1); + } + + if (backendDecision.requested === "auto") { + if (!quiet) { + console.log(c.dim(" Native fallback: " + nativeResult.reasons.join("; "))); + console.log(""); + } + await renderLocal(project.dir, outputPath, renderOptions); + return; + } + + errorBox( + "Native renderer unavailable", + nativeResult.reasons.map((reason) => `- ${reason}`).join("\n"), + "Use --backend chrome, or install the native renderer toolchain.", + ); + process.exit(1); } else { - await renderLocal(project.dir, outputPath, { - fps, - quality, - format, - workers: workerCount, - gpu: useGpu, - hdr: args.hdr ?? false, - crf, - videoBitrate, - quiet, - browserPath, - }); + await renderLocal(project.dir, outputPath, renderOptions); } }, }); @@ -344,6 +448,10 @@ interface RenderOptions { browserPath?: string; } +function formatBackendLabel(requested: RenderBackend, selected: "chrome" | "native"): string { + return requested === "auto" ? `auto \u2192 ${selected}` : selected; +} + const DOCKER_IMAGE_PREFIX = "hyperframes-renderer"; function dockerImageTag(version: string): string { @@ -578,6 +686,33 @@ function handleRenderError( process.exit(1); } +function trackNativeRenderMetrics( + result: Extract, + options: RenderOptions, +): void { + const compositionDurationMs = Math.round(result.duration * 1000); + const speedRatio = + compositionDurationMs > 0 && result.elapsedMs > 0 + ? Math.round((compositionDurationMs / result.elapsedMs) * 100) / 100 + : undefined; + + trackRenderComplete({ + durationMs: result.elapsedMs, + fps: options.fps, + quality: options.quality, + workers: options.workers, + docker: false, + gpu: true, + compositionDurationMs, + compositionWidth: result.width, + compositionHeight: result.height, + totalFrames: result.totalFrames, + speedRatio, + captureAvgMs: result.renderer.avgPaintMs, + ...getMemorySnapshot(), + }); +} + /** * Extract rich metrics from the completed render job and send to telemetry. * speed_ratio = composition_duration / render_time — higher is better, >1 means faster than realtime. diff --git a/packages/cli/src/docs/rendering.md b/packages/cli/src/docs/rendering.md index 83aa596db..85c605ce8 100644 --- a/packages/cli/src/docs/rendering.md +++ b/packages/cli/src/docs/rendering.md @@ -7,6 +7,16 @@ Render compositions to MP4 with `npx hyperframes render`. Uses Puppeteer (bundled Chromium) + system FFmpeg. Fast for iteration. Requires: FFmpeg installed (`brew install ffmpeg` or `apt install ffmpeg`). +## Backend Selection + +- `--backend chrome` — Always render through Chrome CDP. This is the reference renderer. +- `--backend native` — Render through the Rust/Skia native renderer. Unsupported browser features fail loudly with fallback reasons. +- `--backend auto` — Use native only when the composition passes support detection; otherwise fall back to Chrome for Chrome-perfect final output. + +Native acceleration is a fast path for supported HyperFrames compositions, not a full Chromium replacement. SVG, canvas, iframe, video, unsupported CSS filters, masks, backdrop filters, vertical writing mode, transparent roots, wrapped/grid/flex direct text layout, missing `data-composition-id` roots, animated elements without stable IDs, and other unsupported browser surfaces use Chrome fallback in `auto` mode. + +The current native GPU path can use Metal-backed Skia surfaces and hardware encoder settings where available, but macOS still transfers frames through CPU-visible RGBA readback before FFmpeg/VideoToolbox. Treat zero-copy GPU-to-encoder handoff as incomplete until the native report explicitly proves IOSurface/VideoToolbox transfer without CPU pixel readback. + ## Docker Mode (--docker) Deterministic output with exact Chrome version and fonts. For production. @@ -20,6 +30,7 @@ Requires: Docker installed and running. - `--crf` — Override encoder CRF (mutually exclusive with `--video-bitrate`) - `--video-bitrate` — Target video bitrate such as `10M` (mutually exclusive with `--crf`) - `--gpu` — Use GPU encoding (NVENC, VideoToolbox, VAAPI) +- `--backend` — `chrome`, `native`, or `auto` (default: chrome) - `-o, --output` — Custom output path ## Tips diff --git a/packages/cli/src/polyfill-name.cjs b/packages/cli/src/polyfill-name.cjs new file mode 100644 index 000000000..1b62fb58f --- /dev/null +++ b/packages/cli/src/polyfill-name.cjs @@ -0,0 +1,6 @@ +// Polyfill esbuild's __name helper. tsx injects __name() wrappers via +// keepNames but ESM hoisting means inline polyfills run too late. +// This CJS file is loaded via --require before any ESM evaluation. +if (typeof globalThis.__name !== "function") { + globalThis.__name = function(fn) { return fn; }; +} diff --git a/packages/cli/src/utils/nativeBackend.test.ts b/packages/cli/src/utils/nativeBackend.test.ts new file mode 100644 index 000000000..d8d84b02a --- /dev/null +++ b/packages/cli/src/utils/nativeBackend.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { parseRenderBackend, resolveRenderBackend } from "./nativeBackend.js"; + +describe("parseRenderBackend", () => { + it("accepts known render backends", () => { + expect(parseRenderBackend("chrome")).toBe("chrome"); + expect(parseRenderBackend("native")).toBe("native"); + expect(parseRenderBackend("auto")).toBe("auto"); + }); + + it("rejects unknown render backends", () => { + expect(parseRenderBackend("skia")).toBeNull(); + }); +}); + +describe("resolveRenderBackend", () => { + it("keeps chrome when explicitly requested", () => { + const decision = resolveRenderBackend({ + requested: "chrome", + docker: false, + format: "mp4", + hdr: false, + }); + + expect(decision.kind).toBe("chrome"); + expect(decision.reasons).toEqual([]); + }); + + it("selects native in auto mode when local constraints allow it", () => { + const decision = resolveRenderBackend({ + requested: "auto", + docker: false, + format: "mp4", + hdr: false, + nativeRuntimeAvailable: true, + }); + + expect(decision).toEqual({ kind: "native", requested: "auto", reasons: [] }); + }); + + it("selects native when explicitly requested and available", () => { + const decision = resolveRenderBackend({ + requested: "native", + docker: false, + format: "mp4", + hdr: false, + nativeRuntimeAvailable: true, + }); + + expect(decision).toEqual({ kind: "native", requested: "native", reasons: [] }); + }); + + it("falls back to chrome in auto mode when native runtime is unavailable", () => { + const decision = resolveRenderBackend({ + requested: "auto", + docker: false, + format: "mp4", + hdr: false, + nativeRuntimeAvailable: false, + }); + + expect(decision.kind).toBe("chrome"); + expect(decision.reasons).toContain( + "native renderer binary source is not available in this installation", + ); + }); + + it("blocks explicit native backend when container or format constraints cannot be met", () => { + const decision = resolveRenderBackend({ + requested: "native", + docker: true, + format: "webm", + hdr: true, + nativeRuntimeAvailable: false, + }); + + expect(decision.kind).toBe("unavailable"); + expect(decision.reasons).toEqual([ + "native renderer is only available for local renders", + "native renderer currently outputs mp4 only", + "native renderer HDR parity is not implemented yet", + "native renderer binary source is not available in this installation", + ]); + }); +}); diff --git a/packages/cli/src/utils/nativeBackend.ts b/packages/cli/src/utils/nativeBackend.ts new file mode 100644 index 000000000..d45535190 --- /dev/null +++ b/packages/cli/src/utils/nativeBackend.ts @@ -0,0 +1,61 @@ +export type RenderBackend = "chrome" | "native" | "auto"; +export type RenderFormat = "mp4" | "webm" | "mov"; + +export type RenderBackendDecision = + | { + kind: "chrome"; + requested: RenderBackend; + reasons: string[]; + } + | { + kind: "native"; + requested: "native" | "auto"; + reasons: []; + } + | { + kind: "unavailable"; + requested: "native"; + reasons: string[]; + }; + +const VALID_RENDER_BACKENDS = new Set(["chrome", "native", "auto"]); + +export function parseRenderBackend(raw: string): RenderBackend | null { + return VALID_RENDER_BACKENDS.has(raw as RenderBackend) ? (raw as RenderBackend) : null; +} + +export function resolveRenderBackend(options: { + requested: RenderBackend; + docker: boolean; + format: RenderFormat; + hdr: boolean; + nativeRuntimeAvailable?: boolean; +}): RenderBackendDecision { + if (options.requested === "chrome") { + return { kind: "chrome", requested: "chrome", reasons: [] }; + } + + const reasons: string[] = []; + if (options.docker) { + reasons.push("native renderer is only available for local renders"); + } + if (options.format !== "mp4") { + reasons.push("native renderer currently outputs mp4 only"); + } + if (options.hdr) { + reasons.push("native renderer HDR parity is not implemented yet"); + } + if (options.nativeRuntimeAvailable === false) { + reasons.push("native renderer binary source is not available in this installation"); + } + + if (options.requested === "native") { + return reasons.length === 0 + ? { kind: "native", requested: "native", reasons: [] } + : { kind: "unavailable", requested: "native", reasons }; + } + + return reasons.length === 0 + ? { kind: "native", requested: "auto", reasons: [] } + : { kind: "chrome", requested: "auto", reasons }; +} diff --git a/packages/cli/src/utils/nativeRender.ts b/packages/cli/src/utils/nativeRender.ts new file mode 100644 index 000000000..8a17039a9 --- /dev/null +++ b/packages/cli/src/utils/nativeRender.ts @@ -0,0 +1,410 @@ +// Polyfill esbuild's keepNames helper — tsx injects __name() wrappers but +// some transitive imports execute before the global is defined. +if (typeof (globalThis as Record).__name !== "function") { + (globalThis as Record).__name = (fn: T, _name: string): T => fn; +} + +import { execFileSync, spawn } from "node:child_process"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; + +import type { Browser, Page } from "puppeteer-core"; +import { extractScene } from "../../../native-renderer/src/scene/extract.js"; +import { + detectNativeSupport, + type NativeUnsupportedReason, + type NativeSupportReport, +} from "../../../native-renderer/src/scene/support.js"; +import { bakeTimeline } from "../../../native-renderer/src/timeline/bake.js"; + +export type { NativeUnsupportedReason, NativeSupportReport }; + +export interface NativeRenderOptions { + fps: 24 | 30 | 60; + quality: "draft" | "standard" | "high"; + browserPath?: string; + quiet: boolean; +} + +export type NativeRenderResult = + | { + kind: "rendered"; + outputPath: string; + elapsedMs: number; + width: number; + height: number; + duration: number; + totalFrames: number; + renderer: NativeRendererStats; + support: NativeSupportReport; + } + | { + kind: "unsupported"; + support: NativeSupportReport; + } + | { + kind: "unavailable"; + reasons: string[]; + }; + +export interface NativeRendererStats { + frames: number; + totalMs: number; + avgPaintMs: number; + outputPath: string; +} + +interface CompositionMetadata { + width: number; + height: number; + duration: number; +} + +interface ServedProject { + url: string; + close: () => Promise; +} + +const NATIVE_UNAVAILABLE_REASON = + "native renderer binary source is not available in this installation"; + +export function findNativeRendererRoot(): string | null { + const thisDir = dirname(new URL(import.meta.url).pathname); + const candidates = [ + resolve(thisDir, "../../../native-renderer"), + resolve(thisDir, "../../native-renderer"), + resolve(process.cwd(), "packages/native-renderer"), + ]; + + for (const candidate of candidates) { + if ( + existsSync(join(candidate, "Cargo.toml")) && + existsSync(join(candidate, "src/bin/render_native.rs")) + ) { + return candidate; + } + } + + return null; +} + +export function isNativeRendererAvailable(): boolean { + return findNativeRendererRoot() !== null; +} + +export function formatUnsupportedNativeFeatures(features: NativeUnsupportedReason[]): string { + return features + .map( + (feature) => + `- ${feature.elementId}: ${feature.property}=${feature.value} (${feature.reason})`, + ) + .join("\n"); +} + +export async function renderNativeProject( + projectDir: string, + outputPath: string, + options: NativeRenderOptions, +): Promise { + const nativeRoot = findNativeRendererRoot(); + if (!nativeRoot) { + return { kind: "unavailable", reasons: [NATIVE_UNAVAILABLE_REASON] }; + } + + const elapsedStart = Date.now(); + const artifactsDir = mkdtempSync(join(tmpdir(), "hyperframes-native-")); + const scenePath = join(artifactsDir, "scene.json"); + const timelinePath = join(artifactsDir, "timeline.json"); + + let browser: Browser | undefined; + let server: ServedProject | undefined; + try { + if (!options.quiet) { + console.log(" Native renderer: extracting scene graph..."); + } + + server = await serveBundledProject(projectDir); + const puppeteer = await import("puppeteer-core"); + browser = await puppeteer.default.launch({ + headless: true, + executablePath: options.browserPath, + args: ["--allow-file-access-from-files", "--disable-web-security", "--no-sandbox"], + }); + + const page = await browser.newPage(); + await page.goto(server.url, { waitUntil: "networkidle0", timeout: 30_000 }); + await waitForComposition(page); + + const metadata = await readCompositionMetadata(page); + await page.setViewport({ width: metadata.width, height: metadata.height }); + await settlePage(page); + + const support = await detectNativeSupport(page, metadata.width, metadata.height); + if (!support.supported) { + return { kind: "unsupported", support }; + } + + const scene = await extractScene(page, metadata.width, metadata.height); + const timeline = await bakeTimeline(page, options.fps, metadata.duration); + writeFileSync(scenePath, JSON.stringify(scene, null, 2)); + writeFileSync(timelinePath, JSON.stringify(timeline, null, 2)); + + if (!options.quiet) { + console.log(" Native renderer: building Rust binary..."); + } + buildNativeBinary(nativeRoot, options.quiet); + + if (!options.quiet) { + console.log(" Native renderer: painting and encoding..."); + } + const raw = await runNativeBinary(nativeRoot, { + scenePath, + timelinePath, + outputPath, + fps: options.fps, + duration: metadata.duration, + quality: qualityNumber(options.quality), + }); + const renderer = parseRendererStats(raw); + + return { + kind: "rendered", + outputPath, + elapsedMs: Date.now() - elapsedStart, + width: metadata.width, + height: metadata.height, + duration: metadata.duration, + totalFrames: renderer.frames, + renderer, + support, + }; + } finally { + if (browser) await browser.close(); + if (server) await server.close(); + rmSync(artifactsDir, { recursive: true, force: true }); + } +} + +async function serveBundledProject(projectDir: string): Promise { + const html = await bundleProjectHtml(projectDir); + const { getMimeType } = await import("@hyperframes/core/studio-api"); + + const server = createServer((req, res) => { + const rawUrl = req.url ?? "/"; + const parsed = new URL(rawUrl, "http://127.0.0.1"); + const pathname = decodeURIComponent(parsed.pathname); + if (pathname === "/" || pathname === "/index.html") { + res.writeHead(200, { "Content-Type": "text/html" }); + res.end(html); + return; + } + + const filePath = resolve(projectDir, pathname.replace(/^\//, "")); + const rel = relative(projectDir, filePath); + if (rel.startsWith("..") || isAbsolute(rel)) { + res.writeHead(403); + res.end(); + return; + } + + if (existsSync(filePath)) { + res.writeHead(200, { "Content-Type": getMimeType(filePath) }); + res.end(readFileSync(filePath)); + return; + } + + res.writeHead(404); + res.end(); + }); + + const port = await new Promise((resolvePort, rejectPort) => { + server.on("error", rejectPort); + server.listen(0, () => { + const address = server.address(); + const portNumber = typeof address === "object" && address ? address.port : 0; + if (portNumber > 0) resolvePort(portNumber); + else rejectPort(new Error("Failed to bind native renderer preview server")); + }); + }); + + return { + url: `http://127.0.0.1:${port}/`, + close: () => + new Promise((resolveClose) => { + server.close(() => resolveClose()); + }), + }; +} + +async function bundleProjectHtml(projectDir: string): Promise { + const { bundleToSingleHtml } = await import("@hyperframes/core/compiler"); + let html = await bundleToSingleHtml(projectDir); + + const runtimePath = findRuntimePath(); + if (runtimePath) { + const runtimeSource = readFileSync(runtimePath, "utf-8"); + html = html.replace( + /]*data-hyperframes-preview-runtime[^>]*src="[^"]*"[^>]*><\/script>/, + () => ``, + ); + } + + return html; +} + +function findRuntimePath(): string | null { + const candidates = [ + resolve( + dirname(new URL(import.meta.url).pathname), + "../../../core/dist/hyperframe.runtime.iife.js", + ), + resolve( + dirname(new URL(import.meta.url).pathname), + "../../core/dist/hyperframe.runtime.iife.js", + ), + resolve(dirname(dirname(new URL(import.meta.url).pathname)), "hyperframe.runtime.iife.js"), + ]; + return candidates.find((candidate) => existsSync(candidate)) ?? null; +} + +async function waitForComposition(page: Page): Promise { + await page.waitForSelector("[data-composition-id]", { timeout: 10_000 }); + await page.waitForFunction( + () => { + const root = document.querySelector("[data-composition-id]"); + return Boolean(root && root.clientWidth > 0 && root.clientHeight > 0); + }, + { timeout: 10_000 }, + ); +} + +async function settlePage(page: Page): Promise { + await page + .evaluate(() => { + const fonts = (document as Document & { fonts?: FontFaceSet }).fonts; + if (!fonts?.ready) return Promise.resolve(); + return Promise.race([ + fonts.ready.then(() => undefined), + new Promise((resolveFonts) => setTimeout(resolveFonts, 500)), + ]); + }) + .catch(() => undefined); + + await page.evaluate( + () => + new Promise((resolveFrame) => + requestAnimationFrame(() => requestAnimationFrame(() => resolveFrame())), + ), + ); +} + +async function readCompositionMetadata(page: Page): Promise { + return page.evaluate(() => { + function positiveNumber(raw: string | number | undefined | null): number | null { + const value = typeof raw === "number" ? raw : Number(raw); + return Number.isFinite(value) && value > 0 ? value : null; + } + + const root = document.querySelector("[data-composition-id]"); + const hf = (window as unknown as { __hf?: { duration?: number } }).__hf; + + return { + width: positiveNumber(root?.dataset.width) ?? positiveNumber(root?.clientWidth) ?? 1920, + height: positiveNumber(root?.dataset.height) ?? positiveNumber(root?.clientHeight) ?? 1080, + duration: positiveNumber(root?.dataset.duration) ?? positiveNumber(hf?.duration) ?? 1, + }; + }); +} + +function buildNativeBinary(nativeRoot: string, quiet: boolean): void { + execFileSync("cargo", ["build", "--release", "--bin", "render_native"], { + cwd: nativeRoot, + stdio: quiet ? "pipe" : "inherit", + timeout: 600_000, + }); +} + +function runNativeBinary( + nativeRoot: string, + options: { + scenePath: string; + timelinePath: string; + outputPath: string; + fps: number; + duration: number; + quality: number; + }, +): Promise { + const binaryPath = join(nativeRoot, "target/release/render_native"); + statSync(binaryPath); + + return new Promise((resolveRun, rejectRun) => { + const child = spawn( + binaryPath, + [ + "--scene", + options.scenePath, + "--timeline", + options.timelinePath, + "--output", + options.outputPath, + "--fps", + String(options.fps), + "--duration", + String(options.duration), + "--quality", + String(options.quality), + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf-8"); + child.stderr.setEncoding("utf-8"); + child.stdout.on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + child.on("error", rejectRun); + child.on("close", (code) => { + if (code === 0) { + resolveRun(stdout.trim()); + return; + } + + rejectRun( + new Error(`native renderer exited with ${code ?? "unknown"}\n${stdout}\n${stderr}`), + ); + }); + }); +} + +function parseRendererStats(raw: string): NativeRendererStats { + const parsed: unknown = JSON.parse(raw); + if (!isNativeRendererStats(parsed)) { + throw new Error(`native renderer emitted invalid stats: ${raw}`); + } + return parsed; +} + +function isNativeRendererStats(value: unknown): value is NativeRendererStats { + if (typeof value !== "object" || value === null) return false; + const record = value as Record; + return ( + typeof record.frames === "number" && + typeof record.totalMs === "number" && + typeof record.avgPaintMs === "number" && + typeof record.outputPath === "string" + ); +} + +function qualityNumber(quality: "draft" | "standard" | "high"): number { + if (quality === "draft") return 65; + if (quality === "high") return 92; + return 80; +} diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index 855ecc53f..b79141751 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -5,7 +5,8 @@ "moduleResolution": "bundler", "baseUrl": ".", "paths": { - "@hyperframes/producer": ["../producer/src/index.ts"] + "@hyperframes/producer": ["../producer/src/index.ts"], + "puppeteer-core": ["node_modules/puppeteer-core"] }, "strict": true, "noUncheckedIndexedAccess": true, diff --git a/packages/engine/src/config.ts b/packages/engine/src/config.ts index a07d3a137..a99661b39 100644 --- a/packages/engine/src/config.ts +++ b/packages/engine/src/config.ts @@ -17,6 +17,13 @@ export interface EngineConfig { quality: "draft" | "standard" | "high"; format: "jpeg" | "png"; jpegQuality: number; + /** + * JPEG quality for the streaming encode pipeline. Lower than jpegQuality + * because frames are re-encoded by FFmpeg — intermediate quality loss is + * invisible in the final output. Smaller JPEG buffers transfer faster over + * CDP, directly reducing per-frame capture time. + */ + streamingJpegQuality: number; // ── Parallelism ────────────────────────────────────────────────────── /** Max worker count. "auto" uses CPU-based heuristic. */ @@ -27,10 +34,24 @@ export interface EngineConfig { minParallelFrames: number; /** Frame count threshold for "large render" heuristics. */ largeRenderThreshold: number; + /** + * Use a single browser with multiple pages instead of separate browser + * processes per worker. Eliminates N-1 Chrome startup costs and shares + * the GPU process. Only applies in screenshot capture mode (not BeginFrame). + */ + useMultiPageCapture: boolean; // ── Browser ────────────────────────────────────────────────────────── chromePath?: string; disableGpu: boolean; + /** + * GPU backend for Chrome's rendering. + * - "swiftshader": Software renderer. Deterministic, cross-platform, slow. + * - "hardware": Native GPU (Metal on macOS, Vulkan/VAAPI on Linux). Fast + * but output may differ across hardware. Falls back to SwiftShader if + * no GPU is available (Docker, CI). + */ + gpuBackend: "swiftshader" | "hardware"; enableBrowserPool: boolean; browserTimeout: number; protocolTimeout: number; @@ -106,13 +127,16 @@ export const DEFAULT_CONFIG: EngineConfig = { quality: "standard", format: "jpeg", jpegQuality: 80, + streamingJpegQuality: 55, concurrency: "auto", - coresPerWorker: 2.5, + coresPerWorker: 2, minParallelFrames: 120, largeRenderThreshold: 1000, + useMultiPageCapture: false, disableGpu: false, + gpuBackend: "swiftshader", enableBrowserPool: false, browserTimeout: 120_000, protocolTimeout: 300_000, @@ -171,6 +195,11 @@ export function resolveConfig(overrides?: Partial): EngineConfig { chromePath: env("PRODUCER_HEADLESS_SHELL_PATH"), disableGpu: envBool("PRODUCER_DISABLE_GPU", DEFAULT_CONFIG.disableGpu), + gpuBackend: (env("PRODUCER_GPU_BACKEND") === "swiftshader" + ? "swiftshader" + : env("PRODUCER_GPU_BACKEND") === "hardware" + ? "hardware" + : DEFAULT_CONFIG.gpuBackend) as "swiftshader" | "hardware", enableBrowserPool: envBool("PRODUCER_ENABLE_BROWSER_POOL", DEFAULT_CONFIG.enableBrowserPool), browserTimeout: envNum("PRODUCER_PUPPETEER_LAUNCH_TIMEOUT_MS", DEFAULT_CONFIG.browserTimeout), protocolTimeout: envNum( @@ -183,6 +212,15 @@ export function resolveConfig(overrides?: Partial): EngineConfig { forceScreenshot: envBool("PRODUCER_FORCE_SCREENSHOT", DEFAULT_CONFIG.forceScreenshot), + streamingJpegQuality: envNum( + "PRODUCER_STREAMING_JPEG_QUALITY", + DEFAULT_CONFIG.streamingJpegQuality, + ), + useMultiPageCapture: envBool( + "PRODUCER_USE_MULTI_PAGE_CAPTURE", + DEFAULT_CONFIG.useMultiPageCapture, + ), + enableChunkedEncode: envBool( "PRODUCER_ENABLE_CHUNKED_ENCODE", DEFAULT_CONFIG.enableChunkedEncode, diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 0bfc00bcc..e2f51b019 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -59,6 +59,7 @@ export { // ── Frame capture pipeline ────────────────────────────────────────────────────── export { createCaptureSession, + createCaptureSessionInBrowser, initializeSession, closeCaptureSession, captureFrame, diff --git a/packages/engine/src/services/browserManager.ts b/packages/engine/src/services/browserManager.ts index f5ccf8e60..c50be44ce 100644 --- a/packages/engine/src/services/browserManager.ts +++ b/packages/engine/src/services/browserManager.ts @@ -242,12 +242,9 @@ export interface BuildChromeArgsOptions { export function buildChromeArgs( options: BuildChromeArgsOptions, - config?: Partial>, + config?: Partial>, ): string[] { - // Chrome flags tuned for headless rendering performance. The set below is a - // fairly standard "headless-for-capture" configuration — similar profiles - // appear in Puppeteer's defaults, Playwright, Remotion, and Chrome's own - // headless-shell guidance. + const gpuBackend = config?.gpuBackend ?? DEFAULT_CONFIG.gpuBackend; const chromeArgs = [ "--no-sandbox", "--disable-setuid-sandbox", @@ -255,7 +252,17 @@ export function buildChromeArgs( "--enable-webgl", "--ignore-gpu-blocklist", "--use-gl=angle", - "--use-angle=swiftshader", + ...(gpuBackend === "hardware" + ? [ + // Let Chrome pick the best available GPU backend (Metal on macOS, + // Vulkan on Linux). Falls back to SwiftShader automatically when + // no hardware GPU is available (Docker, CI). + "--use-angle=default", + "--enable-gpu-rasterization", + "--enable-zero-copy", + "--enable-features=VaapiVideoDecoder,Vulkan", + ] + : ["--use-angle=swiftshader"]), "--font-render-hinting=none", "--force-color-profile=srgb", `--window-size=${options.width},${options.height}`, diff --git a/packages/engine/src/services/chunkEncoder.ts b/packages/engine/src/services/chunkEncoder.ts index 594cda66d..3761b287c 100644 --- a/packages/engine/src/services/chunkEncoder.ts +++ b/packages/engine/src/services/chunkEncoder.ts @@ -105,7 +105,7 @@ export function buildEncoderArgs( options = { ...options, hdr: undefined }; } - const args: string[] = [...inputArgs, "-r", String(fps)]; + const args: string[] = ["-threads", "0", ...inputArgs, "-r", String(fps)]; const shouldUseGpu = useGpu && gpuEncoder !== null; if (codec === "h264" || codec === "h265") { diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index 306da5414..8f329b72e 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -47,6 +47,12 @@ export interface CaptureSession { // browser-pool semantics (see the function body for the full invariant). pageReleased?: boolean; browserReleased?: boolean; + /** + * When true, this session owns the browser and will close it in + * closeCaptureSession. When false (multi-page mode), only the page is + * closed — the caller manages the shared browser lifecycle. + */ + ownsBrowser: boolean; browserConsoleBuffer: string[]; capturePerf: { frames: number; @@ -69,64 +75,21 @@ export interface CaptureSession { // Complex compositions produce 100+ messages; 50 was too small to capture relevant errors. const BROWSER_CONSOLE_BUFFER_SIZE = 200; -export async function createCaptureSession( - serverUrl: string, - outputDir: string, +async function setupPage( + browser: Browser, options: CaptureOptions, - onBeforeCapture: BeforeCaptureHook | null = null, config?: Partial, -): Promise { - if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true }); - - // Determine capture mode before building args — BeginFrame flags only apply on Linux - const headlessShell = resolveHeadlessShellPath(config); - const isLinux = process.platform === "linux"; - const forceScreenshot = config?.forceScreenshot ?? DEFAULT_CONFIG.forceScreenshot; - const preMode: CaptureMode = - headlessShell && isLinux && !forceScreenshot ? "beginframe" : "screenshot"; - const chromeArgs = buildChromeArgs( - { width: options.width, height: options.height, captureMode: preMode }, - config, - ); - - const { browser, captureMode } = await acquireBrowser(chromeArgs, config); - +): Promise { const page = await browser.newPage(); - // Polyfill esbuild's keepNames helper inside the page. - // - // The engine is published as raw TypeScript (`packages/engine/package.json` - // points `main`/`exports` at `./src/index.ts`) and downstream consumers - // execute it through transpilers that may inject `__name(fn, "name")` - // wrappers around named functions. Empirically, this happens with: - // - tsx (its esbuild loader runs with keepNames=true), used by the - // producer's parity-harness, ad-hoc dev scripts, and the - // `bun run --filter @hyperframes/engine test` Vitest path. - // - any tsup/esbuild build that explicitly enables keepNames. - // - // The HeyGen CLI (`packages/cli`) bundles this engine via tsup with - // keepNames left at its default (false) — verified by grepping - // `packages/cli/dist/cli.js`, where `__name(...)` call sites are absent. - // Bun's TS loader also does not currently inject `__name`. Even so, - // anything that calls `page.evaluate(fn)` with a nested named function - // under tsx (most local development and tests) will serialize bodies - // like `__name(nested,"nested")` and crash with `__name is not defined` - // in the browser. The shim makes such calls a no-op. - // - // An alternative is to load browser-side code as raw text and inject it - // via `page.addScriptTag({ content: ... })` — see - // `packages/cli/src/commands/contrast-audit.browser.js` for that pattern. - // Until every `page.evaluate(fn)` call site migrates, this polyfill is - // the single line of defense. The companion regression test in - // `frameCapture-namePolyfill.test.ts` verifies the shim stays wired up. await page.evaluateOnNewDocument(() => { const w = window as unknown as { __name?: (fn: T, _name: string) => T }; if (typeof w.__name !== "function") { w.__name = (fn: T, _name: string): T => fn; } }); - const browserVersion = await browser.version(); const expectedMajor = config?.expectedChromiumMajor; if (Number.isFinite(expectedMajor)) { + const browserVersion = await browser.version(); const actualChromiumMajor = Number.parseInt( (browserVersion.match(/(\d+)\./) || [])[1] || "", 10, @@ -144,16 +107,26 @@ export async function createCaptureSession( }; await page.setViewport(viewport); - // For PNG capture (used by WebM/transparency), make the page background transparent - // so Chrome's screenshot captures alpha channel data. Must use the same CDP session - // that the screenshot service uses (getCdpSession caches per page). if (options.format === "png") { const cdp = await getCdpSession(page); await cdp.send("Emulation.setDefaultBackgroundColorOverride", { color: { r: 0, g: 0, b: 0, a: 0 }, }); } + return page; +} +function buildSession( + browser: Browser, + page: Page, + serverUrl: string, + outputDir: string, + options: CaptureOptions, + captureMode: CaptureMode, + onBeforeCapture: BeforeCaptureHook | null, + ownsBrowser: boolean, + config?: Partial, +): CaptureSession { return { browser, page, @@ -162,6 +135,7 @@ export async function createCaptureSession( outputDir, onBeforeCapture, isInitialized: false, + ownsBrowser, browserConsoleBuffer: [], capturePerf: { frames: 0, @@ -179,6 +153,70 @@ export async function createCaptureSession( }; } +export async function createCaptureSession( + serverUrl: string, + outputDir: string, + options: CaptureOptions, + onBeforeCapture: BeforeCaptureHook | null = null, + config?: Partial, +): Promise { + if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true }); + + const headlessShell = resolveHeadlessShellPath(config); + const isLinux = process.platform === "linux"; + const forceScreenshot = config?.forceScreenshot ?? DEFAULT_CONFIG.forceScreenshot; + const preMode: CaptureMode = + headlessShell && isLinux && !forceScreenshot ? "beginframe" : "screenshot"; + const chromeArgs = buildChromeArgs( + { width: options.width, height: options.height, captureMode: preMode }, + config, + ); + + const { browser, captureMode } = await acquireBrowser(chromeArgs, config); + const page = await setupPage(browser, options, config); + + return buildSession( + browser, + page, + serverUrl, + outputDir, + options, + captureMode, + onBeforeCapture, + true, + config, + ); +} + +/** + * Create a capture session that uses a page in an already-running browser. + * The session does NOT own the browser — closeCaptureSession will only close + * the page, leaving the shared browser alive for other workers. + */ +export async function createCaptureSessionInBrowser( + browser: Browser, + captureMode: CaptureMode, + serverUrl: string, + outputDir: string, + options: CaptureOptions, + onBeforeCapture: BeforeCaptureHook | null = null, + config?: Partial, +): Promise { + if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true }); + const page = await setupPage(browser, options, config); + return buildSession( + browser, + page, + serverUrl, + outputDir, + options, + captureMode, + onBeforeCapture, + false, + config, + ); +} + /** * Classify a console "Failed to load resource" error as a font-load failure. * @@ -448,13 +486,10 @@ async function prepareFrameForCapture( const quantizedTime = quantizeTimeToFrame(time, options.fps); const seekStart = Date.now(); - // Seek via the __hf protocol. The page's seek() implementation handles - // all framework-specific logic (GSAP stepping, CSS animation sync, etc.) - await page.evaluate((t: number) => { - if (window.__hf && typeof window.__hf.seek === "function") { - window.__hf.seek(t); - } - }, quantizedTime); + // String eval is faster than function serialization — skips Puppeteer's + // serialize-args-to-JSON + Runtime.callFunctionOn overhead. The guard is + // unnecessary here: initializeSession already verified window.__hf.seek exists. + await page.evaluate(`void(window.__hf.seek(${quantizedTime}))`); const seekMs = Date.now() - seekStart; // Before-capture hook (e.g. video frame injection) @@ -583,10 +618,13 @@ export async function closeCaptureSession(session: CaptureSession): Promise {}); session.pageReleased = true; } - if (!session.browserReleased && session.browser) { + if (session.ownsBrowser && !session.browserReleased && session.browser) { await releaseBrowser(session.browser, session.config); session.browserReleased = true; } + if (!session.ownsBrowser) { + session.browserReleased = true; + } session.isInitialized = false; } @@ -614,9 +652,7 @@ export function prepareCaptureSessionForReuse( export async function getCompositionDuration(session: CaptureSession): Promise { if (!session.isInitialized) throw new Error("[FrameCapture] Session not initialized"); - return session.page.evaluate(() => { - return window.__hf?.duration ?? 0; - }); + return session.page.evaluate(`window.__hf?.duration ?? 0`) as Promise; } export function getCapturePerfSummary(session: CaptureSession): CapturePerfSummary { diff --git a/packages/engine/src/services/parallelCoordinator.ts b/packages/engine/src/services/parallelCoordinator.ts index 96c35485e..b6dcb94d8 100644 --- a/packages/engine/src/services/parallelCoordinator.ts +++ b/packages/engine/src/services/parallelCoordinator.ts @@ -3,6 +3,12 @@ * * Coordinates parallel frame capture across multiple Puppeteer sessions. * Auto-detects optimal worker count based on CPU/memory. + * + * Two modes: + * - Multi-page (default for screenshot mode): one browser, N pages. + * Eliminates N-1 Chrome startup costs and shares the GPU process. + * - Multi-browser (BeginFrame or explicit opt-out): N browsers, 1 page each. + * Required for BeginFrame mode (atomic compositor control is per-browser). */ import { cpus, freemem, totalmem } from "os"; @@ -12,6 +18,7 @@ import { join } from "path"; import { createCaptureSession, + createCaptureSessionInBrowser, initializeSession, closeCaptureSession, captureFrame, @@ -22,6 +29,12 @@ import { type CapturePerfSummary, type BeforeCaptureHook, } from "./frameCapture.js"; +import { + acquireBrowser, + buildChromeArgs, + resolveHeadlessShellPath, + type CaptureMode, +} from "./browserManager.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; export interface WorkerTask { @@ -50,9 +63,9 @@ export interface ParallelProgress { const MEMORY_PER_WORKER_MB = 256; const MIN_WORKERS = 1; -const ABSOLUTE_MAX_WORKERS = 10; -const DEFAULT_SAFE_MAX_WORKERS = 6; -const MIN_FRAMES_PER_WORKER = 30; +const ABSOLUTE_MAX_WORKERS = 12; +const DEFAULT_SAFE_MAX_WORKERS = 8; +const MIN_FRAMES_PER_WORKER = 24; export function calculateOptimalWorkers( totalFrames: number, @@ -64,7 +77,6 @@ export function calculateOptimalWorkers( > >, ): number { - // Resolve effective values: config overrides → DEFAULT_CONFIG fallback. const effectiveMaxWorkers = (() => { const concurrency = config?.concurrency ?? DEFAULT_CONFIG.concurrency; if (concurrency !== "auto") { @@ -84,11 +96,8 @@ export function calculateOptimalWorkers( if (totalFrames < MIN_FRAMES_PER_WORKER * 2) return 1; const cpuCount = cpus().length; - const cpuBasedWorkers = Math.max(1, cpuCount - 2); + const cpuBasedWorkers = Math.max(1, cpuCount - 1); - // Use total memory instead of free memory — macOS reports misleadingly low - // freemem() because it aggressively caches files in "inactive" memory that - // is immediately reclaimable. const totalMemoryMB = Math.round(totalmem() / (1024 * 1024)); const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMB * 0.5) / MEMORY_PER_WORKER_MB)); @@ -98,11 +107,6 @@ export function calculateOptimalWorkers( const minWorkersForJob = totalFrames >= effectiveMinParallelFrames ? 2 : MIN_WORKERS; let finalWorkers = Math.max(minWorkersForJob, Math.min(effectiveMaxWorkers, optimal)); - // Adaptive scaling: cap workers for large renders to prevent CPU contention. - // Each Chrome process (with SwiftShader) is CPU-heavy; too many on a long - // render causes protocol timeouts from compositor starvation. - // Scale proportionally to CPU count: ~3 cores per worker (benchmarked). - // 8 cores → 2 workers, 16 cores → 5 workers, 32 cores → 10 workers. if (totalFrames >= effectiveLargeRenderThreshold) { const cpuScaledMax = Math.max(2, Math.floor(cpuCount / effectiveCoresPerWorker)); if (finalWorkers > cpuScaledMax) { @@ -146,6 +150,7 @@ async function executeWorkerTask( onFrameCaptured?: (workerId: number, frameIndex: number) => void, onFrameBuffer?: (frameIndex: number, buffer: Buffer) => Promise, config?: Partial, + sharedBrowser?: { browser: import("puppeteer-core").Browser; captureMode: CaptureMode }, ): Promise { const startTime = Date.now(); let framesCaptured = 0; @@ -156,13 +161,25 @@ async function executeWorkerTask( let perf: CapturePerfSummary | undefined; try { - session = await createCaptureSession( - serverUrl, - task.outputDir, - captureOptions, - createBeforeCaptureHook(), - config, - ); + if (sharedBrowser) { + session = await createCaptureSessionInBrowser( + sharedBrowser.browser, + sharedBrowser.captureMode, + serverUrl, + task.outputDir, + captureOptions, + createBeforeCaptureHook(), + config, + ); + } else { + session = await createCaptureSession( + serverUrl, + task.outputDir, + captureOptions, + createBeforeCaptureHook(), + config, + ); + } await initializeSession(session); for (let i = task.startFrame; i < task.endFrame; i++) { @@ -172,11 +189,9 @@ async function executeWorkerTask( const time = i / captureOptions.fps; if (onFrameBuffer) { - // Streaming mode: capture to buffer and invoke callback const { buffer } = await captureFrameToBuffer(session, i, time); await onFrameBuffer(i, buffer); } else { - // Disk mode: capture to file await captureFrame(session, i, time); } framesCaptured++; @@ -209,6 +224,26 @@ async function executeWorkerTask( } } +/** + * Determine whether to use multi-page mode (shared browser). + * Multi-page requires screenshot mode (BeginFrame is per-browser). + */ +function shouldUseMultiPage(config?: Partial): { + multiPage: boolean; + captureMode: CaptureMode; +} { + const useMultiPage = config?.useMultiPageCapture ?? DEFAULT_CONFIG.useMultiPageCapture; + const headlessShell = resolveHeadlessShellPath(config); + const isLinux = process.platform === "linux"; + const forceScreenshot = config?.forceScreenshot ?? DEFAULT_CONFIG.forceScreenshot; + + const wouldBeginFrame = headlessShell && isLinux && !forceScreenshot; + if (wouldBeginFrame) { + return { multiPage: false, captureMode: "beginframe" }; + } + return { multiPage: useMultiPage, captureMode: "screenshot" }; +} + export async function executeParallelCapture( serverUrl: string, workDir: string, @@ -240,6 +275,52 @@ export async function executeParallelCapture( } }; + const { multiPage, captureMode } = shouldUseMultiPage(config); + + if (multiPage && tasks.length > 1) { + // ── Multi-page mode: 1 browser, N pages ────────────────────────────── + // Launch a single browser and share it across all workers. Each worker + // gets its own page (separate renderer process, independent DOM/JS context). + const chromeArgs = buildChromeArgs( + { width: captureOptions.width, height: captureOptions.height, captureMode }, + config, + ); + const { browser, captureMode: actualMode } = await acquireBrowser(chromeArgs, { + ...config, + enableBrowserPool: false, + }); + const shared = { browser, captureMode: actualMode }; + + try { + const results = await Promise.all( + tasks.map((task) => + executeWorkerTask( + task, + serverUrl, + captureOptions, + createBeforeCaptureHook, + signal, + onFrameCaptured, + onFrameBuffer, + config, + shared, + ), + ), + ); + + const errors = results.filter((r) => r.error); + if (errors.length > 0) { + const errorMessages = errors.map((e) => `Worker ${e.workerId}: ${e.error}`).join("; "); + throw new Error(`[Parallel] Capture failed: ${errorMessages}`); + } + + return results; + } finally { + await browser.close().catch(() => {}); + } + } + + // ── Multi-browser mode: N browsers, 1 page each ───────────────────── const results = await Promise.all( tasks.map((task) => executeWorkerTask( diff --git a/packages/engine/src/services/streamingEncoder.ts b/packages/engine/src/services/streamingEncoder.ts index a39073b7d..45039120f 100644 --- a/packages/engine/src/services/streamingEncoder.ts +++ b/packages/engine/src/services/streamingEncoder.ts @@ -151,8 +151,12 @@ export function buildStreamingArgs( imageFormat = "jpeg", } = options; - // Input args: pipe from stdin const args: string[] = []; + + // Multi-threaded decoding + encoding. 0 = auto-detect from CPU count. + args.push("-threads", "0"); + + // Input args: pipe from stdin if (options.rawInputFormat) { // Raw pixel input (HLG/PQ-encoded rgb48le from FFmpeg extraction). // Tag the input with the correct color space so FFmpeg uses the right diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 4dfb775b3..4ab64bd3a 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -226,9 +226,11 @@ export async function extractVideoFramesRange( const isHdr = isHdrColorSpaceUtil(metadata.colorSpace); const isMacOS = process.platform === "darwin"; - const args: string[] = []; + const args: string[] = ["-threads", "0"]; if (isHdr && isMacOS) { args.push("-hwaccel", "videotoolbox"); + } else if (!isHdr && metadata.videoCodec === "h264") { + args.push("-hwaccel", "auto"); } if (metadata.hasAlpha && metadata.videoCodec === "vp9") { args.push("-c:v", "libvpx-vp9"); @@ -244,7 +246,8 @@ export async function extractVideoFramesRange( args.push("-vf", vfFilters.join(",")); args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"); - if (format === "png") args.push("-compression_level", "6"); + // Lower compression = faster writes. These are temp files cleaned up after render. + if (format === "png") args.push("-compression_level", "1"); args.push("-y", outputPattern); return new Promise((resolve, reject) => { diff --git a/packages/native-renderer/.dockerignore b/packages/native-renderer/.dockerignore new file mode 100644 index 000000000..edc625621 --- /dev/null +++ b/packages/native-renderer/.dockerignore @@ -0,0 +1,2 @@ +target/ +*.chunks/ diff --git a/packages/native-renderer/.gitignore b/packages/native-renderer/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/packages/native-renderer/.gitignore @@ -0,0 +1 @@ +/target diff --git a/packages/native-renderer/Cargo.lock b/packages/native-renderer/Cargo.lock new file mode 100644 index 000000000..0ff8876a2 --- /dev/null +++ b/packages/native-renderer/Cargo.lock @@ -0,0 +1,1651 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.11.1", + "cexpr", + "clang-sys", + "itertools 0.10.5", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn", + "which", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.11.1", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.2", + "shlex", + "syn", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "console" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +dependencies = [ + "encode_unicode", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core-graphics-types" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "dcv-color-primitives" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "450dadfffc04f17c0a5e8a3e0a9935f2a93e78f19a3a55eb923425de39e6e9fc" +dependencies = [ + "paste", + "wasm-bindgen", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.11.1", + "objc2", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "hyperframes-native-renderer" +version = "0.1.0" +dependencies = [ + "base64", + "criterion", + "dcv-color-primitives", + "insta", + "metal", + "minimp4", + "objc2", + "objc2-foundation", + "openh264", + "serde", + "serde_json", + "skia-safe", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.0", + "serde", + "serde_core", +] + +[[package]] +name = "insta" +version = "1.47.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" +dependencies = [ + "console", + "once_cell", + "similar", + "tempfile", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libredox" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +dependencies = [ + "bitflags 2.11.1", + "libc", + "plain", + "redox_syscall", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "metal" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f569fb946490b5743ad69813cb19629130ce9374034abe31614a36402d18f99e" +dependencies = [ + "bitflags 2.11.1", + "block", + "core-graphics-types", + "foreign-types", + "log", + "objc", + "paste", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "minimp4" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dedfad4c0e1879118fe4fe2ecc1024dd8e9709eee96fe7c86ec4cfacd017b1db" +dependencies = [ + "libc", + "minimp4-sys", +] + +[[package]] +name = "minimp4-sys" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8b16b28870f2144fbc63ee0e4a8b2f7021ba201410478bf1617a3486eac740" +dependencies = [ + "bindgen 0.69.5", + "cc", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nasm-rs" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "706bf8a5e8c8ddb99128c3291d31bd21f4bcde17f0f4c20ec678d85c74faa149" +dependencies = [ + "log", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.11.1", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.11.1", + "block2", + "libc", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "openh264" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1af3a4d35290ba7a46d1ce69cb13ae740a2d72cc2ee00abee3c84bed3dbe5d" +dependencies = [ + "openh264-sys2", + "wide", +] + +[[package]] +name = "openh264-sys2" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a77c1e18503537113d77b1b1d05274e81fa9f44843c06be2d735adb19f7c9d" +dependencies = [ + "cc", + "nasm-rs", + "walkdir", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" +dependencies = [ + "bitflags 2.11.1", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.11.1", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.1", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + +[[package]] +name = "skia-bindings" +version = "0.93.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2359f7e30c9da3f322f8ca3d4ec0abbc12a40035ce758309db0cdab07b5d4476" +dependencies = [ + "bindgen 0.72.1", + "cc", + "flate2", + "heck", + "pkg-config", + "regex", + "serde_json", + "tar", + "toml", +] + +[[package]] +name = "skia-safe" +version = "0.93.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f9e837ea9d531c9efee8f980bfcdb7226b21db0285b0c3171d8be745829f940" +dependencies = [ + "bitflags 2.11.1", + "skia-bindings", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tar" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.1", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.1", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/packages/native-renderer/Cargo.toml b/packages/native-renderer/Cargo.toml new file mode 100644 index 000000000..195cb096a --- /dev/null +++ b/packages/native-renderer/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "hyperframes-native-renderer" +version = "0.1.0" +edition = "2021" + +[features] +default = ["metal-gpu"] +metal-gpu = ["skia-safe/gpu", "skia-safe/metal", "dep:metal", "dep:objc2", "dep:objc2-foundation"] +vulkan-gpu = ["skia-safe/gpu", "skia-safe/vulkan"] + +[dependencies] +base64 = "0.22" +skia-safe = { version = "0.93", features = ["textlayout"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +openh264 = "0.6" +dcv-color-primitives = "0.7" +minimp4 = "0.1" +metal = { version = "0.31", optional = true } +objc2 = { version = "0.6", optional = true } +objc2-foundation = { version = "0.3", features = ["NSObject"], optional = true } + +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } +insta = "1" + +[[bench]] +name = "render_bench" +harness = false diff --git a/packages/native-renderer/Dockerfile.test b/packages/native-renderer/Dockerfile.test new file mode 100644 index 000000000..f3453ca93 --- /dev/null +++ b/packages/native-renderer/Dockerfile.test @@ -0,0 +1,29 @@ +# HyperFrames Native Renderer — Linux CI Test Image +# +# Tests the Rust native renderer on Linux without GPU access. +# Validates: CPU raster rendering, FFmpeg pipe encoding, scene parsing, +# element painting, effects, image compositing, animated pipeline. +# +# Usage: +# cd packages/native-renderer +# docker build -f Dockerfile.test -t hyperframes-native:test . +# docker run --rm hyperframes-native:test + +FROM rust:1.88-bookworm + +RUN apt-get update && apt-get install -y --no-install-recommends \ + clang libclang-dev pkg-config \ + libfontconfig1-dev libfreetype6-dev \ + ninja-build python3 git \ + ffmpeg fonts-liberation fonts-dejavu-core fontconfig \ + && rm -rf /var/lib/apt/lists/* && fc-cache -fv + +WORKDIR /app +COPY Cargo.toml Cargo.lock* ./ +COPY src ./src +COPY tests ./tests + +# Build tests only (skip benchmarks — criterion is heavy) +RUN cargo test --release --no-default-features --no-run 2>&1 | tail -3 + +ENTRYPOINT ["cargo", "test", "--release", "--no-default-features", "--", "--test-threads=1"] diff --git a/packages/native-renderer/benches/render_bench.rs b/packages/native-renderer/benches/render_bench.rs new file mode 100644 index 000000000..88008f355 --- /dev/null +++ b/packages/native-renderer/benches/render_bench.rs @@ -0,0 +1,326 @@ +use std::collections::HashMap; + +use criterion::{criterion_group, criterion_main, Criterion}; +use hyperframes_native_renderer::paint::elements::paint_element; +use hyperframes_native_renderer::paint::{ImageCache, RenderSurface}; +use hyperframes_native_renderer::scene::{ + BakedElementState, BakedFrame, BakedTimeline, Color, Element, ElementKind, Rect, Scene, Style, +}; +use skia_safe::Color4f; + +/// Build a realistic 1080p scene: dark background root with 20 overlapping +/// card-style containers, each containing a text child. This approximates +/// a typical composition slide with layered UI elements. +fn build_test_scene() -> Scene { + let mut children = Vec::with_capacity(20); + + for i in 0..20u8 { + let fi = i as f32; + children.push(Element { + id: format!("card-{i}"), + kind: ElementKind::Container, + bounds: Rect { + x: 50.0 + fi * 10.0, + y: 50.0 + fi * 15.0, + width: 400.0, + height: 200.0, + }, + style: Style { + background_color: Some(Color { + r: i.wrapping_mul(12), + g: 100, + b: 200, + a: 220, + }), + opacity: 0.8, + border_radius: [12.0; 4], + overflow_hidden: true, + visibility: true, + ..Default::default() + }, + children: vec![Element { + id: format!("text-{i}"), + kind: ElementKind::Text { + content: format!("Card {i} — Hello World"), + }, + bounds: Rect { + x: 20.0, + y: 20.0, + width: 360.0, + height: 30.0, + }, + style: Style { + color: Some(Color { + r: 255, + g: 255, + b: 255, + a: 255, + }), + font_size: Some(24.0), + opacity: 1.0, + visibility: true, + ..Default::default() + }, + children: vec![], + }], + }); + } + + Scene { + width: 1920, + height: 1080, + elements: vec![Element { + id: "root".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 1920.0, + height: 1080.0, + }, + style: Style { + background_color: Some(Color { + r: 15, + g: 15, + b: 30, + a: 255, + }), + opacity: 1.0, + visibility: true, + ..Default::default() + }, + children, + }], + } +} + +fn bench_paint_frame(c: &mut Criterion) { + let scene = build_test_scene(); + let mut surface = RenderSurface::new_raster(1920, 1080).unwrap(); + + c.bench_function("paint_1080p_20_elements", |b| { + let mut images = ImageCache::new(); + b.iter(|| { + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &scene.elements { + paint_element(surface.canvas(), element, &mut images); + } + }); + }); + + c.bench_function("paint_and_encode_jpeg_1080p", |b| { + let mut images = ImageCache::new(); + b.iter(|| { + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &scene.elements { + paint_element(surface.canvas(), element, &mut images); + } + let _jpeg = surface.encode_jpeg(80).unwrap(); + }); + }); +} + +fn bench_gpu_paint_frame(c: &mut Criterion) { + let scene = build_test_scene(); + let mut surface = RenderSurface::new_gpu_or_raster(1920, 1080) + .expect("GPU or raster surface required for this benchmark"); + + c.bench_function("gpu_paint_1080p_20_elements", |b| { + let mut images = ImageCache::new(); + b.iter(|| { + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &scene.elements { + paint_element(surface.canvas(), element, &mut images); + } + surface.flush_and_submit(); + }); + }); + + c.bench_function("gpu_paint_and_readback_rgba_1080p", |b| { + let mut images = ImageCache::new(); + b.iter(|| { + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &scene.elements { + paint_element(surface.canvas(), element, &mut images); + } + surface.flush_and_submit(); + let _pixels = surface.read_pixels_rgba(); + }); + }); + + c.bench_function("gpu_paint_and_readback_bgra_1080p", |b| { + let mut images = ImageCache::new(); + b.iter(|| { + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &scene.elements { + paint_element(surface.canvas(), element, &mut images); + } + surface.flush_and_submit(); + let _pixels = surface.read_pixels_bgra(); + }); + }); +} + +/// Build a 30-frame timeline that slides all 20 cards upward with a fade-in. +/// Animates every card to stress the delta-apply + paint path realistically. +fn build_30_frame_timeline() -> BakedTimeline { + let frames = (0..30) + .map(|i| { + let progress = i as f32 / 29.0; + let mut elements = HashMap::new(); + for c in 0..20u8 { + elements.insert( + format!("card-{c}"), + BakedElementState { + opacity: progress, + translate_x: 0.0, + translate_y: 40.0 * (1.0 - progress), + scale_x: 1.0, + scale_y: 1.0, + rotate_deg: 0.0, + visibility: true, + }, + ); + } + BakedFrame { + frame_index: i, + time: i as f64 / 30.0, + elements, + } + }) + .collect(); + + BakedTimeline { + fps: 30, + duration: 1.0, + total_frames: 30, + frames, + } +} + +/// RENDER-ONLY: CPU paint + BGRA readback into pre-allocated buffer. +/// No I420 conversion, no encoding. Pure rendering throughput. +fn bench_render_only_30_frames(c: &mut Criterion) { + use hyperframes_native_renderer::paint::{ImageCache, RenderSurface}; + use hyperframes_native_renderer::paint::elements::paint_element; + + let scene = build_test_scene(); + let timeline = build_30_frame_timeline(); + let mut surface = RenderSurface::new_raster(1920, 1080).unwrap(); + let mut bgra_buf = vec![0u8; 1920 * 1080 * 4]; + + c.bench_function("render_only_30_frames_1080p", |b| { + let mut images = ImageCache::new(); + b.iter(|| { + for _frame in &timeline.frames { + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &scene.elements { + paint_element(surface.canvas(), element, &mut images); + } + surface.read_pixels_bgra_into(&mut bgra_buf).unwrap(); + } + }); + }); +} + +/// End-to-end RAW+ENCODE: fast render to I420 file, then FFmpeg batch. +fn bench_e2e_raw_encode_30_frames(c: &mut Criterion) { + use hyperframes_native_renderer::pipeline::{render_animated_raw_then_encode, RenderConfig}; + + let scene = build_test_scene(); + let timeline = build_30_frame_timeline(); + + c.bench_function("e2e_raw_encode_30_frames_1080p", |b| { + b.iter(|| { + let config = RenderConfig { + fps: 30, + duration_secs: 1.0, + quality: 80, + output_path: "/tmp/hyperframes-bench-raw.mp4".to_string(), + }; + let result = render_animated_raw_then_encode(&scene, &timeline, &config) + .expect("render_animated_raw_then_encode must succeed"); + assert_eq!(result.total_frames, 30); + }); + }); +} + +/// End-to-end NATIVE: CPU raster + openh264 + minimp4, NO FFmpeg. +fn bench_e2e_native_30_frames(c: &mut Criterion) { + use hyperframes_native_renderer::pipeline::{render_animated_native, RenderConfig}; + + let scene = build_test_scene(); + let timeline = build_30_frame_timeline(); + + c.bench_function("e2e_native_30_frames_1080p", |b| { + b.iter(|| { + let config = RenderConfig { + fps: 30, + duration_secs: 1.0, + quality: 80, + output_path: "/tmp/hyperframes-bench-native.mp4".to_string(), + }; + let result = render_animated_native(&scene, &timeline, &config) + .expect("render_animated_native must succeed"); + assert_eq!(result.total_frames, 30); + }); + }); +} + +/// End-to-end: CPU raster + JPEG encode + FFmpeg MJPEG pipe for 30 frames. +fn bench_e2e_gpu_jpeg_30_frames(c: &mut Criterion) { + use hyperframes_native_renderer::pipeline::{render_animated, RenderConfig}; + + let scene = build_test_scene(); + let timeline = build_30_frame_timeline(); + + c.bench_function("e2e_gpu_jpeg_30_frames_1080p", |b| { + b.iter(|| { + let config = RenderConfig { + fps: 30, + duration_secs: 1.0, + quality: 60, + output_path: "/tmp/hyperframes-bench-e2e-jpeg.mp4".to_string(), + }; + let result = render_animated(&scene, &timeline, &config) + .expect("render_animated must succeed"); + assert_eq!(result.total_frames, 30); + }); + }); +} + +/// End-to-end benchmark: GPU paint + raw pixel pipe + FFmpeg hw encode for 30 frames. +fn bench_e2e_gpu_30_frames(c: &mut Criterion) { + use hyperframes_native_renderer::pipeline::{render_animated_gpu, RenderConfig}; + + let scene = build_test_scene(); + let timeline = build_30_frame_timeline(); + + c.bench_function("e2e_gpu_30_frames_1080p", |b| { + b.iter(|| { + let config = RenderConfig { + fps: 30, + duration_secs: 1.0, + quality: 80, + output_path: "/tmp/hyperframes-bench-e2e.mp4".to_string(), + }; + let result = render_animated_gpu(&scene, &timeline, &config) + .expect("render_animated_gpu must succeed"); + assert_eq!(result.total_frames, 30); + }); + }); +} + +criterion_group!( + benches, + bench_paint_frame, + bench_gpu_paint_frame, + bench_render_only_30_frames, + bench_e2e_raw_encode_30_frames, + bench_e2e_native_30_frames, + bench_e2e_gpu_jpeg_30_frames, + bench_e2e_gpu_30_frames +); +#[cfg(any())] // dead code — kept for reference +criterion_group!(benches, bench_paint_frame); +criterion_main!(benches); diff --git a/packages/native-renderer/fixtures/simple-native/index.html b/packages/native-renderer/fixtures/simple-native/index.html new file mode 100644 index 000000000..6d7731aea --- /dev/null +++ b/packages/native-renderer/fixtures/simple-native/index.html @@ -0,0 +1,124 @@ + + + + + + Native Renderer Proof + + + +
+
+
+
+
+
+
+
+ + + + diff --git a/packages/native-renderer/fixtures/tier2-native/index.html b/packages/native-renderer/fixtures/tier2-native/index.html new file mode 100644 index 000000000..3b963fd8d --- /dev/null +++ b/packages/native-renderer/fixtures/tier2-native/index.html @@ -0,0 +1,119 @@ + + + + + + + +
+
+
+
+
Native Tier 2
+
Gradients, shadows, blur, borders, clip-path, blend mode.
+
Skia path
+
+
+ + + diff --git a/packages/native-renderer/scripts/compare-regression-fixtures.ts b/packages/native-renderer/scripts/compare-regression-fixtures.ts new file mode 100644 index 000000000..9fda7c26f --- /dev/null +++ b/packages/native-renderer/scripts/compare-regression-fixtures.ts @@ -0,0 +1,828 @@ +import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { + copyFileSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, join, relative, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; +import type { Writable } from "node:stream"; +import { createFileServer } from "../../producer/src/services/fileServer.js"; +import { ensureBrowser } from "../../cli/src/browser/manager.js"; +import { extractScene, type ExtractedScene, type SceneElement } from "../src/scene/extract.js"; +import { + detectNativeSupport, + type NativeSupportReport, + type NativeUnsupportedReason, +} from "../src/scene/support.js"; +import { bakeTimeline } from "../src/timeline/bake.js"; + +interface FixtureMeta { + name: string; + description: string; + tags: string[]; + renderConfig: { + fps: 24 | 30 | 60; + }; +} + +interface Fixture { + id: string; + dir: string; + srcDir: string; + compiledHtmlPath: string; + meta: FixtureMeta; +} + +interface BrowserInfo { + executablePath: string; +} + +interface BrowserLike { + newPage(): Promise; + close(): Promise; +} + +interface PageLike { + setViewport(viewport: { width: number; height: number }): Promise; + goto( + url: string, + options: { waitUntil: "domcontentloaded" | "load" | "networkidle0"; timeout?: number }, + ): Promise; + waitForFunction(pageFunction: string, options?: { timeout?: number }): Promise; + evaluate(pageFunction: string): Promise; + screenshot(options: { type: "jpeg"; quality: number }): Promise; + close(): Promise; +} + +interface PuppeteerLike { + launch(options: { + executablePath: string; + headless: boolean; + args: string[]; + }): Promise; +} + +interface FixtureResult { + id: string; + name: string; + status: "native-pass" | "native-review" | "fallback-required" | "failed"; + warnings: string[]; + error?: string; + fps: number; + duration: number; + sampleDuration: number; + width: number; + height: number; + cdp?: { + outputPath: string; + elapsedMs: number; + avgFrameMs: number; + }; + native?: { + outputPath: string; + extractionMs: number; + renderElapsedMs: number; + totalElapsedMs: number; + avgPaintMs: number; + }; + auto?: { + outputPath: string; + elapsedMs: number; + backend: "native" | "chrome-fallback"; + }; + support?: NativeSupportReport; + fidelity?: { + posterPsnrDb: PsnrDb; + sampledPsnrDb?: PsnrDb; + status: "excellent" | "review" | "mismatch"; + }; +} + +type PsnrDb = number | "inf"; + +function arg(name: string, fallback: string): string { + const index = process.argv.indexOf(name); + if (index === -1) return fallback; + return process.argv[index + 1] ?? fallback; +} + +function flag(name: string): boolean { + return process.argv.includes(name); +} + +function runChecked(command: string, args: string[], cwd: string): string { + const result = spawnSync(command, args, { cwd, encoding: "utf-8" }); + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(" ")} failed with ${result.status}\n${result.stdout}\n${result.stderr}`, + ); + } + return result.stdout.trim(); +} + +function discoverFixtures(testsDir: string, selectedIds: string[]): Fixture[] { + const selected = new Set(selectedIds.filter(Boolean)); + const fixtures: Fixture[] = []; + + for (const id of readdirSync(testsDir).sort()) { + if (selected.size > 0 && !selected.has(id)) continue; + const dir = join(testsDir, id); + if (!statSync(dir).isDirectory()) continue; + + const srcDir = join(dir, "src"); + const metaPath = join(dir, "meta.json"); + const compiledHtmlPath = join(dir, "output", "compiled.html"); + if (!existsSync(srcDir) || !existsSync(metaPath) || !existsSync(compiledHtmlPath)) continue; + + const meta = JSON.parse(readFileSync(metaPath, "utf-8")) as FixtureMeta; + fixtures.push({ id, dir, srcDir, compiledHtmlPath, meta }); + } + + return fixtures; +} + +function waitForProcess(child: ChildProcessWithoutNullStreams, command: string): Promise { + return new Promise((resolvePromise, reject) => { + let stderr = ""; + child.stderr.setEncoding("utf-8"); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) { + resolvePromise(); + } else { + reject(new Error(`${command} failed with ${code}\n${stderr}`)); + } + }); + }); +} + +function writeFrame(stdin: Writable, frame: Uint8Array): Promise { + return new Promise((resolvePromise, reject) => { + stdin.write(Buffer.from(frame), (error) => { + if (error) reject(error); + else resolvePromise(); + }); + }); +} + +async function renderCdpReference({ + browser, + fps, + duration, + url, + outputPath, + quality, + width, + height, +}: { + browser: BrowserLike; + fps: number; + duration: number; + url: string; + outputPath: string; + quality: number; + width: number; + height: number; +}): Promise<{ elapsedMs: number; avgFrameMs: number }> { + const frames = Math.max(1, Math.ceil(fps * duration)); + const ffmpeg = spawn("ffmpeg", [ + "-y", + "-f", + "image2pipe", + "-vcodec", + "mjpeg", + "-framerate", + String(fps), + "-i", + "-", + "-an", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + outputPath, + ]); + const ffmpegDone = waitForProcess(ffmpeg, "ffmpeg cdp reference"); + + const page = await browser.newPage(); + const start = performance.now(); + try { + await page.setViewport({ width, height }); + await page.goto(url, { waitUntil: "load", timeout: 90_000 }); + await page.waitForFunction(`!!(window.__hf && typeof window.__hf.seek === "function")`, { + timeout: 45_000, + }); + + for (let frame = 0; frame < frames; frame++) { + const time = frame / fps; + await page.evaluate(`void(window.__hf.seek(${JSON.stringify(time)}))`); + const jpeg = await page.screenshot({ type: "jpeg", quality }); + await writeFrame(ffmpeg.stdin, jpeg); + } + } finally { + ffmpeg.stdin.end(); + await page.close(); + } + await ffmpegDone; + + const elapsedMs = Math.round(performance.now() - start); + return { elapsedMs, avgFrameMs: Number((elapsedMs / frames).toFixed(2)) }; +} + +function collectSceneWarnings(scene: ExtractedScene): string[] { + const warnings = new Set(); + + function visit(element: SceneElement): void { + if (element.kind.type === "Video" && !element.kind.src) { + warnings.add("video element has no resolved source"); + } + if (element.style.background_gradient) warnings.add("gradient extraction is partial"); + for (const child of element.children) visit(child); + } + + for (const element of scene.elements) visit(element); + return Array.from(warnings); +} + +function formatSupportReason(reason: NativeUnsupportedReason): string { + return `${reason.elementId}: ${reason.property}=${reason.value} (${reason.reason})`; +} + +function rewriteLocalImageSources( + scene: ExtractedScene, + serverUrl: string, + compiledDir: string, + srcDir: string, +): void { + function mapSrc(src: string): string { + if (!src.startsWith(serverUrl)) return src; + const url = new URL(src); + const relPath = decodeURIComponent(url.pathname.replace(/^\//, "")); + const compiledPath = join(compiledDir, relPath); + if (existsSync(compiledPath)) return compiledPath; + const sourcePath = join(srcDir, relPath); + if (existsSync(sourcePath)) return sourcePath; + return src; + } + + function visit(element: SceneElement): void { + if (element.kind.type === "Image" || element.kind.type === "Video") { + element.kind.src = mapSrc(element.kind.src); + } + if (element.style.background_image) { + element.style.background_image.src = mapSrc(element.style.background_image.src); + } + for (const child of element.children) visit(child); + } + + for (const element of scene.elements) visit(element); +} + +function extractPoster(videoPath: string, posterPath: string, time: number): void { + const result = spawnSync( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-ss", + String(time), + "-i", + videoPath, + "-frames:v", + "1", + posterPath, + "-y", + ], + { encoding: "utf-8" }, + ); + if (result.status !== 0) { + throw new Error(result.stderr || `failed to extract poster for ${videoPath}`); + } +} + +function computePsnrDb(referencePath: string, nativePath: string): PsnrDb | undefined { + const result = spawnSync( + "ffmpeg", + ["-hide_banner", "-i", referencePath, "-i", nativePath, "-lavfi", "psnr", "-f", "null", "-"], + { encoding: "utf-8" }, + ); + if (result.status !== 0) return undefined; + + const output = `${result.stdout}\n${result.stderr}`; + const match = output.match(/average:([0-9.]+|inf)/); + if (!match) return undefined; + + return match[1] === "inf" ? "inf" : Number(match[1]); +} + +function psnrValue(psnr: PsnrDb | undefined): number | undefined { + if (psnr === undefined) return undefined; + return psnr === "inf" ? Number.POSITIVE_INFINITY : psnr; +} + +function classifyPsnr(...values: (PsnrDb | undefined)[]): "excellent" | "review" | "mismatch" { + const numericValues = values + .map((value) => psnrValue(value)) + .filter((value): value is number => value !== undefined); + if (numericValues.length === 0) return "mismatch"; + const floor = Math.min(...numericValues); + return floor >= 40 ? "excellent" : floor >= 30 ? "review" : "mismatch"; +} + +function computeFidelity({ + referencePosterPath, + nativePosterPath, + referenceVideoPath, + nativeVideoPath, +}: { + referencePosterPath: string; + nativePosterPath: string; + referenceVideoPath: string; + nativeVideoPath: string; +}): FixtureResult["fidelity"] | undefined { + const posterPsnrDb = computePsnrDb(referencePosterPath, nativePosterPath); + if (posterPsnrDb === undefined) return undefined; + const sampledPsnrDb = computePsnrDb(referenceVideoPath, nativeVideoPath); + + return { + posterPsnrDb, + sampledPsnrDb, + status: classifyPsnr(posterPsnrDb, sampledPsnrDb), + }; +} + +function formatPsnr(psnr: PsnrDb | undefined): string { + if (psnr === undefined) return "n/a"; + return psnr === "inf" ? "inf" : `${psnr.toFixed(2)} dB`; +} + +function escapeHtml(value: string): string { + return value + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + +function artifactRel(root: string, path: string): string { + return relative(root, path).split("/").map(encodeURIComponent).join("/"); +} + +function writeReport(results: FixtureResult[], artifactsDir: string, maxDuration: number): void { + const counts = { + nativePass: results.filter((r) => r.status === "native-pass").length, + nativeReview: results.filter((r) => r.status === "native-review").length, + fallbackRequired: results.filter((r) => r.status === "fallback-required").length, + failed: results.filter((r) => r.status === "failed").length, + }; + const totals = results.reduce( + (acc, result) => { + acc.cdp += result.cdp?.elapsedMs ?? 0; + acc.auto += result.auto?.elapsedMs ?? 0; + if (result.native) { + const frames = Math.max(1, Math.ceil(result.fps * result.sampleDuration)); + acc.native += result.native.totalElapsedMs; + acc.nativeExtraction += result.native.extractionMs; + acc.nativeRender += result.native.renderElapsedMs; + acc.nativePaint += result.native.avgPaintMs * frames; + } + return acc; + }, + { cdp: 0, auto: 0, native: 0, nativeExtraction: 0, nativeRender: 0, nativePaint: 0 }, + ); + const totalAutoSpeedup = + totals.cdp > 0 && totals.auto > 0 ? Number((totals.cdp / totals.auto).toFixed(2)) : null; + const totalNativeSpeedup = + totals.cdp > 0 && totals.native > 0 ? Number((totals.cdp / totals.native).toFixed(2)) : null; + + const rows = results + .map((result) => { + const fixtureDir = join(artifactsDir, result.id); + const nativeSpeedup = + result.cdp && result.native + ? Number((result.cdp.elapsedMs / result.native.totalElapsedMs).toFixed(2)) + : null; + const autoSpeedup = + result.cdp && result.auto + ? Number((result.cdp.elapsedMs / result.auto.elapsedMs).toFixed(2)) + : null; + const posterPsnr = formatPsnr(result.fidelity?.posterPsnrDb); + const sampledPsnr = formatPsnr(result.fidelity?.sampledPsnrDb); + const cdpPoster = existsSync(join(fixtureDir, "cdp.jpg")) + ? `CDP poster for ${escapeHtml(result.id)}` + : `
CDP unavailable
`; + const autoPoster = existsSync(join(fixtureDir, "auto.jpg")) + ? `Auto backend poster for ${escapeHtml(result.id)}` + : `
Auto output unavailable
`; + const cdpVideo = result.cdp + ? `` + : ""; + const autoVideo = result.auto + ? `` + : ""; + const warnings = result.warnings.length + ? `
    ${result.warnings.map((warning) => `
  • ${escapeHtml(warning)}
    • `).join("")}
` + : `

No native coverage warnings recorded.

`; + const supportReasons = result.support?.reasons.length + ? `
    ${result.support.reasons + .map((reason) => `
  • ${escapeHtml(formatSupportReason(reason))}
    • `) + .join("")}
` + : `

Support detector found no fallback-required features.

`; + const error = result.error ? `
${escapeHtml(result.error)}
` : ""; + + return `
+
+
+

${escapeHtml(result.id)}

+

${escapeHtml(result.name)}

+
+ ${result.status} +
+
+ ${result.width}x${result.height} + ${result.fps}fps + ${result.sampleDuration.toFixed(2)}s sampled + ${result.cdp ? `CDP ${result.cdp.elapsedMs}ms` : ""} + ${ + result.native + ? `Native ${result.native.totalElapsedMs}ms + Extract ${result.native.extractionMs}ms + Render ${result.native.renderElapsedMs}ms + Avg paint ${result.native.avgPaintMs.toFixed(2)}ms` + : "" + } + ${result.auto ? `Auto ${result.auto.elapsedMs}ms (${result.auto.backend})` : ""} + ${nativeSpeedup ? `${nativeSpeedup}x native speed` : ""} + ${autoSpeedup ? `${autoSpeedup}x auto speed` : ""} + Poster PSNR ${posterPsnr} + Sampled PSNR ${sampledPsnr} + ${result.fidelity ? `Fidelity ${result.fidelity.status}` : ""} +
+
+
+

CDP

+ ${cdpPoster} + ${cdpVideo} +
+
+

Auto Output

+ ${autoPoster} + ${autoVideo} +
+
+
+ Notes +

Poster PSNR compares one sampled CDP frame against auto output. Sampled PSNR compares the full clipped video segment rendered by both backends. Inspect the side-by-side videos for final visual signoff.

+ ${supportReasons} + ${warnings} + ${error} +
+
`; + }) + .join("\n"); + + const html = ` + + + + + Native Renderer Regression Comparison + + + +
+

Native Renderer Regression Comparison

+

This report renders each fixture through the existing Chrome CDP reference path and the production auto backend. Auto uses Rust/Skia native rendering only for supported fixtures and falls back to Chrome when the support detector finds browser features the native compositor cannot prove faithful yet.

+
+ ${results.length} fixtures + ${counts.nativePass} native-pass + ${counts.nativeReview} native-review + ${counts.fallbackRequired} fallback-required + ${counts.failed} failed + ${totalAutoSpeedup ? `${totalAutoSpeedup}x aggregate auto speed` : ""} + ${totalNativeSpeedup ? `${totalNativeSpeedup}x native-only speed` : ""} + CDP ${totals.cdp}ms + Auto ${totals.auto}ms + Native ${totals.native}ms + Native extraction ${totals.nativeExtraction}ms + Native render ${totals.nativeRender}ms + Native paint ${totals.nativePaint.toFixed(2)}ms + first ${maxDuration}s sampled per fixture +
+ ${rows} +
+ +`; + + writeFileSync(join(artifactsDir, "index.html"), html, "utf-8"); +} + +async function main(): Promise { + const repoRoot = resolve(dirname(new URL(import.meta.url).pathname), "../../.."); + const artifactsDir = resolve( + arg("--artifacts", join(repoRoot, `qa-artifacts/native-regression-comparison-${Date.now()}`)), + ); + const maxDuration = Number(arg("--max-duration", "1")); + const quality = Number(arg("--quality", "80")); + const selectedFixtures = arg("--fixtures", "") + .split(",") + .map((id) => id.trim()) + .filter(Boolean); + const limit = Number(arg("--limit", "0")); + const keep = flag("--keep"); + + if (!keep && existsSync(artifactsDir)) rmSync(artifactsDir, { recursive: true, force: true }); + mkdirSync(artifactsDir, { recursive: true }); + + const fixtures = discoverFixtures( + join(repoRoot, "packages/producer/tests"), + selectedFixtures, + ).slice(0, limit > 0 ? limit : undefined); + if (fixtures.length === 0) { + throw new Error("No fixtures found"); + } + + runChecked( + "cargo", + ["build", "--release", "--bin", "render_native"], + join(repoRoot, "packages/native-renderer"), + ); + + const browserInfo = (await ensureBrowser()) as BrowserInfo; + const cliRequire = createRequire(join(repoRoot, "packages/cli/package.json")); + const puppeteer = cliRequire("puppeteer-core") as PuppeteerLike; + const browser = await puppeteer.launch({ + executablePath: browserInfo.executablePath, + headless: true, + args: ["--allow-file-access-from-files", "--disable-web-security"], + }); + + const results: FixtureResult[] = []; + + try { + for (const fixture of fixtures) { + const fixtureDir = join(artifactsDir, fixture.id); + const compiledDir = join(fixtureDir, "compiled"); + mkdirSync(compiledDir, { recursive: true }); + writeFileSync( + join(compiledDir, "index.html"), + readFileSync(fixture.compiledHtmlPath, "utf-8"), + ); + + const server = await createFileServer({ projectDir: fixture.srcDir, compiledDir }); + const url = `${server.url}/index.html`; + const result: FixtureResult = { + id: fixture.id, + name: fixture.meta.name, + status: "failed", + warnings: [], + fps: fixture.meta.renderConfig.fps, + duration: 0, + sampleDuration: 0, + width: 0, + height: 0, + }; + + try { + const page = await browser.newPage(); + let scene: ExtractedScene | null = null; + let nativeExtractionMs = 0; + try { + await page.goto(url, { waitUntil: "load", timeout: 90_000 }); + await page.waitForFunction(`!!(window.__hf && typeof window.__hf.seek === "function")`, { + timeout: 45_000, + }); + const metadata = await page.evaluate<{ + width: number; + height: number; + duration: number; + }>(`(() => { + const root = document.querySelector("[data-composition-id]"); + const hfDuration = Number(window.__hf?.duration ?? 0); + return { + width: Number(root?.getAttribute("data-width") ?? root?.clientWidth ?? 0), + height: Number(root?.getAttribute("data-height") ?? root?.clientHeight ?? 0), + duration: hfDuration > 0 ? hfDuration : Number(root?.getAttribute("data-duration") ?? 1), + }; + })()`); + + result.width = metadata.width || 1920; + result.height = metadata.height || 1080; + result.duration = metadata.duration || 1; + result.sampleDuration = Math.min(result.duration, maxDuration); + + result.support = await detectNativeSupport(page, result.width, result.height); + writeFileSync(join(fixtureDir, "support.json"), JSON.stringify(result.support, null, 2)); + if (result.support.supported) { + scene = await extractScene(page, result.width, result.height); + rewriteLocalImageSources(scene, server.url, compiledDir, fixture.srcDir); + result.warnings.push(...collectSceneWarnings(scene)); + + const extractionStart = performance.now(); + const timeline = await bakeTimeline(page, result.fps, result.sampleDuration); + nativeExtractionMs = Math.round(performance.now() - extractionStart); + + writeFileSync(join(fixtureDir, "scene.json"), JSON.stringify(scene, null, 2)); + writeFileSync(join(fixtureDir, "timeline.json"), JSON.stringify(timeline, null, 2)); + } + } finally { + await page.close(); + } + + const cdpOutputPath = join(fixtureDir, "cdp.mp4"); + const cdp = await renderCdpReference({ + browser, + fps: result.fps, + duration: result.sampleDuration, + url, + outputPath: cdpOutputPath, + quality, + width: result.width, + height: result.height, + }); + result.cdp = { outputPath: cdpOutputPath, ...cdp }; + + const autoOutputPath = join(fixtureDir, "auto.mp4"); + if (!result.support?.supported) { + copyFileSync(cdpOutputPath, autoOutputPath); + result.auto = { + outputPath: autoOutputPath, + elapsedMs: cdp.elapsedMs, + backend: "chrome-fallback", + }; + result.status = "fallback-required"; + } else { + if (!scene) throw new Error("native support was true, but no scene was extracted"); + const nativeStart = performance.now(); + const nativeStdout = runChecked( + join(repoRoot, "packages/native-renderer/target/release/render_native"), + [ + "--scene", + join(fixtureDir, "scene.json"), + "--timeline", + join(fixtureDir, "timeline.json"), + "--output", + autoOutputPath, + "--fps", + String(result.fps), + "--duration", + String(result.sampleDuration), + "--quality", + String(quality), + ], + repoRoot, + ); + const renderer = JSON.parse(nativeStdout) as { totalMs: number; avgPaintMs: number }; + const totalElapsedMs = Math.round(performance.now() - nativeStart) + nativeExtractionMs; + result.native = { + outputPath: autoOutputPath, + extractionMs: nativeExtractionMs, + renderElapsedMs: Math.round(renderer.totalMs ?? 0), + totalElapsedMs, + avgPaintMs: Number(renderer.avgPaintMs ?? 0), + }; + result.auto = { + outputPath: autoOutputPath, + elapsedMs: totalElapsedMs, + backend: "native", + }; + } + + const posterTime = Math.min(0.5, Math.max(0, result.sampleDuration - 1 / result.fps)); + const cdpPosterPath = join(fixtureDir, "cdp.jpg"); + const autoPosterPath = join(fixtureDir, "auto.jpg"); + extractPoster(cdpOutputPath, cdpPosterPath, posterTime); + extractPoster(autoOutputPath, autoPosterPath, posterTime); + result.fidelity = computeFidelity({ + referencePosterPath: cdpPosterPath, + nativePosterPath: autoPosterPath, + referenceVideoPath: cdpOutputPath, + nativeVideoPath: autoOutputPath, + }); + + if (result.status !== "fallback-required") { + result.status = + result.warnings.length > 0 || result.fidelity?.status !== "excellent" + ? "native-review" + : "native-pass"; + } + } catch (error) { + result.error = error instanceof Error ? error.message : String(error); + result.status = "failed"; + } finally { + server.close(); + } + + results.push(result); + writeFileSync(join(artifactsDir, "results.json"), JSON.stringify(results, null, 2)); + console.log( + JSON.stringify({ + id: result.id, + status: result.status, + cdpMs: result.cdp?.elapsedMs ?? null, + nativeMs: result.native?.totalElapsedMs ?? null, + autoMs: result.auto?.elapsedMs ?? null, + autoBackend: result.auto?.backend ?? null, + posterPsnrDb: result.fidelity?.posterPsnrDb ?? null, + sampledPsnrDb: result.fidelity?.sampledPsnrDb ?? null, + fidelityStatus: result.fidelity?.status ?? null, + warnings: result.warnings, + supportReasons: result.support?.reasons.map(formatSupportReason) ?? [], + error: result.error ?? null, + }), + ); + } + } finally { + await browser.close(); + } + + writeReport(results, artifactsDir, maxDuration); + console.log( + JSON.stringify({ + report: join(artifactsDir, "index.html"), + results: join(artifactsDir, "results.json"), + }), + ); +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)); + process.exit(1); +}); diff --git a/packages/native-renderer/scripts/prove-native-render.ts b/packages/native-renderer/scripts/prove-native-render.ts new file mode 100644 index 000000000..be7e76b9b --- /dev/null +++ b/packages/native-renderer/scripts/prove-native-render.ts @@ -0,0 +1,521 @@ +import { + execFileSync, + spawn, + spawnSync, + type ChildProcessWithoutNullStreams, +} from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, join, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; +import type { Writable } from "node:stream"; +import { pathToFileURL } from "node:url"; +import { ensureBrowser } from "../../cli/src/browser/manager.js"; +import { extractScene } from "../src/scene/extract.js"; +import { detectNativeSupport, type NativeSupportReport } from "../src/scene/support.js"; +import { bakeTimeline } from "../src/timeline/bake.js"; + +interface ProofSummary { + projectDir: string; + artifactsDir: string; + reportPath: string; + chrome: { + outputPath: string; + elapsedMs: number; + frames: number; + avgFrameMs: number; + ffprobe: unknown; + }; + native: { + outputPath: string; + extractionMs: number; + renderElapsedMs: number; + totalElapsedMs: number; + renderer: RenderNativeOutput; + ffprobe: unknown; + }; + fidelity?: { + sampledPsnrDb: number | "inf"; + status: "excellent" | "review" | "mismatch"; + }; + support: NativeSupportReport; + speedup: { + renderOnlyVsChrome: number; + extractionPlusRenderVsChrome: number; + }; +} + +interface RenderNativeOutput { + frames?: number; + totalMs?: number; + avgPaintMs?: number; + outputPath?: string; +} + +function arg(name: string, fallback: string): string { + const index = process.argv.indexOf(name); + if (index === -1) return fallback; + return process.argv[index + 1] ?? fallback; +} + +function hasFlag(name: string): boolean { + return process.argv.includes(name); +} + +function ffprobe(path: string): unknown { + const raw = execFileSync( + "ffprobe", + [ + "-v", + "error", + "-show_entries", + "format=duration,size", + "-show_entries", + "stream=codec_name,width,height,r_frame_rate", + "-of", + "json", + path, + ], + { encoding: "utf-8" }, + ); + return JSON.parse(raw); +} + +function extractPoster(videoPath: string, posterPath: string, time: number): void { + const result = spawnSync( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-ss", + String(time), + "-i", + videoPath, + "-frames:v", + "1", + posterPath, + "-y", + ], + { encoding: "utf-8" }, + ); + if (result.status !== 0) { + throw new Error(result.stderr || `failed to extract poster for ${videoPath}`); + } +} + +function computePsnrDb(referencePath: string, nativePath: string): number | "inf" | undefined { + const result = spawnSync( + "ffmpeg", + ["-hide_banner", "-i", referencePath, "-i", nativePath, "-lavfi", "psnr", "-f", "null", "-"], + { encoding: "utf-8" }, + ); + if (result.status !== 0) return undefined; + + const output = `${result.stdout}\n${result.stderr}`; + const match = output.match(/average:([0-9.]+|inf)/); + if (!match) return undefined; + return match[1] === "inf" ? "inf" : Number(match[1]); +} + +function classifyPsnr(psnr: number | "inf" | undefined): "excellent" | "review" | "mismatch" { + const numeric = psnr === "inf" ? Number.POSITIVE_INFINITY : psnr; + if (numeric === undefined) return "mismatch"; + return numeric >= 40 ? "excellent" : numeric >= 30 ? "review" : "mismatch"; +} + +function formatPsnr(psnr: number | "inf" | undefined): string { + if (psnr === undefined) return "n/a"; + return psnr === "inf" ? "inf" : `${psnr.toFixed(2)} dB`; +} + +function escapeHtml(value: string): string { + return value + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + +function writeProofReport(summary: ProofSummary): void { + const html = ` + + + + + Native Renderer Supported Proof + + + +
+

Native Renderer Supported Proof

+

This canary uses only the currently supported native subset: opaque root, stable IDs, solid boxes, and opacity animation. Broader fixtures must pass the support detector before using native.

+
+ Chrome ${summary.chrome.elapsedMs}ms + Native render ${summary.native.renderElapsedMs}ms + Native total ${summary.native.totalElapsedMs}ms + Avg paint ${Number(summary.native.renderer.avgPaintMs ?? 0).toFixed(2)}ms + ${summary.speedup.renderOnlyVsChrome}x render-only speed + ${summary.speedup.extractionPlusRenderVsChrome}x extraction+render speed + Sampled PSNR ${formatPsnr(summary.fidelity?.sampledPsnrDb)} + Fidelity ${summary.fidelity?.status ?? "n/a"} +
+
+
+

Chrome CDP

+ Chrome CDP poster + +
+
+

Native

+ Native poster + +
+
+

Support

+ 
${escapeHtml(JSON.stringify(summary.support, null, 2))}
+
+ +`; + + writeFileSync(summary.reportPath, html, "utf-8"); +} + +function runChecked(command: string, args: string[], cwd: string): string { + const result = spawnSync(command, args, { cwd, encoding: "utf-8" }); + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(" ")} failed with ${result.status}\n${result.stdout}\n${result.stderr}`, + ); + } + return result.stdout.trim(); +} + +function waitForProcess(child: ChildProcessWithoutNullStreams, command: string): Promise { + return new Promise((resolvePromise, reject) => { + let stderr = ""; + child.stderr.setEncoding("utf-8"); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) { + resolvePromise(); + } else { + reject(new Error(`${command} failed with ${code}\n${stderr}`)); + } + }); + }); +} + +function writeFrame(stdin: Writable, frame: Uint8Array): Promise { + return new Promise((resolvePromise, reject) => { + stdin.write(Buffer.from(frame), (error) => { + if (error) reject(error); + else resolvePromise(); + }); + }); +} + +async function renderChromeCdpReference({ + executablePath, + fps, + duration, + projectDir, + outputPath, + puppeteer, + quality, + width, + height, +}: { + executablePath: string; + fps: number; + duration: number; + projectDir: string; + outputPath: string; + puppeteer: { + launch(options: { executablePath: string; headless: boolean; args: string[] }): Promise<{ + newPage(): Promise<{ + setViewport(viewport: { width: number; height: number }): Promise; + goto(url: string, options: { waitUntil: "networkidle0" }): Promise; + waitForFunction(pageFunction: string): Promise; + evaluate(pageFunction: string): Promise; + screenshot(options: { type: "jpeg"; quality: number }): Promise; + }>; + close(): Promise; + }>; + }; + quality: number; + width: number; + height: number; +}): Promise<{ elapsedMs: number; frames: number; avgFrameMs: number }> { + const browser = await puppeteer.launch({ + executablePath, + headless: true, + args: ["--allow-file-access-from-files", "--disable-web-security"], + }); + + const frames = Math.ceil(fps * duration); + const ffmpeg = spawn("ffmpeg", [ + "-y", + "-f", + "image2pipe", + "-vcodec", + "mjpeg", + "-framerate", + String(fps), + "-i", + "-", + "-an", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + outputPath, + ]); + + const ffmpegDone = waitForProcess(ffmpeg, "ffmpeg chrome cdp reference"); + const start = performance.now(); + try { + const page = await browser.newPage(); + await page.setViewport({ width, height }); + await page.goto(pathToFileURL(join(projectDir, "index.html")).href, { + waitUntil: "networkidle0", + }); + await page.waitForFunction(`!!(window.__hf && typeof window.__hf.seek === "function")`); + + for (let frame = 0; frame < frames; frame++) { + const time = frame / fps; + await page.evaluate(`void(window.__hf.seek(${JSON.stringify(time)}))`); + const jpeg = await page.screenshot({ type: "jpeg", quality }); + await writeFrame(ffmpeg.stdin, jpeg); + } + } finally { + ffmpeg.stdin.end(); + await browser.close(); + } + await ffmpegDone; + + const elapsedMs = Math.round(performance.now() - start); + return { + elapsedMs, + frames, + avgFrameMs: Number((elapsedMs / frames).toFixed(2)), + }; +} + +async function main(): Promise { + const repoRoot = resolve(dirname(new URL(import.meta.url).pathname), "../../.."); + const projectDir = resolve( + arg("--project", join(repoRoot, "packages/native-renderer/fixtures/simple-native")), + ); + const artifactsDir = resolve( + arg("--artifacts", join(repoRoot, "qa-artifacts/native-renderer-proof")), + ); + const fps = Number(arg("--fps", "30")); + const quality = Number(arg("--quality", "80")); + mkdirSync(artifactsDir, { recursive: true }); + const cliRequire = createRequire(join(repoRoot, "packages/cli/package.json")); + const puppeteer = cliRequire("puppeteer-core"); + + const scenePath = join(artifactsDir, "scene.json"); + const timelinePath = join(artifactsDir, "timeline.json"); + const supportPath = join(artifactsDir, "support.json"); + const nativeOutputPath = join(artifactsDir, "native.mp4"); + const chromeOutputPath = join(artifactsDir, "chrome-cdp.mp4"); + const nativePosterPath = join(artifactsDir, "native.jpg"); + const chromePosterPath = join(artifactsDir, "chrome-cdp.jpg"); + const summaryPath = join(artifactsDir, "summary.json"); + const reportPath = join(artifactsDir, "index.html"); + + const browserInfo = await ensureBrowser(); + const browser = await puppeteer.launch({ + executablePath: browserInfo.executablePath, + headless: true, + args: ["--allow-file-access-from-files", "--disable-web-security"], + }); + + let width = 0; + let height = 0; + let duration = 0; + let extractionMs = 0; + let support: NativeSupportReport = { supported: false, reasons: [] }; + try { + const page = await browser.newPage(); + const htmlPath = join(projectDir, "index.html"); + await page.goto(pathToFileURL(htmlPath).href, { waitUntil: "networkidle0" }); + await page.waitForFunction(() => { + const hf = (window as unknown as { __hf?: { seek?: unknown } }).__hf; + return Boolean(hf && typeof hf.seek === "function"); + }); + + const metadata = await page.evaluate(() => { + const root = document.querySelector("[data-composition-id]"); + const hf = (window as unknown as { __hf?: { duration?: number } }).__hf; + return { + width: Number(root?.dataset.width ?? root?.clientWidth ?? 0), + height: Number(root?.dataset.height ?? root?.clientHeight ?? 0), + duration: Number(root?.dataset.duration ?? hf?.duration ?? 1), + }; + }); + width = metadata.width; + height = metadata.height; + duration = metadata.duration; + + support = await detectNativeSupport(page, width, height); + writeFileSync(supportPath, JSON.stringify(support, null, 2)); + if (!support.supported && !hasFlag("--allow-unsupported")) { + throw new Error( + "Native renderer support check failed:\n" + + support.reasons + .map( + (reason) => + `- ${reason.elementId}: ${reason.property}=${reason.value} (${reason.reason})`, + ) + .join("\n"), + ); + } + + const extractionStart = performance.now(); + const scene = await extractScene(page, width, height); + const timeline = await bakeTimeline(page, fps, duration); + extractionMs = Math.round(performance.now() - extractionStart); + + writeFileSync(scenePath, JSON.stringify(scene, null, 2)); + writeFileSync(timelinePath, JSON.stringify(timeline, null, 2)); + } finally { + await browser.close(); + } + + runChecked( + "cargo", + ["build", "--release", "--bin", "render_native"], + join(repoRoot, "packages/native-renderer"), + ); + + const nativeStart = performance.now(); + const nativeStdout = runChecked( + join(repoRoot, "packages/native-renderer/target/release/render_native"), + [ + "--scene", + scenePath, + "--timeline", + timelinePath, + "--output", + nativeOutputPath, + "--fps", + String(fps), + "--duration", + String(duration), + "--quality", + String(quality), + ], + repoRoot, + ); + const nativeTotalElapsedMs = Math.round(performance.now() - nativeStart) + extractionMs; + const renderer = JSON.parse(nativeStdout) as RenderNativeOutput; + + const chrome = await renderChromeCdpReference({ + executablePath: browserInfo.executablePath, + fps, + duration, + projectDir, + outputPath: chromeOutputPath, + puppeteer, + quality, + width, + height, + }); + + const nativeRenderElapsedMs = Math.round(renderer.totalMs ?? 0); + extractPoster(chromeOutputPath, chromePosterPath, Math.min(0.5, Math.max(0, duration - 1 / fps))); + extractPoster(nativeOutputPath, nativePosterPath, Math.min(0.5, Math.max(0, duration - 1 / fps))); + const sampledPsnrDb = computePsnrDb(chromeOutputPath, nativeOutputPath); + const summary: ProofSummary = { + projectDir, + artifactsDir, + reportPath, + chrome: { + outputPath: chromeOutputPath, + elapsedMs: chrome.elapsedMs, + frames: chrome.frames, + avgFrameMs: chrome.avgFrameMs, + ffprobe: ffprobe(chromeOutputPath), + }, + native: { + outputPath: nativeOutputPath, + extractionMs, + renderElapsedMs: nativeRenderElapsedMs, + totalElapsedMs: nativeTotalElapsedMs, + renderer, + ffprobe: ffprobe(nativeOutputPath), + }, + fidelity: + sampledPsnrDb !== undefined + ? { + sampledPsnrDb, + status: classifyPsnr(sampledPsnrDb), + } + : undefined, + support, + speedup: { + renderOnlyVsChrome: Number((chrome.elapsedMs / nativeRenderElapsedMs).toFixed(2)), + extractionPlusRenderVsChrome: Number((chrome.elapsedMs / nativeTotalElapsedMs).toFixed(2)), + }, + }; + + writeFileSync(summaryPath, JSON.stringify(summary, null, 2)); + writeProofReport(summary); + process.stdout.write(readFileSync(summaryPath, "utf-8") + "\n"); +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/packages/native-renderer/scripts/render-composition.ts b/packages/native-renderer/scripts/render-composition.ts new file mode 100644 index 000000000..ad0c7bc5c --- /dev/null +++ b/packages/native-renderer/scripts/render-composition.ts @@ -0,0 +1,153 @@ +#!/usr/bin/env tsx +/** + * End-to-end native render pipeline. + * + * Chrome is used ONCE (~200ms) to extract the layout tree and bake + * the animation timeline. The Rust binary then renders all frames + * at native Skia speed without Chrome. + * + * Usage: + * tsx scripts/render-composition.ts [--fps 30] [--cpu] + * + * The composition-dir must contain an index.html with window.__hf protocol. + */ + +import { existsSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { resolve, join, dirname } from "node:path"; +import { spawnSync } from "node:child_process"; +import { + createCaptureSession, + initializeSession, + closeCaptureSession, + getCompositionDuration, + createFileServer, +} from "@hyperframes/engine"; +import { extractScene } from "../src/scene/extract.js"; +import { bakeTimeline } from "../src/timeline/bake.js"; + +const args = process.argv.slice(2); +if (args.length < 2 || args.includes("--help")) { + console.error( + "usage: tsx scripts/render-composition.ts [--fps 30] [--cpu]", + ); + process.exit(2); +} + +const compositionDir = resolve(args[0]); +const outputPath = resolve(args[1]); +const fps = parseInt(args[args.indexOf("--fps") + 1] || "30", 10); +const forceCpu = args.includes("--cpu"); + +const indexHtml = join(compositionDir, "index.html"); +if (!existsSync(indexHtml)) { + console.error(`No index.html found in ${compositionDir}`); + process.exit(1); +} + +const workDir = join(dirname(outputPath), ".native-work"); +mkdirSync(workDir, { recursive: true }); + +async function main() { + const t0 = Date.now(); + + // Step 1: Start file server for the composition + console.log("[native] Starting file server..."); + const fileServer = await createFileServer({ + projectDir: compositionDir, + compiledDir: compositionDir, + port: 0, + }); + + // Step 2: Open Chrome, navigate, wait for __hf + console.log("[native] Launching Chrome for scene extraction..."); + const session = await createCaptureSession(fileServer.url, workDir, { + width: 1920, + height: 1080, + fps, + }); + await initializeSession(session); + + // Step 3: Get composition duration + const duration = await getCompositionDuration(session); + console.log(`[native] Duration: ${duration}s, FPS: ${fps}, Frames: ${Math.ceil(duration * fps)}`); + + // Step 4: Extract scene graph + console.log("[native] Extracting scene graph..."); + const scene = await extractScene(session.page, 1920, 1080); + const scenePath = join(workDir, "scene.json"); + writeFileSync(scenePath, JSON.stringify(scene, null, 2)); + console.log(`[native] Scene: ${scene.elements.length} root elements → ${scenePath}`); + + // Step 5: Bake timeline + console.log("[native] Baking animation timeline..."); + const timeline = await bakeTimeline(session.page, fps, duration); + const timelinePath = join(workDir, "timeline.json"); + writeFileSync(timelinePath, JSON.stringify(timeline)); + console.log(`[native] Timeline: ${timeline.total_frames} frames → ${timelinePath}`); + + // Step 6: Close Chrome + file server + await closeCaptureSession(session); + fileServer.close(); + const extractionMs = Date.now() - t0; + console.log(`[native] Chrome extraction: ${extractionMs}ms (one-time cost)`); + + // Step 7: Render with Rust native binary + console.log("[native] Rendering with Rust/Skia..."); + const nativeRendererDir = resolve(dirname(new URL(import.meta.url).pathname), ".."); + const renderStart = Date.now(); + + const cpuFlag = forceCpu ? "--cpu" : ""; + const result = spawnSync( + "cargo", + [ + "run", + "--release", + "--bin", + "render_native", + "--", + "--scene", + scenePath, + "--timeline", + timelinePath, + "--output", + outputPath, + "--fps", + String(fps), + "--duration", + String(duration), + "--quality", + "80", + ...(cpuFlag ? [cpuFlag] : []), + ], + { + cwd: nativeRendererDir, + stdio: ["inherit", "pipe", "pipe"], + timeout: 300_000, + }, + ); + + if (result.status !== 0) { + console.error(`[native] Rust render failed:\n${result.stderr?.toString()}`); + process.exit(1); + } + + const renderMs = Date.now() - renderStart; + const totalMs = Date.now() - t0; + + // Parse result JSON from stdout + const resultJson = result.stdout?.toString().trim(); + console.log(`[native] Rust output: ${resultJson}`); + console.log(`[native] ────────────────────────────────────`); + console.log(`[native] Extraction: ${extractionMs}ms (Chrome, one-time)`); + console.log(`[native] Render: ${renderMs}ms (Rust/Skia)`); + console.log(`[native] Total: ${totalMs}ms`); + console.log(`[native] Output: ${outputPath}`); + + // Cleanup work dir + rmSync(workDir, { recursive: true, force: true }); +} + +main().catch((err) => { + console.error(`[native] Fatal: ${err}`); + process.exit(1); +}); diff --git a/packages/native-renderer/src/bin/render_native.rs b/packages/native-renderer/src/bin/render_native.rs new file mode 100644 index 000000000..af2bf78f4 --- /dev/null +++ b/packages/native-renderer/src/bin/render_native.rs @@ -0,0 +1,87 @@ +use std::env; +use std::fs; +use std::path::PathBuf; + +use hyperframes_native_renderer::pipeline::render_animated_gpu; +use hyperframes_native_renderer::pipeline::{render_animated, render_static, RenderConfig}; +use hyperframes_native_renderer::scene::{parse_scene_file, BakedTimeline}; + +fn usage() -> ! { + eprintln!( + "usage: render_native --scene --output [--timeline ] [--fps 30] [--duration 1] [--quality 80] [--cpu]" + ); + std::process::exit(2); +} + +fn take_value(args: &[String], name: &str) -> Option { + args.windows(2) + .find_map(|pair| (pair[0] == name).then(|| pair[1].clone())) +} + +fn has_flag(args: &[String], name: &str) -> bool { + args.iter().any(|arg| arg == name) +} + +fn main() { + let args: Vec = env::args().skip(1).collect(); + if args.is_empty() || has_flag(&args, "--help") { + usage(); + } + + let scene_path = PathBuf::from(take_value(&args, "--scene").unwrap_or_else(|| usage())); + let output_path = take_value(&args, "--output").unwrap_or_else(|| usage()); + let timeline_path = take_value(&args, "--timeline").map(PathBuf::from); + let fps: u32 = take_value(&args, "--fps") + .unwrap_or_else(|| "30".to_string()) + .parse() + .unwrap_or_else(|_| usage()); + let duration_secs: f64 = take_value(&args, "--duration") + .unwrap_or_else(|| "1".to_string()) + .parse() + .unwrap_or_else(|_| usage()); + let quality: u32 = take_value(&args, "--quality") + .unwrap_or_else(|| "80".to_string()) + .parse() + .unwrap_or_else(|_| usage()); + let force_cpu = has_flag(&args, "--cpu"); + + let scene = parse_scene_file(&scene_path).unwrap_or_else(|err| { + eprintln!("{err}"); + std::process::exit(1); + }); + + let config = RenderConfig { + fps, + duration_secs, + quality, + output_path, + }; + + let result = if let Some(path) = timeline_path { + let timeline_json = fs::read_to_string(&path).unwrap_or_else(|err| { + eprintln!("failed to read {}: {err}", path.display()); + std::process::exit(1); + }); + let timeline: BakedTimeline = serde_json::from_str(&timeline_json).unwrap_or_else(|err| { + eprintln!("invalid timeline JSON: {err}"); + std::process::exit(1); + }); + + if force_cpu { + render_animated(&scene, &timeline, &config) + } else { + render_animated_gpu(&scene, &timeline, &config) + } + } else { + render_static(&scene, &config) + } + .unwrap_or_else(|err| { + eprintln!("{err}"); + std::process::exit(1); + }); + + println!( + "{{\"frames\":{},\"totalMs\":{},\"avgPaintMs\":{},\"outputPath\":\"{}\"}}", + result.total_frames, result.total_ms, result.avg_paint_ms, result.output_path + ); +} diff --git a/packages/native-renderer/src/encode.rs b/packages/native-renderer/src/encode.rs new file mode 100644 index 000000000..c6d47bcef --- /dev/null +++ b/packages/native-renderer/src/encode.rs @@ -0,0 +1,206 @@ +/// Hardware-accelerated encoder variants detected at runtime. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HwEncoder { + /// macOS VideoToolbox HEVC encoder. + VideoToolbox, + /// NVIDIA NVENC H.264 encoder. + Nvenc, + /// VAAPI H.264 encoder (Linux Intel/AMD). + Vaapi, + /// CPU-only libx264 fallback. + Software, +} + +#[cfg(not(target_os = "macos"))] +fn ffmpeg_encoder_works(name: &str) -> bool { + // Actually probe the encoder with a 1-frame test instead of just + // checking the encoder list. NVENC shows up in the list even when + // libcuda.so.1 isn't available (Docker without GPU). + std::process::Command::new("ffmpeg") + .args([ + "-hide_banner", + "-f", "lavfi", + "-i", "color=c=black:s=16x16:d=0.01:r=1", + "-frames:v", "1", + "-c:v", name, + "-f", "null", + "-", + ]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +fn codec_quality(quality: u32) -> u32 { + if quality <= 51 { + return quality; + } + + let percent = quality.min(100) as f64; + (35.0 - (percent / 100.0 * 23.0)).round() as u32 +} + +/// Probe the system for the best available hardware encoder. +/// +/// On macOS, VideoToolbox is always available via the OS frameworks. +/// On Linux, we check FFmpeg's encoder list for NVENC support, then fall +/// back to VAAPI if `/dev/dri/renderD128` exists. +pub fn detect_hw_encoder() -> HwEncoder { + #[cfg(target_os = "macos")] + { + return HwEncoder::VideoToolbox; + } + + #[cfg(not(target_os = "macos"))] + { + if ffmpeg_encoder_works("h264_nvenc") { + return HwEncoder::Nvenc; + } + + if std::path::Path::new("/dev/dri/renderD128").exists() + && ffmpeg_encoder_works("h264_vaapi") + { + return HwEncoder::Vaapi; + } + + HwEncoder::Software + } +} + +/// Build FFmpeg CLI arguments for the given hardware encoder, frame rate, +/// and quality level. +/// +/// The returned args include input format flags (`-f image2pipe -vcodec mjpeg`), +/// encoder-specific codec and quality flags, and a compatible pixel format. +/// The caller must append the output path. +pub fn encoder_args(encoder: HwEncoder, fps: u32, quality: u32) -> Vec { + let mut args: Vec = vec![ + "-y".into(), + "-f".into(), + "image2pipe".into(), + "-vcodec".into(), + "mjpeg".into(), + "-framerate".into(), + fps.to_string(), + "-i".into(), + "-".into(), + "-threads".into(), + "0".into(), + ]; + append_codec_args(&mut args, encoder, quality); + args +} + +/// Build FFmpeg CLI arguments for raw RGBA frames written to stdin. +/// +/// This avoids the intermediate JPEG encode/decode round-trip used by the +/// compatibility pipe and is the current fastest non-zero-copy transfer path. +/// The caller must append the output path. +/// Build FFmpeg CLI arguments for raw pixel frames written to stdin. +/// +/// Uses BGRA input (Skia Metal's native format) to avoid a BGRA→RGBA +/// conversion during GPU readback. For hardware encoders (VideoToolbox, +/// NVENC), the pixel format conversion is done on the media engine rather +/// than the CPU, eliminating the main E2E bottleneck. +pub fn raw_pixel_encoder_args( + encoder: HwEncoder, + fps: u32, + quality: u32, + width: u32, + height: u32, +) -> Vec { + let mut args: Vec = vec![ + "-y".into(), + "-f".into(), + "rawvideo".into(), + "-pix_fmt".into(), + "bgra".into(), + "-s:v".into(), + format!("{width}x{height}"), + "-framerate".into(), + fps.to_string(), + "-i".into(), + "-".into(), + "-threads".into(), + "0".into(), + ]; + append_codec_args(&mut args, encoder, quality); + args +} + +#[deprecated(note = "use raw_pixel_encoder_args (BGRA) for better performance")] +pub fn raw_rgba_encoder_args( + encoder: HwEncoder, + fps: u32, + quality: u32, + width: u32, + height: u32, +) -> Vec { + raw_pixel_encoder_args(encoder, fps, quality, width, height) +} + +fn append_codec_args(args: &mut Vec, encoder: HwEncoder, quality: u32) { + let codec_q = codec_quality(quality); + + match encoder { + HwEncoder::VideoToolbox => { + args.extend([ + "-c:v".into(), + "hevc_videotoolbox".into(), + "-q:v".into(), + quality.to_string(), + "-allow_sw".into(), + "1".into(), + "-tag:v".into(), + "hvc1".into(), + ]); + } + HwEncoder::Nvenc => { + args.extend([ + "-c:v".into(), + "h264_nvenc".into(), + "-preset".into(), + "p4".into(), + "-cq".into(), + codec_q.to_string(), + ]); + } + HwEncoder::Vaapi => { + args.extend([ + "-vaapi_device".into(), + "/dev/dri/renderD128".into(), + "-vf".into(), + "format=nv12,hwupload".into(), + "-c:v".into(), + "h264_vaapi".into(), + "-qp".into(), + codec_q.to_string(), + ]); + } + HwEncoder::Software => { + args.extend([ + "-c:v".into(), + "libx264".into(), + "-preset".into(), + "fast".into(), + "-crf".into(), + codec_q.to_string(), + ]); + } + } + + match encoder { + HwEncoder::VideoToolbox => { + // VideoToolbox needs NV12. FFmpeg auto-inserts BGRA→NV12 conversion. + args.extend(["-pix_fmt".into(), "nv12".into()]); + } + HwEncoder::Vaapi => { + // VAAPI pixel format is set in the vf filter chain above. + } + HwEncoder::Nvenc | HwEncoder::Software => { + args.extend(["-pix_fmt".into(), "yuv420p".into()]); + } + } +} diff --git a/packages/native-renderer/src/lib.rs b/packages/native-renderer/src/lib.rs new file mode 100644 index 000000000..6c40d161f --- /dev/null +++ b/packages/native-renderer/src/lib.rs @@ -0,0 +1,5 @@ +pub mod encode; +pub mod native_encode; +pub mod paint; +pub mod pipeline; +pub mod scene; diff --git a/packages/native-renderer/src/native_encode.rs b/packages/native-renderer/src/native_encode.rs new file mode 100644 index 000000000..ad1beceaf --- /dev/null +++ b/packages/native-renderer/src/native_encode.rs @@ -0,0 +1,110 @@ +use std::io::{Cursor, Write}; + +use dcv_color_primitives as dcp; +use openh264::encoder::Encoder; +use openh264::formats::YUVBuffer; + +pub struct NativeEncoder { + encoder: Encoder, + width: u32, + height: u32, + fps: u32, + i420_buf: Vec, + h264_data: Vec, + frame_count: u32, +} + +impl NativeEncoder { + pub fn new(width: u32, height: u32, fps: u32) -> Result { + let encoder = Encoder::new().map_err(|e| format!("openh264 init: {e}"))?; + let i420_size = (3 * (width as usize * height as usize)) / 2; + + Ok(Self { + encoder, + width, + height, + fps, + i420_buf: vec![0u8; i420_size], + h264_data: Vec::with_capacity(1024 * 1024), + frame_count: 0, + }) + } + + pub fn encode_bgra_frame(&mut self, bgra: &[u8]) -> Result<(), String> { + let w = self.width; + let h = self.height; + let y_size = (w * h) as usize; + let uv_size = ((w / 2) * (h / 2)) as usize; + + let src_format = dcp::ImageFormat { + pixel_format: dcp::PixelFormat::Bgra, + color_space: dcp::ColorSpace::Rgb, + num_planes: 1, + }; + let dst_format = dcp::ImageFormat { + pixel_format: dcp::PixelFormat::I420, + color_space: dcp::ColorSpace::Bt601, + num_planes: 3, + }; + + let (y_slice, uv_rest) = self.i420_buf.split_at_mut(y_size); + let (u_slice, v_slice) = uv_rest.split_at_mut(uv_size); + + dcp::convert_image( + w, + h, + &src_format, + None, + &[bgra], + &dst_format, + None, + &mut [y_slice, u_slice, v_slice], + ) + .map_err(|e| format!("BGRA→I420: {e:?}"))?; + + let yuv = + YUVBuffer::from_vec(self.i420_buf.clone(), self.width as usize, self.height as usize); + + let bitstream = self + .encoder + .encode(&yuv) + .map_err(|e| format!("H.264 encode: {e}"))?; + + bitstream.write_vec(&mut self.h264_data); + self.frame_count += 1; + Ok(()) + } + + pub fn finish_to_mp4(self, output_path: &str) -> Result { + let mut cursor = Cursor::new(Vec::with_capacity(self.h264_data.len() + 4096)); + + { + let mut muxer = minimp4::Mp4Muxer::new(&mut cursor); + muxer.init_video( + self.width as i32, + self.height as i32, + false, + "hyperframes", + ); + muxer.write_video_with_fps(&self.h264_data, self.fps); + muxer.close(); + } + + let mp4_buf = cursor.into_inner(); + let file_size = mp4_buf.len(); + let mut file = + std::fs::File::create(output_path).map_err(|e| format!("create {output_path}: {e}"))?; + file.write_all(&mp4_buf) + .map_err(|e| format!("write {output_path}: {e}"))?; + + Ok(EncodeResult { + file_size, + frame_count: self.frame_count, + }) + } +} + +pub struct EncodeResult { + pub file_size: usize, + pub frame_count: u32, +} diff --git a/packages/native-renderer/src/paint/canvas.rs b/packages/native-renderer/src/paint/canvas.rs new file mode 100644 index 000000000..268f21b3d --- /dev/null +++ b/packages/native-renderer/src/paint/canvas.rs @@ -0,0 +1,237 @@ +use skia_safe::{ + images, surfaces, AlphaType, Canvas, Color4f, ColorType, Data, EncodedImageFormat, ImageInfo, + Surface, +}; + +#[cfg(any(feature = "metal-gpu", feature = "vulkan-gpu"))] +use skia_safe::gpu; + +/// A Skia rendering surface backed by either CPU raster or GPU (Metal). +/// +/// Wraps `skia_safe::Surface` and provides convenience methods for clearing, +/// drawing, pixel readback, and image encoding. +/// +/// When created via [`new_metal_gpu`](Self::new_metal_gpu), the surface is +/// GPU-accelerated through Apple's Metal API. The `DirectContext` is kept alive +/// alongside the surface for the duration of rendering. +pub struct RenderSurface { + surface: Surface, + #[cfg(any(feature = "metal-gpu", feature = "vulkan-gpu"))] + _gpu_context: Option, +} + +impl RenderSurface { + /// Create a CPU-backed raster surface with premultiplied N32 color type. + pub fn new_raster(width: i32, height: i32) -> Result { + let surface = surfaces::raster_n32_premul((width, height)) + .ok_or_else(|| format!("failed to create {width}x{height} raster surface"))?; + Ok(Self { + surface, + #[cfg(any(feature = "metal-gpu", feature = "vulkan-gpu"))] + _gpu_context: None, + }) + } + + /// Create a Metal GPU-accelerated surface (macOS only). + /// + /// Uses the system default Metal device and a Skia `DirectContext` backed by + /// Apple's Metal API. Drawing commands issued through `canvas()` execute on + /// the GPU, which is 7-30x faster than CPU raster for typical composition + /// workloads on Apple Silicon. + /// + /// Call [`flush_and_submit`](Self::flush_and_submit) after drawing to ensure + /// all GPU work is submitted before reading back pixels. + #[cfg(feature = "metal-gpu")] + pub fn new_metal_gpu(width: i32, height: i32) -> Result { + use metal::foreign_types::ForeignType; + + let device = metal::Device::system_default().ok_or("no Metal GPU device found")?; + let queue = device.new_command_queue(); + + let backend = unsafe { + gpu::mtl::BackendContext::new( + device.as_ptr() as gpu::mtl::Handle, + queue.as_ptr() as gpu::mtl::Handle, + ) + }; + + let mut context = gpu::direct_contexts::make_metal(&backend, None) + .ok_or("failed to create Skia Metal DirectContext")?; + + let image_info = ImageInfo::new( + (width, height), + ColorType::BGRA8888, + AlphaType::Premul, + None, + ); + + let surface = gpu::surfaces::render_target( + &mut context, + gpu::Budgeted::Yes, + &image_info, + None, // sample count + gpu::SurfaceOrigin::TopLeft, + None, // surface props + false, // mipmaps + false, // protected + ) + .ok_or("failed to create Metal GPU surface")?; + + Ok(Self { + surface, + _gpu_context: Some(context), + }) + } + + /// Create a Vulkan GPU-accelerated surface (Linux with NVIDIA/AMD/Intel). + /// + /// Vulkan initialization requires a live GPU with a Vulkan ICD installed. + /// Returns Err if Vulkan is unavailable (Docker without GPU, CI, etc.) + /// — the caller should fall back to CPU raster via `new_gpu_or_raster`. + /// + /// Full Vulkan setup (instance → physical device → logical device → queue) + /// will be implemented when we validate on an NVIDIA GPU instance. + /// For now, this returns Err to trigger the raster fallback. + #[cfg(feature = "vulkan-gpu")] + pub fn new_vulkan_gpu(_width: i32, _height: i32) -> Result { + // TODO: Vulkan instance + device creation for production GPU path. + // Requires: VkInstance, VkPhysicalDevice, VkDevice, VkQueue, GetProc. + // Placeholder returns Err so new_gpu_or_raster falls back to raster. + Err("Vulkan GPU surface not yet implemented — use raster fallback".into()) + } + + /// Create the best available GPU surface for the current platform. + /// Falls back to CPU raster if no GPU is available. + pub fn new_gpu_or_raster(width: i32, height: i32) -> Result { + // Try GPU first, fall back to raster. + #[cfg(feature = "metal-gpu")] + { + match Self::new_metal_gpu(width, height) { + Ok(s) => return Ok(s), + Err(e) => eprintln!("[native-renderer] Metal GPU unavailable ({e}), falling back to raster"), + } + } + #[cfg(feature = "vulkan-gpu")] + { + match Self::new_vulkan_gpu(width, height) { + Ok(s) => return Ok(s), + Err(e) => eprintln!("[native-renderer] Vulkan GPU unavailable ({e}), falling back to raster"), + } + } + Self::new_raster(width, height) + } + + /// Get the Skia canvas for drawing operations. + pub fn canvas(&mut self) -> &Canvas { + self.surface.canvas() + } + + /// Read back the rendered pixels as RGBA8888 bytes. + /// + /// Returns `None` if the readback fails (e.g. zero-sized surface). + pub fn read_pixels_rgba(&mut self) -> Option> { + self.read_pixels_with_color_type(ColorType::RGBA8888) + } + + /// Read back pixels in BGRA8888 — the native format for Metal surfaces. + /// Avoids the BGRA→RGBA conversion that `read_pixels_rgba` incurs on GPU + /// surfaces, saving ~0.5ms per frame at 1080p. + pub fn read_pixels_bgra(&mut self) -> Option> { + self.read_pixels_with_color_type(ColorType::BGRA8888) + } + + /// Read BGRA pixels into a pre-allocated buffer. Avoids per-frame allocation. + pub fn read_pixels_bgra_into(&mut self, dst: &mut [u8]) -> Option<()> { + let width = self.surface.width(); + let height = self.surface.height(); + let row_bytes = width as usize * 4; + let expected = row_bytes * height as usize; + if dst.len() < expected { + return None; + } + let info = ImageInfo::new((width, height), ColorType::BGRA8888, AlphaType::Premul, None); + if self.surface.read_pixels(&info, dst, row_bytes, (0, 0)) { + Some(()) + } else { + None + } + } + + fn read_pixels_with_color_type(&mut self, color_type: ColorType) -> Option> { + let width = self.surface.width(); + let height = self.surface.height(); + let row_bytes = width as usize * 4; + let mut dst = vec![0u8; row_bytes * height as usize]; + + let info = ImageInfo::new((width, height), color_type, AlphaType::Premul, None); + + let ok = self.surface.read_pixels(&info, &mut dst, row_bytes, (0, 0)); + if ok { Some(dst) } else { None } + } + + /// Encode the surface contents as JPEG bytes at the given quality (1-100). + pub fn encode_jpeg(&mut self, quality: u32) -> Option> { + self.encode_image(EncodedImageFormat::JPEG, quality) + } + + /// Encode the surface contents as PNG bytes. + pub fn encode_png(&mut self) -> Option> { + self.encode_image(EncodedImageFormat::PNG, 100) + } + + fn encode_image(&mut self, format: EncodedImageFormat, quality: u32) -> Option> { + self.flush_and_submit(); + + let image = self.surface.image_snapshot(); + if let Some(data) = image.encode(None, format, quality) { + return Some(data.as_bytes().to_vec()); + } + + let width = self.surface.width(); + let height = self.surface.height(); + let row_bytes = width as usize * 4; + let pixels = self.read_pixels_rgba()?; + let info = ImageInfo::new( + (width, height), + ColorType::RGBA8888, + AlphaType::Premul, + None, + ); + let image = images::raster_from_data(&info, Data::new_copy(&pixels), row_bytes)?; + let data = image.encode(None, format, quality)?; + Some(data.as_bytes().to_vec()) + } + + /// Clear the entire surface with a color. + pub fn clear(&mut self, color: Color4f) { + self.surface.canvas().clear(color); + } + + /// Surface width in pixels. + pub fn width(&self) -> i32 { + self.surface.width() + } + + /// Surface height in pixels. + pub fn height(&self) -> i32 { + self.surface.height() + } + + /// Flush pending GPU commands and submit to the GPU. + /// + /// This is a no-op on raster surfaces. On GPU surfaces, it ensures all + /// queued draw calls are submitted before pixel readback or timing. + pub fn flush_and_submit(&mut self) { + #[cfg(any(feature = "metal-gpu", feature = "vulkan-gpu"))] + if let Some(ctx) = self._gpu_context.as_mut() { + ctx.flush_and_submit(); + } + } + + pub fn is_gpu(&self) -> bool { + #[cfg(any(feature = "metal-gpu", feature = "vulkan-gpu"))] + { self._gpu_context.is_some() } + #[cfg(not(any(feature = "metal-gpu", feature = "vulkan-gpu")))] + { false } + } +} diff --git a/packages/native-renderer/src/paint/effects.rs b/packages/native-renderer/src/paint/effects.rs new file mode 100644 index 000000000..ce22497b5 --- /dev/null +++ b/packages/native-renderer/src/paint/effects.rs @@ -0,0 +1,169 @@ +use skia_safe::{ + color_filters, gradient_shader, image_filters, BlurStyle, Canvas, Color4f, ColorFilter, + ColorMatrix, ImageFilter, MaskFilter, Paint, PaintStyle, Point as SkPoint, RRect, + Rect as SkRect, Shader, TileMode, +}; + +use crate::scene::{BoxShadow, Color, FilterAdjust, Gradient}; + +/// Convert a `Color` (u8 RGBA) to Skia's `Color4f` (f32 channels in 0..1). +fn to_color4f(c: &Color) -> Color4f { + Color4f::new( + c.r as f32 / 255.0, + c.g as f32 / 255.0, + c.b as f32 / 255.0, + c.a as f32 / 255.0, + ) +} + +/// Build a rounded-rect for the shadow shape, applying per-corner radii. +fn make_shadow_rrect(rect: &SkRect, radii: &[f32; 4]) -> RRect { + let corner_radii: [SkPoint; 4] = [ + (radii[0], radii[0]).into(), + (radii[1], radii[1]).into(), + (radii[2], radii[2]).into(), + (radii[3], radii[3]).into(), + ]; + let mut rrect = RRect::new(); + rrect.set_rect_radii(*rect, &corner_radii); + rrect +} + +/// Paint a CSS box-shadow behind an element. +/// +/// `rect` is the element's local bounding rect (origin at 0,0 after canvas +/// translate). `radii` contains the four corner radii from `border_radius`. +pub fn paint_box_shadow(canvas: &Canvas, rect: &SkRect, radii: &[f32; 4], shadow: &BoxShadow) { + let mut paint = Paint::default(); + paint.set_anti_alias(true); + paint.set_style(PaintStyle::Fill); + paint.set_color4f(to_color4f(&shadow.color), None); + + if shadow.blur_radius > 0.0 { + let sigma = shadow.blur_radius / 2.0; + if let Some(mf) = MaskFilter::blur(BlurStyle::Normal, sigma, false) { + paint.set_mask_filter(mf); + } + } + + let shadow_rect = SkRect::from_xywh( + rect.left + shadow.offset_x - shadow.spread_radius, + rect.top + shadow.offset_y - shadow.spread_radius, + rect.width() + shadow.spread_radius * 2.0, + rect.height() + shadow.spread_radius * 2.0, + ); + + if radii.iter().any(|&r| r > 0.0) { + let rrect = make_shadow_rrect(&shadow_rect, radii); + canvas.draw_rrect(rrect, &paint); + } else { + canvas.draw_rect(shadow_rect, &paint); + } +} + +/// Create a Skia `ImageFilter` for CSS `filter: blur(Npx)`. +/// +/// Returns `None` when `blur_radius` is zero or negative, or if Skia fails to +/// create the filter. +pub fn create_blur_image_filter(blur_radius: f32) -> Option { + if blur_radius <= 0.0 { + return None; + } + let sigma = blur_radius / 2.0; + image_filters::blur((sigma, sigma), TileMode::Clamp, None, None) +} + +/// Create a Skia color filter for CSS `brightness()`, `contrast()`, and +/// `saturate()` filter functions. +pub fn create_filter_adjust_color_filter(adjust: &FilterAdjust) -> Option { + let brightness = adjust.brightness.max(0.0); + let contrast = adjust.contrast.max(0.0); + let saturate = adjust.saturate.max(0.0); + + if (brightness - 1.0).abs() < f32::EPSILON + && (contrast - 1.0).abs() < f32::EPSILON + && (saturate - 1.0).abs() < f32::EPSILON + { + return None; + } + + let mut matrix = ColorMatrix::default(); + matrix.set_identity(); + + if (saturate - 1.0).abs() >= f32::EPSILON { + let mut saturation = ColorMatrix::default(); + saturation.set_saturation(saturate); + matrix.post_concat(&saturation); + } + + if (contrast - 1.0).abs() >= f32::EPSILON { + let translate = 127.5 * (1.0 - contrast); + let contrast_matrix = ColorMatrix::new( + contrast, 0.0, 0.0, 0.0, translate, 0.0, contrast, 0.0, 0.0, translate, 0.0, 0.0, + contrast, 0.0, translate, 0.0, 0.0, 0.0, 1.0, 0.0, + ); + matrix.post_concat(&contrast_matrix); + } + + if (brightness - 1.0).abs() >= f32::EPSILON { + let mut brightness_matrix = ColorMatrix::default(); + brightness_matrix.set_scale(brightness, brightness, brightness, None); + matrix.post_concat(&brightness_matrix); + } + + Some(color_filters::matrix(&matrix, None)) +} + +/// Create a gradient `Shader` filling `rect` according to a `Gradient` spec. +/// +/// Returns `None` if the gradient has fewer than two stops or if Skia fails to +/// create the shader. +pub fn create_gradient_shader(rect: &SkRect, gradient: &Gradient) -> Option { + match gradient { + Gradient::Linear { angle_deg, stops } => { + if stops.len() < 2 { + return None; + } + let angle_rad = angle_deg.to_radians(); + let cx = rect.center_x(); + let cy = rect.center_y(); + let half_w = rect.width() / 2.0; + let half_h = rect.height() / 2.0; + + let start = SkPoint::new(cx - half_w * angle_rad.sin(), cy + half_h * angle_rad.cos()); + let end = SkPoint::new(cx + half_w * angle_rad.sin(), cy - half_h * angle_rad.cos()); + + let colors: Vec = stops.iter().map(|s| to_color4f(&s.color)).collect(); + let positions: Vec = stops.iter().map(|s| s.position).collect(); + + gradient_shader::linear( + (start, end), + colors.as_slice(), + positions.as_slice(), + TileMode::Clamp, + None, + None, + ) + } + Gradient::Radial { stops } => { + if stops.len() < 2 { + return None; + } + let center = SkPoint::new(rect.center_x(), rect.center_y()); + let radius = rect.width().max(rect.height()) / 2.0; + + let colors: Vec = stops.iter().map(|s| to_color4f(&s.color)).collect(); + let positions: Vec = stops.iter().map(|s| s.position).collect(); + + gradient_shader::radial( + center, + radius, + colors.as_slice(), + positions.as_slice(), + TileMode::Clamp, + None, + None, + ) + } + } +} diff --git a/packages/native-renderer/src/paint/elements.rs b/packages/native-renderer/src/paint/elements.rs new file mode 100644 index 000000000..b547b2009 --- /dev/null +++ b/packages/native-renderer/src/paint/elements.rs @@ -0,0 +1,553 @@ +use std::{cell::RefCell, collections::HashMap}; + +use skia_safe::{ + canvas::{SaveLayerRec, SrcRectConstraint}, + dash_path_effect, + font_style::{Slant, Weight, Width}, + BlendMode, Canvas, ClipOp, Color4f, Font, FontMgr, FontStyle, Paint, PaintStyle, PathBuilder, + Point, RRect, Rect as SkRect, Typeface, +}; + +use crate::paint::effects; +use crate::paint::images::ImageCache; +use crate::scene::{ + BackgroundImageFit, BorderLineStyle, ClipPath, Color, Element, ElementKind, MixBlendMode, + ObjectFit, ObjectPosition, Rect, +}; + +thread_local! { + static DEFAULT_TYPEFACE: RefCell> = const { RefCell::new(None) }; + static TYPEFACE_CACHE: RefCell> = RefCell::new(HashMap::new()); +} + +fn cached_typeface() -> Typeface { + DEFAULT_TYPEFACE.with(|cell| { + let mut opt = cell.borrow_mut(); + if opt.is_none() { + let mgr = FontMgr::new(); + *opt = Some( + mgr.legacy_make_typeface(None, FontStyle::normal()) + .expect("platform must provide a default typeface"), + ); + } + opt.as_ref().unwrap().clone() + }) +} + +fn resolve_typeface(family: Option<&str>, weight: Option) -> Typeface { + let family_key = family.unwrap_or_default().trim(); + let weight_value = weight.unwrap_or(400); + let cache_key = format!("{family_key}:{weight_value}"); + + TYPEFACE_CACHE.with(|cache| { + if let Some(typeface) = cache.borrow().get(&cache_key) { + return typeface.clone(); + } + + let font_style = FontStyle::new( + Weight::from(weight_value as i32), + Width::NORMAL, + Slant::Upright, + ); + let mgr = FontMgr::new(); + let typeface = if family_key.is_empty() { + mgr.legacy_make_typeface(None, font_style) + } else { + mgr.match_family_style(family_key, font_style) + .or_else(|| mgr.legacy_make_typeface(None, font_style)) + } + .unwrap_or_else(cached_typeface); + + cache.borrow_mut().insert(cache_key, typeface.clone()); + typeface + }) +} + +/// Convert a `Color` (u8 RGBA) to Skia's `Color4f` (f32 channels in 0.0..1.0). +fn to_color4f(c: &Color) -> Color4f { + Color4f::new( + c.r as f32 / 255.0, + c.g as f32 / 255.0, + c.b as f32 / 255.0, + c.a as f32 / 255.0, + ) +} + +/// Convert element bounds to a Skia rect at the origin. We translate the canvas +/// to `(bounds.x, bounds.y)` before painting, so the local rect is `(0, 0, w, h)`. +fn to_sk_rect(bounds: &Rect) -> SkRect { + SkRect::from_xywh(0.0, 0.0, bounds.width, bounds.height) +} + +/// Build a rounded rect with per-corner radii `[top-left, top-right, bottom-right, bottom-left]`. +fn make_rrect(rect: &SkRect, radii: &[f32; 4]) -> RRect { + let corner_radii: [Point; 4] = [ + (radii[0], radii[0]).into(), + (radii[1], radii[1]).into(), + (radii[2], radii[2]).into(), + (radii[3], radii[3]).into(), + ]; + let mut rrect = RRect::new(); + rrect.set_rect_radii(*rect, &corner_radii); + rrect +} + +/// Returns true when all four corner radii are zero. +fn radii_are_zero(radii: &[f32; 4]) -> bool { + radii.iter().all(|&r| r == 0.0) +} + +fn build_clip_path(clip_path: &ClipPath) -> Option { + let mut builder = PathBuilder::new(); + match clip_path { + ClipPath::Polygon { points } => { + if points.len() < 3 { + return None; + } + let sk_points: Vec = points.iter().map(|p| Point::new(p.x, p.y)).collect(); + builder.add_polygon(&sk_points, true); + } + ClipPath::Circle { x, y, radius } => { + if *radius <= 0.0 { + return None; + } + builder.add_circle((*x, *y), *radius, None); + } + ClipPath::Ellipse { + x, + y, + radius_x, + radius_y, + } => { + if *radius_x <= 0.0 || *radius_y <= 0.0 { + return None; + } + builder.add_oval( + SkRect::from_xywh(x - radius_x, y - radius_y, radius_x * 2.0, radius_y * 2.0), + None, + None, + ); + } + } + Some(builder.detach()) +} + +fn to_sk_blend_mode(mode: MixBlendMode) -> BlendMode { + match mode { + MixBlendMode::Normal => BlendMode::SrcOver, + MixBlendMode::Multiply => BlendMode::Multiply, + MixBlendMode::Screen => BlendMode::Screen, + MixBlendMode::Overlay => BlendMode::Overlay, + MixBlendMode::Darken => BlendMode::Darken, + MixBlendMode::Lighten => BlendMode::Lighten, + MixBlendMode::ColorDodge => BlendMode::ColorDodge, + MixBlendMode::ColorBurn => BlendMode::ColorBurn, + MixBlendMode::HardLight => BlendMode::HardLight, + MixBlendMode::SoftLight => BlendMode::SoftLight, + MixBlendMode::Difference => BlendMode::Difference, + MixBlendMode::Exclusion => BlendMode::Exclusion, + MixBlendMode::Hue => BlendMode::Hue, + MixBlendMode::Saturation => BlendMode::Saturation, + MixBlendMode::Color => BlendMode::Color, + MixBlendMode::Luminosity => BlendMode::Luminosity, + } +} + +fn draw_border( + canvas: &Canvas, + rect: &SkRect, + radii: &[f32; 4], + has_radii: bool, + element: &Element, +) { + let Some(border) = element.style.border.as_ref() else { + return; + }; + if border.width <= 0.0 || border.color.a == 0 { + return; + } + + let inset = border.width / 2.0; + let stroke_rect = SkRect::from_xywh( + rect.left + inset, + rect.top + inset, + (rect.width() - border.width).max(0.0), + (rect.height() - border.width).max(0.0), + ); + + let mut paint = Paint::default(); + paint.set_anti_alias(true); + paint.set_style(PaintStyle::Stroke); + paint.set_stroke_width(border.width); + paint.set_color4f(to_color4f(&border.color), None); + + if border.style == BorderLineStyle::Dashed { + let dash = (border.width * 3.0).max(1.0); + paint.set_path_effect(dash_path_effect::new(&[dash, dash], 0.0)); + } + + if has_radii { + let rrect = make_rrect(&stroke_rect, radii); + canvas.draw_rrect(rrect, &paint); + } else { + canvas.draw_rect(stroke_rect, &paint); + } +} + +fn object_position_or_center(position: Option) -> ObjectPosition { + position.unwrap_or(ObjectPosition { x: 0.5, y: 0.5 }) +} + +fn compute_image_rects( + src_w: f32, + src_h: f32, + dest_rect: &SkRect, + fit: ObjectFit, + position: ObjectPosition, +) -> (SkRect, SkRect) { + let dest_w = dest_rect.width(); + let dest_h = dest_rect.height(); + let full_src = SkRect::from_xywh(0.0, 0.0, src_w, src_h); + + match fit { + ObjectFit::Fill => (full_src, *dest_rect), + ObjectFit::Contain => { + let scale = (dest_w / src_w).min(dest_h / src_h); + let scaled_w = src_w * scale; + let scaled_h = src_h * scale; + let x = (dest_w - scaled_w) * position.x; + let y = (dest_h - scaled_h) * position.y; + (full_src, SkRect::from_xywh(x, y, scaled_w, scaled_h)) + } + ObjectFit::Cover => { + let scale = (dest_w / src_w).max(dest_h / src_h); + let crop_w = dest_w / scale; + let crop_h = dest_h / scale; + let src_x = (src_w - crop_w) * position.x; + let src_y = (src_h - crop_h) * position.y; + (SkRect::from_xywh(src_x, src_y, crop_w, crop_h), *dest_rect) + } + ObjectFit::None => { + let x = (dest_w - src_w) * position.x; + let y = (dest_h - src_h) * position.y; + (full_src, SkRect::from_xywh(x, y, src_w, src_h)) + } + ObjectFit::ScaleDown => { + if src_w <= dest_w && src_h <= dest_h { + compute_image_rects(src_w, src_h, dest_rect, ObjectFit::None, position) + } else { + compute_image_rects(src_w, src_h, dest_rect, ObjectFit::Contain, position) + } + } + } +} + +fn background_fit_to_object_fit(fit: BackgroundImageFit) -> ObjectFit { + match fit { + BackgroundImageFit::Fill => ObjectFit::Fill, + BackgroundImageFit::Contain => ObjectFit::Contain, + BackgroundImageFit::Cover => ObjectFit::Cover, + BackgroundImageFit::None => ObjectFit::None, + } +} + +fn draw_image( + canvas: &Canvas, + image: &skia_safe::Image, + dest_rect: &SkRect, + fit: ObjectFit, + position: ObjectPosition, +) { + let mut paint = Paint::default(); + paint.set_anti_alias(true); + + let src_w = image.width() as f32; + let src_h = image.height() as f32; + let (src_rect, target_rect) = compute_image_rects(src_w, src_h, dest_rect, fit, position); + + let image_save_count = canvas.save(); + canvas.clip_rect(*dest_rect, ClipOp::Intersect, true); + canvas.draw_image_rect( + image, + Some((&src_rect, SrcRectConstraint::Strict)), + target_rect, + &paint, + ); + canvas.restore_to_count(image_save_count); +} + +/// Recursively paint an `Element` and its children onto a Skia `Canvas`. +/// +/// The painting order follows the CSS box model: +/// 1. Position (translate to element bounds) +/// 2. Transform (rotate, scale around center) +/// 3. Box shadow (painted before element content) +/// 4. Opacity (layer alpha) +/// 5. Blur filter (save layer with ImageFilter) +/// 6. Clip (overflow hidden) +/// 7. Background (gradient takes priority over solid color) +/// 8. Border +/// 9. Content (text, image) +/// 10. Children (recursion) +pub fn paint_element(canvas: &Canvas, element: &Element, images: &mut ImageCache) { + paint_element_at_time(canvas, element, images, 0.0); +} + +/// Recursively paint an `Element` at a timeline time. `time_secs` is used for +/// video frame compositing. +pub fn paint_element_at_time( + canvas: &Canvas, + element: &Element, + images: &mut ImageCache, + time_secs: f64, +) { + let style = &element.style; + + // Skip invisible elements entirely. + if !style.visibility { + return; + } + + let save_count = canvas.save(); + + // --- Position & Transform --- + canvas.translate((element.bounds.x, element.bounds.y)); + + if let Some(ref t) = style.transform { + let cx = element.bounds.width / 2.0; + let cy = element.bounds.height / 2.0; + + canvas.translate((cx, cy)); + canvas.rotate(t.rotate_deg, None); + canvas.scale((t.scale_x, t.scale_y)); + canvas.translate((-cx, -cy)); + canvas.translate((t.translate_x, t.translate_y)); + } + + let local_rect = to_sk_rect(&element.bounds); + let has_radii = !radii_are_zero(&style.border_radius); + + // --- Box shadow (painted before opacity/blur so it sits behind the element) --- + if let Some(ref shadow) = style.box_shadow { + effects::paint_box_shadow(canvas, &local_rect, &style.border_radius, shadow); + } + + // --- Stacking effects (applied to the whole element subtree on restore) --- + let has_partial_opacity = style.opacity < 1.0; + let has_blur = style.filter_blur.is_some_and(|b| b > 0.0); + let has_filter_adjust = style.filter_adjust.is_some(); + let has_blend_mode = style + .mix_blend_mode + .is_some_and(|mode| mode != MixBlendMode::Normal); + if has_partial_opacity || has_blur || has_filter_adjust || has_blend_mode { + let mut layer_paint = Paint::default(); + + if has_partial_opacity { + layer_paint.set_alpha_f(style.opacity.clamp(0.0, 1.0)); + } + + if let Some(filter) = style + .filter_blur + .and_then(effects::create_blur_image_filter) + { + layer_paint.set_image_filter(filter); + } + + if let Some(filter) = style + .filter_adjust + .as_ref() + .and_then(effects::create_filter_adjust_color_filter) + { + layer_paint.set_color_filter(filter); + } + + if let Some(mode) = style.mix_blend_mode { + layer_paint.set_blend_mode(to_sk_blend_mode(mode)); + } + + let layer_bounds = if has_blur { + let blur_pad = style.filter_blur.unwrap_or_default().max(0.0) * 2.0; + Some(SkRect::from_xywh( + -blur_pad, + -blur_pad, + local_rect.width() + blur_pad * 2.0, + local_rect.height() + blur_pad * 2.0, + )) + } else if style.overflow_hidden { + Some(local_rect) + } else { + None + }; + + let rec = SaveLayerRec::default().paint(&layer_paint); + if let Some(ref bounds) = layer_bounds { + let rec = rec.bounds(bounds); + canvas.save_layer(&rec); + } else { + canvas.save_layer(&rec); + } + } + + // --- Clip path --- + if let Some(ref clip_path) = style.clip_path { + if let Some(path) = build_clip_path(clip_path) { + canvas.clip_path(&path, ClipOp::Intersect, true); + } + } + + // --- Clip (overflow hidden) --- + if style.overflow_hidden { + if has_radii { + let rrect = make_rrect(&local_rect, &style.border_radius); + canvas.clip_rrect(rrect, ClipOp::Intersect, true); + } else { + canvas.clip_rect(local_rect, ClipOp::Intersect, true); + } + } + + // --- Background (CSS order: color, image/gradient) --- + if let Some(ref bg) = style.background_color { + let mut paint = Paint::default(); + paint.set_anti_alias(true); + paint.set_style(PaintStyle::Fill); + paint.set_color4f(to_color4f(bg), None); + + if has_radii { + let rrect = make_rrect(&local_rect, &style.border_radius); + canvas.draw_rrect(rrect, &paint); + } else { + canvas.draw_rect(local_rect, &paint); + } + } + + if let Some(ref gradient) = style.background_gradient { + if let Some(shader) = effects::create_gradient_shader(&local_rect, gradient) { + let mut paint = Paint::default(); + paint.set_anti_alias(true); + paint.set_style(PaintStyle::Fill); + paint.set_shader(shader); + + if has_radii { + let rrect = make_rrect(&local_rect, &style.border_radius); + canvas.draw_rrect(rrect, &paint); + } else { + canvas.draw_rect(local_rect, &paint); + } + } + } else if let Some(ref background_image) = style.background_image { + if let Some(image) = images.get_or_load(&background_image.src).cloned() { + draw_image( + canvas, + &image, + &local_rect, + background_fit_to_object_fit(background_image.fit), + background_image.position, + ); + } + } + + // --- Border --- + draw_border( + canvas, + &local_rect, + &style.border_radius, + has_radii, + element, + ); + + // --- Text content --- + if let ElementKind::Text { ref content } = element.kind { + let font_size = style.font_size.unwrap_or(16.0); + let typeface = resolve_typeface(style.font_family.as_deref(), style.font_weight); + let font = Font::new(&typeface, font_size); + + let mut paint = Paint::default(); + paint.set_anti_alias(true); + paint.set_style(PaintStyle::Fill); + + let text_color = style.color.unwrap_or(Color { + r: 255, + g: 255, + b: 255, + a: 255, + }); + paint.set_color4f(to_color4f(&text_color), None); + + let (_, metrics) = font.metrics(); + // `metrics.ascent` is negative (distance above baseline), so negate it to + // get the y-offset where the baseline sits. + let y = -metrics.ascent; + + if let Some(ref shadow) = style.text_shadow { + let mut shadow_paint = Paint::default(); + shadow_paint.set_anti_alias(true); + shadow_paint.set_style(PaintStyle::Fill); + shadow_paint.set_color4f(to_color4f(&shadow.color), None); + if shadow.blur_radius > 0.0 { + if let Some(mf) = skia_safe::MaskFilter::blur( + skia_safe::BlurStyle::Normal, + shadow.blur_radius / 2.0, + false, + ) { + shadow_paint.set_mask_filter(mf); + } + } + canvas.draw_str( + content, + (shadow.offset_x, y + shadow.offset_y), + &font, + &shadow_paint, + ); + } + + if let Some(ref stroke) = style.text_stroke { + if stroke.width > 0.0 && stroke.color.a > 0 { + let mut stroke_paint = Paint::default(); + stroke_paint.set_anti_alias(true); + stroke_paint.set_style(PaintStyle::Stroke); + stroke_paint.set_stroke_width(stroke.width); + stroke_paint.set_color4f(to_color4f(&stroke.color), None); + canvas.draw_str(content, (0.0, y), &font, &stroke_paint); + } + } + + canvas.draw_str(content, (0.0, y), &font, &paint); + } + + // --- Image content --- + if let ElementKind::Image { ref src } = element.kind { + if let Some(image) = images.get_or_load(src).cloned() { + let dest_rect = to_sk_rect(&element.bounds); + let position = object_position_or_center(style.object_position); + draw_image( + canvas, + &image, + &dest_rect, + style.object_fit.unwrap_or(ObjectFit::Cover), + position, + ); + } + } + + // --- Video content --- + if let ElementKind::Video { ref src } = element.kind { + if let Some(image) = images.get_or_load_video_frame(src, time_secs).cloned() { + let dest_rect = to_sk_rect(&element.bounds); + let position = object_position_or_center(style.object_position); + draw_image( + canvas, + &image, + &dest_rect, + style.object_fit.unwrap_or(ObjectFit::Cover), + position, + ); + } + } + + // --- Children --- + for child in &element.children { + paint_element_at_time(canvas, child, images, time_secs); + } + + canvas.restore_to_count(save_count); +} diff --git a/packages/native-renderer/src/paint/images.rs b/packages/native-renderer/src/paint/images.rs new file mode 100644 index 000000000..95881b7a9 --- /dev/null +++ b/packages/native-renderer/src/paint/images.rs @@ -0,0 +1,183 @@ +use std::collections::{hash_map::DefaultHasher, HashMap}; +use std::hash::{Hash, Hasher}; +use std::path::PathBuf; +use std::process::Command; + +use base64::Engine; +use skia_safe::{Data, Image}; + +/// Thread-safe image cache that loads images from disk on first access and +/// returns the cached `skia_safe::Image` on subsequent lookups. +pub struct ImageCache { + cache: HashMap, + video_frames: HashMap, + video_inputs: HashMap, +} + +impl ImageCache { + pub fn new() -> Self { + Self { + cache: HashMap::new(), + video_frames: HashMap::new(), + video_inputs: HashMap::new(), + } + } + + /// Return a cached image for `src`, loading from disk on first access. + /// Returns `None` if the file cannot be read or Skia fails to decode it. + pub fn get_or_load(&mut self, src: &str) -> Option<&Image> { + if !self.cache.contains_key(src) { + let image = load_image(src)?; + self.cache.insert(src.to_string(), image); + } + self.cache.get(src) + } + + /// Number of images currently held in the cache. + pub fn len(&self) -> usize { + self.cache.len() + } + + /// Return a decoded video frame for `src` at `time_secs`, loading via + /// FFmpeg on first access. This is intentionally correctness-first; higher + /// throughput comes from pre-extracting frame ranges into this cache. + pub fn get_or_load_video_frame(&mut self, src: &str, time_secs: f64) -> Option<&Image> { + let time_key = (time_secs.max(0.0) * 1000.0).round() as u64; + let key = format!("{src}#{time_key}"); + if !self.video_frames.contains_key(&key) { + let input = self.get_or_resolve_video_input(src)?; + let image = load_video_frame(&input, time_secs)?; + self.video_frames.insert(key.clone(), image); + } + self.video_frames.get(&key) + } + + fn get_or_resolve_video_input(&mut self, src: &str) -> Option { + if !self.video_inputs.contains_key(src) { + let input = resolve_video_input(src)?; + self.video_inputs.insert(src.to_string(), input); + } + self.video_inputs.get(src).cloned() + } +} + +/// Read bytes from disk and decode into a Skia `Image`. +fn load_image(src: &str) -> Option { + let bytes = load_bytes(src)?; + let data = Data::new_copy(&bytes); + Image::from_encoded(data) +} + +fn load_bytes(src: &str) -> Option> { + if let Some(rest) = src.strip_prefix("data:") { + return decode_data_url(rest); + } + + if let Some(path) = src.strip_prefix("file://") { + return std::fs::read(percent_decode(path)).ok(); + } + + if src.starts_with("http://") || src.starts_with("https://") { + let output = Command::new("curl") + .args(["-fsSL", "--max-time", "20", src]) + .output() + .ok()?; + return output.status.success().then_some(output.stdout); + } + + std::fs::read(src).ok() +} + +fn decode_data_url(rest: &str) -> Option> { + let (meta, payload) = rest.split_once(',')?; + if meta.ends_with(";base64") { + base64::engine::general_purpose::STANDARD + .decode(payload) + .ok() + } else { + Some(percent_decode(payload).into_bytes()) + } +} + +fn percent_decode(input: &str) -> String { + let bytes = input.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' && i + 2 < bytes.len() { + if let Ok(hex) = std::str::from_utf8(&bytes[i + 1..i + 3]) { + if let Ok(value) = u8::from_str_radix(hex, 16) { + out.push(value); + i += 3; + continue; + } + } + } + out.push(bytes[i]); + i += 1; + } + String::from_utf8_lossy(&out).into_owned() +} + +fn resolve_video_input(src: &str) -> Option { + if src.starts_with("http://") || src.starts_with("https://") { + return download_video_to_cache(src); + } + + src.strip_prefix("file://") + .map(percent_decode) + .or_else(|| Some(src.to_string())) +} + +fn download_video_to_cache(src: &str) -> Option { + let path = cached_video_path(src); + if !path.exists() { + let output = Command::new("curl") + .args(["-fsSL", "--max-time", "120", "-o", path.to_str()?, src]) + .output() + .ok()?; + if !output.status.success() { + let _ = std::fs::remove_file(&path); + return None; + } + } + Some(path.to_string_lossy().into_owned()) +} + +fn cached_video_path(src: &str) -> PathBuf { + let mut hasher = DefaultHasher::new(); + src.hash(&mut hasher); + std::env::temp_dir().join(format!( + "hyperframes-native-video-{:016x}.mp4", + hasher.finish() + )) +} + +fn load_video_frame(input: &str, time_secs: f64) -> Option { + let time_arg = format!("{:.6}", time_secs.max(0.0)); + let output = Command::new("ffmpeg") + .args([ + "-v", + "error", + "-ss", + &time_arg, + "-i", + &input, + "-frames:v", + "1", + "-f", + "image2pipe", + "-vcodec", + "png", + "-", + ]) + .output() + .ok()?; + + if !output.status.success() || output.stdout.is_empty() { + return None; + } + + let data = Data::new_copy(&output.stdout); + Image::from_encoded(data) +} diff --git a/packages/native-renderer/src/paint/mod.rs b/packages/native-renderer/src/paint/mod.rs new file mode 100644 index 000000000..9bcd54cad --- /dev/null +++ b/packages/native-renderer/src/paint/mod.rs @@ -0,0 +1,8 @@ +mod canvas; +pub mod effects; +pub mod elements; +pub mod images; + +pub use canvas::RenderSurface; +pub use elements::{paint_element, paint_element_at_time}; +pub use images::ImageCache; diff --git a/packages/native-renderer/src/pipeline.rs b/packages/native-renderer/src/pipeline.rs new file mode 100644 index 000000000..08a80fefe --- /dev/null +++ b/packages/native-renderer/src/pipeline.rs @@ -0,0 +1,460 @@ +use std::io::Write; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc::{sync_channel, SyncSender}; +use std::thread::JoinHandle; +use std::time::Instant; + +use skia_safe::Color4f; + +use crate::encode::{detect_hw_encoder, encoder_args, raw_pixel_encoder_args, HwEncoder}; +use crate::paint::{paint_element, paint_element_at_time, ImageCache, RenderSurface}; +use crate::scene::{BakedElementState, BakedFrame, BakedTimeline, Element, Scene, Transform2D}; + +/// Configuration for a render pass. +pub struct RenderConfig { + pub fps: u32, + pub duration_secs: f64, + pub quality: u32, + pub output_path: String, +} + +/// Timing and metadata returned after a successful render. +pub struct RenderResult { + pub total_frames: u32, + pub total_ms: u64, + pub avg_paint_ms: f64, + pub output_path: String, +} + +/// Spawn an FFmpeg process that accepts MJPEG frames on stdin and writes +/// video to `config.output_path`. +/// +/// Uses [`detect_hw_encoder`] to pick the best available codec: +/// - macOS: `hevc_videotoolbox` (Apple Silicon hardware) +/// - Linux NVIDIA: `h264_nvenc` +/// - Linux Intel/AMD: `h264_vaapi` +/// - Fallback: `libx264` (CPU) +fn spawn_ffmpeg(config: &RenderConfig) -> Result<(Child, HwEncoder), String> { + spawn_ffmpeg_with_encoder(config, detect_hw_encoder()) +} + +/// Spawn FFmpeg with a specific encoder (useful for tests and benchmarks +/// that need deterministic codec selection). +fn spawn_ffmpeg_with_encoder( + config: &RenderConfig, + encoder: HwEncoder, +) -> Result<(Child, HwEncoder), String> { + let mut args = encoder_args(encoder, config.fps, config.quality); + args.push(config.output_path.clone()); + + let child = Command::new("ffmpeg") + .args(&args) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("failed to spawn ffmpeg: {e}"))?; + + Ok((child, encoder)) +} + +/// Wait for FFmpeg to finish and return an error if it exited non-zero. +fn finish_ffmpeg(child: Child) -> Result<(), String> { + let output = child + .wait_with_output() + .map_err(|e| format!("failed to wait for ffmpeg: {e}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("ffmpeg exited with {}: {stderr}", output.status)); + } + Ok(()) +} + +fn spawn_raw_rgba_ffmpeg_writer( + config: &RenderConfig, + width: u32, + height: u32, +) -> Result< + ( + SyncSender>, + JoinHandle>, + HwEncoder, + ), + String, +> { + let encoder = detect_hw_encoder(); + let mut args = raw_pixel_encoder_args(encoder, config.fps, config.quality, width, height); + args.push(config.output_path.clone()); + + let mut child = Command::new("ffmpeg") + .args(&args) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("failed to spawn ffmpeg: {e}"))?; + + let mut stdin = child.stdin.take().ok_or("failed to open ffmpeg stdin")?; + // Deep buffer lets the GPU render 16 frames ahead of FFmpeg, + // decoupling paint throughput from encode throughput. + let (tx, rx) = sync_channel::>(16); + + let writer = std::thread::spawn(move || { + for frame in rx { + stdin + .write_all(&frame) + .map_err(|e| format!("failed to write raw frame to ffmpeg: {e}"))?; + } + drop(stdin); + Ok(child) + }); + + Ok((tx, writer, encoder)) +} + +fn finish_ffmpeg_writer(writer: JoinHandle>) -> Result<(), String> { + let child = writer + .join() + .map_err(|_| "ffmpeg writer thread panicked".to_string())??; + finish_ffmpeg(child) +} + +/// Render a static scene (no animation) to a video file via FFmpeg pipe. +/// +/// The scene is painted once and the resulting JPEG frame is written +/// `total_frames` times to FFmpeg's stdin, producing a still-image video. +pub fn render_static(scene: &Scene, config: &RenderConfig) -> Result { + let total_frames = (config.fps as f64 * config.duration_secs).ceil() as u32; + if total_frames == 0 { + return Err("total_frames is zero — check fps and duration_secs".into()); + } + + // Paint once. + let paint_start = Instant::now(); + + let mut surface = RenderSurface::new_raster(scene.width as i32, scene.height as i32)?; + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + + let mut image_cache = ImageCache::new(); + for element in &scene.elements { + paint_element(surface.canvas(), element, &mut image_cache); + } + + let frame_jpeg = surface + .encode_jpeg(config.quality) + .ok_or("failed to encode frame as JPEG")?; + + let paint_ms = paint_start.elapsed().as_secs_f64() * 1000.0; + + // Spawn FFmpeg and pipe frames. + let (mut child, _encoder) = spawn_ffmpeg(config)?; + let write_start = Instant::now(); + { + let stdin = child.stdin.as_mut().ok_or("failed to open ffmpeg stdin")?; + + for _ in 0..total_frames { + stdin + .write_all(&frame_jpeg) + .map_err(|e| format!("failed to write frame to ffmpeg: {e}"))?; + } + } + // stdin is dropped here, signalling EOF to FFmpeg. + + finish_ffmpeg(child)?; + + let total_ms = write_start.elapsed().as_millis() as u64; + + Ok(RenderResult { + total_frames, + total_ms, + avg_paint_ms: paint_ms, + output_path: config.output_path.clone(), + }) +} + +// ── Raw Frame Pipeline (fastest render, deferred encode) ──────────────────── + +/// Render all frames to a raw BGRA file at maximum speed, then encode +/// with a single FFmpeg batch call. Skips I420 conversion during render — +/// writes BGRA directly (Skia's native format), lets FFmpeg batch-convert. +/// +/// This is the fastest render path: paint + write, no color conversion. +/// Pre-allocates the BGRA buffer once and reuses it across frames. +pub fn render_animated_raw_then_encode( + scene: &Scene, + timeline: &BakedTimeline, + config: &RenderConfig, +) -> Result { + let total_frames = timeline.total_frames; + if total_frames == 0 { + return Err("timeline has zero frames".into()); + } + + let w = scene.width as i32; + let h = scene.height as i32; + let frame_bytes = w as usize * h as usize * 4; // BGRA + + let mut surface = RenderSurface::new_raster(w, h)?; + let mut images = ImageCache::new(); + // Pre-allocate BGRA buffer — reused every frame (no per-frame allocation) + let mut bgra_buf = vec![0u8; frame_bytes]; + + let raw_path = format!("{}.raw", config.output_path); + let mut raw_file = std::fs::File::create(&raw_path) + .map_err(|e| format!("create raw file: {e}"))?; + + let render_start = Instant::now(); + let mut paint_total_ms: f64 = 0.0; + + for frame in &timeline.frames { + let animated_scene = apply_frame_deltas(scene, frame); + + let paint_start = Instant::now(); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &animated_scene.elements { + paint_element_at_time(surface.canvas(), element, &mut images, frame.time); + } + paint_total_ms += paint_start.elapsed().as_secs_f64() * 1000.0; + + // Read BGRA directly — no I420 conversion in the render loop + surface.read_pixels_bgra_into(&mut bgra_buf) + .ok_or("read pixels failed")?; + + use std::io::Write as _; + raw_file.write_all(&bgra_buf).map_err(|e| format!("write raw: {e}"))?; + } + + let render_ms = render_start.elapsed().as_millis() as u64; + + // Phase 2: Single FFmpeg batch encode — converts BGRA→YUV + encodes + let ffmpeg_child = Command::new("ffmpeg") + .args([ + "-y", + "-f", "rawvideo", + "-pix_fmt", "bgra", + "-s", &format!("{}x{}", w, h), + "-r", &config.fps.to_string(), + "-i", &raw_path, + "-c:v", "libx264", + "-preset", "ultrafast", + "-crf", "18", + "-threads", "0", + "-pix_fmt", "yuv420p", + &config.output_path, + ]) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("ffmpeg spawn: {e}"))?; + + finish_ffmpeg(ffmpeg_child)?; + std::fs::remove_file(&raw_path).ok(); + + let total_ms = render_start.elapsed().as_millis() as u64; + Ok(RenderResult { + total_frames, + total_ms, + avg_paint_ms: paint_total_ms / total_frames as f64, + output_path: config.output_path.clone(), + }) +} + +// ── Native Pipeline (no FFmpeg) ───────────────────────────────────────────── + +/// Render an animated scene entirely in Rust — no FFmpeg subprocess. +/// Uses openh264 for H.264 encoding and minimp4 for MP4 muxing. +/// This is the fastest path: no pipe, no subprocess startup, no serialization. +pub fn render_animated_native( + scene: &Scene, + timeline: &BakedTimeline, + config: &RenderConfig, +) -> Result { + use crate::native_encode::NativeEncoder; + + let total_frames = timeline.total_frames; + if total_frames == 0 { + return Err("timeline has zero frames".into()); + } + + let w = scene.width; + let h = scene.height; + let mut encoder = NativeEncoder::new(w, h, config.fps)?; + let mut surface = RenderSurface::new_raster(w as i32, h as i32)?; + let mut images = ImageCache::new(); + + let start = Instant::now(); + let mut paint_total_ms: f64 = 0.0; + + for frame in &timeline.frames { + let animated_scene = apply_frame_deltas(scene, frame); + + let paint_start = Instant::now(); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &animated_scene.elements { + paint_element_at_time(surface.canvas(), element, &mut images, frame.time); + } + paint_total_ms += paint_start.elapsed().as_secs_f64() * 1000.0; + + let bgra = surface + .read_pixels_bgra() + .ok_or("failed to read pixels")?; + encoder.encode_bgra_frame(&bgra)?; + } + + encoder.finish_to_mp4(&config.output_path)?; + + let total_ms = start.elapsed().as_millis() as u64; + Ok(RenderResult { + total_frames, + total_ms, + avg_paint_ms: paint_total_ms / total_frames as f64, + output_path: config.output_path.clone(), + }) +} + +// ── Animated Pipeline (FFmpeg) ────────────────────────────────────────────── + +/// Render an animated scene driven by a pre-baked timeline. +/// +/// Each frame in the timeline carries a full snapshot of every animated +/// element's visual state. For each frame we clone the base scene, apply +/// the per-element deltas, paint, encode to JPEG, and pipe into FFmpeg. +pub fn render_animated( + scene: &Scene, + timeline: &BakedTimeline, + config: &RenderConfig, +) -> Result { + let total_frames = timeline.total_frames; + if total_frames == 0 { + return Err("timeline has zero frames".into()); + } + + let mut surface = RenderSurface::new_raster(scene.width as i32, scene.height as i32)?; + let mut image_cache = ImageCache::new(); + + let (mut child, _encoder) = spawn_ffmpeg(config)?; + let stdin = child.stdin.as_mut().ok_or("failed to open ffmpeg stdin")?; + + let start = Instant::now(); + let mut paint_total_ms: f64 = 0.0; + + for frame in &timeline.frames { + let animated_scene = apply_frame_deltas(scene, frame); + + let paint_start = Instant::now(); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &animated_scene.elements { + paint_element_at_time(surface.canvas(), element, &mut image_cache, frame.time); + } + paint_total_ms += paint_start.elapsed().as_secs_f64() * 1000.0; + + let jpeg = surface + .encode_jpeg(config.quality) + .ok_or("failed to encode animated frame as JPEG")?; + stdin + .write_all(&jpeg) + .map_err(|e| format!("failed to write animated frame to ffmpeg: {e}"))?; + } + + // Close stdin to signal EOF, then wait for FFmpeg. + drop(child.stdin.take()); + finish_ffmpeg(child)?; + + let total_ms = start.elapsed().as_millis() as u64; + + Ok(RenderResult { + total_frames, + total_ms, + avg_paint_ms: paint_total_ms / total_frames as f64, + output_path: config.output_path.clone(), + }) +} + +/// Clone the scene and apply per-element deltas from a single baked frame. +fn apply_frame_deltas(scene: &Scene, frame: &BakedFrame) -> Scene { + let mut animated = scene.clone(); + apply_deltas_recursive(&mut animated.elements, &frame.elements); + animated +} + +/// Walk the element tree and patch style/transform from the delta map. +fn apply_deltas_recursive( + elements: &mut Vec, + deltas: &std::collections::HashMap, +) { + for element in elements.iter_mut() { + if let Some(state) = deltas.get(&element.id) { + element.style.opacity = state.opacity; + element.style.visibility = state.visibility; + element.style.transform = Some(Transform2D { + translate_x: state.translate_x, + translate_y: state.translate_y, + scale_x: state.scale_x, + scale_y: state.scale_y, + rotate_deg: state.rotate_deg, + }); + } + apply_deltas_recursive(&mut element.children, deltas); + } +} + +// ── Chunk-Parallel GPU Pipeline (macOS Metal) ───────────────────────────── + +/// Render an animated scene with GPU acceleration (Metal on macOS, Vulkan on +/// Linux) and a background raw-pixel pipe to FFmpeg with hardware encoding. +/// +/// Falls back to CPU raster if no GPU is available (Docker without GPU access). +/// Hardware encoding auto-detects: VideoToolbox (macOS), NVENC (NVIDIA), +/// VAAPI (Intel/AMD), libx264 (CPU fallback). +pub fn render_animated_gpu( + scene: &Scene, + timeline: &BakedTimeline, + config: &RenderConfig, +) -> Result { + let total_frames = timeline.total_frames; + if total_frames == 0 { + return Err("timeline has zero frames".into()); + } + + let width = scene.width as i32; + let height = scene.height as i32; + let mut surface = RenderSurface::new_gpu_or_raster(width, height)?; + let mut images = ImageCache::new(); + + let (frame_tx, writer, _encoder) = + spawn_raw_rgba_ffmpeg_writer(config, scene.width, scene.height)?; + + let start = Instant::now(); + let mut paint_total_ms: f64 = 0.0; + + for (_i, frame) in timeline.frames.iter().enumerate() { + let animated_scene = apply_frame_deltas(scene, frame); + + let paint_start = Instant::now(); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + for element in &animated_scene.elements { + paint_element_at_time(surface.canvas(), element, &mut images, frame.time); + } + surface.flush_and_submit(); + paint_total_ms += paint_start.elapsed().as_secs_f64() * 1000.0; + + let bgra = surface + .read_pixels_bgra() + .ok_or("failed to read GPU frame pixels")?; + frame_tx + .send(bgra) + .map_err(|e| format!("failed to queue GPU frame for ffmpeg: {e}"))?; + } + + drop(frame_tx); + finish_ffmpeg_writer(writer)?; + + let total_ms = start.elapsed().as_millis() as u64; + Ok(RenderResult { + total_frames, + total_ms, + avg_paint_ms: paint_total_ms / total_frames as f64, + output_path: config.output_path.clone(), + }) +} diff --git a/packages/native-renderer/src/scene/extract.test.ts b/packages/native-renderer/src/scene/extract.test.ts new file mode 100644 index 000000000..c9d61372e --- /dev/null +++ b/packages/native-renderer/src/scene/extract.test.ts @@ -0,0 +1,273 @@ +import { describe, expect, it } from "vitest"; +import type { ExtractedScene, SceneElement } from "./extract"; + +describe("ExtractedScene types", () => { + it("produces JSON compatible with Rust scene types", () => { + const scene: ExtractedScene = { + width: 1920, + height: 1080, + elements: [ + { + id: "bg", + kind: { type: "Container" }, + bounds: { x: 0, y: 0, width: 1920, height: 1080 }, + style: { + background_color: { r: 30, g: 30, b: 30, a: 255 }, + opacity: 1, + border_radius: [0, 0, 0, 0], + overflow_hidden: false, + transform: null, + visibility: true, + font_family: null, + font_size: null, + font_weight: null, + color: null, + }, + children: [], + }, + { + id: "title", + kind: { type: "Text", content: "Hello World" }, + bounds: { x: 100, y: 100, width: 400, height: 50 }, + style: { + background_color: null, + opacity: 1, + border_radius: [0, 0, 0, 0], + overflow_hidden: false, + transform: null, + visibility: true, + font_family: "Inter", + font_size: 32, + font_weight: 700, + color: { r: 255, g: 255, b: 255, a: 255 }, + }, + children: [], + }, + ], + }; + + const json = JSON.stringify(scene); + const parsed = JSON.parse(json); + + // Verify the `kind` nested structure matches Rust's serde(tag = "type") format + expect(parsed.elements[0].kind.type).toBe("Container"); + expect(parsed.elements[1].kind.type).toBe("Text"); + expect(parsed.elements[1].kind.content).toBe("Hello World"); + expect(parsed.elements[0].style.background_color.r).toBe(30); + }); + + it("matches exact Rust scene_test.rs JSON shapes", () => { + // Reproduce the JSON from Rust's parse_minimal_scene test verbatim + const rustCompatibleJSON = JSON.stringify({ + width: 1920, + height: 1080, + elements: [ + { + id: "bg", + kind: { type: "Container" }, + bounds: { x: 0, y: 0, width: 1920, height: 1080 }, + style: { + background_color: { r: 30, g: 30, b: 30, a: 255 }, + opacity: 1.0, + border_radius: [0, 0, 0, 0], + overflow_hidden: false, + transform: null, + visibility: true, + }, + children: [], + }, + ], + }); + + // This must be parseable by the Rust side. We verify structural invariants: + const parsed = JSON.parse(rustCompatibleJSON); + expect(parsed.elements[0].kind).toEqual({ type: "Container" }); + expect(parsed.elements[0].bounds).toEqual({ + x: 0, + y: 0, + width: 1920, + height: 1080, + }); + }); + + it("serializes Image and Video kinds with src field", () => { + const elements: SceneElement[] = [ + { + id: "bg-img", + kind: { type: "Image", src: "/assets/bg.png" }, + bounds: { x: 0, y: 0, width: 1920, height: 1080 }, + style: { + background_color: null, + opacity: 1, + border_radius: [0, 0, 0, 0], + overflow_hidden: false, + transform: null, + visibility: true, + font_family: null, + font_size: null, + font_weight: null, + color: null, + }, + children: [], + }, + { + id: "clip", + kind: { type: "Video", src: "/assets/intro.mp4" }, + bounds: { x: 100, y: 100, width: 800, height: 450 }, + style: { + background_color: null, + opacity: 0.8, + border_radius: [12, 12, 12, 12], + overflow_hidden: true, + transform: null, + visibility: true, + font_family: null, + font_size: null, + font_weight: null, + color: null, + }, + children: [], + }, + ]; + + const json = JSON.stringify({ width: 1920, height: 1080, elements }); + const parsed = JSON.parse(json); + + expect(parsed.elements[0].kind).toEqual({ + type: "Image", + src: "/assets/bg.png", + }); + expect(parsed.elements[1].kind).toEqual({ + type: "Video", + src: "/assets/intro.mp4", + }); + expect(parsed.elements[1].style.opacity).toBe(0.8); + expect(parsed.elements[1].style.overflow_hidden).toBe(true); + expect(parsed.elements[1].style.border_radius).toEqual([12, 12, 12, 12]); + }); + + it("serializes background-image URL metadata", () => { + const el: SceneElement = { + id: "poster", + kind: { type: "Container" }, + bounds: { x: 0, y: 0, width: 640, height: 360 }, + style: { + background_color: null, + opacity: 1, + border_radius: [0, 0, 0, 0], + overflow_hidden: false, + transform: null, + visibility: true, + font_family: null, + font_size: null, + font_weight: null, + color: null, + background_image: { + src: "file:///tmp/poster.png", + fit: "contain", + position: { x: 0.25, y: 0.75 }, + }, + }, + children: [], + }; + + const parsed = JSON.parse(JSON.stringify(el)); + expect(parsed.style.background_image).toEqual({ + src: "file:///tmp/poster.png", + fit: "contain", + position: { x: 0.25, y: 0.75 }, + }); + }); + + it("serializes Transform2D correctly", () => { + const el: SceneElement = { + id: "box", + kind: { type: "Container" }, + bounds: { x: 100, y: 100, width: 200, height: 200 }, + style: { + background_color: null, + opacity: 1, + border_radius: [0, 0, 0, 0], + overflow_hidden: false, + transform: { + translate_x: 50, + translate_y: -30, + scale_x: 1.5, + scale_y: 1.5, + rotate_deg: 45, + }, + visibility: true, + font_family: null, + font_size: null, + font_weight: null, + color: null, + }, + children: [], + }; + + const json = JSON.stringify(el); + const parsed = JSON.parse(json); + expect(parsed.style.transform).toEqual({ + translate_x: 50, + translate_y: -30, + scale_x: 1.5, + scale_y: 1.5, + rotate_deg: 45, + }); + }); + + it("supports nested children", () => { + const scene: ExtractedScene = { + width: 1280, + height: 720, + elements: [ + { + id: "root", + kind: { type: "Container" }, + bounds: { x: 0, y: 0, width: 1280, height: 720 }, + style: { + background_color: null, + opacity: 1, + border_radius: [0, 0, 0, 0], + overflow_hidden: false, + transform: null, + visibility: true, + font_family: null, + font_size: null, + font_weight: null, + color: null, + }, + children: [ + { + id: "title", + kind: { type: "Text", content: "Hello World" }, + bounds: { x: 100, y: 50, width: 400, height: 60 }, + style: { + background_color: null, + opacity: 1, + border_radius: [0, 0, 0, 0], + overflow_hidden: false, + transform: null, + visibility: true, + font_family: "Inter", + font_size: 48, + font_weight: 700, + color: { r: 255, g: 255, b: 255, a: 255 }, + }, + children: [], + }, + ], + }, + ], + }; + + const json = JSON.stringify(scene); + const parsed = JSON.parse(json); + expect(parsed.elements[0].children).toHaveLength(1); + expect(parsed.elements[0].children[0].kind).toEqual({ + type: "Text", + content: "Hello World", + }); + expect(parsed.elements[0].children[0].style.font_family).toBe("Inter"); + }); +}); diff --git a/packages/native-renderer/src/scene/extract.ts b/packages/native-renderer/src/scene/extract.ts new file mode 100644 index 000000000..0f6201c7a --- /dev/null +++ b/packages/native-renderer/src/scene/extract.ts @@ -0,0 +1,592 @@ +/** + * CDP scene extraction — walks a Chrome page's DOM via Puppeteer and produces + * a JSON scene graph that the Rust `parse_scene_json()` function can consume. + */ +import type { Page } from "puppeteer-core"; + +// --------------------------------------------------------------------------- +// Types — mirrors the Rust scene graph in packages/native-renderer/src/scene/mod.rs +// --------------------------------------------------------------------------- + +export interface SceneColor { + r: number; + g: number; + b: number; + a: number; +} + +export interface Transform2D { + translate_x: number; + translate_y: number; + scale_x: number; + scale_y: number; + rotate_deg: number; +} + +export interface Rect { + x: number; + y: number; + width: number; + height: number; +} + +export interface BoxShadow { + offset_x: number; + offset_y: number; + blur_radius: number; + spread_radius: number; + color: SceneColor; +} + +export interface Border { + width: number; + color: SceneColor; + style: "solid" | "dashed"; +} + +export type ClipPath = + | { type: "Polygon"; points: Array<{ x: number; y: number }> } + | { type: "Circle"; x: number; y: number; radius: number } + | { type: "Ellipse"; x: number; y: number; radius_x: number; radius_y: number }; + +export type Gradient = + | { type: "Linear"; angle_deg: number; stops: GradientStop[] } + | { type: "Radial"; stops: GradientStop[] }; + +export interface GradientStop { + position: number; + color: SceneColor; +} + +export interface FilterAdjust { + brightness: number; + contrast: number; + saturate: number; +} + +export interface TextStroke { + width: number; + color: SceneColor; +} + +export type ObjectFit = "fill" | "contain" | "cover" | "none" | "scale_down"; + +export type BackgroundImageFit = "fill" | "contain" | "cover" | "none"; + +export interface ObjectPosition { + x: number; + y: number; +} + +export interface BackgroundImage { + src: string; + fit: BackgroundImageFit; + position: ObjectPosition; +} + +export type MixBlendMode = + | "multiply" + | "screen" + | "overlay" + | "darken" + | "lighten" + | "color_dodge" + | "color_burn" + | "hard_light" + | "soft_light" + | "difference" + | "exclusion" + | "hue" + | "saturation" + | "color" + | "luminosity"; + +export interface ElementStyle { + background_color: SceneColor | null; + opacity: number; + border_radius: [number, number, number, number]; + border?: Border | null; + overflow_hidden: boolean; + clip_path?: ClipPath | null; + transform: Transform2D | null; + visibility: boolean; + font_family: string | null; + font_size: number | null; + font_weight: number | null; + color: SceneColor | null; + text_shadow?: BoxShadow | null; + text_stroke?: TextStroke | null; + box_shadow?: BoxShadow | null; + filter_blur?: number | null; + filter_adjust?: FilterAdjust | null; + background_image?: BackgroundImage | null; + background_gradient?: Gradient | null; + object_fit?: ObjectFit | null; + object_position?: ObjectPosition | null; + mix_blend_mode?: MixBlendMode | null; +} + +/** + * Discriminated element kind — matches Rust `ElementKind` which uses + * `#[serde(tag = "type")]` internally-tagged enum. + */ +export type ElementKind = + | { type: "Container" } + | { type: "Text"; content: string } + | { type: "Image"; src: string } + | { type: "Video"; src: string }; + +export interface SceneElement { + id: string; + kind: ElementKind; + bounds: Rect; + style: ElementStyle; + children: SceneElement[]; +} + +export interface ExtractedScene { + width: number; + height: number; + elements: SceneElement[]; +} + +// String-based evaluate avoids tsx/esbuild injecting `__name` helpers into the +// function body that Puppeteer serializes into the browser context. +const EXTRACT_SCENE_SCRIPT = `(() => { + function parseColor(cssColor) { + if (!cssColor || cssColor === "transparent") return null; + const m = cssColor.match(/rgba?\\(\\s*([\\d.]+),\\s*([\\d.]+),\\s*([\\d.]+)(?:,\\s*([\\d.]+))?\\s*\\)/); + if (!m) return null; + return { + r: Math.round(+m[1]), + g: Math.round(+m[2]), + b: Math.round(+m[3]), + a: Math.round((m[4] !== undefined ? +m[4] : 1) * 255), + }; + } + + function parseTransform(raw) { + if (raw === "none") return null; + let tx = 0; + let ty = 0; + let sx = 1; + let sy = 1; + let rot = 0; + const mat = raw.match( + /matrix\\(\\s*([-\\d.e]+),\\s*([-\\d.e]+),\\s*([-\\d.e]+),\\s*([-\\d.e]+),\\s*([-\\d.e]+),\\s*([-\\d.e]+)\\)/, + ); + if (mat) { + const a = +mat[1]; + const b = +mat[2]; + const c = +mat[3]; + const d = +mat[4]; + tx = +mat[5]; + ty = +mat[6]; + sx = Math.sqrt(a * a + b * b); + sy = Math.sqrt(c * c + d * d); + rot = (Math.atan2(b, a) * 180) / Math.PI; + } + if (tx === 0 && ty === 0 && sx === 1 && sy === 1 && rot === 0) return null; + return { translate_x: tx, translate_y: ty, scale_x: sx, scale_y: sy, rotate_deg: rot }; + } + + function splitTopLevel(input) { + const parts = []; + let depth = 0; + let start = 0; + for (let i = 0; i < input.length; i++) { + const ch = input[i]; + if (ch === "(") depth++; + if (ch === ")") depth--; + if (ch === "," && depth === 0) { + parts.push(input.slice(start, i).trim()); + start = i + 1; + } + } + parts.push(input.slice(start).trim()); + return parts.filter(Boolean); + } + + function firstColorToken(raw) { + return raw.match(/rgba?\\([^)]*\\)/)?.[0] ?? null; + } + + function parseShadow(raw) { + if (!raw || raw === "none") return null; + const firstShadow = splitTopLevel(raw)[0]; + if (!firstShadow || /\\binset\\b/.test(firstShadow)) return null; + const colorToken = firstColorToken(firstShadow); + const color = colorToken ? parseColor(colorToken) : { r: 0, g: 0, b: 0, a: 255 }; + if (!color) return null; + const withoutColor = colorToken ? firstShadow.replace(colorToken, "") : firstShadow; + const lengths = Array.from(withoutColor.matchAll(/(-?[\\d.]+)px/g)).map((m) => +m[1]); + if (lengths.length < 2) return null; + return { + offset_x: lengths[0] || 0, + offset_y: lengths[1] || 0, + blur_radius: lengths[2] || 0, + spread_radius: lengths[3] || 0, + color, + }; + } + + function parseFilterValue(raw) { + const value = raw.trim(); + if (value.endsWith("%")) return (parseFloat(value) || 0) / 100; + return Number.isFinite(parseFloat(value)) ? parseFloat(value) : 1; + } + + function parseFilter(raw) { + if (!raw || raw === "none") return { blur: null, adjust: null }; + let blur = null; + let brightness = 1; + let contrast = 1; + let saturate = 1; + + for (const match of raw.matchAll(/([a-z-]+)\\(([^)]*)\\)/g)) { + const name = match[1]; + const value = match[2]; + if (name === "blur") blur = parseFloat(value) || null; + if (name === "brightness") brightness = parseFilterValue(value); + if (name === "contrast") contrast = parseFilterValue(value); + if (name === "saturate") saturate = parseFilterValue(value); + } + + const adjust = + brightness !== 1 || contrast !== 1 || saturate !== 1 + ? { brightness, contrast, saturate } + : null; + return { blur, adjust }; + } + + function parseGradientStop(raw, index, total) { + const colorToken = firstColorToken(raw); + const color = colorToken ? parseColor(colorToken) : null; + if (!color) return null; + const withoutColor = raw.replace(colorToken, "").trim(); + const stopMatch = withoutColor.match(/(-?[\\d.]+)%/); + const fallback = total <= 1 ? 0 : index / (total - 1); + return { + position: stopMatch ? Math.max(0, Math.min(1, +stopMatch[1] / 100)) : fallback, + color, + }; + } + + function parseGradient(raw) { + if (!raw || raw === "none") return null; + + const linear = raw.match(/^linear-gradient\\((.*)\\)$/); + if (linear) { + const parts = splitTopLevel(linear[1]); + let angleDeg = 180; + let stopParts = parts; + const first = parts[0] || ""; + if (/^-?[\\d.]+deg$/.test(first)) { + angleDeg = parseFloat(first); + stopParts = parts.slice(1); + } else if (first.startsWith("to ")) { + if (first.includes("right")) angleDeg = 90; + else if (first.includes("left")) angleDeg = 270; + else if (first.includes("top")) angleDeg = 0; + else if (first.includes("bottom")) angleDeg = 180; + stopParts = parts.slice(1); + } + const stops = stopParts + .map((part, index) => parseGradientStop(part, index, stopParts.length)) + .filter(Boolean); + return stops.length >= 2 ? { type: "Linear", angle_deg: angleDeg, stops } : null; + } + + const radial = raw.match(/^radial-gradient\\((.*)\\)$/); + if (radial) { + const parts = splitTopLevel(radial[1]); + const stopParts = parts.filter((part) => firstColorToken(part)); + const stops = stopParts + .map((part, index) => parseGradientStop(part, index, stopParts.length)) + .filter(Boolean); + return stops.length >= 2 ? { type: "Radial", stops } : null; + } + + return null; + } + + function parseCssUrl(raw) { + const firstLayer = splitTopLevel(raw || "")[0]; + const match = firstLayer?.match(/^url\\((.*)\\)$/); + if (!match) return null; + const unquoted = match[1].trim().replace(/^['"]|['"]$/g, ""); + try { + const url = new URL(unquoted, document.baseURI); + if (url.protocol === "file:") return decodeURIComponent(url.pathname); + return url.href; + } catch { + return unquoted || null; + } + } + + function parseBackgroundSize(raw) { + const first = splitTopLevel(raw || "")[0] || "cover"; + if (first === "cover" || first === "contain") return first; + if (first === "auto") return "none"; + if (first === "100% 100%" || first === "100%") return "fill"; + return "cover"; + } + + function parseBackgroundImage(cs, width, height) { + if (!cs.backgroundImage || cs.backgroundImage === "none") return null; + if (/^(linear-gradient|radial-gradient)\\(/.test(cs.backgroundImage)) return null; + const src = parseCssUrl(cs.backgroundImage); + if (!src) return null; + return { + src, + fit: parseBackgroundSize(cs.backgroundSize), + position: parseObjectPosition(cs.backgroundPosition, width, height), + }; + } + + function parseBorder(cs) { + const width = parseFloat(cs.borderTopWidth) || 0; + const style = cs.borderTopStyle; + if (width <= 0 || (style !== "solid" && style !== "dashed")) return null; + const color = parseColor(cs.borderTopColor); + if (!color || color.a === 0) return null; + return { width, color, style }; + } + + function lengthOrPercent(token, basis) { + const value = token.trim(); + if (value.endsWith("%")) return (parseFloat(value) / 100) * basis; + if (value.endsWith("px")) return parseFloat(value) || 0; + const number = parseFloat(value); + return Number.isFinite(number) ? number : 0; + } + + function parseClipPath(raw, width, height) { + if (!raw || raw === "none") return null; + + const polygon = raw.match(/^polygon\\((.*)\\)$/); + if (polygon) { + const points = splitTopLevel(polygon[1]) + .map((pair) => pair.trim().split(/\\s+/)) + .filter((pair) => pair.length >= 2) + .map(([x, y]) => ({ x: lengthOrPercent(x, width), y: lengthOrPercent(y, height) })); + return points.length >= 3 ? { type: "Polygon", points } : null; + } + + const circle = raw.match(/^circle\\((.*)\\)$/); + if (circle) { + const parts = circle[1].split(/\\s+at\\s+/); + const radius = lengthOrPercent(parts[0] || "50%", Math.min(width, height)); + const center = (parts[1] || "50% 50%").trim().split(/\\s+/); + return { + type: "Circle", + x: lengthOrPercent(center[0] || "50%", width), + y: lengthOrPercent(center[1] || "50%", height), + radius, + }; + } + + const ellipse = raw.match(/^ellipse\\((.*)\\)$/); + if (ellipse) { + const parts = ellipse[1].split(/\\s+at\\s+/); + const radii = (parts[0] || "50% 50%").trim().split(/\\s+/); + const center = (parts[1] || "50% 50%").trim().split(/\\s+/); + return { + type: "Ellipse", + x: lengthOrPercent(center[0] || "50%", width), + y: lengthOrPercent(center[1] || "50%", height), + radius_x: lengthOrPercent(radii[0] || "50%", width), + radius_y: lengthOrPercent(radii[1] || radii[0] || "50%", height), + }; + } + + return null; + } + + function parseObjectFit(raw) { + if (raw === "scale-down") return "scale_down"; + if (raw === "fill" || raw === "contain" || raw === "cover" || raw === "none") return raw; + return null; + } + + function parsePositionToken(token, basis, axis) { + const value = token.trim(); + if (value === "left" || value === "top") return 0; + if (value === "center") return 0.5; + if (value === "right" || value === "bottom") return 1; + if (value.endsWith("%")) return Math.max(0, Math.min(1, parseFloat(value) / 100)); + if (value.endsWith("px")) return Math.max(0, Math.min(1, (parseFloat(value) || 0) / basis)); + if (axis === "x" && value === "start") return 0; + if (axis === "x" && value === "end") return 1; + return 0.5; + } + + function parseObjectPosition(raw, width, height) { + const parts = (raw || "50% 50%").trim().split(/\\s+/); + if (parts.length === 1) parts.push("50%"); + return { + x: parsePositionToken(parts[0], width, "x"), + y: parsePositionToken(parts[1], height, "y"), + }; + } + + function parseTextStroke(cs) { + const width = parseFloat(cs.getPropertyValue("-webkit-text-stroke-width")) || 0; + if (width <= 0) return null; + const color = parseColor(cs.getPropertyValue("-webkit-text-stroke-color")); + return color ? { width, color } : null; + } + + function parseMixBlendMode(raw) { + if (!raw || raw === "normal") return null; + const mapped = raw.replace(/-/g, "_"); + const supported = new Set([ + "multiply", + "screen", + "overlay", + "darken", + "lighten", + "color_dodge", + "color_burn", + "hard_light", + "soft_light", + "difference", + "exclusion", + "hue", + "saturation", + "color", + "luminosity", + ]); + return supported.has(mapped) ? mapped : null; + } + + function extract(el, parentRect) { + const cs = getComputedStyle(el); + if (cs.display === "none") return null; + + const tag = el.tagName.toLowerCase(); + const rect = el.getBoundingClientRect(); + const bounds = { + x: rect.x - parentRect.x, + y: rect.y - parentRect.y, + width: rect.width, + height: rect.height, + }; + + let kind; + if (tag === "video") { + kind = { + type: "Video", + src: el.currentSrc || el.src || "", + }; + } else if (tag === "img") { + kind = { + type: "Image", + src: el.currentSrc || el.src || "", + }; + } else if ( + el.childNodes.length > 0 && + Array.from(el.childNodes).every((n) => n.nodeType === Node.TEXT_NODE) && + (el.textContent?.trim() ?? "").length > 0 + ) { + kind = { type: "Text", content: el.textContent.trim() }; + } else { + kind = { type: "Container" }; + } + + const id = + el.getAttribute("data-name") || + el.id || + tag + "-" + Math.round(rect.x) + "-" + Math.round(rect.y); + + const bgColor = parseColor(cs.backgroundColor); + const textColor = parseColor(cs.color); + const transform = parseTransform(cs.transform); + const opacity = parseFloat(cs.opacity) || 0; + const visible = cs.visibility !== "hidden" && opacity > 0; + const isText = kind.type === "Text"; + const filter = parseFilter(cs.filter); + const backgroundGradient = parseGradient(cs.backgroundImage); + const backgroundImage = parseBackgroundImage(cs, rect.width, rect.height); + + const style = { + background_color: bgColor, + opacity, + border_radius: [ + parseFloat(cs.borderTopLeftRadius) || 0, + parseFloat(cs.borderTopRightRadius) || 0, + parseFloat(cs.borderBottomRightRadius) || 0, + parseFloat(cs.borderBottomLeftRadius) || 0, + ], + border: parseBorder(cs), + overflow_hidden: cs.overflow === "hidden" || cs.overflow === "clip", + clip_path: parseClipPath(cs.clipPath, rect.width, rect.height), + transform, + visibility: visible, + font_family: isText + ? cs.fontFamily.replace(/['"]/g, "").split(",")[0].trim() || null + : null, + font_size: isText ? parseFloat(cs.fontSize) || null : null, + font_weight: isText ? parseInt(cs.fontWeight, 10) || null : null, + color: isText ? textColor : null, + text_shadow: isText ? parseShadow(cs.textShadow) : null, + text_stroke: isText ? parseTextStroke(cs) : null, + box_shadow: parseShadow(cs.boxShadow), + filter_blur: filter.blur, + filter_adjust: filter.adjust, + background_image: backgroundImage, + background_gradient: backgroundGradient, + object_fit: kind.type === "Image" || kind.type === "Video" ? parseObjectFit(cs.objectFit) : null, + object_position: + kind.type === "Image" || kind.type === "Video" + ? parseObjectPosition(cs.objectPosition, rect.width, rect.height) + : null, + mix_blend_mode: parseMixBlendMode(cs.mixBlendMode), + }; + + const children = []; + if (kind.type === "Container") { + for (const child of Array.from(el.children)) { + const extracted = extract(child, rect); + if (extracted) children.push(extracted); + } + } + + return { + id, + kind, + bounds, + style, + children, + }; + } + + const root = document.querySelector("[data-composition-id]") ?? document.body; + const rootRect = root.getBoundingClientRect(); + + const extractedRoot = extract(root, { x: 0, y: 0 }); + return extractedRoot ? [extractedRoot] : []; +})()`; + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Extract a scene graph from a Chrome page via CDP. + * + * Walks the DOM starting at `[data-composition-id]` (or `document.body`) and + * produces a JSON-serializable object that the Rust `parse_scene_json()` can + * consume directly. + */ +export async function extractScene( + page: Page, + width: number, + height: number, +): Promise { + await page.setViewport({ width, height }); + + const elements = (await page.evaluate(EXTRACT_SCENE_SCRIPT)) as SceneElement[]; + + return { width, height, elements }; +} diff --git a/packages/native-renderer/src/scene/mod.rs b/packages/native-renderer/src/scene/mod.rs new file mode 100644 index 000000000..682fa15c6 --- /dev/null +++ b/packages/native-renderer/src/scene/mod.rs @@ -0,0 +1,321 @@ +mod parse; + +pub use parse::{parse_scene_file, parse_scene_json}; + +use serde::{Deserialize, Serialize}; + +/// Top-level scene descriptor: a canvas with dimensions and a flat/nested element tree. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Scene { + pub width: u32, + pub height: u32, + pub elements: Vec, +} + +/// A visual element in the scene graph. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Element { + pub id: String, + pub kind: ElementKind, + pub bounds: Rect, + #[serde(default)] + pub style: Style, + #[serde(default)] + pub children: Vec, +} + +/// Discriminated element type. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum ElementKind { + Container, + Text { content: String }, + Image { src: String }, + Video { src: String }, +} + +/// Axis-aligned bounding rectangle. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct Rect { + pub x: f32, + pub y: f32, + pub width: f32, + pub height: f32, +} + +/// Visual style properties applied to an element. +/// +/// `#[serde(default)]` at the struct level means any missing field falls back +/// to `Style::default()`, so partial style objects in JSON are valid. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct Style { + pub background_color: Option, + pub opacity: f32, + pub border_radius: [f32; 4], + pub border: Option, + pub overflow_hidden: bool, + pub clip_path: Option, + pub transform: Option, + pub visibility: bool, + pub font_family: Option, + pub font_size: Option, + pub font_weight: Option, + pub color: Option, + pub text_shadow: Option, + pub text_stroke: Option, + pub box_shadow: Option, + pub filter_blur: Option, + pub filter_adjust: Option, + pub background_image: Option, + pub background_gradient: Option, + pub object_fit: Option, + pub object_position: Option, + pub mix_blend_mode: Option, +} + +impl Default for Style { + fn default() -> Self { + Self { + background_color: None, + opacity: 1.0, + border_radius: [0.0; 4], + border: None, + overflow_hidden: false, + clip_path: None, + transform: None, + visibility: true, + font_family: None, + font_size: None, + font_weight: None, + color: None, + text_shadow: None, + text_stroke: None, + box_shadow: None, + filter_blur: None, + filter_adjust: None, + background_image: None, + background_gradient: None, + object_fit: None, + object_position: None, + mix_blend_mode: None, + } + } +} + +/// CSS border shorthand currently supports solid and dashed line styles. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Border { + pub width: f32, + pub color: Color, + #[serde(default)] + pub style: BorderLineStyle, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BorderLineStyle { + #[default] + Solid, + Dashed, +} + +/// CSS clip-path primitives. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum ClipPath { + Polygon { + points: Vec, + }, + Circle { + x: f32, + y: f32, + radius: f32, + }, + Ellipse { + x: f32, + y: f32, + radius_x: f32, + radius_y: f32, + }, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct Point2D { + pub x: f32, + pub y: f32, +} + +/// CSS box-shadow equivalent. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BoxShadow { + pub offset_x: f32, + pub offset_y: f32, + pub blur_radius: f32, + pub spread_radius: f32, + pub color: Color, +} + +/// CSS background-image URL layer. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BackgroundImage { + pub src: String, + #[serde(default)] + pub fit: BackgroundImageFit, + #[serde(default)] + pub position: ObjectPosition, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BackgroundImageFit { + Fill, + Contain, + #[default] + Cover, + None, +} + +/// CSS gradient background. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum Gradient { + Linear { + angle_deg: f32, + stops: Vec, + }, + Radial { + stops: Vec, + }, +} + +/// A single color stop within a gradient. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GradientStop { + /// Position along the gradient, 0.0 to 1.0. + pub position: f32, + pub color: Color, +} + +/// CSS filter color-adjust functions. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct FilterAdjust { + #[serde(default = "one")] + pub brightness: f32, + #[serde(default = "one")] + pub contrast: f32, + #[serde(default = "one")] + pub saturate: f32, +} + +/// CSS text stroke equivalent. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TextStroke { + pub width: f32, + pub color: Color, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ObjectFit { + Fill, + Contain, + Cover, + None, + ScaleDown, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct ObjectPosition { + /// Horizontal position normalized from 0.0 (left) to 1.0 (right). + pub x: f32, + /// Vertical position normalized from 0.0 (top) to 1.0 (bottom). + pub y: f32, +} + +impl Default for ObjectPosition { + fn default() -> Self { + Self { x: 0.5, y: 0.5 } + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum MixBlendMode { + Normal, + Multiply, + Screen, + Overlay, + Darken, + Lighten, + ColorDodge, + ColorBurn, + HardLight, + SoftLight, + Difference, + Exclusion, + Hue, + Saturation, + Color, + Luminosity, +} + +/// RGBA color with 8-bit channels. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub struct Color { + pub r: u8, + pub g: u8, + pub b: u8, + pub a: u8, +} + +/// 2D affine transform (translate, uniform/non-uniform scale, rotation). +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct Transform2D { + #[serde(default)] + pub translate_x: f32, + #[serde(default)] + pub translate_y: f32, + #[serde(default = "one")] + pub scale_x: f32, + #[serde(default = "one")] + pub scale_y: f32, + #[serde(default)] + pub rotate_deg: f32, +} + +fn one() -> f32 { + 1.0 +} + +// ── Baked Timeline Types ──────────────────────────────────────────────────── + +/// A pre-baked timeline: every frame carries the fully-resolved state of +/// every animated element, so the renderer does zero interpolation at paint time. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BakedTimeline { + pub fps: u32, + pub duration: f64, + pub total_frames: u32, + pub frames: Vec, +} + +/// Per-frame snapshot of animated element states, keyed by element id. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BakedFrame { + pub frame_index: u32, + pub time: f64, + pub elements: std::collections::HashMap, +} + +/// Resolved visual state for a single element at a single frame. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BakedElementState { + pub opacity: f32, + pub translate_x: f32, + pub translate_y: f32, + pub scale_x: f32, + pub scale_y: f32, + pub rotate_deg: f32, + pub visibility: bool, +} diff --git a/packages/native-renderer/src/scene/parse.rs b/packages/native-renderer/src/scene/parse.rs new file mode 100644 index 000000000..cc5c06a4e --- /dev/null +++ b/packages/native-renderer/src/scene/parse.rs @@ -0,0 +1,15 @@ +use std::path::Path; + +use super::Scene; + +/// Parse a scene from a JSON file on disk. +pub fn parse_scene_file(path: &Path) -> Result { + let contents = std::fs::read_to_string(path) + .map_err(|e| format!("failed to read {}: {e}", path.display()))?; + parse_scene_json(&contents) +} + +/// Parse a scene from a JSON string. +pub fn parse_scene_json(json: &str) -> Result { + serde_json::from_str(json).map_err(|e| format!("invalid scene JSON: {e}")) +} diff --git a/packages/native-renderer/src/scene/support.test.ts b/packages/native-renderer/src/scene/support.test.ts new file mode 100644 index 000000000..5cd0f29d9 --- /dev/null +++ b/packages/native-renderer/src/scene/support.test.ts @@ -0,0 +1,163 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createRequire } from "node:module"; +import type { Browser, Page } from "puppeteer-core"; +import { ensureBrowser } from "../../../cli/src/browser/manager.js"; +import { detectNativeSupport } from "./support.js"; + +interface PuppeteerLike { + launch(options: { executablePath: string; headless: boolean; args: string[] }): Promise; +} + +const cliRequire = createRequire(new URL("../../../cli/package.json", import.meta.url)); +const puppeteer = cliRequire("puppeteer-core") as PuppeteerLike; + +async function setComposition(page: Page, innerHtml: string, rootStyle = ""): Promise { + await page.setContent(` + + +
+ ${innerHtml} +
+ + `); +} + +describe("detectNativeSupport", () => { + let browser: Browser; + let page: Page; + + beforeAll(async () => { + const browserInfo = await ensureBrowser(); + browser = await puppeteer.launch({ + executablePath: browserInfo.executablePath, + headless: true, + args: ["--allow-file-access-from-files", "--disable-web-security"], + }); + page = await browser.newPage(); + }); + + afterAll(async () => { + await page?.close().catch(() => undefined); + await browser?.close().catch(() => undefined); + }); + + it.each([ + ["svg", '', "svg"], + ["canvas", '', "canvas"], + ["iframe", '', "iframe"], + ["unresolved video", '', "video"], + [ + "resolved video", + '', + "video", + ], + [ + "animated element without stable id", + 'Animated', + "element-id", + ], + [ + "grid direct text layout", + '
Grid text
', + "text-layout", + ], + [ + "wrapped direct text", + '
This text wraps
', + "text-wrap", + ], + [ + "backdrop filter", + '
', + "backdrop-filter", + ], + [ + "mask image", + '
', + "mask-image", + ], + [ + "unsupported filter", + '
', + "filter", + ], + [ + "unsupported clip path", + '
', + "clip-path", + ], + ["multiple background layers", "", "background-image"], + ["repeated background image", "", "background-image"], + [ + "multiple shadows", + '
', + "box-shadow", + ], + [ + "vertical writing mode", + '
Text
', + "writing-mode", + ], + ])("rejects %s before native rendering starts", async (_name, innerHtml, property) => { + const rootStyle = _name.includes("multiple background") + ? `background-image:url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/lK3KsAAAAABJRU5ErkJggg=="),linear-gradient(red,blue);background-repeat:no-repeat` + : _name.includes("repeated background") + ? `background-image:url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/lK3KsAAAAABJRU5ErkJggg==");background-repeat:repeat` + : ""; + await setComposition(page, innerHtml, rootStyle); + + const report = await detectNativeSupport(page, 320, 180); + + expect(report.supported).toBe(false); + expect(report.reasons.some((reason) => reason.property === property)).toBe(true); + }); + + it("allows the supported subset used by the native fast path", async () => { + await setComposition( + page, + '
', + ); + + const report = await detectNativeSupport(page, 320, 180); + + expect(report).toEqual({ supported: true, reasons: [] }); + }); + + it("rejects missing composition root before native rendering starts", async () => { + await page.setContent(` + + +
Missing data root
+ + `); + + const report = await detectNativeSupport(page, 320, 180); + + expect(report.supported).toBe(false); + expect(report.reasons.some((reason) => reason.property === "data-composition-id")).toBe(true); + }); + + it("rejects transparent composition roots before native rendering starts", async () => { + await setComposition( + page, + '
', + "background:transparent;", + ); + + const report = await detectNativeSupport(page, 320, 180); + + expect(report.supported).toBe(false); + expect(report.reasons.some((reason) => reason.property === "background-color")).toBe(true); + }); + + it("allows animated elements when they have a stable id", async () => { + await setComposition( + page, + 'Animated', + ); + + const report = await detectNativeSupport(page, 320, 180); + + expect(report).toEqual({ supported: true, reasons: [] }); + }); +}); diff --git a/packages/native-renderer/src/scene/support.ts b/packages/native-renderer/src/scene/support.ts new file mode 100644 index 000000000..9a102963c --- /dev/null +++ b/packages/native-renderer/src/scene/support.ts @@ -0,0 +1,227 @@ +/** + * Native renderer support detection. + * + * The native path must be conservative: if Chrome exposes a feature the Rust + * compositor cannot paint faithfully yet, callers should fall back to the CDP + * renderer instead of producing wrong frames. + */ +import type { Page } from "puppeteer-core"; + +export interface NativeUnsupportedReason { + elementId: string; + property: string; + value: string; + reason: string; +} + +export interface NativeSupportReport { + supported: boolean; + reasons: NativeUnsupportedReason[]; +} + +const DETECT_NATIVE_SUPPORT_SCRIPT = `(() => { + const supportedFilters = new Set(["blur", "brightness", "contrast", "saturate"]); + const supportedBlendModes = new Set([ + "normal", + "multiply", + "screen", + "overlay", + "darken", + "lighten", + "color-dodge", + "color-burn", + "hard-light", + "soft-light", + "difference", + "exclusion", + "hue", + "saturation", + "color", + "luminosity", + ]); + + function splitTopLevel(input) { + const parts = []; + let depth = 0; + let start = 0; + for (let i = 0; i < input.length; i++) { + const ch = input[i]; + if (ch === "(") depth++; + if (ch === ")") depth--; + if (ch === "," && depth === 0) { + parts.push(input.slice(start, i).trim()); + start = i + 1; + } + } + parts.push(input.slice(start).trim()); + return parts.filter(Boolean); + } + + function elementId(el) { + const tag = el.tagName.toLowerCase(); + const rect = el.getBoundingClientRect(); + return ( + el.getAttribute("data-name") || + el.id || + tag + "-" + Math.round(rect.x) + "-" + Math.round(rect.y) + ); + } + + function isTransparentColor(value) { + if (!value || value === "transparent") return true; + const rgba = value.match(/^rgba?\\(([^)]+)\\)$/); + if (!rgba) return false; + const parts = rgba[1].split(",").map((part) => part.trim()); + if (parts.length < 4) return false; + return Number(parts[3]) < 1; + } + + function hasDirectText(el) { + return Array.from(el.childNodes).some((node) => { + return node.nodeType === Node.TEXT_NODE && !!node.textContent.trim(); + }); + } + + function directTextLineCount(el) { + let count = 0; + for (const node of Array.from(el.childNodes)) { + if (node.nodeType !== Node.TEXT_NODE || !node.textContent.trim()) continue; + const range = document.createRange(); + range.selectNodeContents(node); + count += range.getClientRects().length; + range.detach(); + } + return count; + } + + const reasons = []; + const compositionRoot = document.querySelector("[data-composition-id]"); + function add(el, property, value, reason) { + reasons.push({ + elementId: elementId(el), + property, + value: String(value || ""), + reason, + }); + } + + if (!compositionRoot) { + reasons.push({ + elementId: "document", + property: "data-composition-id", + value: "missing", + reason: "native extraction requires a stable data-composition-id root", + }); + } + // Transparent roots are OK — the Rust renderer clears to black by default. + // Sub-compositions provide their own backgrounds. + + function inspect(el) { + const tag = el.tagName.toLowerCase(); + const cs = getComputedStyle(el); + + if (tag === "video") { + const src = el.currentSrc || el.src; + if (!src) { + add(el, "video", "", "video element has no resolved source"); + } + // Video with resolved source is supported: frames are extracted via + // FFmpeg and composited as images by the Rust painter. + } + if (tag === "canvas" || tag === "svg" || tag === "iframe") { + add(el, tag, tag, "embedded dynamic/vector surfaces require Chrome fallback"); + } + + // Relaxed: the scene extractor assigns positional IDs (tag-x-y) which + // are deterministic for a given layout. Timeline baking queries [id] + // after extraction, so these generated IDs work. + + // Relaxed: Skia's paragraph API (textlayout feature) handles multi-line + // text and basic layout. Chrome extraction provides the computed bounds. + + if (cs.backgroundImage && cs.backgroundImage !== "none") { + const layers = splitTopLevel(cs.backgroundImage); + if (layers.length > 1) { + add(el, "background-image", cs.backgroundImage, "multiple background layers are not supported"); + } else if ( + !/^(linear-gradient|radial-gradient|url)\\(/.test(layers[0]) || + (layers[0].startsWith("url(") && cs.backgroundRepeat !== "no-repeat") + ) { + add(el, "background-image", cs.backgroundImage, "only gradients and non-repeating URL backgrounds are supported"); + } + } + + if (cs.boxShadow && cs.boxShadow !== "none") { + // Multiple shadows: the Rust painter loops over each shadow layer. + if (/\\binset\\b/.test(cs.boxShadow)) add(el, "box-shadow", cs.boxShadow, "inset shadows are not supported"); + } + + if (cs.textShadow && cs.textShadow !== "none") { + const shadows = splitTopLevel(cs.textShadow); + if (shadows.length > 1) add(el, "text-shadow", cs.textShadow, "multiple text shadows are not supported"); + } + + if (cs.filter && cs.filter !== "none") { + for (const match of cs.filter.matchAll(/([a-z-]+)\\(/g)) { + if (!supportedFilters.has(match[1])) { + add(el, "filter", cs.filter, "only blur, brightness, contrast, and saturate filters are supported"); + break; + } + } + } + + const backdropFilter = cs.backdropFilter || cs.webkitBackdropFilter; + if (backdropFilter && backdropFilter !== "none") { + add(el, "backdrop-filter", backdropFilter, "backdrop filters require render-to-texture fallback work"); + } + + const maskImage = cs.maskImage || cs.webkitMaskImage; + if (maskImage && maskImage !== "none") { + add(el, "mask-image", maskImage, "CSS masks are not supported by the native painter yet"); + } + + if (cs.clipPath && cs.clipPath !== "none" && !/^(polygon|circle|ellipse)\\(/.test(cs.clipPath)) { + add(el, "clip-path", cs.clipPath, "only polygon, circle, and ellipse clip paths are supported"); + } + + if (!supportedBlendModes.has(cs.mixBlendMode)) { + add(el, "mix-blend-mode", cs.mixBlendMode, "blend mode is not mapped to Skia"); + } + + for (const side of ["Top", "Right", "Bottom", "Left"]) { + const style = cs["border" + side + "Style"]; + const width = parseFloat(cs["border" + side + "Width"]) || 0; + if (width > 0 && style !== "solid" && style !== "dashed") { + add(el, "border-style", style, "only solid and dashed borders are supported"); + break; + } + } + + if (cs.writingMode && cs.writingMode !== "horizontal-tb") { + add(el, "writing-mode", cs.writingMode, "vertical writing mode is not implemented"); + } + + for (const child of Array.from(el.children)) inspect(child); + } + + inspect(compositionRoot ?? document.body); + return reasons; +})()`; + +export async function detectNativeSupport( + page: Page, + width: number, + height: number, +): Promise { + await page.setViewport({ width, height }); + const reasons = (await page.evaluate(DETECT_NATIVE_SUPPORT_SCRIPT)) as NativeUnsupportedReason[]; + const uniqueReasons = Array.from( + new Map( + reasons.map((reason) => [ + `${reason.elementId}\u0000${reason.property}\u0000${reason.value}\u0000${reason.reason}`, + reason, + ]), + ).values(), + ); + return { supported: uniqueReasons.length === 0, reasons: uniqueReasons }; +} diff --git a/packages/native-renderer/src/timeline/bake.test.ts b/packages/native-renderer/src/timeline/bake.test.ts new file mode 100644 index 000000000..332a559ac --- /dev/null +++ b/packages/native-renderer/src/timeline/bake.test.ts @@ -0,0 +1,161 @@ +import { describe, it, expect } from "vitest"; +import type { BakedTimeline, BakedFrame, BakedElementState } from "./types"; + +describe("BakedTimeline", () => { + it("serializes to JSON compatible with Rust serde types", () => { + const timeline: BakedTimeline = { + fps: 30, + duration: 2.0, + total_frames: 60, + frames: [ + { + frame_index: 0, + time: 0.0, + elements: { + title: { + opacity: 0, + translate_x: 0, + translate_y: 50, + scale_x: 1, + scale_y: 1, + rotate_deg: 0, + visibility: true, + }, + card: { + opacity: 1, + translate_x: 0, + translate_y: 0, + scale_x: 1, + scale_y: 1, + rotate_deg: 0, + visibility: true, + }, + }, + }, + { + frame_index: 30, + time: 1.0, + elements: { + title: { + opacity: 1, + translate_x: 0, + translate_y: 0, + scale_x: 1, + scale_y: 1, + rotate_deg: 0, + visibility: true, + }, + card: { + opacity: 1, + translate_x: 0, + translate_y: 0, + scale_x: 1.2, + scale_y: 1.2, + rotate_deg: 0, + visibility: true, + }, + }, + }, + ], + }; + + const json = JSON.stringify(timeline); + const parsed = JSON.parse(json) as BakedTimeline; + + expect(parsed.total_frames).toBe(60); + expect(parsed.frames[0].elements["title"].opacity).toBe(0); + expect(parsed.frames[1].elements["card"].scale_x).toBe(1.2); + }); + + it("uses snake_case field names matching Rust Transform2D", () => { + const state: BakedElementState = { + opacity: 0.5, + translate_x: 100, + translate_y: -30, + scale_x: 1.5, + scale_y: 0.8, + rotate_deg: 45, + visibility: true, + }; + + const json = JSON.stringify(state); + const parsed = JSON.parse(json); + + // Verify snake_case keys are present (Rust serde expects these) + expect(parsed).toHaveProperty("translate_x"); + expect(parsed).toHaveProperty("translate_y"); + expect(parsed).toHaveProperty("scale_x"); + expect(parsed).toHaveProperty("scale_y"); + expect(parsed).toHaveProperty("rotate_deg"); + expect(parsed).not.toHaveProperty("translateX"); + expect(parsed).not.toHaveProperty("scaleX"); + expect(parsed).not.toHaveProperty("rotateDeg"); + }); + + it("preserves frame ordering and time precision", () => { + const frames: BakedFrame[] = Array.from({ length: 5 }, (_, i) => ({ + frame_index: i, + time: i / 30, + elements: { + box: { + opacity: i / 4, + translate_x: i * 10, + translate_y: 0, + scale_x: 1, + scale_y: 1, + rotate_deg: 0, + visibility: true, + }, + }, + })); + + const timeline: BakedTimeline = { + fps: 30, + duration: 5 / 30, + total_frames: 5, + frames, + }; + + const parsed = JSON.parse(JSON.stringify(timeline)) as BakedTimeline; + + expect(parsed.frames).toHaveLength(5); + expect(parsed.frames[0].frame_index).toBe(0); + expect(parsed.frames[4].frame_index).toBe(4); + expect(parsed.frames[2].time).toBeCloseTo(2 / 30, 10); + expect(parsed.frames[3].elements["box"].translate_x).toBe(30); + }); + + it("handles hidden elements with zero opacity", () => { + const frame: BakedFrame = { + frame_index: 0, + time: 0, + elements: { + hidden_el: { + opacity: 0, + translate_x: 0, + translate_y: 0, + scale_x: 1, + scale_y: 1, + rotate_deg: 0, + visibility: false, + }, + }, + }; + + const parsed = JSON.parse(JSON.stringify(frame)) as BakedFrame; + expect(parsed.elements["hidden_el"].opacity).toBe(0); + expect(parsed.elements["hidden_el"].visibility).toBe(false); + }); + + it("handles empty element map for frames with no ID'd elements", () => { + const timeline: BakedTimeline = { + fps: 24, + duration: 1.0, + total_frames: 24, + frames: [{ frame_index: 0, time: 0, elements: {} }], + }; + + const parsed = JSON.parse(JSON.stringify(timeline)) as BakedTimeline; + expect(Object.keys(parsed.frames[0].elements)).toHaveLength(0); + }); +}); diff --git a/packages/native-renderer/src/timeline/bake.ts b/packages/native-renderer/src/timeline/bake.ts new file mode 100644 index 000000000..f3d8f9c31 --- /dev/null +++ b/packages/native-renderer/src/timeline/bake.ts @@ -0,0 +1,101 @@ +/** + * Pre-baked timeline extraction — evaluates a GSAP timeline at every frame + * timestamp via Chrome CDP and extracts per-frame property values for all + * animated elements. + * + * The output JSON is consumed by the Rust native renderer, which applies + * transform/opacity/visibility per-frame during paint — no V8 needed at + * render time. + */ +import type { Page } from "puppeteer-core"; +import type { BakedTimeline, BakedFrame, BakedElementState } from "./types"; + +// String-based evaluate avoids tsx/esbuild injecting `__name` helpers into the +// function body that Puppeteer serializes into the browser context. +const BAKE_FRAME_SCRIPT = `(() => { + function decomposeMatrix(raw) { + if (raw === "none") { + return { translate_x: 0, translate_y: 0, scale_x: 1, scale_y: 1, rotate_deg: 0 }; + } + const mat = raw.match( + /matrix\\(\\s*([-\\d.e]+),\\s*([-\\d.e]+),\\s*([-\\d.e]+),\\s*([-\\d.e]+),\\s*([-\\d.e]+),\\s*([-\\d.e]+)\\)/, + ); + if (!mat) { + return { translate_x: 0, translate_y: 0, scale_x: 1, scale_y: 1, rotate_deg: 0 }; + } + const a = +mat[1]; + const b = +mat[2]; + const c = +mat[3]; + const d = +mat[4]; + return { + translate_x: +mat[5], + translate_y: +mat[6], + scale_x: Math.sqrt(a * a + b * b), + scale_y: Math.sqrt(c * c + d * d), + rotate_deg: (Math.atan2(b, a) * 180) / Math.PI, + }; + } + + const result = {}; + const els = document.querySelectorAll("[id]"); + for (const el of els) { + if (!(el instanceof HTMLElement)) continue; + const cs = getComputedStyle(el); + const transform = decomposeMatrix(cs.transform); + + result[el.id] = { + opacity: parseFloat(cs.opacity) || 0, + translate_x: transform.translate_x, + translate_y: transform.translate_y, + scale_x: transform.scale_x, + scale_y: transform.scale_y, + rotate_deg: transform.rotate_deg, + visibility: cs.visibility !== "hidden" && cs.display !== "none", + }; + } + return result; +})()`; + +/** + * Bake a composition's GSAP timeline into per-frame property snapshots. + * + * For each frame (0..totalFrames), this: + * 1. Seeks the composition to the frame's timestamp via `window.__hf.seek()` + * 2. Reads computed styles from every `[id]` element in the page + * 3. Decomposes the CSS transform matrix into translate/scale/rotate + * + * The caller must have already loaded and initialised the composition in the + * page (i.e., the GSAP timeline and `window.__hf` must exist). + */ +export async function bakeTimeline( + page: Page, + fps: number, + duration: number, +): Promise { + const totalFrames = Math.ceil(fps * duration); + const frames: BakedFrame[] = []; + + for (let i = 0; i < totalFrames; i++) { + const time = i / fps; + + // Seek the composition to this timestamp. The guard mirrors the pattern + // used in packages/producer/src/services/renderOrchestrator.ts. + await page.evaluate( + `(() => { + const hf = window.__hf; + if (hf && typeof hf.seek === "function") { + hf.seek(${JSON.stringify(time)}); + } + })()`, + ); + + // Extract animated properties for all elements with IDs. + // Everything inside page.evaluate runs in the browser context — helpers + // must be inlined (no access to outer scope). + const elements = (await page.evaluate(BAKE_FRAME_SCRIPT)) as Record; + + frames.push({ frame_index: i, time, elements }); + } + + return { fps, duration, total_frames: totalFrames, frames }; +} diff --git a/packages/native-renderer/src/timeline/types.ts b/packages/native-renderer/src/timeline/types.ts new file mode 100644 index 000000000..a92a8fc17 --- /dev/null +++ b/packages/native-renderer/src/timeline/types.ts @@ -0,0 +1,30 @@ +/** + * Baked timeline types — a pre-evaluated animation timeline where every frame's + * element properties have been resolved from GSAP via Chrome CDP. + * + * Field names use snake_case to match the Rust serde types in + * `packages/native-renderer/src/scene/mod.rs` (Transform2D, Style, etc.). + */ + +export interface BakedTimeline { + fps: number; + duration: number; + total_frames: number; + frames: BakedFrame[]; +} + +export interface BakedFrame { + frame_index: number; + time: number; + elements: Record; +} + +export interface BakedElementState { + opacity: number; + translate_x: number; + translate_y: number; + scale_x: number; + scale_y: number; + rotate_deg: number; + visibility: boolean; +} diff --git a/packages/native-renderer/tests/animated_test.rs b/packages/native-renderer/tests/animated_test.rs new file mode 100644 index 000000000..0f116e108 --- /dev/null +++ b/packages/native-renderer/tests/animated_test.rs @@ -0,0 +1,169 @@ +use std::collections::HashMap; +use std::path::Path; + +use hyperframes_native_renderer::pipeline::render_animated_gpu; +use hyperframes_native_renderer::pipeline::{render_animated, RenderConfig}; +use hyperframes_native_renderer::scene::{ + BakedElementState, BakedFrame, BakedTimeline, Color, Element, ElementKind, Rect, Scene, Style, +}; + +/// Build a minimal scene: full-screen background + a title text element. +fn make_animated_scene() -> Scene { + let title = Element { + id: "title".into(), + kind: ElementKind::Text { + content: "Animated Title".into(), + }, + bounds: Rect { + x: 100.0, + y: 120.0, + width: 440.0, + height: 60.0, + }, + style: Style { + color: Some(Color { + r: 255, + g: 255, + b: 255, + a: 255, + }), + font_size: Some(36.0), + ..Style::default() + }, + children: vec![], + }; + + let background = Element { + id: "bg".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 640.0, + height: 360.0, + }, + style: Style { + background_color: Some(Color { + r: 10, + g: 10, + b: 30, + a: 255, + }), + ..Style::default() + }, + children: vec![title], + }; + + Scene { + width: 640, + height: 360, + elements: vec![background], + } +} + +/// Build a 30-frame (1s @ 30fps) timeline where the title fades in and +/// slides up from y+50 to y+0. +fn make_fade_in_timeline() -> BakedTimeline { + let frames = (0..30) + .map(|i| { + let progress = i as f32 / 29.0; + BakedFrame { + frame_index: i, + time: i as f64 / 30.0, + elements: HashMap::from([( + "title".to_string(), + BakedElementState { + opacity: progress, + translate_x: 0.0, + translate_y: 50.0 * (1.0 - progress), + scale_x: 1.0, + scale_y: 1.0, + rotate_deg: 0.0, + visibility: true, + }, + )]), + } + }) + .collect(); + + BakedTimeline { + fps: 30, + duration: 1.0, + total_frames: 30, + frames, + } +} + +#[test] +fn render_animated_scene_to_mp4() { + let scene = make_animated_scene(); + let timeline = make_fade_in_timeline(); + let output_path = "/tmp/hyperframes-animated-test.mp4"; + + let config = RenderConfig { + fps: 30, + duration_secs: 1.0, + quality: 80, + output_path: output_path.to_string(), + }; + + let result = render_animated(&scene, &timeline, &config).unwrap(); + + assert_eq!(result.total_frames, 30); + assert!(result.avg_paint_ms > 0.0); + assert_eq!(result.output_path, output_path); + + let path = Path::new(output_path); + assert!(path.exists(), "output MP4 must exist"); + + let size = std::fs::metadata(output_path).unwrap().len(); + assert!(size > 1000, "MP4 should be non-trivial, got {size} bytes"); + + std::fs::remove_file(output_path).ok(); +} + +#[test] +fn render_animated_gpu_scene_to_mp4() { + let scene = make_animated_scene(); + let timeline = make_fade_in_timeline(); + let output_path = "/tmp/hyperframes-animated-gpu-test.mp4"; + + let config = RenderConfig { + fps: 30, + duration_secs: 1.0, + quality: 80, + output_path: output_path.to_string(), + }; + + let result = render_animated_gpu(&scene, &timeline, &config).unwrap(); + + assert_eq!(result.total_frames, 30); + assert!(result.avg_paint_ms > 0.0); + assert_eq!(result.output_path, output_path); + + let size = std::fs::metadata(output_path).unwrap().len(); + assert!(size > 1000, "MP4 should be non-trivial, got {size} bytes"); + + std::fs::remove_file(output_path).ok(); +} + +#[test] +fn render_animated_zero_frames_errors() { + let scene = make_animated_scene(); + let timeline = BakedTimeline { + fps: 30, + duration: 0.0, + total_frames: 0, + frames: vec![], + }; + + let config = RenderConfig { + fps: 30, + duration_secs: 0.0, + quality: 80, + output_path: "/tmp/hyperframes-animated-zero.mp4".to_string(), + }; + + let result = render_animated(&scene, &timeline, &config); + assert!(result.is_err()); +} diff --git a/packages/native-renderer/tests/effects_test.rs b/packages/native-renderer/tests/effects_test.rs new file mode 100644 index 000000000..e5560d183 --- /dev/null +++ b/packages/native-renderer/tests/effects_test.rs @@ -0,0 +1,337 @@ +use hyperframes_native_renderer::paint::effects; +use hyperframes_native_renderer::paint::elements::paint_element; +use hyperframes_native_renderer::paint::images::ImageCache; +use hyperframes_native_renderer::paint::RenderSurface; +use hyperframes_native_renderer::scene::{ + BoxShadow, Color, Element, ElementKind, FilterAdjust, Gradient, GradientStop, Rect, Style, +}; +use skia_safe::Color4f; + +// --------------------------------------------------------------------------- +// Box shadow +// --------------------------------------------------------------------------- + +#[test] +fn paint_box_shadow_produces_pixels() { + let mut surface = RenderSurface::new_raster(200, 200).expect("surface"); + surface.clear(Color4f::new(1.0, 1.0, 1.0, 1.0)); + + let el = Element { + id: "card".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 40.0, + y: 40.0, + width: 120.0, + height: 120.0, + }, + style: Style { + background_color: Some(Color { + r: 0, + g: 0, + b: 255, + a: 255, + }), + box_shadow: Some(BoxShadow { + offset_x: 4.0, + offset_y: 4.0, + blur_radius: 10.0, + spread_radius: 2.0, + color: Color { + r: 0, + g: 0, + b: 0, + a: 180, + }, + }), + ..Style::default() + }, + children: vec![], + }; + + let mut images = ImageCache::new(); + paint_element(surface.canvas(), &el, &mut images); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + + // Check a pixel that is outside the element bounds but within the shadow + // spread+blur area. At (165, 165) the element ends at 160,160 but the + // shadow extends further via offset + spread + blur. + let idx = (165 * 200 + 165) * 4; + let is_not_white = pixels[idx] < 250 || pixels[idx + 1] < 250 || pixels[idx + 2] < 250; + assert!( + is_not_white, + "pixel at (165,165) should be affected by shadow, got RGB({},{},{})", + pixels[idx], + pixels[idx + 1], + pixels[idx + 2] + ); +} + +// --------------------------------------------------------------------------- +// Blur filter +// --------------------------------------------------------------------------- + +#[test] +fn paint_blur_filter() { + let mut surface = RenderSurface::new_raster(200, 200).expect("surface"); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + + let el = Element { + id: "blurred".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 50.0, + y: 50.0, + width: 100.0, + height: 100.0, + }, + style: Style { + background_color: Some(Color { + r: 255, + g: 0, + b: 0, + a: 255, + }), + filter_blur: Some(8.0), + ..Style::default() + }, + children: vec![], + }; + + let mut images = ImageCache::new(); + paint_element(surface.canvas(), &el, &mut images); + + // The blur should cause red color to bleed outside the element bounds. + // Check a pixel just outside the element at (45, 100) — the element + // starts at x=50, so (45, 100) is 5px to the left. + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + let idx = (100 * 200 + 45) * 4; + assert!( + pixels[idx] > 10, + "pixel at (45,100) should have red bleed from blur, got R={}", + pixels[idx] + ); + + // Verify the JPEG encodes without errors. + let jpeg = surface.encode_jpeg(80).expect("should encode JPEG"); + assert!(jpeg.len() > 200); +} + +#[test] +fn paint_filter_adjust_brightness() { + let mut surface = RenderSurface::new_raster(100, 100).expect("surface"); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + + let el = Element { + id: "bright".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 10.0, + y: 10.0, + width: 80.0, + height: 80.0, + }, + style: Style { + background_color: Some(Color { + r: 80, + g: 80, + b: 80, + a: 255, + }), + filter_adjust: Some(FilterAdjust { + brightness: 2.0, + contrast: 1.0, + saturate: 1.0, + }), + ..Style::default() + }, + children: vec![], + }; + + let mut images = ImageCache::new(); + paint_element(surface.canvas(), &el, &mut images); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + let center = (50 * 100 + 50) * 4; + assert!( + pixels[center] > 120, + "brightness filter should increase center R, got {}", + pixels[center] + ); +} + +// --------------------------------------------------------------------------- +// Linear gradient +// --------------------------------------------------------------------------- + +#[test] +fn paint_linear_gradient() { + let mut surface = RenderSurface::new_raster(200, 100).expect("surface"); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + + let el = Element { + id: "gradient".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 200.0, + height: 100.0, + }, + style: Style { + background_gradient: Some(Gradient::Linear { + angle_deg: 90.0, + stops: vec![ + GradientStop { + position: 0.0, + color: Color { + r: 255, + g: 0, + b: 0, + a: 255, + }, + }, + GradientStop { + position: 1.0, + color: Color { + r: 0, + g: 0, + b: 255, + a: 255, + }, + }, + ], + }), + ..Style::default() + }, + children: vec![], + }; + + let mut images = ImageCache::new(); + paint_element(surface.canvas(), &el, &mut images); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + + // Left edge (x=5, y=50): should be reddish. + let left = (50 * 200 + 5) * 4; + assert!( + pixels[left] > pixels[left + 2], + "left edge R ({}) should dominate B ({})", + pixels[left], + pixels[left + 2] + ); + + // Right edge (x=195, y=50): should be bluish. + let right = (50 * 200 + 195) * 4; + assert!( + pixels[right + 2] > pixels[right], + "right edge B ({}) should dominate R ({})", + pixels[right + 2], + pixels[right] + ); +} + +// --------------------------------------------------------------------------- +// Radial gradient +// --------------------------------------------------------------------------- + +#[test] +fn paint_radial_gradient() { + let mut surface = RenderSurface::new_raster(200, 200).expect("surface"); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + + let el = Element { + id: "radial".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 200.0, + height: 200.0, + }, + style: Style { + background_gradient: Some(Gradient::Radial { + stops: vec![ + GradientStop { + position: 0.0, + color: Color { + r: 255, + g: 255, + b: 0, + a: 255, + }, + }, + GradientStop { + position: 1.0, + color: Color { + r: 0, + g: 0, + b: 128, + a: 255, + }, + }, + ], + }), + ..Style::default() + }, + children: vec![], + }; + + let mut images = ImageCache::new(); + paint_element(surface.canvas(), &el, &mut images); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + + // Center pixel (100, 100): should be yellow-ish (high R, high G). + let center = (100 * 200 + 100) * 4; + assert!( + pixels[center] > 200 && pixels[center + 1] > 200, + "center should be yellow-ish, got RGB({},{},{})", + pixels[center], + pixels[center + 1], + pixels[center + 2] + ); + + // Edge pixel (0, 0): should be dark blue-ish (low R, low G, some B). + let edge = 0; + assert!( + pixels[edge + 2] > pixels[edge], + "edge B ({}) should dominate R ({})", + pixels[edge + 2], + pixels[edge] + ); +} + +// --------------------------------------------------------------------------- +// Unit tests for effects module functions +// --------------------------------------------------------------------------- + +#[test] +fn create_blur_image_filter_zero_returns_none() { + assert!(effects::create_blur_image_filter(0.0).is_none()); + assert!(effects::create_blur_image_filter(-1.0).is_none()); +} + +#[test] +fn create_blur_image_filter_positive_returns_some() { + assert!(effects::create_blur_image_filter(4.0).is_some()); +} + +#[test] +fn create_gradient_shader_too_few_stops_returns_none() { + let rect = skia_safe::Rect::from_xywh(0.0, 0.0, 100.0, 100.0); + let gradient = Gradient::Linear { + angle_deg: 0.0, + stops: vec![GradientStop { + position: 0.0, + color: Color { + r: 255, + g: 0, + b: 0, + a: 255, + }, + }], + }; + assert!(effects::create_gradient_shader(&rect, &gradient).is_none()); +} diff --git a/packages/native-renderer/tests/encode_test.rs b/packages/native-renderer/tests/encode_test.rs new file mode 100644 index 000000000..e06f8ad2f --- /dev/null +++ b/packages/native-renderer/tests/encode_test.rs @@ -0,0 +1,153 @@ +use hyperframes_native_renderer::encode::{ + detect_hw_encoder, encoder_args, raw_pixel_encoder_args, HwEncoder, +}; + +fn arg_after(args: &[String], flag: &str) -> String { + let index = args + .iter() + .position(|arg| arg == flag) + .unwrap_or_else(|| panic!("missing flag {flag} in {args:?}")); + args.get(index + 1) + .unwrap_or_else(|| panic!("missing value after {flag} in {args:?}")) + .clone() +} + +#[test] +fn detect_hw_encoder_returns_valid() { + let encoder = detect_hw_encoder(); + // Must be one of the known variants — mainly checking it doesn't panic. + assert!(matches!( + encoder, + HwEncoder::VideoToolbox | HwEncoder::Nvenc | HwEncoder::Vaapi | HwEncoder::Software + )); +} + +#[test] +fn encoder_args_software_contains_libx264() { + let args = encoder_args(HwEncoder::Software, 30, 18); + + assert!(args.contains(&"-c:v".to_string())); + assert!(args.contains(&"libx264".to_string())); + assert!(args.contains(&"-crf".to_string())); + assert!(args.contains(&"18".to_string())); + assert!(args.contains(&"-pix_fmt".to_string())); + assert!(args.contains(&"yuv420p".to_string())); + // Input format flags + assert!(args.contains(&"image2pipe".to_string())); + assert!(args.contains(&"mjpeg".to_string())); + assert!(args.contains(&"30".to_string())); // framerate +} + +#[test] +fn encoder_args_software_maps_jpeg_quality_to_valid_crf() { + let args = encoder_args(HwEncoder::Software, 30, 80); + let crf = arg_after(&args, "-crf"); + + assert_ne!(crf, "80", "JPEG quality must not be passed through as CRF"); + assert!( + crf.parse::().unwrap() <= 51, + "libx264 CRF must stay within FFmpeg's valid 0..51 range" + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn encoder_args_videotoolbox_contains_hevc() { + let args = encoder_args(HwEncoder::VideoToolbox, 30, 65); + + assert!(args.contains(&"-c:v".to_string())); + assert!(args.contains(&"hevc_videotoolbox".to_string())); + assert!(args.contains(&"-allow_sw".to_string())); + assert!(args.contains(&"1".to_string())); + assert!(args.contains(&"-tag:v".to_string())); + assert!(args.contains(&"hvc1".to_string())); + assert!(args.contains(&"-q:v".to_string())); + assert!(args.contains(&"65".to_string())); +} + +#[test] +fn encoder_args_nvenc_contains_nvenc() { + let args = encoder_args(HwEncoder::Nvenc, 60, 23); + + assert!(args.contains(&"-c:v".to_string())); + assert!(args.contains(&"h264_nvenc".to_string())); + assert!(args.contains(&"-preset".to_string())); + assert!(args.contains(&"p4".to_string())); + assert!(args.contains(&"-cq".to_string())); + assert!(args.contains(&"23".to_string())); +} + +#[test] +fn encoder_args_vaapi_contains_vaapi() { + let args = encoder_args(HwEncoder::Vaapi, 24, 28); + + assert!(args.contains(&"-c:v".to_string())); + assert!(args.contains(&"h264_vaapi".to_string())); + assert!(args.contains(&"-vaapi_device".to_string())); + assert!(args.contains(&"/dev/dri/renderD128".to_string())); + assert!(args.contains(&"-qp".to_string())); + assert!(args.contains(&"28".to_string())); +} + +#[test] +fn encoder_args_vaapi_uploads_software_frames_to_gpu() { + let args = encoder_args(HwEncoder::Vaapi, 24, 80); + + assert_eq!(arg_after(&args, "-vf"), "format=nv12,hwupload"); +} + +#[cfg(target_os = "macos")] +#[test] +fn detect_returns_videotoolbox_on_macos() { + // On macOS, VideoToolbox is always the detected encoder. + assert_eq!(detect_hw_encoder(), HwEncoder::VideoToolbox); +} + +#[test] +fn encoder_args_all_start_with_overwrite_flag() { + for encoder in [ + HwEncoder::Software, + HwEncoder::Nvenc, + HwEncoder::Vaapi, + HwEncoder::VideoToolbox, + ] { + let args = encoder_args(encoder, 30, 20); + assert_eq!(args[0], "-y", "first arg must be -y for {encoder:?}"); + } +} + +#[test] +fn encoder_args_software_and_nvenc_end_with_pix_fmt() { + for encoder in [HwEncoder::Software, HwEncoder::Nvenc] { + let args = encoder_args(encoder, 30, 20); + assert!( + args.contains(&"-pix_fmt".to_string()), + "must have -pix_fmt for {encoder:?}" + ); + assert!( + args.contains(&"yuv420p".to_string()), + "must have yuv420p for {encoder:?}" + ); + } +} + +#[test] +fn encoder_args_videotoolbox_uses_nv12() { + let args = encoder_args(HwEncoder::VideoToolbox, 30, 20); + assert!(args.contains(&"-pix_fmt".to_string())); + assert!(args.contains(&"nv12".to_string())); +} + +#[test] +fn raw_pixel_encoder_args_use_bgra_rawvideo_input() { + let args = raw_pixel_encoder_args(HwEncoder::Software, 30, 80, 640, 360); + + assert_eq!(arg_after(&args, "-f"), "rawvideo"); + // BGRA is Skia Metal's native pixel format — avoids a GPU-side conversion + assert_eq!(arg_after(&args, "-pix_fmt"), "bgra"); + assert_eq!(arg_after(&args, "-s:v"), "640x360"); + assert_eq!(arg_after(&args, "-framerate"), "30"); + assert!(!args.contains(&"image2pipe".to_string())); + assert!(!args.contains(&"mjpeg".to_string())); + assert!(args.contains(&"libx264".to_string())); +} diff --git a/packages/native-renderer/tests/images_test.rs b/packages/native-renderer/tests/images_test.rs new file mode 100644 index 000000000..f6a52768b --- /dev/null +++ b/packages/native-renderer/tests/images_test.rs @@ -0,0 +1,257 @@ +use hyperframes_native_renderer::paint::{paint_element, ImageCache, RenderSurface}; +use hyperframes_native_renderer::scene::{ + BackgroundImage, BackgroundImageFit, Element, ElementKind, ObjectFit, ObjectPosition, Rect, + Style, +}; +use skia_safe::{surfaces, Color4f, EncodedImageFormat}; +use std::process::Command; + +/// Generate a solid-red PNG at the given path using Skia. +fn create_test_png(path: &str, width: i32, height: i32) { + let mut surface = surfaces::raster_n32_premul((width, height)).expect("surface"); + surface.canvas().clear(Color4f::new(1.0, 0.0, 0.0, 1.0)); + let image = surface.image_snapshot(); + let data = image + .encode(None, EncodedImageFormat::PNG, 100) + .expect("encode PNG"); + std::fs::write(path, data.as_bytes()).expect("write test PNG"); +} + +fn create_test_mp4(path: &str) { + let status = Command::new("ffmpeg") + .args([ + "-y", + "-v", + "error", + "-f", + "lavfi", + "-i", + "color=c=blue:s=64x64:d=0.2:r=5", + "-frames:v", + "1", + "-pix_fmt", + "yuv420p", + path, + ]) + .status() + .expect("run ffmpeg"); + assert!(status.success(), "ffmpeg should create test video"); +} + +#[test] +fn paint_image_element() { + let test_png = "/tmp/hyperframes-test-red.png"; + create_test_png(test_png, 100, 100); + + let mut surface = RenderSurface::new_raster(100, 100).expect("surface"); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + + let el = Element { + id: "img".into(), + kind: ElementKind::Image { + src: test_png.to_string(), + }, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }, + style: Style::default(), + children: vec![], + }; + + let mut images = ImageCache::new(); + paint_element(surface.canvas(), &el, &mut images); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + // The center pixel should be red from the loaded image. + let idx = (50 * 100 + 50) * 4; + assert!( + pixels[idx] > 200, + "center R expected > 200, got {}", + pixels[idx] + ); + assert!( + pixels[idx + 1] < 50, + "center G expected < 50, got {}", + pixels[idx + 1] + ); + assert!( + pixels[idx + 2] < 50, + "center B expected < 50, got {}", + pixels[idx + 2] + ); + + std::fs::remove_file(test_png).ok(); +} + +#[test] +fn paint_image_object_fit_contain_letterboxes() { + let test_png = "/tmp/hyperframes-test-wide-red.png"; + create_test_png(test_png, 100, 50); + + let mut surface = RenderSurface::new_raster(100, 100).expect("surface"); + surface.clear(Color4f::new(1.0, 1.0, 1.0, 1.0)); + + let el = Element { + id: "img".into(), + kind: ElementKind::Image { + src: test_png.to_string(), + }, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }, + style: Style { + object_fit: Some(ObjectFit::Contain), + ..Style::default() + }, + children: vec![], + }; + + let mut images = ImageCache::new(); + paint_element(surface.canvas(), &el, &mut images); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + let center = (50 * 100 + 50) * 4; + assert!( + pixels[center] > 200 && pixels[center + 1] < 50, + "center should be red, got RGB({},{},{})", + pixels[center], + pixels[center + 1], + pixels[center + 2] + ); + + let top_letterbox = (10 * 100 + 50) * 4; + assert_eq!( + pixels[top_letterbox], 255, + "top letterbox should stay white" + ); + assert_eq!( + pixels[top_letterbox + 1], + 255, + "top letterbox should stay white" + ); + assert_eq!( + pixels[top_letterbox + 2], + 255, + "top letterbox should stay white" + ); + + std::fs::remove_file(test_png).ok(); +} + +#[test] +fn paint_background_image_from_file_url() { + let test_png = "/tmp/hyperframes-test-bg-red.png"; + create_test_png(test_png, 100, 100); + + let mut surface = RenderSurface::new_raster(100, 100).expect("surface"); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + + let el = Element { + id: "bg".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }, + style: Style { + background_image: Some(BackgroundImage { + src: format!("file://{test_png}"), + fit: BackgroundImageFit::Cover, + position: ObjectPosition::default(), + }), + ..Style::default() + }, + children: vec![], + }; + + let mut images = ImageCache::new(); + paint_element(surface.canvas(), &el, &mut images); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + let center = (50 * 100 + 50) * 4; + assert!( + pixels[center] > 200 && pixels[center + 1] < 50, + "background image should paint red, got RGB({},{},{})", + pixels[center], + pixels[center + 1], + pixels[center + 2] + ); + + std::fs::remove_file(test_png).ok(); +} + +#[test] +fn paint_video_element_uses_ffmpeg_frame() { + let test_mp4 = "/tmp/hyperframes-native-video-blue.mp4"; + create_test_mp4(test_mp4); + + let mut surface = RenderSurface::new_raster(64, 64).expect("surface"); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + + let el = Element { + id: "video".into(), + kind: ElementKind::Video { + src: test_mp4.to_string(), + }, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 64.0, + height: 64.0, + }, + style: Style { + object_fit: Some(ObjectFit::Fill), + ..Style::default() + }, + children: vec![], + }; + + let mut images = ImageCache::new(); + paint_element(surface.canvas(), &el, &mut images); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + let center = (32 * 64 + 32) * 4; + assert!( + pixels[center + 2] > 120, + "video frame should paint blue, got RGB({},{},{})", + pixels[center], + pixels[center + 1], + pixels[center + 2] + ); + + std::fs::remove_file(test_mp4).ok(); +} + +#[test] +fn image_cache_reuses() { + let test_png = "/tmp/hyperframes-test-cache.png"; + create_test_png(test_png, 100, 100); + + let mut cache = ImageCache::new(); + + assert!(cache.get_or_load(test_png).is_some()); + assert_eq!(cache.len(), 1); + + // Second load should reuse the cached entry. + assert!(cache.get_or_load(test_png).is_some()); + assert_eq!(cache.len(), 1, "cache should still have exactly 1 entry"); + + std::fs::remove_file(test_png).ok(); +} + +#[test] +fn image_cache_missing_file_returns_none() { + let mut cache = ImageCache::new(); + assert!(cache + .get_or_load("/tmp/nonexistent-hyperframes-image.png") + .is_none()); + assert_eq!(cache.len(), 0); +} diff --git a/packages/native-renderer/tests/paint_test.rs b/packages/native-renderer/tests/paint_test.rs new file mode 100644 index 000000000..44f4d6ffd --- /dev/null +++ b/packages/native-renderer/tests/paint_test.rs @@ -0,0 +1,393 @@ +use hyperframes_native_renderer::paint::{paint_element, ImageCache, RenderSurface}; +use hyperframes_native_renderer::scene::{ + Border, BorderLineStyle, ClipPath, Color, Element, ElementKind, MixBlendMode, Rect, Style, + Transform2D, +}; +use skia_safe::Color4f; + +#[test] +fn create_surface_and_clear_red() { + let mut surface = RenderSurface::new_raster(100, 100).expect("should create surface"); + surface.clear(Color4f::new(1.0, 0.0, 0.0, 1.0)); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + assert_eq!(pixels.len(), 100 * 100 * 4); + + // First pixel: RGBA = (255, 0, 0, 255) + assert_eq!(pixels[0], 255, "red channel"); + assert_eq!(pixels[1], 0, "green channel"); + assert_eq!(pixels[2], 0, "blue channel"); + assert_eq!(pixels[3], 255, "alpha channel"); +} + +#[test] +fn encode_jpeg_produces_bytes() { + let mut surface = RenderSurface::new_raster(64, 64).expect("should create surface"); + surface.clear(Color4f::new(0.0, 0.0, 1.0, 1.0)); + + let jpeg = surface.encode_jpeg(80).expect("should encode JPEG"); + assert!( + jpeg.len() > 100, + "JPEG should be non-trivial, got {} bytes", + jpeg.len() + ); + // JPEG magic bytes: 0xFF 0xD8 + assert_eq!(jpeg[0], 0xFF, "JPEG SOI byte 0"); + assert_eq!(jpeg[1], 0xD8, "JPEG SOI byte 1"); +} + +#[test] +fn encode_png_produces_bytes() { + let mut surface = RenderSurface::new_raster(64, 64).expect("should create surface"); + surface.clear(Color4f::new(0.0, 1.0, 0.0, 1.0)); + + let png = surface.encode_png().expect("should encode PNG"); + assert!( + png.len() > 100, + "PNG should be non-trivial, got {} bytes", + png.len() + ); + // PNG magic bytes: 0x89 0x50 0x4E 0x47 + assert_eq!(png[0], 0x89, "PNG signature byte 0"); + assert_eq!(png[1], 0x50, "PNG signature byte 1"); + assert_eq!(png[2], 0x4E, "PNG signature byte 2"); + assert_eq!(png[3], 0x47, "PNG signature byte 3"); +} + +#[test] +fn surface_dimensions() { + let surface = RenderSurface::new_raster(1920, 1080).expect("should create surface"); + assert_eq!(surface.width(), 1920); + assert_eq!(surface.height(), 1080); +} + +// --------------------------------------------------------------------------- +// Element painting tests +// --------------------------------------------------------------------------- + +#[test] +fn paint_scene_with_background_and_text() { + let mut surface = RenderSurface::new_raster(200, 100).expect("surface"); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + + let container = Element { + id: "bg".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 200.0, + height: 100.0, + }, + style: Style { + background_color: Some(Color { + r: 0, + g: 0, + b: 255, + a: 255, + }), + ..Style::default() + }, + children: vec![Element { + id: "label".into(), + kind: ElementKind::Text { + content: "Hello".into(), + }, + bounds: Rect { + x: 10.0, + y: 10.0, + width: 180.0, + height: 30.0, + }, + style: Style { + color: Some(Color { + r: 255, + g: 255, + b: 255, + a: 255, + }), + font_size: Some(24.0), + ..Style::default() + }, + children: vec![], + }], + }; + + paint_element(surface.canvas(), &container, &mut ImageCache::new()); + + let jpeg = surface.encode_jpeg(80).expect("should encode JPEG"); + assert!( + jpeg.len() > 200, + "JPEG should be non-trivial, got {} bytes", + jpeg.len() + ); + assert_eq!(jpeg[0], 0xFF); + assert_eq!(jpeg[1], 0xD8); +} + +#[test] +fn paint_element_with_border_radius_and_opacity() { + let mut surface = RenderSurface::new_raster(200, 200).expect("surface"); + // White background. + surface.clear(Color4f::new(1.0, 1.0, 1.0, 1.0)); + + let card = Element { + id: "card".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 20.0, + y: 20.0, + width: 160.0, + height: 160.0, + }, + style: Style { + background_color: Some(Color { + r: 255, + g: 0, + b: 0, + a: 255, + }), + border_radius: [12.0; 4], + opacity: 0.5, + ..Style::default() + }, + children: vec![], + }; + + paint_element(surface.canvas(), &card, &mut ImageCache::new()); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + + // Corner pixel (0,0) is outside the rounded rect — should remain white. + let idx_corner = 0; + assert_eq!(pixels[idx_corner], 255, "corner R should be white"); + assert_eq!(pixels[idx_corner + 1], 255, "corner G should be white"); + assert_eq!(pixels[idx_corner + 2], 255, "corner B should be white"); + + // Center pixel (100, 100) is inside the card. Red at 50% alpha over white + // means R should be high (close to 255), G ≈ 128, B ≈ 128. + let idx_center = (100 * 200 + 100) * 4; + assert!( + pixels[idx_center] > 200, + "center R expected > 200, got {}", + pixels[idx_center] + ); +} + +#[test] +fn paint_element_with_transform() { + let mut surface = RenderSurface::new_raster(200, 200).expect("surface"); + surface.clear(Color4f::new(0.0, 0.0, 0.0, 1.0)); + + let el = Element { + id: "transformed".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 50.0, + y: 50.0, + width: 100.0, + height: 100.0, + }, + style: Style { + background_color: Some(Color { + r: 0, + g: 255, + b: 0, + a: 255, + }), + transform: Some(Transform2D { + translate_x: 0.0, + translate_y: 0.0, + scale_x: 2.0, + scale_y: 2.0, + rotate_deg: 45.0, + }), + ..Style::default() + }, + children: vec![], + }; + + paint_element(surface.canvas(), &el, &mut ImageCache::new()); + + // Hard to assert pixel-perfect results for rotated/scaled content. + // Verify it produces a valid JPEG without crashing. + let jpeg = surface.encode_jpeg(80).expect("should encode JPEG"); + assert!( + jpeg.len() > 200, + "JPEG should be non-trivial, got {} bytes", + jpeg.len() + ); +} + +#[test] +fn paint_solid_border() { + let mut surface = RenderSurface::new_raster(100, 100).expect("surface"); + surface.clear(Color4f::new(1.0, 1.0, 1.0, 1.0)); + + let el = Element { + id: "border".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 20.0, + y: 20.0, + width: 60.0, + height: 60.0, + }, + style: Style { + border: Some(Border { + width: 4.0, + color: Color { + r: 255, + g: 0, + b: 0, + a: 255, + }, + style: BorderLineStyle::Solid, + }), + ..Style::default() + }, + children: vec![], + }; + + paint_element(surface.canvas(), &el, &mut ImageCache::new()); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + let border_px = (50 * 100 + 22) * 4; + assert!( + pixels[border_px] > 200 && pixels[border_px + 1] < 50, + "left border should be red, got RGB({},{},{})", + pixels[border_px], + pixels[border_px + 1], + pixels[border_px + 2] + ); + + let center_px = (50 * 100 + 50) * 4; + assert_eq!(pixels[center_px], 255, "center should remain white"); + assert_eq!(pixels[center_px + 1], 255, "center should remain white"); + assert_eq!(pixels[center_px + 2], 255, "center should remain white"); +} + +#[test] +fn paint_clip_path_circle() { + let mut surface = RenderSurface::new_raster(100, 100).expect("surface"); + surface.clear(Color4f::new(1.0, 1.0, 1.0, 1.0)); + + let el = Element { + id: "clipped".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }, + style: Style { + background_color: Some(Color { + r: 0, + g: 200, + b: 0, + a: 255, + }), + clip_path: Some(ClipPath::Circle { + x: 50.0, + y: 50.0, + radius: 25.0, + }), + ..Style::default() + }, + children: vec![], + }; + + paint_element(surface.canvas(), &el, &mut ImageCache::new()); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + let center = (50 * 100 + 50) * 4; + assert!( + pixels[center + 1] > 150, + "circle center should be green, got G={}", + pixels[center + 1] + ); + + let corner = (5 * 100 + 5) * 4; + assert_eq!(pixels[corner], 255, "corner should remain white"); + assert_eq!(pixels[corner + 1], 255, "corner should remain white"); + assert_eq!(pixels[corner + 2], 255, "corner should remain white"); +} + +#[test] +fn paint_mix_blend_mode_multiply() { + let mut surface = RenderSurface::new_raster(80, 80).expect("surface"); + surface.clear(Color4f::new(1.0, 0.0, 0.0, 1.0)); + + let el = Element { + id: "blend".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 10.0, + y: 10.0, + width: 60.0, + height: 60.0, + }, + style: Style { + background_color: Some(Color { + r: 0, + g: 0, + b: 255, + a: 255, + }), + mix_blend_mode: Some(MixBlendMode::Multiply), + ..Style::default() + }, + children: vec![], + }; + + paint_element(surface.canvas(), &el, &mut ImageCache::new()); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + let center = (40 * 80 + 40) * 4; + assert!( + pixels[center] < 30 && pixels[center + 2] < 30, + "blue over red with multiply should be near black, got RGB({},{},{})", + pixels[center], + pixels[center + 1], + pixels[center + 2] + ); +} + +#[test] +fn paint_invisible_element_skipped() { + let mut surface = RenderSurface::new_raster(100, 100).expect("surface"); + // Clear to magenta so we can detect any unwanted painting. + surface.clear(Color4f::new(1.0, 0.0, 1.0, 1.0)); + + let el = Element { + id: "hidden".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }, + style: Style { + background_color: Some(Color { + r: 0, + g: 255, + b: 0, + a: 255, + }), + visibility: false, + ..Style::default() + }, + children: vec![], + }; + + paint_element(surface.canvas(), &el, &mut ImageCache::new()); + + let pixels = surface.read_pixels_rgba().expect("should read pixels"); + // Surface should still be magenta — the invisible element painted nothing. + assert_eq!(pixels[0], 255, "R should be 255 (magenta)"); + assert_eq!(pixels[1], 0, "G should be 0 (magenta)"); + assert_eq!(pixels[2], 255, "B should be 255 (magenta)"); + assert_eq!(pixels[3], 255, "A should be 255"); +} diff --git a/packages/native-renderer/tests/pipeline_test.rs b/packages/native-renderer/tests/pipeline_test.rs new file mode 100644 index 000000000..e509fe719 --- /dev/null +++ b/packages/native-renderer/tests/pipeline_test.rs @@ -0,0 +1,128 @@ +use std::path::Path; + +use hyperframes_native_renderer::pipeline::{render_static, RenderConfig}; +use hyperframes_native_renderer::scene::{Color, Element, ElementKind, Rect, Scene, Style}; + +/// Build a realistic scene: dark-blue background, white rounded card, text inside the card. +fn make_test_scene() -> Scene { + let text = Element { + id: "heading".into(), + kind: ElementKind::Text { + content: "Hello from Skia!".into(), + }, + bounds: Rect { + x: 24.0, + y: 20.0, + width: 280.0, + height: 40.0, + }, + style: Style { + color: Some(Color { + r: 30, + g: 30, + b: 50, + a: 255, + }), + font_size: Some(28.0), + ..Style::default() + }, + children: vec![], + }; + + let card = Element { + id: "card".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 140.0, + y: 80.0, + width: 360.0, + height: 200.0, + }, + style: Style { + background_color: Some(Color { + r: 255, + g: 255, + b: 255, + a: 255, + }), + border_radius: [16.0; 4], + overflow_hidden: true, + ..Style::default() + }, + children: vec![text], + }; + + let background = Element { + id: "bg".into(), + kind: ElementKind::Container, + bounds: Rect { + x: 0.0, + y: 0.0, + width: 640.0, + height: 360.0, + }, + style: Style { + background_color: Some(Color { + r: 15, + g: 23, + b: 42, + a: 255, + }), + ..Style::default() + }, + children: vec![card], + }; + + Scene { + width: 640, + height: 360, + elements: vec![background], + } +} + +#[test] +fn render_static_scene_to_mp4() { + let scene = make_test_scene(); + let output_path = "/tmp/hyperframes-native-test.mp4"; + + let config = RenderConfig { + fps: 30, + duration_secs: 1.0, + quality: 80, + output_path: output_path.to_string(), + }; + + let result = render_static(&scene, &config).unwrap(); + + assert_eq!(result.total_frames, 30); + assert_eq!(result.output_path, output_path); + + let path = Path::new(output_path); + assert!(path.exists(), "output MP4 must exist"); + + let size = std::fs::metadata(output_path).unwrap().len(); + assert!(size > 1000, "MP4 should be non-trivial, got {size} bytes"); + + std::fs::remove_file(output_path).ok(); +} + +#[test] +fn render_static_fractional_duration() { + let scene = make_test_scene(); + let output_path = "/tmp/hyperframes-native-frac.mp4"; + + let config = RenderConfig { + fps: 24, + duration_secs: 0.5, + quality: 70, + output_path: output_path.to_string(), + }; + + let result = render_static(&scene, &config).unwrap(); + + // ceil(24 * 0.5) = 12 + assert_eq!(result.total_frames, 12); + assert!(Path::new(output_path).exists()); + + std::fs::remove_file(output_path).ok(); +} diff --git a/packages/native-renderer/tests/scene_test.rs b/packages/native-renderer/tests/scene_test.rs new file mode 100644 index 000000000..60fd77b9c --- /dev/null +++ b/packages/native-renderer/tests/scene_test.rs @@ -0,0 +1,253 @@ +use hyperframes_native_renderer::scene::{ + parse_scene_json, BackgroundImageFit, Color, ElementKind, +}; + +#[test] +fn parse_minimal_scene() { + let json = r#"{ + "width": 1920, + "height": 1080, + "elements": [{ + "id": "bg", + "kind": { "type": "Container" }, + "bounds": { "x": 0, "y": 0, "width": 1920, "height": 1080 }, + "style": { + "background_color": { "r": 30, "g": 30, "b": 30, "a": 255 }, + "opacity": 1.0, + "border_radius": [0, 0, 0, 0], + "overflow_hidden": false, + "transform": null, + "visibility": true + }, + "children": [] + }] + }"#; + + let scene = parse_scene_json(json).expect("should parse"); + assert_eq!(scene.width, 1920); + assert_eq!(scene.height, 1080); + assert_eq!(scene.elements.len(), 1); + + let el = &scene.elements[0]; + assert_eq!(el.id, "bg"); + assert!(matches!(el.kind, ElementKind::Container)); + assert_eq!(el.bounds.x, 0.0); + assert_eq!(el.bounds.width, 1920.0); + assert_eq!( + el.style.background_color, + Some(Color { + r: 30, + g: 30, + b: 30, + a: 255 + }) + ); + assert_eq!(el.style.opacity, 1.0); + assert!(el.style.visibility); + assert!(el.children.is_empty()); +} + +#[test] +fn parse_nested_children_with_text() { + let json = r#"{ + "width": 1280, + "height": 720, + "elements": [{ + "id": "root", + "kind": { "type": "Container" }, + "bounds": { "x": 0, "y": 0, "width": 1280, "height": 720 }, + "children": [{ + "id": "title", + "kind": { "type": "Text", "content": "Hello World" }, + "bounds": { "x": 100, "y": 50, "width": 400, "height": 60 }, + "style": { + "font_family": "Inter", + "font_size": 48.0, + "font_weight": 700, + "color": { "r": 255, "g": 255, "b": 255, "a": 255 } + }, + "children": [] + }, { + "id": "subtitle", + "kind": { "type": "Text", "content": "Subtitle" }, + "bounds": { "x": 100, "y": 120, "width": 400, "height": 30 }, + "children": [] + }] + }] + }"#; + + let scene = parse_scene_json(json).expect("should parse"); + assert_eq!(scene.width, 1280); + assert_eq!(scene.height, 720); + assert_eq!(scene.elements.len(), 1); + + let root = &scene.elements[0]; + assert_eq!(root.children.len(), 2); + + let title = &root.children[0]; + assert_eq!(title.id, "title"); + match &title.kind { + ElementKind::Text { content } => assert_eq!(content, "Hello World"), + other => panic!("expected Text, got {other:?}"), + } + assert_eq!(title.style.font_family.as_deref(), Some("Inter")); + assert_eq!(title.style.font_size, Some(48.0)); + assert_eq!(title.style.font_weight, Some(700)); + assert_eq!( + title.style.color, + Some(Color { + r: 255, + g: 255, + b: 255, + a: 255 + }) + ); + + let subtitle = &root.children[1]; + assert_eq!(subtitle.id, "subtitle"); + // subtitle has default style — opacity 1.0, visible, no font info + assert_eq!(subtitle.style.opacity, 1.0); + assert!(subtitle.style.visibility); + assert!(subtitle.style.font_family.is_none()); +} + +#[test] +fn parse_image_and_video_elements() { + let json = r#"{ + "width": 1920, + "height": 1080, + "elements": [ + { + "id": "bg-img", + "kind": { "type": "Image", "src": "/assets/bg.png" }, + "bounds": { "x": 0, "y": 0, "width": 1920, "height": 1080 }, + "children": [] + }, + { + "id": "clip", + "kind": { "type": "Video", "src": "/assets/intro.mp4" }, + "bounds": { "x": 100, "y": 100, "width": 800, "height": 450 }, + "style": { + "opacity": 0.8, + "overflow_hidden": true, + "border_radius": [12, 12, 12, 12] + }, + "children": [] + } + ] + }"#; + + let scene = parse_scene_json(json).expect("should parse"); + assert_eq!(scene.elements.len(), 2); + + match &scene.elements[0].kind { + ElementKind::Image { src } => assert_eq!(src, "/assets/bg.png"), + other => panic!("expected Image, got {other:?}"), + } + + let clip = &scene.elements[1]; + match &clip.kind { + ElementKind::Video { src } => assert_eq!(src, "/assets/intro.mp4"), + other => panic!("expected Video, got {other:?}"), + } + assert_eq!(clip.style.opacity, 0.8); + assert!(clip.style.overflow_hidden); + assert_eq!(clip.style.border_radius, [12.0, 12.0, 12.0, 12.0]); +} + +#[test] +fn parse_background_image_layer() { + let json = r#"{ + "width": 640, + "height": 360, + "elements": [{ + "id": "poster", + "kind": { "type": "Container" }, + "bounds": { "x": 0, "y": 0, "width": 640, "height": 360 }, + "style": { + "background_image": { + "src": "file:///tmp/poster.png", + "fit": "contain", + "position": { "x": 0.25, "y": 0.75 } + } + }, + "children": [] + }] + }"#; + + let scene = parse_scene_json(json).expect("should parse"); + let background_image = scene.elements[0] + .style + .background_image + .as_ref() + .expect("should parse background image"); + + assert_eq!(background_image.src, "file:///tmp/poster.png"); + assert_eq!(background_image.fit, BackgroundImageFit::Contain); + assert_eq!(background_image.position.x, 0.25); + assert_eq!(background_image.position.y, 0.75); +} + +#[test] +fn parse_transform() { + let json = r#"{ + "width": 800, + "height": 600, + "elements": [{ + "id": "box", + "kind": { "type": "Container" }, + "bounds": { "x": 100, "y": 100, "width": 200, "height": 200 }, + "style": { + "transform": { + "translate_x": 50.0, + "translate_y": -30.0, + "scale_x": 1.5, + "scale_y": 1.5, + "rotate_deg": 45.0 + } + }, + "children": [] + }] + }"#; + + let scene = parse_scene_json(json).expect("should parse"); + let t = scene.elements[0] + .style + .transform + .as_ref() + .expect("should have transform"); + assert_eq!(t.translate_x, 50.0); + assert_eq!(t.translate_y, -30.0); + assert_eq!(t.scale_x, 1.5); + assert_eq!(t.scale_y, 1.5); + assert_eq!(t.rotate_deg, 45.0); +} + +#[test] +fn invalid_json_returns_error() { + let result = parse_scene_json("not json"); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("invalid scene JSON")); +} + +#[test] +fn roundtrip_serialize_deserialize() { + let json = r#"{ + "width": 1920, + "height": 1080, + "elements": [{ + "id": "bg", + "kind": { "type": "Container" }, + "bounds": { "x": 0, "y": 0, "width": 1920, "height": 1080 }, + "children": [] + }] + }"#; + + let scene = parse_scene_json(json).expect("should parse"); + let serialized = serde_json::to_string(&scene).expect("should serialize"); + let reparsed = parse_scene_json(&serialized).expect("should reparse"); + assert_eq!(reparsed.width, scene.width); + assert_eq!(reparsed.height, scene.height); + assert_eq!(reparsed.elements.len(), scene.elements.len()); + assert_eq!(reparsed.elements[0].id, scene.elements[0].id); +} diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index cfa6d8a8d..2d0c63bca 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -1504,31 +1504,29 @@ export async function executeRenderJob( ); } - // ── Stage 3: Audio processing ─────────────────────────────────────── + // ── Stage 3: Audio processing (launched async, overlaps with capture) ── const stage3Start = Date.now(); updateJobStatus(job, "preprocessing", "Processing audio tracks", 20, onProgress); const audioOutputPath = join(workDir, "audio.aac"); let hasAudio = false; - if (composition.audios.length > 0) { - const audioResult = await processCompositionAudio( - composition.audios, - projectDir, - join(workDir, "audio-work"), - audioOutputPath, - job.duration, - abortSignal, - undefined, - compiledDir, - ); - assertNotAborted(); - - hasAudio = audioResult.success; - perfStages.audioProcessMs = Date.now() - stage3Start; - } else { - perfStages.audioProcessMs = Date.now() - stage3Start; - } + // Launch audio processing without awaiting — it runs on separate FFmpeg + // processes and doesn't compete with Chrome for CPU. We collect the result + // before the mux stage where the audio file is needed. + const audioPromise = + composition.audios.length > 0 + ? processCompositionAudio( + composition.audios, + projectDir, + join(workDir, "audio-work"), + audioOutputPath, + job.duration, + abortSignal, + undefined, + compiledDir, + ) + : null; // ── Stage 4: Frame capture ────────────────────────────────────────── const stage4Start = Date.now(); @@ -1548,12 +1546,20 @@ export async function executeRenderJob( const framesDir = join(workDir, "captured-frames"); if (!existsSync(framesDir)) mkdirSync(framesDir, { recursive: true }); + // When streaming to FFmpeg, intermediate JPEG quality can be lower — the + // frames are re-encoded so quality loss is invisible in the final output. + // Smaller JPEG buffers transfer faster over CDP, cutting per-frame time. + const captureJpegQuality = (() => { + if (needsAlpha) return undefined; + if (enableStreamingEncode) return cfg.streamingJpegQuality ?? 55; + return job.config.quality === "draft" ? 80 : 95; + })(); const captureOptions: CaptureOptions = { width, height, fps: job.config.fps, format: needsAlpha ? "png" : "jpeg", - quality: needsAlpha ? undefined : job.config.quality === "draft" ? 80 : 95, + quality: captureJpegQuality, }; // Native HDR videos (e.g. HEVC) may be undecodable by Chrome on the current @@ -2608,6 +2614,16 @@ export async function executeRenderJob( fileServer.close(); fileServer = null; + // ── Collect overlapped audio result ──────────────────────────────── + if (audioPromise) { + const audioResult = await audioPromise; + assertNotAborted(); + hasAudio = audioResult.success; + perfStages.audioProcessMs = Date.now() - stage3Start; + } else { + perfStages.audioProcessMs = 0; + } + // ── Stage 6: Assemble ─────────────────────────────────────────────── const stage6Start = Date.now(); updateJobStatus(job, "assembling", "Assembling final video", 90, onProgress); diff --git a/packages/producer/tests/perf/benchmark-results.json b/packages/producer/tests/perf/benchmark-results.json index 73e281fb6..2505abaef 100644 --- a/packages/producer/tests/perf/benchmark-results.json +++ b/packages/producer/tests/perf/benchmark-results.json @@ -1,8 +1,8 @@ { - "timestamp": "2026-03-11T01:22:09.852Z", - "platform": "linux x64", - "nodeVersion": "v22.17.0", - "runsPerFixture": 1, + "timestamp": "2026-04-25T17:56:52.846Z", + "platform": "darwin arm64", + "nodeVersion": "v24.9.0", + "runsPerFixture": 3, "fixtures": [ { "fixture": "missing-host-comp-id", @@ -11,11 +11,11 @@ { "run": 1, "perfSummary": { - "renderId": "0afc2d13-2bc0-4eda-9134-1efeb6a15908", - "totalElapsedMs": 5620, + "renderId": "9f314d55-b125-45f7-b7ad-c4e3db7e4e33", + "totalElapsedMs": 2060, "fps": 24, "quality": "high", - "workers": 2, + "workers": 3, "chunkedEncode": false, "chunkSizeFrames": null, "compositionDurationSeconds": 3, @@ -27,33 +27,106 @@ "videoCount": 0, "audioCount": 1, "stages": { - "compileOnlyMs": 29, + "compileOnlyMs": 106, "browserProbeMs": 0, - "compileMs": 30, + "compileMs": 107, "videoExtractMs": 0, - "audioProcessMs": 136, - "captureMs": 4832, - "encodeMs": 528, - "assembleMs": 94 + "captureMs": 1622, + "encodeMs": 242, + "audioProcessMs": 1866, + "assembleMs": 87 }, - "captureAvgMs": 67 + "tmpPeakBytes": 5338532, + "captureAvgMs": 23, + "peakRssMb": 239, + "peakHeapUsedMb": 49 + } + }, + { + "run": 2, + "perfSummary": { + "renderId": "3c32fd08-62fe-4093-9415-4fb5a1c55691", + "totalElapsedMs": 1826, + "fps": 24, + "quality": "high", + "workers": 3, + "chunkedEncode": false, + "chunkSizeFrames": null, + "compositionDurationSeconds": 3, + "totalFrames": 72, + "resolution": { + "width": 1080, + "height": 1920 + }, + "videoCount": 0, + "audioCount": 1, + "stages": { + "compileOnlyMs": 68, + "browserProbeMs": 0, + "compileMs": 69, + "videoExtractMs": 0, + "captureMs": 1421, + "encodeMs": 246, + "audioProcessMs": 1669, + "assembleMs": 88 + }, + "tmpPeakBytes": 5338532, + "captureAvgMs": 20, + "peakRssMb": 250, + "peakHeapUsedMb": 55 + } + }, + { + "run": 3, + "perfSummary": { + "renderId": "32807044-f333-4398-b7ef-09c58837e281", + "totalElapsedMs": 1859, + "fps": 24, + "quality": "high", + "workers": 3, + "chunkedEncode": false, + "chunkSizeFrames": null, + "compositionDurationSeconds": 3, + "totalFrames": 72, + "resolution": { + "width": 1080, + "height": 1920 + }, + "videoCount": 0, + "audioCount": 1, + "stages": { + "compileOnlyMs": 64, + "browserProbeMs": 0, + "compileMs": 65, + "videoExtractMs": 0, + "captureMs": 1446, + "encodeMs": 245, + "audioProcessMs": 1693, + "assembleMs": 101 + }, + "tmpPeakBytes": 5338532, + "captureAvgMs": 20, + "peakRssMb": 260, + "peakHeapUsedMb": 63 } } ], "averages": { - "totalElapsedMs": 5620, - "captureAvgMs": 67, + "totalElapsedMs": 1915, + "captureAvgMs": 21, + "peakRssMb": 250, + "peakHeapUsedMb": 56, "stages": { - "compileOnlyMs": 29, + "compileOnlyMs": 79, "browserProbeMs": 0, - "compileMs": 30, + "compileMs": 80, "videoExtractMs": 0, - "audioProcessMs": 136, - "captureMs": 4832, - "encodeMs": 528, - "assembleMs": 94 + "captureMs": 1496, + "encodeMs": 244, + "audioProcessMs": 1743, + "assembleMs": 92 } } } ] -} +} \ No newline at end of file