diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05af975..4389883 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,9 +22,13 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy + targets: wasm32-unknown-unknown - uses: Swatinem/rust-cache@v2 - run: cargo fmt --all --check - run: cargo clippy --workspace --all-targets --all-features -- -D warnings + # Browser bindings live in wasm/; this locked check keeps the root + # crate and every parser dependency buildable for that target. + - run: cargo check -p anydoc --target wasm32-unknown-unknown --locked # Root package only: the node/ and python/ binding crates are cdylib-only # and their test harnesses cannot link outside a Node/Python host; they # are covered by the node and python jobs instead. @@ -78,7 +82,70 @@ jobs: curl -sSfL https://github.com/wasm-bindgen/wasm-pack/releases/download/v0.15.0/wasm-pack-v0.15.0-x86_64-unknown-linux-musl.tar.gz | tar xz --strip-components=1 -C /usr/local/bin wasm-pack-v0.15.0-x86_64-unknown-linux-musl/wasm-pack - run: wasm-pack build wasm --release --target web --scope firecrawl + # The suite is told which build it is looking at, so the test guarding + # the OCR feature gate has an expectation of its own to check. + - run: node --test wasm/test.mjs + env: + ANYDOC_WASM_OCR_BUILD: '0' + + # Cargo.lock pins pdf-inspector through [patch.crates-io] to a git + # revision of my fork's PR branch (firecrawl/pdf-inspector#280), so + # every lane below runs fine from a clean clone. The remaining step + # before merge is switching that patch to the released pdf-inspector + # crate once #280 ships. + ocr: + name: OCR + runs-on: blacksmith-8vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + components: clippy + - uses: Swatinem/rust-cache@v2 + with: + key: ocr + # The model-gated Rust tests are #[ignore]d, so they never run here: + # the models are a separate CC-BY-SA-4.0 download CI does not carry. + - run: cargo test --locked --features ocr + - run: cargo check --locked -p anydoc --features ocr --target wasm32-unknown-unknown + - run: cargo clippy -p anydoc-wasm --features ocr --target wasm32-unknown-unknown -- -D warnings + - uses: actions/setup-node@v7 + with: + node-version: 26 + - name: Install wasm-pack + run: > + curl -sSfL https://github.com/wasm-bindgen/wasm-pack/releases/download/v0.15.0/wasm-pack-v0.15.0-x86_64-unknown-linux-musl.tar.gz + | tar xz --strip-components=1 -C /usr/local/bin wasm-pack-v0.15.0-x86_64-unknown-linux-musl/wasm-pack + # wasm-pack takes cargo flags after `--`; passing --features directly + # makes it forward its own arguments to cargo and fail. + - run: wasm-pack build wasm --release --target web --scope firecrawl -- --features ocr - run: node --test wasm/test.mjs + env: + ANYDOC_WASM_OCR_BUILD: '1' + + # The crate compiles on 1.88; the ocr feature pulls in a renderer that needs + # 1.92. Both floors are pinned so a dependency cannot raise either quietly. + compiler-floor: + name: Compiler floor (${{ matrix.rust }}) + runs-on: blacksmith-8vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + include: + - rust: 1.88.0 + check: cargo check --locked -p anydoc + - rust: 1.92.0 + check: cargo check --locked -p anydoc --features ocr + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + - uses: Swatinem/rust-cache@v2 + with: + key: compiler-floor-${{ matrix.rust }} + - run: ${{ matrix.check }} python: name: Python bindings diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 4764bce..120f168 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -29,13 +29,25 @@ jobs: curl -sSfL https://github.com/wasm-bindgen/wasm-pack/releases/download/v0.15.0/wasm-pack-v0.15.0-x86_64-unknown-linux-musl.tar.gz | tar xz --strip-components=1 -C /usr/local/bin wasm-pack-v0.15.0-x86_64-unknown-linux-musl/wasm-pack # --out-dir is crate-relative: the module lands in wasm/www/pkg, where - # index.html imports it from. - - run: wasm-pack build wasm --release --target web --no-pack --out-dir www/pkg + # index.html imports it from. The demo ships OCR, so the models below + # are fetched pinned by hash at deploy time and never live in git. + - run: wasm-pack build wasm --release --target web --no-pack --out-dir www/pkg -- --features ocr + # SIMD kernels for the OCR inference; supported by all current browsers. + env: + RUSTFLAGS: -C target-feature=+simd128 - name: Assemble site run: | mkdir -p _site cp -r wasm/www/. _site/ rm -f _site/pkg/.gitignore + - name: Fetch OCR models + run: | + mkdir -p _site/models + base=https://ocrs-models.s3-accelerate.amazonaws.com + curl -sSfL -o _site/models/text-detection.rten $base/text-detection.rten + curl -sSfL -o _site/models/text-recognition.rten $base/text-recognition.rten + echo 'f15cfb56bd02c4bf478a20343986504a1f01e1665c2b3a0ad66340f054b1b5ca _site/models/text-detection.rten' | sha256sum -c - + echo 'e484866d4cce403175bd8d00b128feb08ab42e208de30e42cd9889d8f1735a6e _site/models/text-recognition.rten' | sha256sum -c - - uses: actions/upload-pages-artifact@v4 with: path: _site diff --git a/.gitignore b/.gitignore index f84ace4..2549908 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ __pycache__/ # wasm-pack build output: pkg/ is the npm package, www/pkg/ the demo site copy /wasm/pkg/ /wasm/www/pkg/ +# OCR models for a local run of the demo: CI fetches them at deploy time +/wasm/www/models/ diff --git a/Cargo.lock b/Cargo.lock index f1b1f7e..36d6979 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -96,10 +111,13 @@ dependencies = [ "csv", "encoding_rs", "flate2", + "image", "insta", "log", + "ocrs", "pdf-inspector", "quick-xml", + "rten", "sha2 0.11.0", "zip", ] @@ -134,6 +152,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "atoi_simd" version = "0.18.1" @@ -189,18 +219,65 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f65693059b6b9c588b9f62fed1cedbf0a8b805631457ea162d68f0de186f3de5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "calamine" version = "0.36.1" @@ -297,6 +374,15 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "color" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ec7c5eb7a16992b1904d76c517d170ab353b0e0b3d5a0c81a8a0cd1037893cf" +dependencies = [ + "bytemuck", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -387,6 +473,12 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -566,6 +658,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + [[package]] name = "fast-float2" version = "0.2.3" @@ -578,12 +679,43 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "fearless_simd" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flatbuffers" +version = "24.12.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + [[package]] name = "flate2" version = "1.1.9" @@ -601,6 +733,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "font-types" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b38ad915f6dadd993ced50848a8291a543bd41ca62bc10740d5e64e2ab4cfd7" +dependencies = [ + "bytemuck", +] + [[package]] name = "futures" version = "0.3.33" @@ -713,11 +860,141 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "glifo" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d99fc21d493812643aae86d53b7bbd02f376434a90317e8a790bc209fdd6605e" +dependencies = [ + "bytemuck", + "foldhash", + "hashbrown", + "log", + "peniko", + "png", + "skrifa", + "smallvec", + "vello_common 0.0.9", +] + +[[package]] +name = "guillotiere" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b17e70c989c36bad147b27a58d148c0741c51448aa5653436547323e524d0ab" +dependencies = [ + "euclid", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hayro" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4caa128ab87fd48ffb7490617cf93f77f606820dffbc9fd9ef6ab0ed077f56d" +dependencies = [ + "bytemuck", + "hayro-interpret", + "image", + "kurbo", + "pic-scale", + "vello_cpu", +] + +[[package]] +name = "hayro-ccitt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f4d0e94ddd48749f06bbe4e5389fb9799a0c45bcaf00495042076ef05e3241a" + +[[package]] +name = "hayro-cmap" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d285dc30731c8485de5fa732fbdf2b3affdf01e4da7c2135022ca6fa664bf6" +dependencies = [ + "brotli", + "hayro-postscript", +] + +[[package]] +name = "hayro-interpret" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2613d0406898995042d0794c4245b2f2fba1246490c8ac593769bba6551129d" +dependencies = [ + "bitflags 2.13.1", + "hayro-cmap", + "hayro-syntax", + "kurbo", + "moxcms", + "phf", + "rustc-hash", + "siphasher", + "skrifa", + "smallvec", + "yoke", +] + +[[package]] +name = "hayro-jbig2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69374b3668dd45aeb3d3145cda68f2c7b4f223aaa2511e67d076f1c7d741388d" +dependencies = [ + "fearless_simd", + "hayro-ccitt", +] + +[[package]] +name = "hayro-jpeg2000" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ab947623ef4ccaa7acf0579edf7cbb5a73838e3839a7be73335e522f433a1" +dependencies = [ + "fearless_simd", +] + +[[package]] +name = "hayro-postscript" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "885c5ef0654933139a9b9546fc2c69e18d37f38aa2520f079092b1be18f1fcaa" + +[[package]] +name = "hayro-syntax" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0edeafd70aa2db743de8ede8637d07ec87db05efe69cde371d03f1b185fcef27" +dependencies = [ + "flate2", + "hayro-ccitt", + "hayro-jbig2", + "hayro-jpeg2000", + "memchr", + "rustc-hash", + "smallvec", + "zune-jpeg", +] [[package]] name = "heck" @@ -725,6 +1002,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hybrid-array" version = "0.4.14" @@ -758,6 +1041,33 @@ dependencies = [ "cc", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "image-webp", + "moxcms", + "num-traits", + "png", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + [[package]] name = "include_dir" version = "0.7.4" @@ -885,6 +1195,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kurbo" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2" +dependencies = [ + "arrayvec", + "euclid", + "polycool", + "smallvec", +] + [[package]] name = "libc" version = "0.2.189" @@ -901,6 +1223,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "linebender_resource_handle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -970,6 +1298,16 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "napi" version = "3.12.0" @@ -1057,6 +1395,31 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "ocrs" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5379fdd3f11522b5a2ff53017a189463dabf5d0a9c915cb3eb97fabec4ea11c" +dependencies = [ + "anyhow", + "rayon", + "rten", + "rten-imageproc", + "rten-tensor", + "thiserror", + "wasm-bindgen", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1071,11 +1434,12 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "pdf-inspector" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd2f755e49ad38eafbc82ba2bec1c59d57b5bf829b13a8f35e4263bc8913df3a" +version = "0.1.7" +source = "git+https://github.com/massimodeluisa/pdf-inspector.git?rev=433629e84cc880b450ac71ccf968e10df2b7299d#433629e84cc880b450ac71ccf968e10df2b7299d" dependencies = [ + "bytemuck", "env_logger", + "hayro", "include_dir", "log", "lopdf", @@ -1087,12 +1451,99 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "peniko" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839c8299360d2e998bdb106dc0a6cd71dcc5f4df51df1b620361bf50e283cca6" +dependencies = [ + "bytemuck", + "color", + "kurbo", + "linebender_resource_handle", + "smallvec", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pic-scale" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c80d88d5c31215ceec2862abb2504860c715b8e2545a5ab15b62a3d097f30d" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.13.1", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "polycool" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6" +dependencies = [ + "arrayvec", +] + [[package]] name = "portable-atomic" version = "1.14.0" @@ -1123,6 +1574,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "pyo3" version = "0.29.1" @@ -1180,6 +1637,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-xml" version = "0.41.0" @@ -1248,6 +1711,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "read-fonts" +version = "0.39.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4ed38b89c2c77ff968c524145ad65fb010f38af5c7a224b53b81d47ac2daa81" +dependencies = [ + "bytemuck", + "font-types", +] + [[package]] name = "regex" version = "1.13.1" @@ -1277,12 +1750,121 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rten" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43c230fa4ade87c913f61dbd911b7eb0d49460ceff3f1e4fabc837fac191137c" +dependencies = [ + "flatbuffers", + "num_cpus", + "rayon", + "rten-base", + "rten-gemm", + "rten-model-file", + "rten-shape-inference", + "rten-simd", + "rten-tensor", + "rten-vecmath", + "rustc-hash", + "smallvec", + "typeid", + "wasm-bindgen", +] + +[[package]] +name = "rten-base" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2738cf8bb4c27f828ac788d01ccf4e367e8e773cfec6851f81851b5211de6a79" +dependencies = [ + "rayon", +] + +[[package]] +name = "rten-gemm" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a81a0ca209fb5ce21bd17efa0bd287d5881c6cebfbff0b21c4294a1a14a9e" +dependencies = [ + "rayon", + "rten-base", + "rten-simd", + "rten-tensor", +] + +[[package]] +name = "rten-imageproc" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f148e7e941fb5727b9046a5fa1b45525543d5105f14b384fd9261df0ee49bc" +dependencies = [ + "rten-tensor", +] + +[[package]] +name = "rten-model-file" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2f8d270f07ab1bbfff47250c6039f6caa5da59d6da7d74f66aa48559aa6fea" +dependencies = [ + "flatbuffers", + "rten-base", +] + +[[package]] +name = "rten-shape-inference" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8a913c7ca40e2bfbb2a0cd447cce56b33ab19435f56693271a2ef37cf58984" +dependencies = [ + "rten-tensor", + "smallvec", +] + +[[package]] +name = "rten-simd" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b19a0032dfcb70dd20960c1c51a37674b237586cbc1ce586f45b46605d108e82" + +[[package]] +name = "rten-tensor" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05dc744a270aa32d154f1a3df8e48740ccc1be9dfbcf23295ada66d83aa98de6" +dependencies = [ + "rayon", + "rten-base", + "smallvec", + "typeid", +] + +[[package]] +name = "rten-vecmath" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9574ddebf5671bc08ceb76e2e1638fadc57fdeff318634eab2c29e9a803cff64" +dependencies = [ + "rten-base", + "rten-simd", +] + [[package]] name = "rustc-hash" version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.4" @@ -1405,12 +1987,40 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "skrifa" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c34617370ae968efb7161bb2beb517d9084659aae19e24b89e3db25b46e4564" +dependencies = [ + "bytemuck", + "read-fonts", +] + [[package]] name = "slab" version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "stringprep" version = "0.1.5" @@ -1444,6 +2054,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "target-lexicon" version = "0.13.5" @@ -1483,6 +2104,20 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "time" version = "0.3.55" @@ -1540,6 +2175,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.20.1" @@ -1595,6 +2236,53 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vello_common" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3361bff7f7d82c0c496b92048db83846691f0e844cc28dee92b1c824291b55ee" +dependencies = [ + "bytemuck", + "fearless_simd", + "guillotiere", + "hashbrown", + "log", + "peniko", + "png", + "smallvec", + "thiserror", +] + +[[package]] +name = "vello_common" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d672facaa2d697285a786cd9d44d614cd2ce54cdc022504bf339f8fff3b750" +dependencies = [ + "bytemuck", + "fearless_simd", + "guillotiere", + "hashbrown", + "log", + "peniko", + "png", + "smallvec", + "thiserror", +] + +[[package]] +name = "vello_cpu" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d8ded630e8316bb94a55881256506d1f3b9947b5f66db8a7d32ca7ba02decd0" +dependencies = [ + "bytemuck", + "glifo", + "hashbrown", + "png", + "vello_common 0.0.8", +] + [[package]] name = "version_check" version = "0.9.5" @@ -1730,6 +2418,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + [[package]] name = "zip" version = "8.6.0" @@ -1761,3 +2513,18 @@ dependencies = [ "log", "simd-adler32", ] + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index 97d57d0..e0ebe36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,10 @@ keywords = ["markdown", "docx", "pdf", "converter", "document"] categories = ["parser-implementations", "text-processing"] include = ["/src", "/examples/convert.rs", "/README.md", "/LICENSE"] +[features] +default = [] +ocr = ["dep:image", "dep:ocrs", "dep:rten", "pdf-inspector/render"] + [dev-dependencies] insta = "1" sha2 = "0.11" @@ -27,11 +31,17 @@ cfb = "0.14.0" csv = "1.4.0" flate2 = "1" encoding_rs = "0.8.35" +image = { version = "0.25.10", optional = true, default-features = false, features = ["bmp", "jpeg", "png", "tiff", "webp"] } log = "0.4" -pdf-inspector = "0.1.8" +ocrs = { version = "0.12.2", optional = true } +pdf-inspector = "0.1.7" quick-xml = "0.41.0" +rten = { version = "0.24.0", default-features = false, features = ["rten_format"], optional = true } zip = { version = "8.6.0", default-features = false, features = ["deflate"] } [profile.release] lto = "thin" strip = "symbols" + +[patch.crates-io] +pdf-inspector = { git = "https://github.com/massimodeluisa/pdf-inspector.git", rev = "433629e84cc880b450ac71ccf968e10df2b7299d" } diff --git a/README.md b/README.md index ed4cb9e..6c84b98 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Fast Rust library that converts documents (Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF) into clean GitHub-Flavored Markdown. Includes bindings for [Node.js](node/README.md), [Python](python/README.md), and the [browser](wasm/README.md) (WebAssembly). -Built by [Firecrawl](https://firecrawl.dev) to turn any office document into LLM-ready Markdown in single-digit milliseconds, with one consistent output no matter which format goes in. It powers [Firecrawl Parse](https://firecrawl.dev/parse), so if you'd rather not run it yourself, the hosted API gives you the same conversion plus our OCR models for the scanned pages anydoc can't read on its own. +Built by [Firecrawl](https://firecrawl.dev) to turn any office document into LLM-ready Markdown in single-digit milliseconds, with one consistent output no matter which format goes in. It powers [Firecrawl Parse](https://firecrawl.dev/parse), so if you'd rather not run it yourself, the hosted API gives you the same conversion with the OCR models managed for you — anydoc reads scanned pages too, with [local OCR](#local-ocr-optional) you enable and supply models for. **[Try it in your browser](https://firecrawl.github.io/anydoc/)**: the demo page runs the library as WebAssembly, so files are converted locally and never leave your machine. @@ -133,7 +133,7 @@ let document = anydoc::to_document(&bytes, None)?; - **Content-based format detection.** The format is read from the bytes themselves (PDF header, RTF open group, OLE stream names, ZIP package mimetype), so mislabeled files still convert correctly. - **Fast.** Pure Rust, no ML models, no external services. Median conversion time is under 5ms per document. - **Bindings that stay out of the way.** Node.js conversion runs on the libuv thread pool and never blocks the event loop; Python releases the GIL so other threads keep running. TypeScript types and Python stubs ship with the packages. -- **PDF support built in.** Text-based PDFs convert locally through [pdf-inspector](https://github.com/firecrawl/pdf-inspector), no OCR service required. +- **PDF support built in.** Text-based PDFs convert locally through [pdf-inspector](https://github.com/firecrawl/pdf-inspector), no OCR service required. Scanned and mixed PDFs, and image documents, can be read with [optional local OCR](#local-ocr-optional) when you supply the models. - **Agent ready.** Ships as an [Agent Skill](#agent-skill): one `npx skills add firecrawl/anydoc` and any agent can read office documents. ## Supported formats @@ -148,6 +148,63 @@ let document = anydoc::to_document(&bytes, None)?; | EPUB | `.epub` | | CSV | `.csv` | | PDF | `.pdf` | +| Images | `.png`, `.jpg`, `.jpeg`, `.webp`, `.tif`, `.tiff`, `.bmp` | + +Text-based PDF pages convert locally with no OCR. Pages that carry no extractable text, and image documents, need [local OCR](#local-ocr-optional): without it they report as unsupported. + +## Local OCR (optional) + +anydoc extracts text; it does not read pixels. A scanned PDF page or an image document has no text to extract, so reading one needs OCR. That runs locally, behind the `ocr` cargo feature, and stays off unless you ask for it: the default build is pure Rust with no models and no network, exactly as before. + +The models are yours to supply — the library never downloads anything. Parse them once into a reusable converter and every conversion made with it shares that engine. + +```rust +// Rust: cargo add anydoc --features ocr +let converter = anydoc::Converter::builder() + .with_ocr_models(std::fs::read("text-detection.rten")?, std::fs::read("text-recognition.rten")?)? + .build(); +let markdown = converter.to_markdown_bytes(&std::fs::read("scan.pdf")?, anydoc::Format::Pdf)?; +``` + +```js +// Node.js +const converter = await Converter.create({ detectionModel, recognitionModel }) +const markdown = await converter.toMarkdownBytes(await readFile('scan.pdf')) +``` + +```js +// Browser: build the wasm package with --features ocr, then run it in a Worker +const converter = new Converter({ detectionModel, recognitionModel }) +const markdown = converter.toMarkdownBytes(pdfBytes, 'pdf') +``` + +### Models + +Two files from the [ocrs](https://github.com/robertknight/ocrs) project, in RTen format, published at `https://ocrs-models.s3-accelerate.amazonaws.com/`: + +| File | SHA-256 | +| ------------------------ | ------------------------------------------------------------------ | +| `text-detection.rten` | `f15cfb56bd02c4bf478a20343986504a1f01e1665c2b3a0ad66340f054b1b5ca` | +| `text-recognition.rten` | `e484866d4cce403175bd8d00b128feb08ab42e208de30e42cd9889d8f1735a6e` | + +The models are licensed CC-BY-SA-4.0, separately from anydoc's MIT license, so check that it suits your use before shipping them. Verify the digests after downloading: anydoc parses whatever bytes you hand it. + +The code the feature pulls in is permissively licensed throughout: `ocrs` and `rten` are MIT OR Apache-2.0, `image` is MIT OR Apache-2.0, and the PDF renderer behind it (`hayro`, `vello_cpu`) is Apache-2.0 OR MIT with `bytemuck` adding Zlib to that choice. Nothing in the graph is copyleft; the models are the only CC-BY-SA-4.0 component. + +### What to expect + +Local OCR is an early preview, and it is worth knowing its limits before relying on it: + +- The current models read **Latin scripts only**. +- Output is **plain text**: no headings, tables, or lists are inferred from a recognized page, only the lines as read. +- Accuracy varies with scan quality. **Overlapping text garbles lines** — a watermark across body text can merge into one unreadable line. +- An image with no text can still produce **short spurious fragments**. +- **EXIF orientation is not applied**: a photo stored rotated is read as stored. +- A scanned document with no extractable text at all has **every page** rendered and recognized, not just some: there is no per-page text to tell the pages apart. +- Recognition is **CPU-bound** and costs roughly a tenth of a second per page — a one-page scan converts in about 155 ms after a ~20 ms engine startup, against ~5 ms for a text document. In a browser, run it in a Web Worker or the page freezes while it works. +- Building with `ocr` needs **Rust 1.92**; the crate without it keeps its 1.88 minimum. + +For hosted OCR with no models to manage, [Firecrawl Parse](https://firecrawl.dev) runs the same conversions as a service. ## Benchmark @@ -222,9 +279,10 @@ match anydoc::to_markdown(path) { | `Encrypted` | Encrypted or password-protected | | `ResourceLimit` | Crossed a fixed safety limit (decompression, nesting, node count) | | `MissingPart` | A part required for any meaningful output is absent | +| `Ocr` | OCR ran on every page carrying content and read no text | | `Io` | The file could not be read, from `to_markdown` only | -Node and wasm publish the variant name on `error.code`; Python raises one `anydoc.ConvertError` subclass per variant, or `OSError` when the file cannot be read. +Node and wasm publish the variant name on `error.code`; Python raises one `anydoc.ConvertError` subclass per named variant (a variant added later, like the OCR ones, raises the base class until it gets a name), or `OSError` when the file cannot be read. ## How it works diff --git a/node/Cargo.toml b/node/Cargo.toml index 406ef42..945307c 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -11,7 +11,8 @@ publish = false crate-type = ["cdylib"] [dependencies] -anydoc = { path = ".." } +# The published binaries ship OCR support; the models stay caller-supplied. +anydoc = { path = "..", features = ["ocr"] } napi = { version = "3", default-features = false, features = ["napi4"] } napi-derive = "3" diff --git a/node/README.md b/node/README.md index 5263150..20f0e46 100644 --- a/node/README.md +++ b/node/README.md @@ -3,7 +3,7 @@ [![npm](https://img.shields.io/npm/v/@firecrawl/anydoc.svg)](https://www.npmjs.com/package/@firecrawl/anydoc) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/firecrawl/anydoc/blob/main/LICENSE) -Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF files into clean GitHub-Flavored Markdown. Node.js bindings for the [anydoc](https://github.com/firecrawl/anydoc) Rust crate, built by [Firecrawl](https://firecrawl.dev). Also available as a hosted API through [Firecrawl Parse](https://firecrawl.dev/parse), which adds our OCR models for the scanned pages anydoc can't read on its own. +Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF files into clean GitHub-Flavored Markdown. Node.js bindings for the [anydoc](https://github.com/firecrawl/anydoc) Rust crate, built by [Firecrawl](https://firecrawl.dev). Scanned pages and image documents are read with [local OCR](#ocr-converter) when you supply the models. Also available as a hosted API through [Firecrawl Parse](https://firecrawl.dev/parse), which manages those models for you. Every format parses into one shared document model and renders through a single Markdown serializer, so headings, tables, lists, and footnotes come out the same no matter which format goes in. Conversion runs on the libuv thread pool and never blocks the event loop. TypeScript types ship with the package. @@ -54,6 +54,26 @@ const fromCsv = await toMarkdownBytes(bytes, 'csv'); const document = await toDocument(bytes); ``` +## OCR converter + +A scanned PDF page or an image document carries no text to extract, so reading one needs OCR. Supply the two [ocrs](https://github.com/robertknight/ocrs) models as bytes: they are parsed once, and every conversion made with that converter reuses the engine. Nothing is downloaded for you, and model parsing and conversion both run off the event loop. + +```js +import { readFile } from 'node:fs/promises'; +import { Converter } from '@firecrawl/anydoc'; + +const converter = await Converter.create({ + detectionModel: await readFile('text-detection.rten'), + recognitionModel: await readFile('text-recognition.rten'), +}); + +const markdown = await converter.toMarkdownBytes(await readFile('scan.pdf')); +``` + +Models that cannot be loaded reject with `code` `'ocrInit'`. The module-level functions never use OCR: through them a scanned PDF stays `'unsupported'`. See the [root README](../README.md#local-ocr-optional) for model digests, licensing, and what OCR can and cannot do. + +OCR is compiled into the published binary, so it carries the recognition and rendering code whether or not you build a `Converter`: about 4.75 MB on top of a 5.6 MB native library, roughly doubling it. + ## Errors A conversion rejects only when no meaningful Markdown could come out of the file. The rejection is an `Error` whose `code` names what went wrong: @@ -78,6 +98,8 @@ try { | `encrypted` | Encrypted or password-protected | | `resourceLimit` | Crossed a fixed safety limit (decompression, nesting, node count) | | `missingPart` | A part required for any meaningful output is absent | +| `ocr` | OCR ran on every page carrying content and read no text | +| `ocrInit` | The OCR models could not be loaded, from `Converter.create` only | | `io` | The file could not be read, from `toMarkdown` only | `error.message` carries the detail, naming the package part at fault where the format identifies one. TypeScript gets the union as `ConvertErrorCode`. diff --git a/node/cli.js b/node/cli.js index bfba762..a7fbf9c 100644 --- a/node/cli.js +++ b/node/cli.js @@ -27,7 +27,8 @@ Options: The format is detected from the file content; the file extension is the fallback for signature-less formats (CSV). stdin has no extension, so CSV input from stdin needs --format csv. Scanned or image-only PDFs need OCR, -which anydoc does not do, and error as unsupported. +which this CLI does not carry, and error as unsupported; the library API +reads them when you build a Converter with OCR models. Exit codes: 0 success diff --git a/node/dts-header.d.ts b/node/dts-header.d.ts index 8f30c14..617e537 100644 --- a/node/dts-header.d.ts +++ b/node/dts-header.d.ts @@ -18,3 +18,12 @@ export type ConvertErrorCode = | 'missingPart' /** The file could not be read, from `toMarkdown` only. */ | 'io' + /** + * Local OCR was required for every page carrying content and recovered no + * text from any of them. Only a `Converter` conversion produces this. + */ + | 'ocr' + /** + * The OCR models could not be loaded, from `Converter.create` only. + */ + | 'ocrInit' diff --git a/node/index.d.ts b/node/index.d.ts index 27ceba0..a3822ed 100644 --- a/node/index.d.ts +++ b/node/index.d.ts @@ -18,6 +18,42 @@ export type ConvertErrorCode = | 'missingPart' /** The file could not be read, from `toMarkdown` only. */ | 'io' + /** + * Local OCR was required for every page carrying content and recovered no + * text from any of them. Only a `Converter` conversion produces this. + */ + | 'ocr' + /** + * The OCR models could not be loaded, from `Converter.create` only. + */ + | 'ocrInit' +/** + * A reusable converter that carries local OCR. + * + * The models are parsed once, when the converter is created, and every + * conversion made with it shares that engine; nothing is rebuilt per call. + * The module-level functions keep their own behavior, which never uses OCR. + */ +export declare class Converter { + /** + * Parse the OCR models and build a converter that reuses them. Parsing + * runs off the event loop. + * + * Rejects with an `Error` carrying a `ConvertErrorCode` on `code`: + * `'ocrInit'` when the models cannot be loaded. + */ + static create(models: OcrModels): Promise + /** + * Convert an in-memory document to Markdown, recognizing the pages that + * need OCR. Without a format, it is detected from the content, which + * signature-less formats (CSV) have to name explicitly. Conversion runs + * off the event loop. + * + * Rejects with an `Error` carrying a `ConvertErrorCode` on `code`. + */ + toMarkdownBytes(bytes: Uint8Array, format?: Format | undefined | null): Promise +} + /** * An embedded binary asset (image, object payload). Bytes are always * retained, so a document stays self-contained. @@ -102,8 +138,9 @@ export declare const enum Format { odt = 'odt', /** * Converted with pdf-inspector, which emits Markdown directly: - * `toDocument` is unsupported for PDFs. Scanned or image-only PDFs - * (needing OCR) error as unsupported. + * `toDocument` is unsupported for PDFs. Pages with no extractable text + * need OCR: the module-level functions report them as unsupported, + * while a `Converter` recognizes exactly those pages. */ pdf = 'pdf', ppt = 'ppt', @@ -113,7 +150,13 @@ export declare const enum Format { xlsx = 'xlsx', ods = 'ods', odp = 'odp', - csv = 'csv' + csv = 'csv', + /** + * Raster image documents (PNG, JPEG, WebP, TIFF, BMP). Recognized with + * local OCR where it is configured, and unsupported otherwise; + * `toDocument` is unsupported for images the same way it is for PDFs. + */ + image = 'image' } /** @@ -235,6 +278,17 @@ export declare const enum NoteKind { endnote = 'endnote' } +/** + * The RTen models local OCR needs, as bytes the caller supplies. anydoc + * performs no download and no filesystem access of its own. + */ +export interface OcrModels { + /** The text-detection model. */ + detectionModel: Uint8Array + /** The text-recognition model. */ + recognitionModel: Uint8Array +} + /** Fully resolved character style. */ export interface Style { bold: boolean diff --git a/node/index.js b/node/index.js index ca22b8b..e77dd8b 100644 --- a/node/index.js +++ b/node/index.js @@ -700,6 +700,7 @@ if (!nativeBinding) { } module.exports = nativeBinding +module.exports.Converter = nativeBinding.Converter module.exports.BlockKind = nativeBinding.BlockKind module.exports.CellSlotKind = nativeBinding.CellSlotKind module.exports.Format = nativeBinding.Format diff --git a/node/src/lib.rs b/node/src/lib.rs index 01df83a..542cce3 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -1,5 +1,7 @@ //! Node.js bindings for anydoc. +use std::sync::Arc; + use napi::bindgen_prelude::*; use napi_derive::napi; @@ -18,8 +20,9 @@ pub enum Format { docx, odt, /// Converted with pdf-inspector, which emits Markdown directly: - /// `toDocument` is unsupported for PDFs. Scanned or image-only PDFs - /// (needing OCR) error as unsupported. + /// `toDocument` is unsupported for PDFs. Pages with no extractable text + /// need OCR: the module-level functions report them as unsupported, + /// while a `Converter` recognizes exactly those pages. pdf, ppt, pptx, @@ -29,6 +32,10 @@ pub enum Format { ods, odp, csv, + /// Raster image documents (PNG, JPEG, WebP, TIFF, BMP). Recognized with + /// local OCR where it is configured, and unsupported otherwise; + /// `toDocument` is unsupported for images the same way it is for PDFs. + image, } impl From for anydoc::Format { @@ -46,6 +53,7 @@ impl From for anydoc::Format { Format::ods => anydoc::Format::Ods, Format::odp => anydoc::Format::Odp, Format::csv => anydoc::Format::Csv, + Format::image => anydoc::Format::Image, } } } @@ -65,6 +73,7 @@ impl From for Format { anydoc::Format::Ods => Format::ods, anydoc::Format::Odp => Format::odp, anydoc::Format::Csv => Format::csv, + anydoc::Format::Image => Format::image, } } } @@ -134,6 +143,118 @@ pub fn to_document(bytes: Uint8Array, format: Option) -> AsyncTask, +} + +#[napi] +impl Converter { + /// Parse the OCR models and build a converter that reuses them. Parsing + /// runs off the event loop. + /// + /// Rejects with an `Error` carrying a `ConvertErrorCode` on `code`: + /// `'ocrInit'` when the models cannot be loaded. + #[napi(ts_return_type = "Promise")] + pub fn create(models: OcrModels) -> AsyncTask { + AsyncTask::new(ConverterTask { + detection_model: models.detection_model.to_vec(), + recognition_model: models.recognition_model.to_vec(), + failure: Failure::default(), + }) + } + + /// Convert an in-memory document to Markdown, recognizing the pages that + /// need OCR. Without a format, it is detected from the content, which + /// signature-less formats (CSV) have to name explicitly. Conversion runs + /// off the event loop. + /// + /// Rejects with an `Error` carrying a `ConvertErrorCode` on `code`. + #[napi(ts_return_type = "Promise")] + pub fn to_markdown_bytes( + &self, + bytes: Uint8Array, + format: Option, + ) -> AsyncTask { + AsyncTask::new(ConverterMarkdownTask { + converter: Arc::clone(&self.inner), + bytes: bytes.to_vec(), + format: format.map(Into::into), + failure: Failure::default(), + }) + } +} + +pub struct ConverterTask { + detection_model: Vec, + recognition_model: Vec, + failure: Failure, +} + +impl Task for ConverterTask { + type Output = anydoc::Converter; + type JsValue = Converter; + + fn compute(&mut self) -> Result { + let detection = std::mem::take(&mut self.detection_model); + let recognition = std::mem::take(&mut self.recognition_model); + let builder = anydoc::Converter::builder() + .with_ocr_models(detection, recognition) + .map_err(|e| self.failure.capture_init(e))?; + Ok(builder.build()) + } + + fn resolve(&mut self, _env: Env, output: Self::Output) -> Result { + Ok(Converter { inner: Arc::new(output) }) + } + + fn reject(&mut self, env: Env, error: Error) -> Result { + Err(self.failure.reject(env, error)) + } +} + +pub struct ConverterMarkdownTask { + converter: Arc, + bytes: Vec, + format: Option, + failure: Failure, +} + +impl Task for ConverterMarkdownTask { + type Output = String; + type JsValue = String; + + fn compute(&mut self) -> Result { + self.converter + .to_markdown_bytes(&self.bytes, self.format) + .map_err(|e| self.failure.capture(e)) + } + + fn resolve(&mut self, _env: Env, output: Self::Output) -> Result { + Ok(output) + } + + fn reject(&mut self, env: Env, error: Error) -> Result { + Err(self.failure.reject(env, error)) + } +} + /// The kind of a failed conversion, held between the two threads a rejection /// crosses: `compute` runs on the libuv pool, where there is no `Env` to build /// a JS error with, and `reject` runs on the JS thread, where there is. @@ -147,6 +268,13 @@ impl Failure { Error::from_reason(error.to_string()) } + /// Loading OCR models fails before any conversion starts, so it carries + /// its own code rather than one of `ConvertError`'s. + fn capture_init(&mut self, error: anydoc::OcrInitError) -> Error { + self.0 = Some("ocrInit"); + Error::from_reason(error.to_string()) + } + /// Rebuild the rejection as an error whose `code` is the `ConvertError` /// kind. napi fills `code` from the error's status, so the status here is /// a plain string rather than the `Status` enum it defaults to. Anything diff --git a/node/test.mjs b/node/test.mjs index ec98a3c..e33d861 100644 --- a/node/test.mjs +++ b/node/test.mjs @@ -9,6 +9,7 @@ import { test } from 'node:test' import { promisify } from 'node:util' import { + Converter, formatFromBytes, formatFromExtension, formatFromPath, @@ -139,3 +140,57 @@ test('cli help and version go to stdout and exit 0', async () => { const version = await runCli(['--version']) assert.match(version.stdout.trim(), /^\d+\.\d+\.\d+/) }) + +// The OCR models are external files, so the converter tests only run where +// they are configured; everywhere else they report as skipped. +const DETECTION_MODEL = process.env.ANYDOC_OCR_DETECTION_MODEL +const RECOGNITION_MODEL = process.env.ANYDOC_OCR_RECOGNITION_MODEL +const withoutModels = DETECTION_MODEL && RECOGNITION_MODEL + ? false + : 'set ANYDOC_OCR_DETECTION_MODEL and ANYDOC_OCR_RECOGNITION_MODEL to run' + +const readModels = async () => + Promise.all([readFile(DETECTION_MODEL), readFile(RECOGNITION_MODEL)]) + +test('Converter.create parses the models once and reuses them', { skip: withoutModels }, async () => { + const [detectionModel, recognitionModel] = await readModels() + const converter = await Converter.create({ detectionModel, recognitionModel }) + + const first = await converter.toMarkdownBytes(await readFile(OUTLINE)) + const second = await converter.toMarkdownBytes(await readFile(CSV), 'csv') + + assert.match(first, /^# /m) + assert.match(second, /\| --- \|/) +}) + +test('a converter conversion matches the free function where no OCR is needed', { skip: withoutModels }, async () => { + const [detectionModel, recognitionModel] = await readModels() + const converter = await Converter.create({ detectionModel, recognitionModel }) + const bytes = await readFile(RICH) + + assert.equal(await converter.toMarkdownBytes(bytes), await toMarkdownBytes(bytes)) +}) + +test('a converter rejects unconvertible input with a coded Error', { skip: withoutModels }, async () => { + const [detectionModel, recognitionModel] = await readModels() + const converter = await Converter.create({ detectionModel, recognitionModel }) + + await assert.rejects(converter.toMarkdownBytes(await readFile(ENCRYPTED), 'odt'), (error) => { + assert.equal(error.code, 'encrypted') + return true + }) +}) + +test('Converter.create rejects invalid model bytes with a coded Error', async () => { + await assert.rejects( + Converter.create({ + detectionModel: Buffer.from([0, 1, 2, 3]), + recognitionModel: Buffer.from([4, 5, 6, 7]), + }), + (error) => { + assert.equal(error.code, 'ocrInit') + assert.match(error.message, /detection model/) + return true + }, + ) +}) diff --git a/python/README.md b/python/README.md index fa97971..76b755d 100644 --- a/python/README.md +++ b/python/README.md @@ -3,7 +3,10 @@ [![PyPI](https://img.shields.io/pypi/v/firecrawl-anydoc.svg)](https://pypi.org/project/firecrawl-anydoc/) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/firecrawl/anydoc/blob/main/LICENSE) -Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF files into clean GitHub-Flavored Markdown. Python bindings for the [anydoc](https://github.com/firecrawl/anydoc) Rust crate, built by [Firecrawl](https://firecrawl.dev). Also available as a hosted API through [Firecrawl Parse](https://firecrawl.dev/parse), which adds our OCR models for the scanned pages anydoc can't read on its own. +Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF files into clean GitHub-Flavored Markdown. Python bindings for the [anydoc](https://github.com/firecrawl/anydoc) Rust crate, built by [Firecrawl](https://firecrawl.dev). Also available as a hosted API through [Firecrawl Parse](https://firecrawl.dev/parse), which adds OCR on top of the same conversion. + +OCR is available in the Rust, Node and wasm packages; this binding +does not expose it yet. Every format parses into one shared document model and renders through a single Markdown serializer, so headings, tables, lists, and footnotes come out the same no matter which format goes in. Conversion releases the GIL, so other threads keep running. Type stubs ship with the package. @@ -66,7 +69,7 @@ except (anydoc.EncryptedError, anydoc.UnsupportedError) as error: | `MissingPartError` | A part required for any meaningful output is absent | | `OSError` | The file could not be read, from `to_markdown` only | -The five conversion failures subclass `anydoc.ConvertError`, so catching that handles all of them at once. `MalformedError.part` and `MissingPartError.part` name the package part at fault, `ResourceLimitError.limit` names the limit crossed, and `str(error)` carries the whole message. A `format` argument naming no supported format raises `ValueError`. +Every conversion failure subclasses `anydoc.ConvertError`, so catching that handles all of them at once. `MalformedError.part` and `MissingPartError.part` name the package part at fault, `ResourceLimitError.limit` names the limit crossed, and `str(error)` carries the whole message. A `format` argument naming no supported format raises `ValueError`. ## Format detection diff --git a/python/anydoc/__init__.py b/python/anydoc/__init__.py index f7c01d7..8e41217 100644 --- a/python/anydoc/__init__.py +++ b/python/anydoc/__init__.py @@ -31,7 +31,8 @@ ) Format = Literal[ - "doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv" + "doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv", + "image" ] """Input format, named after the extension that identifies it. Container variants that share a parser (`.docm`, `.xlsm`, `.ppsx`, ...) map onto these diff --git a/python/anydoc/_anydoc.pyi b/python/anydoc/_anydoc.pyi index 19dadba..ae63561 100644 --- a/python/anydoc/_anydoc.pyi +++ b/python/anydoc/_anydoc.pyi @@ -4,7 +4,8 @@ import os from typing import Literal, final Format = Literal[ - "doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv" + "doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv", + "image" ] class ConvertError(Exception): @@ -14,7 +15,7 @@ class ConvertError(Exception): class UnsupportedError(ConvertError): """The format is unknown, or cannot be converted at all: a scanned or - image-only PDF needs OCR, which anydoc does not do.""" + image-only PDF needs OCR, which this binding does not expose yet.""" class MalformedError(ConvertError): """The document is structurally unusable: no meaningful content could be diff --git a/python/src/lib.rs b/python/src/lib.rs index abcbbd6..53e8e1c 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -22,7 +22,7 @@ create_exception!( UnsupportedError, ConvertError, "The format is unknown, or cannot be converted at all: a scanned or \ - image-only PDF needs OCR, which anydoc does not do." + image-only PDF needs OCR, which this binding does not expose yet." ); create_exception!( @@ -59,7 +59,7 @@ create_exception!( /// Format names, as the extension that identifies each format. Container /// variants that share a parser (`.docm`, `.xlsm`, `.ppsx`, ...) map onto /// these via `format_from_bytes` or `format_from_extension`. -const FORMATS: [(&str, anydoc::Format); 12] = [ +const FORMATS: [(&str, anydoc::Format); 13] = [ ("doc", anydoc::Format::Doc), ("docx", anydoc::Format::Docx), ("odt", anydoc::Format::Odt), @@ -72,6 +72,7 @@ const FORMATS: [(&str, anydoc::Format); 12] = [ ("ods", anydoc::Format::Ods), ("odp", anydoc::Format::Odp), ("csv", anydoc::Format::Csv), + ("image", anydoc::Format::Image), ]; fn parse_format(name: &str) -> PyResult { diff --git a/skills/convert-documents-to-markdown/SKILL.md b/skills/convert-documents-to-markdown/SKILL.md index d4a0591..9306660 100644 --- a/skills/convert-documents-to-markdown/SKILL.md +++ b/skills/convert-documents-to-markdown/SKILL.md @@ -22,5 +22,5 @@ Rules: 2. The format is detected from the file content. Pass `--format ` only when detection cannot work: CSV from stdin, or a missing or wrong extension. 3. Exit codes: 0 success, 1 the document could not be converted, 2 usage error. Failures print one `anydoc: ` line to stderr. The CLI never prompts. 4. For a large document, write to a file with `-o` and read the parts you need instead of streaming everything into context. -5. Scanned and image-only PDFs need OCR, which anydoc does not do; they fail as unsupported. The hosted [Firecrawl Parse](https://firecrawl.dev/parse) API handles those. +5. Scanned and image-only PDFs need OCR, which this CLI does not carry; they fail as unsupported. The Rust, Node, and browser APIs can read them with local OCR when the caller supplies models, and the hosted [Firecrawl Parse](https://firecrawl.dev/parse) API handles them with no setup. 6. Inside a Node, Python, or Rust codebase, prefer the library over shelling out: `@firecrawl/anydoc` on npm, `firecrawl-anydoc` on PyPI, `anydoc` on crates.io. Each exposes the same `to_markdown` / `toMarkdown` API. diff --git a/src/error.rs b/src/error.rs index 37c8a67..9acbde5 100644 --- a/src/error.rs +++ b/src/error.rs @@ -38,6 +38,12 @@ pub enum ConvertError { /// The part or stream that was absent. part: String, }, + /// Local OCR was required for every page carrying content and recovered + /// no text from any of them. + Ocr { + /// What was recognized or rendered, and why nothing usable remained. + reason: String, + }, /// The input could not be read. Io(std::io::Error), } @@ -57,6 +63,7 @@ impl fmt::Display for ConvertError { write!(f, "resource limit exceeded ({limit}): {detail}") } ConvertError::MissingPart { part } => write!(f, "missing required part: {part}"), + ConvertError::Ocr { reason } => write!(f, "OCR recovered no text: {reason}"), ConvertError::Io(e) => write!(f, "io error: {e}"), } } @@ -88,6 +95,7 @@ impl ConvertError { ConvertError::Encrypted => "encrypted", ConvertError::ResourceLimit { .. } => "resourceLimit", ConvertError::MissingPart { .. } => "missingPart", + ConvertError::Ocr { .. } => "ocr", ConvertError::Io(_) => "io", } } @@ -122,6 +130,7 @@ mod tests { let limit = ConvertError::ResourceLimit { limit: "max_entry_bytes", detail: String::new() }; assert_eq!(limit.code(), "resourceLimit"); assert_eq!(ConvertError::MissingPart { part: String::new() }.code(), "missingPart"); + assert_eq!(ConvertError::Ocr { reason: String::new() }.code(), "ocr"); assert_eq!(ConvertError::Io(std::io::ErrorKind::NotFound.into()).code(), "io"); } } diff --git a/src/formats/detect.rs b/src/formats/detect.rs index 727a69f..b4305c7 100644 --- a/src/formats/detect.rs +++ b/src/formats/detect.rs @@ -14,6 +14,9 @@ //! type of the part the package-level officeDocument relationship //! designates as the main document (with the main part's mandated root //! element as the authority when content types are stale or generic). +//! - Raster images: the signature each codec mandates (PNG, JPEG, WebP, +//! TIFF, BMP). They are checked last, after every container, and all map +//! to one format: the decoder identifies the codec itself. //! //! Plain-text formats (CSV) carry no signature and are never detected; //! callers fall back to the file extension. Detection never errors: any @@ -44,7 +47,35 @@ pub(crate) fn from_bytes(bytes: &[u8]) -> Option { if bytes[..bytes.len().min(1024)].windows(5).any(|w| w == b"%PDF-") { return Some(Format::Pdf); } - None + detect_image(bytes) +} + +/// Raster image signatures, checked after every container. They share one +/// format: which codec it is only matters to the decoder. Detection runs +/// ahead of the extension fallback, so a signature that also matches +/// ordinary text would take formats like CSV away from it. +fn detect_image(bytes: &[u8]) -> Option { + let is_image = bytes.starts_with(b"\x89PNG\r\n\x1a\n") + || bytes.starts_with(&[0xFF, 0xD8, 0xFF]) + || (bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP")) + || bytes.starts_with(b"II*\0") + || bytes.starts_with(b"MM\0*") + || is_bmp(bytes); + is_image.then_some(Format::Image) +} + +/// `BM` alone is two ordinary letters, and text files do start with them +/// (`BMI,weight,height`), so the DIB header behind it has to be one of the +/// sizes the format specifies before this claims a bitmap. +fn is_bmp(bytes: &[u8]) -> bool { + const DIB_HEADER_SIZES: [u32; 7] = [12, 40, 52, 56, 64, 108, 124]; + + bytes.starts_with(b"BM") + && bytes + .get(14..18) + .and_then(|size| size.try_into().ok()) + .map(u32::from_le_bytes) + .is_some_and(|size| DIB_HEADER_SIZES.contains(&size)) } /// Classify an OLE compound file by its mandated content stream. Encrypted @@ -240,6 +271,47 @@ mod tests { assert_eq!(from_bytes(b""), None); } + /// `BM`, then a file header and a DIB header of `dib_size` bytes. + fn bmp_of(dib_size: u32) -> Vec { + let mut bytes = vec![0_u8; 14]; + bytes[..2].copy_from_slice(b"BM"); + bytes.extend_from_slice(&dib_size.to_le_bytes()); + bytes.resize(14 + dib_size as usize, 0); + bytes + } + + #[test] + fn image_codecs_are_identified_by_their_signature() { + for signature in [ + b"\x89PNG\r\n\x1a\n".as_slice(), + &[0xFF, 0xD8, 0xFF, 0xE0], + b"RIFF\x24\0\0\0WEBPVP8 ", + b"II*\0\x08\0\0\0", + b"MM\0*\0\0\0\x08", + &bmp_of(40), + &bmp_of(124), + ] { + assert_eq!(from_bytes(signature), Some(Format::Image), "{signature:?}"); + } + } + + #[test] + fn near_misses_are_not_images() { + // `BM` needs a DIB header of a specified size behind it. A CSV whose + // first column is BMI used to convert and has to keep converting. + assert_eq!(from_bytes(b"BMI,weight,height\n22.5,70,1.76\n21.0,65,1.76\n"), None); + assert_eq!(from_bytes(&bmp_of(41)), None); + assert_eq!(from_bytes(b"BM"), None); + // RIFF containers that are not WebP (this one is a WAV). + assert_eq!(from_bytes(b"RIFF\x24\0\0\0WAVEfmt "), None); + assert_eq!(from_bytes(b"RIFF"), None); + // One byte off each remaining signature. + assert_eq!(from_bytes(b"\x89PNG\r\n\x1a\x00"), None); + assert_eq!(from_bytes(&[0xFF, 0xD8, 0xFE, 0xE0]), None); + assert_eq!(from_bytes(b"II\0*\x08\0\0\0"), None); + assert_eq!(from_bytes(b"MM*\0\0\0\0\x08"), None); + } + #[test] fn container_signature_wins_over_an_early_embedded_pdf() { let pkg = zip_of(&[("embedded.pdf", b"%PDF-1.7\n"), ("word/document.xml", b"")]); diff --git a/src/formats/image.rs b/src/formats/image.rs new file mode 100644 index 0000000..3b05c07 --- /dev/null +++ b/src/formats/image.rs @@ -0,0 +1,289 @@ +//! Standalone raster images, read with local OCR. +//! +//! An image document carries no text layer, so it converts only when the +//! converter has an OCR engine; the crate reports it as unsupported +//! otherwise. Decoding is bounded before any pixel buffer exists: the header +//! alone decides whether the image is within the fixed dimension and area +//! limits below. Recognized text follows the same output policy as +//! recognized PDF pages. +//! +//! EXIF orientation is not applied, so an image stored rotated is recognized +//! in the orientation it is stored in. + +use crate::error::ConvertError; +use crate::ocr::PageOcr; +use image::error::{ImageError, LimitErrorKind}; +use image::{ImageReader, Limits}; +use std::io::Cursor; + +/// Widest and tallest image accepted, in pixels. +const MAX_DIMENSION: u32 = 16_384; + +/// Largest image area accepted, in pixels. +const MAX_PIXELS: u64 = 25_000_000; + +pub fn to_markdown_with_ocr( + bytes: &[u8], + engine: &crate::ocr::Engine, +) -> Result { + to_markdown_with_page_ocr(bytes, engine) +} + +fn to_markdown_with_page_ocr(bytes: &[u8], ocr: &O) -> Result { + // Check the header first, so a huge image is rejected before to decode + // anything. + let (width, height) = reader(bytes)?.into_dimensions().map_err(map_error)?; + let pixels = u64::from(width) + .checked_mul(u64::from(height)) + .filter(|&pixels| pixels <= MAX_PIXELS) + .ok_or_else(|| ConvertError::ResourceLimit { + limit: "max_image_pixels", + detail: format!("{width}x{height} exceeds {MAX_PIXELS} pixels"), + })?; + log::debug!("recognizing a {width}x{height} image ({pixels} pixels)"); + + let decoded = reader(bytes)?.decode().map_err(map_error)?.into_rgba8(); + let (width, height) = (decoded.width(), decoded.height()); + let recognized = ocr + .recognize_page(width, height, decoded.as_raw()) + .map_err(|error| ConvertError::Ocr { reason: error.to_string() })?; + + let lines = super::pdf::recognized_lines(&recognized); + if lines.is_empty() { + log::debug!("no text was recognized in the image"); + return Ok(String::new()); + } + let mut markdown = lines.join("\n"); + markdown.push('\n'); + Ok(markdown) +} + +/// A reader with the codec guessed from the signature and the limits set. +fn reader(bytes: &[u8]) -> Result>, ConvertError> { + let mut reader = ImageReader::new(Cursor::new(bytes)) + .with_guessed_format() + .map_err(|error| ConvertError::malformed(format!("unreadable image: {error}")))?; + let mut limits = Limits::default(); + limits.max_image_width = Some(MAX_DIMENSION); + limits.max_image_height = Some(MAX_DIMENSION); + reader.limits(limits); + Ok(reader) +} + +fn map_error(error: ImageError) -> ConvertError { + match error { + ImageError::Limits(error) => ConvertError::ResourceLimit { + limit: match error.kind() { + LimitErrorKind::DimensionError => "max_image_dimension", + _ => "max_image_alloc", + }, + detail: format!("image exceeds the decoder limits: {error}"), + }, + ImageError::Unsupported(error) => { + ConvertError::Unsupported(format!("image codec: {error}")) + } + // Truncated or corrupt bytes arrive as a decoding or IO error, and + // either way the input is at fault, not the environment. + error => ConvertError::malformed(format!("undecodable image: {error}")), + } +} + +#[cfg(test)] +mod tests { + use super::{MAX_DIMENSION, to_markdown_with_page_ocr}; + use crate::error::ConvertError; + use crate::ocr::{OcrPageError, PageOcr}; + use image::{DynamicImage, ImageFormat, RgbaImage}; + use std::cell::RefCell; + use std::io::Cursor; + + /// Every codec the `ocr` feature enables. + const CODECS: [ImageFormat; 5] = [ + ImageFormat::Png, + ImageFormat::Jpeg, + ImageFormat::WebP, + ImageFormat::Tiff, + ImageFormat::Bmp, + ]; + + /// Records what it was handed and replays a scripted result. + struct FakeOcr { + result: Result, ()>, + images: RefCell>, + } + + impl FakeOcr { + fn recognizing(lines: &[&str]) -> Self { + let lines = lines.iter().map(|line| (*line).to_string()).collect(); + Self { result: Ok(lines), images: RefCell::new(Vec::new()) } + } + + fn failing() -> Self { + Self { result: Err(()), images: RefCell::new(Vec::new()) } + } + + fn images(&self) -> Vec<(u32, u32)> { + self.images.borrow().clone() + } + } + + impl PageOcr for FakeOcr { + fn recognize_page( + &self, + width: u32, + height: u32, + rgba: &[u8], + ) -> Result, OcrPageError> { + assert_eq!(rgba.len(), width as usize * height as usize * 4); + self.images.borrow_mut().push((width, height)); + self.result + .clone() + .map_err(|()| OcrPageError::Recognition { detail: "fake failure".into() }) + } + } + + /// A 3x2 image with a distinct pixel per position, encoded with `format`. + fn sample_image(format: ImageFormat) -> Vec { + let mut pixels = RgbaImage::new(3, 2); + for (index, pixel) in pixels.pixels_mut().enumerate() { + let step = index as u8 * 40; + *pixel = image::Rgba([step, 255 - step, 128 + step / 2, 255]); + } + let sample = match format { + // JPEG has no alpha channel. + ImageFormat::Jpeg => { + DynamicImage::ImageRgb8(DynamicImage::ImageRgba8(pixels).to_rgb8()) + } + _ => DynamicImage::ImageRgba8(pixels), + }; + let mut encoded = Cursor::new(Vec::new()); + sample.write_to(&mut encoded, format).expect("encode the sample image"); + encoded.into_inner() + } + + #[test] + fn every_enabled_codec_reaches_the_engine_with_its_decoded_pixels() { + for format in CODECS { + let ocr = FakeOcr::recognizing(&["Recognized line"]); + let markdown = to_markdown_with_page_ocr(&sample_image(format), &ocr) + .unwrap_or_else(|error| panic!("{format:?} failed: {error}")); + + assert_eq!(ocr.images(), [(3, 2)], "{format:?}"); + assert_eq!(markdown, "Recognized line\n", "{format:?}"); + } + } + + /// Every codec this crate decodes is also detected from its signature, so + /// no image document depends on its file extension. + #[test] + fn every_enabled_codec_is_detected_from_its_bytes() { + for format in CODECS { + assert_eq!( + crate::Format::from_bytes(&sample_image(format)), + Some(crate::Format::Image), + "{format:?}" + ); + } + } + + #[test] + fn recognized_text_follows_the_shared_output_policy() { + let ocr = FakeOcr::recognizing(&["# not a heading", "", "1. not a list"]); + let markdown = to_markdown_with_page_ocr(&sample_image(ImageFormat::Png), &ocr).unwrap(); + + assert_eq!(markdown, "\\# not a heading\n1\\. not a list\n"); + } + + #[test] + fn an_image_without_text_converts_to_nothing() { + let ocr = FakeOcr::recognizing(&[]); + let markdown = to_markdown_with_page_ocr(&sample_image(ImageFormat::Png), &ocr).unwrap(); + + assert_eq!(markdown, ""); + } + + #[test] + fn a_failed_recognition_is_terminal_for_a_single_page_document() { + let ocr = FakeOcr::failing(); + let result = to_markdown_with_page_ocr(&sample_image(ImageFormat::Png), &ocr); + + assert!(matches!(result, Err(ConvertError::Ocr { .. })), "{result:?}"); + } + + #[test] + fn malformed_bytes_are_a_typed_error() { + let ocr = FakeOcr::recognizing(&["unexpected"]); + let mut truncated = sample_image(ImageFormat::Png); + truncated.truncate(20); + let result = to_markdown_with_page_ocr(&truncated, &ocr); + + assert!(matches!(result, Err(ConvertError::Malformed { .. })), "{result:?}"); + assert!(ocr.images().is_empty()); + } + + /// The declared dimensions are rejected from the header alone, so no + /// buffer for 30000x30000 pixels is ever requested. + #[test] + fn oversized_dimensions_are_rejected_before_decoding() { + let ocr = FakeOcr::recognizing(&["unexpected"]); + let result = to_markdown_with_page_ocr(&png_declaring(30_000, 30_000), &ocr); + + let Err(ConvertError::ResourceLimit { limit, .. }) = result else { + panic!("an oversized image was accepted: {result:?}"); + }; + assert_eq!(limit, "max_image_dimension"); + assert!(ocr.images().is_empty()); + } + + /// Within the per-side limit, the area limit still rejects the image. + #[test] + fn oversized_area_is_rejected_before_decoding() { + let ocr = FakeOcr::recognizing(&["unexpected"]); + let result = to_markdown_with_page_ocr(&png_declaring(MAX_DIMENSION, MAX_DIMENSION), &ocr); + + let Err(ConvertError::ResourceLimit { limit, .. }) = result else { + panic!("an oversized image was accepted: {result:?}"); + }; + assert_eq!(limit, "max_image_pixels"); + assert!(ocr.images().is_empty()); + } + + /// A PNG whose header declares this size over an empty image body. The + /// limits act on the header, so the declared size costs nothing to build. + fn png_declaring(width: u32, height: u32) -> Vec { + let mut ihdr = Vec::from(b"IHDR".as_slice()); + ihdr.extend_from_slice(&width.to_be_bytes()); + ihdr.extend_from_slice(&height.to_be_bytes()); + // Bit depth 8, colour type 6 (RGBA), deflate, no filter, no interlace. + ihdr.extend_from_slice(&[8, 6, 0, 0, 0]); + + let mut png = Vec::from(b"\x89PNG\r\n\x1a\n".as_slice()); + push_chunk(&mut png, &ihdr); + // An empty deflate stream; nothing here is ever inflated. + push_chunk(&mut png, b"IDAT\x78\x01\x03\x00\x00\x00\x00\x01"); + push_chunk(&mut png, b"IEND"); + png + } + + /// Append a PNG chunk: its length, the type and data, then the CRC. + fn push_chunk(png: &mut Vec, chunk: &[u8]) { + let data_len = chunk.len() - b"IHDR".len(); + png.extend_from_slice(&(data_len as u32).to_be_bytes()); + png.extend_from_slice(chunk); + png.extend_from_slice(&crc32(chunk).to_be_bytes()); + } + + /// PNG chunk CRC-32 (IEEE, reflected), computed directly so the fixture + /// needs no dependency. + fn crc32(bytes: &[u8]) -> u32 { + let mut crc = u32::MAX; + for byte in bytes { + crc ^= u32::from(*byte); + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xEDB8_8320 & mask); + } + } + !crc + } +} diff --git a/src/formats/mod.rs b/src/formats/mod.rs index d570b26..dce2ccd 100644 --- a/src/formats/mod.rs +++ b/src/formats/mod.rs @@ -5,6 +5,8 @@ pub mod detect; mod doc; mod docx; mod epub; +#[cfg(feature = "ocr")] +pub mod image; mod odf; pub mod pdf; mod ppt; @@ -34,5 +36,10 @@ pub fn parse(bytes: &[u8], format: Format) -> Result { Format::Pdf => Err(ConvertError::Unsupported( "PDF converts directly to Markdown; use to_markdown or to_markdown_bytes".to_string(), )), + // Recognized text is Markdown already; an image has no document + // model. `to_markdown_bytes` routes them to `image`. + Format::Image => Err(ConvertError::Unsupported( + "image converts directly to Markdown; use to_markdown or to_markdown_bytes".to_string(), + )), } } diff --git a/src/formats/pdf.rs b/src/formats/pdf.rs index 74a36e1..9c32d2d 100644 --- a/src/formats/pdf.rs +++ b/src/formats/pdf.rs @@ -1,15 +1,31 @@ //! PDF via [pdf-inspector]: classification plus direct Markdown extraction. //! //! Unlike the other frontends, pdf-inspector emits Markdown itself, so PDFs -//! bypass the document model and the shared GFM writer. Scanned and -//! image-only PDFs need OCR, which is out of scope here; they error as -//! unsupported. Pages flagged for OCR in an otherwise text-based document -//! degrade with a log, consistent with the crate-wide recovery policy. +//! bypass the document model and the shared GFM writer. Without a configured +//! OCR engine the pages it routes to OCR stay unavailable: image-only PDFs +//! error as unsupported, and pages flagged in an otherwise text-based document +//! degrade with a log, consistent with the crate-wide recovery policy. With an +//! engine, exactly those pages are rasterized one at a time and recognized, +//! then merged back into the untouched structured pages in source order. A +//! document classified as scanned that extracts to no text at all is read in +//! full instead, page by page: the little text such a page carries is usually +//! a watermark rather than its content, so the richer of the two wins. //! //! [pdf-inspector]: https://github.com/firecrawl/pdf-inspector use crate::error::ConvertError; -use pdf_inspector::PdfError; +use pdf_inspector::{PdfError, PdfProcessResult}; + +#[cfg(feature = "ocr")] +use crate::ocr::PageOcr; +#[cfg(feature = "ocr")] +use pdf_inspector::{PageMarkdown, PdfType, RenderOptions, RenderWarning}; +#[cfg(feature = "ocr")] +use std::collections::BTreeMap; + +#[cfg(all(test, feature = "ocr"))] +#[path = "../../tests/support/pdf_fixture.rs"] +mod pdf_fixture; pub fn to_markdown(bytes: &[u8]) -> Result { let result = pdf_inspector::process_pdf_mem(bytes).map_err(map_error)?; @@ -20,9 +36,221 @@ pub fn to_markdown(bytes: &[u8]) -> Result { result.page_count ); } - if result.has_encoding_issues { - log::warn!("broken font encodings detected; extracted text may be garbled"); + warn_on_encoding_issues(&result); + extracted_markdown(result) +} + +/// Convert with local OCR for the pages that need it: the ones pdf-inspector +/// flags, or every page of a scanned document that extracts to nothing. A +/// document with no such page produces the same bytes as [`to_markdown`]. +#[cfg(feature = "ocr")] +pub fn to_markdown_with_ocr( + bytes: &[u8], + engine: &crate::ocr::Engine, +) -> Result { + to_markdown_with_page_ocr(bytes, engine) +} + +#[cfg(feature = "ocr")] +fn to_markdown_with_page_ocr(bytes: &[u8], ocr: &O) -> Result { + let result = pdf_inspector::process_pdf_mem(bytes).map_err(map_error)?; + warn_on_encoding_issues(&result); + if result.pdf_type == PdfType::TextBased { + return extracted_markdown(result); + } + + let extracted = pdf_inspector::extract_pages_markdown_mem(bytes, None).map_err(map_error)?; + // A scanned document that extracts to nothing at the document level can + // still have text on its pages (a watermark, a tool footer) which the + // per-page flags accept as real and skip OCR for. So every page of such + // a document is recognized and whichever text carry more content wins. + let scanned_without_text = matches!(result.pdf_type, PdfType::Scanned | PdfType::ImageBased) + && has_no_extractable_text(&result); + let recognize = |page: &PageMarkdown| scanned_without_text || page.needs_ocr; + let required = extracted.pages.iter().filter(|page| recognize(page)).count(); + if required == 0 { + // Nothing to recognize after all. Re-using the single-pass Markdown + // costs a second parse but is worth it: the bytes match exactly a + // converter built without an engine. + return extracted_markdown(result); + } + + // Page ids arrive 0-based in document order, so pages are recognized in + // ascending order and the merge map is keyed by that same index. + let mut blocks: BTreeMap = BTreeMap::new(); + let mut failed = 0_usize; + for page in &extracted.pages { + let extracted_text = page.markdown.trim(); + if !recognize(page) { + if !extracted_text.is_empty() { + blocks.insert(page.page, extracted_text.to_string()); + } + continue; + } + // One page per render call keeps peak memory at a single page of + // pixels, which are released before the next one is rasterized. + let recognized = match recognize_page(bytes, page.page, ocr) { + Ok(lines) => Some(lines.join("\n")), + Err(reason) => { + failed += 1; + if extracted_text.is_empty() { + log::warn!("page {} was skipped: {reason}", page.page + 1); + } else { + log::warn!("page {} kept its extracted text: {reason}", page.page + 1); + } + None + } + }; + match select_page_text(page.needs_ocr, extracted_text, recognized.as_deref()) { + PageText::Recognized => { + if let Some(recognized) = recognized { + if !page.needs_ocr { + log::debug!("OCR replaced the extracted text of page {}", page.page + 1); + } + blocks.insert(page.page, recognized); + } + } + PageText::Extracted => { + blocks.insert(page.page, extracted_text.to_string()); + } + PageText::Nothing => { + if recognized.is_some() { + log::debug!("no text was recognized on page {}", page.page + 1); + } + } + } + } + + if blocks.is_empty() { + // Recognizing no text is an answer, not a failure: a page that was + // rendered and read without finding any is a blank page. Only pages + // that could not be processed at all make the document terminal. + if failed == 0 { + return Ok(String::new()); + } + return Err(ConvertError::Ocr { + reason: format!("{failed} of {required} pages could not be processed"), + }); + } + Ok(merged_pages(&blocks)) +} + +/// Recognize one page, or say why it has to be skipped. +#[cfg(feature = "ocr")] +fn recognize_page(bytes: &[u8], page: u32, ocr: &O) -> Result, String> { + let mut rendered = pdf_inspector::render_pages_mem(bytes, &[page], RenderOptions::new()) + .map_err(|error| format!("the page could not be rendered: {error}"))?; + let Some(rendered) = rendered.pop() else { + return Err("the renderer returned no page".to_string()); + }; + if rendered.warnings.contains(&RenderWarning::ImageDecodeFailure) { + return Err("an image on the page could not be decoded".to_string()); + } + if rendered.warnings.contains(&RenderWarning::UnsupportedFont) { + log::debug!("page {} uses an unsupported font; glyphs may be missing", page + 1); + } + let lines = ocr + .recognize_page(rendered.width, rendered.height, &rendered.pixels) + .map_err(|error| error.to_string())?; + Ok(recognized_lines(&lines)) +} + +/// The Markdown lines recognized text contributes, shared by every OCR +/// frontend: line endings normalized, blank lines dropped, syntax escaped. +#[cfg(feature = "ocr")] +pub(super) fn recognized_lines(lines: &[String]) -> Vec { + lines + .iter() + .flat_map(|line| line.split(['\r', '\n'])) + .filter(|line| !line.trim().is_empty()) + .map(escape_block_syntax) + .collect() +} + +/// Escape whatever a recognized line opens with that GFM would read as a +/// block. OCR carries no structure, so the text is otherwise left as-is. +#[cfg(feature = "ocr")] +fn escape_block_syntax(line: &str) -> String { + const TRIGGERS: [char; 10] = ['#', '>', '-', '+', '*', '`', '|', '=', '_', '~']; + + let (indent, rest) = line.split_at(line.len() - line.trim_start().len()); + let mut escaped = String::with_capacity(line.len() + 1); + escaped.push_str(indent); + match rest.chars().next() { + Some(first) if TRIGGERS.contains(&first) => { + escaped.push('\\'); + escaped.push_str(rest); + } + // An ordered list opens with digits and either a dot or a paren. + Some(first) if first.is_ascii_digit() => { + let tail = rest.trim_start_matches(|c: char| c.is_ascii_digit()); + escaped.push_str(&rest[..rest.len() - tail.len()]); + let mut after_digits = tail.chars(); + match after_digits.next() { + Some(marker @ ('.' | ')')) => { + escaped.push('\\'); + escaped.push(marker); + escaped.push_str(after_digits.as_str()); + } + _ => escaped.push_str(tail), + } + } + _ => escaped.push_str(rest), } + escaped +} + +/// Page blocks in source order, one blank line apart, one trailing newline. +#[cfg(feature = "ocr")] +fn merged_pages(blocks: &BTreeMap) -> String { + let mut markdown = blocks.values().map(String::as_str).collect::>().join("\n\n"); + markdown.push('\n'); + markdown +} + +/// Whether document-level extraction found nothing worth keeping, which is +/// what makes a converter without an engine call the document unsupported. +#[cfg(feature = "ocr")] +fn has_no_extractable_text(result: &PdfProcessResult) -> bool { + result.markdown.as_ref().is_none_or(|markdown| markdown.trim().is_empty()) +} + +/// Which of a page's two readings ends up in the output. +#[cfg(feature = "ocr")] +#[derive(Debug, PartialEq, Eq)] +enum PageText { + Recognized, + Extracted, + Nothing, +} + +/// Pick between what the page extracted and what OCR read on it. A flagged +/// page always takes the recognition, anywhere else the richer text wins. +#[cfg(feature = "ocr")] +fn select_page_text(flagged: bool, extracted: &str, recognized: Option<&str>) -> PageText { + let keeps_extraction = match recognized { + None => true, + Some(recognized) => !flagged && text_weight(recognized) <= text_weight(extracted), + }; + if keeps_extraction { + return if extracted.is_empty() { PageText::Nothing } else { PageText::Extracted }; + } + match recognized { + Some(recognized) if !recognized.is_empty() => PageText::Recognized, + _ => PageText::Nothing, + } +} + +/// How much text a block carries. Counting alphanumerics ignores case and +/// layout for free, so neither can tip the comparison. +#[cfg(feature = "ocr")] +fn text_weight(text: &str) -> usize { + text.chars().filter(|character| character.is_alphanumeric()).count() +} + +/// The extracted Markdown with exactly one trailing newline, or the terminal +/// error a document without extractable text produces. +fn extracted_markdown(result: PdfProcessResult) -> Result { match result.markdown { Some(mut markdown) if !markdown.trim().is_empty() => { if !markdown.ends_with('\n') { @@ -37,6 +265,12 @@ pub fn to_markdown(bytes: &[u8]) -> Result { } } +fn warn_on_encoding_issues(result: &PdfProcessResult) { + if result.has_encoding_issues { + log::warn!("broken font encodings detected; extracted text may be garbled"); + } +} + fn map_error(e: PdfError) -> ConvertError { match e { PdfError::Encrypted => ConvertError::Encrypted, @@ -46,3 +280,275 @@ fn map_error(e: PdfError) -> ConvertError { PdfError::Parse(detail) => ConvertError::malformed(detail), } } + +#[cfg(all(test, feature = "ocr"))] +mod tests { + use super::pdf_fixture::{ + WATERMARK, image_only_pdf, mixed_text_and_image_pdf, text_only_pdf, watermarked_image_pdf, + }; + use super::{ + ConvertError, PageText, PdfType, escape_block_syntax, select_page_text, text_weight, + to_markdown, to_markdown_with_page_ocr, + }; + use crate::ocr::{OcrPageError, PageOcr}; + use std::cell::RefCell; + + /// The corpus PDF, which pdf-inspector classifies as `Mixed` while + /// flagging none of its pages: the second routing branch. + fn corpus_text_pdf() -> Vec { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("fixtures") + .join("pdf") + .join("text.pdf"); + std::fs::read(path).unwrap() + } + + /// Deterministic stand-in for the model-backed engine. It records the + /// pages it was handed, checking that each one arrives as a complete RGBA + /// buffer, and replays a scripted result. + struct FakeOcr { + result: Result, ()>, + pages: RefCell>, + } + + impl FakeOcr { + fn recognizing(lines: &[&str]) -> Self { + let lines = lines.iter().map(|line| (*line).to_string()).collect(); + Self { result: Ok(lines), pages: RefCell::new(Vec::new()) } + } + + fn failing() -> Self { + Self { result: Err(()), pages: RefCell::new(Vec::new()) } + } + + fn pages(&self) -> Vec<(u32, u32)> { + self.pages.borrow().clone() + } + } + + impl PageOcr for FakeOcr { + fn recognize_page( + &self, + width: u32, + height: u32, + rgba: &[u8], + ) -> Result, OcrPageError> { + assert_eq!(rgba.len(), width as usize * height as usize * 4); + self.pages.borrow_mut().push((width, height)); + self.result + .clone() + .map_err(|()| OcrPageError::Recognition { detail: "fake failure".into() }) + } + } + + /// The routing the whole pipeline rests on: pdf-inspector has to see the + /// generated image page as scanned and both text pages as extractable. + #[test] + fn the_fixtures_are_classified_as_the_pipeline_expects() { + let image_only = image_only_pdf(); + let classified = pdf_inspector::process_pdf_mem(&image_only).unwrap(); + assert_eq!(classified.pages_needing_ocr, [1]); + let pages = pdf_inspector::extract_pages_markdown_mem(&image_only, None).unwrap(); + assert!(pages.pages[0].needs_ocr); + + let mixed = mixed_text_and_image_pdf(); + let pages = pdf_inspector::extract_pages_markdown_mem(&mixed, None).unwrap(); + let needing_ocr: Vec = + pages.pages.iter().filter(|page| page.needs_ocr).map(|page| page.page).collect(); + assert_eq!(needing_ocr, [1]); + } + + /// Configuring an engine may not change a document that has no page to + /// recognize, whether the detector called it text-based (this case) or + /// not (the corpus PDF below). + #[test] + fn a_text_document_converts_exactly_as_it_does_without_an_engine() { + let pdf = text_only_pdf(); + let ocr = FakeOcr::recognizing(&["unexpected"]); + let with_engine = to_markdown_with_page_ocr(&pdf, &ocr).unwrap(); + + assert!(ocr.pages().is_empty()); + assert_eq!(with_engine, to_markdown(&pdf).unwrap()); + assert!(with_engine.contains("Structured first page")); + } + + #[test] + fn a_document_with_no_flagged_page_converts_exactly_as_it_does_without_an_engine() { + let pdf = corpus_text_pdf(); + let ocr = FakeOcr::recognizing(&["unexpected"]); + let with_engine = to_markdown_with_page_ocr(&pdf, &ocr).unwrap(); + + assert!(ocr.pages().is_empty()); + assert_eq!(with_engine, to_markdown(&pdf).unwrap()); + } + + /// The mixed fixture is a scanned document that extracts to nothing at the + /// document level, so every page is read. Its text pages extract far more + /// than this recognizer returns and keep their Markdown; the scanned page + /// has none to compare against and takes the recognized text. + #[test] + fn a_scanned_document_reads_every_page_and_keeps_the_richer_text() { + let ocr = FakeOcr::recognizing(&["Recognized page"]); + let markdown = to_markdown_with_page_ocr(&mixed_text_and_image_pdf(), &ocr).unwrap(); + + // 612 x 792 points at the default 200 DPI, once per page. + assert_eq!(ocr.pages(), [(1699, 2200), (1699, 2200), (1699, 2200)]); + assert_eq!( + markdown, + "Structured first page with two extracted lines.\n\ + \n\ + Recognized page\n\ + \n\ + Structured third page with two extracted lines.\n" + ); + } + + #[test] + fn a_page_without_recognized_text_contributes_nothing() { + let ocr = FakeOcr::recognizing(&[]); + let markdown = to_markdown_with_page_ocr(&mixed_text_and_image_pdf(), &ocr).unwrap(); + + assert_eq!(ocr.pages().len(), 3); + assert_eq!( + markdown, + "Structured first page with two extracted lines.\n\ + \n\ + Structured third page with two extracted lines.\n" + ); + } + + /// A page that cannot be recognized keeps whatever was extracted from it, + /// so a failure never costs text the document already had. + #[test] + fn a_failed_page_falls_back_to_its_extracted_text() { + let ocr = FakeOcr::failing(); + let markdown = to_markdown_with_page_ocr(&mixed_text_and_image_pdf(), &ocr).unwrap(); + + // The scanned page had nothing to fall back to and is skipped. + assert_eq!( + markdown, + "Structured first page with two extracted lines.\n\ + \n\ + Structured third page with two extracted lines.\n" + ); + } + + /// A flagged page's extraction was already judged unusable, so however + /// little OCR reads there, it is still the better of the two. + #[test] + fn a_flagged_page_takes_its_recognition_however_thin() { + assert_eq!(select_page_text(true, WATERMARK, Some("Trial")), PageText::Recognized); + assert_eq!(select_page_text(true, "", Some("Trial")), PageText::Recognized); + } + + /// An unflagged page has text worth keeping, so recognition has to read + /// strictly more to take its place. + #[test] + fn an_unflagged_page_takes_recognition_only_when_it_reads_richer() { + let richer = "Open a ticket, assign it to yourself, fix the issue and close it"; + assert!(text_weight(richer) > text_weight(WATERMARK)); + + assert_eq!(select_page_text(false, WATERMARK, Some(richer)), PageText::Recognized); + assert_eq!(select_page_text(false, WATERMARK, Some("Trial")), PageText::Extracted); + // A tie leaves the extraction in place. + assert_eq!(select_page_text(false, WATERMARK, Some(WATERMARK)), PageText::Extracted); + } + + /// Loss prevention: a page that could not be recognized keeps whatever + /// was extracted from it, and contributes nothing only when it had none. + #[test] + fn a_failed_page_keeps_whatever_it_had() { + assert_eq!(select_page_text(false, WATERMARK, None), PageText::Extracted); + assert_eq!(select_page_text(true, WATERMARK, None), PageText::Extracted); + assert_eq!(select_page_text(true, "", None), PageText::Nothing); + assert_eq!(select_page_text(true, "", Some("")), PageText::Nothing); + } + + /// The synthetic watermark fixture as pdf-inspector now reads it: the + /// page is flagged with no extractable text of its own, so recognition + /// stands alone. It no longer matches corpus 018's shape (unflagged, with + /// the watermark as its text), which the ignored corpus test now owns. + #[test] + fn a_scanned_page_contributes_what_was_recognized_on_it() { + let pdf = watermarked_image_pdf(); + + let classified = pdf_inspector::process_pdf_mem(&pdf).unwrap(); + let extracted = pdf_inspector::extract_pages_markdown_mem(&pdf, None).unwrap(); + assert_eq!(classified.pdf_type, PdfType::ImageBased); + assert!(classified.markdown.is_none()); + assert!(extracted.pages[0].needs_ocr); + assert!(extracted.pages[0].markdown.is_empty()); + + let recognized = ["Open a ticket, assign it to yourself,", "fix the issue and close it"]; + let ocr = FakeOcr::recognizing(&recognized); + let markdown = to_markdown_with_page_ocr(&pdf, &ocr).unwrap(); + + assert_eq!(ocr.pages(), [(1699, 2200)]); + assert_eq!(markdown, format!("{}\n{}\n", recognized[0], recognized[1])); + assert!(!markdown.contains("Trial Version")); + } + + #[test] + fn a_document_whose_pages_all_failed_is_a_terminal_error() { + let failing = to_markdown_with_page_ocr(&image_only_pdf(), &FakeOcr::failing()); + + let Err(ConvertError::Ocr { reason }) = failing else { + panic!("a document with no recoverable content converted"); + }; + assert!(reason.contains("1 of 1 pages"), "{reason}"); + } + + /// Reading a page and finding nothing on it is not a failure, so the + /// document converts to the empty output a content-free document has. + #[test] + fn a_document_whose_pages_recognize_no_text_converts_to_nothing() { + let empty = to_markdown_with_page_ocr(&image_only_pdf(), &FakeOcr::recognizing(&[])); + + assert_eq!(empty.unwrap(), ""); + } + + #[test] + fn recognized_lines_keep_their_text_but_not_block_syntax() { + for (line, expected) in [ + ("# not a heading", "\\# not a heading"), + ("> not a quote", "\\> not a quote"), + ("- not a list", "\\- not a list"), + ("+ not a list", "\\+ not a list"), + ("* not a list", "\\* not a list"), + ("`not code`", "\\`not code`"), + ("| not | a table |", "\\| not | a table |"), + ("1. not a list", "1\\. not a list"), + (" 12. not a list", " 12\\. not a list"), + // Underlining a line above turns it into a heading, and a run of + // any of these opens a break or a fence. + ("=== not an underline", "\\=== not an underline"), + ("___ not a signature rule", "\\___ not a signature rule"), + ("~~~ not a fence", "\\~~~ not a fence"), + ("1) not a list", "1\\) not a list"), + // A trigger is escaped wherever it opens the line, as the dash + // already was: "~5 pages" reads the same either way. + ("~5 pages", "\\~5 pages"), + // Away from the start of the line these characters are ordinary. + ("a - b", "a - b"), + ("x #1", "x #1"), + ("2 items", "2 items"), + ("a = b", "a = b"), + ("x_y", "x_y"), + ("call (a) now", "call (a) now"), + ("Größe 12 µm", "Größe 12 µm"), + ] { + assert_eq!(escape_block_syntax(line), expected, "escaping {line:?}"); + } + } + + /// Only the characters that carry content count, so layout and case never + /// decide which of two readings of a page is kept. + #[test] + fn text_weight_counts_content_characters_only() { + assert_eq!(text_weight("Open a ticket"), 11); + assert_eq!(text_weight("OPEN A\nTICKET"), 11); + assert_eq!(text_weight(" ...--- \t\n "), 0); + assert_eq!(text_weight("Größe 12 µm"), 5 + 2 + 2); + } +} diff --git a/src/lib.rs b/src/lib.rs index efba6ff..cf0428b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,16 +10,191 @@ pub mod model; mod error; mod formats; +#[cfg(feature = "ocr")] +mod ocr; mod package; mod render; mod shared; pub use error::ConvertError; +#[cfg(feature = "ocr")] +pub use ocr::OcrInitError; use render::markdown::document_to_markdown; use std::path::Path; +/// Reusable document converter. +/// +/// The default converter has the same behavior as the crate-level conversion +/// functions. Build a configured converter with [`Converter::builder`] when a +/// conversion capability needs reusable initialization. +/// +/// Marked non-exhaustive so optional features can add fields without +/// breaking construction in downstream crates. +#[non_exhaustive] +pub struct Converter { + #[cfg(feature = "ocr")] + ocr: Option, +} + +impl Converter { + /// Create a converter with the default capabilities. + pub fn new() -> Self { + Self { + #[cfg(feature = "ocr")] + ocr: None, + } + } + + /// Start building a configured converter. + pub fn builder() -> ConverterBuilder { + ConverterBuilder::new() + } + + /// Whether this converter has local OCR configured. + pub fn has_ocr(&self) -> bool { + #[cfg(feature = "ocr")] + { + self.ocr.is_some() + } + #[cfg(not(feature = "ocr"))] + { + false + } + } + + /// Convert a document file to Markdown. The format is detected from the + /// file content ([`Format::from_bytes`]); the extension is the fallback for + /// signature-less formats (CSV) and unrecognizable containers. + pub fn to_markdown(&self, path: impl AsRef) -> Result { + let path = path.as_ref(); + let bytes = std::fs::read(path)?; + let Some(format) = Format::from_bytes(&bytes).or_else(|| Format::from_path(path)) else { + return Err(ConvertError::Unsupported(format!( + "unrecognized file content and extension: {}", + path.display() + ))); + }; + self.to_markdown_bytes(&bytes, format) + } + + /// Convert an in-memory document to Markdown. Pass a [`Format`] to select + /// the parser, or `None` to detect it from the content + /// ([`Format::from_bytes`]), which signature-less formats (CSV) have to + /// name explicitly. + pub fn to_markdown_bytes( + &self, + bytes: &[u8], + format: impl Into>, + ) -> Result { + let format = resolve_format(bytes, format.into())?; + // PDFs and images convert to Markdown directly (pdf-inspector and + // local OCR) without passing through the document model. + if format == Format::Pdf { + return self.pdf_to_markdown(bytes); + } + if format == Format::Image { + return self.image_to_markdown(bytes); + } + Ok(document_to_markdown(&self.to_document(bytes, format)?)) + } + + /// Route the PDF through local OCR when this converter has an engine; the + /// unconfigured path is the crate's default behavior. + fn pdf_to_markdown(&self, bytes: &[u8]) -> Result { + #[cfg(feature = "ocr")] + if let Some(engine) = self.ocr.as_ref() { + return formats::pdf::to_markdown_with_ocr(bytes, engine); + } + formats::pdf::to_markdown(bytes) + } + + /// Recognize an image document when this converter has an engine. An + /// image has no text layer to fall back on, so without one there is + /// nothing to convert. + fn image_to_markdown(&self, bytes: &[u8]) -> Result { + #[cfg(feature = "ocr")] + if let Some(engine) = self.ocr.as_ref() { + return formats::image::to_markdown_with_ocr(bytes, engine); + } + #[cfg(not(feature = "ocr"))] + let _ = bytes; + Err(ConvertError::Unsupported("image has no extractable text: OCR is required".to_string())) + } + + /// Parse an in-memory document into the document model. Pass a [`Format`] + /// to select the parser, or `None` to detect it from the content. + /// + /// Unsupported for [`Format::Pdf`]: PDF conversion produces Markdown + /// directly and has no document-model form; use + /// [`Converter::to_markdown_bytes`]. + pub fn to_document( + &self, + bytes: &[u8], + format: impl Into>, + ) -> Result { + formats::parse(bytes, resolve_format(bytes, format.into())?) + } +} + +impl Default for Converter { + fn default() -> Self { + Self::new() + } +} + +/// Builder for a reusable [`Converter`]. +/// +/// Marked non-exhaustive so optional features can add fields without +/// breaking construction in downstream crates. +#[non_exhaustive] +pub struct ConverterBuilder { + #[cfg(feature = "ocr")] + ocr: Option, +} + +impl ConverterBuilder { + /// Create a converter builder with the default capabilities. + pub fn new() -> Self { + Self { + #[cfg(feature = "ocr")] + ocr: None, + } + } + + /// Load the RTen detection and recognition models used for local OCR. + /// + /// Model bytes are parsed immediately and retained by the converter built + /// from this builder. The library performs no download or filesystem + /// access. This method is available with the `ocr` Cargo feature. The + /// size limit is checked after the bytes are owned, so an oversized + /// borrowed slice is still copied once before rejection. + #[cfg(feature = "ocr")] + pub fn with_ocr_models( + mut self, + detection_model: impl Into>, + recognition_model: impl Into>, + ) -> Result { + self.ocr = Some(ocr::Engine::from_bytes(detection_model.into(), recognition_model.into())?); + Ok(self) + } + + /// Build the converter. + pub fn build(self) -> Converter { + Converter { + #[cfg(feature = "ocr")] + ocr: self.ocr, + } + } +} + +impl Default for ConverterBuilder { + fn default() -> Self { + Self::new() + } +} + /// Input format. Selects the parser; container variants that share a parser /// (docm, xlsm, ...) map onto these via [`Format::from_bytes`] or /// [`Format::from_extension`]. @@ -32,8 +207,10 @@ pub enum Format { /// OpenDocument Text (`.odt`). Odt, /// Converted with [pdf-inspector], which emits Markdown directly: - /// [`to_document`] is unsupported for PDFs. Scanned/image-only PDFs - /// (needing OCR) error as unsupported. + /// [`to_document`] is unsupported for PDFs. Pages that carry no + /// extractable text need OCR: a converter without an engine reports them + /// as unsupported, while one built with OCR models (the `ocr` feature) + /// recognizes exactly those pages and merges them with the rest. /// /// [pdf-inspector]: https://github.com/firecrawl/pdf-inspector Pdf, @@ -55,6 +232,16 @@ pub enum Format { /// Delimiter-separated text (`.csv`). Carries no signature, so it has to /// be named rather than detected. Csv, + /// Raster image documents (PNG, JPEG, WebP, TIFF, BMP), detected by their + /// content signature. An image carries no text layer, so it converts with + /// local OCR when the converter has an engine configured and returns + /// [`ConvertError::Unsupported`] when it does not. [`to_document`] is + /// unsupported, as for [`Format::Pdf`]: recognition produces Markdown + /// directly. + /// + /// EXIF orientation is not applied in this version, so a photo stored + /// rotated is recognized in the orientation it is stored in. + Image, } impl Format { @@ -83,6 +270,7 @@ impl Format { "ods" => Format::Ods, "odp" => Format::Odp, "csv" => Format::Csv, + "png" | "jpg" | "jpeg" | "webp" | "tif" | "tiff" | "bmp" => Format::Image, _ => return None, }) } @@ -98,15 +286,7 @@ impl Format { /// file content ([`Format::from_bytes`]); the extension is the fallback for /// signature-less formats (CSV) and unrecognizable containers. pub fn to_markdown(path: impl AsRef) -> Result { - let path = path.as_ref(); - let bytes = std::fs::read(path)?; - let Some(format) = Format::from_bytes(&bytes).or_else(|| Format::from_path(path)) else { - return Err(ConvertError::Unsupported(format!( - "unrecognized file content and extension: {}", - path.display() - ))); - }; - to_markdown_bytes(&bytes, format) + Converter::new().to_markdown(path) } /// Convert an in-memory document to Markdown. Pass a [`Format`] to select the @@ -116,13 +296,7 @@ pub fn to_markdown_bytes( bytes: &[u8], format: impl Into>, ) -> Result { - let format = resolve_format(bytes, format.into())?; - // PDFs convert to Markdown directly (pdf-inspector) without passing - // through the document model. - if format == Format::Pdf { - return formats::pdf::to_markdown(bytes); - } - Ok(document_to_markdown(&to_document(bytes, format)?)) + Converter::new().to_markdown_bytes(bytes, format) } /// Parse an in-memory document into the document model. Pass a [`Format`] to @@ -134,7 +308,7 @@ pub fn to_document( bytes: &[u8], format: impl Into>, ) -> Result { - formats::parse(bytes, resolve_format(bytes, format.into())?) + Converter::new().to_document(bytes, format) } fn resolve_format(bytes: &[u8], format: Option) -> Result { @@ -142,3 +316,63 @@ fn resolve_format(bytes: &[u8], format: Option) -> Result, + recognition_model: Vec, + ) -> Result { + validate_model_size("detection", detection_model.len())?; + validate_model_size("recognition", recognition_model.len())?; + let detection_model = load_model("detection", detection_model)?; + let recognition_model = load_model("recognition", recognition_model)?; + let inner = OcrEngine::new(OcrEngineParams { + detection_model: Some(detection_model), + recognition_model: Some(recognition_model), + ..Default::default() + }) + .map_err(|error| OcrInitError::Engine { detail: error.to_string() })?; + Ok(Self { inner }) + } + + /// One string per recognized line, in reading order, blank ones dropped. + /// `rgba` is row-major RGBA8, so it has to be `width * height * 4` long. + pub(crate) fn recognize_rgba_page( + &self, + width: u32, + height: u32, + rgba: &[u8], + ) -> Result, OcrPageError> { + let expected = rgba_len(width, height).ok_or(OcrPageError::PageSize { + width, + height, + bytes: rgba.len(), + })?; + if rgba.len() != expected { + return Err(OcrPageError::PageSize { width, height, bytes: rgba.len() }); + } + let source = ImageSource::from_bytes(rgba, (width, height)) + .map_err(|error| OcrPageError::Recognition { detail: error.to_string() })?; + let input = self + .inner + .prepare_input(source) + .map_err(|error| OcrPageError::Recognition { detail: error.to_string() })?; + let words = self + .inner + .detect_words(&input) + .map_err(|error| OcrPageError::Recognition { detail: error.to_string() })?; + let lines = self.inner.find_text_lines(&input, &words); + let recognized = self + .inner + .recognize_text(&input, &lines) + .map_err(|error| OcrPageError::Recognition { detail: error.to_string() })?; + Ok(recognized + .into_iter() + .flatten() + .map(|line| line.to_string()) + .filter(|line| !line.trim().is_empty()) + .collect()) + } +} + +/// Recognition of one page, so tests can stand in for the real engine. +pub(crate) trait PageOcr { + /// See [`Engine::recognize_rgba_page`]. + fn recognize_page( + &self, + width: u32, + height: u32, + rgba: &[u8], + ) -> Result, OcrPageError>; +} + +impl PageOcr for Engine { + fn recognize_page( + &self, + width: u32, + height: u32, + rgba: &[u8], + ) -> Result, OcrPageError> { + self.recognize_rgba_page(width, height, rgba) + } +} + +/// Why one page could not be recognized. +#[derive(Debug)] +pub(crate) enum OcrPageError { + /// The buffer does not describe a non-empty RGBA8 image of that size, + /// which is an internal error and never bad input. + PageSize { width: u32, height: u32, bytes: usize }, + /// The OCR pipeline rejected the page. + Recognition { detail: String }, +} + +impl fmt::Display for OcrPageError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + OcrPageError::PageSize { width, height, bytes } => { + write!(f, "{bytes} pixel bytes do not describe a {width}x{height} RGBA image") + } + OcrPageError::Recognition { detail } => write!(f, "recognition failed: {detail}"), + } + } +} + +/// Byte length of a non-empty RGBA8 buffer this size, or `None` when the +/// dimensions are empty or overflow the `u32` the image source multiplies in. +fn rgba_len(width: u32, height: u32) -> Option { + let pixels = width.checked_mul(height).filter(|&pixels| pixels != 0)?; + usize::try_from(pixels).ok()?.checked_mul(4) +} + +/// Why local OCR initialization failed. +#[derive(Debug)] +#[non_exhaustive] +pub enum OcrInitError { + /// A model exceeded the fixed configuration-size limit. + ModelTooLarge { + /// Whether this was the `detection` or `recognition` model. + model: &'static str, + /// Supplied model size in bytes. + bytes: usize, + /// Maximum accepted model size in bytes. + max_bytes: usize, + }, + /// RTen could not parse or validate a model. + InvalidModel { + /// Whether this was the `detection` or `recognition` model. + model: &'static str, + /// Loader diagnostic. + detail: String, + }, + /// The loaded model pair could not initialize the OCR engine. + Engine { + /// Engine diagnostic. + detail: String, + }, +} + +impl fmt::Display for OcrInitError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + OcrInitError::ModelTooLarge { model, bytes, max_bytes } => { + write!(f, "OCR {model} model is too large: {bytes} bytes exceeds {max_bytes} bytes") + } + OcrInitError::InvalidModel { model, detail } => { + write!(f, "invalid OCR {model} model: {detail}") + } + OcrInitError::Engine { detail } => write!(f, "could not initialize OCR: {detail}"), + } + } +} + +impl std::error::Error for OcrInitError {} + +fn load_model(model: &'static str, bytes: Vec) -> Result { + if bytes.is_empty() { + return Err(OcrInitError::InvalidModel { model, detail: "model bytes are empty".into() }); + } + ModelOptions::with_ops(ocr_ops()) + .load(bytes) + .map_err(|error| OcrInitError::InvalidModel { model, detail: error.to_string() }) +} + +fn validate_model_size(model: &'static str, bytes: usize) -> Result<(), OcrInitError> { + if bytes > MAX_MODEL_BYTES { + return Err(OcrInitError::ModelTooLarge { model, bytes, max_bytes: MAX_MODEL_BYTES }); + } + Ok(()) +} + +fn ocr_ops() -> OpRegistry { + op_registry!( + Add, + AveragePool, + Cast, + Concat, + ConstantOfShape, + Conv, + ConvTranspose, + GRU, + Gather, + LogSoftmax, + MatMul, + MaxPool, + Pad, + Relu, + Reshape, + Shape, + Sigmoid, + Slice, + Transpose, + Unsqueeze + ) +} + +#[cfg(test)] +mod tests { + use super::{Engine, MAX_MODEL_BYTES, OcrInitError, rgba_len, validate_model_size}; + + #[test] + fn model_size_limit_is_fixed() { + assert!(validate_model_size("detection", MAX_MODEL_BYTES).is_ok()); + assert!(matches!( + validate_model_size("detection", MAX_MODEL_BYTES + 1), + Err(OcrInitError::ModelTooLarge { model: "detection", .. }) + )); + } + + #[test] + fn engine_is_safe_to_share_between_conversions() { + fn assert_send_sync() {} + assert_send_sync::(); + } + + /// The page recognizer accepts a buffer only when it exactly describes the + /// stated RGBA8 image, and computes that size without overflowing. + #[test] + fn rgba_length_is_checked_for_empty_and_oversized_pages() { + assert_eq!(rgba_len(2, 3), Some(24)); + assert_eq!(rgba_len(0, 3), None); + assert_eq!(rgba_len(3, 0), None); + assert_eq!(rgba_len(u32::MAX, 2), None); + } + + #[test] + fn oversized_recognition_model_is_rejected_before_detection_is_parsed() { + let invalid_detection = vec![0u8, 1, 2, 3]; + let oversized_recognition = vec![0u8; MAX_MODEL_BYTES + 1]; + let result = Engine::from_bytes(invalid_detection, oversized_recognition); + assert!(matches!(result, Err(OcrInitError::ModelTooLarge { model: "recognition", .. }))); + } +} diff --git a/tests/ocr.rs b/tests/ocr.rs new file mode 100644 index 0000000..232989a --- /dev/null +++ b/tests/ocr.rs @@ -0,0 +1,68 @@ +//! End-to-end OCR tests against the standard models. + +#![cfg(feature = "ocr")] + +#[path = "support/pdf_fixture.rs"] +mod pdf_fixture; + +use sha2::Digest as _; +use std::path::PathBuf; + +fn model_path(variable: &str) -> PathBuf { + std::env::var_os(variable) + .map(PathBuf::from) + .unwrap_or_else(|| panic!("set {variable} to the downloaded RTen model")) +} + +fn model_converter() -> anydoc::Converter { + let detection = std::fs::read(model_path("ANYDOC_OCR_DETECTION_MODEL")).unwrap(); + let recognition = std::fs::read(model_path("ANYDOC_OCR_RECOGNITION_MODEL")).unwrap(); + anydoc::Converter::builder().with_ocr_models(detection, recognition).unwrap().build() +} + +#[test] +#[ignore = "requires the external ocrs detection and recognition models"] +fn standard_models_initialize() { + assert!(model_converter().has_ocr()); +} + +/// The synthetic page carries no text, so the expected result is a successful +/// conversion to little or nothing. Only the shape can be asserted: the whole +/// render-and-recognize path runs, without panicking, to a bounded result. +#[test] +#[ignore = "requires the external ocrs detection and recognition models"] +fn standard_models_ocr_scanned_pdf() { + /// One synthetic page cannot legitimately recognize more text than this. + const MAX_RECOGNIZED_BYTES: usize = 64 * 1024; + + let markdown = model_converter() + .to_markdown_bytes(&pdf_fixture::image_only_pdf(), anydoc::Format::Pdf) + .expect("the scanned page converts"); + + assert!(markdown.len() < MAX_RECOGNIZED_BYTES, "{} bytes recognized", markdown.len()); + assert!(!markdown.contains("\n\n\n")); + assert!(markdown.is_empty() || markdown.ends_with('\n')); +} + +/// A scanned page whose only extractable text is a tool watermark has to be +/// recognized anyway, and what it says has to reach the output. The file is +/// pinned by content hash so a corpus update cannot quietly change the claim. +#[test] +#[ignore = "requires the external ocrs models and the pinned sample-files corpus"] +fn corpus_scanned_watermark_routes_to_ocr() { + const SHA256: &str = "aaad90df16fce40ec768629d2135479b98f65b39bb27c7f80fb106393187d619"; + + let corpus = std::env::var_os("ANYDOC_PDF_SAMPLE_FILES") + .map(PathBuf::from) + .expect("set ANYDOC_PDF_SAMPLE_FILES to the pinned sample-files checkout"); + let pdf = std::fs::read(corpus.join("018-base64-image").join("base64image.pdf")).unwrap(); + let digest: String = + sha2::Sha256::digest(&pdf).iter().map(|byte| format!("{byte:02x}")).collect(); + assert_eq!(digest, SHA256, "the pinned corpus file changed"); + + let markdown = + model_converter().to_markdown_bytes(&pdf, anydoc::Format::Pdf).expect("the scan converts"); + let normalized = markdown.to_lowercase().split_whitespace().collect::>().join(" "); + + assert!(normalized.contains("fix the issue and close it"), "recognized: {markdown}"); +} diff --git a/tests/support/pdf_fixture.rs b/tests/support/pdf_fixture.rs new file mode 100644 index 0000000..8393f73 --- /dev/null +++ b/tests/support/pdf_fixture.rs @@ -0,0 +1,203 @@ +//! Deterministic in-memory PDF fixtures for the selective OCR pipeline. +//! +//! The bytes are generated rather than committed so no separately licensed +//! binary fixture is needed and every build target exercises the same input. +//! Image pages carry a raw-RGB XObject large enough for pdf-inspector to +//! classify the page as scanned; text pages use an uncompressed content +//! stream and a base-14 font. + +// Each consumer uses a subset of the builders. +#![allow(dead_code)] + +/// Width in pixels of the raw-RGB image placed on image pages. Together with +/// [`IMAGE_HEIGHT`] this clears the detector's scanned-page image-area +/// threshold. +pub const IMAGE_WIDTH: u32 = 1_700; + +/// Height in pixels of the raw-RGB image placed on image pages. +pub const IMAGE_HEIGHT: u32 = 2_200; + +/// US Letter, in PDF points. +const PAGE_WIDTH: u32 = 612; +const PAGE_HEIGHT: u32 = 792; + +enum Page<'a> { + /// Lines of ASCII text drawn with the base-14 Helvetica font. + Text(&'a [&'a str]), + /// The raw-RGB image scaled to fill the page. + Image, + /// The same image with one short line of text over it, the shape a + /// scanned page carrying a tool watermark has. + WatermarkedImage(&'a str), +} + +/// A single-page PDF whose only content is a page-filling raw-RGB image. +pub fn image_only_pdf() -> Vec { + build_pdf(&[Page::Image]) +} + +/// A three-page PDF: a text page, the image page, then another text page. +pub fn mixed_text_and_image_pdf() -> Vec { + build_pdf(&[ + Page::Text(&["Structured first page", "with two extracted lines."]), + Page::Image, + Page::Text(&["Structured third page", "with two extracted lines."]), + ]) +} + +/// The watermark a scanned page of [`watermarked_image_pdf`] carries. +pub const WATERMARK: &str = "Produced with a Trial Version of PDF Annotator"; + +/// A single-page PDF whose page is the raw-RGB image with nothing but a tool +/// watermark as text: real content only OCR can read. +pub fn watermarked_image_pdf() -> Vec { + build_pdf(&[Page::WatermarkedImage(WATERMARK)]) +} + +/// A two-page PDF with text on every page and no images. +pub fn text_only_pdf() -> Vec { + build_pdf(&[ + Page::Text(&["Structured first page", "with two extracted lines."]), + Page::Text(&["Structured second page", "with two extracted lines."]), + ]) +} + +fn build_pdf(pages: &[Page]) -> Vec { + let has_text = + pages.iter().any(|page| matches!(page, Page::Text(_) | Page::WatermarkedImage(_))); + let has_image = + pages.iter().any(|page| matches!(page, Page::Image | Page::WatermarkedImage(_))); + // Objects 1 and 2 are the catalog and the page tree; each page then takes + // a page dictionary and a content stream, and the shared font and image + // resources follow. + let font_id = 3 + pages.len() * 2; + let image_id = font_id + usize::from(has_text); + + let mut pdf = b"%PDF-1.4\n".to_vec(); + let mut offsets = vec![0_usize]; + + add_object(&mut pdf, &mut offsets, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); + + let kids = (0..pages.len()) + .map(|index| format!("{} 0 R", 3 + index * 2)) + .collect::>() + .join(" "); + add_object( + &mut pdf, + &mut offsets, + 2, + format!("<< /Type /Pages /Kids [{kids}] /Count {} >>", pages.len()).as_bytes(), + ); + + for (index, page) in pages.iter().enumerate() { + let page_id = 3 + index * 2; + let content_id = page_id + 1; + let (resources, content) = match page { + Page::Text(lines) => { + (format!("<< /Font << /F1 {font_id} 0 R >> >>"), text_content_stream(lines)) + } + Page::Image => ( + format!("<< /XObject << /Im0 {image_id} 0 R >> >>"), + format!("q {PAGE_WIDTH} 0 0 {PAGE_HEIGHT} 0 0 cm /Im0 Do Q"), + ), + Page::WatermarkedImage(watermark) => ( + format!("<< /Font << /F1 {font_id} 0 R >> /XObject << /Im0 {image_id} 0 R >> >>"), + format!( + "q {PAGE_WIDTH} 0 0 {PAGE_HEIGHT} 0 0 cm /Im0 Do Q {}", + text_content_stream(&[watermark]) + ), + ), + }; + add_object( + &mut pdf, + &mut offsets, + page_id, + format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {PAGE_WIDTH} {PAGE_HEIGHT}] \ + /Resources {resources} /Contents {content_id} 0 R >>" + ) + .as_bytes(), + ); + add_object( + &mut pdf, + &mut offsets, + content_id, + format!("<< /Length {} >>\nstream\n{content}\nendstream", content.len()).as_bytes(), + ); + } + + if has_text { + add_object( + &mut pdf, + &mut offsets, + font_id, + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>", + ); + } + if has_image { + let image = checkerboard_rgb(); + let mut stream = format!( + "<< /Type /XObject /Subtype /Image /Width {IMAGE_WIDTH} /Height {IMAGE_HEIGHT} \ + /ColorSpace /DeviceRGB /BitsPerComponent 8 /Length {} >>\nstream\n", + image.len() + ) + .into_bytes(); + stream.extend_from_slice(&image); + stream.extend_from_slice(b"\nendstream"); + add_object(&mut pdf, &mut offsets, image_id, &stream); + } + + let xref_start = pdf.len(); + pdf.extend_from_slice(format!("xref\n0 {}\n", offsets.len()).as_bytes()); + pdf.extend_from_slice(b"0000000000 65535 f \n"); + for offset in offsets.iter().skip(1) { + pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + pdf.extend_from_slice( + format!( + "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref_start}\n%%EOF", + offsets.len() + ) + .as_bytes(), + ); + pdf +} + +fn add_object(pdf: &mut Vec, offsets: &mut Vec, id: usize, body: &[u8]) { + assert_eq!(id, offsets.len()); + offsets.push(pdf.len()); + pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes()); + pdf.extend_from_slice(body); + pdf.extend_from_slice(b"\nendobj\n"); +} + +fn text_content_stream(lines: &[&str]) -> String { + let mut content = String::from("BT /F1 18 Tf 72 700 Td"); + for line in lines { + assert!( + line.is_ascii() && !line.contains(['(', ')', '\\']), + "fixture text has to be escape-free ASCII" + ); + content.push_str(&format!(" ({line}) Tj 0 -24 Td")); + } + content.push_str(" ET"); + content +} + +/// Row-major RGB pixels forming coarse light and dark blocks. +fn checkerboard_rgb() -> Vec { + const BLOCK: u32 = 100; + let row = |offset: u32| { + (0..IMAGE_WIDTH) + .flat_map(|x| { + if (x / BLOCK + offset).is_multiple_of(2) { [16, 16, 16] } else { [240, 240, 240] } + }) + .collect::>() + }; + let (even, odd) = (row(0), row(1)); + let mut image = Vec::with_capacity((IMAGE_WIDTH * IMAGE_HEIGHT * 3) as usize); + for y in 0..IMAGE_HEIGHT { + image.extend_from_slice(if (y / BLOCK).is_multiple_of(2) { &even } else { &odd }); + } + image +} diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index 9ea8841..128de0e 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -15,9 +15,17 @@ publish = false crate-type = ["cdylib", "rlib"] # rustc emits bulk-memory and nontrapping-fptoint by default; wasm-pack's -# bundled wasm-opt rejects them unless enabled explicitly. +# bundled wasm-opt rejects them unless enabled explicitly, and the same for +# the simd128 the demo build asks for. Enabling a feature here only lets +# wasm-opt accept it: a build that emits none is unaffected. [package.metadata.wasm-pack.profile.release] -wasm-opt = ["-O", "--enable-bulk-memory", "--enable-nontrapping-float-to-int"] +wasm-opt = ["-O", "--enable-bulk-memory", "--enable-nontrapping-float-to-int", "--enable-simd"] + +[features] +default = [] +# Local OCR in the browser. Off by default: it multiplies the module size, +# and the models are a separate download the caller supplies. +ocr = ["anydoc/ocr"] [dependencies] anydoc = { path = ".." } diff --git a/wasm/README.md b/wasm/README.md index a2e0dfd..d341dbc 100644 --- a/wasm/README.md +++ b/wasm/README.md @@ -58,14 +58,49 @@ try { | `encrypted` | Encrypted or password-protected | | `resourceLimit` | Crossed a fixed safety limit (decompression, nesting, node count) | | `missingPart` | A part required for any meaningful output is absent | +| `ocr` | OCR ran on every page carrying content and read no text | +| `ocrInit` | The OCR models could not be loaded, from `new Converter` only | `error.message` carries the detail, naming the package part at fault where the format identifies one. TypeScript gets the union as `ConvertErrorCode`. The crate's `io` code has no counterpart here: there is no filesystem to read from. +## Local OCR + +Scanned PDFs and image documents carry no text to extract, so reading them needs OCR. It ships behind the `ocr` cargo feature, which is off by default because it multiplies the module size; build with `--features ocr` to get the `Converter` class. + +The two RTen models are a separate download that you supply as `Uint8Array`s. They are parsed once, when the converter is constructed, and every conversion made with it reuses that engine: + +```js +const [detectionModel, recognitionModel] = await Promise.all([ + fetch('/models/text-detection.rten').then((r) => r.arrayBuffer()), + fetch('/models/text-recognition.rten').then((r) => r.arrayBuffer()), +]) + +const converter = new Converter({ + detectionModel: new Uint8Array(detectionModel), + recognitionModel: new Uint8Array(recognitionModel), +}) + +const markdown = converter.toMarkdownBytes(new Uint8Array(pdfBytes), 'pdf') +``` + +A failed model load throws an `Error` with `'ocrInit'` on `code`. + +Run it in a Web Worker. Recognition is CPU-bound and blocks whatever thread it runs on, so doing this on the UI thread freezes the page for as long as a document takes. Load the module and build the converter inside the worker, keep the converter alive between messages so the models are parsed once, and post only the bytes and the Markdown across. + +[`examples/worker/`](examples/worker) is that setup in about 30 lines: [`worker.js`](examples/worker/worker.js) fetches the models and holds the converter, and [`index.html`](examples/worker/index.html) transfers the file bytes to it and prints the Markdown. No dependencies, no build step beyond the package itself. + ## Building ```bash wasm-pack build wasm --release --target web --scope firecrawl -node --test wasm/test.mjs +ANYDOC_WASM_OCR_BUILD=0 node --test wasm/test.mjs +``` + +`ANYDOC_WASM_OCR_BUILD` tells the suite which build it is looking at, so the test that guards the feature gate has an expectation of its own to check. Build and test the OCR package the same way: + +```bash +wasm-pack build wasm --release --target web --scope firecrawl -- --features ocr +ANYDOC_WASM_OCR_BUILD=1 node --test wasm/test.mjs ``` This produces the npm package in `wasm/pkg/`: the module, the JS glue, and TypeScript definitions. Publishing runs from [`../.github/workflows/release.yml`](../.github/workflows/release.yml) on release tags. diff --git a/wasm/examples/worker/index.html b/wasm/examples/worker/index.html new file mode 100644 index 0000000..7e81917 --- /dev/null +++ b/wasm/examples/worker/index.html @@ -0,0 +1,47 @@ + + + + + anydoc: OCR in a Web Worker + + +

anydoc: OCR in a Web Worker

+

+ Pick a scanned PDF or an image. Recognition runs in a worker, so this + page stays responsive while it works. +

+ + + 

+
+    
+  
+
diff --git a/wasm/examples/worker/worker.js b/wasm/examples/worker/worker.js
new file mode 100644
index 0000000..32fb9a3
--- /dev/null
+++ b/wasm/examples/worker/worker.js
@@ -0,0 +1,33 @@
+// Runs anydoc off the UI thread. Recognition is CPU-bound, so a converter
+// built here keeps the page responsive while a document is read.
+//
+// The module and the two models are fetched once; the converter is kept for
+// the lifetime of the worker so the models are parsed once too.
+import init, { Converter } from '../../pkg/anydoc_wasm.js'
+
+const MODELS = {
+  detection: '/models/text-detection.rten',
+  recognition: '/models/text-recognition.rten',
+}
+
+const bytes = async (url) => new Uint8Array(await (await fetch(url)).arrayBuffer())
+
+const ready = (async () => {
+  await init()
+  const [detectionModel, recognitionModel] = await Promise.all([
+    bytes(MODELS.detection),
+    bytes(MODELS.recognition),
+  ])
+  return new Converter({ detectionModel, recognitionModel })
+})()
+
+self.onmessage = async ({ data: { id, document, format } }) => {
+  try {
+    const converter = await ready
+    self.postMessage({ id, markdown: converter.toMarkdownBytes(document, format) })
+  } catch (error) {
+    // `code` is what the page branches on: 'ocrInit' means the models are
+    // wrong, anything else is the document.
+    self.postMessage({ id, error: error.message, code: error.code })
+  }
+}
diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs
index 5a710a5..1eeab45 100644
--- a/wasm/src/lib.rs
+++ b/wasm/src/lib.rs
@@ -20,8 +20,10 @@ pub enum Format {
     Docx = "docx",
     Odt = "odt",
     /// Converted with pdf-inspector, which emits Markdown directly:
-    /// `toDocument` is unsupported for PDFs. Scanned or image-only PDFs
-    /// (needing OCR) error as unsupported.
+    /// `toDocument` is unsupported for PDFs. Pages with no extractable text
+    /// need OCR: the module-level functions report them as unsupported,
+    /// while a `Converter`, in a build with the `ocr` feature, recognizes
+    /// exactly those pages.
     Pdf = "pdf",
     Ppt = "ppt",
     Pptx = "pptx",
@@ -31,6 +33,10 @@ pub enum Format {
     Ods = "ods",
     Odp = "odp",
     Csv = "csv",
+    /// Raster image documents (PNG, JPEG, WebP, TIFF, BMP). Recognized with
+    /// local OCR where it is configured, and unsupported otherwise;
+    /// `toDocument` is unsupported for images the same way it is for PDFs.
+    Image = "image",
 }
 
 impl From for anydoc::Format {
@@ -48,6 +54,7 @@ impl From for anydoc::Format {
             Format::Ods => anydoc::Format::Ods,
             Format::Odp => anydoc::Format::Odp,
             Format::Csv => anydoc::Format::Csv,
+            Format::Image => anydoc::Format::Image,
             Format::__Invalid => unreachable!("wasm-bindgen rejects invalid enum strings"),
         }
     }
@@ -68,6 +75,7 @@ impl From for Format {
             anydoc::Format::Ods => Format::Ods,
             anydoc::Format::Odp => Format::Odp,
             anydoc::Format::Csv => Format::Csv,
+            anydoc::Format::Image => Format::Image,
         }
     }
 }
@@ -118,12 +126,82 @@ pub fn to_document(bytes: &[u8], format: Option) -> Result Result {
+        let detection = model_bytes(&models, "detectionModel")?;
+        let recognition = model_bytes(&models, "recognitionModel")?;
+        let built = anydoc::Converter::builder()
+            .with_ocr_models(detection, recognition)
+            .map_err(|error| coded_error(&error.to_string(), "ocrInit"))?;
+        Ok(Converter { inner: built.build() })
+    }
+
+    /// Convert an in-memory document to Markdown, recognizing the pages that
+    /// need OCR. Without a format, it is detected from the content, which
+    /// signature-less formats (CSV) have to name explicitly.
+    ///
+    /// Throws an `Error` carrying a `ConvertErrorCode` on `code`.
+    #[wasm_bindgen(js_name = toMarkdownBytes)]
+    pub fn to_markdown_bytes(
+        &self,
+        bytes: &[u8],
+        format: Option,
+    ) -> Result {
+        self.inner.to_markdown_bytes(bytes, format.map(anydoc::Format::from)).map_err(convert_error)
+    }
+}
+
+/// One model out of the options object, as owned bytes.
+#[cfg(feature = "ocr")]
+fn model_bytes(models: &JsValue, key: &str) -> Result, JsValue> {
+    use wasm_bindgen::JsCast as _;
+
+    let value = js_sys::Reflect::get(models, &JsValue::from_str(key))
+        .map_err(|_| coded_error(&format!("{key} could not be read"), "ocrInit"))?;
+    if value.is_undefined() || value.is_null() {
+        return Err(coded_error(&format!("{key} is required"), "ocrInit"));
+    }
+    value
+        .dyn_into::()
+        .map(|model| model.to_vec())
+        .map_err(|_| coded_error(&format!("{key} has to be a Uint8Array"), "ocrInit"))
+}
+
 /// The thrown value: a JS `Error` carrying the crate's message, with the
 /// variant name on `code` for callers to branch on.
 fn convert_error(error: anydoc::ConvertError) -> JsValue {
-    let thrown = js_sys::Error::new(&error.to_string());
+    coded_error(&error.to_string(), error.code())
+}
+
+fn coded_error(message: &str, code: &str) -> JsValue {
+    let thrown = js_sys::Error::new(message);
     // Only fails on a non-object target, which `thrown` is not.
-    let _ =
-        js_sys::Reflect::set(&thrown, &JsValue::from_str("code"), &JsValue::from_str(error.code()));
+    let _ = js_sys::Reflect::set(&thrown, &JsValue::from_str("code"), &JsValue::from_str(code));
     thrown.into()
 }
diff --git a/wasm/src/typescript.rs b/wasm/src/typescript.rs
index 9a5cba6..c77b6e7 100644
--- a/wasm/src/typescript.rs
+++ b/wasm/src/typescript.rs
@@ -26,6 +26,28 @@ export type ConvertErrorCode =
   | 'resourceLimit'
   /** A part required for any meaningful output is absent. */
   | 'missingPart'
+  /**
+   * Local OCR was required for every page carrying content and recovered no
+   * text from any of them. Only a `Converter` conversion produces this, so
+   * never in a build without the `ocr` feature.
+   */
+  | 'ocr'
+  /**
+   * The OCR models could not be loaded: one is missing, is not a
+   * `Uint8Array`, or the pair could not initialize an engine. Only the
+   * `Converter` constructor produces this.
+   */
+  | 'ocrInit'
+
+/**
+ * The RTen models local OCR needs, as bytes the caller fetched. Passed to
+ * `new Converter(...)`, which is present only in a build with the `ocr`
+ * feature.
+ */
+export interface OcrModels {
+  detectionModel: Uint8Array
+  recognitionModel: Uint8Array
+}
 
 export interface Document {
   blocks: Array
diff --git a/wasm/test-ocr.mjs b/wasm/test-ocr.mjs
new file mode 100644
index 0000000..98ae2d6
--- /dev/null
+++ b/wasm/test-ocr.mjs
@@ -0,0 +1,73 @@
+// Real-model OCR test: the browser converter reading a scanned PDF.
+//
+// Needs an OCR build and three paths from the environment, and skips itself
+// when any is missing:
+//   wasm-pack build wasm --release --target web -- --features ocr
+//   ANYDOC_OCR_DETECTION_MODEL=... ANYDOC_OCR_RECOGNITION_MODEL=... \
+//   ANYDOC_PDF_SAMPLE_FILES=... node --test wasm/test-ocr.mjs
+import assert from 'node:assert/strict'
+import { createHash } from 'node:crypto'
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { test } from 'node:test'
+
+import * as anydoc from './pkg/anydoc_wasm.js'
+
+const DETECTION_MODEL = process.env.ANYDOC_OCR_DETECTION_MODEL
+const RECOGNITION_MODEL = process.env.ANYDOC_OCR_RECOGNITION_MODEL
+const SAMPLE_FILES = process.env.ANYDOC_PDF_SAMPLE_FILES
+
+// The pinned scanned page: its only extractable text is a tool watermark, so
+// everything asserted below has to come from recognition.
+const SCAN = '018-base64-image/base64image.pdf'
+const SCAN_SHA256 = 'aaad90df16fce40ec768629d2135479b98f65b39bb27c7f80fb106393187d619'
+
+const missing = [
+  ['ANYDOC_OCR_DETECTION_MODEL', DETECTION_MODEL],
+  ['ANYDOC_OCR_RECOGNITION_MODEL', RECOGNITION_MODEL],
+  ['ANYDOC_PDF_SAMPLE_FILES', SAMPLE_FILES],
+]
+  .filter(([, value]) => !value)
+  .map(([name]) => name)
+
+const skip = missing.length
+  ? `set ${missing.join(', ')} to run`
+  : anydoc.Converter
+    ? false
+    : 'built without --features ocr'
+
+if (!skip) {
+  anydoc.initSync({
+    module: await readFile(fileURLToPath(new URL('./pkg/anydoc_wasm_bg.wasm', import.meta.url))),
+  })
+}
+
+const normalize = (markdown) => markdown.toLowerCase().split(/\s+/).filter(Boolean).join(' ')
+
+const build = async () =>
+  new anydoc.Converter({
+    detectionModel: new Uint8Array(await readFile(DETECTION_MODEL)),
+    recognitionModel: new Uint8Array(await readFile(RECOGNITION_MODEL)),
+  })
+
+const scan = async () => {
+  const bytes = new Uint8Array(await readFile(join(SAMPLE_FILES, SCAN)))
+  const digest = createHash('sha256').update(bytes).digest('hex')
+  assert.equal(digest, SCAN_SHA256, 'the pinned corpus file changed')
+  return bytes
+}
+
+test('the converter recognizes a scanned page in wasm', { skip }, async () => {
+  const markdown = (await build()).toMarkdownBytes(await scan(), 'pdf')
+
+  console.log(`018 output (verbatim):\n${markdown}`)
+  assert.match(normalize(markdown), /fix the issue and close it/)
+})
+
+test('one converter reads several documents', { skip }, async () => {
+  const converter = await build()
+  const bytes = await scan()
+
+  assert.equal(converter.toMarkdownBytes(bytes, 'pdf'), converter.toMarkdownBytes(bytes, 'pdf'))
+})
diff --git a/wasm/test.mjs b/wasm/test.mjs
index afcebfc..c92af55 100644
--- a/wasm/test.mjs
+++ b/wasm/test.mjs
@@ -5,14 +5,16 @@ import { readFile } from 'node:fs/promises'
 import { fileURLToPath } from 'node:url'
 import { test } from 'node:test'
 
-import {
+import * as anydoc from './pkg/anydoc_wasm.js'
+
+const {
   initSync,
   formatFromBytes,
   formatFromExtension,
   formatFromPath,
   toDocument,
   toMarkdownBytes,
-} from './pkg/anydoc_wasm.js'
+} = anydoc
 
 const fixture = (name) => fileURLToPath(new URL(`../tests/fixtures/${name}`, import.meta.url))
 
@@ -83,3 +85,37 @@ test('conversion errors throw a coded Error', () => {
   throws(() => toMarkdownBytes(ENCRYPTED, 'odt'), 'encrypted', /encrypted/)
   throws(() => toDocument(ENCRYPTED, 'odt'), 'encrypted', /encrypted/)
 })
+
+// The Converter exists only in a build with the `ocr` feature, so the same
+// suite covers both. Which build this is has to come from the caller:
+// ANYDOC_WASM_OCR_BUILD=1 for an OCR build, 0 for a plain one. A test that
+// read it off the module could not tell a broken feature gate from a build
+// that never had the feature.
+const { Converter } = anydoc
+const OCR_BUILD = process.env.ANYDOC_WASM_OCR_BUILD
+
+test('the OCR converter is present exactly when the build says it is', {
+  skip: OCR_BUILD === undefined && 'set ANYDOC_WASM_OCR_BUILD=1 or 0 to assert the feature gate',
+}, () => {
+  assert.ok(['0', '1'].includes(OCR_BUILD), `ANYDOC_WASM_OCR_BUILD has to be 1 or 0, got ${OCR_BUILD}`)
+
+  if (OCR_BUILD === '1') {
+    assert.equal(typeof Converter, 'function', 'an OCR build has to export Converter')
+  } else {
+    assert.equal(Converter, undefined, 'a build without --features ocr may not export Converter')
+  }
+})
+
+test('the converter rejects model bytes it cannot use', { skip: !Converter && 'built without --features ocr' }, () => {
+  const throws = (models, message) =>
+    assert.throws(() => new Converter(models), (error) => {
+      assert.ok(error instanceof Error)
+      assert.equal(error.code, 'ocrInit')
+      assert.match(error.message, message)
+      return true
+    })
+
+  throws({ detectionModel: new Uint8Array([0, 1, 2, 3]), recognitionModel: new Uint8Array([4, 5]) }, /detection model/)
+  throws({ recognitionModel: new Uint8Array([4, 5]) }, /detectionModel is required/)
+  throws({ detectionModel: 'not bytes', recognitionModel: new Uint8Array([4, 5]) }, /detectionModel has to be a Uint8Array/)
+})
diff --git a/wasm/www/index.html b/wasm/www/index.html
index 2e610db..7c34f2c 100644
--- a/wasm/www/index.html
+++ b/wasm/www/index.html
@@ -702,9 +702,11 @@ 
About

               Text-based PDFs convert locally through
               pdf-inspector. Scanned pages need OCR, which
-              Firecrawl Parse adds on
-              top of this same conversion.
+              >. Scanned pages need OCR: this page loads the models on demand
+              (12 MB, only when you drop a scan). The package ships an optional
+              ocr feature that reads them in the browser, and
+              Firecrawl Parse adds it
+              on top of this same conversion as a hosted API.
             
 
             
This page

@@ -770,6 +772,9 @@ 
Install

         Source
         Issues
         Built by Firecrawl
+        ocrs models by Robert Knight, CC-BY-SA-4.0
       
     
 
@@ -779,6 +784,9 @@ 
Install

         formatFromPath,
         toMarkdownBytes,
       } from './pkg/anydoc_wasm.js';
+      // Also as a namespace: only a build with the `ocr` feature exports
+      // Converter, and naming a missing export fails the whole import.
+      import * as anydoc from './pkg/anydoc_wasm.js';
 
       const $ = (id) => document.getElementById(id);
       const dropTitle = $('drop-title');
@@ -788,6 +796,18 @@ 
Install

       let markdown = '';
       let baseName = 'document';
 
+      const hasOcr = 'Converter' in anydoc;
+      // 12 MB of models, so they are fetched the first time a document
+      // actually needs them and kept for the rest of the session. The sizes
+      // are pinned: a truncated or wrong file fails here, saying so, instead
+      // of somewhere inside the engine.
+      const OCR_MODELS = [
+        ['detectionModel', 'models/text-detection.rten', 2510284],
+        ['recognitionModel', 'models/text-recognition.rten', 9716568],
+      ];
+      const OCR_BYTES = 12226852;
+      let converterReady;
+
       function showResult(name, format, stats, text, isError) {
         $('result').hidden = false;
         $('result-name').textContent = name;
@@ -800,29 +820,103 @@ 
Install

         $('result').scrollIntoView({ behavior: 'smooth', block: 'nearest' });
       }
 
+      // Progress reuses the error layout — one line, no buttons — without
+      // the error color.
+      function showStatus(name, text) {
+        showResult(name, '', '', text, true);
+        output.classList.remove('error');
+      }
+
+      function showFailure(name, format, error) {
+        showResult(
+          name,
+          format,
+          '',
+          `Could not convert this file: ${error.message ?? error}`,
+          true,
+        );
+      }
+
+      // The one render path: time the conversion and put the Markdown in the
+      // preview with its stats. Scans reach it through the OCR converter,
+      // every other document through the module functions.
+      function show(name, format, run) {
+        const started = performance.now();
+        markdown = run();
+        const ms = Math.max(1, Math.round(performance.now() - started));
+        const chars = markdown.length.toLocaleString('en-US');
+        showResult(name, format, `${chars} chars · ${ms} ms`, markdown, false);
+      }
+
+      // A scanned PDF reports 'unsupported' and says OCR is required; an
+      // image has no text layer to fall back on in the first place.
+      const needsOcr = (error, format) =>
+        format === 'image' ||
+        (error?.code === 'unsupported' &&
+          String(error.message ?? '').includes('OCR is required'));
+
       function convert(name, bytes) {
         const format = formatFromBytes(bytes) ?? formatFromPath(name);
         baseName = name.replace(/\.[^.]*$/, '') || 'document';
         try {
-          const started = performance.now();
-          markdown = toMarkdownBytes(bytes, format);
-          const ms = Math.max(1, Math.round(performance.now() - started));
-          const chars = markdown.length.toLocaleString('en-US');
-          showResult(
-            name,
-            format,
-            `${chars} chars · ${ms} ms`,
-            markdown,
-            false,
-          );
+          show(name, format, () => toMarkdownBytes(bytes, format));
         } catch (error) {
+          if (hasOcr && needsOcr(error, format)) recognize(name, bytes, format);
+          else showFailure(name, format, error);
+        }
+      }
+
+      async function loadModels(onProgress) {
+        onProgress('Loading OCR models (12 MB)…');
+        const models = {};
+        let loaded = 0;
+        for (const [key, path, size] of OCR_MODELS) {
+          const response = await fetch(path);
+          if (!response.ok) {
+            throw new Error(`${path} responded ${response.status}`);
+          }
+          const bytes = new Uint8Array(await response.arrayBuffer());
+          if (bytes.length !== size) {
+            throw new Error(`${path} is ${bytes.length} bytes, not ${size}`);
+          }
+          models[key] = bytes;
+          loaded += size;
+          const percent = Math.round((loaded / OCR_BYTES) * 100);
+          onProgress(`Loading OCR models (12 MB)… ${percent}%`);
+        }
+        return new anydoc.Converter(models);
+      }
+
+      // Recognition is CPU-bound and runs on the UI thread here, which one
+      // dropped document can afford. An app should keep it off the thread:
+      // wasm/examples/worker is that setup.
+      async function recognize(name, bytes, format) {
+        let converter;
+        try {
+          converterReady ??= loadModels((text) => showStatus(name, text));
+          converter = await converterReady;
+        } catch (error) {
+          converterReady = undefined;
           showResult(
             name,
             format,
             '',
-            `Could not convert this file: ${error.message ?? error}`,
+            `Could not load the OCR models: ${error.message ?? error}\n` +
+              'This page reads them from its models/ directory, which needs ' +
+              'text-detection.rten and text-recognition.rten in it.',
             true,
           );
+          return;
+        }
+        showStatus(name, 'Reading the page with OCR…');
+        // Let that paint: the recognition below blocks the thread.
+        await new Promise((resolve) =>
+          requestAnimationFrame(() => setTimeout(resolve, 0)),
+        );
+        try {
+          show(name, format, () => converter.toMarkdownBytes(bytes, format));
+        } catch (error) {
+          showFailure(name, format, error);
         }
       }