refactor: concentrate bulk-decode unsafe in a Utf8Writer

.github/workflows/nix.yml

@@ -129,6 +129,49 @@ jobs:
       - name: Run clippy (all features)
         run: nix develop --command cargo clippy --all-targets --all-features -- -D warnings
 
+  # Miri validates the hand-written `unsafe` in the bulk decoder's `Utf8Writer`
+  # (overshoot writes into uninitialized capacity, pointer provenance, cursor
+  # arithmetic, endianness). The libFuzzer targets can't run under Miri, so the
+  # deterministic `writer_stress` sweep is the harness that exercises it. Uses
+  # the nightly-with-Miri toolchain from the `.#miri` dev shell (flake.nix).
+  miri:
+    runs-on: ubuntu-latest
+    env:
+      MIRIFLAGS: -Zmiri-strict-provenance
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v21
+
+      - name: Setup Nix cache
+        uses: DeterminateSystems/magic-nix-cache-action@v13
+        with:
+          use-flakehub: false
+
+      - name: Miri setup
+        run: nix develop .#miri --command cargo miri setup
+
+      # Host (little-endian), strict Stacked Borrows.
+      - name: Miri — writer stress (Stacked Borrows)
+        run: nix develop .#miri --command cargo miri test --test writer_stress
+
+      # Tree Borrows: the stricter aliasing model Rust is moving toward.
+      - name: Miri — writer stress (Tree Borrows)
+        run: nix develop .#miri --command cargo miri test --test writer_stress
+        env:
+          MIRIFLAGS: -Zmiri-tree-borrows -Zmiri-strict-provenance
+
+      # Big-endian target: guards the `to_ne_bytes` / `to_le_bytes` split in the
+      # writer — a byte-order mistake shows up here but not on little-endian x86.
+      - name: Miri — writer stress (big-endian s390x)
+        run: nix develop .#miri --command cargo miri test --target s390x-unknown-linux-gnu --test writer_stress
+
+      # Lib unit tests under Miri cover the nonstandard-ASCII and
+      # ASCII-optimized-codepage paths through the same allocator.
+      - name: Miri — lib unit tests
+        run: nix develop .#miri --command cargo miri test --lib --all-features
+
   codegen:
     runs-on: ubuntu-latest
     steps:
@@ -159,12 +202,12 @@ jobs:
   # Final job that depends on all other jobs - use this in branch protection rules
   ci-success:
     runs-on: ubuntu-latest
-    needs: [build-and-test, no-std, fuzz, codegen, format, clippy]
+    needs: [build-and-test, no-std, fuzz, miri, codegen, format, clippy]
     if: always()
     steps:
       - name: Check all jobs passed
         run: |
-          if [[ "${{ needs.build-and-test.result }}" != "success" || "${{ needs.no-std.result }}" != "success" || "${{ needs.fuzz.result }}" != "success" || "${{ needs.codegen.result }}" != "success" || "${{ needs.format.result }}" != "success" || "${{ needs.clippy.result }}" != "success" ]]; then
+          if [[ "${{ needs.build-and-test.result }}" != "success" || "${{ needs.no-std.result }}" != "success" || "${{ needs.fuzz.result }}" != "success" || "${{ needs.miri.result }}" != "success" || "${{ needs.codegen.result }}" != "success" || "${{ needs.format.result }}" != "success" || "${{ needs.clippy.result }}" != "success" ]]; then
             echo "One or more jobs failed"
             exit 1
           fi

flake.nix

@@ -20,6 +20,17 @@
           extensions = [ "rust-src" "rust-analyzer" ];
           targets = [ "thumbv7em-none-eabi" ];
         };
+
+        # Nightly toolchain with Miri, used to validate the bulk decoder's
+        # hand-written `unsafe` (the `Utf8Writer`). `selectLatestNightlyWith`
+        # picks the most recent nightly on which `miri` actually built, so CI
+        # never trips over a nightly that shipped without the component.
+        # `rust-src` is required for Miri to build its sysroot (incl. the
+        # big-endian cross target).
+        miriToolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain:
+          toolchain.default.override {
+            extensions = [ "miri" "rust-src" ];
+          });
       in
       {
         devShells.default = pkgs.mkShell {
@@ -38,6 +49,14 @@
             echo "Rust: $(rustc --version)"
           '';
         };
+
+        # `nix develop .#miri --command cargo miri test ...`
+        devShells.miri = pkgs.mkShell {
+          packages = [ miriToolchain ];
+
+          RUST_SRC_PATH = "${miriToolchain}/lib/rustlib/src/rust/library";
+          RUST_BACKTRACE = "1";
+        };
       }
     );
 }

src/decoder.rs

@@ -60,38 +60,119 @@ impl Entry {
         unsafe { char::from_u32_unchecked(cp) }
     }
 
-    /// Store the entry's UTF-8 bytes at `*dst` and advance `dst` by `len`.
+    /// The entry packed as `[b0, b1, b2, len]` — little-endian on every target,
+    /// so the UTF-8 bytes land in sequence. Only the first [`len`](Self::len)
+    /// bytes are meaningful; the trailing `len` byte is overshoot to overwrite.
     #[cfg(feature = "alloc")]
     #[inline]
-    pub unsafe fn write_to(self, dst: &mut *mut u8) {
-        // SAFETY: the caller guarantees >= 4 writable bytes at `*dst`.
-        // `write_unaligned` needs no alignment (`dst` is a byte pointer into a
-        // `Vec<u8>`), and `to_le_bytes` fixes the in-memory order to
-        // `[b0, b1, b2, len]` on every target, so the UTF-8 bytes land in
-        // sequence regardless of host endianness.
-        dst.cast::<[u8; 4]>()
-            .write_unaligned(self.0.get().to_le_bytes());
-        // SAFETY: `len` is `1..=3`, so the advanced pointer stays within the
-        // reserved allocation
-        *dst = dst.add(self.len());
+    pub const fn utf8_word(self) -> [u8; 4] {
+        self.0.get().to_le_bytes()
     }
 }
 
 #[cfg(feature = "alloc")]
 const USIZE_SIZE: usize = mem::size_of::<usize>();
 
-/// Given [`buffer`] and end-ptr [`ptr`] set new length and shrink allocation
+/// A cursor that appends decoded UTF-8 into a [`Vec<u8>`]'s reserved capacity,
+/// then hands back a [`String`].
 ///
-/// # Safety
-///
-/// [`dst`] must be within allocated capacity of [`res`]
+/// It concentrates the bulk decoder's `unsafe` behind three primitives
+/// ([`push_entry`](Self::push_entry), [`push_ascii_word`](Self::push_ascii_word),
+/// [`finish`](Self::finish)); each documents the slack the caller must
+/// guarantee, and the cursor never leaves the allocation. The overshoot trick —
+/// `push_entry` stores a full 4-byte word but advances by the codepoint's
+/// `1..=3` bytes — lives here alone. Debug builds assert the slack on every
+/// write, so Miri and the fuzzers check it per byte.
 #[cfg(feature = "alloc")]
-#[inline]
-unsafe fn finalize_string(mut buffer: Vec<u8>, dst: *const u8) -> String {
-    let length = dst.offset_from(buffer.as_ptr()) as usize;
-    buffer.set_len(length);
-    buffer.shrink_to_fit();
-    String::from_utf8_unchecked(buffer)
+pub(crate) struct Utf8Writer {
+    buf: Vec<u8>,
+    /// Write cursor into `buf`'s spare capacity.
+    dst: *mut u8,
+    /// One-past-the-end sentinel; only its address is read, so the slack check
+    /// never reborrows `buf` (keeping provenance clean).
+    end: *mut u8,
+}
+
+#[cfg(feature = "alloc")]
+impl Utf8Writer {
+    /// Reserve enough capacity to decode `input_len` bytes.
+    ///
+    /// Each byte yields at most 3 UTF-8 bytes and the final
+    /// [`push_entry`](Self::push_entry) may overshoot by a full 4-byte word, so
+    /// `input_len * 3 + 1` is always enough and never leaves under 4 bytes of
+    /// slack before the last entry — the `push_*` preconditions then hold for
+    /// any full decode by construction.
+    #[inline]
+    pub fn for_input_len(input_len: usize) -> Self {
+        let cap = input_len * 3 + 1;
+        let mut buf: Vec<u8> = Vec::with_capacity(cap);
+        let dst = buf.as_mut_ptr();
+        // SAFETY: `Vec::with_capacity` reserved `cap` bytes, so `dst.add(cap)` is
+        // the one-past-the-end sentinel of a single allocation.
+        let end = unsafe { dst.add(cap) };
+        Self { buf, dst, end }
+    }
+
+    /// Bytes of reserved capacity still available at the cursor. Address-only
+    /// arithmetic, so it neither dereferences nor reborrows `buf`.
+    #[inline]
+    fn remaining(&self) -> usize {
+        self.end as usize - self.dst as usize
+    }
+
+    /// Append one decoded codepoint.
+    ///
+    /// Stores a fixed 4-byte word (the 3 UTF-8 bytes plus a length byte the next
+    /// write or [`finish`](Self::finish) overwrites) and advances by the real
+    /// length. The unconditional store is what keeps the loop branchless.
+    ///
+    /// # Safety
+    /// At least 4 bytes of capacity must remain at the cursor.
+    #[inline]
+    pub unsafe fn push_entry(&mut self, entry: Entry) {
+        debug_assert!(self.remaining() >= 4, "push_entry overran buffer");
+        // SAFETY: caller guarantees >= 4 writable bytes; `write_unaligned` needs
+        // no alignment for a byte cursor.
+        self.dst
+            .cast::<[u8; 4]>()
+            .write_unaligned(entry.utf8_word());
+        // SAFETY: `len` is `1..=3`, so the cursor stays within the allocation.
+        self.dst = self.dst.add(entry.len());
+    }
+
+    /// Append a whole `usize` word of ASCII bytes verbatim (word-at-a-time fast
+    /// path). `to_ne_bytes` keeps the source word's in-memory order, matching a
+    /// raw byte copy without a transmute.
+    ///
+    /// # Safety
+    /// At least `size_of::<usize>()` bytes of capacity must remain at the cursor.
+    #[inline]
+    pub unsafe fn push_ascii_word(&mut self, word: usize) {
+        debug_assert!(
+            self.remaining() >= USIZE_SIZE,
+            "push_ascii_word overran buffer"
+        );
+        // SAFETY: caller guarantees >= USIZE_SIZE writable bytes.
+        self.dst
+            .cast::<[u8; USIZE_SIZE]>()
+            .write_unaligned(word.to_ne_bytes());
+        self.dst = self.dst.add(USIZE_SIZE);
+    }
+
+    /// Set the final length, shrink, and reinterpret as a `String`.
+    ///
+    /// # Safety
+    /// Every byte written so far must form valid UTF-8 (the decode tables
+    /// guarantee this).
+    #[inline]
+    pub unsafe fn finish(mut self) -> String {
+        // SAFETY: `dst` and `buf`'s base share the same allocation, and `dst`
+        // never moved past `end`, so the offset is the written length.
+        let length = self.dst.offset_from(self.buf.as_ptr()) as usize;
+        self.buf.set_len(length);
+        self.buf.shrink_to_fit();
+        String::from_utf8_unchecked(self.buf)
+    }
 }
 
 //lifted from std internal

src/decoder/complete.rs

@@ -1,13 +1,11 @@
 #[cfg(feature = "alloc")]
 use alloc::borrow::Cow;
 #[cfg(feature = "alloc")]
-use alloc::vec::Vec;
-#[cfg(feature = "alloc")]
 use core::mem;
 
 use super::Entry;
 #[cfg(feature = "alloc")]
-use super::{contains_nonascii, finalize_string, USIZE_SIZE};
+use super::{contains_nonascii, Utf8Writer, USIZE_SIZE};
 
 pub(crate) type Table = [Entry; 256];
 
@@ -19,10 +17,7 @@ pub(crate) fn decode_helper<'a>(table: &Table, src: &'a [u8]) -> Cow<'a, str> {
         return s.into();
     }
 
-    // +1 for branchless 4-byte write which may overshoot by 1 byte
-    let mut buffer: Vec<u8> = Vec::with_capacity(src.len() * 3 + 1);
-    // SAFETY: decode_slice expects buffer.len() >= src.len() * 3
-    let mut dst = buffer.as_mut_ptr();
+    let mut writer = Utf8Writer::for_input_len(src.len());
 
     // If we wouldn't gain anything from the word-at-a-time implementation, fall
     // back to a scalar loop.
@@ -31,27 +26,22 @@ pub(crate) fn decode_helper<'a>(table: &Table, src: &'a [u8]) -> Cow<'a, str> {
     // sufficient alignment for `usize`, because it's a weird edge case.
     unsafe {
         if src.len() < USIZE_SIZE || USIZE_SIZE < mem::align_of::<usize>() {
-            decode_slice(table, src, &mut dst);
-            return finalize_string(buffer, dst).into();
+            decode_slice(table, src, &mut writer);
+            return writer.finish().into();
         }
 
         let (prefix, aligned_bytes, suffix) = src.align_to::<usize>();
-        decode_slice(table, prefix, &mut dst);
+        decode_slice(table, prefix, &mut writer);
         for chunk in aligned_bytes {
             if contains_nonascii(*chunk) {
-                decode_slice(
-                    table,
-                    mem::transmute::<&usize, &[u8; USIZE_SIZE]>(chunk),
-                    &mut dst,
-                );
+                decode_slice(table, &chunk.to_ne_bytes(), &mut writer);
             } else {
-                dst.copy_from_nonoverlapping(chunk as *const usize as *const u8, USIZE_SIZE);
-                dst = dst.add(USIZE_SIZE)
+                writer.push_ascii_word(*chunk);
             }
         }
 
-        decode_slice(table, suffix, &mut dst);
-        finalize_string(buffer, dst).into()
+        decode_slice(table, suffix, &mut writer);
+        writer.finish().into()
     }
 }
 
@@ -60,23 +50,21 @@ pub(crate) fn decode_helper<'a>(table: &Table, src: &'a [u8]) -> Cow<'a, str> {
 #[cfg(feature = "alloc")]
 #[inline(always)]
 pub(crate) fn decode_helper_non_ascii<'a>(table: &Table, bytes: &'a [u8]) -> Cow<'a, str> {
-    // +1 for branchless 4-byte write which may overshoot by 1 byte
-    let mut buffer: Vec<u8> = Vec::with_capacity(bytes.len() * 3 + 1);
-    // SAFETY: decode_slice expects buffer.len() >= src.len() * 3
-    let mut dst = buffer.as_mut_ptr();
-    unsafe { decode_slice(table, bytes, &mut dst) };
-    unsafe { finalize_string(buffer, dst) }.into()
+    let mut writer = Utf8Writer::for_input_len(bytes.len());
+    unsafe { decode_slice(table, bytes, &mut writer) };
+    unsafe { writer.finish() }.into()
 }
 
-/// Lookup every byte in [`src`] using provided [`table`] and write resulting bytes to [`dst`]
+/// Look up every byte in [`src`] using [`table`] and append the decoded UTF-8 to
+/// [`writer`].
+///
 /// # Safety
 ///
-/// This function is unsafe because it assumes that the buffer pointed to by [`dst`] has a length >= src.len() * 3
+/// `writer` must have at least `src.len() * 3 + 1` bytes of capacity remaining.
 #[cfg(feature = "alloc")]
 #[inline]
-unsafe fn decode_slice(table: &Table, src: &[u8], dst: &mut *mut u8) {
-    for b in src {
-        let entry = table[*b as usize];
-        entry.write_to(dst);
+unsafe fn decode_slice(table: &Table, src: &[u8], writer: &mut Utf8Writer) {
+    for &b in src {
+        writer.push_entry(table[b as usize]);
     }
 }