fix(pdf): codepoint-aware symbol fallback; footnote cells keep fallback (#72)

CHANGELOG.md

@@ -22,6 +22,16 @@ Format follows [Keep a Changelog](https://keepachangelog.com/).
   footnote markers match the exported HTML/PDF instead of showing as ordinary inline
   text. Footnote markers are clickable via the existing `LinkClicked` event. The new
   `AvaloniaRenderTheme.BodyFontSize` is the reference size for sizing super/subscript.
+- **PDF: symbol font-fallback now handles non-BMP characters and footnote-bearing cells (#72).**
+  Two cases still emitted `?` despite the #52 fallback: (A) a non-BMP character
+  (e.g. `🛇`) came out as `??` because the fallback iterated UTF-16 code
+  units instead of Unicode codepoints, so each surrogate half became a `?`; and
+  (B) a `footnote:[…]` in a table cell disabled symbol fallback for the rest of
+  that cell (a regression from #69's segment-based cell rendering), so `⇒`/`✓`
+  next to a footnote dropped back to `?`. The fallback, glyph encoder, and WinAnsi
+  escaping now iterate by codepoint (a non-BMP glyph routes to the fallback font,
+  or collapses to a single missing-glyph indicator — never `??`), and the
+  footnote-bearing cell path runs the same fallback expansion as body text.
 
 ## [1.0.18] - 2026-06-19
 

src/AdocNet.Converters.Pdf/PdfFontEmbedder.cs

@@ -149,12 +149,36 @@ internal static string BuildToUnicodeCMap(TrueTypeFont font, HashSet<int> usedCo
         return sb.ToString();
     }
 
+    /// <summary>
+    /// Enumerates the Unicode codepoints of <paramref name="text"/>, combining a
+    /// surrogate pair into the single codepoint it encodes. Iterating codepoints
+    /// (rather than UTF-16 code units) keeps non-BMP characters whole, so they map
+    /// to one glyph rather than two missing-glyph halves (issue #72).
+    /// </summary>
+    internal static IEnumerable<int> EnumerateCodePoints(string text)
+    {
+        for (int i = 0; i < text.Length;)
+        {
+            char c = text[i];
+            if (char.IsHighSurrogate(c) && i + 1 < text.Length && char.IsLowSurrogate(text[i + 1]))
+            {
+                yield return char.ConvertToUtf32(c, text[i + 1]);
+                i += 2;
+            }
+            else
+            {
+                yield return c;
+                i++;
+            }
+        }
+    }
+
     internal static string EncodeTextAsGlyphIds(string text, TrueTypeFont font)
     {
         var sb = new StringBuilder(text.Length * 4);
-        foreach (var ch in text)
+        foreach (var cp in EnumerateCodePoints(text))
         {
-            var gid = font.GetGlyphId(ch);
+            var gid = font.GetGlyphId(cp);
             sb.Append($"{gid:X4}");
         }
         return sb.ToString();
@@ -164,8 +188,8 @@ internal static void TrackCodePoints(Dictionary<string, HashSet<int>> usedCodePo
     {
         if (usedCodePoints.TryGetValue(fontKey, out var codePoints))
         {
-            foreach (var ch in text)
-                codePoints.Add(ch);
+            foreach (var cp in EnumerateCodePoints(text))
+                codePoints.Add(cp);
         }
     }
 

src/AdocNet.Converters.Pdf/PdfRenderer.Blocks.cs

@@ -679,6 +679,11 @@ private void RenderCellSegments(PdfWriter w, PlacedCell cell, float x, float cel
                 segments.Add(new TextSegment(run.Text, font, fontSize));
         }
 
+        // Route symbols the cell font can't show (✓, ⇒, …) to the Unicode fallback
+        // font, exactly as the body-text path does — a footnote in the cell must not
+        // disable symbol fallback for its sibling runs (issues #52, #72).
+        segments = w.ExpandSegmentsForFallback(segments);
+
         float lineY = baseY;
         foreach (var line in w.WrapSegments(segments, availWidth))
         {

src/AdocNet.Converters.Pdf/PdfWriter.Fallback.cs

@@ -53,26 +53,28 @@ private List<FallbackCandidate> FallbackCandidates()
         return _fallbackCandidates;
     }
 
-    /// <summary>True when <paramref name="ch"/> cannot be shown by <paramref name="primaryFont"/>.</summary>
-    private bool NeedsFallback(char ch, string primaryFont)
+    /// <summary>True when <paramref name="codePoint"/> cannot be shown by <paramref name="primaryFont"/>.</summary>
+    private bool NeedsFallback(int codePoint, string primaryFont)
     {
         if (_embeddedFonts.TryGetValue(primaryFont, out var ttf))
-            return ttf.GetGlyphId(ch) == 0;
-        // Standard WinAnsi base font: representable when ≤ 0xFF or explicitly mapped.
-        return ch > 0xFF && MapUnicodeToWinAnsi(ch) == "?";
+            return ttf.GetGlyphId(codePoint) == 0;
+        // Standard WinAnsi base font: a non-BMP codepoint is never representable;
+        // a BMP one is representable when ≤ 0xFF or explicitly mapped.
+        if (codePoint > 0xFFFF) return true;
+        return codePoint > 0xFF && MapUnicodeToWinAnsi((char)codePoint) == "?";
     }
 
     /// <summary>
-    /// Resolves the font key that should render <paramref name="ch"/>: the primary
-    /// font when it can show it, otherwise the first fallback font that has the
-    /// glyph (registered on first use), otherwise the primary font (renders '?').
+    /// Resolves the font key that should render <paramref name="codePoint"/>: the
+    /// primary font when it can show it, otherwise the first fallback font that has
+    /// the glyph (registered on first use), otherwise the primary font (renders '?').
     /// </summary>
-    private string FontForChar(char ch, string primaryFont)
+    private string FontForCodePoint(int codePoint, string primaryFont)
     {
-        if (!NeedsFallback(ch, primaryFont)) return primaryFont;
+        if (!NeedsFallback(codePoint, primaryFont)) return primaryFont;
         foreach (var cand in FallbackCandidates())
         {
-            if (cand.Font.GetGlyphId(ch) != 0)
+            if (cand.Font.GetGlyphId(codePoint) != 0)
             {
                 cand.Key ??= RegisterEmbeddedFont($"__fb{_embeddedFonts.Count}", cand.Font);
                 return cand.Key;
@@ -84,6 +86,8 @@ private string FontForChar(char ch, string primaryFont)
     /// <summary>
     /// Splits <paramref name="text"/> into consecutive runs that share a render
     /// font, routing characters the primary font can't show to a fallback font.
+    /// Iterates by Unicode codepoint so a non-BMP character (a UTF-16 surrogate
+    /// pair) is treated as one glyph rather than two missing halves (issue #72).
     /// Returns <c>null</c> (fast path) when the whole string renders in
     /// <paramref name="primaryFont"/> — the common case — so ordinary text is
     /// measured and emitted exactly as before.
@@ -93,23 +97,23 @@ private string FontForChar(char ch, string primaryFont)
         if (string.IsNullOrEmpty(text)) return null;
 
         bool needsAny = false;
-        foreach (var ch in text)
-            if (NeedsFallback(ch, primaryFont)) { needsAny = true; break; }
+        foreach (var cp in PdfFontEmbedder.EnumerateCodePoints(text))
+            if (NeedsFallback(cp, primaryFont)) { needsAny = true; break; }
         if (!needsAny) return null;
 
         var runs = new List<(string, string)>();
         var sb = new StringBuilder();
         string runFont = primaryFont;
-        foreach (var ch in text)
+        foreach (var cp in PdfFontEmbedder.EnumerateCodePoints(text))
         {
-            string f = FontForChar(ch, primaryFont);
+            string f = FontForCodePoint(cp, primaryFont);
             if (sb.Length > 0 && f != runFont)
             {
                 runs.Add((sb.ToString(), runFont));
                 sb.Clear();
             }
             if (sb.Length == 0) runFont = f;
-            sb.Append(ch);
+            sb.Append(char.ConvertFromUtf32(cp));
         }
         if (sb.Length > 0) runs.Add((sb.ToString(), runFont));
         return runs;

src/AdocNet.Converters.Pdf/PdfWriter.Rendering.cs

@@ -679,30 +679,32 @@ private static void ReplaceTotalPagesPlaceholderTrueType(byte[] data, int totalP
     private static string EscapePdfString(string text)
     {
         var sb = new StringBuilder(text.Length);
-        foreach (var ch in text)
+        // Iterate by codepoint so a non-BMP character (a surrogate pair) becomes a
+        // single missing-glyph indicator rather than two '?'s (issue #72).
+        foreach (var cp in PdfFontEmbedder.EnumerateCodePoints(text))
         {
-            switch (ch)
+            if (cp == '(') sb.Append("\\(");
+            else if (cp == ')') sb.Append("\\)");
+            else if (cp == '\\') sb.Append("\\\\");
+            else if (cp < 128)
             {
-                case '(':  sb.Append("\\("); break;
-                case ')':  sb.Append("\\)"); break;
-                case '\\': sb.Append("\\\\"); break;
-                default:
-                    if (ch < 128)
-                    {
-                        sb.Append(ch);
-                    }
-                    else if (ch <= 255)
-                    {
-                        // WinAnsiEncoding: emit as octal escape
-                        sb.Append('\\');
-                        sb.Append(Convert.ToString(ch, 8).PadLeft(3, '0'));
-                    }
-                    else
-                    {
-                        // Outside WinAnsi range — best effort: try to map common Unicode chars
-                        sb.Append(MapUnicodeToWinAnsi(ch));
-                    }
-                    break;
+                sb.Append((char)cp);
+            }
+            else if (cp <= 255)
+            {
+                // WinAnsiEncoding: emit as octal escape
+                sb.Append('\\');
+                sb.Append(Convert.ToString(cp, 8).PadLeft(3, '0'));
+            }
+            else if (cp <= 0xFFFF)
+            {
+                // Outside WinAnsi range — best effort: map common Unicode chars
+                sb.Append(MapUnicodeToWinAnsi((char)cp));
+            }
+            else
+            {
+                // Non-BMP, not representable in a WinAnsi base font.
+                sb.Append('?');
             }
         }
         return sb.ToString();

tests/AdocNet.Tests/Converters/Pdf/PdfSymbolFallbackTests.cs

@@ -0,0 +1,90 @@
+using System.Text;
+using System.Text.RegularExpressions;
+using AdocNet.Converters.Pdf;
+using AdocNet.Parser;
+
+namespace AdocNet.Tests.Converters.Pdf;
+
+/// <summary>
+/// Regression tests for issue #72: the PDF symbol font-fallback must (A) treat a
+/// non-BMP character as one codepoint — emitting a single missing-glyph indicator
+/// rather than two <c>?</c> — and (B) keep routing symbols to the fallback font in
+/// a block that also contains a footnote (the table-cell path regressed in #69).
+/// </summary>
+[TestFixture]
+public class PdfSymbolFallbackTests
+{
+    private static byte[] Render(string adoc) =>
+        new PdfRenderer().RenderToBytes(AdocParser.Parse(adoc).Document, PdfRenderOptions.A4);
+
+    private static string Raw(byte[] pdf) => Encoding.GetEncoding("ISO-8859-1").GetString(pdf);
+
+    // The concatenated literal (parenthesized) Tj strings — the WinAnsi/base-font
+    // runs, where a glyph the font can't show appears as a literal '?'. Symbols
+    // routed to the embedded fallback font are emitted as hex <…> Tj instead, so
+    // they do not appear here.
+    private static string LiteralTjText(string raw)
+    {
+        var sb = new StringBuilder();
+        foreach (Match m in Regex.Matches(raw, @"\(((?:[^()\\]|\\.)*)\)\s*Tj"))
+            sb.Append(m.Groups[1].Value);
+        return sb.ToString();
+    }
+
+    private static int HexTjRuns(string raw) => Regex.Matches(raw, @"<[0-9A-Fa-f]+>\s*Tj").Count;
+
+    [Test]
+    public void Non_bmp_char_emits_a_single_missing_glyph_not_double()
+    {
+        // U+1F6C7 (PROHIBITED SIGN) is a non-BMP codepoint not covered by the
+        // fallback font — it must collapse to one '?', never '??' (two surrogate
+        // halves). The BMP arrow ⇒ on the same line still routes to the fallback.
+        var raw = Raw(Render("= T\n\nProhibited &#x1F6C7; sign and arrow => Z.\n"));
+        var literal = LiteralTjText(raw);
+
+        Assert.That(literal, Does.Not.Contain("??"), "a non-BMP char must not become two question marks");
+        Assert.That(literal.Count(c => c == '?'), Is.EqualTo(1), "exactly one missing-glyph indicator");
+        Assert.That(HexTjRuns(raw), Is.GreaterThanOrEqualTo(1), "the ⇒ arrow should route to the embedded fallback font");
+    }
+
+    [Test]
+    public void Footnote_in_table_cell_does_not_disable_symbol_fallback()
+    {
+        // The ⇒ in a cell that also has a footnote must still render via the
+        // fallback font (embedded), not regress to '?' in the base font (#72/#69).
+        var doc =
+            "= T\n\n" +
+            "|===\n" +
+            "| Plain => K alone\n" +
+            "| => K footnote:[a note] mixed\n" +
+            "|===\n";
+        var raw = Raw(Render(doc));
+
+        Assert.That(LiteralTjText(raw), Does.Not.Contain("?"),
+            "no symbol should fall back to a base-font '?' — the footnote cell's ⇒ must use the fallback font");
+        Assert.That(HexTjRuns(raw), Is.GreaterThanOrEqualTo(1), "the arrows should be emitted via the embedded fallback font");
+    }
+
+    [Test]
+    public void Check_mark_in_footnote_cell_routes_to_fallback()
+    {
+        var doc =
+            "= T\n\n" +
+            "|===\n" +
+            "| ✓ done footnote:[a note]\n" +
+            "|===\n";
+        var raw = Raw(Render(doc));
+
+        Assert.That(LiteralTjText(raw), Does.Not.Contain("?"),
+            "the check mark in a footnote cell must route to the fallback font, not '?'");
+    }
+
+    [Test]
+    public void Symbol_in_a_plain_paragraph_still_routes_to_fallback()
+    {
+        // Regression guard for the #52 behaviour that must remain intact.
+        var raw = Raw(Render("= T\n\nArrow => Z and check ✓ here.\n"));
+        Assert.That(LiteralTjText(raw), Does.Not.Contain("?"));
+        Assert.That(HexTjRuns(raw), Is.GreaterThanOrEqualTo(1));
+    }
+}