Approach A: moi scratch diagram — declarative diagrams with ELK auto-layout

docs/moi-scratchpad.md

@@ -38,11 +38,63 @@ The agent works the canvas through a `moi scratch` CLI — it both **sees** and
 
 `read` is for logic; `view` / `read-image` are for vision.
 
-### Drawing
+### Drawing diagrams
 
-The agent doesn't emit raw tldraw records. `moi scratch` exposes a small **primitive layer**
-that maps onto tldraw's own shape API — friendly to drive and stable against tldraw's
-internal schema:
+For anything boxes-and-arrows shaped — architectures, flows, pipelines, how-X-works — the
+agent shouldn't place shapes by hand. `moi scratch diagram` takes a **declarative JSON
+spec** (nodes, groups, labeled edges, a title) and compiles it onto the canvas: every label
+is measured with the actual canvas font, nodes are sized to fit, and positions come from
+ELK's layered auto-layout. Geometry is non-overlapping, aligned, and evenly spaced by
+construction — no coordinates in the spec at all.
+
+```
+moi scratch diagram [--spec file.json] [--at x,y] [--id prefix]
+```
+
+No `--spec` reads the spec from stdin (pipe a heredoc). `--at` pins the diagram's top-left
+corner; omitted, it auto-places below whatever is already on the canvas. `--id` sets the
+prefix of the created shape ids — every shape lands as `<prefix>-<nodeId>`,
+`<prefix>-title`, `<prefix>-edge-<i>` (printed on success), so the agent can `move`, `set`,
+or `delete` the pieces afterwards.
+
+The spec:
+
+```jsonc
+{
+  "title": "How to expose a Tailscale service", // optional; big text above the diagram
+  "direction": "right", // main flow: "right" (default) or "down"
+  "nodes": [
+    // shape: "rect" (default, sized to fit its label) | "note" (sticky, fixed 200×200);
+    // color/fill take the same values as the CLI flags; width = optional wrap-width hint
+    { "id": "browser", "label": "Browser (internet)", "color": "blue" }
+  ],
+  "groups": [
+    // a container rect drawn around its children (nodes or nested groups)
+    {
+      "id": "tailnet",
+      "label": "Tailscale network (private)",
+      "color": "grey",
+      "children": ["svc"]
+    }
+  ],
+  "edges": [
+    // arrows bound to their endpoint shapes; "elbow": true for right-angle routing
+    { "from": "browser", "to": "proxy", "label": "https://app.yourdomain.com", "color": "blue" }
+  ]
+}
+```
+
+Edges are real bound arrows (they follow their shapes when moved later), groups render as
+outline rects with the label at the top, and the whole diagram goes in as one batch —
+either all of it lands or none. Spec mistakes (unknown edge endpoint, duplicate id, bad
+color) come back as descriptive errors naming the offending entry.
+
+### Drawing primitives
+
+For **annotations and one-off marks** — a note next to the user's sketch, a single arrow, a
+label — the agent uses the primitive layer directly. It doesn't emit raw tldraw records:
+`moi scratch` exposes a small set that maps onto tldraw's own shape API — friendly to drive
+and stable against tldraw's internal schema:
 
 ```
 moi scratch add text   --at <x,y> --text "..."          [--id NAME] [--color C] [--font-size S]
@@ -80,8 +132,8 @@ moi scratch clear                           # wipe the whole canvas
 
 The set is deliberately small — text, rect, note, arrow (with color plus each shape's
 fill/font-size/stroke), plus
-move/set/delete/clear. Enough to lay out a diagram or annotate the user's drawing; not a full
-tldraw API.
+move/set/delete/clear. Enough to annotate the user's drawing or drop a one-off shape; for
+laying out a whole diagram, `moi scratch diagram` does the geometry instead.
 
 ## How it works
 
@@ -104,6 +156,9 @@ on the server (the mutations). Only `view` — rendering pixels — genuinely re
   `add image` additionally resizes the file through `sharp` (the same dep the icon pipeline uses)
   before embedding it. (We drive the store, not an `Editor`, because the Editor needs a DOM + text
   measurement the server runtime doesn't have. See `server/scratchpad-executor.ts`.)
+  `diagram` runs through the same path with two extra stages up front: labels are measured with
+  the real canvas font (`server/scratchpad-metrics.ts`), then `elkjs` computes the layout in a
+  worker before the records are built (`server/scratchpad-diagram.ts`).
 - **Viewing** (`moi scratch view`) is the one op still relayed to a connected tab: only the
   browser can rasterize the canvas (`editor.toImageDataUrl`). With no tab showing **this**
   workspace's scratchpad it returns "No live canvas" — every other op still works off disk.

lib/types.ts

@@ -65,6 +65,50 @@ export type ScratchStyle = { color?: ScratchColor; size?: ScratchSize; fill?: Sc
 // the canvas light), 'hi' allows more pixels when detail matters.
 export type ScratchImageQuality = 'lo' | 'hi'
 
+// A declarative boxes-and-arrows diagram, compiled server-side by `moi scratch
+// diagram`: labels are measured, nodes sized to fit, and positions computed by
+// ELK auto-layout — the agent declares structure and never touches a coordinate.
+// `color`/`fill` carry the CLI's user-facing vocabulary (a palette name or hex;
+// none|semi|pattern|solid) as raw strings: the spec is agent input, so parsing
+// and validation happen in one place server-side (server/scratchpad-diagram.ts)
+// and errors travel back over the control socket.
+export type ScratchDiagramNode = {
+  id: string
+  label: string
+  // 'rect' (default): a geo rectangle sized to fit its label. 'note': a sticky
+  // note — fixed 200×200, so keep note labels short.
+  shape?: 'rect' | 'note'
+  color?: string
+  fill?: string
+  // Preferred label wrap width in px (default 260). A hint: the rect can come
+  // out wider for an unbreakable word, or narrower for a short label.
+  width?: number
+}
+export type ScratchDiagramGroup = {
+  id: string
+  label?: string
+  color?: string
+  // Ids of the nodes (or nested groups) drawn inside this container.
+  children: string[]
+}
+export type ScratchDiagramEdge = {
+  from: string
+  to: string
+  label?: string
+  color?: string
+  // Right-angle routing, like `add arrow --elbow`; default is a curved arc.
+  elbow?: boolean
+}
+export type ScratchDiagramSpec = {
+  // Big heading placed above the diagram bounds.
+  title?: string
+  // Main flow direction: 'right' (default) lays layers left→right, 'down' top→bottom.
+  direction?: 'right' | 'down'
+  nodes: ScratchDiagramNode[]
+  groups?: ScratchDiagramGroup[]
+  edges?: ScratchDiagramEdge[]
+}
+
 export type ScratchOp =
   | ({ kind: 'add-text'; name: string; x: number; y: number; text: string } & ScratchStyle)
   | ({
@@ -96,15 +140,25 @@ export type ScratchOp =
       // Right-angle (orthogonal) routing for clean diagrams; default is a curved arc.
       elbow?: boolean
     } & ScratchStyle)
+  // Compile a declarative diagram spec into laid-out shapes (see
+  // ScratchDiagramSpec). `name` prefixes every created shape id —
+  // `<name>-<nodeId>`, `<name>-title`, `<name>-edge-<i>` — so the agent can
+  // address the pieces afterwards. Omitted x/y auto-places the diagram below
+  // the existing canvas content.
+  | { kind: 'diagram'; name: string; spec: ScratchDiagramSpec; x?: number; y?: number }
   | { kind: 'move'; name: string; x: number; y: number }
   | { kind: 'set'; name: string; text: string }
   | { kind: 'delete'; name: string }
   | { kind: 'clear' }
   | { kind: 'view' }
 
-// What a tab returns after running an op: a shape's `name` for add ops, a PNG
-// data URL for `view`, or a bare ack for mutations.
-export type ScratchOpResult = { name: string } | { image: string } | { ok: true }
+// What a tab returns after running an op: a shape's `name` for add ops (plus
+// the full list of `created` shape ids for `diagram`), a PNG data URL for
+// `view`, or a bare ack for mutations.
+export type ScratchOpResult =
+  | { name: string; created?: string[] }
+  | { image: string }
+  | { ok: true }
 
 // An attachment uploaded ahead of a chat message. The bytes live server-side in
 // an in-memory upload store (see server/uploads.ts); a chat frame references it

package.json

@@ -70,6 +70,7 @@
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
     "devalue": "^5.6.4",
+    "elkjs": "^0.11.1",
     "env-paths": "^4.0.0",
     "fontkit": "^2.0.4",
     "frimousse": "^0.3.0",

server/cli.ts

@@ -10,11 +10,9 @@ import { COLOR_THEMES, FONT_THEMES } from '@/lib/themes'
 import type { ColorTheme, FontTheme } from '@/lib/themes'
 import type {
   ScratchArrowEnd,
-  ScratchColor,
-  ScratchFill,
+  ScratchDiagramSpec,
   ScratchImageQuality,
   ScratchOp,
-  ScratchSize,
   ScratchStyle
 } from '@/lib/types'
 
@@ -27,6 +25,16 @@ import {
 } from './cli-env'
 import { columns } from './cli-ui'
 import { CONTROL_PORT, PORT } from './constants'
+import {
+  COLOR_NAMES,
+  FILL_NAMES,
+  FONT_SIZE_NAMES,
+  STROKE_NAMES,
+  parseColor,
+  parseFill,
+  parseFontSize,
+  parseStroke
+} from './scratchpad-style'
 import { type OpenClawAgent, discoverOpenClawAgents } from './openclaw'
 import { liftToWorkspaceRoot, registerWorkspace } from './registry'
 import { serverCwd } from './server-cwd'
@@ -1109,87 +1117,8 @@ function parseEnd(s: string): ScratchArrowEnd {
   return { name: s }
 }
 
-// The Scratchpad palette (matches the UI toolbar's six swatches) and each color's
-// light-theme solid hex — used to snap an arbitrary `--color #rrggbb` to the nearest
-// palette entry (tldraw shapes can't hold free hex). Keep in sync with the swatches
-// in client/components/Scratchpad.tsx.
-const COLOR_HEX: Record<ScratchColor, string> = {
-  black: '#1d1d1d',
-  red: '#e03131',
-  yellow: '#f1ac4b',
-  green: '#099268',
-  blue: '#4465e9',
-  grey: '#9fa8b2'
-}
-const COLOR_NAMES = Object.keys(COLOR_HEX) as ScratchColor[]
-
-// Arrows expose tldraw's size as a line weight; the CLI mirrors the UI's two sizes.
-const STROKE_SIZES: Record<string, ScratchSize> = { small: 'm', large: 'xl' }
-const STROKE_NAMES = Object.keys(STROKE_SIZES)
-
-// Text & notes expose the same size style as a label font size, under friendlier names.
-const FONT_SIZES: Record<string, ScratchSize> = { regular: 'm', big: 'xl' }
-const FONT_SIZE_NAMES = Object.keys(FONT_SIZES)
-
-// Rectangle fills — the UI toolbar's four options. Each user-facing name maps onto a
-// tldraw DefaultFillStyle value (see ScratchFill for tldraw's semi/solid quirk). Keep
-// in sync with FILL_OPTIONS in client/components/Scratchpad.tsx.
-const FILL_STYLES: Record<string, ScratchFill> = {
-  none: 'none',
-  semi: 'solid',
-  pattern: 'pattern',
-  solid: 'fill'
-}
-const FILL_NAMES = Object.keys(FILL_STYLES)
-
-function hexToRgb(hex: string): [number, number, number] | null {
-  let h = hex.trim().replace(/^#/, '')
-  if (h.length === 3) h = h.replace(/(.)/g, '$1$1')
-  if (!/^[0-9a-fA-F]{6}$/.test(h)) return null
-  return [parseInt(h.slice(0, 2), 16), parseInt(h.slice(2, 4), 16), parseInt(h.slice(4, 6), 16)]
-}
-
-// Accept a palette name as-is, or snap any hex to the nearest palette color by
-// squared RGB distance. Throws on anything else.
-function parseColor(s: string): ScratchColor {
-  const lower = s.trim().toLowerCase()
-  if ((COLOR_NAMES as string[]).includes(lower)) return lower as ScratchColor
-  const rgb = hexToRgb(s)
-  if (!rgb) {
-    throw new Error(
-      `Unknown color "${s}". Use a hex like "#4465e9" or one of: ${COLOR_NAMES.join(', ')}.`
-    )
-  }
-  let best: ScratchColor = 'black'
-  let bestDist = Infinity
-  for (const name of COLOR_NAMES) {
-    const [r, g, b] = hexToRgb(COLOR_HEX[name])!
-    const d = (r - rgb[0]) ** 2 + (g - rgb[1]) ** 2 + (b - rgb[2]) ** 2
-    if (d < bestDist) {
-      bestDist = d
-      best = name
-    }
-  }
-  return best
-}
-
-function parseStroke(s: string): ScratchSize {
-  const size = STROKE_SIZES[s.trim().toLowerCase()]
-  if (!size) throw new Error(`Unknown stroke "${s}". Use one of: ${STROKE_NAMES.join(', ')}.`)
-  return size
-}
-
-function parseFontSize(s: string): ScratchSize {
-  const size = FONT_SIZES[s.trim().toLowerCase()]
-  if (!size) throw new Error(`Unknown font size "${s}". Use one of: ${FONT_SIZE_NAMES.join(', ')}.`)
-  return size
-}
-
-function parseFill(s: string): ScratchFill {
-  const fill = FILL_STYLES[s.trim().toLowerCase()]
-  if (!fill) throw new Error(`Unknown fill "${s}". Use one of: ${FILL_NAMES.join(', ')}.`)
-  return fill
-}
+// The palette / fill / size vocabulary is shared with the `diagram` spec
+// compiler, so it lives in scratchpad-style.ts (imported at the top of this file).
 
 // Resize preset for `add image` — defaults to 'lo' (keep the canvas light).
 function parseImageQuality(s: string | undefined): ScratchImageQuality {
@@ -1519,6 +1448,65 @@ const scratchAdd = defineCommand({
   }
 })
 
+// The declarative diagram compiler: a JSON spec in (file or stdin), measured +
+// auto-laid-out shapes on the canvas out. See server/scratchpad-diagram.ts.
+const scratchDiagram = defineCommand({
+  meta: {
+    name: 'diagram',
+    description:
+      'Compile a declarative diagram (JSON spec) onto the canvas with measured labels and ELK auto-layout — no coordinates needed. ' +
+      'Spec: {"title":"...", "direction":"right|down", ' +
+      '"nodes":[{"id":"a","label":"...","shape":"rect|note","color":"blue","fill":"semi","width":260}], ' +
+      '"groups":[{"id":"g","label":"...","color":"grey","children":["a"]}], ' +
+      '"edges":[{"from":"a","to":"b","label":"...","elbow":true}]}'
+  },
+  args: {
+    spec: { type: 'string', description: 'Path to the JSON spec file (default: read stdin)' },
+    at: {
+      type: 'string',
+      description: 'Top-left anchor "x,y" (default: auto-place below existing content)'
+    },
+    id: {
+      type: 'string',
+      description: 'Id prefix for created shapes (<prefix>-<nodeId>, <prefix>-title, ...)'
+    },
+    dir: dirArg
+  },
+  async run({ args }) {
+    const raw = args.spec ? await Bun.file(resolve(args.spec)).text() : await Bun.stdin.text()
+    let spec: unknown
+    try {
+      spec = JSON.parse(raw)
+    } catch (err) {
+      const source = args.spec ? args.spec : 'stdin'
+      console.error(
+        '\n' +
+          pc.red('✗') +
+          ` The spec (${source}) is not valid JSON: ${err instanceof Error ? err.message : String(err)}\n`
+      )
+      process.exit(1)
+    }
+    const at = args.at ? parseXY(args.at) : undefined
+    sendScratch(
+      resolve(args.dir),
+      {
+        kind: 'diagram',
+        name: args.id ?? '',
+        // Validated server-side (compileDiagramSpec) so errors surface in one
+        // place; the CLI only guarantees it's JSON.
+        spec: spec as ScratchDiagramSpec,
+        ...(at ? { x: at.x, y: at.y } : {})
+      },
+      res => {
+        const result = res.result as { name?: string; created?: string[] } | undefined
+        console.log('\n' + pc.green('✓') + ' diagram ' + pc.bold(result?.name ?? '(diagram)'))
+        for (const id of result?.created ?? []) console.log('  ' + id)
+        console.log('')
+      }
+    )
+  }
+})
+
 const scratchMove = defineCommand({
   meta: { name: 'move', description: 'Move a shape to a new position' },
   args: {
@@ -1581,6 +1569,7 @@ const scratch = defineCommand({
     'read-image': scratchReadImage,
     view: scratchView,
     add: scratchAdd,
+    diagram: scratchDiagram,
     move: scratchMove,
     set: scratchSet,
     delete: scratchDelete,