Step 4: Validation

src/engine/run.ts

@@ -2,6 +2,7 @@
 // written against this and never changes — each workshop step fills in the
 // internals so runQuery() starts returning real results instead of NotImplemented.
 import { synthesize } from "../synthesis/synthesize.js";
+import { validatePlan } from "../validation/validate.js";
 import { executePlan } from "./execute.js";
 
 /** A synthesized query plan: which collection to run against, and the aggregation pipeline. */
@@ -10,11 +11,17 @@ export interface QueryPlan {
   pipeline: Record<string, unknown>[];
 }
 
+/** One synthesis attempt: the plan, and why it was rejected (empty errors = it passed validation). */
+export interface Attempt {
+  plan: QueryPlan;
+  errors: string[];
+}
+
 /** The result of running a natural-language query through the engine. */
 export type Outcome =
-  | { type: "Success"; rows: Record<string, unknown>[]; plan: QueryPlan; attempts: number }
-  | { type: "NoResults"; plan: QueryPlan; attempts: number }
-  | { type: "MaxAttemptsExceeded"; errors: string[]; plan: QueryPlan; attempts: number }
+  | { type: "Success"; rows: Record<string, unknown>[]; plan: QueryPlan; attempts: number; trace: Attempt[] }
+  | { type: "NoResults"; plan: QueryPlan; attempts: number; trace: Attempt[] }
+  | { type: "MaxAttemptsExceeded"; errors: string[]; plan: QueryPlan; attempts: number; trace: Attempt[] }
   | { type: "ResolutionFailed"; reason: string; attempts: number }
   | { type: "NotImplemented"; message: string };
 
@@ -29,7 +36,7 @@ export interface RunOptions {
 
 /**
  * Turn a natural-language question into a MongoDB query, run it, return the outcome.
- * Step 3: synthesize -> execute -> format. (Validation and retry come in steps 4-5.)
+ * Step 4: synthesize -> validate -> execute. (The retry loop comes in step 5.)
  */
 export async function runQuery(query: string, opts: RunOptions = {}): Promise<Outcome> {
   let plan: QueryPlan;
@@ -40,10 +47,19 @@ export async function runQuery(query: string, opts: RunOptions = {}): Promise<Ou
   }
 
   opts.onPlan?.(plan, 1);
+
+  // The trace records each attempt (plan + why it was rejected) so the UI can show
+  // the validation outcome. Step 4 has a single attempt; step 5 appends per retry.
+  const validation = await validatePlan(plan);
+  const trace: Attempt[] = [{ plan, errors: validation.ok ? [] : validation.errors }];
+  if (!validation.ok) {
+    return { type: "MaxAttemptsExceeded", errors: validation.errors, plan, attempts: 1, trace };
+  }
+
   const rows = await executePlan(plan);
   return rows.length
-    ? { type: "Success", rows, plan, attempts: 1 }
-    : { type: "NoResults", plan, attempts: 1 };
+    ? { type: "Success", rows, plan, attempts: 1, trace }
+    : { type: "NoResults", plan, attempts: 1, trace };
 }
 
 function errorText(err: unknown): string {

src/public/app.js

@@ -22,6 +22,36 @@ const queryForm = document.getElementById("query-form");
 const queryPlan = document.getElementById("query-plan");
 const queryRows = document.getElementById("query-rows");
 const queryMeta = document.getElementById("query-meta");
+const queryTrace = document.getElementById("query-trace");
+
+// Show the validate → reject → retry timeline. Skipped when a single attempt
+// passed straight away (nothing interesting to show).
+function renderTrace(trace) {
+  queryTrace.innerHTML = "";
+  if (!trace || trace.length === 0) return;
+  if (trace.length === 1 && trace[0].errors.length === 0) return;
+  const h = document.createElement("h3");
+  h.textContent = "Attempts";
+  queryTrace.appendChild(h);
+  trace.forEach((a, i) => {
+    const card = document.createElement("div");
+    card.className = `attempt ${a.errors.length ? "rejected" : "accepted"}`;
+    const head = document.createElement("div");
+    head.className = "attempt-head";
+    head.textContent = `Attempt ${i + 1} — ${a.errors.length ? "✗ rejected by validation" : "✓ valid"}`;
+    const pre = document.createElement("pre");
+    pre.className = "output code";
+    pre.textContent = JSON.stringify(a.plan.pipeline);
+    card.append(head, pre);
+    if (a.errors.length) {
+      const err = document.createElement("div");
+      err.className = "attempt-err";
+      err.textContent = a.errors.join("  ·  ");
+      card.appendChild(err);
+    }
+    queryTrace.appendChild(card);
+  });
+}
 
 queryForm.addEventListener("submit", async (e) => {
   e.preventDefault();
@@ -30,12 +60,14 @@ queryForm.addEventListener("submit", async (e) => {
   queryPlan.textContent = "…";
   queryRows.textContent = "";
   queryMeta.textContent = "";
+  queryTrace.innerHTML = "";
   const outcome = await (await fetch("/api/query", {
     method: "POST",
     headers: { "Content-Type": "application/json" },
     body: JSON.stringify({ question }),
   })).json();
 
+  renderTrace(outcome.trace);
   queryPlan.textContent = outcome.plan
     ? `${outcome.plan.collection}\n${JSON.stringify(outcome.plan.pipeline, null, 2)}`
     : "(no plan)";

src/public/index.html

@@ -27,6 +27,7 @@ <h1>Document Query Engine</h1>
           <summary>Discovered document schema — what the synthesizer is told</summary>
           <pre id="schema-body" class="output code">Loading…</pre>
         </details>
+        <div id="query-trace" class="trace"></div>
         <div class="query-result">
           <div>
             <h3>Generated pipeline</h3>

src/public/styles.css

@@ -98,6 +98,22 @@ label { color: var(--muted); font-size: 13px; display: flex; gap: 8px; align-ite
 .schema[open] summary { color: var(--accent); }
 .schema .output { margin-top: 8px; }
 
+.trace { margin-bottom: 16px; }
+.trace h3 { font-size: 13px; color: var(--muted); text-transform: uppercase; letter-spacing: .04em; margin: 0 0 8px; }
+.attempt {
+  border: 1px solid var(--border);
+  border-left: 3px solid var(--border);
+  border-radius: 8px;
+  padding: 10px 12px;
+  margin-bottom: 8px;
+  background: var(--panel);
+}
+.attempt.rejected { border-left-color: #f85149; }
+.attempt.accepted { border-left-color: var(--green); }
+.attempt-head { font-size: 13px; color: var(--fg); margin-bottom: 6px; }
+.attempt .output { margin: 0; max-height: 120px; padding: 8px 10px; }
+.attempt-err { color: #f85149; font-size: 12.5px; font-family: "SF Mono", ui-monospace, monospace; margin-top: 6px; }
+
 .query-result { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
 .query-result h3 { font-size: 13px; color: var(--muted); text-transform: uppercase; letter-spacing: .04em; }
 #query-meta { color: var(--green); text-transform: none; letter-spacing: 0; }

src/validation/validate.ts

@@ -0,0 +1,138 @@
+import { COLLECTIONS, type CollectionName } from "../models/index.js";
+import { collectFieldPaths } from "../schema/describe.js";
+import type { QueryPlan } from "../engine/run.js";
+
+const ALLOWED_STAGES = new Set([
+  "$match",
+  "$sort",
+  "$limit",
+  "$count",
+  "$group",
+  "$project",
+  "$unwind",
+]);
+
+const ALLOWED_OPERATORS = new Set([
+  // query operators
+  "$eq", "$ne", "$gt", "$gte", "$lt", "$lte", "$in", "$nin",
+  "$and", "$or", "$not", "$nor", "$exists", "$size", "$elemMatch",
+  // accumulators / group expressions
+  "$sum", "$avg", "$min", "$max", "$first", "$last", "$push", "$addToSet",
+]);
+
+/** Field paths a query may never reference. */
+function isBlocked(path: string): boolean {
+  return path === "_id" || path === "metadata" || path.startsWith("metadata.");
+}
+
+export type ValidationResult = { ok: true } | { ok: false; errors: string[] };
+
+/**
+ * Statically validates a query plan before execution: known collection, only
+ * allow-listed stages and operators, and every referenced field exists in the
+ * schema and isn't blocked. Returns structured errors so the retry loop (Step 5)
+ * can feed them back to the model.
+ */
+export async function validatePlan(plan: QueryPlan): Promise<ValidationResult> {
+  const errors: string[] = [];
+
+  if (!(plan.collection in COLLECTIONS)) {
+    return { ok: false, errors: [`Unknown collection "${plan.collection}".`] };
+  }
+  if (!Array.isArray(plan.pipeline) || plan.pipeline.length === 0) {
+    return { ok: false, errors: ["Pipeline must be a non-empty array of stages."] };
+  }
+
+  const fields = await collectFieldPaths(plan.collection as CollectionName);
+
+  // $group/$project rebuild the document shape, so downstream stages reference
+  // computed fields (e.g. a $sum alias) that aren't in the source schema. We only
+  // validate field existence against the schema up to the first reshaping stage.
+  let reshaped = false;
+
+  for (const stage of plan.pipeline) {
+    const keys = Object.keys(stage);
+    if (keys.length !== 1) {
+      errors.push(`Each stage must have exactly one operator; got [${keys.join(", ")}].`);
+      continue;
+    }
+    const stageName = keys[0]!;
+    if (!ALLOWED_STAGES.has(stageName)) {
+      errors.push(`Stage "${stageName}" is not allowed.`);
+      continue;
+    }
+    checkOperators(stage[stageName], errors);
+    if (!reshaped) checkFields(stageName, stage[stageName], fields, errors);
+    if (stageName === "$group" || stageName === "$project") reshaped = true;
+  }
+
+  return errors.length ? { ok: false, errors } : { ok: true };
+}
+
+/** Recursively rejects any $-prefixed key that isn't an allow-listed operator/accumulator. */
+function checkOperators(value: unknown, errors: string[]): void {
+  if (Array.isArray(value)) {
+    value.forEach((v) => checkOperators(v, errors));
+    return;
+  }
+  if (value && typeof value === "object") {
+    for (const [key, v] of Object.entries(value)) {
+      if (key.startsWith("$") && !ALLOWED_OPERATORS.has(key)) {
+        errors.push(`Operator "${key}" is not allowed.`);
+      }
+      checkOperators(v, errors);
+    }
+  }
+}
+
+/** Validates field references for the field-bearing stages. */
+function checkFields(stage: string, value: unknown, fields: Set<string>, errors: string[]): void {
+  const check = (path: string) => {
+    if (isBlocked(path)) errors.push(`Field "${path}" is not accessible.`);
+    else if (!fields.has(path)) errors.push(`Unknown field "${path}" in ${stage}.`);
+  };
+
+  switch (stage) {
+    case "$match":
+      collectMatchFields(value).forEach(check);
+      break;
+    case "$sort":
+      // _id is always a valid sort key (document order). Other keys must be real fields.
+      if (value && typeof value === "object") Object.keys(value).filter((k) => k !== "_id").forEach(check);
+      break;
+    // $project is intentionally not field-checked: it renames/computes output fields
+    // (e.g. {login: "$user.login"}), so its keys are new names, not source paths.
+    case "$unwind":
+      if (typeof value === "string") check(value.replace(/^\$/, ""));
+      else if (value && typeof value === "object" && "path" in value)
+        check(String((value as { path: unknown }).path).replace(/^\$/, ""));
+      break;
+    default:
+      break; // $limit, $count, $group keys are not document field paths
+  }
+}
+
+/** Pulls dotted document field paths out of a $match expression, descending through $and/$or/$not and nested docs. */
+function collectMatchFields(value: unknown): string[] {
+  const out: string[] = [];
+  const isPlainDoc = (v: unknown): v is Record<string, unknown> =>
+    !!v && typeof v === "object" && !Array.isArray(v) && Object.keys(v).every((k) => !k.startsWith("$"));
+
+  const visit = (v: unknown, prefix: string) => {
+    if (Array.isArray(v)) return v.forEach((x) => visit(x, prefix));
+    if (v && typeof v === "object") {
+      for (const [key, child] of Object.entries(v)) {
+        if (key.startsWith("$")) {
+          visit(child, prefix); // logical operator — recurse, keep prefix
+        } else {
+          const path = prefix ? `${prefix}.${key}` : key;
+          // A nested doc value (e.g. {user: {login: "x"}}) extends the dotted path.
+          if (isPlainDoc(child)) visit(child, path);
+          else out.push(path);
+        }
+      }
+    }
+  };
+  visit(value, "");
+  return out;
+}