From 1b797793f7e516501ff2e17053892db11bfba432 Mon Sep 17 00:00:00 2001 From: Peter Werry Date: Fri, 26 Jun 2026 14:54:11 -0700 Subject: [PATCH] =?UTF-8?q?Step=203:=20synthesis=20=E2=80=94=20the=20engin?= =?UTF-8?q?e=20returns=20real=20results?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/engine/execute.ts | 9 ++++ src/engine/run.ts | 32 ++++++++----- src/synthesis/prompt.ts | 32 +++++++++++++ src/synthesis/synthesize.ts | 93 +++++++++++++++++++++++++++++++++++++ src/synthesis/tool.ts | 27 +++++++++++ 5 files changed, 182 insertions(+), 11 deletions(-) create mode 100644 src/engine/execute.ts create mode 100644 src/synthesis/prompt.ts create mode 100644 src/synthesis/synthesize.ts create mode 100644 src/synthesis/tool.ts diff --git a/src/engine/execute.ts b/src/engine/execute.ts new file mode 100644 index 0000000..a627b74 --- /dev/null +++ b/src/engine/execute.ts @@ -0,0 +1,9 @@ +import { getCollection } from "../db.js"; +import type { CollectionName } from "../models/index.js"; +import type { QueryPlan } from "./run.js"; + +/** Runs a query plan's aggregation pipeline and returns the matching documents. */ +export async function executePlan(plan: QueryPlan): Promise[]> { + const col = await getCollection(plan.collection as CollectionName); + return col.aggregate(plan.pipeline).toArray(); +} diff --git a/src/engine/run.ts b/src/engine/run.ts index 98d7167..ec89d81 100644 --- a/src/engine/run.ts +++ b/src/engine/run.ts @@ -1,6 +1,8 @@ // The query engine's public interface. The UX (server, chat agent, frontend) is // written against this and never changes — each workshop step fills in the // internals so runQuery() starts returning real results instead of NotImplemented. +import { synthesize } from "../synthesis/synthesize.js"; +import { executePlan } from "./execute.js"; /** A synthesized query plan: which collection to run against, and the aggregation pipeline. */ export interface QueryPlan { @@ -26,16 +28,24 @@ export interface RunOptions { } /** - * Turn a natural-language question into a MongoDB query, run it, and return the - * outcome. - * - * TODO (workshop): this is a stub. You'll build the real engine across the steps — - * schema description, identity resolution, synthesis, validation, retry, full-text — - * and wire it up here so it returns Success/NoResults instead of NotImplemented. + * Turn a natural-language question into a MongoDB query, run it, return the outcome. + * Step 3: synthesize -> execute -> format. (Validation and retry come in steps 4-5.) */ -export async function runQuery(_query: string, _opts: RunOptions = {}): Promise { - return { - type: "NotImplemented", - message: "TODO: Wire up the query engine", - }; +export async function runQuery(query: string, opts: RunOptions = {}): Promise { + let plan: QueryPlan; + try { + ({ plan } = await synthesize(query, { names: opts.names })); + } catch (err) { + return { type: "ResolutionFailed", reason: errorText(err), attempts: 1 }; + } + + opts.onPlan?.(plan, 1); + const rows = await executePlan(plan); + return rows.length + ? { type: "Success", rows, plan, attempts: 1 } + : { type: "NoResults", plan, attempts: 1 }; +} + +function errorText(err: unknown): string { + return err instanceof Error ? err.message : String(err); } diff --git a/src/synthesis/prompt.ts b/src/synthesis/prompt.ts new file mode 100644 index 0000000..a8cac9a --- /dev/null +++ b/src/synthesis/prompt.ts @@ -0,0 +1,32 @@ +/** Builds the system prompt for query synthesis from the schema context. */ +export function buildSystemPrompt(schemaContext: string): string { + return `You translate natural-language questions into MongoDB aggregation pipelines over GitHub data. + +${schemaContext} + +# Rules +- You MUST call the \`execute_mongo_query\` tool exactly once. Output no prose. +- Use ONLY fields that appear in the schema above. Never invent fields. +- Pick the single collection that best answers the question. +- Allowed pipeline stages: $match, $sort, $limit, $count, $group, $project, $unwind. +- Allowed filter operators: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin, $and, $or, $not, $exists, $size, $elemMatch. +- Do NOT use $regex on free-text fields (title, body, comments.body). Match exact field values instead. +- Timestamps are ISO-8601 strings; ISO strings sort lexicographically, so compare them directly with $gte/$lte. +- For relative dates ("today", "last week", "past 3 days"), compute the cutoff from the "Current time" given in the user message — never guess the current date. +- A PR is "merged" when merged is true (or merged_at is not null). "Open"/"closed" use the state field. +- After a $group stage, only _id and the accumulator fields you defined exist — the original document fields are gone. Any later $sort, $match, or $project must reference only those produced fields. +- Default to a $limit of 50 unless the question implies a count or aggregation. +- Prefer fields that appear in the index hints for $match and $sort. + +# Workflow +1. Identify the target collection and the filters, sorts, and limits the question implies. +2. If a "Resolved User" section is present, the person's name has already been resolved to GitHub logins — filter on those exact logins (e.g. user.login or assignees.login), never on the raw name. +3. Emit the pipeline via the tool call.`; +} + +/** Builds the user message: current time, the question, and any resolved-user context. */ +export function buildUserMessage(query: string, resolvedUsersSection: string, nowIso: string): string { + const parts = [`Current time (UTC): ${nowIso}`, query]; + if (resolvedUsersSection) parts.push(resolvedUsersSection); + return parts.join("\n\n"); +} diff --git a/src/synthesis/synthesize.ts b/src/synthesis/synthesize.ts new file mode 100644 index 0000000..7249163 --- /dev/null +++ b/src/synthesis/synthesize.ts @@ -0,0 +1,93 @@ +import { buildSchemaContext } from "../schema/describe.js"; +import { resolveUsers, formatResolvedUsers } from "../identity/resolver.js"; +import { buildSystemPrompt, buildUserMessage } from "./prompt.js"; +import { mongoQueryTool } from "./tool.js"; +import { forceTool, userMessage, MODELS, type LlmMessage, type ToolSpec } from "../llm/provider.js"; +import type { QueryPlan } from "../engine/run.js"; + +export interface SynthesizeOptions { + /** Named people to resolve to logins (beyond automatic "me" handling). */ + names?: string[]; + /** Prior turns to continue from — used by the retry loop (Step 5). */ + history?: LlmMessage[]; +} + +export interface SynthesisResult { + plan: QueryPlan; + toolUseId: string; + /** The full message history including this turn, for feeding a retry. */ + messages: LlmMessage[]; +} + +/** Forced tool the extractor must call — removes the model's option to refuse via prose. */ +const peopleTool: ToolSpec = { + name: "report_people", + description: "Report the specific people the question is about (PR/issue authors, reviewers, assignees).", + schema: { + type: "object", + properties: { + names: { + type: "array", + items: { type: "string" }, + description: + 'Each specific person named in the question, exactly as written. Bare first names, last ' + + 'names, and usernames all count (e.g. "peter", "alice", "octocat"). Empty array if no ' + + "person is named. Exclude generic self-references (me/my/I/mine).", + }, + }, + required: ["names"], + }, +}; + +const EXTRACT_SYSTEM = "Identify the specific people the question is about (PR/issue authors, reviewers, assignees)."; + +/** + * Pulls names of specific people out of a query via a forced tool call, so the + * query console (which, unlike the chat agent, has no tool layer to extract a + * user_name) can still resolve "peter's PRs" to a login. A forced tool call + + * temperature 0 makes this deterministic. Names that don't match the directory + * resolve to nothing downstream, so over-extraction is safe. + */ +async function extractNames(query: string): Promise { + const { input } = await forceTool({ + system: EXTRACT_SYSTEM, + messages: [userMessage(query)], + tool: peopleTool, + model: MODELS.synth, + temperature: 0, + maxTokens: 256, + }); + const names = (input as { names?: unknown }).names; + return Array.isArray(names) ? names.filter((x): x is string => typeof x === "string") : []; +} + +/** + * Turns a natural-language query into a MongoDB query plan via a single forced + * tool call. Does not validate or execute the plan — that's Steps 4 and 5. + */ +export async function synthesize(query: string, opts: SynthesizeOptions = {}): Promise { + const schemaContext = await buildSchemaContext(); + const system = buildSystemPrompt(schemaContext); + + let messages: LlmMessage[]; + if (opts.history) { + messages = [...opts.history]; + } else { + // Caller-provided names (the chat agent's extracted user_name) take priority; + // otherwise extract them from the query so the console resolves names too. + const names = opts.names?.length ? opts.names : await extractNames(query); + const resolved = await resolveUsers(query, names); + messages = [userMessage(buildUserMessage(query, formatResolvedUsers(resolved), new Date().toISOString()))]; + } + + const { input, toolCallId, assistant } = await forceTool({ + system, + messages, + tool: mongoQueryTool, + model: MODELS.synth, + temperature: 0, + maxTokens: 1024, + }); + + return { plan: input as QueryPlan, toolUseId: toolCallId, messages: [...messages, ...assistant] }; +} diff --git a/src/synthesis/tool.ts b/src/synthesis/tool.ts new file mode 100644 index 0000000..242a2b2 --- /dev/null +++ b/src/synthesis/tool.ts @@ -0,0 +1,27 @@ +import { COLLECTIONS } from "../models/index.js"; +import type { ToolSpec } from "../llm/provider.js"; + +export const TOOL_NAME = "execute_mongo_query"; + +/** The single tool the model must call. Its input IS the query plan. */ +export const mongoQueryTool: ToolSpec = { + name: TOOL_NAME, + description: + "Execute a MongoDB aggregation pipeline against one collection and return the matching documents.", + schema: { + type: "object", + properties: { + collection: { + type: "string", + enum: Object.keys(COLLECTIONS), + description: "The collection to query.", + }, + pipeline: { + type: "array", + items: { type: "object" }, + description: "A MongoDB aggregation pipeline (array of stage objects).", + }, + }, + required: ["collection", "pipeline"], + }, +};