From c4cc764f8e693f8b6dbc29e2833b2cff68e79554 Mon Sep 17 00:00:00 2001 From: zahlekhan Date: Thu, 30 Jul 2026 18:51:42 +0530 Subject: [PATCH 1/5] feat(docs): track demo agent interactions --- docs/README.md | 29 +++ docs/app/api/chat/route.ts | 3 +- .../agent-surfaces/cloud-agent-surface.tsx | 10 + .../agent-surfaces/oss-agent-surface.tsx | 11 + .../agent-surfaces/cloud-agent-surface.tsx | 5 + .../agent-surfaces/oss-agent-surface.tsx | 5 + .../_components/compare-page-client.tsx | 10 +- .../_components/comparison-analytics.test.ts | 63 +++++ .../_components/comparison-analytics.ts | 31 +++ .../_components/comparison-controls.tsx | 14 +- docs/app/demo/github/constants.ts | 2 +- docs/app/demo/github/page.tsx | 30 ++- docs/app/demos/constants.ts | 8 +- docs/app/demos/page.tsx | 7 + docs/lib/demo-analytics.test.ts | 216 ++++++++++++++++++ docs/lib/demo-analytics.ts | 87 +++++++ docs/lib/demo-models.ts | 16 ++ docs/lib/thesys-link-attribution.test.ts | 53 +++++ docs/package.json | 4 +- docs/plans/frontend-posthog-demo-analytics.md | 185 +++++++++++++++ packages/react-headless/src/index.ts | 3 + .../react-headless/src/store/ChatProvider.tsx | 4 +- .../store/__tests__/__helpers/makeStore.ts | 7 +- .../store/__tests__/createChatStore.test.ts | 43 ++++ .../src/store/createChatStore.ts | 13 +- packages/react-headless/src/store/types.ts | 20 +- .../AgentInterface/AgentInterface.tsx | 2 + .../AgentInterface/ConversationStarter.tsx | 11 +- .../src/components/AgentInterface/README.md | 17 +- .../AgentInterface/components/Composer.tsx | 11 +- .../components/DesktopWelcomeComposer.tsx | 11 +- .../OpenUIChat/CustomComposerAdapter.tsx | 2 +- .../OpenUIChat/GenUIAssistantMessage.tsx | 11 +- .../OpenUIChat/withChatProvider.tsx | 2 + pnpm-lock.yaml | 3 + 35 files changed, 892 insertions(+), 57 deletions(-) create mode 100644 docs/app/compare/_components/comparison-analytics.test.ts create mode 100644 docs/app/compare/_components/comparison-analytics.ts create mode 100644 docs/lib/demo-analytics.test.ts create mode 100644 docs/lib/demo-analytics.ts create mode 100644 docs/lib/demo-models.ts create mode 100644 docs/lib/thesys-link-attribution.test.ts create mode 100644 docs/plans/frontend-posthog-demo-analytics.md diff --git a/docs/README.md b/docs/README.md index 332d32d37..470bb0057 100644 --- a/docs/README.md +++ b/docs/README.md @@ -45,6 +45,35 @@ a shared, cross-instance session-and-IP rate limiter, Cloud organization budgets an approved conversation retention/deletion process are in place. Same-origin validation is an additional browser safeguard, not a substitute for those cost controls. +### Demo interaction analytics + +The four hosted demos (`/compare`, `/demo/github`, `/chat`, and `/demos`) emit +`demo_agent_interaction` PostHog events for accepted user interactions. Most turns emit one event; +`/compare` emits one per visible panel. The instrumentation supplies only these custom dimensions +(PostHog may also attach its standard page context): + +- `demo`: the stable demo identifier +- `variant`: the visible comparison mode or chat mode when applicable +- `model`: the canonical provider/model ID requested for the interaction +- `interaction_source`: `composer`, `starter`, or `rendered_action` + +`/compare` emits two events for a shared prompt, one per available mode in the selected visible +pair. Its hidden third controller still receives the prompt in the background, but that work is +not counted as a visible user interaction. A generated `continue_conversation` action emits one +event for the panel it targets. Prefill-only chips, blocked submissions, restored messages, and +local generated-UI state changes do not emit events. + +Demo analytics never add prompt or message content, generated output, action payloads, form state, +GitHub usernames, action or tool URLs, thread or Cloud user IDs, tool arguments, or raw errors. The +docs site uses the shared, fail-open PostHog client in `lib/analytics.ts`; published OpenUI packages +do not initialize or send analytics. + +OpenUI visitors remain anonymous. Outbound `thesys.dev` links carry PostHog's current anonymous +distinct ID and session ID as handoff query parameters, not event properties. The receiving console +validates and removes those parameters before analytics initialization, bootstraps the distinct ID +as anonymous, and later merges the demo history through its canonical `posthog.identify(user.id)` +call. Both applications must use the same PostHog project, and the console resets PostHog on logout. + ## Project structure ``` diff --git a/docs/app/api/chat/route.ts b/docs/app/api/chat/route.ts index e49212f6e..2528e9ef7 100644 --- a/docs/app/api/chat/route.ts +++ b/docs/app/api/chat/route.ts @@ -1,3 +1,4 @@ +import { DEFAULT_MODEL } from "@/lib/openui-cloud/models"; import { readFileSync } from "fs"; import { NextRequest } from "next/server"; import OpenAI from "openai"; @@ -293,7 +294,7 @@ export async function POST(req: NextRequest) { apiKey, baseURL: "https://openrouter.ai/api/v1", }); - const MODEL = "openai/gpt-5.4"; + const MODEL = DEFAULT_MODEL; const cleanMessages = (messages as any[]) .filter( diff --git a/docs/app/chat/_components/agent-surfaces/cloud-agent-surface.tsx b/docs/app/chat/_components/agent-surfaces/cloud-agent-surface.tsx index 16125deb7..9574c4e70 100644 --- a/docs/app/chat/_components/agent-surfaces/cloud-agent-surface.tsx +++ b/docs/app/chat/_components/agent-surfaces/cloud-agent-surface.tsx @@ -1,5 +1,6 @@ "use client"; +import { captureDemoAgentInteraction } from "@/lib/demo-analytics"; import { createCloudChatLLM } from "@/lib/openui-cloud/chat-llm"; import { DEFAULT_MODEL } from "@/lib/openui-cloud/models"; import { CLOUD_USER_ID_HEADER, getOrCreateCloudUserId } from "@/lib/openui-cloud/user-id"; @@ -82,6 +83,15 @@ export function CloudAgentSurface() { scrollOnLoad={false} starterVariant="short" starters={CLOUD_STARTERS} + onUserMessageAccepted={({ source }) => { + if (source === "programmatic") return; + captureDemoAgentInteraction({ + demo: "openui_chat", + variant: "cloud", + model: selectedModel, + interaction_source: source, + }); + }} > { + if (source === "programmatic") return; + captureDemoAgentInteraction({ + demo: "openui_chat", + variant: "oss", + model: DEFAULT_MODEL, + interaction_source: source, + }); + }} > { + if (source !== "rendered_action") return; + captureComparisonRenderedAction("cloud"); + }} > diff --git a/docs/app/compare/_components/agent-surfaces/oss-agent-surface.tsx b/docs/app/compare/_components/agent-surfaces/oss-agent-surface.tsx index 20ef99a7b..7498ab67d 100644 --- a/docs/app/compare/_components/agent-surfaces/oss-agent-surface.tsx +++ b/docs/app/compare/_components/agent-surfaces/oss-agent-surface.tsx @@ -2,6 +2,7 @@ import { AgentInterface } from "@openuidev/react-ui"; import { openuiChatLibrary } from "@openuidev/react-ui/genui-lib"; +import { captureComparisonRenderedAction } from "../comparison-analytics"; import type { ComparisonControllerRegistry } from "../comparison-mode-controller"; import { ComparisonModeControllerBridge } from "../comparison-mode-controller"; import { ComparisonSurfaceWelcome } from "./comparison-surface-welcome"; @@ -22,6 +23,10 @@ export function OssAgentSurface({ onCreditsExhausted, registry }: OssAgentSurfac componentLibrary={openuiChatLibrary} agentName="OpenUI OSS" scrollVariant="always" + onUserMessageAccepted={({ source }) => { + if (source !== "rendered_action") return; + captureComparisonRenderedAction("oss"); + }} > diff --git a/docs/app/compare/_components/compare-page-client.tsx b/docs/app/compare/_components/compare-page-client.tsx index 5d6059246..225f549f6 100644 --- a/docs/app/compare/_components/compare-page-client.tsx +++ b/docs/app/compare/_components/compare-page-client.tsx @@ -23,7 +23,8 @@ import { getComparisonPair, type ComparisonPair, } from "./chat-types"; -import { ComparisonControls } from "./comparison-controls"; +import { captureComparisonPromptInteractions } from "./comparison-analytics"; +import { ComparisonControls, type ComparisonSubmissionSource } from "./comparison-controls"; import type { ComparisonMode, ComparisonModeController, @@ -148,15 +149,18 @@ export function ComparePageClient({ }, [registry, unavailableModes]); const submitToAll = useCallback( - (content: string) => { + (content: string, source: ComparisonSubmissionSource) => { const controllers = getReadyControllers(); if (!controllers) return; const states = controllers.map((controller) => controller.getSnapshot()); if (states.some((state) => !state.isReady || state.isRunning)) return; + const availableModes = new Set(controllers.map(({ mode }) => mode)); + const visibleModes = getComparisonPair(pair).modes.filter((mode) => availableModes.has(mode)); + captureComparisonPromptInteractions(visibleModes, source); void Promise.allSettled(controllers.map((controller) => controller.send(content))); }, - [getReadyControllers], + [getReadyControllers, pair], ); const stopAll = useCallback(() => { diff --git a/docs/app/compare/_components/comparison-analytics.test.ts b/docs/app/compare/_components/comparison-analytics.test.ts new file mode 100644 index 000000000..5a1f688a3 --- /dev/null +++ b/docs/app/compare/_components/comparison-analytics.test.ts @@ -0,0 +1,63 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_MODEL } from "../../../lib/openui-cloud/models"; + +const { captureDemoAgentInteraction } = vi.hoisted(() => ({ + captureDemoAgentInteraction: vi.fn(), +})); + +vi.mock("../../../lib/demo-analytics", () => ({ + captureDemoAgentInteraction, +})); + +import { + captureComparisonPromptInteractions, + captureComparisonRenderedAction, +} from "./comparison-analytics"; + +beforeEach(() => { + captureDemoAgentInteraction.mockReset(); +}); + +describe("comparison analytics", () => { + it("captures one model-attributed event for each visible comparison mode", () => { + captureComparisonPromptInteractions(["markdown", "cloud"], "composer"); + + expect(captureDemoAgentInteraction).toHaveBeenCalledTimes(2); + expect(captureDemoAgentInteraction).toHaveBeenNthCalledWith(1, { + demo: "compare", + variant: "markdown", + model: DEFAULT_MODEL, + interaction_source: "composer", + }); + expect(captureDemoAgentInteraction).toHaveBeenNthCalledWith(2, { + demo: "compare", + variant: "cloud", + model: DEFAULT_MODEL, + interaction_source: "composer", + }); + }); + + it("captures only modes that actually remain available", () => { + captureComparisonPromptInteractions(["oss"], "starter"); + + expect(captureDemoAgentInteraction).toHaveBeenCalledOnce(); + expect(captureDemoAgentInteraction).toHaveBeenCalledWith({ + demo: "compare", + variant: "oss", + model: DEFAULT_MODEL, + interaction_source: "starter", + }); + }); + + it("captures a rendered action only for its target panel", () => { + captureComparisonRenderedAction("cloud"); + + expect(captureDemoAgentInteraction).toHaveBeenCalledOnce(); + expect(captureDemoAgentInteraction).toHaveBeenCalledWith({ + demo: "compare", + variant: "cloud", + model: DEFAULT_MODEL, + interaction_source: "rendered_action", + }); + }); +}); diff --git a/docs/app/compare/_components/comparison-analytics.ts b/docs/app/compare/_components/comparison-analytics.ts new file mode 100644 index 000000000..9357a5aa6 --- /dev/null +++ b/docs/app/compare/_components/comparison-analytics.ts @@ -0,0 +1,31 @@ +import { + captureDemoAgentInteraction, + type DemoInteractionSource, +} from "../../../lib/demo-analytics"; +import { DEFAULT_MODEL } from "../../../lib/openui-cloud/models"; +import type { ComparisonMode } from "./comparison-mode-controller"; + +export type ComparisonPromptSource = Extract; + +export function captureComparisonPromptInteractions( + modes: readonly ComparisonMode[], + interactionSource: ComparisonPromptSource, +): void { + modes.forEach((mode) => { + captureDemoAgentInteraction({ + demo: "compare", + variant: mode, + model: DEFAULT_MODEL, + interaction_source: interactionSource, + }); + }); +} + +export function captureComparisonRenderedAction(mode: ComparisonMode): void { + captureDemoAgentInteraction({ + demo: "compare", + variant: mode, + model: DEFAULT_MODEL, + interaction_source: "rendered_action", + }); +} diff --git a/docs/app/compare/_components/comparison-controls.tsx b/docs/app/compare/_components/comparison-controls.tsx index 6edd68063..6bd9b00cf 100644 --- a/docs/app/compare/_components/comparison-controls.tsx +++ b/docs/app/compare/_components/comparison-controls.tsx @@ -65,12 +65,14 @@ const COMPARISON_SUGGESTIONS = [ }, ] as const; +export type ComparisonSubmissionSource = "composer" | "starter"; + interface ComparisonControlsProps { comparisonPair: ComparisonPair; isReady: boolean; isRunning: boolean; hasStarted: boolean; - onSubmit: (content: string) => void; + onSubmit: (content: string, source: ComparisonSubmissionSource) => void; onStop: () => void; onReset: () => void; isDegraded: boolean; @@ -121,11 +123,11 @@ export function ComparisonControls({ return () => observer.disconnect(); }, [text]); - const submit = (content: string) => { + const submit = (content: string, source: ComparisonSubmissionSource) => { const next = content.trim(); if (!next || !isReady || isRunning) return; - onSubmit(next); + onSubmit(next, source); setText(""); }; @@ -152,7 +154,7 @@ export function ComparisonControls({ key={suggestion.label} type="button" className={styles.suggestionButton} - onClick={() => submit(suggestion.prompt)} + onClick={() => submit(suggestion.prompt, "starter")} disabled={!isReady || isRunning || (needsCloud && !cloudEnabled)} title={ needsCloud && !cloudEnabled @@ -217,14 +219,14 @@ export function ComparisonControls({ onKeyDown={(event) => { if (event.key === "Enter" && !event.shiftKey) { event.preventDefault(); - submit(text); + submit(text, "composer"); } }} />