diff --git a/extensions/cli/src/commands/serve.ts b/extensions/cli/src/commands/serve.ts index f77e86f41bb..f0a41469b3f 100644 --- a/extensions/cli/src/commands/serve.ts +++ b/extensions/cli/src/commands/serve.ts @@ -387,10 +387,10 @@ export async function serve(prompt?: string, options: ServeOptions = {}) { server.close(async () => { telemetryService.stopActiveTime(); - // Update metadata one final time before exiting + // Update metadata one final time before exiting (with completion flag) try { const history = services.chatHistory?.getHistory(); - await updateAgentMetadata(history); + await updateAgentMetadata({ history, isComplete: true }); } catch (err) { logger.debug("Failed to update metadata (non-critical)", err as any); } @@ -596,10 +596,10 @@ export async function serve(prompt?: string, options: ServeOptions = {}) { server.close(async () => { telemetryService.stopActiveTime(); - // Update metadata one final time before exiting + // Update metadata one final time before exiting (with completion flag) try { const history = services.chatHistory?.getHistory(); - await updateAgentMetadata(history); + await updateAgentMetadata({ history, isComplete: true }); } catch (err) { logger.debug("Failed to update metadata (non-critical)", err as any); } @@ -628,10 +628,10 @@ export async function serve(prompt?: string, options: ServeOptions = {}) { server.close(async () => { telemetryService.stopActiveTime(); - // Update metadata one final time before exiting + // Update metadata one final time before exiting (with completion flag) try { const history = services.chatHistory?.getHistory(); - await updateAgentMetadata(history); + await updateAgentMetadata({ history, isComplete: true }); } catch (err) { logger.debug("Failed to update metadata (non-critical)", err as any); } diff --git a/extensions/cli/src/util/exit.test.ts b/extensions/cli/src/util/exit.test.ts new file mode 100644 index 00000000000..a45e9b4fc30 --- /dev/null +++ b/extensions/cli/src/util/exit.test.ts @@ -0,0 +1,627 @@ +import type { ChatHistoryItem } from "core/index.js"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import * as git from "./git.js"; +import * as metadata from "./metadata.js"; +import { updateAgentMetadata, UpdateAgentMetadataOptions } from "./exit.js"; + +// Mock dependencies +vi.mock("./git.js"); +vi.mock("./metadata.js"); +vi.mock("./logger.js", () => ({ + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})); +vi.mock("../session.js", () => ({ + getSessionUsage: vi.fn(() => ({ + totalCost: 0, + promptTokens: 0, + completionTokens: 0, + promptTokensDetails: {}, + })), +})); + +// Helper to create a mock chat history item +function createMockChatHistoryItem( + content: string, + role: "user" | "assistant" | "system" = "assistant", +): ChatHistoryItem { + return { + message: { + role, + content, + }, + } as ChatHistoryItem; +} + +describe("updateAgentMetadata", () => { + const mockAgentId = "test-agent-123"; + + beforeEach(() => { + vi.clearAllMocks(); + + // Default mocks + vi.spyOn(metadata, "getAgentIdFromArgs").mockReturnValue(mockAgentId); + vi.spyOn(metadata, "postAgentMetadata").mockResolvedValue(); + vi.spyOn(metadata, "calculateDiffStats").mockReturnValue({ + additions: 0, + deletions: 0, + }); + vi.spyOn(metadata, "extractSummary").mockReturnValue(undefined); + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "", + repoFound: false, + }); + }); + + describe("backward compatibility with old signature", () => { + it("should accept history array directly (old signature)", async () => { + const history = [ + createMockChatHistoryItem("Hello", "user"), + createMockChatHistoryItem("Hi there!", "assistant"), + ]; + + await updateAgentMetadata(history); + + expect(metadata.extractSummary).toHaveBeenCalledWith(history); + }); + + it("should treat history array as isComplete=false", async () => { + const history = [createMockChatHistoryItem("Test message", "assistant")]; + + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "diff content", + repoFound: true, + }); + vi.spyOn(metadata, "calculateDiffStats").mockReturnValue({ + additions: 5, + deletions: 2, + }); + + await updateAgentMetadata(history); + + // Should not include isComplete or hasChanges + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.objectContaining({ + additions: 5, + deletions: 2, + }), + ); + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.not.objectContaining({ + isComplete: expect.anything(), + hasChanges: expect.anything(), + }), + ); + }); + }); + + describe("new signature with UpdateAgentMetadataOptions", () => { + it("should accept options object with history", async () => { + const history = [createMockChatHistoryItem("Test", "assistant")]; + const options: UpdateAgentMetadataOptions = { + history, + isComplete: false, + }; + + await updateAgentMetadata(options); + + expect(metadata.extractSummary).toHaveBeenCalledWith(history); + }); + + it("should accept options object with isComplete=true", async () => { + const options: UpdateAgentMetadataOptions = { + isComplete: true, + }; + + await updateAgentMetadata(options); + + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.objectContaining({ + isComplete: true, + hasChanges: false, + }), + ); + }); + + it("should accept options object with both history and isComplete", async () => { + const history = [createMockChatHistoryItem("Done!", "assistant")]; + const options: UpdateAgentMetadataOptions = { + history, + isComplete: true, + }; + + vi.spyOn(metadata, "extractSummary").mockReturnValue("Done!"); + + await updateAgentMetadata(options); + + expect(metadata.extractSummary).toHaveBeenCalledWith(history); + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.objectContaining({ + isComplete: true, + summary: "Done!", + }), + ); + }); + }); + + describe("isComplete flag behavior", () => { + it("should set isComplete=true and hasChanges=false when no changes", async () => { + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "", + repoFound: true, + }); + + await updateAgentMetadata({ isComplete: true }); + + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.objectContaining({ + isComplete: true, + hasChanges: false, + }), + ); + }); + + it("should set isComplete=true and hasChanges=true when changes exist", async () => { + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "some diff content", + repoFound: true, + }); + vi.spyOn(metadata, "calculateDiffStats").mockReturnValue({ + additions: 10, + deletions: 5, + }); + + await updateAgentMetadata({ isComplete: true }); + + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.objectContaining({ + isComplete: true, + hasChanges: true, + additions: 10, + deletions: 5, + }), + ); + }); + + it("should set hasChanges=true even with only additions", async () => { + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "diff content", + repoFound: true, + }); + vi.spyOn(metadata, "calculateDiffStats").mockReturnValue({ + additions: 5, + deletions: 0, + }); + + await updateAgentMetadata({ isComplete: true }); + + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.objectContaining({ + hasChanges: true, + additions: 5, + }), + ); + }); + + it("should set hasChanges=true even with only deletions", async () => { + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "diff content", + repoFound: true, + }); + vi.spyOn(metadata, "calculateDiffStats").mockReturnValue({ + additions: 0, + deletions: 3, + }); + + await updateAgentMetadata({ isComplete: true }); + + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.objectContaining({ + hasChanges: true, + deletions: 3, + }), + ); + }); + + it("should not include isComplete/hasChanges when isComplete=false", async () => { + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "some diff", + repoFound: true, + }); + vi.spyOn(metadata, "calculateDiffStats").mockReturnValue({ + additions: 5, + deletions: 2, + }); + + await updateAgentMetadata({ isComplete: false }); + + const callArgs = (metadata.postAgentMetadata as any).mock.calls[0][1]; + expect(callArgs).not.toHaveProperty("isComplete"); + expect(callArgs).not.toHaveProperty("hasChanges"); + }); + + it("should default to isComplete=false when not specified in options", async () => { + await updateAgentMetadata({}); + + const callArgs = (metadata.postAgentMetadata as any).mock.calls[0][1]; + expect(callArgs).not.toHaveProperty("isComplete"); + expect(callArgs).not.toHaveProperty("hasChanges"); + }); + }); + + describe("diff stats collection", () => { + it("should collect diff stats when repo found with changes", async () => { + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "diff content", + repoFound: true, + }); + vi.spyOn(metadata, "calculateDiffStats").mockReturnValue({ + additions: 15, + deletions: 8, + }); + + await updateAgentMetadata({}); + + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.objectContaining({ + additions: 15, + deletions: 8, + }), + ); + }); + + it("should not include diff stats when no changes", async () => { + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "", + repoFound: true, + }); + vi.spyOn(metadata, "calculateDiffStats").mockReturnValue({ + additions: 0, + deletions: 0, + }); + + await updateAgentMetadata({}); + + const callArgs = (metadata.postAgentMetadata as any).mock.calls[0][1]; + expect(callArgs).not.toHaveProperty("additions"); + expect(callArgs).not.toHaveProperty("deletions"); + }); + + it("should not include diff stats when repo not found", async () => { + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "", + repoFound: false, + }); + + await updateAgentMetadata({}); + + expect(metadata.calculateDiffStats).not.toHaveBeenCalled(); + }); + + it("should handle git diff errors gracefully", async () => { + vi.spyOn(git, "getGitDiffSnapshot").mockRejectedValue( + new Error("Git error"), + ); + + await expect(updateAgentMetadata({})).resolves.not.toThrow(); + + // Should still call postAgentMetadata but without diff stats + expect(metadata.postAgentMetadata).toHaveBeenCalled(); + }); + }); + + describe("summary extraction", () => { + it("should extract summary from history", async () => { + const history = [createMockChatHistoryItem("Summary text", "assistant")]; + vi.spyOn(metadata, "extractSummary").mockReturnValue("Summary text"); + + await updateAgentMetadata({ history }); + + expect(metadata.extractSummary).toHaveBeenCalledWith(history); + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.objectContaining({ + summary: "Summary text", + }), + ); + }); + + it("should not include summary when history is empty", async () => { + await updateAgentMetadata({ history: [] }); + + expect(metadata.extractSummary).toHaveBeenCalledWith([]); + const callArgs = (metadata.postAgentMetadata as any).mock.calls[0][1]; + expect(callArgs).not.toHaveProperty("summary"); + }); + + it("should not include summary when history is undefined", async () => { + await updateAgentMetadata({}); + + expect(metadata.extractSummary).toHaveBeenCalledWith(undefined); + const callArgs = (metadata.postAgentMetadata as any).mock.calls[0][1]; + expect(callArgs).not.toHaveProperty("summary"); + }); + + it("should handle extractSummary errors gracefully", async () => { + const history = [createMockChatHistoryItem("Test", "assistant")]; + vi.spyOn(metadata, "extractSummary").mockImplementation(() => { + throw new Error("Extract error"); + }); + + await expect(updateAgentMetadata({ history })).resolves.not.toThrow(); + }); + }); + + describe("session usage collection", () => { + it("should include usage when cost > 0", async () => { + const { getSessionUsage } = await import("../session.js"); + vi.mocked(getSessionUsage).mockReturnValue({ + totalCost: 0.05, + promptTokens: 1000, + completionTokens: 500, + promptTokensDetails: { + cachedTokens: 100, + cacheWriteTokens: 50, + }, + }); + + await updateAgentMetadata({}); + + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.objectContaining({ + usage: { + totalCost: 0.05, + promptTokens: 1000, + completionTokens: 500, + cachedTokens: 100, + cacheWriteTokens: 50, + }, + }), + ); + }); + + it("should not include usage when cost is 0", async () => { + const { getSessionUsage } = await import("../session.js"); + vi.mocked(getSessionUsage).mockReturnValue({ + totalCost: 0, + promptTokens: 0, + completionTokens: 0, + promptTokensDetails: {}, + }); + + await updateAgentMetadata({}); + + const callArgs = (metadata.postAgentMetadata as any).mock.calls[0][1]; + expect(callArgs).not.toHaveProperty("usage"); + }); + + it("should round totalCost to 6 decimal places", async () => { + const { getSessionUsage } = await import("../session.js"); + vi.mocked(getSessionUsage).mockReturnValue({ + totalCost: 0.123456789, + promptTokens: 1000, + completionTokens: 500, + promptTokensDetails: {}, + }); + + await updateAgentMetadata({}); + + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.objectContaining({ + usage: expect.objectContaining({ + totalCost: 0.123457, // Rounded to 6 decimals + }), + }), + ); + }); + + it("should omit cachedTokens when not present", async () => { + const { getSessionUsage } = await import("../session.js"); + vi.mocked(getSessionUsage).mockReturnValue({ + totalCost: 0.01, + promptTokens: 100, + completionTokens: 50, + promptTokensDetails: {}, + }); + + await updateAgentMetadata({}); + + const callArgs = (metadata.postAgentMetadata as any).mock.calls[0][1]; + expect(callArgs.usage).not.toHaveProperty("cachedTokens"); + expect(callArgs.usage).not.toHaveProperty("cacheWriteTokens"); + }); + + it("should handle getSessionUsage errors gracefully", async () => { + const { getSessionUsage } = await import("../session.js"); + vi.mocked(getSessionUsage).mockImplementation(() => { + throw new Error("Session error"); + }); + + await expect(updateAgentMetadata({})).resolves.not.toThrow(); + }); + }); + + describe("agent ID handling", () => { + it("should skip metadata update when no agent ID", async () => { + vi.spyOn(metadata, "getAgentIdFromArgs").mockReturnValue(undefined); + + await updateAgentMetadata({}); + + expect(metadata.postAgentMetadata).not.toHaveBeenCalled(); + }); + + it("should use agent ID from process args", async () => { + const customAgentId = "custom-agent-456"; + vi.spyOn(metadata, "getAgentIdFromArgs").mockReturnValue(customAgentId); + + await updateAgentMetadata({}); + + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + customAgentId, + expect.any(Object), + ); + }); + }); + + describe("metadata posting", () => { + it("should not post when metadata object is empty", async () => { + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "", + repoFound: false, + }); + const { getSessionUsage } = await import("../session.js"); + vi.mocked(getSessionUsage).mockReturnValue({ + totalCost: 0, + promptTokens: 0, + completionTokens: 0, + promptTokensDetails: {}, + }); + + await updateAgentMetadata({}); + + expect(metadata.postAgentMetadata).not.toHaveBeenCalled(); + }); + + it("should post metadata when any field is present", async () => { + const history = [createMockChatHistoryItem("Test", "assistant")]; + vi.spyOn(metadata, "extractSummary").mockReturnValue("Test"); + + await updateAgentMetadata({ history }); + + expect(metadata.postAgentMetadata).toHaveBeenCalledWith( + mockAgentId, + expect.objectContaining({ + summary: "Test", + }), + ); + }); + + it("should handle postAgentMetadata errors gracefully", async () => { + vi.spyOn(metadata, "postAgentMetadata").mockRejectedValue( + new Error("API error"), + ); + const history = [createMockChatHistoryItem("Test", "assistant")]; + vi.spyOn(metadata, "extractSummary").mockReturnValue("Test"); + + await expect(updateAgentMetadata({ history })).resolves.not.toThrow(); + }); + }); + + describe("comprehensive integration scenarios", () => { + it("should collect all metadata when available with isComplete=true", async () => { + const history = [ + createMockChatHistoryItem("User request", "user"), + createMockChatHistoryItem("Assistant response", "assistant"), + ]; + + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "diff content", + repoFound: true, + }); + vi.spyOn(metadata, "calculateDiffStats").mockReturnValue({ + additions: 20, + deletions: 10, + }); + vi.spyOn(metadata, "extractSummary").mockReturnValue( + "Assistant response", + ); + + const { getSessionUsage } = await import("../session.js"); + vi.mocked(getSessionUsage).mockReturnValue({ + totalCost: 0.075, + promptTokens: 2000, + completionTokens: 1000, + promptTokensDetails: { + cachedTokens: 500, + cacheWriteTokens: 100, + }, + }); + + await updateAgentMetadata({ history, isComplete: true }); + + expect(metadata.postAgentMetadata).toHaveBeenCalledWith(mockAgentId, { + additions: 20, + deletions: 10, + isComplete: true, + hasChanges: true, + summary: "Assistant response", + usage: { + totalCost: 0.075, + promptTokens: 2000, + completionTokens: 1000, + cachedTokens: 500, + cacheWriteTokens: 100, + }, + }); + }); + + it("should handle partial metadata with isComplete=false", async () => { + vi.spyOn(git, "getGitDiffSnapshot").mockResolvedValue({ + diff: "", + repoFound: true, + }); + + const { getSessionUsage } = await import("../session.js"); + vi.mocked(getSessionUsage).mockReturnValue({ + totalCost: 0.01, + promptTokens: 100, + completionTokens: 50, + promptTokensDetails: {}, + }); + + await updateAgentMetadata({ isComplete: false }); + + expect(metadata.postAgentMetadata).toHaveBeenCalledWith(mockAgentId, { + usage: { + totalCost: 0.01, + promptTokens: 100, + completionTokens: 50, + }, + }); + }); + }); + + describe("edge cases", () => { + it("should handle undefined options", async () => { + await expect(updateAgentMetadata(undefined)).resolves.not.toThrow(); + }); + + it("should handle empty options object", async () => { + await expect(updateAgentMetadata({})).resolves.not.toThrow(); + }); + + it("should handle null history in options", async () => { + await expect( + updateAgentMetadata({ history: null as any }), + ).resolves.not.toThrow(); + }); + + it("should distinguish between empty array and undefined history", async () => { + // Empty array + await updateAgentMetadata({ history: [] }); + expect(metadata.extractSummary).toHaveBeenCalledWith([]); + + vi.clearAllMocks(); + + // Undefined history + await updateAgentMetadata({}); + expect(metadata.extractSummary).toHaveBeenCalledWith(undefined); + }); + }); +}); diff --git a/extensions/cli/src/util/exit.ts b/extensions/cli/src/util/exit.ts index 2d2c052d9f0..a378cd8de64 100644 --- a/extensions/cli/src/util/exit.ts +++ b/extensions/cli/src/util/exit.ts @@ -14,16 +14,101 @@ import { postAgentMetadata, } from "./metadata.js"; +/** + * Options for updating agent metadata + */ +export interface UpdateAgentMetadataOptions { + /** Chat history to extract summary from */ + history?: ChatHistoryItem[]; + /** Set to true when the agent is exiting/completing */ + isComplete?: boolean; +} + +/** + * Collect diff stats and add to metadata + * @returns Whether there were any changes + */ +async function collectDiffStats( + metadata: Record, +): Promise { + try { + const { diff, repoFound } = await getGitDiffSnapshot(); + if (repoFound && diff) { + const { additions, deletions } = calculateDiffStats(diff); + if (additions > 0 || deletions > 0) { + metadata.additions = additions; + metadata.deletions = deletions; + return true; + } + } + } catch (err) { + logger.debug("Failed to calculate diff stats (non-critical)", err as any); + } + return false; +} + +/** + * Extract summary from conversation history and add to metadata + */ +function collectSummary( + metadata: Record, + history: ChatHistoryItem[] | undefined, +): void { + if (!history || history.length === 0) { + return; + } + try { + const summary = extractSummary(history); + if (summary) { + metadata.summary = summary; + } + } catch (err) { + logger.debug( + "Failed to extract conversation summary (non-critical)", + err as any, + ); + } +} + +/** + * Extract session usage and add to metadata + */ +function collectSessionUsage(metadata: Record): void { + try { + const usage = getSessionUsage(); + if (usage.totalCost > 0) { + metadata.usage = { + totalCost: parseFloat(usage.totalCost.toFixed(6)), + promptTokens: usage.promptTokens, + completionTokens: usage.completionTokens, + ...(usage.promptTokensDetails?.cachedTokens && { + cachedTokens: usage.promptTokensDetails.cachedTokens, + }), + ...(usage.promptTokensDetails?.cacheWriteTokens && { + cacheWriteTokens: usage.promptTokensDetails.cacheWriteTokens, + }), + }; + } + } catch (err) { + logger.debug("Failed to get session usage (non-critical)", err as any); + } +} + /** * Update agent session metadata in control plane * Collects diff stats and conversation summary, posts to control plane * This is called both during execution (after each turn) and before exit * - * @param history - Chat history to extract summary from (optional, will fetch if not provided) + * @param options - Options including history and completion state */ export async function updateAgentMetadata( - history?: ChatHistoryItem[], + options?: ChatHistoryItem[] | UpdateAgentMetadataOptions, ): Promise { + // Support both old signature (history array) and new signature (options object) + const { history, isComplete } = Array.isArray(options) + ? { history: options, isComplete: false } + : { history: options?.history, isComplete: options?.isComplete ?? false }; + try { const agentId = getAgentIdFromArgs(); if (!agentId) { @@ -32,57 +117,16 @@ export async function updateAgentMetadata( } const metadata: Record = {}; + const hasChanges = await collectDiffStats(metadata); - // Calculate diff stats - try { - const { diff, repoFound } = await getGitDiffSnapshot(); - if (repoFound && diff) { - const { additions, deletions } = calculateDiffStats(diff); - if (additions > 0 || deletions > 0) { - metadata.additions = additions; - metadata.deletions = deletions; - } - } - } catch (err) { - logger.debug("Failed to calculate diff stats (non-critical)", err as any); - } - - // Extract summary from conversation - if (history && history.length > 0) { - try { - const summary = extractSummary(history); - if (summary) { - metadata.summary = summary; - } - } catch (err) { - logger.debug( - "Failed to extract conversation summary (non-critical)", - err as any, - ); - } + if (isComplete) { + metadata.isComplete = true; + metadata.hasChanges = hasChanges; } - // Extract session usage (cost and token counts) - try { - const usage = getSessionUsage(); - if (usage.totalCost > 0) { - metadata.usage = { - totalCost: parseFloat(usage.totalCost.toFixed(6)), - promptTokens: usage.promptTokens, - completionTokens: usage.completionTokens, - ...(usage.promptTokensDetails?.cachedTokens && { - cachedTokens: usage.promptTokensDetails.cachedTokens, - }), - ...(usage.promptTokensDetails?.cacheWriteTokens && { - cacheWriteTokens: usage.promptTokensDetails.cacheWriteTokens, - }), - }; - } - } catch (err) { - logger.debug("Failed to get session usage (non-critical)", err as any); - } + collectSummary(metadata, history); + collectSessionUsage(metadata); - // Post metadata if we have any if (Object.keys(metadata).length > 0) { await postAgentMetadata(agentId, metadata); }