Show models powering auto in attribution footers

actions/setup/js/copilot_model_selection.cjs

@@ -10,6 +10,9 @@ const { normalizeForCopilotCLI, splitModelIdentifier } = require("./resolve_mode
 const { copilotSessionContext } = require("./copilot_session.cjs");
 const { getErrorMessage } = require("./error_helpers.cjs");
 
+// Bounds the per-run model history persisted in the selection log (16 KiB) and aw_info.json.
+const MAX_OBSERVED_MODELS = 32;
+
 // Exit code the SDK driver uses to report a main-session model verification failure (substitution
 // or unverifiable evidence) so the harness stops without retrying, as it does in CLI mode.
 const COPILOT_MODEL_VERIFICATION_EXIT_CODE = 79;
@@ -42,6 +45,32 @@ function recordCopilotModelSelection(selection, env = process.env) {
   }
 }
 
+/**
+ * Models already observed by an earlier attempt in this agent phase, so a retried or resumed
+ * session extends the history instead of overwriting it.
+ * @param {NodeJS.ProcessEnv} env
+ * @returns {string[]}
+ */
+function previouslyObservedModels(env) {
+  const file = path.join(env.GH_AW_TMP_DIR || "/tmp/gh-aw", "sandbox/agent/logs/copilot-model-selection.json");
+  try {
+    if (!fs.existsSync(file)) return [];
+    const record = JSON.parse(fs.readFileSync(file, "utf8"));
+    if (record?.status !== "observed") return [];
+    const models = Array.isArray(record.observed_models) ? record.observed_models : [record.effective_model];
+    return [
+      ...new Set(
+        models
+          .filter(model => typeof model === "string")
+          .map(modelID)
+          .filter(Boolean)
+      ),
+    ].slice(-MAX_OBSERVED_MODELS);
+  } catch {
+    return [];
+  }
+}
+
 // Only the runner post-step updates workflow metadata; the harness writes a log.
 function persistCopilotModelSelection(env = process.env) {
   if (env.GH_AW_PHASE && env.GH_AW_PHASE !== "agent") return;
@@ -60,6 +89,17 @@ function persistCopilotModelSelection(env = process.env) {
       resolved_model: typeof record?.resolved_model === "string" ? modelID(record.resolved_model) : "",
       effective_model: typeof record?.effective_model === "string" ? modelID(record.effective_model) : "",
     };
+    const observedModels = Array.isArray(record?.observed_models)
+      ? [
+          ...new Set(
+            record.observed_models
+              .filter(model => typeof model === "string")
+              .map(modelID)
+              .filter(Boolean)
+          ),
+        ].slice(0, MAX_OBSERVED_MODELS)
+      : [];
+    if (observedModels.length > 1) selection.observed_models = observedModels;
     if (!selection.configured_model || !["observed", "substituted", "unavailable"].includes(selection.status)) throw new Error("Invalid Copilot model selection");
     if (selection.status === "unavailable") {
       selection.resolved_model = "";
@@ -92,6 +132,7 @@ function createCopilotModelObserver({ configuredModel, resolvedModel, sessionId,
   let observedModel = "";
   const expected = modelID(resolvedModel);
   if (!expected || (env.GH_AW_PHASE && env.GH_AW_PHASE !== "agent")) return { check: () => "" };
+  const observedModels = new Set(previouslyObservedModels(env));
   if (!sessionId || !/^[a-z0-9_-]+$/i.test(sessionId)) throw new Error("Copilot model verification requires an exact invocation session ID");
   const ownedSessionId = sessionId;
 
@@ -144,8 +185,22 @@ function createCopilotModelObserver({ configuredModel, resolvedModel, sessionId,
           const model = modelID(event.type === "session.model_change" && event.data?.source === "startup" ? event.data.newModel : event.type === "assistant.message" ? event.data?.model : "");
           if (!model || model === observedModel) continue;
           observedModel = model;
+          if (!observedModels.has(model)) {
+            const oldest = observedModels.values().next().value;
+            if (observedModels.size >= MAX_OBSERVED_MODELS && oldest !== undefined) observedModels.delete(oldest);
+            observedModels.add(model);
+          }
           const substituted = expected.toLowerCase() !== "auto" && model.toLowerCase() !== expected.toLowerCase();
-          recordCopilotModelSelection({ status: substituted ? "substituted" : "observed", configured_model: configuredModel, resolved_model: resolvedModel, effective_model: model }, env);
+          recordCopilotModelSelection(
+            {
+              status: substituted ? "substituted" : "observed",
+              configured_model: configuredModel,
+              resolved_model: resolvedModel,
+              effective_model: model,
+              ...(observedModels.size > 1 ? { observed_models: [...observedModels] } : {}),
+            },
+            env
+          );
           if (substituted) {
             diagnostic = modelSubstitutionDiagnostic(expected, model);
             logger(diagnostic);
@@ -200,6 +255,7 @@ function createCopilotSDKModelObserver({ configuredModel, expectedModel, session
   let sessionId = "";
   let diagnostic = "";
   let recorded = "";
+  const observedModels = new Set();
   const initialOffsets = new Map();
   if (restoredSessionId) {
     if (!/^[a-z0-9_-]+$/i.test(restoredSessionId)) throw new Error("Copilot model verification requires an exact restored session ID");
@@ -227,7 +283,20 @@ function createCopilotSDKModelObserver({ configuredModel, expectedModel, session
       const effective = matches ? (unconstrained ? model : expected) : validateModelIdentifier(base) || model;
       if (matches && effective === recorded) return "";
       recorded = effective;
-      recordCopilotModelSelection({ status: matches ? "observed" : "substituted", configured_model: configuredModel, resolved_model: resolvedModel, effective_model: effective }, env);
+      if (!observedModels.has(effective)) {
+        if (observedModels.size >= MAX_OBSERVED_MODELS) observedModels.delete(observedModels.values().next().value);
+        observedModels.add(effective);
+      }
+      recordCopilotModelSelection(
+        {
+          status: matches ? "observed" : "substituted",
+          configured_model: configuredModel,
+          resolved_model: resolvedModel,
+          effective_model: effective,
+          ...(observedModels.size > 1 ? { observed_models: [...observedModels] } : {}),
+        },
+        env
+      );
       if (matches) return "";
       diagnostic = modelSubstitutionDiagnostic(expected, effective);
     } catch (error) {

actions/setup/js/copilot_model_selection.test.cjs

@@ -55,6 +55,18 @@ describe("Copilot configured versus observed model", () => {
     });
   });
 
+  it("keeps models observed before a retry when the resumed session switches models", () => {
+    const autoObserver = () => createCopilotModelObserver({ configuredModel: "auto", resolvedModel: "auto", sessionId: "session", env });
+    const first = autoObserver();
+    append({ type: "assistant.message", data: { model: "gemini-3.8-flash" } });
+    expect(first.check()).toBe("");
+    const resumed = autoObserver();
+    append({ type: "assistant.message", data: { model: "claude-sonnet-5" } });
+    expect(resumed.check()).toBe("");
+    expect(readInfo().model_selection).toMatchObject({ status: "observed", effective_model: "claude-sonnet-5", observed_models: ["gemini-3.8-flash", "claude-sonnet-5"] });
+    expect(getEffectiveModelLabel(infoPath, "agent", { GH_AW_ENGINE_MODEL: "auto" })).toBe("auto (Gemini 3.8 Flash, Claude Sonnet 5)");
+  });
+
   it("records an unavailable model without creating workflow metadata in the sandbox", () => {
     fs.rmSync(infoPath);
     recordCopilotModelSelection({ status: "unavailable", configured_model: "gemini-3.8-flash", resolved_model: "", effective_model: "" }, env);

actions/setup/js/copilot_sdk_model_observation.test.cjs

@@ -11,6 +11,7 @@ const { COPILOT_MODEL_VERIFICATION_EXIT_CODE, createCopilotSDKModelObserver, rea
 const { runWithCopilotSDK } = require("./copilot_sdk_session.cjs");
 const { qualifyModelForMultiProvider } = require("./copilot_sdk_multi_provider.cjs");
 const { detectErrors } = require("./agent_error_patterns.cjs");
+const { getEffectiveModelLabel } = require("./model_attribution.cjs");
 
 // Captured, credential-free model-bearing records from Smoke Copilot SDK Routed run 39529495361
 // (native envelopes and main/sub-agent scope preserved) and the CLI control run 39529502581.
@@ -142,7 +143,13 @@ describe("Copilot SDK main-session model observation", () => {
     // Derived from the 39529495361 fixture: the main session switches to the same id on an unconfigured `copilot` provider.
     const diagnostic = watch.observe({ ...derivedMainModelChange(), data: { ...derivedMainModelChange().data, newModel: "copilot/gpt-5.6-luna" } });
     expect(diagnostic).toContain("Copilot substituted 'copilot/gpt-5.6-luna'");
-    expect(selection()).toEqual({ status: "substituted", configured_model: "gpt-5.6-luna", resolved_model: "gpt-5.6-luna", effective_model: "copilot/gpt-5.6-luna" });
+    expect(selection()).toEqual({
+      status: "substituted",
+      configured_model: "gpt-5.6-luna",
+      resolved_model: "gpt-5.6-luna",
+      effective_model: "copilot/gpt-5.6-luna",
+      observed_models: ["gpt-5.6-luna", "copilot/gpt-5.6-luna"],
+    });
     expect(watch.observe(firstMessage)).toBe(diagnostic);
     expect(selection().status).toBe("substituted");
   });
@@ -263,7 +270,13 @@ runWithCopilotSDK({
     expect(result.stderr).toContain("stopping SDK session: Copilot model verification failed");
     expect(result.stderr).toContain("Copilot substituted 'copilot-completions/claude-haiku-4.5'. Refusing silent model substitution.");
     expect(result.stderr).not.toContain("retry 1/3");
-    expect(selection()).toEqual({ status: "substituted", configured_model: "gpt-5.6-luna", resolved_model: "gpt-5.6-luna", effective_model: "copilot-completions/claude-haiku-4.5" });
+    expect(selection()).toEqual({
+      status: "substituted",
+      configured_model: "gpt-5.6-luna",
+      resolved_model: "gpt-5.6-luna",
+      effective_model: "copilot-completions/claude-haiku-4.5",
+      observed_models: ["gpt-5.6-luna", "copilot-completions/claude-haiku-4.5"],
+    });
     const diagnostic = readCopilotModelSubstitution();
     expect(result.stderr).toContain(diagnostic);
     expect(detectErrors(diagnostic).modelNotSupportedError).toBe(true);
@@ -304,17 +317,46 @@ runWithCopilotSDK({
       },
     });
     expect(result.exitCode).toBe(exitCode);
-    expect(selection()).toEqual({ status, configured_model: "auto", resolved_model: "gpt-5.6-luna", effective_model: mainModel });
+    expect(selection()).toEqual({
+      status,
+      configured_model: "auto",
+      resolved_model: "gpt-5.6-luna",
+      effective_model: mainModel,
+      ...(status === "substituted" ? { observed_models: ["gpt-5.6-luna", "gpt-5.6-sol"] } : {}),
+    });
   });
 
   it("sdk_auto_does_not_report_substitution", () => {
     const watch = createCopilotSDKModelObserver({ configuredModel: "auto", expectedModel: "auto", providerConfig: null, sessionDirs: [] });
     for (const event of mainEvents()) expect(watch.observe(event)).toBe("");
     expect(watch.finalize(SESSION_ID)).toBe("");
-    expect(selection()).toEqual({ status: "observed", configured_model: "auto", resolved_model: "auto", effective_model: "gpt-5.6-luna" });
+    expect(selection()).toEqual({
+      status: "observed",
+      configured_model: "auto",
+      resolved_model: "auto",
+      effective_model: "gpt-5.6-luna",
+      observed_models: ["copilot-responses/gpt-5.6-luna", "gpt-5.6-luna"],
+    });
     expect(readCopilotModelSubstitution()).toBe("");
   });
 
+  it("preserves multiple native models for auto attribution without firewall usage", () => {
+    const infoPath = path.join(root, "aw_info.json");
+    const watch = createCopilotSDKModelObserver({ configuredModel: "auto", expectedModel: "auto", providerConfig: null, sessionDirs: [] });
+    expect(watch.observe({ type: "assistant.message", data: { model: "gemini-3.8-flash" } })).toBe("");
+    expect(watch.observe({ type: "assistant.message", data: { model: "claude-sonnet-5" } })).toBe("");
+    expect(watch.finalize(SESSION_ID)).toBe("");
+    expect(selection()).toEqual({
+      status: "observed",
+      configured_model: "auto",
+      resolved_model: "auto",
+      effective_model: "claude-sonnet-5",
+      observed_models: ["gemini-3.8-flash", "claude-sonnet-5"],
+    });
+    persistCopilotModelSelection();
+    expect(getEffectiveModelLabel(infoPath, "agent", process.env)).toBe("auto (Gemini 3.8 Flash, Claude Sonnet 5)");
+  });
+
   it("sdk_subagent_model_events_do_not_change_main_observation", () => {
     expect(haikuEvents().map(event => [event.type, event.data.newModel || event.data.model])).toEqual([
       ["session.model_change", "copilot-completions/claude-haiku-4.5"],

actions/setup/js/messages.test.cjs

@@ -11,6 +11,8 @@
  */
 import { describe, it, expect, beforeEach, vi } from "vitest";
 import path from "path";
+import fs from "node:fs";
+import os from "node:os";
 import { fileURLToPath } from "url";
 
 // Mock core for GitHub Actions environment
@@ -177,6 +179,41 @@ describe("messages.cjs", () => {
   });
 
   describe("getFooterMessage", () => {
+    it.each([false, true])("shows the model powering auto in default and custom credit footers (custom: %s)", async custom => {
+      const root = fs.mkdtempSync(path.join(os.tmpdir(), "gh-aw-auto-footer-"));
+      const environmentKeys = ["GH_AW_SAFE_OUTPUT_MESSAGES", "GH_AW_TMP_DIR", "GH_AW_ENGINE_ID", "GH_AW_ENGINE_MODEL", "GH_AW_AGENT_AIC", "GH_AW_THREAT_DETECTION_AIC", "GH_AW_AMBIENT_CONTEXT"];
+      const environment = new Map(environmentKeys.map(key => [key, process.env[key]]));
+      try {
+        process.env.GH_AW_TMP_DIR = root;
+        process.env.GH_AW_ENGINE_ID = "copilot";
+        process.env.GH_AW_ENGINE_MODEL = "auto";
+        process.env.GH_AW_AGENT_AIC = "14";
+        process.env.GH_AW_THREAT_DETECTION_AIC = "1";
+        process.env.GH_AW_AMBIENT_CONTEXT = "9200";
+        if (custom) process.env.GH_AW_SAFE_OUTPUT_MESSAGES = JSON.stringify({ footer: "> Generated by [{workflow_name}]({run_url}){ai_credits_suffix}{history_link}" });
+        fs.writeFileSync(path.join(root, "aw_info.json"), JSON.stringify({ requested_model: "auto", model_selection: { status: "observed", effective_model: "gemini-3.8-flash" } }));
+        fs.mkdirSync(path.join(root, "usage"), { recursive: true });
+        fs.writeFileSync(
+          path.join(root, "usage/aw_session.jsonl"),
+          [
+            { type: "firewall.token_usage", data: { model: "gemini-3.8-flash", purpose: "agent" }, provenance: { component: "firewall", phase: "agent" } },
+            { type: "firewall.token_usage", data: { model: "claude-sonnet-5", purpose: "agent" }, provenance: { component: "firewall", phase: "agent" } },
+          ]
+            .map(event => JSON.stringify(event))
+            .join("\n")
+        );
+        const { getFooterMessage } = await import("./messages.cjs");
+        const result = getFooterMessage({ workflowName: "Test Workflow", runUrl: "https://github.com/test/repo/actions/runs/123", historyUrl: "https://github.com/search" });
+        expect(result).toContain(" · copilot · auto (Gemini 3.8 Flash, Claude Sonnet 5) · \\~14 AIC · ⌖ \\~1 AIC · ⊞ 9.2K · [◷]");
+      } finally {
+        fs.rmSync(root, { recursive: true, force: true });
+        for (const [key, value] of environment) {
+          if (value === undefined) delete process.env[key];
+          else process.env[key] = value;
+        }
+      }
+    });
+
     it("should return default footer when no custom config", async () => {
       const { getFooterMessage } = await import("./messages.cjs");