From 11d416eeba54aecae9c2b06501f2963f9f7b074a Mon Sep 17 00:00:00 2001 From: Jayesh Betala Date: Sun, 7 Jun 2026 16:20:38 +0530 Subject: [PATCH] fix(model-benchmark): don't drop the claude provider on macOS subscription installs ClaudeAdapter.available() treated Claude auth as present only when ~/.claude/.credentials.json exists or ANTHROPIC_API_KEY is set. On the default macOS Claude Code subscription install the credential lives in the login Keychain, not that file, so available() returned NOT READY for a provider whose run() path (`claude -p --output-format json`) authenticates fine via the same subscription session. model-benchmark and the LLM-judge path silently dropped the claude provider for that whole population. An absent ~/.claude/.credentials.json on macOS is not evidence of missing auth. Keep the positive signals (creds file or ANTHROPIC_API_KEY) as a fast yes, but on darwin be optimistic when the binary resolves and defer the real auth decision to run(), which already classifies a genuinely logged-out state as an `auth` error. Non-macOS keeps the strict file/key check, where ~/.claude/.credentials.json is the actual credential store. Adds test/claude-adapter-available.test.ts covering: macOS no-creds-no-key now available; non-macOS no-creds-no-key still not available; API key and a present creds file available on any platform; unresolvable binary still reported before the auth sniff. The macOS case fails on main and passes with this change. Fixes #1890 Co-Authored-By: Claude Opus 4.8 (1M context) --- test/claude-adapter-available.test.ts | 88 +++++++++++++++++++++++++++ test/helpers/providers/claude.ts | 19 ++++-- 2 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 test/claude-adapter-available.test.ts diff --git a/test/claude-adapter-available.test.ts b/test/claude-adapter-available.test.ts new file mode 100644 index 0000000000..7207a11aed --- /dev/null +++ b/test/claude-adapter-available.test.ts @@ -0,0 +1,88 @@ +/** + * Unit tests for ClaudeAdapter.available() auth detection (issue #1890). + * + * The adapter must not drop the claude provider for a logged-in macOS + * subscription install, where the credential lives in the login Keychain and + * ~/.claude/.credentials.json is absent yet `claude -p` works. available() + * stays strict on non-macOS, where ~/.claude/.credentials.json is the actual + * credential store. + * + * Does NOT exercise the live CLI — resolveClaudeCommand is satisfied with a + * real binary via GSTACK_CLAUDE_BIN, os.homedir is pointed at an empty dir so + * the creds file is reliably absent, and process.platform is overridden. + */ + +import { test, expect, beforeEach, afterEach, spyOn } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { ClaudeAdapter } from './helpers/providers/claude'; + +const adapter = new ClaudeAdapter(); +const origPlatform = process.platform; +const origKey = process.env.ANTHROPIC_API_KEY; +const origBin = process.env.GSTACK_CLAUDE_BIN; + +let emptyHome: string; +let homedirSpy: ReturnType; + +function setPlatform(value: NodeJS.Platform) { + Object.defineProperty(process, 'platform', { value, configurable: true }); +} + +beforeEach(() => { + // A home with no ~/.claude/.credentials.json, regardless of the real machine. + emptyHome = fs.mkdtempSync(path.join(os.tmpdir(), 'claude-adapter-test-')); + homedirSpy = spyOn(os, 'homedir').mockReturnValue(emptyHome); + // resolveClaudeCommand resolves an existing absolute binary (not the live CLI). + process.env.GSTACK_CLAUDE_BIN = process.execPath; + delete process.env.ANTHROPIC_API_KEY; +}); + +afterEach(() => { + homedirSpy.mockRestore(); + setPlatform(origPlatform); + if (origKey === undefined) delete process.env.ANTHROPIC_API_KEY; + else process.env.ANTHROPIC_API_KEY = origKey; + if (origBin === undefined) delete process.env.GSTACK_CLAUDE_BIN; + else process.env.GSTACK_CLAUDE_BIN = origBin; + fs.rmSync(emptyHome, { recursive: true, force: true }); +}); + +test('macOS subscription install (no creds file, no key) reports available', async () => { + setPlatform('darwin'); + const check = await adapter.available(); + expect(check.ok).toBe(true); +}); + +test('non-macOS with no creds file and no key reports not available', async () => { + setPlatform('linux'); + const check = await adapter.available(); + expect(check.ok).toBe(false); + expect(check.reason).toMatch(/No Claude auth found/); +}); + +test('ANTHROPIC_API_KEY makes the adapter available on any platform', async () => { + process.env.ANTHROPIC_API_KEY = 'sk-test'; + setPlatform('linux'); + expect((await adapter.available()).ok).toBe(true); + setPlatform('darwin'); + expect((await adapter.available()).ok).toBe(true); +}); + +test('a present .credentials.json makes the adapter available on non-macOS', async () => { + setPlatform('linux'); + fs.mkdirSync(path.join(emptyHome, '.claude'), { recursive: true }); + fs.writeFileSync(path.join(emptyHome, '.claude', '.credentials.json'), '{}'); + expect((await adapter.available()).ok).toBe(true); +}); + +test('an unresolvable binary still reports not available before the auth sniff', async () => { + setPlatform('darwin'); + // A bare (non-absolute) name forces a PATH lookup, which fails — an absolute + // override is trusted as-is by the resolver and would not exercise this path. + process.env.GSTACK_CLAUDE_BIN = 'no-such-claude-binary-xyz'; + const check = await adapter.available(); + expect(check.ok).toBe(false); + expect(check.reason).toMatch(/claude CLI not found/); +}); diff --git a/test/helpers/providers/claude.ts b/test/helpers/providers/claude.ts index 5e3c1acb1a..1ce36f7574 100644 --- a/test/helpers/providers/claude.ts +++ b/test/helpers/providers/claude.ts @@ -25,14 +25,25 @@ export class ClaudeAdapter implements ProviderAdapter { if (!resolved) { return { ok: false, reason: 'claude CLI not found on PATH. Install from https://claude.ai/download or npm i -g @anthropic-ai/claude-code (or set GSTACK_CLAUDE_BIN)' }; } - // Auth sniff: ~/.claude/.credentials.json OR ANTHROPIC_API_KEY + // Auth sniff. Positive signals: a logged-in file credential or an explicit + // API key. const credsPath = path.join(os.homedir(), '.claude', '.credentials.json'); const hasCreds = fs.existsSync(credsPath); const hasKey = !!process.env.ANTHROPIC_API_KEY; - if (!hasCreds && !hasKey) { - return { ok: false, reason: 'No Claude auth found. Log in via `claude` interactive session, or export ANTHROPIC_API_KEY.' }; + if (hasCreds || hasKey) { + return { ok: true }; } - return { ok: true }; + // On macOS the default Claude Code subscription install stores its + // credential in the login Keychain, not ~/.claude/.credentials.json — so an + // absent file is NOT evidence of missing auth. run() drives the same + // `claude -p` path and already classifies a genuinely logged-out state as an + // `auth` error, so be optimistic here and defer the real auth decision to + // run() rather than dropping a provider whose run() path would succeed. (A + // subscription session needs no ANTHROPIC_API_KEY for `claude -p`.) + if (process.platform === 'darwin') { + return { ok: true }; + } + return { ok: false, reason: 'No Claude auth found. Log in via `claude` interactive session, or export ANTHROPIC_API_KEY.' }; } async run(opts: RunOpts): Promise {