diff --git a/docs/ci-integration.md b/docs/ci-integration.md index b9d526f..5d7253d 100644 --- a/docs/ci-integration.md +++ b/docs/ci-integration.md @@ -50,7 +50,18 @@ Checks that fail cause the test to fail. Checks that warn still pass (they're in ## Running a subset of checks -If certain checks don't apply to your site (for example, you don't serve markdown), limit which checks run in the config: +To exclude specific checks, use `skipChecks`. For example, a local preview server may not implement the content negotiation or cache headers supplied by your production server: + +```yaml +url: https://docs.example.com +skipChecks: + - content-negotiation + - cache-header-hygiene +``` + +Both helpers honor `skipChecks`. Excluded checks do not run; the per-check helper logs their results with a `skip` status. Prefer this exclude-list for ongoing CI: checks added in later AFDocs versions still run automatically. + +For a deliberately narrow run, use the `checks` include-list instead: ```yaml url: https://docs.example.com @@ -69,7 +80,12 @@ Checks not in the list show as skipped in the test output. ```yaml url: https://docs.example.com -# Optional: run only specific checks (omit to run all 28) +# Optional: skip specific checks (run everything else, including future checks) +# skipChecks: +# - content-negotiation +# - cache-header-hygiene + +# Optional: run only specific checks # checks: # - llms-txt-exists # - llms-txt-valid @@ -208,7 +224,7 @@ on: - 'docs/**' ``` -One limitation to plan around: `skipChecks` is currently honored by the CLI and ignored by the vitest helpers ([afdocs#133](https://github.com/agent-ecosystem/afdocs/issues/133)). Until that is fixed, run the localhost target through `afdocs check --config agent-docs.local.yml` rather than the helpers. +The CLI and both vitest helpers honor `skipChecks`. AFDocs checks its own docs site this way; the two workflows are [agent-docs.yml](https://github.com/agent-ecosystem/afdocs/blob/main/.github/workflows/agent-docs.yml) and [agent-docs-live.yml](https://github.com/agent-ecosystem/afdocs/blob/main/.github/workflows/agent-docs-live.yml). diff --git a/src/helpers/vitest-runner.ts b/src/helpers/vitest-runner.ts index e4b8b41..f0146c0 100644 --- a/src/helpers/vitest-runner.ts +++ b/src/helpers/vitest-runner.ts @@ -54,6 +54,7 @@ export function describeAgentDocs( : undefined; const report = await runChecks(config.url, { checkIds: config.checks, + skipCheckIds: config.skipChecks, ...config.options, ...(inferredStrategy && { samplingStrategy: inferredStrategy as 'curated' }), curatedPages: config.pages, @@ -104,6 +105,7 @@ export function describeAgentDocsPerCheck( : undefined; report = await runChecks(config.url, { checkIds: config.checks, + skipCheckIds: config.skipChecks, ...config.options, ...(inferredStrategy && { samplingStrategy: inferredStrategy as 'curated' }), curatedPages: config.pages, diff --git a/src/runner.ts b/src/runner.ts index c0396e0..24d2b76 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -173,7 +173,7 @@ export async function runChecks( id: check.id, category: check.category, status: 'skip', - message: 'Check skipped (excluded via --skip-checks)', + message: 'Check explicitly skipped', }; storeInPreviousResults = false; } else if ( diff --git a/test/integration/dependency-chains.test.ts b/test/integration/dependency-chains.test.ts index f2ac05b..7e7ae33 100644 --- a/test/integration/dependency-chains.test.ts +++ b/test/integration/dependency-chains.test.ts @@ -200,7 +200,7 @@ describe('--skip-checks interaction with dependencies', () => { // markdown-url-support should be explicitly skipped const mdUrl = report.results.find((r) => r.id === 'markdown-url-support')!; expect(mdUrl.status).toBe('skip'); - expect(mdUrl.message).toContain('--skip-checks'); + expect(mdUrl.message).toBe('Check explicitly skipped'); // content-negotiation should still run (not affected by skip) const cn = report.results.find((r) => r.id === 'content-negotiation')!; @@ -234,7 +234,7 @@ describe('--skip-checks interaction with dependencies', () => { const exists = report.results.find((r) => r.id === 'llms-txt-exists')!; expect(exists.status).toBe('skip'); - expect(exists.message).toContain('--skip-checks'); + expect(exists.message).toBe('Check explicitly skipped'); // The runner does NOT block llms-txt-valid (dep "never ran" from runner's // perspective, since --skip-checks results aren't stored in previousResults). diff --git a/test/unit/helpers/vitest-runner.test.ts b/test/unit/helpers/vitest-runner.test.ts new file mode 100644 index 0000000..2e9f5b0 --- /dev/null +++ b/test/unit/helpers/vitest-runner.test.ts @@ -0,0 +1,101 @@ +import { expect, vi } from 'vitest'; +import type * as Vitest from 'vitest'; +import { + describeAgentDocs, + describeAgentDocsPerCheck, +} from '../../../src/helpers/vitest-runner.js'; +import { loadConfig } from '../../../src/helpers/config.js'; +import { runChecks } from '../../../src/runner.js'; +import type { AgentDocsConfig } from '../../../src/types.js'; + +const { describe, it, beforeEach } = await vi.importActual('vitest'); + +const { runCallbacks } = vi.hoisted(() => ({ + runCallbacks: [] as Array<() => Promise>, +})); + +vi.mock('vitest', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + describe: (_name: string, register: () => void) => register(), + it: (name: string, run: () => Promise) => { + if (name === 'should run checks') runCallbacks.push(run); + }, + beforeAll: (run: () => Promise) => runCallbacks.push(run), + }; +}); + +vi.mock('../../../src/helpers/config.js', () => ({ loadConfig: vi.fn() })); +vi.mock('../../../src/runner.js', () => ({ + runChecks: vi.fn(async () => ({ results: [] })), +})); + +beforeEach(() => { + vi.clearAllMocks(); + runCallbacks.length = 0; +}); + +describe.each([ + { name: 'describeAgentDocs', helper: describeAgentDocs }, + { name: 'describeAgentDocsPerCheck', helper: describeAgentDocsPerCheck }, +])('$name', ({ helper }) => { + const config: AgentDocsConfig = { + url: 'https://docs.example.com', + checks: ['llms-txt-exists', 'content-negotiation'], + skipChecks: ['content-negotiation'], + pages: [ + 'https://docs.example.com/quickstart', + { url: 'https://docs.example.com/api/auth', tag: 'api-reference' }, + ], + options: { maxLinksToTest: 5 }, + }; + + it.each(['inline', 'directory', 'default'] as const)( + 'forwards checks, skipChecks, pages, and options from %s config', + async (source) => { + vi.mocked(loadConfig).mockResolvedValue(config); + const input = source === 'inline' ? config : source === 'directory' ? '/docs' : undefined; + + helper(input); + expect(runCallbacks).toHaveLength(1); + await runCallbacks[0](); + + expect(runChecks).toHaveBeenCalledExactlyOnceWith(config.url, { + checkIds: config.checks, + skipCheckIds: config.skipChecks, + curatedPages: config.pages, + maxLinksToTest: 5, + samplingStrategy: 'curated', + }); + if (source === 'inline') { + expect(loadConfig).not.toHaveBeenCalled(); + } else { + expect(loadConfig).toHaveBeenCalledExactlyOnceWith(input); + } + }, + ); + + it('leaves selection and sampling unset when not configured', async () => { + helper({ url: config.url }); + await runCallbacks[0](); + + expect(runChecks).toHaveBeenCalledExactlyOnceWith(config.url, { + checkIds: undefined, + skipCheckIds: undefined, + curatedPages: undefined, + }); + }); + + it('preserves an explicit sampling strategy with curated pages', async () => { + helper({ ...config, options: { samplingStrategy: 'deterministic' } }); + await runCallbacks[0](); + + expect(runChecks).toHaveBeenCalledExactlyOnceWith(config.url, { + checkIds: config.checks, + skipCheckIds: config.skipChecks, + curatedPages: config.pages, + samplingStrategy: 'deterministic', + }); + }); +}); diff --git a/test/unit/runner.test.ts b/test/unit/runner.test.ts index 7f75655..96fedda 100644 --- a/test/unit/runner.test.ts +++ b/test/unit/runner.test.ts @@ -482,7 +482,7 @@ describe('runner', () => { const skipped = report.results.find((r) => r.id === 'llms-txt-valid'); expect(skipped).toBeDefined(); expect(skipped?.status).toBe('skip'); - expect(skipped?.message).toContain('--skip-checks'); + expect(skipped?.message).toBe('Check explicitly skipped'); // llms-txt-exists should still run (not in skipCheckIds) const exists = report.results.find((r) => r.id === 'llms-txt-exists'); @@ -520,7 +520,7 @@ describe('runner', () => { const exists = report.results.find((r) => r.id === 'llms-txt-exists'); expect(exists?.status).toBe('skip'); - expect(exists?.message).toContain('--skip-checks'); + expect(exists?.message).toBe('Check explicitly skipped'); // llms-txt-valid should run in standalone mode, not cascade-skip const valid = report.results.find((r) => r.id === 'llms-txt-valid'); diff --git a/vitest.config.ts b/vitest.config.ts index c3fc3eb..626a7d8 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -7,7 +7,6 @@ export default defineConfig({ coverage: { provider: 'v8', include: ['src/**/*.ts'], - exclude: ['src/helpers/vitest-runner.ts'], }, }, });