Generate config example (#128)

* Make content generation ready for other use-cases

* Separate generic genarator from cli option specifics

* Fetch example config from main repository

* Update config documentation & extract version note

* Apply suggestions made by @mre

* Apply suggestion by @katrinafyi

Author: thomas-zahner

astro.config.mjs

@@ -1,7 +1,7 @@
 import starlight from "@astrojs/starlight";
 import { defineConfig } from "astro/config";
 import smartypants from "remark-smartypants";
-import { generateCliOptionsIntegration } from "./src/generate-cli-options";
+import { generateFilesFromMainRepo } from "./src/generate/generate";
 
 // https://astro.build/config
 export default defineConfig({
@@ -14,7 +14,7 @@ export default defineConfig({
 		],
 	},
 	integrations: [
-		generateCliOptionsIntegration("src/content/docs/guides/_cli.md"),
+		generateFilesFromMainRepo(),
 		starlight({
 			expressiveCode: {
 				themes: ["catppuccin-frappe", "catppuccin-latte"],

src/content/docs/guides/.gitignore

@@ -1 +1,2 @@
 /cli.md
+/config.md

src/content/docs/guides/_cli.md

@@ -27,4 +27,3 @@ lychee --exclude https://example.com --exclude https://example.org README.md
 
 To specify multiple values in this way, the argument flag should be repeated.
 Otherwise, the extra values would be treated as link checking inputs.
-

src/content/docs/guides/_config.md

@@ -0,0 +1,30 @@
+---
+title: Configuration File
+---
+
+The configuration file is a [TOML](https://toml.io) file that can be used to specify the options that are also available on the command line.
+It comes in handy when you want to specify a lot of options, or when you want to configure lychee for continuous integration as part of a repository.
+
+By default `lychee.toml` is used if it exists in the current working directory.
+If `--config <PATH>` is specified, then the configuration file at the specified path will be used.
+
+```bash
+lychee --config config.toml
+```
+
+For example, the default value for the `--timeout` option is `20` seconds.
+If you want to change this value, you can do so in the configuration file.
+
+```toml title="lychee.toml"
+timeout = 30
+```
+
+## Example
+
+Here is an example configuration file making use of every single option.
+
+CONFIG-VERSION-NOTE-PLACEHOLDER
+
+```toml title="lychee.toml"
+CONFIG-PLACEHOLDER
+```

src/content/docs/guides/config.md

@@ -1,13 +1,5 @@
 import assert from "node:assert";
-import { readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
-import { basename, dirname, join } from "node:path";
-import type { AstroIntegration } from "astro";
-import { LYCHEE_VERSION } from "./lychee-version";
-
-// https://raw.githubusercontent.com/lycheeverse/lychee/master/README.md
-const url = `https://raw.githubusercontent.com/lycheeverse/lychee/refs/tags/${LYCHEE_VERSION}/README.md`;
-
-const TEMPLATE = "README-OPTIONS-PLACEHOLDER";
+import { versionNote as versionGenerationNote } from "./generate";
 
 function extractHelpFromReadme(readme: string) {
 	const [, section] = readme.split(/### Commandline Parameters/, 2);
@@ -50,12 +42,7 @@ function* generateMarkdown(lines: string[]) {
 			yield "```bash";
 			yield line.replace(/^Usage: /, "");
 			yield "```";
-			yield `
-:::note
-This page is up-to-date as of
-[${LYCHEE_VERSION}](https://github.com/lycheeverse/lychee/releases/tag/${LYCHEE_VERSION}).
-:::
-`;
+			yield versionGenerationNote();
 		} else if ((match = line.match(headingRegex))) {
 			yield `## ${match[1]}`;
 		} else if ((match = line.match(optionRegex))) {
@@ -82,12 +69,9 @@ This page is up-to-date as of
 }
 // biome-ignore-end lint/suspicious/noAssignInExpressions: using assignment expressions for regex match is conventional
 
-export async function generateCliOptionsMarkdown() {
-	const readme = await fetch(url);
-	assert(readme.ok, `${readme.status} when fetching ${url}`);
-
-	const rawUsageText = extractHelpFromReadme(await readme.text());
-	const usageText = [...generateMarkdown(splitLines(rawUsageText))].join("\n");
+export async function generate(readmeContents: string) {
+	const rawHelpText = extractHelpFromReadme(readmeContents);
+	const usageText = [...generateMarkdown(splitLines(rawHelpText))].join("\n");
 
 	assert(
 		usageText.match("\n## Options\n"),
@@ -108,37 +92,3 @@ export async function generateCliOptionsMarkdown() {
 
 	return usageText;
 }
-
-export function generateCliOptionsIntegration(
-	templatePath: string,
-): AstroIntegration {
-	const [dir, file] = [dirname(templatePath), basename(templatePath)];
-
-	const outputPath = join(dir, file.replace("_", ""));
-
-	return {
-		name: "lycheeverse:generate-cli-page",
-		hooks: {
-			"astro:config:setup": async ({ logger, addWatchFile }) => {
-				logger.info(`Using template file ${templatePath}`);
-
-				addWatchFile(realpathSync(templatePath));
-				addWatchFile(import.meta.filename);
-
-				logger.info(`Fetching from git tag ${LYCHEE_VERSION}`);
-				rmSync(outputPath, { force: true });
-				const usageText = generateCliOptionsMarkdown();
-
-				const docTemplateText = readFileSync(templatePath, "utf-8");
-				const docOutput = docTemplateText.replace(TEMPLATE, await usageText);
-
-				assert(
-					docOutput !== docTemplateText,
-					`Placeholder ${TEMPLATE} not found in template file`,
-				);
-				logger.info(`Writing output file ${outputPath}`);
-				writeFileSync(outputPath, docOutput);
-			},
-		},
-	};
-}