Validate generated reference docs in CI

.github/workflows/flaky-tests.yml

@@ -54,6 +54,6 @@ jobs:
           time-range: "7d"
           top-k: "3"
           skip-posting-issues: ${{ github.event_name == 'workflow_dispatch' && inputs.skip-posting-issues || 'false' }}
-          ignored-tests: "TestOurUpstreamTestCasesAreInSyncWithUpstream,TestConfigDescriptorIsUpToDate" # TestOurUpstreamTestCasesAreInSyncWithUpstream is supposed to block upstream mimir-prometheus updates, so it's also expected to fail. TestConfigDescriptorIsUpToDate is flaky.
+          ignored-tests: "TestOurUpstreamTestCasesAreInSyncWithUpstream" # TestOurUpstreamTestCasesAreInSyncWithUpstream is supposed to block upstream mimir-prometheus updates, so it's also expected to fail.
         env:
           GITHUB_TOKEN: ${{ steps.github-token.outputs.token }}

.github/workflows/test-build-deploy.yml

@@ -79,6 +79,8 @@ jobs:
         run: make BUILD_IN_CONTAINER=false check-protos
       - name: Check Generated Documentation
         run: make BUILD_IN_CONTAINER=false check-doc
+      - name: Check Reference Help Documentation
+        run: make BUILD_IN_CONTAINER=false check-reference-help
       - name: Check White Noise
         run: make BUILD_IN_CONTAINER=false check-white-noise
       - name: Check License Header

Makefile

@@ -10,7 +10,7 @@ help:
 # WARNING: do not commit to a repository!
 -include Makefile.local
 
-.PHONY: all test test-with-race integration-tests cover clean images protos exes dist doc clean-doc check-doc push-multiarch-build-image license check-license format check-mixin check-mixin-jb check-mixin-mixtool check-mixin-runbooks check-mixin-mimirtool-rules build-mixin format-mixin check-jsonnet-manifests format-jsonnet-manifests push-multiarch-mimir list-image-targets check-jsonnet-getting-started mixin-screenshots
+.PHONY: all test test-with-race integration-tests cover clean images protos exes dist doc clean-doc check-doc check-reference-help push-multiarch-build-image license check-license format check-mixin check-mixin-jb check-mixin-mixtool check-mixin-runbooks check-mixin-mimirtool-rules build-mixin format-mixin check-jsonnet-manifests format-jsonnet-manifests push-multiarch-mimir list-image-targets check-jsonnet-getting-started mixin-screenshots
 .DEFAULT_GOAL := all
 
 # Version number
@@ -647,6 +647,15 @@ check-doc: doc
 	@find . -name "*.md" | xargs git diff --exit-code -- \
 	|| (echo "Please update generated documentation by running 'make doc' and committing the changes" && false)
 
+check-reference-help: ## Check the reference help documentation is up to date.
+check-reference-help: reference-help
+	@git diff --exit-code -- \
+		cmd/mimir/help.txt.tmpl \
+		cmd/mimir/help-all.txt.tmpl \
+		cmd/mimir/config-descriptor.json \
+		operations/mimir/mimir-flags-defaults.json \
+	|| (echo "Please update generated reference documentation by running 'make reference-help' and committing the changes" && false)
+
 # Tool is developed in the grafana/technical-documentation repository:
 # https://github.com/grafana/technical-documentation/tree/main/tools/doc-validator
 check-doc-validator: ## Check documentation using doc-validator tool

cmd/mimir/main_test.go

@@ -248,64 +248,6 @@ ruler_storage:
 	}
 }
 
-func TestHelp(t *testing.T) {
-	for _, tc := range []struct {
-		name     string
-		arg      string
-		filename string
-	}{
-		{
-			name:     "basic",
-			arg:      "-h",
-			filename: "help.txt.tmpl",
-		},
-		{
-			name:     "all",
-			arg:      "-help-all",
-			filename: "help-all.txt.tmpl",
-		},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			oldArgs, oldStdout, oldStderr, oldTestMode, oldCmdLine := os.Args, os.Stdout, os.Stderr, testMode, flag.CommandLine
-			restored := false
-			restoreIfNeeded := func() {
-				if restored {
-					return
-				}
-
-				os.Stdout = oldStdout
-				os.Stderr = oldStderr
-				os.Args = oldArgs
-				testMode = oldTestMode
-				flag.CommandLine = oldCmdLine
-				restored = true
-			}
-			t.Cleanup(restoreIfNeeded)
-
-			testMode = true
-			co := test.CaptureOutput(t)
-
-			const cmd = "./cmd/mimir/mimir"
-			os.Args = []string{cmd, tc.arg}
-
-			// reset default flags
-			flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
-
-			main()
-
-			stdout, stderr := co.Done()
-
-			// Restore stdout and stderr before reporting errors to make them visible.
-			restoreIfNeeded()
-
-			expected, err := os.ReadFile(tc.filename)
-			require.NoError(t, err)
-			assert.Equalf(t, string(expected), string(stdout), "%s %s output changed; try `make reference-help`", cmd, tc.arg)
-			assert.Empty(t, stderr)
-		})
-	}
-}
-
 func testSingle(t *testing.T, arguments []string, configYAML string, stdoutMessage, stderrMessage, stdoutExcluded, stderrExcluded string, assertConfig func(*testing.T, *mimir.Config)) {
 	t.Helper()
 	oldArgs, oldStdout, oldStderr, oldTestMode := os.Args, os.Stdout, os.Stderr, testMode

operations/mimir/mimir-flags-defaults.json

@@ -35,9 +35,9 @@
   "server.grpc-max-recv-msg-size-bytes": 104857600,
   "server.grpc-max-send-msg-size-bytes": 104857600,
   "server.grpc-max-concurrent-streams": 100,
-  "server.grpc.keepalive.max-connection-idle": 9223372036854776000,
-  "server.grpc.keepalive.max-connection-age": 9223372036854776000,
-  "server.grpc.keepalive.max-connection-age-grace": 9223372036854776000,
+  "server.grpc.keepalive.max-connection-idle": 9223372036854775807,
+  "server.grpc.keepalive.max-connection-age": 9223372036854775807,
+  "server.grpc.keepalive.max-connection-age-grace": 9223372036854775807,
   "server.grpc.keepalive.time": 7200000000000,
   "server.grpc.keepalive.timeout": 20000000000,
   "server.grpc.keepalive.min-time-between-pings": 10000000000,