[None][infra] Echo test output to the console on post-merge multi-GPU stages

jenkins/L0_Test.groovy

@@ -48,6 +48,8 @@ UPLOAD_PATH = env.uploadPath ? env.uploadPath : "sw-tensorrt-generic/llm-artifac
 URM_ARTIFACTORY_BASE = "https://urm.nvidia.com/artifactory"
 ENABLE_UPLOAD_TEST_RESULTS = params.enableUploadTestResults != null ? params.enableUploadTestResults : true
 ENABLE_S3_ECHO_STDOUT = params.enableS3EchoStdout != null ? params.enableS3EchoStdout : false
+// Kill switch for the scoped post-merge multi-GPU echo (shouldEchoTestOutputToConsole).
+DISABLE_POST_MERGE_STDOUT_ECHO = params.disableStdoutEchoOnPostMerge != null ? params.disableStdoutEchoOnPostMerge : false
 
 X86_64_TRIPLE = "x86_64-linux-gnu"
 AARCH64_TRIPLE = "aarch64-linux-gnu"
@@ -1249,13 +1251,52 @@ def getPytestBaseCommandLine(
     testCmdLine += ["--unittest-markexpr='${unittestMarkExpr}'"]
     if (ENABLE_UPLOAD_TEST_RESULTS) {
         testCmdLine += ["-o console_output_style=progress-even-when-capture-no"]
+        // ENABLE_S3_ECHO_STDOUT already appends this at the call site; don't duplicate.
+        if (!ENABLE_S3_ECHO_STDOUT && shouldEchoTestOutputToConsole(stageName)) {
+            testCmdLine += ["--s3-echo-stdout"]
+        }
     }
     if (extraArgs) {
         testCmdLine += extraArgs
     }
     return testCmdLine as String[]
 }
 
+// Whether a stage should echo per-test stdout/stderr to the console as well as
+// capturing it for S3.
+//
+// pytest capture is already off (-s), but the S3 log plugin then spools every
+// byte a test -- and every MPI worker rank that inherited its fds -- writes
+// into a session file, and publishes it per test on completion. A test that
+// wedges never completes: pytest's --timeout hard-kills the process with
+// os._exit, so that test's output is never published and the stage log holds
+// no trace of the wedge. The output is not destroyed (the spool ships inside
+// results-<stage>.tar.gz), but recovering it means knowing to download and
+// unpack an artifact, which is not how a stage failure gets triaged.
+//
+// What echoing recovers, precisely: everything written up to roughly a quarter
+// second before the process dies, which covers a HangDetector report. It does
+// not recover pytest-timeout's own stack dump, which is written immediately
+// before os._exit -- that still only reaches the spool, as it does today.
+//
+// Echoing costs log volume, so it is limited to where the evidence is worth
+// most: post-merge multi-GPU stages. Those are the stages whose timeouts burn
+// the most GPU-hours per occurrence, and the wedges there are the ones that
+// leave nothing behind today. PerfSanity stages are excluded so that timing
+// runs keep their current, quieter console. Set the disableStdoutEchoOnPostMerge
+// build parameter to turn this off without a code change; the plugin also caps
+// the echoed bytes per session.
+def shouldEchoTestOutputToConsole(String stageName) {
+    if (DISABLE_POST_MERGE_STDOUT_ECHO) {
+        return false
+    }
+    if (!stageName.contains("Post-Merge") || stageName.contains("PerfSanity")) {
+        return false
+    }
+    def taskConfig = parseTaskConfigFromStageName(stageName)
+    return taskConfig != null && (taskConfig.system_gpu_count as Integer) > 1
+}
+
 def getMountListForSlurmTest(SlurmCluster cluster, boolean useSbatch = false)
 {
     def mounts = []