testuite: update tests for new shutdown script

Problem: some tests assume that clearing the rc1/rc3 paths
is sufficient to bypass all default startup and shutdown activities,
but now the shutdown script path must also be cleared.

Modify test scripts to take the shutdown script into
consideration when modifying rc1/rc3 paths.

Author: garlick

t/issues/t2284-initial-program-format-chars.sh

@@ -3,7 +3,7 @@
 
 for s in %h %g %%h %f; do
     echo "Running flux broker echo $s"
-    output=$(flux broker -Sbroker.rc1_path= -Sbroker.rc3_path= /bin/echo $s)
+    output=$(flux broker -Sbroker.rc1_path= -Sbroker.shutdown_path= -Sbroker.rc3_path= /bin/echo $s)
     test "$output" = "$s"
 done
 exit 0

t/issues/t4482-flush-list-corruption.sh

@@ -39,5 +39,6 @@ chmod +x t4482.sh
 
 flux start -s 1 \
     -o,--setattr=broker.rc1_path= \
+    -o,--setattr=broker.shutdown_path= \
     -o,--setattr=broker.rc3_path= \
     ./t4482.sh

t/t0001-basic.t

@@ -132,7 +132,7 @@ test_expect_success 'flux fortune with art works' '
 
 # Minimal is sufficient for these tests, but test_under_flux unavailable
 # clear the RC paths
-ARGS="-o,-Sbroker.rc1_path=,-Sbroker.rc3_path="
+ARGS="-o,-Sbroker.rc1_path=,-Sbroker.rc3_path=,-Sbroker.shutdown_path="
 
 test_expect_success 'flux-start in exec mode works' "
 	flux start ${ARGS} flux getattr size | grep -x 1

t/t0003-module.t

@@ -286,7 +286,7 @@ test_expect_success 'module: remove testmod if loaded' '
 '
 test_expect_success 'module: load without unload causes broker failure' '
 	test_must_fail flux start \
-	    -o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
+	    -o,-Sbroker.rc1_path=,-Sbroker.rc3_path=,-Sbroker.shutdown_path= \
 	    flux module load content 2>nounload.err
 '
 test_expect_success 'module: module name is called out' '

t/t0012-content-sqlite.t

@@ -20,6 +20,7 @@ RPC=${FLUX_BUILD_DIR}/t/request/rpc
 SPAMUTIL="${FLUX_BUILD_DIR}/t/kvs/content-spam"
 rc1_kvs=$SHARNESS_TEST_SRCDIR/rc/rc1-kvs
 rc3_kvs=$SHARNESS_TEST_SRCDIR/rc/rc3-kvs
+shutdown_kvs=$SHARNESS_TEST_SRCDIR/rc/shutdown-kvs
 
 test_expect_success 'load content module with lower purge/age thresholds' '
 	flux exec flux module load content \
@@ -333,20 +334,29 @@ test_expect_success 'reload module with no options and verify modes' '
 
 
 test_expect_success 'run flux without statedir and verify modes' '
-	flux start -o,-Sbroker.rc1_path=$rc1_kvs,-Sbroker.rc3_path=$rc3_kvs \
+	flux start \
+	    -o,-Sbroker.rc1_path=$rc1_kvs \
+	    -o,-Sbroker.shutdown_path=$shutdown_kvs \
+	    -o,-Sbroker.rc3_path=$rc3_kvs \
 	    flux dmesg >logs3 &&
 	grep "journal_mode=OFF synchronous=OFF" logs3
 '
 test_expect_success 'run flux with statedir and verify modes' '
-	flux start -o,-Sbroker.rc1_path=$rc1_kvs,-Sbroker.rc3_path=$rc3_kvs \
+	flux start \
+	    -o,-Sbroker.rc1_path=$rc1_kvs \
+	    -o,-Sbroker.shutdown_path=$shutdown_kvs \
+	    -o,-Sbroker.rc3_path=$rc3_kvs \
 	    -o,-Sstatedir=$(pwd) flux dmesg >logs4  &&
 	grep "journal_mode=WAL synchronous=NORMAL" logs4
 '
 
 # Will create in WAL mode since statedir is set
 recreate_database()
 {
-	flux start -o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
+	flux start \
+	    -o,-Sbroker.rc1_path= \
+	    -o,-Sbroker.shutdown_path= \
+	    -o,-Sbroker.rc3_path= \
 	    -o,-Sstatedir=$(pwd) bash -c \
 	    "flux module load content &&
 	    flux module load content-sqlite truncate && \
@@ -355,7 +365,10 @@ recreate_database()
 }
 load_module_xfail()
 {
-	flux start -o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
+	flux start \
+	    -o,-Sbroker.rc1_path= \
+	    -o,-Sbroker.shutdown_path= \
+	    -o,-Sbroker.rc3_path= \
 	    -o,-Sstatedir=$(pwd) bash -c \
 	    "flux module load content; \
 	    flux module load content-sqlite; \

t/t0013-config-file.t

@@ -10,7 +10,7 @@ test_description='Test config file overlay bootstrap'
 
 
 # Avoid loading unnecessary modules in back to back broker tests
-ARGS="-Sbroker.rc1_path= -Sbroker.rc3_path="
+ARGS="-Sbroker.rc1_path= -Sbroker.rc3_path= -Sbroker.shutdown_path="
 
 # This option is compiled out of flux if zeromq is too old
 if flux broker ${ARGS} flux getattr tbon.tcp_user_timeout >/dev/null 2>&1; then
@@ -227,6 +227,7 @@ test_expect_success 'start size=2 instance with ipc://' '
 	EOT
 	flux start -s2 --test-hosts=fake[0-1] \
 		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
+		-o,-Sbroker.shutdown_path= \
 		-o,--config-path=conf8 \
 		./attrdump.sh >ipc.out &&
 	cat <<-EXP >ipc.exp &&
@@ -257,6 +258,7 @@ test_expect_success 'start size=3 instance with ipc:// and custom topology' '
 	EOT
 	flux start --test-size=3 --test-hosts=fake[0-2] \
 		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
+		-o,-Sbroker.shutdown_path= \
 		-o,--config-path=conf8a \
 		flux getattr tbon.maxlevel >conf8a.out &&
 	echo 2 >conf8a.exp &&
@@ -288,8 +290,8 @@ test_expect_success NO_CHAIN_LINT 'a warning is printed when upstream URI has un
 	host = "fake1"
 	EOT
 	FLUX_FAKE_HOSTNAME=fake1 \
-		flux broker -vv -Sbroker.rc1_path=,-Sbroker.rc3_path= \
-		--config-path=conf8b 2>warn.err &
+		flux broker -vv -Sbroker.rc1_path= -Sbroker.rc3_path= \
+		-Sbroker.shutdown_path= --config-path=conf8b 2>warn.err &
 	echo $! >warn.pid &&
 	waitgrep "unable to resolve upstream peer" warn.err 30
 '
@@ -325,6 +327,7 @@ test_expect_success 'start size=4 instance with tcp://' '
 	EOT
 	flux start -s4 --test-hosts=fake[0-3] \
 		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
+		-o,-Sbroker.shutdown_path= \
 		-o,--config-path=conf9 \
 		./attrdump.sh >tcp.out &&
 	cat <<-EXP >tcp.exp &&

t/t0014-runlevel.t

@@ -10,13 +10,17 @@ test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile --debug
 
 test_expect_success 'initial program is run when rc1/rc3 are nullified' '
 	flux start -o,-Slog-stderr-level=6 \
-		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
+		-o,-Sbroker.rc1_path= \
+		-o,-Sbroker.rc3_path= \
+		-o,-Sbroker.shutdown_path= \
 		/bin/true 2>normal.log
 '
 
 test_expect_success 'rc1 failure causes instance failure' '
 	test_expect_code 1 flux start \
-		-o,-Sbroker.rc1_path=/bin/false,-Sbroker.rc3_path= \
+		-o,-Sbroker.rc1_path=/bin/false \
+		-o,-Sbroker.rc3_path= \
+		-o,-Sbroker.shutdown_path= \
 		-o,-Slog-stderr-level=6 \
 		sleep 3600 2>rc1_failure.log
 '
@@ -25,7 +29,9 @@ test_expect_success 'rc1 bad path handled same as failure' '
 	(
 	  SHELL=/bin/sh &&
 	  test_expect_code 127 flux start \
-		-o,-Sbroker.rc1_path=rc1-nonexist,-Sbroker.rc3_path= \
+		-o,-Sbroker.rc1_path=rc1-nonexist \
+		-o,-Sbroker.rc3_path= \
+		-o,-Sbroker.shutdown_path= \
 		-o,-Slog-stderr-level=6 \
 		/bin/true 2>bad1.log
 	)
@@ -35,15 +41,20 @@ test_expect_success 'default initial program is $SHELL' '
 	run_timeout --env=SHELL=/bin/sh 60 \
 		flux $SHARNESS_TEST_SRCDIR/scripts/runpty.py -i none \
 		flux start -o,-Slog-stderr-level=6 \
-		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
+		-o,-Sbroker.rc1_path= \
+		-o,,-Sbroker.rc3_path= \
+		-o,,-Sbroker.shutdown_path= \
 		>shell.log &&
 	grep "rc2.0: /bin/sh Exited" shell.log
 '
 
 test_expect_success 'rc2 failure if stdin not a tty' '
 	test_expect_code 1 \
-		flux start -o,-Slog-stderr-level=6 \
-		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
+		flux start \
+		-o,-Slog-stderr-level=6 \
+		-o,-Sbroker.rc1_path= \
+		-o,-Sbroker.rc3_path= \
+		-o,-Sbroker.shutdown_path= \
                 2>shell-notty.log &&
 	grep "not a tty" shell-notty.log
 '
@@ -59,67 +70,46 @@ test_expect_success 'broker.rc2_none terminates by signal without error' '
 	for timeout in 0.5 1 2 4; do
 	    run_timeout -s ALRM $timeout flux start \
 		-o,-Slog-stderr-level=6 \
-		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path=,-Sbroker.rc2_none &&
+		-o,-Sbroker.rc1_path= \
+		-o,-Sbroker.rc3_path= \
+		-o,-Sbroker.shutdown_path= \
+		-o,-Sbroker.rc2_none &&
 	    break
 	done
 '
 
-test_expect_success 'flux admin cleanup-push /bin/true works' '
+test_expect_success 'shutdown=/bin/true works' '
 	flux start -o,-Slog-stderr-level=6 \
-		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
-		flux admin cleanup-push /bin/true
-'
-
-test_expect_success 'flux admin cleanup-push /bin/false causes instance failure' '
-	test_expect_code 1 flux start -o,-Slog-stderr-level=6 \
-		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
-		flux admin cleanup-push /bin/false
-'
-
-test_expect_success 'cleanup does not run if rc1 fails' '
-	test_expect_code 1 flux start -o,-Slog-stderr-level=6 \
-		-o,-Sbroker.rc1_path=/bin/false,-Sbroker.rc3_path= \
-		flux admin cleanup-push memorable-string 2>nocleanup.err && \
-	test_must_fail grep memorable-string nocleanup.err
+		-o,-Sbroker.rc1_path= \
+		-o,-Sbroker.rc3_path= \
+		-o,-Sbroker.shutdown_path=/bin/true \
+		/bin/true
 '
-
-test_expect_success 'flux admin cleanup-push (empty) fails' '
+test_expect_success 'shutdown=/bin/false causes instance failure' '
 	test_expect_code 1 flux start \
-		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
-		flux admin cleanup-push "" 2>push.err &&
-	grep "cannot push an empty command line" push.err
+		-o,-Slog-stderr-level=6 \
+		-o,-Sbroker.rc1_path= \
+		-o,-Sbroker.rc3_path= \
+		-o,-Sbroker.shutdown_path=/bin/false \
+		/bin/true
 '
 
-test_expect_success 'flux admin cleanup-push with no commands fails' '
+test_expect_success 'shutdown does not run if rc1 fails' '
 	test_expect_code 1 flux start \
-		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
-		flux admin cleanup-push </dev/null 2>push2.err &&
-	grep "commands array is empty" push2.err
-'
-
-test_expect_success 'flux admin cleanup-push (stdin) works' '
-	echo /bin/true | flux start -o,-Slog-stderr-level=6 \
-		-o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
-		flux admin cleanup-push 2>push-stdin.err &&
-	grep cleanup.0 push-stdin.err
-'
-
-test_expect_success 'flux admin cleanup-push (stdin) retains cmd block order' '
-	flux start -o,-Sbroker.rc1_path=,-Sbroker.rc3_path= \
 		-o,-Slog-stderr-level=6 \
-		flux admin cleanup-push <<-EOT 2>hello.err &&
-	echo Hello world
-	echo Hello solar system
-	EOT
-	grep "cleanup.0: Hello world" hello.err &&
-	grep "cleanup.1: Hello solar system" hello.err
+		-o,-Sbroker.rc1_path=/bin/false \
+		-o,-Sbroker.rc3_path= \
+		-o,-Sbroker.shutdown_path=memorable-string \
+		/bin/true 2>nocleanup.err &&
+	test_must_fail grep memorable-string nocleanup.err
 '
 
-test_expect_success 'capture the environment for all three rc scripts' '
+test_expect_success 'capture the environment for all rc scripts' '
 	SLURM_FOO=42 flux start \
 		-o,-Slog-stderr-level=6 \
 		-o,-Sbroker.rc1_path="bash -c printenv >rc1.env" \
 		-o,-Sbroker.rc3_path="bash -c printenv >rc3.env" \
+		-o,-Sbroker.shutdown_path="bash -c printenv >shutdown.env" \
 		"bash -c printenv >rc2.env"
 '
 
@@ -166,6 +156,7 @@ test_expect_success 'capture the environment for instance run as a job' '
 		-o,-Slog-stderr-level=6 \
 		-o,-Sbroker.rc1_path="bash -c printenv >rc1.env2" \
 		-o,-Sbroker.rc3_path="bash -c printenv >rc3.env2" \
+		-o,-Sbroker.shutdown_path="bash -c printenv >shutdown.env2" \
 		"bash -c printenv >rc2.env2"
 '
 

t/t0024-content-s3.t

@@ -349,12 +349,14 @@ EOF
 test_expect_success 'run instance with content-s3 module loaded' '
 	flux start -o,--setattr=broker.rc1_path=$(pwd)/rc1-content-s3 \
                    -o,--setattr=broker.rc3_path=$(pwd)/rc3-content-s3 \
+		   -o,--setattr=broker.shutdown_path= \
 	           flux kvs put testkey=43
 '
 
 test_expect_success 're-run instance with content-s3 module loaded' '
 	flux start -o,--setattr=broker.rc1_path=$(pwd)/rc1-content-s3 \
                    -o,--setattr=broker.rc3_path=$(pwd)/rc3-content-s3 \
+		   -o,--setattr=broker.shutdown_path= \
 	           flux kvs get testkey >gets3.out
 '
 
@@ -366,6 +368,7 @@ test_expect_success 'content from previous instance survived (s3)' '
 test_expect_success 're-run instance, verify checkpoint date saved (s3)' '
 	flux start -o,--setattr=broker.rc1_path=$(pwd)/rc1-content-s3 \
                    -o,--setattr=broker.rc3_path=$(pwd)/rc3-content-s3 \
+		   -o,--setattr=broker.shutdown_path= \
 	           flux dmesg >dmesgs3.out
 '
 

t/t0025-broker-state-machine.t

@@ -9,7 +9,7 @@ test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
 
 RPC=${FLUX_BUILD_DIR}/t/request/rpc
 SRPC=${FLUX_BUILD_DIR}/t/request/rpc_stream
-ARGS="-o,-Sbroker.rc1_path=,-Sbroker.rc3_path="
+ARGS="-o,-Sbroker.rc1_path=,-Sbroker.rc3_path=,-Sbroker.shutdown_path="
 GROUPSCMD="flux python ${SHARNESS_TEST_SRCDIR}/scripts/groups.py"
 
 test_expect_success 'quorum reached on instance with 1 TBON level' '
@@ -83,6 +83,7 @@ test_expect_success 'instance functions with late-joiner' '
 		flux start -s2 \
 		-o,-Slog-stderr-level=6 \
 		-o,-Sbroker.rc1_path="$(pwd)/rc1_block" \
+		-o,-Sbroker.shutdown_path= \
 		-o,-Sbroker.rc3_path= \
 		-o,-Sbroker.quorum=1 \
 		$(pwd)/rc2_unblock >late.out &&
@@ -115,6 +116,7 @@ test_expect_success 'monitor reports INIT(2) in rc1' '
 	echo 2 >rc1.exp &&
 	flux start \
 		-o,-Sbroker.rc1_path=$(pwd)/rc_getstate \
+		-o,-Sbroker.shutdown_path= \
 		-o,-Sbroker.rc3_path= \
 		/bin/true &&
 	test_cmp rc1.exp rc.out
@@ -124,24 +126,27 @@ test_expect_success 'monitor reports RUN(4) in rc2' '
 	echo 4 >rc2.exp &&
 	flux start \
 		-o,-Sbroker.rc1_path= \
+		-o,-Sbroker.shutdown_path= \
 		-o,-Sbroker.rc3_path= \
 		$(pwd)/rc_getstate &&
 	test_cmp rc2.exp rc.out
 '
 
-test_expect_success 'monitor reports CLEANUP(5) in cleanup script' '
+test_expect_success 'monitor reports CLEANUP(5) in shutdown script' '
 	echo 5 >cleanup.exp &&
 	flux start \
 		-o,-Sbroker.rc1_path= \
+		-o,-Sbroker.shutdown_path=$(pwd)/rc_getstate \
 		-o,-Sbroker.rc3_path= \
-		bash -c "echo $(pwd)/rc_getstate | flux admin cleanup-push" &&
+		/bin/true &&
 	test_cmp cleanup.exp rc.out
 '
 
 test_expect_success 'monitor reports FINALIZE(7) in rc3' '
 	echo 7 >rc3.exp &&
 	flux start \
 		-o,-Sbroker.rc1_path= \
+		-o,-Sbroker.shutdown_path= \
 		-o,-Sbroker.rc3_path=$(pwd)/rc_getstate \
 		/bin/true &&
 	test_cmp rc3.exp rc.out
@@ -232,6 +237,7 @@ test_expect_success 'capture state transitions from instance with rc3 failure' '
 	test_expect_code 1 flux start \
 	    -o,-Slog-filename=states_rc3.log \
 	    -o,-Sbroker.rc1_path= \
+	    -o,-Sbroker.shutdown_path= \
 	    -o,-Sbroker.rc3_path=/bin/false \
 	    /bin/true
 '

t/t2004-hydra.t

@@ -7,7 +7,7 @@ test_description='Test that MPICH Hydra can launch Flux'
 test -n "$FLUX_TESTS_LOGFILE" && set -- "$@" --logfile
 . `dirname $0`/sharness.sh
 PMI_INFO=${FLUX_BUILD_DIR}/src/common/libpmi/test_pmi_info
-ARGS="-o,-Sbroker.rc1_path=,-Sbroker.rc3_path="
+ARGS="-o,-Sbroker.rc1_path=,-Sbroker.rc3_path=,-Sbroker.shutdown_path="
 
 if ! which mpiexec.hydra 2>/dev/null; then
     skip_all='skipping hydra-launching-flux tests, mpiexec.hydra unavailable'