From ea13f42719f741dea2ab73da299d7ce7d6f59644 Mon Sep 17 00:00:00 2001 From: Alex Deiter Date: Mon, 9 Jan 2023 20:19:26 +0400 Subject: [PATCH] LU-16399 tests: add subtests setup/cleanup records * Added setup/cleanup records for subtests Lustre-change: https://review.whamcloud.com/49582 Lustre-commit: d0ae0079d747d05f74f733fb594d8edb512f8b16 Change-Id: Icb203a864fa8785e423a073b4ee0f02ea3d3ac77 Signed-off-by: Alex Deiter Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/51406 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- lustre/tests/conf-sanity.sh | 4 +- lustre/tests/dne_sanity.sh | 2 +- lustre/tests/functions.sh | 4 +- lustre/tests/ha.sh | 2 +- lustre/tests/lnet-selftest.sh | 2 +- lustre/tests/lustre-rsync-test.sh | 6 +- lustre/tests/sanity-gss.sh | 2 +- lustre/tests/sanity-pcc.sh | 2 +- lustre/tests/sanity-pfl.sh | 2 +- lustre/tests/sanity.sh | 6 +- lustre/tests/sanityn.sh | 10 ++-- lustre/tests/test-framework.sh | 121 ++++++++++++++++++++++++++++++-------- 12 files changed, 117 insertions(+), 46 deletions(-) diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index cd2860c..e237a5d 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -2613,8 +2613,8 @@ test_33b() { # was test_34 run_test 33b "Drop cancel during umount" test_34a() { - setup - do_facet client "sh runmultiop_bg_pause $DIR/file O_c" + setup + do_facet client "bash runmultiop_bg_pause $DIR/file O_c" manual_umount_client rc=$? do_facet client killall -USR1 multiop diff --git a/lustre/tests/dne_sanity.sh b/lustre/tests/dne_sanity.sh index 823249e..99fc91f 100644 --- a/lustre/tests/dne_sanity.sh +++ b/lustre/tests/dne_sanity.sh @@ -75,7 +75,7 @@ test_sanity() for index in $(seq $PARALLEL_RUNS); do DIR=$ORIGIN_DIR/dir${index} PARALLEL=yes \ EXT2_DEV="$TMP/SANITY.LOOP_${index}" \ - LOGDIR="${LOGDIR}_${index}" YAML_LOG="" sh sanity.sh & + LOGDIR="${LOGDIR}_${index}" YAML_LOG="" bash sanity.sh & pid=$! echo start sanity: $pid rpids="$rpids $pid" diff --git a/lustre/tests/functions.sh b/lustre/tests/functions.sh index d775999..a2cc69b 100644 --- a/lustre/tests/functions.sh +++ b/lustre/tests/functions.sh @@ -177,7 +177,7 @@ mpi_run () { if [ -n "$MPI_USER" -a "$MPI_USER" != root -a -n "$mpirun" ]; then echo "+ chmod 0777 $MOUNT" chmod 0777 $MOUNT - command="su $MPI_USER sh -c \"$command \"" + command="su $MPI_USER bash -c \"$command \"" fi ls -ald $MOUNT @@ -575,7 +575,7 @@ run_connectathon() { fi echo "tests: $tests" for test in $tests; do - local cmd="sh ./runtests -N $cnt_NRUN $test -f $testdir" + local cmd="bash ./runtests -N $cnt_NRUN $test -f $testdir" local rc=0 log "$cmd" diff --git a/lustre/tests/ha.sh b/lustre/tests/ha.sh index d30424a..fe2d6f8 100755 --- a/lustre/tests/ha.sh +++ b/lustre/tests/ha.sh @@ -452,7 +452,7 @@ ha_repeat_mpi_load() ha_on $client $LFS setstripe $stripeparams $dir && ha_on $client $LFS getstripe $dir && ha_on $client chmod a+xwr $dir && - ha_on $client "su $mpiuser sh -c \" $mpirun $ha_mpirun_options \ + ha_on $client "su $mpiuser bash -c \" $mpirun $ha_mpirun_options \ -np $((${#ha_clients[@]} * mpi_threads_per_client )) \ $machines $cmd \" " || rc=$? [[ -n "$ha_postcmd" ]] && ha_info "$ha_postcmd" && diff --git a/lustre/tests/lnet-selftest.sh b/lustre/tests/lnet-selftest.sh index b4fcaa1..43eb256 100755 --- a/lustre/tests/lnet-selftest.sh +++ b/lustre/tests/lnet-selftest.sh @@ -167,7 +167,7 @@ run_lst () { export LST_SESSION=$$ # start lst - sh $file + bash $file } check_lst_err () { diff --git a/lustre/tests/lustre-rsync-test.sh b/lustre/tests/lustre-rsync-test.sh index 5382ea6..6c7b2d1 100644 --- a/lustre/tests/lustre-rsync-test.sh +++ b/lustre/tests/lustre-rsync-test.sh @@ -273,7 +273,7 @@ test_2a() { init_changelog # Run dbench - sh rundbench -C -D $DIR/$tdir 2 -t $DBENCH_TIME || error "dbench failed" + bash rundbench -C -D $DIR/$tdir 2 -t $DBENCH_TIME || error "dbench failed" local LRSYNC_LOG=$(generate_logname "lrsync_log") # Replicate the changes to $TGT @@ -297,7 +297,7 @@ test_2b() { init_changelog # Run dbench - sh rundbench -C -D $DIR/$tdir 2 -t $DBENCH_TIME & + bash rundbench -C -D $DIR/$tdir 2 -t $DBENCH_TIME & # wait for dbench to start wait_for_function 'child_pid=$(pgrep dbench)' 360 # let dbench run for a bit @@ -347,7 +347,7 @@ test_2c() { init_changelog # Run dbench - sh rundbench -C -D $DIR/$tdir 2 -t $DBENCH_TIME & + bash rundbench -C -D $DIR/$tdir 2 -t $DBENCH_TIME & local LRSYNC_LOG=$(generate_logname "lrsync_log") # Replicate the changes to $TGT diff --git a/lustre/tests/sanity-gss.sh b/lustre/tests/sanity-gss.sh index 3b6fc56..13b7a26 100755 --- a/lustre/tests/sanity-gss.sh +++ b/lustre/tests/sanity-gss.sh @@ -52,7 +52,7 @@ start_dbench() { NPROC=`cat /proc/cpuinfo 2>/dev/null | grep ^processor | wc -l` [ $NPROC -gt 2 ] && NPROC=2 - sh rundbench $NPROC 1>/dev/null & + bash rundbench $NPROC 1>/dev/null & DBENCH_PID=$! sleep 2 diff --git a/lustre/tests/sanity-pcc.sh b/lustre/tests/sanity-pcc.sh index 0527c34..8063372 100644 --- a/lustre/tests/sanity-pcc.sh +++ b/lustre/tests/sanity-pcc.sh @@ -4715,7 +4715,7 @@ test_101a() { # File is owned by root, make it accessible to RUNAS user chmod a+rw $DIR/$tdir/$tfile.shell stack_trap 'rm -f $DIR/$tdir/$tfile.shell' EXIT - do_facet $SINGLEAGT nsenter -t $PID -U -m "sh $DIR/$tdir/$tfile.shell" + do_facet $SINGLEAGT nsenter -t $PID -U -m "bash $DIR/$tdir/$tfile.shell" do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER \ $file || error "RW-PCC attach $file failed" check_lpcc_state $file "readwrite" diff --git a/lustre/tests/sanity-pfl.sh b/lustre/tests/sanity-pfl.sh index 572b6d3..bfff04e 100644 --- a/lustre/tests/sanity-pfl.sh +++ b/lustre/tests/sanity-pfl.sh @@ -515,7 +515,7 @@ test_8() { -E -1 -c 4 -S 4M $parent || error "Set default layout to $parent failed" - sh rundbench -C -D $parent 2 || error "dbench failed" + bash rundbench -C -D $parent 2 || error "dbench failed" rm -fr $parent || error "Delete dir $parent failed" } diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 7044b9f..8a08408 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -8581,7 +8581,7 @@ run_test 64a "verify filter grant calculations (in kernel) =====" test_64b () { [ $PARALLEL == "yes" ] && skip "skip parallel run" - sh oos.sh $MOUNT || error "oos.sh failed: $?" + bash oos.sh $MOUNT || error "oos.sh failed: $?" } run_test 64b "check out-of-space detection on client" @@ -9372,7 +9372,7 @@ run_test 69 "verify oa2dentry return -ENOENT doesn't LBUG ======" test_71() { test_mkdir $DIR/$tdir $LFS setdirstripe -D -c$MDSCOUNT $DIR/$tdir - sh rundbench -C -D $DIR/$tdir 2 || error "dbench failed!" + bash rundbench -C -D $DIR/$tdir 2 || error "dbench failed!" } run_test 71 "Running dbench on lustre (don't segment fault) ====" @@ -25701,7 +25701,7 @@ test_411() { # Should not LBUG, just be killed by oom-killer # dd will return 0 even allocation failure in some environment. # So don't check return value - sh -c "echo \$$ > $cgdir/tasks && dd if=$DIR/$tfile of=/dev/null" + bash -c "echo \$$ > $cgdir/tasks && dd if=$DIR/$tfile of=/dev/null" cleanup_test411_cgroup $cgdir return 0 diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh index 72ed7e1..3be059e 100755 --- a/lustre/tests/sanityn.sh +++ b/lustre/tests/sanityn.sh @@ -305,7 +305,7 @@ test_11() { run_test 11 "execution of file opened for write should return error ====" test_12() { - DIR=$DIR DIR2=$DIR2 sh lockorder.sh + DIR=$DIR DIR2=$DIR2 bash lockorder.sh } run_test 12 "test lock ordering (link, stat, unlink)" @@ -383,7 +383,7 @@ run_test 14d "chmod of executing file is still possible ========" test_15() { # bug 974 - ENOSPC echo "PATH=$PATH" - sh oos2.sh $MOUNT1 $MOUNT2 + bash oos2.sh $MOUNT1 $MOUNT2 wait_delete_completed grant_error=$(dmesg | grep "< tot_grant") [ -z "$grant_error" ] || error "$grant_error" @@ -983,7 +983,7 @@ run_test 28 "read/write/truncate file with lost stripes" test_30() { #b=11110, LU-2523 test_mkdir $DIR1/$tdir cp -f /bin/bash $DIR1/$tdir/bash - /bin/sh -c 'sleep 1; rm -f $DIR2/$tdir/bash; cp /bin/bash $DIR2/$tdir' & + /bin/bash -c 'sleep 1; rm -f $DIR2/$tdir/bash; cp /bin/bash $DIR2/$tdir' & $DIR1/$tdir/bash -c 'sleep 2; openfile -f O_RDONLY /proc/$$/exe >& /dev/null; echo $?' wait @@ -1283,9 +1283,9 @@ op_trigger_cos() { # trigger CoS twice in case transaction commit before unlock for i in 1 2; do - sh -c "$1" + bash -c "$1" do_nodes $nodes "lctl set_param -n mdt.*.async_commit_count=0" - sh -c "$2" + bash -c "$2" commit_nr=$(do_nodes $nodes \ "lctl get_param -n mdt.*.async_commit_count" | calc_sum) total=$((total + commit_nr)); diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 862f305..9db7850 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -4122,7 +4122,7 @@ do_node() { eval $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests; PATH=\$PATH:/sbin:/usr/sbin; cd $RPWD; - LUSTRE=\"$RLUSTRE\" sh -c \"$@\") || + LUSTRE=\"$RLUSTRE\" bash -c \"$@\") || echo command failed >$command_status" [[ -n "$($myPDSH $HOST cat $command_status)" ]] && return 1 || return 0 @@ -4131,12 +4131,12 @@ do_node() { if $verbose ; then # print HOSTNAME for myPDSH="no_dsh" if [[ $myPDSH = no_dsh ]]; then - $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" sh -c \"$@\")" | sed -e "s/^/${HOSTNAME}: /" + $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" bash -c \"$@\")" | sed -e "s/^/${HOSTNAME}: /" else - $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" sh -c \"$@\")" + $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" bash -c \"$@\")" fi else - $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" sh -c \"$@\")" | sed "s/^${HOST}: //" + $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" bash -c \"$@\")" | sed "s/^${HOST}: //" fi return ${PIPESTATUS[0]} } @@ -4162,7 +4162,7 @@ do_node_vp() { # *Important* to set LC_CTYPE to be able to test UTF-8 symbols if [[ "$host" == "$HOSTNAME" ]]; then - sh -c "export \"LC_CTYPE=en_US.UTF-8\"; $(printf -- ' %q' "$@")" + bash -c "export \"LC_CTYPE=en_US.UTF-8\"; $(printf -- ' %q' "$@")" return $? fi @@ -4257,9 +4257,9 @@ do_nodes() { # do not replace anything from pdsh output if -N is used # -N Disable hostname: prefix on lines of output. if $verbose || [[ $myPDSH = *-N* ]]; then - $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" $(get_env_vars) sh -c \"$@\")" + $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" $(get_env_vars) bash -c \"$@\")" else - $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" $(get_env_vars) sh -c \"$@\")" | sed -re "s/^[^:]*: //g" + $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" $(get_env_vars) bash -c \"$@\")" | sed -re "s/^[^:]*: //g" fi return ${PIPESTATUS[0]} } @@ -5552,7 +5552,9 @@ is_empty_fs() { return 0 } -check_and_setup_lustre() { +do_check_and_setup_lustre() { + echo "=== $TESTSUITE: start setup $(date +'%H:%M:%S (%s)') ===" + sanitize_parameters nfs_client_mode && return cifs_client_mode && return @@ -5631,17 +5633,50 @@ check_and_setup_lustre() { if [ -n "$fs_STRIPEPARAMS" ]; then setstripe_getstripe $MOUNT $fs_STRIPEPARAMS fi + if $GSS_SK; then set_flavor_all null elif $GSS; then set_flavor_all $SEC fi - if [ "$ONLY" == "setup" ]; then + echo "=== $TESTSUITE: finish setup $(date +'%H:%M:%S (%s)') ===" + + if [[ "$ONLY" == "setup" ]]; then exit 0 fi } +check_and_setup_lustre() { + local start_stamp=$(date +%s) + local saved_umask=$(umask) + local log=$TESTLOG_PREFIX.test_setup.test_log.$(hostname -s).log + local status='PASS' + local stop_stamp=0 + local duration=0 + local error='' + local rc=0 + + umask 0022 + + log_sub_test_begin test_setup + + if ! do_check_and_setup_lustre 2>&1 > >(tee -i $log); then + error=$(tail -1 $log) + status='FAIL' + rc=1 + fi + + stop_stamp=$(date +%s) + duration=$((stop_stamp - start_stamp)) + + log_sub_test_end "$status" "$duration" "$rc" "$error" + + umask $saved_umask + + return $rc +} + restore_mount () { local clients=${CLIENTS:-$HOSTNAME} local mntpt=$1 @@ -5657,14 +5692,16 @@ cleanup_mount () { } cleanup_and_setup_lustre() { - if [ "$ONLY" == "cleanup" -o "`mount | grep $MOUNT`" ]; then - lctl set_param debug=0 || true - cleanupall - if [ "$ONLY" == "cleanup" ]; then - exit 0 - fi - fi - check_and_setup_lustre + if [[ "$ONLY" == "cleanup" ]] || grep -q "$MOUNT" /proc/mounts; then + lctl set_param debug=0 || true + cleanupall + + if [[ "$ONLY" == "cleanup" ]]; then + exit 0 + fi + fi + + do_check_and_setup_lustre } # Get all of the server target devices from a given server node and type. @@ -5851,34 +5888,68 @@ log_zfs_info() { fi } -check_and_cleanup_lustre() { - if [ "$LFSCK_ALWAYS" = "yes" -a "$TESTSUITE" != "sanity-lfsck" -a \ - "$TESTSUITE" != "sanity-scrub" ]; then +do_check_and_cleanup_lustre() { + echo "=== $TESTSUITE: start cleanup $(date +'%H:%M:%S (%s)') ===" + + if [[ "$LFSCK_ALWAYS" == "yes" && "$TESTSUITE" != "sanity-lfsck" && \ + "$TESTSUITE" != "sanity-scrub" ]]; then run_lfsck fi if is_mounted $MOUNT; then if $DO_CLEANUP; then - [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* || + [[ -n "$DIR" ]] && rm -rf $DIR/[Rdfs][0-9]* || error "remove sub-test dirs failed" else echo "skip cleanup" fi - [ "$ENABLE_QUOTA" ] && restore_quota || true + [[ -n "$ENABLE_QUOTA" ]] && restore_quota || true fi - if [ "$I_UMOUNTED2" = "yes" ]; then + if [[ "$I_UMOUNTED2" == "yes" ]]; then restore_mount $MOUNT2 || error "restore $MOUNT2 failed" fi - if [ "$I_MOUNTED2" = "yes" ]; then + if [[ "$I_MOUNTED2" == "yes" ]]; then cleanup_mount $MOUNT2 fi - if [[ "$I_MOUNTED" = "yes" ]] && ! $AUSTER_CLEANUP; then + if [[ "$I_MOUNTED" == "yes" ]] && ! $AUSTER_CLEANUP; then cleanupall -f || error "cleanup failed" unset I_MOUNTED fi + + echo "=== $TESTSUITE: finish cleanup $(date +'%H:%M:%S (%s)') ===" +} + +check_and_cleanup_lustre() { + local start_stamp=$(date +%s) + local saved_umask=$(umask) + local log=$TESTLOG_PREFIX.test_cleanup.test_log.$(hostname -s).log + local status='PASS' + local stop_stamp=0 + local duration=0 + local error='' + local rc=0 + + umask 0022 + + log_sub_test_begin test_cleanup + + if ! do_check_and_cleanup_lustre 2>&1 > >(tee -i $log); then + error=$(tail -1 $log) + status='FAIL' + rc=1 + fi + + stop_stamp=$(date +%s) + duration=$((stop_stamp - start_stamp)) + + log_sub_test_end "$status" "$duration" "$rc" "$error" + + umask $saved_umask + + return $rc } ####### -- 1.8.3.1