LU-7273 tests: dump stacks upon CT stop failure

[fs/lustre-release.git] / lustre / tests / sanity-hsm.sh

diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh
index e6a4534..c9704e7 100755 (executable)
--- a/lustre/tests/sanity-hsm.sh
+++ b/lustre/tests/sanity-hsm.sh
@@ -145,15 +145,40 @@ get_mdt_devices() {
 }
 
 search_copytools() {
-       local agents=${1:-$(facet_active_host $SINGLEAGT)}
-       do_nodesv $agents "pgrep -x $HSMTOOL_BASE"
+       local hosts=${1:-$(facet_active_host $SINGLEAGT)}
+       do_nodesv $hosts "pgrep -x $HSMTOOL_BASE"
 }
 
-search_and_kill_copytool() {
-       local agents=${1:-$(facet_active_host $SINGLEAGT)}
+kill_copytools() {
+       local hosts=${1:-$(facet_active_host $SINGLEAGT)}
 
-       echo "Killing existing copytools on $agents"
-       do_nodesv $agents "killall -q $HSMTOOL_BASE" || true
+       echo "Killing existing copytools on $hosts"
+       do_nodesv $hosts "killall -q $HSMTOOL_BASE" || true
+}
+
+wait_copytools() {
+       local hosts=${1:-$(facet_active_host $SINGLEAGT)}
+       local wait_timeout=200
+       local wait_start=$SECONDS
+       local wait_end=$((wait_start + wait_timeout))
+
+       while ((SECONDS < wait_end)); do
+               sleep 2
+               if ! search_copytools $hosts; then
+                       echo "copytools stopped in $((SECONDS - wait_start))s"
+                       return 0
+               fi
+
+               echo "copytools still running on $hosts"
+       done
+
+       # try to dump Copytool's stack
+       do_nodesv $hosts "echo 1 >/proc/sys/kernel/sysrq ; " \
+                        "echo t >/proc/sysrq-trigger"
+
+       echo "copytools failed to stop in ${wait_timeout}s"
+
+       return 1
 }
 
 copytool_monitor_setup() {
@@ -275,31 +300,14 @@ copytool_cleanup() {
        trap - EXIT
        local agt_facet=$SINGLEAGT
        local agt_hosts=${1:-$(facet_active_host $agt_facet)}
-       local end_wait=$(( SECONDS + TIMEOUT ))
        local hsm_root=$(copytool_device $agt_facet)
        local i
        local facet
        local param
        local -a state
 
-       do_nodesv $agt_hosts "pkill -INT -x $HSMTOOL_BASE" || return 0
-
-       while (( SECONDS < end_wait )); do
-               sleep 2
-               do_nodesv $agt_hosts "pgrep -x $HSMTOOL_BASE"
-               if [ $? -ne 0 ]; then
-                       echo "copytool is stopped on $agt_hosts"
-                       break
-               fi
-               echo "copytool still running on $agt_hosts"
-       done
-
-       if do_nodesv $agt_hosts "pgrep -x $HSMTOOL_BASE"; then
-               error "copytool failed to stop in ${TIMEOUT}s"
-       else
-               echo "copytool has stopped in " \
-                    "$((TIMEOUT - (end_wait - SECONDS)))s"
-       fi
+       kill_copytools $agt_hosts
+       wait_copytools $agt_hosts || error "copytools failed to stop"
 
        # Clean all CDTs orphans requests from previous tests that
        # would otherwise need to timeout to clear.
@@ -328,7 +336,6 @@ copytool_cleanup() {
                do_facet $facet "$LCTL set_param $param=${state[$i]}"
        done
 
-
        for ((i = 0; i < MDSCOUNT; i++)); do
                # Only check CDTs that we stopped in the first loop.
                [[ "${state[$i]}" =~ ^stop ]] && continue
@@ -794,7 +801,7 @@ get_mdt_devices
 init_agt_vars
 
 # cleanup from previous bad setup
-search_and_kill_copytool
+kill_copytools
 
 # for recovery tests, coordinator needs to be started at mount
 # so force it
@@ -1494,9 +1501,8 @@ test_12q() {
        $LFS hsm_release $f || error "could not release file"
        check_hsm_flags $f "0x0000000d"
 
-       search_and_kill_copytool
-       sleep 5
-       search_copytools && error "Copytool should have stopped"
+       kill_copytools
+       wait_copytools || error "copytool failed to stop"
 
        cat $f > /dev/null &
 
@@ -4613,6 +4619,66 @@ test_405() {
 }
 run_test 405 "archive and release under striped directory"
 
+test_406() {
+       [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return 0
+
+       [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.64) ] &&
+               skip "need MDS version at least 2.7.64" && return 0
+
+       local fid
+       local mdt_index
+
+       copytool_setup
+       mkdir -p $DIR/$tdir
+       fid=$(make_small $DIR/$tdir/$tfile)
+       echo "old fid $fid"
+
+       $LFS hsm_archive $DIR/$tdir/$tfile
+       wait_request_state "$fid" ARCHIVE SUCCEED
+       $LFS hsm_release $DIR/$tdir/$tfile
+
+       # Should migrate $tdir but not $tfile.
+       $LFS mv -M1 $DIR/$tdir &&
+               error "migrating HSM an archived file should fail"
+
+       $LFS hsm_restore $DIR/$tdir/$tfile
+       wait_request_state "$fid" RESTORE SUCCEED
+
+       $LFS hsm_remove $DIR/$tdir/$tfile
+       wait_request_state "$fid" REMOVE SUCCEED
+
+       cat $DIR/$tdir/$tfile > /dev/null ||
+               error "cannot read $DIR/$tdir/$tfile"
+
+       $LFS mv -M1 $DIR/$tdir ||
+               error "cannot complete migration after HSM remove"
+
+       mdt_index=$($LFS getstripe -M $DIR/$tdir)
+       if ((mdt_index != 1)); then
+               error "expected MDT index 1, got $mdt_index"
+       fi
+
+       # Refresh fid after migration.
+       fid=$(path2fid $DIR/$tdir/$tfile)
+       echo "new fid $fid"
+
+       $LFS hsm_archive $DIR/$tdir/$tfile
+       wait_request_state "$fid" ARCHIVE SUCCEED 1
+
+       lctl set_param debug=+trace
+       $LFS hsm_release $DIR/$tdir/$tfile ||
+               error "cannot release $DIR/$tdir/$tfile"
+
+       $LFS hsm_restore $DIR/$tdir/$tfile
+       wait_request_state "$fid" RESTORE SUCCEED 1
+
+       cat $DIR/$tdir/$tfile > /dev/null ||
+               error "cannot read $DIR/$tdir/$tfile"
+
+       copytool_cleanup
+}
+run_test 406 "attempting to migrate HSM archived files is safe"
+
 test_500()
 {
        [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.6.92) ] &&