LU-15132 hsm: Protect against parallel HSM restore requests

[fs/lustre-release.git] / lustre / tests / sanity-hsm.sh

diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh
index 740e6f6..b234123 100755 (executable)
--- a/lustre/tests/sanity-hsm.sh
+++ b/lustre/tests/sanity-hsm.sh
@@ -11,7 +11,7 @@ ONLY=${ONLY:-"$*"}
 
 LUSTRE=${LUSTRE:-$(dirname $0)/..}
 . $LUSTRE/tests/test-framework.sh
-init_test_env $@
+init_test_env "$@"
 init_logging
 
 ALWAYS_EXCEPT="$SANITY_HSM_EXCEPT "
@@ -84,7 +84,7 @@ CLIENT2=${CLIENT2:-$CLIENT1}
 
 search_copytools() {
        local hosts=${1:-$(facet_active_host $SINGLEAGT)}
-       do_nodesv $hosts "libtool execute pgrep -x $HSMTOOL"
+       do_nodesv $hosts "pgrep --pidfile=$HSMTOOL_PID_FILE hsmtool"
 }
 
 wait_copytools() {
@@ -147,10 +147,13 @@ fid2archive()
 {
        local fid="$1"
 
-       case "$HSMTOOL" in
-       lhsmtool_posix)
-               printf "%s" "$(hsm_root)/*/*/*/*/*/*/$fid"
-               ;;
+       case "$HSMTOOL_ARCHIVE_FORMAT" in
+               v1)
+                       printf "%s" "$(hsm_root)/*/*/*/*/*/*/$fid"
+                       ;;
+               v2)
+                       printf "%s" "$(hsm_root)/*/$fid"
+                       ;;
        esac
 }
 
@@ -168,9 +171,8 @@ get_copytool_event_log() {
 copytool_suspend() {
        local agents=${1:-$(facet_active_host $SINGLEAGT)}
 
-       stack_trap \
-               "do_nodesv $agents libtool execute pkill -CONT -x '$HSMTOOL' || true" EXIT
-       do_nodesv $agents "libtool execute pkill -STOP -x $HSMTOOL" || return 0
+       stack_trap "pkill_copytools $agents CONT || true" EXIT
+       pkill_copytools $agents STOP || return 0
        echo "Copytool is suspended on $agents"
 }
 
@@ -488,7 +490,8 @@ get_agent_uuid() {
 
        # Lustre mount-point is mandatory and last parameter on
        # copytool cmd-line.
-       local mntpnt=$(do_rpc_nodes $agent libtool execute ps -C $HSMTOOL -o args= |
+       local mntpnt=$(do_rpc_nodes $agent \
+                       pgrep --pidfile=$HSMTOOL_PID_FILE --list-full hsmtool |
                       awk '{print $NF}')
        [ -n "$mntpnt" ] || error "Found no Agent or with no mount-point "\
                                  "parameter"
@@ -555,6 +558,8 @@ test_1A() { # was test_1
 run_test 1A "lfs hsm flags root/non-root access"
 
 test_1a() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
 
@@ -599,7 +604,7 @@ test_1bde_base() {
 }
 
 test_1b() {
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        $LFS setstripe -E 1M -S 1M -E 64M -c 2 -E -1 -c 4 $DIR/$tdir ||
                error "failed to set default stripe"
        local f=$DIR/$tdir/$tfile
@@ -635,8 +640,8 @@ test_1c() {
                error "wrong archive number, $st != $LOCAL_HSM_ARCHIVE_NUMBER"
 
        LOCAL_HSM_ARCHIVE_NUMBER=33
-       if [ $(lustre_version_code client) -ge $(version_code 2.11.56) ] &&
-          [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.11.56) ]; then
+       if [ "$CLIENT_VERSION" -ge $(version_code 2.11.56) ] &&
+          [ "$MDS1_VERSION" -ge $(version_code 2.11.56) ]; then
                # lustre in the new version supports unlimited archiveID.
                # Test whether setting archive number > 32 is supported
                $LFS hsm_set --exists --archive-id $LOCAL_HSM_ARCHIVE_NUMBER $f ||
@@ -672,7 +677,7 @@ test_1d() {
        [ $MDS1_VERSION -lt $(version_code 2.10.59) ] &&
                skip "need MDS version at least 2.10.59"
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        $LFS setstripe -E 1M -L mdt -E -1 -c 2 $DIR/$tdir ||
                error "failed to set default stripe"
        local f=$DIR/$tdir/$tfile
@@ -682,10 +687,10 @@ test_1d() {
 run_test 1d "Archive, Release and Restore DoM file"
 
 test_1e() {
-       [ $(lustre_version_code $SINGLEMDS) -lt $(version_code $SEL_VER) ] &&
+       [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
                skip "skipped for lustre < $SEL_VER"
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        $LFS setstripe -E 1G -z 64M -E 10G -z 512M -E -1 -z 1G $DIR/$tdir ||
                error "failed to set default stripe"
        local comp_file=$DIR/$tdir/$tfile
@@ -723,6 +728,7 @@ run_test 1e "Archive, Release and Restore SEL file"
 test_2() {
        local f=$DIR/$tdir/$tfile
 
+       mkdir_on_mdt0 $DIR/$tdir
        create_empty_file "$f"
        # New files are not dirty
        check_hsm_flags $f "0x00000000"
@@ -749,7 +755,7 @@ test_2() {
 run_test 2 "Check file dirtyness when doing setattr"
 
 test_3() {
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        f=$DIR/$tdir/$tfile
 
        # New files are not dirty
@@ -817,7 +823,7 @@ test_8() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        $LFS hsm_archive $f
@@ -837,7 +843,8 @@ test_9A() { # was test_9
        local uuid=$(get_agent_uuid $(facet_active_host $SINGLEAGT))
        check_agent_registered $uuid
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        $LFS hsm_archive --archive $archive_id $f
@@ -859,6 +866,8 @@ test_9a() {
                copytool setup --facet agt$n
        done
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        # archive files
        for n in $(seq $AGTCOUNT); do
                file=$DIR/$tdir/$tfile.$n
@@ -875,6 +884,7 @@ test_10a() {
        # test needs a running copytool
        copytool setup
 
+       mkdir_on_mdt0 $DIR/$tdir
        mkdir -p $DIR/$tdir/d1
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
@@ -903,7 +913,7 @@ test_10b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
        $LFS hsm_archive $f || error "archive request failed"
@@ -933,7 +943,7 @@ test_10d() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
        $LFS hsm_archive $f || error "cannot archive $f"
@@ -947,7 +957,7 @@ test_10d() {
 run_test 10d "Archive a file on the default archive id"
 
 test_11a() {
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/hosts $tdir/$tfile
        local f=$DIR/$tdir/$tfile
 
@@ -976,7 +986,7 @@ test_11b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
        $LFS hsm_archive -a $HSM_ARCHIVE_NUMBER $f ||
@@ -994,11 +1004,23 @@ test_11b() {
 }
 run_test 11b "Import a deleted file using its FID"
 
+test_11c() {
+       pool_add $TESTNAME || error "Pool creation failed"
+       pool_add_targets $TESTNAME 1 1 || error "pool_add_targets failed"
+
+       mkdir -p $DIR/$tdir
+       $LFS setstripe -p "$TESTNAME" $DIR/$tdir
+
+       copy2archive /etc/hosts $tdir/$tfile
+       copytool import $tdir/$tfile $DIR/$tdir/$tfile
+}
+run_test 11c "Import a file to a directory with a pool"
+
 test_12a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/hosts $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -1024,7 +1046,7 @@ test_12b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/hosts $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -1050,7 +1072,7 @@ test_12c() {
        copytool setup
 
        local f=$DIR/$tdir/$tfile
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        $LFS setstripe -c 2 "$f"
        local fid=$(create_file "$f" 1M 5)
 
@@ -1070,7 +1092,7 @@ test_12d() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
@@ -1094,7 +1116,7 @@ test_12e() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
        $LFS hsm_archive $f || error "archive request failed"
@@ -1114,7 +1136,7 @@ test_12f() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1137,7 +1159,7 @@ test_12g() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1161,7 +1183,7 @@ test_12h() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1183,7 +1205,7 @@ test_12m() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        $LFS hsm_archive $f || error "archive of $f failed"
@@ -1218,7 +1240,7 @@ test_12o() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1267,7 +1289,7 @@ test_12p() {
        # test needs a running copytool
        copytool setup
 
-       mkdir $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1292,6 +1314,8 @@ test_12q() {
        # test needs a running copytool
        copytool setup -m "$MOUNT3"
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local f2=$DIR2/$tdir/$tfile
        local fid=$(create_small_file $f)
@@ -1340,6 +1364,57 @@ test_12q() {
 }
 run_test 12q "file attributes are refreshed after restore"
 
+test_12r() {
+       # test needs a running copytool
+       copytool setup
+
+       mkdir_on_mdt0 $DIR/$tdir
+       local f=$DIR/$tdir/$tfile
+       local fid=$(copy_file /etc/hosts $f)
+
+       $LFS hsm_archive $f || error "archive of $f failed"
+       wait_request_state $fid ARCHIVE SUCCEED
+       $LFS hsm_release $f || error "release of $f failed"
+
+       offset=$(lseek_test -d 7 $f)
+
+       # we check we had a restore done
+       wait_request_state $fid RESTORE SUCCEED
+       [[ $offset == 7 ]] || error "offset $offset != 7"
+}
+run_test 12r "lseek restores released file"
+
+test_12s() {
+       local f=$DIR/$tdir/$tfile
+       local fid
+       local pid1 pid2
+
+       (( MDS1_VERSION >= $(version_code 2.15.50) )) ||
+               skip "Need MDS version newer than 2.15.50"
+
+       # test needs a running copytool
+       copytool setup
+
+       mkdir_on_mdt0 $DIR/$tdir
+       fid=$(copy_file /etc/hosts $f)
+
+       $LFS hsm_archive $f || error "archive of $f failed"
+       wait_request_state $fid ARCHIVE SUCCEED
+       $LFS hsm_release $f || error "release of $f failed"
+
+#define OBD_FAIL_ONCE|OBD_FAIL_MDS_HSM_RESTORE_RACE 0x8000018b
+       do_facet mds1 $LCTL set_param fail_loc=0x8000018b
+       cat $f > /dev/null & pid1=$!
+       cat $f > /dev/null & pid2=$!
+
+       wait $pid1 || error "cat process 1 fail (pid: $pid1)"
+       wait $pid2 || error "cat process 2 fail (pid: $pid2)"
+
+       # Race exists if more than 1 restore requests is registered
+       assert_request_count $fid RESTORE 1
+}
+run_test 12s "race between restore requests"
+
 test_13() {
        local -i i j k=0
        for i in {1..10}; do
@@ -1376,6 +1451,8 @@ test_14() {
        # test needs a running copytool
        copytool setup
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        # archive a file
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
@@ -1404,6 +1481,8 @@ test_15() {
        # test needs a running copytool
        copytool setup
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        # archive files
        local f=$DIR/$tdir/$tfile
        local count=5
@@ -1456,7 +1535,7 @@ test_16() {
        local goal=20
        dd if=/dev/zero of=$ref bs=1M count=20
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file $ref $f)
        rm $ref
@@ -1502,6 +1581,7 @@ run_test 20 "Release is not permitted"
 test_21() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/test_release
 
@@ -1511,7 +1591,7 @@ test_21() {
 
        # LU-4388/LU-4389 - ZFS does not report full number of blocks
        # used until file is flushed to disk
-       if [  $(facet_fstype ost1) == "zfs" ]; then
+       if [ "$ost1_FSTYPE" == "zfs" ]; then
            # this causes an OST_SYNC rpc to be sent
            dd if=/dev/zero of=$f bs=512 count=1 oflag=sync conv=notrunc,fsync
            # clear locks to reread file data
@@ -1568,6 +1648,7 @@ run_test 21 "Simple release tests"
 test_22() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/test_release
        local swap=$DIR/$tdir/test_swap
@@ -1593,6 +1674,7 @@ run_test 22 "Could not swap a release file"
 test_23() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/test_mtime
 
@@ -1629,6 +1711,7 @@ test_24a() {
 
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        fid=$(create_small_file $file)
 
@@ -1731,6 +1814,7 @@ test_24b() {
 
        # Test needs a running copytool.
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        # Check that root can do HSM actions on a regular user's file.
        fid=$(create_small_file $file)
@@ -1776,8 +1860,7 @@ test_24c() {
 
        # test needs a running copytool
        copytool setup
-
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        # Save the default masks and check that cleanup_24c will
        # restore the request masks correctly.
@@ -1839,6 +1922,8 @@ test_24d() {
        local fid1
        local fid2
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        fid1=$(create_small_file $file1)
 
        echo $fid1
@@ -1882,6 +1967,7 @@ run_test 24d "check that read-only mounts are respected"
 
 test_24e() {
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid
@@ -1901,6 +1987,7 @@ run_test 24e "tar succeeds on HSM released files" # LU-6213
 test_24f() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        mkdir -p $DIR/$tdir/d1
        local f=$DIR/$tdir/$tfile
@@ -1933,7 +2020,7 @@ test_24g() {
 
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        chmod ugo+rwx $DIR/$tdir
 
        echo "Please listen carefully as our options have changed." | tee $file
@@ -1974,7 +2061,7 @@ test_25b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -1995,6 +2082,7 @@ run_test 25b "Restore lost file (HS_LOST flag) after release"\
 test_26A() { # was test_26
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2015,8 +2103,8 @@ test_26a() {
 
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
-       mkdir -p $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -2065,8 +2153,8 @@ run_test 26a "Remove Archive On Last Unlink (RAoLU) policy"
 test_26b() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
-       mkdir -p $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -2097,8 +2185,8 @@ run_test 26b "RAoLU policy when CDT off"
 test_26c() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
-       mkdir -p $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -2144,8 +2232,8 @@ run_test 26c "RAoLU effective when file closed"
 test_26d() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
-       mkdir -p $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
 
@@ -2197,6 +2285,7 @@ run_test 27a "Remove the archive of an imported file (Operation not permitted)"
 test_27b() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2214,6 +2303,7 @@ run_test 27b "Remove the archive of a relased file (Operation not permitted)"
 test_28() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2253,6 +2343,7 @@ run_test 29a "Tests --mntpath and --archive options"
 test_29b() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
@@ -2270,6 +2361,7 @@ run_test 29b "Archive/delete/remove by FID from the archive."
 test_29c() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local fid1=$(create_small_file $DIR/$tdir/$tfile-1)
        local fid2=$(create_small_file $DIR/$tdir/$tfile-2)
@@ -2307,6 +2399,8 @@ test_29d() {
                copytool setup -f agt$n -a $n
        done
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        # archive files
        file=$DIR/$tdir/$tfile
        fid=$(create_small_file $file)
@@ -2394,7 +2488,7 @@ test_30b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/true
        local fid=$(copy_file /bin/true $f)
        chmod 755 $f
@@ -2422,7 +2516,7 @@ test_30c() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/SLEEP
        local slp_sum1=$(md5sum /bin/sleep)
        local fid=$(copy_file /bin/sleep $f)
@@ -2490,6 +2584,7 @@ restore_and_check_size() {
 test_31a() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        create_archive_file $tdir/$tfile
        local f=$DIR/$tdir/$tfile
@@ -2508,6 +2603,7 @@ run_test 31a "Import a large file and check size during restore"
 test_31b() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1MB 39)
@@ -2526,6 +2622,7 @@ run_test 31b "Restore a large unaligned file and check size during restore"
 test_31c() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1M 39)
@@ -2543,6 +2640,8 @@ run_test 31c "Restore a large aligned file and check size during restore"
 
 test_33() {
        local f=$DIR/$tdir/$tfile
+
+       mkdir_on_mdt0 $DIR/$tdir
        local fid=$(create_empty_file "$f")
 
        copytool setup
@@ -2572,6 +2671,7 @@ run_test 33 "Kill a restore waiting process"
 test_34() {
        # test needs a running copytool
        copytool setup -b 1
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2606,6 +2706,7 @@ run_test 34 "Remove file during restore"
 test_35() {
        # test needs a running copytool
        copytool setup -b 1
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local f1=$DIR/$tdir/$tfile-1
@@ -2642,6 +2743,7 @@ run_test 35 "Overwrite file during restore"
 test_36() {
        # test needs a running copytool
        copytool setup -b 1
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2674,6 +2776,7 @@ run_test 36 "Move file during restore"
 test_37() {
        # LU-5683: check that an archived dirty file can be rearchived.
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid
@@ -2749,11 +2852,96 @@ test_40() {
 }
 run_test 40 "Parallel archive requests"
 
+hsm_archive_batch() {
+       local files_num=$1
+       local batch_max=$2
+       local filebase=$3
+       local batch_num=0
+       local fileset=""
+       local i=0
+
+       while [ $i -lt $files_num ]; do
+               if [ $batch_num -eq $batch_max ]; then
+                       $LFS hsm_archive $fileset || error "HSM archive failed"
+                       # Reset the batch container.
+                       fileset=""
+                       batch_num=0
+               fi
+
+               fileset+="${filebase}$i "
+               batch_num=$(( batch_num + 1 ))
+               i=$(( i + 1 ))
+       done
+
+       if [ $batch_num -ne 0 ]; then
+               $LFS hsm_archive $fileset || error "HSM archive failed"
+               fileset=""
+               batch_num=0
+       fi
+}
+
+test_50() {
+       local dir=$DIR/$tdir
+       local batch_max=50
+
+       set_hsm_param max_requests 1000000
+       mkdir $dir || error "mkdir $dir failed"
+       df -i $MOUNT
+
+       local start
+       local elapsed
+       local files_num
+       local filebase
+
+       files_num=10000
+       filebase="$dir/$tfile.start."
+       createmany -m $filebase $files_num ||
+               error "createmany -m $filebase failed: $?"
+
+       start=$SECONDS
+       hsm_archive_batch $files_num $batch_max "$filebase"
+       elapsed=$((SECONDS - start))
+       do_facet $SINGLEMDS "$LCTL get_param -n \
+                $HSM_PARAM.actions | grep WAITING | wc -l"
+       unlinkmany $filebase $files_num || error "unlinkmany $filabase failed"
+       echo "Start Phase files_num: $files_num time: $elapsed"
+
+       files_num=20000
+       filebase="$dir/$tfile.in."
+       createmany -m $filebase $files_num ||
+               error "createmany -m $filebase failed: $?"
+       start=$SECONDS
+       hsm_archive_batch  $files_num $batch_max "$filebase"
+       elapsed=$((SECONDS - start))
+       unlinkmany $filebase $files_num || error "unlinkmany $filabase failed"
+       echo "Middle Phase files_num: $files_num time: $elapsed"
+
+       files_num=10000
+       filebase="$dir/$tfile.end."
+       createmany -m $filebase $files_num ||
+               error "createmany -m $filebase failed: $?"
+
+       start=$SECONDS
+       hsm_archive_batch $files_num $batch_max "$filebase"
+       elapsed=$((SECONDS - start))
+       do_facet $SINGLEMDS "$LCTL get_param -n \
+                $HSM_PARAM.actions | grep WAITING | wc -l"
+
+       unlinkmany $filebase $files_num || error "unlinkmany $filebase failed"
+       echo "End Phase files_num: $files_num time: $elapsed"
+
+       do_facet $SINGLEMDS "$LCTL get_param -n \
+                $HSM_PARAM.actions | grep WAITING | wc -l"
+
+       cdt_purge
+}
+run_test 50 "Archive with large number of pending HSM actions"
+
 test_52() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
 
@@ -2778,7 +2966,7 @@ test_53() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
 
@@ -2801,6 +2989,8 @@ test_53() {
 run_test 53 "Opened for read file on an evicted client should not be set dirty"
 
 test_54() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1MB 39)
 
@@ -2825,6 +3015,8 @@ test_54() {
 run_test 54 "Write during an archive cancels it"
 
 test_55() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1MB 39)
 
@@ -2849,6 +3041,8 @@ test_55() {
 run_test 55 "Truncate during an archive cancels it"
 
 test_56() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1MB 39)
 
@@ -2878,7 +3072,7 @@ test_57() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/test_archive_remote
        # Create a file on a remote node
        do_node $CLIENT2 "dd if=/dev/urandom of=$f bs=1M "\
@@ -2940,7 +3134,7 @@ test_58() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local sz=$(stat -c %s /etc/passwd)
 
@@ -2974,6 +3168,8 @@ test_60() {
        # This test validates the fix for LU-4512. Ensure that the -u
        # option changes the progress reporting interval from the
        # default (30 seconds) to the user-specified interval.
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1M 10)
 
@@ -3009,7 +3205,7 @@ test_60() {
        local sleep=1
 
        echo -n "Expecting a progress update within $progress_timeout seconds... "
-       while [ true ]; do
+       while true; do
                RESULT=$(do_node $(facet_active_host $mds) "$cmd")
                if [ -n "$RESULT" ] && [ "$RESULT" -gt 0 ]; then
                        echo "$RESULT bytes copied in $WAIT seconds."
@@ -3040,7 +3236,7 @@ test_61() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        cdt_disable
@@ -3108,6 +3304,8 @@ test_71() {
        # Just start and stop the copytool to generate events.
        cdt_clear_no_retry
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file "$f")
 
@@ -3180,7 +3378,7 @@ test_72() {
                error "cannot create $test_file on $SINGLEAGT"
        copy2archive $test_file $tdir/$tfile
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        copytool import $tdir/$tfile $f
        f=$DIR2/$tdir/$tfile
@@ -3262,7 +3460,7 @@ run_test 72 "Copytool logs JSON restore events to FIFO"
 
 test_90() {
        file_count=51 # Max number of files constrained by LNET message size
-       mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+       mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
        local f=$DIR/$tdir/$tfile
        local FILELIST=/tmp/filelist.txt
        local i=""
@@ -3279,12 +3477,12 @@ test_90() {
        wait_for_grace_delay
        $LFS hsm_archive --filelist $FILELIST ||
                error "cannot archive a file list"
-       wait_all_done 100
+       wait_all_done 200
        $LFS hsm_release --filelist $FILELIST ||
                error "cannot release a file list"
        $LFS hsm_restore --filelist $FILELIST ||
                error "cannot restore a file list"
-       wait_all_done 100
+       wait_all_done 200
 }
 run_test 90 "Archive/restore a file list"
 
@@ -3349,9 +3547,71 @@ test_103() {
 }
 run_test 103 "Purge all requests"
 
+test_103a() {
+       (( MDS1_VERSION >= $(version_code 2.14.56) )) ||
+               skip "Need MDS version at least 2.14.56"
+
+       cdt_clear_non_blocking_restore
+
+       # test needs a running copytool
+       copytool setup
+
+       local -a fids=()
+       local i
+       local rpcs_inflight=$($LCTL get_param -n \
+               "mdc.$(facet_svc mds1)*.max_rpcs_in_flight" |
+               head -n1)
+
+       mkdir_on_mdt0 $DIR/$tdir
+       for ((i=0; i < rpcs_inflight; i++)); do
+               fids+=( $(copy_file /etc/passwd $DIR/$tdir/${tfile}_$i) )
+       done
+       $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $DIR/$tdir/*
+
+       local time=0
+       local cnt=0
+       local grep_regex="($(tr ' ' '|' <<< "${fids[*]}")).*action=ARCHIVE.*status=SUCCEED"
+       echo $grep_regex
+       while [[ $time -lt 5 ]] && [[ $cnt -ne ${#fids[@]} ]]; do
+               cnt=$(do_facet mds1 "$LCTL get_param -n $HSM_PARAM.actions |
+                       grep -c -E '$grep_regex'")
+               sleep 1
+               ((++time))
+       done
+       [[ $cnt -eq ${#fids[@]} ]] || error "Fail to archive files $cnt/${#fids[@]}"
+
+       $LFS hsm_release $DIR/$tdir/*
+
+       kill_copytools
+       wait_copytools || error "Copytool failed to stop"
+
+       local -a pids=()
+       for i in "${fids[@]}"; do
+               cat $DIR/.lustre/fid/$i > /dev/null & pids+=($!)
+       done
+
+       cdt_purge
+       grep_regex="($(tr ' ' '|' <<< "${fids[*]}")).*action=RESTORE.*status=CANCELED"
+       cnt=$(do_facet mds1 "$LCTL get_param -n $HSM_PARAM.actions |
+               grep -cE '$grep_regex'")
+
+       [[ "$cnt" -eq ${#fids[@]} ]] ||
+               error "Some request have not been canceled ($cnt/${#fids[@]} canceled)"
+
+       # cat cmds should not hang and should fail
+       for i in "${!pids[@]}"; do
+               wait ${pids[$i]} &&
+                       error "Restore for ${tfile}_$i (${pids[$i]}) should fail" ||
+                       true
+       done
+}
+run_test 103a "Purge pending restore requests"
+
 DATA=CEA
 DATAHEX='[434541]'
 test_104() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3369,7 +3629,7 @@ run_test 104 "Copy tool data field"
 
 test_105() {
        local max_requests=$(get_hsm_param max_requests)
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local i=""
 
        stack_trap "set_hsm_param max_requests $max_requests" EXIT
@@ -3424,7 +3684,7 @@ test_107() {
        # test needs a running copytool
        copytool setup
        # create and archive file
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f1=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f1)
        $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f1
@@ -3479,7 +3739,7 @@ test_110a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        copy2archive /etc/passwd $tdir/$tfile
 
@@ -3506,7 +3766,7 @@ test_110b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f
@@ -3532,7 +3792,7 @@ test_111a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/passwd $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -3561,7 +3821,7 @@ test_111b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        stack_trap cdt_clear_no_retry EXIT
@@ -3586,7 +3846,7 @@ test_112() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        cdt_disable
@@ -3604,6 +3864,8 @@ test_112() {
 run_test 112 "State of recorded request"
 
 test_113() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local file1=$DIR/$tdir/$tfile
        local file2=$DIR2/$tdir/$tfile
 
@@ -3640,6 +3902,8 @@ test_113() {
 run_test 113 "wrong stat after restore"
 
 test_200() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3665,6 +3929,7 @@ run_test 200 "Register/Cancel archive"
 test_201() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        create_archive_file $tdir/$tfile
@@ -3684,6 +3949,8 @@ test_201() {
 run_test 201 "Register/Cancel restore"
 
 test_202() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3707,7 +3974,7 @@ test_220A() { # was test_220
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -3726,7 +3993,7 @@ test_220a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -3755,6 +4022,8 @@ test_220a() {
 run_test 220a "Changelog for failed archive"
 
 test_221() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3781,7 +4050,7 @@ test_222a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/passwd $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -3802,7 +4071,7 @@ test_222b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -3824,7 +4093,7 @@ test_222c() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/passwd $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -3858,7 +4127,7 @@ test_222d() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -3881,6 +4150,7 @@ run_test 222d "Changelog for failed implicit restore"
 test_223a() {
        # test needs a running copytool
        copytool setup -b 1
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        create_archive_file $tdir/$tfile
@@ -3902,6 +4172,8 @@ test_223a() {
 run_test 223a "Changelog for restore canceled (import case)"
 
 test_223b() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3932,7 +4204,7 @@ test_224A() { # was test_224
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -3953,7 +4225,7 @@ test_224a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -3993,6 +4265,7 @@ test_225() {
 
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -4020,7 +4293,7 @@ test_226() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f1=$DIR/$tdir/$tfile-1
        local f2=$DIR/$tdir/$tfile-2
@@ -4095,6 +4368,7 @@ run_test 227 "changelog when explicit setting of HSM flags"
 test_228() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local fid=$(create_small_sync_file $DIR/$tfile)
        $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $DIR/$tfile
@@ -4141,6 +4415,9 @@ test_250() {
 
        # send 1 requests of each kind twice
        copytool setup
+
+       mkdir_on_mdt0 $DIR/$tdir
+
        # setup the files
        for action in archive restore remove; do
                local filepath="$file"-to-$action
@@ -4189,6 +4466,8 @@ test_250() {
 run_test 250 "Coordinator max request"
 
 test_251() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -4218,6 +4497,8 @@ test_251() {
 run_test 251 "Coordinator request timeout"
 
 test_252() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -4247,7 +4528,7 @@ test_253() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
 
        dd if=/dev/zero of=$f bs=1MB count=10
@@ -4311,6 +4592,8 @@ test_254b()
                "set_hsm_param max_requests $(get_hsm_param max_requests)" EXIT
        set_hsm_param max_requests "$request_count"
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        local timeout
        local count
        for request_type in archive restore remove; do
@@ -4371,6 +4654,8 @@ test_255()
        [ $MDS1_VERSION -lt $(version_code 2.12.0) ] &&
                skip "Need MDS version at least 2.12.0"
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        local file="$DIR/$tdir/$tfile"
        local fid=$(create_empty_file "$file")
 
@@ -4428,6 +4713,8 @@ test_260a()
        local -a files=("$DIR/$tdir/$tfile".{0..15})
        local file
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        for file in "${files[@]}"; do
                create_small_file "$file"
        done
@@ -4494,6 +4781,8 @@ test_260b()
        local -a files=("$DIR/$tdir/$tfile".{0..15})
        local file
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        for file in "${files[@]}"; do
                create_small_file "$file"
        done
@@ -4561,6 +4850,8 @@ test_260c()
        local -a files=("$DIR/$tdir/$tfile".{0..15})
        local file
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        for file in "${files[@]}"; do
                create_small_file "$file"
        done
@@ -4631,6 +4922,36 @@ test_260c()
 }
 run_test 260c "Requests are not reordered on the 'hot' path of the coordinator"
 
+test_261() {
+       local file=$DIR/$tdir/$tfile
+       local size
+       local fid
+
+       copytool setup
+       mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+
+       dd if=/dev/zero of=$file bs=4k count=2 || error "Write $file failed"
+       fid=$(path2fid $file)
+       $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $file
+       wait_request_state $fid ARCHIVE SUCCEED
+
+       $LFS hsm_state $file
+       $LFS hsm_release $file
+       $LFS hsm_restore $file
+       wait_request_state $fid RESTORE SUCCEED
+       $LFS hsm_release $file
+       size=$(stat -c %s $file)
+       [[ $size == 8192 ]] || error "Size after HSM release: $size"
+
+       $LFS hsm_release $file
+       $LFS hsm_restore $file
+       $LFS hsm_release $file
+       size=$(stat -c %s $file)
+       [[ $size == 8192 ]] || error "Size after HSM release: $size"
+       $LFS hsm_state $file
+}
+run_test 261 "Report 0 bytes size after HSM release"
+
 test_300() {
        [ "$CLIENTONLY" ] && skip "CLIENTONLY mode" && return
 
@@ -4710,7 +5031,7 @@ test_400() {
 
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local dir_mdt0=$DIR/$tdir/mdt0
        local dir_mdt1=$DIR/$tdir/mdt1
@@ -4742,7 +5063,7 @@ test_401() {
 
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local dir_mdt0=$DIR/$tdir/mdt0
        local dir_mdt1=$DIR/$tdir/mdt1
@@ -4800,7 +5121,7 @@ run_test 402a "Copytool start fails if all MDTs are inactive"
 test_402b() {
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        touch $f || error "touch $f failed"
@@ -4853,7 +5174,7 @@ test_404() {
        copytool setup
 
        # create files on both MDT0000 and MDT0001
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local dir_mdt0=$DIR/$tdir/mdt0
        stack_trap "rm -rf $dir_mdt0" EXIT
@@ -4882,7 +5203,7 @@ test_405() {
 
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local striped_dir=$DIR/$tdir/striped_dir
 
@@ -4934,6 +5255,8 @@ test_406() {
        local fid
        local mdt_index
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        fid=$(create_small_file $DIR/$tdir/$tfile)
        echo "old fid $fid"
 
@@ -4984,6 +5307,8 @@ test_406() {
 run_test 406 "attempting to migrate HSM archived files is safe"
 
 test_407() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local f2=$DIR2/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -5006,6 +5331,7 @@ test_407() {
        md5sum $f2 &
        sleep 2
 
+       do_facet $SINGLEMDS "$LCTL get_param $HSM_PARAM.actions"
        # after umount hsm_actions->O/x/x log shouldn't have
        # double RESTORE records like below
        #[0x200000401:0x1:0x0]...0x58d03a0d/0x58d03a0c action=RESTORE...WAITING
@@ -5013,21 +5339,24 @@ test_407() {
        sleep 30 &&
                do_facet $SINGLEMDS "$LCTL get_param $HSM_PARAM.actions"&
        fail $SINGLEMDS
+       do_facet $SINGLEMDS $LCTL set_param fail_loc=0
+
+       do_facet $SINGLEMDS "$LCTL get_param $HSM_PARAM.actions"
 
        copytool_continue
-       wait_request_state $fid RESTORE SUCCEED
+       wait_all_done 100 $fid
 }
 run_test 407 "Check for double RESTORE records in llog"
 
 test_500()
 {
-       [ $MDS1_VERSION -lt $(version_code 2.6.92) ] &&
+       [ "$MDS1_VERSION" -lt $(version_code 2.6.92) ] &&
                skip "HSM migrate is not supported"
 
        test_mkdir -p $DIR/$tdir
 
-       if [ $(lustre_version_code client) -lt $(version_code 2.11.56) ] ||
-            [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.11.56) ];
+       if [ "$CLIENT_VERSION" -lt $(version_code 2.11.56) ] ||
+            [ "$MDS1_VERSION" -lt $(version_code 2.11.56) ];
        then
                llapi_hsm_test -d $DIR/$tdir -b ||
                        error "One llapi HSM test failed"
@@ -5039,7 +5368,7 @@ test_500()
 run_test 500 "various LLAPI HSM tests"
 
 test_600() {
-       [ $MDS1_VERSION -lt $(version_code 2.10.58) ] &&
+       [ "$MDS1_VERSION" -lt $(version_code 2.10.58) ] &&
                skip "need MDS version at least 2.10.58"
 
        mkdir -p $DIR/$tdir
@@ -5098,6 +5427,9 @@ test_602() {
        [ $MDS1_VERSION -lt $(version_code 2.10.58) ] &&
                skip "need MDS version at least 2.10.58"
 
+       stack_trap "restore_opencache" EXIT
+       disable_opencache
+
        mkdir -p $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
@@ -5170,7 +5502,7 @@ test_604() {
        [ $MDS1_VERSION -lt $(version_code 2.10.58) ] &&
                skip "need MDS version at least 2.10.58"
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local f2=$DIR2/$tdir/$tfile
@@ -5246,6 +5578,9 @@ test_605() {
        [ $MDS1_VERSION -lt $(version_code 2.10.58) ] &&
                skip "need MDS version at least 2.10.58"
 
+       stack_trap "restore_opencache" EXIT
+       disable_opencache
+
        mkdir -p $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
@@ -5331,10 +5666,9 @@ test_606() {
        local llog_reader=$(do_facet mgs "which llog_reader 2> /dev/null")
        llog_reader=${llog_reader:-$LUSTRE/utils/llog_reader}
        [ -z $(do_facet mgs ls -d $llog_reader 2> /dev/null) ] &&
-                       skip_env "missing llog_reader" && return
-       local fstype=$(facet_fstype mds1)
+                       skip_env "missing llog_reader"
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
 
@@ -5352,8 +5686,9 @@ test_606() {
        local entry
 
        #remount mds1 as ldiskfs or zfs type
-       stack_trap "stop mds1; start mds1 $(mdsdevname 1) $MDS_MOUNT_OPTS" EXIT
        stop mds1 || error "stop mds1 failed"
+       stack_trap "unmount_fstype mds1; start mds1 $(mdsdevname 1)\
+               $MDS_MOUNT_OPTS" EXIT
        mount_fstype mds1 || error "remount mds1 failed"
 
        for ((i = 0; i < 1; i++)); do
@@ -5379,7 +5714,7 @@ test_606() {
        [ -n "$nid" ] || error "nid is empty"
        echo "Got NID $nid"
        [ -n "$nid" ] && [[ "${CLIENT_NIDS[*]}" =~ $nid ]] ||
-               error "nid '$nid' does not match any NID ${CLIENT_NIDS[@]}"
+               error "nid '$nid' does not match any NID ${CLIENT_NIDS[*]}"
 }
 run_test 606 "llog_reader groks changelog fields"