LU-15132 hsm: Protect against parallel HSM restore requests

[fs/lustre-release.git] / lustre / tests / sanity-hsm.sh

diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh
index be7276b..b234123 100755 (executable)
--- a/lustre/tests/sanity-hsm.sh
+++ b/lustre/tests/sanity-hsm.sh
@@ -11,7 +11,7 @@ ONLY=${ONLY:-"$*"}
 
 LUSTRE=${LUSTRE:-$(dirname $0)/..}
 . $LUSTRE/tests/test-framework.sh
-init_test_env $@
+init_test_env "$@"
 init_logging
 
 ALWAYS_EXCEPT="$SANITY_HSM_EXCEPT "
@@ -147,10 +147,13 @@ fid2archive()
 {
        local fid="$1"
 
-       case "$HSMTOOL" in
-       *lhsmtool_posix)
-               printf "%s" "$(hsm_root)/*/*/*/*/*/*/$fid"
-               ;;
+       case "$HSMTOOL_ARCHIVE_FORMAT" in
+               v1)
+                       printf "%s" "$(hsm_root)/*/*/*/*/*/*/$fid"
+                       ;;
+               v2)
+                       printf "%s" "$(hsm_root)/*/$fid"
+                       ;;
        esac
 }
 
@@ -555,6 +558,8 @@ test_1A() { # was test_1
 run_test 1A "lfs hsm flags root/non-root access"
 
 test_1a() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
 
@@ -599,7 +604,7 @@ test_1bde_base() {
 }
 
 test_1b() {
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        $LFS setstripe -E 1M -S 1M -E 64M -c 2 -E -1 -c 4 $DIR/$tdir ||
                error "failed to set default stripe"
        local f=$DIR/$tdir/$tfile
@@ -672,7 +677,7 @@ test_1d() {
        [ $MDS1_VERSION -lt $(version_code 2.10.59) ] &&
                skip "need MDS version at least 2.10.59"
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        $LFS setstripe -E 1M -L mdt -E -1 -c 2 $DIR/$tdir ||
                error "failed to set default stripe"
        local f=$DIR/$tdir/$tfile
@@ -685,7 +690,7 @@ test_1e() {
        [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
                skip "skipped for lustre < $SEL_VER"
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        $LFS setstripe -E 1G -z 64M -E 10G -z 512M -E -1 -z 1G $DIR/$tdir ||
                error "failed to set default stripe"
        local comp_file=$DIR/$tdir/$tfile
@@ -723,6 +728,7 @@ run_test 1e "Archive, Release and Restore SEL file"
 test_2() {
        local f=$DIR/$tdir/$tfile
 
+       mkdir_on_mdt0 $DIR/$tdir
        create_empty_file "$f"
        # New files are not dirty
        check_hsm_flags $f "0x00000000"
@@ -749,7 +755,7 @@ test_2() {
 run_test 2 "Check file dirtyness when doing setattr"
 
 test_3() {
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        f=$DIR/$tdir/$tfile
 
        # New files are not dirty
@@ -817,7 +823,7 @@ test_8() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        $LFS hsm_archive $f
@@ -837,7 +843,8 @@ test_9A() { # was test_9
        local uuid=$(get_agent_uuid $(facet_active_host $SINGLEAGT))
        check_agent_registered $uuid
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        $LFS hsm_archive --archive $archive_id $f
@@ -859,6 +866,8 @@ test_9a() {
                copytool setup --facet agt$n
        done
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        # archive files
        for n in $(seq $AGTCOUNT); do
                file=$DIR/$tdir/$tfile.$n
@@ -875,6 +884,7 @@ test_10a() {
        # test needs a running copytool
        copytool setup
 
+       mkdir_on_mdt0 $DIR/$tdir
        mkdir -p $DIR/$tdir/d1
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
@@ -903,7 +913,7 @@ test_10b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
        $LFS hsm_archive $f || error "archive request failed"
@@ -933,7 +943,7 @@ test_10d() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
        $LFS hsm_archive $f || error "cannot archive $f"
@@ -947,7 +957,7 @@ test_10d() {
 run_test 10d "Archive a file on the default archive id"
 
 test_11a() {
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/hosts $tdir/$tfile
        local f=$DIR/$tdir/$tfile
 
@@ -976,7 +986,7 @@ test_11b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
        $LFS hsm_archive -a $HSM_ARCHIVE_NUMBER $f ||
@@ -1010,7 +1020,7 @@ test_12a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/hosts $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -1036,7 +1046,7 @@ test_12b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/hosts $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -1062,7 +1072,7 @@ test_12c() {
        copytool setup
 
        local f=$DIR/$tdir/$tfile
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        $LFS setstripe -c 2 "$f"
        local fid=$(create_file "$f" 1M 5)
 
@@ -1082,7 +1092,7 @@ test_12d() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
@@ -1106,7 +1116,7 @@ test_12e() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
        $LFS hsm_archive $f || error "archive request failed"
@@ -1126,7 +1136,7 @@ test_12f() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1149,7 +1159,7 @@ test_12g() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1173,7 +1183,7 @@ test_12h() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1195,7 +1205,7 @@ test_12m() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        $LFS hsm_archive $f || error "archive of $f failed"
@@ -1230,7 +1240,7 @@ test_12o() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1279,7 +1289,7 @@ test_12p() {
        # test needs a running copytool
        copytool setup
 
-       mkdir $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1304,6 +1314,8 @@ test_12q() {
        # test needs a running copytool
        copytool setup -m "$MOUNT3"
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local f2=$DIR2/$tdir/$tfile
        local fid=$(create_small_file $f)
@@ -1356,7 +1368,7 @@ test_12r() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1372,6 +1384,37 @@ test_12r() {
 }
 run_test 12r "lseek restores released file"
 
+test_12s() {
+       local f=$DIR/$tdir/$tfile
+       local fid
+       local pid1 pid2
+
+       (( MDS1_VERSION >= $(version_code 2.15.50) )) ||
+               skip "Need MDS version newer than 2.15.50"
+
+       # test needs a running copytool
+       copytool setup
+
+       mkdir_on_mdt0 $DIR/$tdir
+       fid=$(copy_file /etc/hosts $f)
+
+       $LFS hsm_archive $f || error "archive of $f failed"
+       wait_request_state $fid ARCHIVE SUCCEED
+       $LFS hsm_release $f || error "release of $f failed"
+
+#define OBD_FAIL_ONCE|OBD_FAIL_MDS_HSM_RESTORE_RACE 0x8000018b
+       do_facet mds1 $LCTL set_param fail_loc=0x8000018b
+       cat $f > /dev/null & pid1=$!
+       cat $f > /dev/null & pid2=$!
+
+       wait $pid1 || error "cat process 1 fail (pid: $pid1)"
+       wait $pid2 || error "cat process 2 fail (pid: $pid2)"
+
+       # Race exists if more than 1 restore requests is registered
+       assert_request_count $fid RESTORE 1
+}
+run_test 12s "race between restore requests"
+
 test_13() {
        local -i i j k=0
        for i in {1..10}; do
@@ -1408,6 +1451,8 @@ test_14() {
        # test needs a running copytool
        copytool setup
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        # archive a file
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
@@ -1436,6 +1481,8 @@ test_15() {
        # test needs a running copytool
        copytool setup
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        # archive files
        local f=$DIR/$tdir/$tfile
        local count=5
@@ -1488,7 +1535,7 @@ test_16() {
        local goal=20
        dd if=/dev/zero of=$ref bs=1M count=20
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file $ref $f)
        rm $ref
@@ -1534,6 +1581,7 @@ run_test 20 "Release is not permitted"
 test_21() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/test_release
 
@@ -1600,6 +1648,7 @@ run_test 21 "Simple release tests"
 test_22() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/test_release
        local swap=$DIR/$tdir/test_swap
@@ -1625,6 +1674,7 @@ run_test 22 "Could not swap a release file"
 test_23() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/test_mtime
 
@@ -1661,6 +1711,7 @@ test_24a() {
 
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        fid=$(create_small_file $file)
 
@@ -1763,6 +1814,7 @@ test_24b() {
 
        # Test needs a running copytool.
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        # Check that root can do HSM actions on a regular user's file.
        fid=$(create_small_file $file)
@@ -1808,8 +1860,7 @@ test_24c() {
 
        # test needs a running copytool
        copytool setup
-
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        # Save the default masks and check that cleanup_24c will
        # restore the request masks correctly.
@@ -1871,6 +1922,8 @@ test_24d() {
        local fid1
        local fid2
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        fid1=$(create_small_file $file1)
 
        echo $fid1
@@ -1914,6 +1967,7 @@ run_test 24d "check that read-only mounts are respected"
 
 test_24e() {
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid
@@ -1933,6 +1987,7 @@ run_test 24e "tar succeeds on HSM released files" # LU-6213
 test_24f() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        mkdir -p $DIR/$tdir/d1
        local f=$DIR/$tdir/$tfile
@@ -1965,7 +2020,7 @@ test_24g() {
 
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        chmod ugo+rwx $DIR/$tdir
 
        echo "Please listen carefully as our options have changed." | tee $file
@@ -2006,7 +2061,7 @@ test_25b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -2027,6 +2082,7 @@ run_test 25b "Restore lost file (HS_LOST flag) after release"\
 test_26A() { # was test_26
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2047,8 +2103,8 @@ test_26a() {
 
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
-       mkdir -p $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -2097,8 +2153,8 @@ run_test 26a "Remove Archive On Last Unlink (RAoLU) policy"
 test_26b() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
-       mkdir -p $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -2129,8 +2185,8 @@ run_test 26b "RAoLU policy when CDT off"
 test_26c() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
-       mkdir -p $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -2176,8 +2232,8 @@ run_test 26c "RAoLU effective when file closed"
 test_26d() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
-       mkdir -p $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
 
@@ -2229,6 +2285,7 @@ run_test 27a "Remove the archive of an imported file (Operation not permitted)"
 test_27b() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2246,6 +2303,7 @@ run_test 27b "Remove the archive of a relased file (Operation not permitted)"
 test_28() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2285,6 +2343,7 @@ run_test 29a "Tests --mntpath and --archive options"
 test_29b() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
@@ -2302,6 +2361,7 @@ run_test 29b "Archive/delete/remove by FID from the archive."
 test_29c() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local fid1=$(create_small_file $DIR/$tdir/$tfile-1)
        local fid2=$(create_small_file $DIR/$tdir/$tfile-2)
@@ -2339,6 +2399,8 @@ test_29d() {
                copytool setup -f agt$n -a $n
        done
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        # archive files
        file=$DIR/$tdir/$tfile
        fid=$(create_small_file $file)
@@ -2426,7 +2488,7 @@ test_30b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/true
        local fid=$(copy_file /bin/true $f)
        chmod 755 $f
@@ -2454,7 +2516,7 @@ test_30c() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/SLEEP
        local slp_sum1=$(md5sum /bin/sleep)
        local fid=$(copy_file /bin/sleep $f)
@@ -2522,6 +2584,7 @@ restore_and_check_size() {
 test_31a() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        create_archive_file $tdir/$tfile
        local f=$DIR/$tdir/$tfile
@@ -2540,6 +2603,7 @@ run_test 31a "Import a large file and check size during restore"
 test_31b() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1MB 39)
@@ -2558,6 +2622,7 @@ run_test 31b "Restore a large unaligned file and check size during restore"
 test_31c() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1M 39)
@@ -2575,6 +2640,8 @@ run_test 31c "Restore a large aligned file and check size during restore"
 
 test_33() {
        local f=$DIR/$tdir/$tfile
+
+       mkdir_on_mdt0 $DIR/$tdir
        local fid=$(create_empty_file "$f")
 
        copytool setup
@@ -2604,6 +2671,7 @@ run_test 33 "Kill a restore waiting process"
 test_34() {
        # test needs a running copytool
        copytool setup -b 1
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2638,6 +2706,7 @@ run_test 34 "Remove file during restore"
 test_35() {
        # test needs a running copytool
        copytool setup -b 1
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local f1=$DIR/$tdir/$tfile-1
@@ -2674,6 +2743,7 @@ run_test 35 "Overwrite file during restore"
 test_36() {
        # test needs a running copytool
        copytool setup -b 1
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2706,6 +2776,7 @@ run_test 36 "Move file during restore"
 test_37() {
        # LU-5683: check that an archived dirty file can be rearchived.
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid
@@ -2781,11 +2852,96 @@ test_40() {
 }
 run_test 40 "Parallel archive requests"
 
+hsm_archive_batch() {
+       local files_num=$1
+       local batch_max=$2
+       local filebase=$3
+       local batch_num=0
+       local fileset=""
+       local i=0
+
+       while [ $i -lt $files_num ]; do
+               if [ $batch_num -eq $batch_max ]; then
+                       $LFS hsm_archive $fileset || error "HSM archive failed"
+                       # Reset the batch container.
+                       fileset=""
+                       batch_num=0
+               fi
+
+               fileset+="${filebase}$i "
+               batch_num=$(( batch_num + 1 ))
+               i=$(( i + 1 ))
+       done
+
+       if [ $batch_num -ne 0 ]; then
+               $LFS hsm_archive $fileset || error "HSM archive failed"
+               fileset=""
+               batch_num=0
+       fi
+}
+
+test_50() {
+       local dir=$DIR/$tdir
+       local batch_max=50
+
+       set_hsm_param max_requests 1000000
+       mkdir $dir || error "mkdir $dir failed"
+       df -i $MOUNT
+
+       local start
+       local elapsed
+       local files_num
+       local filebase
+
+       files_num=10000
+       filebase="$dir/$tfile.start."
+       createmany -m $filebase $files_num ||
+               error "createmany -m $filebase failed: $?"
+
+       start=$SECONDS
+       hsm_archive_batch $files_num $batch_max "$filebase"
+       elapsed=$((SECONDS - start))
+       do_facet $SINGLEMDS "$LCTL get_param -n \
+                $HSM_PARAM.actions | grep WAITING | wc -l"
+       unlinkmany $filebase $files_num || error "unlinkmany $filabase failed"
+       echo "Start Phase files_num: $files_num time: $elapsed"
+
+       files_num=20000
+       filebase="$dir/$tfile.in."
+       createmany -m $filebase $files_num ||
+               error "createmany -m $filebase failed: $?"
+       start=$SECONDS
+       hsm_archive_batch  $files_num $batch_max "$filebase"
+       elapsed=$((SECONDS - start))
+       unlinkmany $filebase $files_num || error "unlinkmany $filabase failed"
+       echo "Middle Phase files_num: $files_num time: $elapsed"
+
+       files_num=10000
+       filebase="$dir/$tfile.end."
+       createmany -m $filebase $files_num ||
+               error "createmany -m $filebase failed: $?"
+
+       start=$SECONDS
+       hsm_archive_batch $files_num $batch_max "$filebase"
+       elapsed=$((SECONDS - start))
+       do_facet $SINGLEMDS "$LCTL get_param -n \
+                $HSM_PARAM.actions | grep WAITING | wc -l"
+
+       unlinkmany $filebase $files_num || error "unlinkmany $filebase failed"
+       echo "End Phase files_num: $files_num time: $elapsed"
+
+       do_facet $SINGLEMDS "$LCTL get_param -n \
+                $HSM_PARAM.actions | grep WAITING | wc -l"
+
+       cdt_purge
+}
+run_test 50 "Archive with large number of pending HSM actions"
+
 test_52() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
 
@@ -2810,7 +2966,7 @@ test_53() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
 
@@ -2833,6 +2989,8 @@ test_53() {
 run_test 53 "Opened for read file on an evicted client should not be set dirty"
 
 test_54() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1MB 39)
 
@@ -2857,6 +3015,8 @@ test_54() {
 run_test 54 "Write during an archive cancels it"
 
 test_55() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1MB 39)
 
@@ -2881,6 +3041,8 @@ test_55() {
 run_test 55 "Truncate during an archive cancels it"
 
 test_56() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1MB 39)
 
@@ -2910,7 +3072,7 @@ test_57() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/test_archive_remote
        # Create a file on a remote node
        do_node $CLIENT2 "dd if=/dev/urandom of=$f bs=1M "\
@@ -2972,7 +3134,7 @@ test_58() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local sz=$(stat -c %s /etc/passwd)
 
@@ -3006,6 +3168,8 @@ test_60() {
        # This test validates the fix for LU-4512. Ensure that the -u
        # option changes the progress reporting interval from the
        # default (30 seconds) to the user-specified interval.
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1M 10)
 
@@ -3041,7 +3205,7 @@ test_60() {
        local sleep=1
 
        echo -n "Expecting a progress update within $progress_timeout seconds... "
-       while [ true ]; do
+       while true; do
                RESULT=$(do_node $(facet_active_host $mds) "$cmd")
                if [ -n "$RESULT" ] && [ "$RESULT" -gt 0 ]; then
                        echo "$RESULT bytes copied in $WAIT seconds."
@@ -3072,7 +3236,7 @@ test_61() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        cdt_disable
@@ -3140,6 +3304,8 @@ test_71() {
        # Just start and stop the copytool to generate events.
        cdt_clear_no_retry
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file "$f")
 
@@ -3212,7 +3378,7 @@ test_72() {
                error "cannot create $test_file on $SINGLEAGT"
        copy2archive $test_file $tdir/$tfile
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        copytool import $tdir/$tfile $f
        f=$DIR2/$tdir/$tfile
@@ -3294,7 +3460,7 @@ run_test 72 "Copytool logs JSON restore events to FIFO"
 
 test_90() {
        file_count=51 # Max number of files constrained by LNET message size
-       mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+       mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
        local f=$DIR/$tdir/$tfile
        local FILELIST=/tmp/filelist.txt
        local i=""
@@ -3381,9 +3547,71 @@ test_103() {
 }
 run_test 103 "Purge all requests"
 
+test_103a() {
+       (( MDS1_VERSION >= $(version_code 2.14.56) )) ||
+               skip "Need MDS version at least 2.14.56"
+
+       cdt_clear_non_blocking_restore
+
+       # test needs a running copytool
+       copytool setup
+
+       local -a fids=()
+       local i
+       local rpcs_inflight=$($LCTL get_param -n \
+               "mdc.$(facet_svc mds1)*.max_rpcs_in_flight" |
+               head -n1)
+
+       mkdir_on_mdt0 $DIR/$tdir
+       for ((i=0; i < rpcs_inflight; i++)); do
+               fids+=( $(copy_file /etc/passwd $DIR/$tdir/${tfile}_$i) )
+       done
+       $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $DIR/$tdir/*
+
+       local time=0
+       local cnt=0
+       local grep_regex="($(tr ' ' '|' <<< "${fids[*]}")).*action=ARCHIVE.*status=SUCCEED"
+       echo $grep_regex
+       while [[ $time -lt 5 ]] && [[ $cnt -ne ${#fids[@]} ]]; do
+               cnt=$(do_facet mds1 "$LCTL get_param -n $HSM_PARAM.actions |
+                       grep -c -E '$grep_regex'")
+               sleep 1
+               ((++time))
+       done
+       [[ $cnt -eq ${#fids[@]} ]] || error "Fail to archive files $cnt/${#fids[@]}"
+
+       $LFS hsm_release $DIR/$tdir/*
+
+       kill_copytools
+       wait_copytools || error "Copytool failed to stop"
+
+       local -a pids=()
+       for i in "${fids[@]}"; do
+               cat $DIR/.lustre/fid/$i > /dev/null & pids+=($!)
+       done
+
+       cdt_purge
+       grep_regex="($(tr ' ' '|' <<< "${fids[*]}")).*action=RESTORE.*status=CANCELED"
+       cnt=$(do_facet mds1 "$LCTL get_param -n $HSM_PARAM.actions |
+               grep -cE '$grep_regex'")
+
+       [[ "$cnt" -eq ${#fids[@]} ]] ||
+               error "Some request have not been canceled ($cnt/${#fids[@]} canceled)"
+
+       # cat cmds should not hang and should fail
+       for i in "${!pids[@]}"; do
+               wait ${pids[$i]} &&
+                       error "Restore for ${tfile}_$i (${pids[$i]}) should fail" ||
+                       true
+       done
+}
+run_test 103a "Purge pending restore requests"
+
 DATA=CEA
 DATAHEX='[434541]'
 test_104() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3401,7 +3629,7 @@ run_test 104 "Copy tool data field"
 
 test_105() {
        local max_requests=$(get_hsm_param max_requests)
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local i=""
 
        stack_trap "set_hsm_param max_requests $max_requests" EXIT
@@ -3456,7 +3684,7 @@ test_107() {
        # test needs a running copytool
        copytool setup
        # create and archive file
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f1=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f1)
        $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f1
@@ -3511,7 +3739,7 @@ test_110a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        copy2archive /etc/passwd $tdir/$tfile
 
@@ -3538,7 +3766,7 @@ test_110b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f
@@ -3564,7 +3792,7 @@ test_111a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/passwd $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -3593,7 +3821,7 @@ test_111b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        stack_trap cdt_clear_no_retry EXIT
@@ -3618,7 +3846,7 @@ test_112() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        cdt_disable
@@ -3636,6 +3864,8 @@ test_112() {
 run_test 112 "State of recorded request"
 
 test_113() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local file1=$DIR/$tdir/$tfile
        local file2=$DIR2/$tdir/$tfile
 
@@ -3672,6 +3902,8 @@ test_113() {
 run_test 113 "wrong stat after restore"
 
 test_200() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3697,6 +3929,7 @@ run_test 200 "Register/Cancel archive"
 test_201() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        create_archive_file $tdir/$tfile
@@ -3716,6 +3949,8 @@ test_201() {
 run_test 201 "Register/Cancel restore"
 
 test_202() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3739,7 +3974,7 @@ test_220A() { # was test_220
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -3758,7 +3993,7 @@ test_220a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -3787,6 +4022,8 @@ test_220a() {
 run_test 220a "Changelog for failed archive"
 
 test_221() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3813,7 +4050,7 @@ test_222a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/passwd $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -3834,7 +4071,7 @@ test_222b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -3856,7 +4093,7 @@ test_222c() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/passwd $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -3890,7 +4127,7 @@ test_222d() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -3913,6 +4150,7 @@ run_test 222d "Changelog for failed implicit restore"
 test_223a() {
        # test needs a running copytool
        copytool setup -b 1
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        create_archive_file $tdir/$tfile
@@ -3934,6 +4172,8 @@ test_223a() {
 run_test 223a "Changelog for restore canceled (import case)"
 
 test_223b() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3964,7 +4204,7 @@ test_224A() { # was test_224
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -3985,7 +4225,7 @@ test_224a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -4025,6 +4265,7 @@ test_225() {
 
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -4052,7 +4293,7 @@ test_226() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f1=$DIR/$tdir/$tfile-1
        local f2=$DIR/$tdir/$tfile-2
@@ -4127,6 +4368,7 @@ run_test 227 "changelog when explicit setting of HSM flags"
 test_228() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local fid=$(create_small_sync_file $DIR/$tfile)
        $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $DIR/$tfile
@@ -4173,6 +4415,9 @@ test_250() {
 
        # send 1 requests of each kind twice
        copytool setup
+
+       mkdir_on_mdt0 $DIR/$tdir
+
        # setup the files
        for action in archive restore remove; do
                local filepath="$file"-to-$action
@@ -4221,6 +4466,8 @@ test_250() {
 run_test 250 "Coordinator max request"
 
 test_251() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -4250,6 +4497,8 @@ test_251() {
 run_test 251 "Coordinator request timeout"
 
 test_252() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -4279,7 +4528,7 @@ test_253() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
 
        dd if=/dev/zero of=$f bs=1MB count=10
@@ -4343,6 +4592,8 @@ test_254b()
                "set_hsm_param max_requests $(get_hsm_param max_requests)" EXIT
        set_hsm_param max_requests "$request_count"
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        local timeout
        local count
        for request_type in archive restore remove; do
@@ -4403,6 +4654,8 @@ test_255()
        [ $MDS1_VERSION -lt $(version_code 2.12.0) ] &&
                skip "Need MDS version at least 2.12.0"
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        local file="$DIR/$tdir/$tfile"
        local fid=$(create_empty_file "$file")
 
@@ -4460,6 +4713,8 @@ test_260a()
        local -a files=("$DIR/$tdir/$tfile".{0..15})
        local file
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        for file in "${files[@]}"; do
                create_small_file "$file"
        done
@@ -4526,6 +4781,8 @@ test_260b()
        local -a files=("$DIR/$tdir/$tfile".{0..15})
        local file
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        for file in "${files[@]}"; do
                create_small_file "$file"
        done
@@ -4593,6 +4850,8 @@ test_260c()
        local -a files=("$DIR/$tdir/$tfile".{0..15})
        local file
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        for file in "${files[@]}"; do
                create_small_file "$file"
        done
@@ -4663,6 +4922,36 @@ test_260c()
 }
 run_test 260c "Requests are not reordered on the 'hot' path of the coordinator"
 
+test_261() {
+       local file=$DIR/$tdir/$tfile
+       local size
+       local fid
+
+       copytool setup
+       mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+
+       dd if=/dev/zero of=$file bs=4k count=2 || error "Write $file failed"
+       fid=$(path2fid $file)
+       $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $file
+       wait_request_state $fid ARCHIVE SUCCEED
+
+       $LFS hsm_state $file
+       $LFS hsm_release $file
+       $LFS hsm_restore $file
+       wait_request_state $fid RESTORE SUCCEED
+       $LFS hsm_release $file
+       size=$(stat -c %s $file)
+       [[ $size == 8192 ]] || error "Size after HSM release: $size"
+
+       $LFS hsm_release $file
+       $LFS hsm_restore $file
+       $LFS hsm_release $file
+       size=$(stat -c %s $file)
+       [[ $size == 8192 ]] || error "Size after HSM release: $size"
+       $LFS hsm_state $file
+}
+run_test 261 "Report 0 bytes size after HSM release"
+
 test_300() {
        [ "$CLIENTONLY" ] && skip "CLIENTONLY mode" && return
 
@@ -4742,7 +5031,7 @@ test_400() {
 
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local dir_mdt0=$DIR/$tdir/mdt0
        local dir_mdt1=$DIR/$tdir/mdt1
@@ -4774,7 +5063,7 @@ test_401() {
 
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local dir_mdt0=$DIR/$tdir/mdt0
        local dir_mdt1=$DIR/$tdir/mdt1
@@ -4832,7 +5121,7 @@ run_test 402a "Copytool start fails if all MDTs are inactive"
 test_402b() {
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        touch $f || error "touch $f failed"
@@ -4885,7 +5174,7 @@ test_404() {
        copytool setup
 
        # create files on both MDT0000 and MDT0001
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local dir_mdt0=$DIR/$tdir/mdt0
        stack_trap "rm -rf $dir_mdt0" EXIT
@@ -4914,7 +5203,7 @@ test_405() {
 
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local striped_dir=$DIR/$tdir/striped_dir
 
@@ -4966,6 +5255,8 @@ test_406() {
        local fid
        local mdt_index
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        fid=$(create_small_file $DIR/$tdir/$tfile)
        echo "old fid $fid"
 
@@ -5016,6 +5307,8 @@ test_406() {
 run_test 406 "attempting to migrate HSM archived files is safe"
 
 test_407() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local f2=$DIR2/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -5134,6 +5427,9 @@ test_602() {
        [ $MDS1_VERSION -lt $(version_code 2.10.58) ] &&
                skip "need MDS version at least 2.10.58"
 
+       stack_trap "restore_opencache" EXIT
+       disable_opencache
+
        mkdir -p $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
@@ -5206,7 +5502,7 @@ test_604() {
        [ $MDS1_VERSION -lt $(version_code 2.10.58) ] &&
                skip "need MDS version at least 2.10.58"
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local f2=$DIR2/$tdir/$tfile
@@ -5282,6 +5578,9 @@ test_605() {
        [ $MDS1_VERSION -lt $(version_code 2.10.58) ] &&
                skip "need MDS version at least 2.10.58"
 
+       stack_trap "restore_opencache" EXIT
+       disable_opencache
+
        mkdir -p $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
@@ -5369,7 +5668,7 @@ test_606() {
        [ -z $(do_facet mgs ls -d $llog_reader 2> /dev/null) ] &&
                        skip_env "missing llog_reader"
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
 
@@ -5387,8 +5686,9 @@ test_606() {
        local entry
 
        #remount mds1 as ldiskfs or zfs type
-       stack_trap "stop mds1; start mds1 $(mdsdevname 1) $MDS_MOUNT_OPTS" EXIT
        stop mds1 || error "stop mds1 failed"
+       stack_trap "unmount_fstype mds1; start mds1 $(mdsdevname 1)\
+               $MDS_MOUNT_OPTS" EXIT
        mount_fstype mds1 || error "remount mds1 failed"
 
        for ((i = 0; i < 1; i++)); do
@@ -5414,7 +5714,7 @@ test_606() {
        [ -n "$nid" ] || error "nid is empty"
        echo "Got NID $nid"
        [ -n "$nid" ] && [[ "${CLIENT_NIDS[*]}" =~ $nid ]] ||
-               error "nid '$nid' does not match any NID ${CLIENT_NIDS[@]}"
+               error "nid '$nid' does not match any NID ${CLIENT_NIDS[*]}"
 }
 run_test 606 "llog_reader groks changelog fields"