LU-14793 hsm: record index for further HSM action scanning

[fs/lustre-release.git] / lustre / tests / sanity-hsm.sh

diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh
index 37ebd70..d739235 100755 (executable)
--- a/lustre/tests/sanity-hsm.sh
+++ b/lustre/tests/sanity-hsm.sh
@@ -147,10 +147,13 @@ fid2archive()
 {
        local fid="$1"
 
-       case "$HSMTOOL" in
-       *lhsmtool_posix)
-               printf "%s" "$(hsm_root)/*/*/*/*/*/*/$fid"
-               ;;
+       case "$HSMTOOL_ARCHIVE_FORMAT" in
+               v1)
+                       printf "%s" "$(hsm_root)/*/*/*/*/*/*/$fid"
+                       ;;
+               v2)
+                       printf "%s" "$(hsm_root)/*/$fid"
+                       ;;
        esac
 }
 
@@ -555,6 +558,8 @@ test_1A() { # was test_1
 run_test 1A "lfs hsm flags root/non-root access"
 
 test_1a() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
 
@@ -599,7 +604,7 @@ test_1bde_base() {
 }
 
 test_1b() {
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        $LFS setstripe -E 1M -S 1M -E 64M -c 2 -E -1 -c 4 $DIR/$tdir ||
                error "failed to set default stripe"
        local f=$DIR/$tdir/$tfile
@@ -635,8 +640,8 @@ test_1c() {
                error "wrong archive number, $st != $LOCAL_HSM_ARCHIVE_NUMBER"
 
        LOCAL_HSM_ARCHIVE_NUMBER=33
-       if [ $(lustre_version_code client) -ge $(version_code 2.11.56) ] &&
-          [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.11.56) ]; then
+       if [ "$CLIENT_VERSION" -ge $(version_code 2.11.56) ] &&
+          [ "$MDS1_VERSION" -ge $(version_code 2.11.56) ]; then
                # lustre in the new version supports unlimited archiveID.
                # Test whether setting archive number > 32 is supported
                $LFS hsm_set --exists --archive-id $LOCAL_HSM_ARCHIVE_NUMBER $f ||
@@ -672,7 +677,7 @@ test_1d() {
        [ $MDS1_VERSION -lt $(version_code 2.10.59) ] &&
                skip "need MDS version at least 2.10.59"
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        $LFS setstripe -E 1M -L mdt -E -1 -c 2 $DIR/$tdir ||
                error "failed to set default stripe"
        local f=$DIR/$tdir/$tfile
@@ -682,10 +687,10 @@ test_1d() {
 run_test 1d "Archive, Release and Restore DoM file"
 
 test_1e() {
-       [ $(lustre_version_code $SINGLEMDS) -lt $(version_code $SEL_VER) ] &&
+       [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
                skip "skipped for lustre < $SEL_VER"
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        $LFS setstripe -E 1G -z 64M -E 10G -z 512M -E -1 -z 1G $DIR/$tdir ||
                error "failed to set default stripe"
        local comp_file=$DIR/$tdir/$tfile
@@ -723,6 +728,7 @@ run_test 1e "Archive, Release and Restore SEL file"
 test_2() {
        local f=$DIR/$tdir/$tfile
 
+       mkdir_on_mdt0 $DIR/$tdir
        create_empty_file "$f"
        # New files are not dirty
        check_hsm_flags $f "0x00000000"
@@ -749,7 +755,7 @@ test_2() {
 run_test 2 "Check file dirtyness when doing setattr"
 
 test_3() {
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        f=$DIR/$tdir/$tfile
 
        # New files are not dirty
@@ -817,7 +823,7 @@ test_8() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        $LFS hsm_archive $f
@@ -837,7 +843,8 @@ test_9A() { # was test_9
        local uuid=$(get_agent_uuid $(facet_active_host $SINGLEAGT))
        check_agent_registered $uuid
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        $LFS hsm_archive --archive $archive_id $f
@@ -859,6 +866,8 @@ test_9a() {
                copytool setup --facet agt$n
        done
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        # archive files
        for n in $(seq $AGTCOUNT); do
                file=$DIR/$tdir/$tfile.$n
@@ -875,6 +884,7 @@ test_10a() {
        # test needs a running copytool
        copytool setup
 
+       mkdir_on_mdt0 $DIR/$tdir
        mkdir -p $DIR/$tdir/d1
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
@@ -903,7 +913,7 @@ test_10b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
        $LFS hsm_archive $f || error "archive request failed"
@@ -933,7 +943,7 @@ test_10d() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
        $LFS hsm_archive $f || error "cannot archive $f"
@@ -947,7 +957,7 @@ test_10d() {
 run_test 10d "Archive a file on the default archive id"
 
 test_11a() {
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/hosts $tdir/$tfile
        local f=$DIR/$tdir/$tfile
 
@@ -976,7 +986,7 @@ test_11b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
        $LFS hsm_archive -a $HSM_ARCHIVE_NUMBER $f ||
@@ -994,11 +1004,23 @@ test_11b() {
 }
 run_test 11b "Import a deleted file using its FID"
 
+test_11c() {
+       pool_add $TESTNAME || error "Pool creation failed"
+       pool_add_targets $TESTNAME 1 1 || error "pool_add_targets failed"
+
+       mkdir -p $DIR/$tdir
+       $LFS setstripe -p "$TESTNAME" $DIR/$tdir
+
+       copy2archive /etc/hosts $tdir/$tfile
+       copytool import $tdir/$tfile $DIR/$tdir/$tfile
+}
+run_test 11c "Import a file to a directory with a pool"
+
 test_12a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/hosts $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -1024,7 +1046,7 @@ test_12b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/hosts $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -1050,7 +1072,7 @@ test_12c() {
        copytool setup
 
        local f=$DIR/$tdir/$tfile
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        $LFS setstripe -c 2 "$f"
        local fid=$(create_file "$f" 1M 5)
 
@@ -1070,7 +1092,7 @@ test_12d() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
@@ -1094,7 +1116,7 @@ test_12e() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
        $LFS hsm_archive $f || error "archive request failed"
@@ -1114,7 +1136,7 @@ test_12f() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1137,7 +1159,7 @@ test_12g() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1161,7 +1183,7 @@ test_12h() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1183,7 +1205,7 @@ test_12m() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        $LFS hsm_archive $f || error "archive of $f failed"
@@ -1218,7 +1240,7 @@ test_12o() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1267,7 +1289,7 @@ test_12p() {
        # test needs a running copytool
        copytool setup
 
-       mkdir $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1292,6 +1314,8 @@ test_12q() {
        # test needs a running copytool
        copytool setup -m "$MOUNT3"
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local f2=$DIR2/$tdir/$tfile
        local fid=$(create_small_file $f)
@@ -1344,7 +1368,7 @@ test_12r() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/hosts $f)
 
@@ -1396,6 +1420,8 @@ test_14() {
        # test needs a running copytool
        copytool setup
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        # archive a file
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
@@ -1424,6 +1450,8 @@ test_15() {
        # test needs a running copytool
        copytool setup
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        # archive files
        local f=$DIR/$tdir/$tfile
        local count=5
@@ -1476,7 +1504,7 @@ test_16() {
        local goal=20
        dd if=/dev/zero of=$ref bs=1M count=20
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file $ref $f)
        rm $ref
@@ -1522,6 +1550,7 @@ run_test 20 "Release is not permitted"
 test_21() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/test_release
 
@@ -1531,7 +1560,7 @@ test_21() {
 
        # LU-4388/LU-4389 - ZFS does not report full number of blocks
        # used until file is flushed to disk
-       if [  $(facet_fstype ost1) == "zfs" ]; then
+       if [ "$ost1_FSTYPE" == "zfs" ]; then
            # this causes an OST_SYNC rpc to be sent
            dd if=/dev/zero of=$f bs=512 count=1 oflag=sync conv=notrunc,fsync
            # clear locks to reread file data
@@ -1588,6 +1617,7 @@ run_test 21 "Simple release tests"
 test_22() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/test_release
        local swap=$DIR/$tdir/test_swap
@@ -1613,6 +1643,7 @@ run_test 22 "Could not swap a release file"
 test_23() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/test_mtime
 
@@ -1649,6 +1680,7 @@ test_24a() {
 
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        fid=$(create_small_file $file)
 
@@ -1751,6 +1783,7 @@ test_24b() {
 
        # Test needs a running copytool.
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        # Check that root can do HSM actions on a regular user's file.
        fid=$(create_small_file $file)
@@ -1796,8 +1829,7 @@ test_24c() {
 
        # test needs a running copytool
        copytool setup
-
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        # Save the default masks and check that cleanup_24c will
        # restore the request masks correctly.
@@ -1859,6 +1891,8 @@ test_24d() {
        local fid1
        local fid2
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        fid1=$(create_small_file $file1)
 
        echo $fid1
@@ -1902,6 +1936,7 @@ run_test 24d "check that read-only mounts are respected"
 
 test_24e() {
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid
@@ -1921,6 +1956,7 @@ run_test 24e "tar succeeds on HSM released files" # LU-6213
 test_24f() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        mkdir -p $DIR/$tdir/d1
        local f=$DIR/$tdir/$tfile
@@ -1953,7 +1989,7 @@ test_24g() {
 
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        chmod ugo+rwx $DIR/$tdir
 
        echo "Please listen carefully as our options have changed." | tee $file
@@ -1994,7 +2030,7 @@ test_25b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -2015,6 +2051,7 @@ run_test 25b "Restore lost file (HS_LOST flag) after release"\
 test_26A() { # was test_26
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2035,8 +2072,8 @@ test_26a() {
 
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
-       mkdir -p $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -2085,8 +2122,8 @@ run_test 26a "Remove Archive On Last Unlink (RAoLU) policy"
 test_26b() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
-       mkdir -p $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -2117,8 +2154,8 @@ run_test 26b "RAoLU policy when CDT off"
 test_26c() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
-       mkdir -p $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -2164,8 +2201,8 @@ run_test 26c "RAoLU effective when file closed"
 test_26d() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
-       mkdir -p $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
 
@@ -2217,6 +2254,7 @@ run_test 27a "Remove the archive of an imported file (Operation not permitted)"
 test_27b() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2234,6 +2272,7 @@ run_test 27b "Remove the archive of a relased file (Operation not permitted)"
 test_28() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2273,6 +2312,7 @@ run_test 29a "Tests --mntpath and --archive options"
 test_29b() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
@@ -2290,6 +2330,7 @@ run_test 29b "Archive/delete/remove by FID from the archive."
 test_29c() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local fid1=$(create_small_file $DIR/$tdir/$tfile-1)
        local fid2=$(create_small_file $DIR/$tdir/$tfile-2)
@@ -2327,6 +2368,8 @@ test_29d() {
                copytool setup -f agt$n -a $n
        done
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        # archive files
        file=$DIR/$tdir/$tfile
        fid=$(create_small_file $file)
@@ -2414,7 +2457,7 @@ test_30b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/true
        local fid=$(copy_file /bin/true $f)
        chmod 755 $f
@@ -2442,7 +2485,7 @@ test_30c() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/SLEEP
        local slp_sum1=$(md5sum /bin/sleep)
        local fid=$(copy_file /bin/sleep $f)
@@ -2510,6 +2553,7 @@ restore_and_check_size() {
 test_31a() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        create_archive_file $tdir/$tfile
        local f=$DIR/$tdir/$tfile
@@ -2528,6 +2572,7 @@ run_test 31a "Import a large file and check size during restore"
 test_31b() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1MB 39)
@@ -2546,6 +2591,7 @@ run_test 31b "Restore a large unaligned file and check size during restore"
 test_31c() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1M 39)
@@ -2563,6 +2609,8 @@ run_test 31c "Restore a large aligned file and check size during restore"
 
 test_33() {
        local f=$DIR/$tdir/$tfile
+
+       mkdir_on_mdt0 $DIR/$tdir
        local fid=$(create_empty_file "$f")
 
        copytool setup
@@ -2592,6 +2640,7 @@ run_test 33 "Kill a restore waiting process"
 test_34() {
        # test needs a running copytool
        copytool setup -b 1
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2626,6 +2675,7 @@ run_test 34 "Remove file during restore"
 test_35() {
        # test needs a running copytool
        copytool setup -b 1
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local f1=$DIR/$tdir/$tfile-1
@@ -2662,6 +2712,7 @@ run_test 35 "Overwrite file during restore"
 test_36() {
        # test needs a running copytool
        copytool setup -b 1
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -2694,6 +2745,7 @@ run_test 36 "Move file during restore"
 test_37() {
        # LU-5683: check that an archived dirty file can be rearchived.
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid
@@ -2769,11 +2821,96 @@ test_40() {
 }
 run_test 40 "Parallel archive requests"
 
+hsm_archive_batch() {
+       local files_num=$1
+       local batch_max=$2
+       local filebase=$3
+       local batch_num=0
+       local fileset=""
+       local i=0
+
+       while [ $i -lt $files_num ]; do
+               if [ $batch_num -eq $batch_max ]; then
+                       $LFS hsm_archive $fileset || error "HSM archive failed"
+                       # Reset the batch container.
+                       fileset=""
+                       batch_num=0
+               fi
+
+               fileset+="${filebase}$i "
+               batch_num=$(( batch_num + 1 ))
+               i=$(( i + 1 ))
+       done
+
+       if [ $batch_num -ne 0 ]; then
+               $LFS hsm_archive $fileset || error "HSM archive failed"
+               fileset=""
+               batch_num=0
+       fi
+}
+
+test_50() {
+       local dir=$DIR/$tdir
+       local batch_max=50
+
+       set_hsm_param max_requests 1000000
+       mkdir $dir || error "mkdir $dir failed"
+       df -i $MOUNT
+
+       local start
+       local elapsed
+       local files_num
+       local filebase
+
+       files_num=10000
+       filebase="$dir/$tfile.start."
+       createmany -m $filebase $files_num ||
+               error "createmany -m $filebase failed: $?"
+
+       start=$SECONDS
+       hsm_archive_batch $files_num $batch_max "$filebase"
+       elapsed=$((SECONDS - start))
+       do_facet $SINGLEMDS "$LCTL get_param -n \
+                $HSM_PARAM.actions | grep WAITING | wc -l"
+       unlinkmany $filebase $files_num || error "unlinkmany $filabase failed"
+       echo "Start Phase files_num: $files_num time: $elapsed"
+
+       files_num=20000
+       filebase="$dir/$tfile.in."
+       createmany -m $filebase $files_num ||
+               error "createmany -m $filebase failed: $?"
+       start=$SECONDS
+       hsm_archive_batch  $files_num $batch_max "$filebase"
+       elapsed=$((SECONDS - start))
+       unlinkmany $filebase $files_num || error "unlinkmany $filabase failed"
+       echo "Middle Phase files_num: $files_num time: $elapsed"
+
+       files_num=10000
+       filebase="$dir/$tfile.end."
+       createmany -m $filebase $files_num ||
+               error "createmany -m $filebase failed: $?"
+
+       start=$SECONDS
+       hsm_archive_batch $files_num $batch_max "$filebase"
+       elapsed=$((SECONDS - start))
+       do_facet $SINGLEMDS "$LCTL get_param -n \
+                $HSM_PARAM.actions | grep WAITING | wc -l"
+
+       unlinkmany $filebase $files_num || error "unlinkmany $filebase failed"
+       echo "End Phase files_num: $files_num time: $elapsed"
+
+       do_facet $SINGLEMDS "$LCTL get_param -n \
+                $HSM_PARAM.actions | grep WAITING | wc -l"
+
+       cdt_purge
+}
+run_test 50 "Archive with large number of pending HSM actions"
+
 test_52() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
 
@@ -2798,7 +2935,7 @@ test_53() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file $f)
 
@@ -2821,6 +2958,8 @@ test_53() {
 run_test 53 "Opened for read file on an evicted client should not be set dirty"
 
 test_54() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1MB 39)
 
@@ -2845,6 +2984,8 @@ test_54() {
 run_test 54 "Write during an archive cancels it"
 
 test_55() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1MB 39)
 
@@ -2869,6 +3010,8 @@ test_55() {
 run_test 55 "Truncate during an archive cancels it"
 
 test_56() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1MB 39)
 
@@ -2898,7 +3041,7 @@ test_57() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/test_archive_remote
        # Create a file on a remote node
        do_node $CLIENT2 "dd if=/dev/urandom of=$f bs=1M "\
@@ -2960,7 +3103,7 @@ test_58() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local sz=$(stat -c %s /etc/passwd)
 
@@ -2994,6 +3137,8 @@ test_60() {
        # This test validates the fix for LU-4512. Ensure that the -u
        # option changes the progress reporting interval from the
        # default (30 seconds) to the user-specified interval.
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_file "$f" 1M 10)
 
@@ -3060,7 +3205,7 @@ test_61() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        cdt_disable
@@ -3128,6 +3273,8 @@ test_71() {
        # Just start and stop the copytool to generate events.
        cdt_clear_no_retry
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_small_file "$f")
 
@@ -3200,7 +3347,7 @@ test_72() {
                error "cannot create $test_file on $SINGLEAGT"
        copy2archive $test_file $tdir/$tfile
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        copytool import $tdir/$tfile $f
        f=$DIR2/$tdir/$tfile
@@ -3282,7 +3429,7 @@ run_test 72 "Copytool logs JSON restore events to FIFO"
 
 test_90() {
        file_count=51 # Max number of files constrained by LNET message size
-       mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+       mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
        local f=$DIR/$tdir/$tfile
        local FILELIST=/tmp/filelist.txt
        local i=""
@@ -3372,6 +3519,8 @@ run_test 103 "Purge all requests"
 DATA=CEA
 DATAHEX='[434541]'
 test_104() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3389,7 +3538,7 @@ run_test 104 "Copy tool data field"
 
 test_105() {
        local max_requests=$(get_hsm_param max_requests)
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local i=""
 
        stack_trap "set_hsm_param max_requests $max_requests" EXIT
@@ -3444,7 +3593,7 @@ test_107() {
        # test needs a running copytool
        copytool setup
        # create and archive file
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f1=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f1)
        $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f1
@@ -3499,7 +3648,7 @@ test_110a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        copy2archive /etc/passwd $tdir/$tfile
 
@@ -3526,7 +3675,7 @@ test_110b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f
@@ -3552,7 +3701,7 @@ test_111a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/passwd $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -3581,7 +3730,7 @@ test_111b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        stack_trap cdt_clear_no_retry EXIT
@@ -3606,7 +3755,7 @@ test_112() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
        cdt_disable
@@ -3624,6 +3773,8 @@ test_112() {
 run_test 112 "State of recorded request"
 
 test_113() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local file1=$DIR/$tdir/$tfile
        local file2=$DIR2/$tdir/$tfile
 
@@ -3660,6 +3811,8 @@ test_113() {
 run_test 113 "wrong stat after restore"
 
 test_200() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3685,6 +3838,7 @@ run_test 200 "Register/Cancel archive"
 test_201() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        create_archive_file $tdir/$tfile
@@ -3704,6 +3858,8 @@ test_201() {
 run_test 201 "Register/Cancel restore"
 
 test_202() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3727,7 +3883,7 @@ test_220A() { # was test_220
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -3746,7 +3902,7 @@ test_220a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -3775,6 +3931,8 @@ test_220a() {
 run_test 220a "Changelog for failed archive"
 
 test_221() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3801,7 +3959,7 @@ test_222a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/passwd $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -3822,7 +3980,7 @@ test_222b() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -3844,7 +4002,7 @@ test_222c() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        copy2archive /etc/passwd $tdir/$tfile
 
        local f=$DIR/$tdir/$tfile
@@ -3878,7 +4036,7 @@ test_222d() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
 
@@ -3901,6 +4059,7 @@ run_test 222d "Changelog for failed implicit restore"
 test_223a() {
        # test needs a running copytool
        copytool setup -b 1
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        create_archive_file $tdir/$tfile
@@ -3922,6 +4081,8 @@ test_223a() {
 run_test 223a "Changelog for restore canceled (import case)"
 
 test_223b() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -3952,7 +4113,7 @@ test_224A() { # was test_224
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -3973,7 +4134,7 @@ test_224a() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(copy_file /etc/passwd $f)
@@ -4013,6 +4174,7 @@ test_225() {
 
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -4040,7 +4202,7 @@ test_226() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f1=$DIR/$tdir/$tfile-1
        local f2=$DIR/$tdir/$tfile-2
@@ -4115,6 +4277,7 @@ run_test 227 "changelog when explicit setting of HSM flags"
 test_228() {
        # test needs a running copytool
        copytool setup
+       mkdir_on_mdt0 $DIR/$tdir
 
        local fid=$(create_small_sync_file $DIR/$tfile)
        $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $DIR/$tfile
@@ -4161,6 +4324,9 @@ test_250() {
 
        # send 1 requests of each kind twice
        copytool setup
+
+       mkdir_on_mdt0 $DIR/$tdir
+
        # setup the files
        for action in archive restore remove; do
                local filepath="$file"-to-$action
@@ -4209,6 +4375,8 @@ test_250() {
 run_test 250 "Coordinator max request"
 
 test_251() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -4238,6 +4406,8 @@ test_251() {
 run_test 251 "Coordinator request timeout"
 
 test_252() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local fid=$(create_empty_file "$f")
 
@@ -4267,7 +4437,7 @@ test_253() {
        # test needs a running copytool
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
        local f=$DIR/$tdir/$tfile
 
        dd if=/dev/zero of=$f bs=1MB count=10
@@ -4331,6 +4501,8 @@ test_254b()
                "set_hsm_param max_requests $(get_hsm_param max_requests)" EXIT
        set_hsm_param max_requests "$request_count"
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        local timeout
        local count
        for request_type in archive restore remove; do
@@ -4391,6 +4563,8 @@ test_255()
        [ $MDS1_VERSION -lt $(version_code 2.12.0) ] &&
                skip "Need MDS version at least 2.12.0"
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        local file="$DIR/$tdir/$tfile"
        local fid=$(create_empty_file "$file")
 
@@ -4448,6 +4622,8 @@ test_260a()
        local -a files=("$DIR/$tdir/$tfile".{0..15})
        local file
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        for file in "${files[@]}"; do
                create_small_file "$file"
        done
@@ -4514,6 +4690,8 @@ test_260b()
        local -a files=("$DIR/$tdir/$tfile".{0..15})
        local file
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        for file in "${files[@]}"; do
                create_small_file "$file"
        done
@@ -4581,6 +4759,8 @@ test_260c()
        local -a files=("$DIR/$tdir/$tfile".{0..15})
        local file
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        for file in "${files[@]}"; do
                create_small_file "$file"
        done
@@ -4730,7 +4910,7 @@ test_400() {
 
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local dir_mdt0=$DIR/$tdir/mdt0
        local dir_mdt1=$DIR/$tdir/mdt1
@@ -4762,7 +4942,7 @@ test_401() {
 
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local dir_mdt0=$DIR/$tdir/mdt0
        local dir_mdt1=$DIR/$tdir/mdt1
@@ -4820,7 +5000,7 @@ run_test 402a "Copytool start fails if all MDTs are inactive"
 test_402b() {
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        touch $f || error "touch $f failed"
@@ -4873,7 +5053,7 @@ test_404() {
        copytool setup
 
        # create files on both MDT0000 and MDT0001
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local dir_mdt0=$DIR/$tdir/mdt0
        stack_trap "rm -rf $dir_mdt0" EXIT
@@ -4902,7 +5082,7 @@ test_405() {
 
        copytool setup
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local striped_dir=$DIR/$tdir/striped_dir
 
@@ -4954,6 +5134,8 @@ test_406() {
        local fid
        local mdt_index
 
+       mkdir_on_mdt0 $DIR/$tdir
+
        fid=$(create_small_file $DIR/$tdir/$tfile)
        echo "old fid $fid"
 
@@ -5004,6 +5186,8 @@ test_406() {
 run_test 406 "attempting to migrate HSM archived files is safe"
 
 test_407() {
+       mkdir_on_mdt0 $DIR/$tdir
+
        local f=$DIR/$tdir/$tfile
        local f2=$DIR2/$tdir/$tfile
        local fid=$(create_empty_file "$f")
@@ -5045,13 +5229,13 @@ run_test 407 "Check for double RESTORE records in llog"
 
 test_500()
 {
-       [ $MDS1_VERSION -lt $(version_code 2.6.92) ] &&
+       [ "$MDS1_VERSION" -lt $(version_code 2.6.92) ] &&
                skip "HSM migrate is not supported"
 
        test_mkdir -p $DIR/$tdir
 
-       if [ $(lustre_version_code client) -lt $(version_code 2.11.56) ] ||
-            [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.11.56) ];
+       if [ "$CLIENT_VERSION" -lt $(version_code 2.11.56) ] ||
+            [ "$MDS1_VERSION" -lt $(version_code 2.11.56) ];
        then
                llapi_hsm_test -d $DIR/$tdir -b ||
                        error "One llapi HSM test failed"
@@ -5063,7 +5247,7 @@ test_500()
 run_test 500 "various LLAPI HSM tests"
 
 test_600() {
-       [ $MDS1_VERSION -lt $(version_code 2.10.58) ] &&
+       [ "$MDS1_VERSION" -lt $(version_code 2.10.58) ] &&
                skip "need MDS version at least 2.10.58"
 
        mkdir -p $DIR/$tdir
@@ -5122,6 +5306,9 @@ test_602() {
        [ $MDS1_VERSION -lt $(version_code 2.10.58) ] &&
                skip "need MDS version at least 2.10.58"
 
+       stack_trap "restore_opencache" EXIT
+       disable_opencache
+
        mkdir -p $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
@@ -5194,7 +5381,7 @@ test_604() {
        [ $MDS1_VERSION -lt $(version_code 2.10.58) ] &&
                skip "need MDS version at least 2.10.58"
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
        local f2=$DIR2/$tdir/$tfile
@@ -5270,6 +5457,9 @@ test_605() {
        [ $MDS1_VERSION -lt $(version_code 2.10.58) ] &&
                skip "need MDS version at least 2.10.58"
 
+       stack_trap "restore_opencache" EXIT
+       disable_opencache
+
        mkdir -p $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
@@ -5355,10 +5545,9 @@ test_606() {
        local llog_reader=$(do_facet mgs "which llog_reader 2> /dev/null")
        llog_reader=${llog_reader:-$LUSTRE/utils/llog_reader}
        [ -z $(do_facet mgs ls -d $llog_reader 2> /dev/null) ] &&
-                       skip_env "missing llog_reader" && return
-       local fstype=$(facet_fstype mds1)
+                       skip_env "missing llog_reader"
 
-       mkdir -p $DIR/$tdir
+       mkdir_on_mdt0 $DIR/$tdir
 
        local f=$DIR/$tdir/$tfile
 
@@ -5376,8 +5565,9 @@ test_606() {
        local entry
 
        #remount mds1 as ldiskfs or zfs type
-       stack_trap "stop mds1; start mds1 $(mdsdevname 1) $MDS_MOUNT_OPTS" EXIT
        stop mds1 || error "stop mds1 failed"
+       stack_trap "unmount_fstype mds1; start mds1 $(mdsdevname 1)\
+               $MDS_MOUNT_OPTS" EXIT
        mount_fstype mds1 || error "remount mds1 failed"
 
        for ((i = 0; i < 1; i++)); do