LU-15132 hsm: Protect against parallel HSM restore requests

[fs/lustre-release.git] / lustre / tests / sanity-hsm.sh

diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh
index d739235..b234123 100755 (executable)
--- a/lustre/tests/sanity-hsm.sh
+++ b/lustre/tests/sanity-hsm.sh
@@ -11,7 +11,7 @@ ONLY=${ONLY:-"$*"}
 
 LUSTRE=${LUSTRE:-$(dirname $0)/..}
 . $LUSTRE/tests/test-framework.sh
-init_test_env $@
+init_test_env "$@"
 init_logging
 
 ALWAYS_EXCEPT="$SANITY_HSM_EXCEPT "
@@ -1384,6 +1384,37 @@ test_12r() {
 }
 run_test 12r "lseek restores released file"
 
+test_12s() {
+       local f=$DIR/$tdir/$tfile
+       local fid
+       local pid1 pid2
+
+       (( MDS1_VERSION >= $(version_code 2.15.50) )) ||
+               skip "Need MDS version newer than 2.15.50"
+
+       # test needs a running copytool
+       copytool setup
+
+       mkdir_on_mdt0 $DIR/$tdir
+       fid=$(copy_file /etc/hosts $f)
+
+       $LFS hsm_archive $f || error "archive of $f failed"
+       wait_request_state $fid ARCHIVE SUCCEED
+       $LFS hsm_release $f || error "release of $f failed"
+
+#define OBD_FAIL_ONCE|OBD_FAIL_MDS_HSM_RESTORE_RACE 0x8000018b
+       do_facet mds1 $LCTL set_param fail_loc=0x8000018b
+       cat $f > /dev/null & pid1=$!
+       cat $f > /dev/null & pid2=$!
+
+       wait $pid1 || error "cat process 1 fail (pid: $pid1)"
+       wait $pid2 || error "cat process 2 fail (pid: $pid2)"
+
+       # Race exists if more than 1 restore requests is registered
+       assert_request_count $fid RESTORE 1
+}
+run_test 12s "race between restore requests"
+
 test_13() {
        local -i i j k=0
        for i in {1..10}; do
@@ -3174,7 +3205,7 @@ test_60() {
        local sleep=1
 
        echo -n "Expecting a progress update within $progress_timeout seconds... "
-       while [ true ]; do
+       while true; do
                RESULT=$(do_node $(facet_active_host $mds) "$cmd")
                if [ -n "$RESULT" ] && [ "$RESULT" -gt 0 ]; then
                        echo "$RESULT bytes copied in $WAIT seconds."
@@ -3516,6 +3547,66 @@ test_103() {
 }
 run_test 103 "Purge all requests"
 
+test_103a() {
+       (( MDS1_VERSION >= $(version_code 2.14.56) )) ||
+               skip "Need MDS version at least 2.14.56"
+
+       cdt_clear_non_blocking_restore
+
+       # test needs a running copytool
+       copytool setup
+
+       local -a fids=()
+       local i
+       local rpcs_inflight=$($LCTL get_param -n \
+               "mdc.$(facet_svc mds1)*.max_rpcs_in_flight" |
+               head -n1)
+
+       mkdir_on_mdt0 $DIR/$tdir
+       for ((i=0; i < rpcs_inflight; i++)); do
+               fids+=( $(copy_file /etc/passwd $DIR/$tdir/${tfile}_$i) )
+       done
+       $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $DIR/$tdir/*
+
+       local time=0
+       local cnt=0
+       local grep_regex="($(tr ' ' '|' <<< "${fids[*]}")).*action=ARCHIVE.*status=SUCCEED"
+       echo $grep_regex
+       while [[ $time -lt 5 ]] && [[ $cnt -ne ${#fids[@]} ]]; do
+               cnt=$(do_facet mds1 "$LCTL get_param -n $HSM_PARAM.actions |
+                       grep -c -E '$grep_regex'")
+               sleep 1
+               ((++time))
+       done
+       [[ $cnt -eq ${#fids[@]} ]] || error "Fail to archive files $cnt/${#fids[@]}"
+
+       $LFS hsm_release $DIR/$tdir/*
+
+       kill_copytools
+       wait_copytools || error "Copytool failed to stop"
+
+       local -a pids=()
+       for i in "${fids[@]}"; do
+               cat $DIR/.lustre/fid/$i > /dev/null & pids+=($!)
+       done
+
+       cdt_purge
+       grep_regex="($(tr ' ' '|' <<< "${fids[*]}")).*action=RESTORE.*status=CANCELED"
+       cnt=$(do_facet mds1 "$LCTL get_param -n $HSM_PARAM.actions |
+               grep -cE '$grep_regex'")
+
+       [[ "$cnt" -eq ${#fids[@]} ]] ||
+               error "Some request have not been canceled ($cnt/${#fids[@]} canceled)"
+
+       # cat cmds should not hang and should fail
+       for i in "${!pids[@]}"; do
+               wait ${pids[$i]} &&
+                       error "Restore for ${tfile}_$i (${pids[$i]}) should fail" ||
+                       true
+       done
+}
+run_test 103a "Purge pending restore requests"
+
 DATA=CEA
 DATAHEX='[434541]'
 test_104() {
@@ -4831,6 +4922,36 @@ test_260c()
 }
 run_test 260c "Requests are not reordered on the 'hot' path of the coordinator"
 
+test_261() {
+       local file=$DIR/$tdir/$tfile
+       local size
+       local fid
+
+       copytool setup
+       mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+
+       dd if=/dev/zero of=$file bs=4k count=2 || error "Write $file failed"
+       fid=$(path2fid $file)
+       $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $file
+       wait_request_state $fid ARCHIVE SUCCEED
+
+       $LFS hsm_state $file
+       $LFS hsm_release $file
+       $LFS hsm_restore $file
+       wait_request_state $fid RESTORE SUCCEED
+       $LFS hsm_release $file
+       size=$(stat -c %s $file)
+       [[ $size == 8192 ]] || error "Size after HSM release: $size"
+
+       $LFS hsm_release $file
+       $LFS hsm_restore $file
+       $LFS hsm_release $file
+       size=$(stat -c %s $file)
+       [[ $size == 8192 ]] || error "Size after HSM release: $size"
+       $LFS hsm_state $file
+}
+run_test 261 "Report 0 bytes size after HSM release"
+
 test_300() {
        [ "$CLIENTONLY" ] && skip "CLIENTONLY mode" && return
 
@@ -5593,7 +5714,7 @@ test_606() {
        [ -n "$nid" ] || error "nid is empty"
        echo "Got NID $nid"
        [ -n "$nid" ] && [[ "${CLIENT_NIDS[*]}" =~ $nid ]] ||
-               error "nid '$nid' does not match any NID ${CLIENT_NIDS[@]}"
+               error "nid '$nid' does not match any NID ${CLIENT_NIDS[*]}"
 }
 run_test 606 "llog_reader groks changelog fields"