X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Fsanity-hsm.sh;h=0ee67b103990e87f6b8eafec33406a983b066a4d;hp=218012af74b7680780671a70fd71d5a204c113a9;hb=f625f670afbe954030ff81f0f8522137d6cdd335;hpb=8641822bdb248a034bf4065c7e2e60cdb7a47041 diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh index 218012a..0ee67b1 100755 --- a/lustre/tests/sanity-hsm.sh +++ b/lustre/tests/sanity-hsm.sh @@ -13,10 +13,6 @@ export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/utils:$PATH:/sbin:/usr/sbin ONLY=${ONLY:-"$*"} # bug number for skipped test: LU-3815 ALWAYS_EXCEPT="$SANITY_HSM_EXCEPT 34 35 36" -# bug number for skipped test:LU-4178 -ALWAYS_EXCEPT="$ALWAYS_EXCEPT 200 221 223b" -# bug number for skipped test:LU-3852 -ALWAYS_EXCEPT="$ALWAYS_EXCEPT 251" # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} @@ -340,7 +336,7 @@ make_archive() { local file=$HSM_ARCHIVE/$1 do_facet $SINGLEAGT mkdir -p $(dirname $file) do_facet $SINGLEAGT dd if=/dev/urandom of=$file count=32 bs=1000000 || - error "cannot create $file" + file_creation_failure dd $file $? } copy2archive() { @@ -515,17 +511,19 @@ needclients() { path2fid() { $LFS path2fid $1 | tr -d '[]' + return ${PIPESTATUS[0]} } get_hsm_flags() { local f=$1 local u=$2 + local st if [[ $u == "user" ]]; then - local st=$($RUNAS $LFS hsm_state $f) + st=$($RUNAS $LFS hsm_state $f) else - local st=$($LFS hsm_state $f) u=root + st=$($LFS hsm_state $f) fi [[ $? == 0 ]] || error "$LFS hsm_state $f failed (run as $u)" @@ -536,7 +534,8 @@ get_hsm_flags() { get_hsm_archive_id() { local f=$1 - local st=$($LFS hsm_state $f) + local st + st=$($LFS hsm_state $f) [[ $? == 0 ]] || error "$LFS hsm_state $f failed" local ar=$(echo $st | grep "archive_id" | cut -f5 -d" " | @@ -560,6 +559,15 @@ check_hsm_flags_user() { [[ $st == $fl ]] || error "hsm flags on $f are $st != $fl" } +file_creation_failure() { + local cmd=$1 + local f=$2 + local err=$3 + + df $MOUNT $MOUNT2 >&2 + error "cannot create $f with $cmd, status=$err" +} + copy_file() { local f= @@ -573,20 +581,22 @@ copy_file() { f=${f/$DIR/$DIR2} fi rm -f $f - cp $1 $f || error "cannot copy $1 to $f" + cp $1 $f || file_creation_failure cp $f $? + path2fid $f || error "cannot get fid on $f" } make_small() { local file2=${1/$DIR/$DIR2} dd if=/dev/urandom of=$file2 count=2 bs=1M conv=fsync || - error "cannot create $file2" + file_creation_failure dd $file2 $? + path2fid $1 || error "cannot get fid on $1" } make_small_sync() { dd if=/dev/urandom of=$1 count=1 bs=1M conv=sync || - error "cannot create $1" + file_creation_failure dd $1 $? path2fid $1 || error "cannot get fid on $1" } @@ -596,14 +606,27 @@ cleanup_large_files() { [ $ratio -gt 50 ] && find $MOUNT -size +10M -exec rm -f {} \; } +check_enough_free_space() { + local nb=$1 + local unit=$2 + local need=$((nb * unit /1024)) + local free=$(df -kP $MOUNT | tail -1 | awk '{print $4}') + (( $need >= $free )) && return 1 + return 0 +} + make_large_for_striping() { local file2=${1/$DIR/$DIR2} local sz=$($LCTL get_param -n lov.*-clilov-*.stripesize | head -n1) cleanup_large_files + check_enough_free_space 5 $sz + [ $? != 0 ] && return $? + dd if=/dev/urandom of=$file2 count=5 bs=$sz conv=fsync || - error "cannot create $file2" + file_creation_failure dd $file2 $? + path2fid $1 || error "cannot get fid on $1" } @@ -612,12 +635,16 @@ make_large_for_progress() { cleanup_large_files + check_enough_free_space 39 1000000 + [ $? != 0 ] && return $? + # big file is large enough, so copy time is > 30s # so copytool make 1 progress # size is not a multiple of 1M to avoid stripe # aligment dd if=/dev/urandom of=$file2 count=39 bs=1000000 conv=fsync || - error "cannot create $file2" + file_creation_failure dd $file2 $? + path2fid $1 || error "cannot get fid on $1" } @@ -626,12 +653,15 @@ make_large_for_progress_aligned() { cleanup_large_files + check_enough_free_space 33 1048576 + [ $? != 0 ] && return $? + # big file is large enough, so copy time is > 30s # so copytool make 1 progress # size is a multiple of 1M to have stripe # aligment dd if=/dev/urandom of=$file2 count=33 bs=1M conv=fsync || - error "cannot create $file2" + file_creation_failure dd $file2 $? path2fid $1 || error "cannot get fid on $1" } @@ -640,9 +670,12 @@ make_large_for_cancel() { cleanup_large_files + check_enough_free_space 103 1048576 + [ $? != 0 ] && return $? + # Copy timeout is 100s. 105MB => 105s dd if=/dev/urandom of=$file2 count=103 bs=1M conv=fsync || - error "cannot create $file2" + file_creation_failure dd $file2 $? path2fid $1 || error "cannot get fid on $1" } @@ -663,7 +696,7 @@ wait_request_state() { local cmd="$LCTL get_param -n ${MDT_PREFIX}${mdtidx}.hsm.actions" cmd+=" | awk '/'$fid'.*action='$request'/ {print \\\$13}' | cut -f2 -d=" - wait_result $mds "$cmd" $state 100 || + wait_result $mds "$cmd" $state 200 || error "request on $fid is not $state on $mds" } @@ -827,7 +860,7 @@ test_3() { error "user could not change hsm flags" dd if=/etc/passwd of=$f.append bs=1 count=3\ conv=notrunc oflag=append status=noxfer || - error "could not append to test file" + file_creation_failure dd $f.append $? check_hsm_flags $f.append "0x00000003" # Modify a file sets it dirty @@ -836,7 +869,7 @@ test_3() { error "user could not change hsm flags" dd if=/dev/zero of=$f.modify bs=1 count=3\ conv=notrunc status=noxfer || - error "could not modify test file" + file_creation_failure dd $f.modify $? check_hsm_flags $f.modify "0x00000003" # Open O_TRUNC sets dirty @@ -897,7 +930,7 @@ test_9() { copytool_cleanup } -run_test 9 "Use of explict archive number, with dedicated copytool" +run_test 9 "Use of explicit archive number, with dedicated copytool" test_9a() { needclients 3 || return 0 @@ -1122,7 +1155,10 @@ test_12c() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile $LFS setstripe -c 2 $f - local fid=$(make_large_for_striping $f) + local fid + fid=$(make_large_for_striping $f) + [ $? != 0 ] && skip "not enough free space" && return + local FILE_CRC=$(md5sum $f) $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f @@ -1349,6 +1385,26 @@ test_12o() { } run_test 12o "Layout-swap failure during Restore leaves file released" +test_12p() { + # test needs a running copytool + copytool_setup + + mkdir $DIR/$tdir + local f=$DIR/$tdir/$tfile + local fid=$(copy_file /etc/hosts $f) + + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f + wait_request_state $fid ARCHIVE SUCCEED + do_facet $SINGLEAGT cat $f > /dev/null || error "cannot cat $f" + $LFS hsm_release $f || error "cannot release $f" + do_facet $SINGLEAGT cat $f > /dev/null || error "cannot cat $f" + $LFS hsm_release $f || error "cannot release $f" + do_facet $SINGLEAGT cat $f > /dev/null || error "cannot cat $f" + + copytool_cleanup +} +run_test 12p "implicit restore of a file on copytool mount point" + test_13() { # test needs a running copytool copytool_setup @@ -1778,7 +1834,7 @@ test_24b() { copytool_setup mkdir -p $DIR/$tdir - # Check that root can do HSM actions on a ordinary user's file. + # Check that root can do HSM actions on a regular user's file. rm -f $file fid=$(make_small $file) sum0=$(md5sum $file) @@ -1997,7 +2053,10 @@ test_26() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f wait_request_state $fid ARCHIVE SUCCEED @@ -2034,7 +2093,10 @@ test_27b() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f wait_request_state $fid ARCHIVE SUCCEED $LFS hsm_release $f @@ -2053,7 +2115,10 @@ test_28() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f wait_request_state $fid ARCHIVE SUCCEED @@ -2142,6 +2207,7 @@ test_30c() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/SLEEP + local slp_sum1=$(md5sum /bin/sleep) local fid=$(copy_file /bin/sleep $f) chmod 755 $f $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f @@ -2158,7 +2224,12 @@ test_30c() { wait $pid [[ $? == 0 ]] || error "Execution failed during run" cmp /bin/sleep $f - [[ $? == 0 ]] || error "Binary overwritten during exec" + if [[ $? != 0 ]]; then + local slp_sum2=$(md5sum /bin/sleep) + # in case sleep file is modified during the test + [[ $slp_sum1 == $slp_sum2 ]] && + error "Binary overwritten during exec" + fi # cleanup # remove no try action mode @@ -2231,7 +2302,10 @@ test_31b() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f wait_request_state $fid ARCHIVE SUCCEED $LFS hsm_release $f @@ -2252,7 +2326,10 @@ test_31c() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress_aligned $f) + local fid + fid=$(make_large_for_progress_aligned $f) + [ $? != 0 ] && skip "not enough free space" && return + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f wait_request_state $fid ARCHIVE SUCCEED $LFS hsm_release $f @@ -2273,7 +2350,10 @@ test_33() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f wait_request_state $fid ARCHIVE SUCCEED $LFS hsm_release $f @@ -2337,7 +2417,10 @@ test_34() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f wait_request_state $fid ARCHIVE SUCCEED $LFS hsm_release $f @@ -2370,7 +2453,10 @@ test_35() { local f=$DIR/$tdir/$tfile local f1=$DIR/$tdir/$tfile-1 - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return + local fid1=$(copy_file /etc/passwd $f1) $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f wait_request_state $fid ARCHIVE SUCCEED @@ -2406,7 +2492,10 @@ test_36() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f wait_request_state $fid ARCHIVE SUCCEED $LFS hsm_release $f @@ -2599,7 +2688,9 @@ test_56() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f || error "could not archive file" @@ -2721,7 +2812,9 @@ test_60() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return local mdtidx=0 local mdt=${MDT_PREFIX}${mdtidx} @@ -2736,6 +2829,16 @@ test_60() { $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f || error "could not archive file" + local agent=$(facet_active_host $SINGLEAGT) + local prefix=$TESTLOG_PREFIX + [[ -z "$TESTNAME" ]] || prefix=$prefix.$TESTNAME + local copytool_log=$prefix.copytool_log.$agent.log + + + wait_update $agent \ + "grep -o start.copy $copytool_log" "start copy" 100 || + error "copytool failed to start" + local cmd="$LCTL get_param -n ${mdt}.hsm.active_requests" cmd+=" | awk '/'$fid'.*action=ARCHIVE/ {print \\\$12}' | cut -f2 -d=" @@ -2828,7 +2931,9 @@ test_71() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f || error "could not archive file" @@ -3104,7 +3209,10 @@ test_104() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return + # if cdt is on, it can serve too quickly the request cdt_disable $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER --data $DATA $f @@ -3415,7 +3523,10 @@ test_200() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_cancel $f) + local fid + fid=$(make_large_for_cancel $f) + [ $? != 0 ] && skip "not enough free space" && return + # test with cdt on is made in test_221 cdt_disable $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f @@ -3456,7 +3567,10 @@ test_202() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f wait_request_state $fid ARCHIVE SUCCEED @@ -3501,7 +3615,9 @@ test_221() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_cancel $f) + local fid + fid=$(make_large_for_cancel $f) + [ $? != 0 ] && skip "not enough free space" && return changelog_setup @@ -3608,7 +3724,9 @@ test_223b() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return changelog_setup $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f @@ -3668,7 +3786,9 @@ test_225() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_progress $f) + local fid + fid=$(make_large_for_progress $f) + [ $? != 0 ] && skip "not enough free space" && return changelog_setup $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f @@ -3871,7 +3991,9 @@ test_251() { mkdir -p $DIR/$tdir local f=$DIR/$tdir/$tfile - local fid=$(make_large_for_cancel $f) + local fid + fid=$(make_large_for_cancel $f) + [ $? != 0 ] && skip "not enough free space" && return cdt_disable # to have a short test @@ -3883,6 +4005,10 @@ test_251() { set_hsm_param loop_period 2 cdt_enable + # clear locks to avoid extra delay caused by flush/cancel + # and thus prevent early copytool death to timeout. + cancel_lru_locks osc + $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f wait_request_state $fid ARCHIVE STARTED sleep 5 @@ -4175,6 +4301,19 @@ test_405() { } run_test 405 "archive and release under striped directory" +test_500() +{ + [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.6.92) ] && + skip "HSM migrate is not supported" && return + + # Stop the existing copytool + copytool_cleanup + + test_mkdir -p $DIR/$tdir + llapi_hsm_test -d $DIR/$tdir || error "One llapi HSM test failed" +} +run_test 500 "various LLAPI HSM tests" + copytool_cleanup complete $SECONDS