LU-3726 tests: make sanity-hsm compatible with MDSCOUNT >= 2.

Add generic functions to perform set_param and check_param
on all MDTs, and persistent set_param on the mgs.
CL_USER variable replaced by CL_USERS array (1 value per MDT).
MDT0 variable replaced by MDT array (1 value per MDT).

Test-Parameters: mdtcount=2 mdscount=2 testlist=sanity-hsm
Signed-off-by: Thomas Leibovici <thomas.leibovici@cea.fr>
Signed-off-by: Aurelien Degremont <aurelien.degremont@cea.fr>
Change-Id: I0d8610a74427423dda25a6defd85f5c52c535850
Reviewed-on: http://review.whamcloud.com/7437
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: jacques-Charles Lafoucriere <jacques-charles.lafoucriere@cea.fr>
Reviewed-by: Faccini Bruno <bruno.faccini@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/tests/sanity-hsm.sh b/lustre/tests/sanity-hsm.sh
index d4ba709..ae96f81 100644 (file)
--- a/lustre/tests/sanity-hsm.sh
+++ b/lustre/tests/sanity-hsm.sh
@@ -30,9 +30,9 @@ MCREATE=${MCREATE:-mcreate}
 MOUNT_2=${MOUNT_2:-"yes"}
 FAIL_ON_ERROR=false
 
-if [[ $MDSCOUNT -ge 2 ]]; then
-       skip_env "Only run with single MDT for now" && exit
-fi
+# script only handles up to 10 MDTs (because of MDT_PREFIX)
+[ $MDSCOUNT -gt 9 ] &&
+       error "script cannot handle more than 9 MDTs, please fix" && exit
 
 check_and_setup_lustre
 
@@ -94,8 +94,9 @@ init_agt_vars() {
        HSM_ARCHIVE=$(copytool_device $SINGLEAGT)
        HSM_ARCHIVE_NUMBER=2
 
-       MDT_PARAM="mdt.$FSNAME-MDT0000"
-       HSM_PARAM="$MDT_PARAM.hsm"
+       # The test only support up to 10 MDTs
+       MDT_PREFIX="mdt.$FSNAME-MDT000"
+       HSM_PARAM="${MDT_PREFIX}0.hsm"
 
        # archive is purged at copytool setup
        HSM_ARCHIVE_PURGE=true
@@ -116,6 +117,17 @@ cleanup() {
        cdt_set_sanity_policy
 }
 
+get_mdt_devices() {
+       local mdtno
+       # get MDT device for each mdc
+       for mdtno in $(seq 1 $MDSCOUNT); do
+               local idx=$(($mdtno - 1))
+               MDT[$idx]=$($LCTL get_param -n \
+                       mdc.$FSNAME-MDT000${idx}-mdc-*.mds_server_uuid |
+                       awk '{gsub(/_UUID/,""); print $1}' | head -1)
+       done
+}
+
 search_and_kill_copytool() {
        local agents=${1:-$(facet_active_host $SINGLEAGT)}
 
@@ -227,20 +239,66 @@ copy2archive() {
        do_facet $SINGLEAGT cp -p $1 $file || error "cannot copy $1 to $file"
 }
 
+mdts_set_param() {
+       local arg=$1
+       local key=$2
+       local value=$3
+       local mdtno
+       local rc=0
+       if [[ "$value" != "" ]]; then
+               value="=$value"
+       fi
+       for mdtno in $(seq 1 $MDSCOUNT); do
+               local idx=$(($mdtno - 1))
+               local facet=mds${mdtno}
+               # if $arg include -P option, run 1 set_param per MDT on the MGS
+               # else, run set_param on each MDT
+               [[ $arg = *"-P"* ]] && facet=mgs
+               do_facet $facet $LCTL set_param $arg mdt.${MDT[$idx]}.$key$value
+               [[ $? != 0 ]] && rc=1
+       done
+       return $rc
+}
+
+mdts_check_param() {
+       local key="$1"
+       local target="$2"
+       local timeout="$3"
+       local mdtno
+       for mdtno in $(seq 1 $MDSCOUNT); do
+               local idx=$(($mdtno - 1))
+               wait_result mds${mdtno} \
+                       "$LCTL get_param -n $MDT_PREFIX${idx}.$key" "$target" \
+                       $timeout ||
+                       error "$key state is not '$target' on mds${mdtno}"
+       done
+}
+
 changelog_setup() {
-       CL_USER=$(do_facet $SINGLEMDS $LCTL --device $MDT0\
-                 changelog_register -n)
-       do_facet $SINGLEMDS lctl set_param mdd.$MDT0.changelog_mask="+hsm"
-       $LFS changelog_clear $MDT0 $CL_USER 0
+       CL_USERS=()
+       local mdtno
+       for mdtno in $(seq 1 $MDSCOUNT); do
+               local idx=$(($mdtno - 1))
+               local cl_user=$(do_facet mds${mdtno} $LCTL \
+                            --device ${MDT[$idx]} \
+                            changelog_register -n)
+               CL_USERS+=($cl_user)
+               do_facet mds${mdtno} lctl set_param \
+                       mdd.${MDT[$idx]}.changelog_mask="+hsm"
+               $LFS changelog_clear ${MDT[$idx]} $cl_user 0
+       done
 }
 
 changelog_cleanup() {
-#      $LFS changelog $MDT0
-       [[ -n "$CL_USER" ]] || return 0
-
-       $LFS changelog_clear $MDT0 $CL_USER 0
-       do_facet $SINGLEMDS lctl --device $MDT0 changelog_deregister $CL_USER
-       CL_USER=
+       local mdtno
+       for mdtno in $(seq 1 $MDSCOUNT); do
+               local idx=$(($mdtno - 1))
+               [[ -z  ${CL_USERS[$idx]} ]] && continue
+               $LFS changelog_clear ${MDT[$idx]} ${CL_USERS[$idx]} 0
+               do_facet mds${mdtno} lctl --device ${MDT[$idx]} \
+                       changelog_deregister ${CL_USERS[$idx]}
+       done
+       CL_USERS=()
 }
 
 changelog_get_flags() {
@@ -261,64 +319,57 @@ set_hsm_param() {
        local param=$1
        local value=$2
        local opt=$3
-       if [[ "$value" != "" ]]; then
-               value="=$value"
-       fi
-       do_facet $SINGLEMDS $LCTL set_param $opt -n $HSM_PARAM.$param$value
+       mdts_set_param "$opt -n" "hsm.$param" "$value"
        return $?
 }
 
 set_test_state() {
        local cmd=$1
        local target=$2
-       do_facet $SINGLEMDS $LCTL set_param $MDT_PARAM.hsm_control=$cmd
-       wait_result $SINGLEMDS "$LCTL get_param -n $MDT_PARAM.hsm_control"\
-               $target 10 || error "cdt state is not $target"
+       mdts_set_param "" hsm_control "$cmd"
+       mdts_check_param hsm_control "$target" 10
 }
 
 cdt_set_sanity_policy() {
        if [[ "$CDT_POLICY_HAD_CHANGED" ]]
        then
                # clear all
-               do_facet $SINGLEMDS $LCTL set_param $HSM_PARAM.policy=+NRA
-               do_facet $SINGLEMDS $LCTL set_param $HSM_PARAM.policy=-NBR
+               mdts_set_param "" hsm.policy "+NRA"
+               mdts_set_param "" hsm.policy "-NBR"
                CDT_POLICY_HAD_CHANGED=
        fi
 }
 
 cdt_set_no_retry() {
-       do_facet $SINGLEMDS $LCTL set_param $HSM_PARAM.policy=+NRA
+       mdts_set_param "" hsm.policy "+NRA"
        CDT_POLICY_HAD_CHANGED=true
 }
 
 cdt_clear_no_retry() {
-       do_facet $SINGLEMDS $LCTL set_param $HSM_PARAM.policy=-NRA
+       mdts_set_param "" hsm.policy "-NRA"
        CDT_POLICY_HAD_CHANGED=true
 }
 
 cdt_set_non_blocking_restore() {
-       do_facet $SINGLEMDS $LCTL set_param $HSM_PARAM.policy=+NBR
+       mdts_set_param "" hsm.policy "+NBR"
        CDT_POLICY_HAD_CHANGED=true
 }
 
 cdt_clear_non_blocking_restore() {
-       do_facet $SINGLEMDS $LCTL set_param $HSM_PARAM.policy=-NBR
+       mdts_set_param "" hsm.policy "-NBR"
        CDT_POLICY_HAD_CHANGED=true
 }
 
 cdt_clear_mount_state() {
-       do_facet $SINGLEMDS $LCTL set_param -d -P $MDT_PARAM.hsm_control
+       mdts_set_param "-P -d" hsm_control ""
 }
 
 cdt_set_mount_state() {
-       do_facet $SINGLEMDS $LCTL set_param -P $MDT_PARAM.hsm_control=$1
+       mdts_set_param "-P" hsm_control "$1"
 }
 
 cdt_check_state() {
-       local target=$1
-       wait_result $SINGLEMDS\
-               "$LCTL get_param -n $MDT_PARAM.hsm_control" "$target" 20 ||
-                       error "cdt state is not $target"
+       mdts_check_param hsm_control "$1" 20
 }
 
 cdt_disable() {
@@ -528,8 +579,8 @@ wait_for_grace_delay() {
        sleep $val
 }
 
-MDT0=$($LCTL get_param -n mdc.*.mds_server_uuid |
-       awk '{gsub(/_UUID/,""); print $1}' | head -1)
+# populate MDT device array
+get_mdt_devices
 
 # initiate variables
 init_agt_vars
@@ -2826,7 +2877,7 @@ test_220() {
        $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $f
        wait_request_state $fid ARCHIVE SUCCEED
 
-       local flags=$(changelog_get_flags $MDT0 HSM $fid | tail -1)
+       local flags=$(changelog_get_flags ${MDT[0]} HSM $fid | tail -1)
        changelog_cleanup
 
        local target=0x0
@@ -2853,7 +2904,7 @@ test_221() {
        wait_request_state $fid ARCHIVE CANCELED
        wait_request_state $fid CANCEL SUCCEED
 
-       local flags=$(changelog_get_flags $MDT0 HSM $fid | tail -1)
+       local flags=$(changelog_get_flags ${MDT[0]} HSM $fid | tail -1)
 
        local target=0x7d
        [[ $flags == $target ]] || error "Changelog flag is $flags not $target"
@@ -2878,7 +2929,7 @@ test_222a() {
        $LFS hsm_restore $f
        wait_request_state $fid RESTORE SUCCEED
 
-       local flags=$(changelog_get_flags $MDT0 HSM $fid | tail -1)
+       local flags=$(changelog_get_flags ${MDT[0]} HSM $fid | tail -1)
 
        local target=0x80
        [[ $flags == $target ]] || error "Changelog flag is $flags not $target"
@@ -2904,7 +2955,7 @@ test_222b() {
 
        wait_request_state $fid RESTORE SUCCEED
 
-       local flags=$(changelog_get_flags $MDT0 HSM $fid | tail -1)
+       local flags=$(changelog_get_flags ${MDT[0]} HSM $fid | tail -1)
 
        local target=0x80
        [[ $flags == $target ]] || error "Changelog flag is $flags not $target"
@@ -2933,7 +2984,7 @@ test_223a() {
        wait_request_state $fid RESTORE CANCELED
        wait_request_state $fid CANCEL SUCCEED
 
-       local flags=$(changelog_get_flags $MDT0 HSM $fid | tail -1)
+       local flags=$(changelog_get_flags ${MDT[0]} HSM $fid | tail -1)
 
        local target=0xfd
        [[ $flags == $target ]] ||
@@ -2962,7 +3013,7 @@ test_223b() {
        wait_request_state $fid RESTORE CANCELED
        wait_request_state $fid CANCEL SUCCEED
 
-       local flags=$(changelog_get_flags $MDT0 HSM $fid | tail -1)
+       local flags=$(changelog_get_flags ${MDT[0]} HSM $fid | tail -1)
 
        local target=0xfd
        [[ $flags == $target ]] ||
@@ -2988,7 +3039,7 @@ test_224() {
        $LFS hsm_remove $f
        wait_request_state $fid REMOVE SUCCEED
 
-       local flags=$(changelog_get_flags $MDT0 HSM $fid | tail -1)
+       local flags=$(changelog_get_flags ${MDT[0]} HSM $fid | tail -n 1)
 
        local target=0x200
        [[ $flags == $target ]] ||
@@ -3024,9 +3075,9 @@ test_225() {
        wait_request_state $fid REMOVE CANCELED
        wait_request_state $fid CANCEL SUCCEED
 
-       flags=$(changelog_get_flags $MDT0 RENME $fid2)
-       local flags=$($LFS changelog $MDT0 | grep HSM | grep $fid | tail -1 |
-               awk '{print $5}')
+       flags=$(changelog_get_flags ${MDT[0]} RENME $fid2)
+       local flags=$($LFS changelog ${MDT[0]} | grep HSM | grep $fid |
+               tail -n 1 | awk '{print $5}')
 
        local target=0x27d
        [[ $flags == $target ]] ||
@@ -3058,7 +3109,7 @@ test_226() {
 
        rm $f1 || error "rm $f1 failed"
 
-       local flags=$(changelog_get_flags $MDT0 UNLNK $fid1)
+       local flags=$(changelog_get_flags ${MDT[0]} UNLNK $fid1)
 
        local target=0x3
        [[ $flags == $target ]] ||
@@ -3066,7 +3117,7 @@ test_226() {
 
        mv $f3 $f2 || error "mv $f3 $f2 failed"
 
-       flags=$(changelog_get_flags $MDT0 RENME $fid2)
+       flags=$(changelog_get_flags ${MDT[0]} RENME $fid2)
 
        target=0x3
        [[ $flags == $target ]] ||
@@ -3086,7 +3137,7 @@ check_flags_changes() {
        local target=0x280
        $LFS hsm_set --$hsm_flag $f ||
                error "Cannot set $hsm_flag on $f"
-       local flags=($(changelog_get_flags $MDT0 HSM $fid))
+       local flags=($(changelog_get_flags ${MDT[0]} HSM $fid))
        local seen=${#flags[*]}
        cnt=$((fst + cnt))
        [[ $seen == $cnt ]] ||
@@ -3097,7 +3148,7 @@ check_flags_changes() {
 
        $LFS hsm_clear --$hsm_flag $f ||
                error "Cannot clear $hsm_flag on $f"
-       flags=($(changelog_get_flags $MDT0 HSM $fid))
+       flags=($(changelog_get_flags ${MDT[0]} HSM $fid))
        seen=${#flags[*]}
        cnt=$(($cnt + 1))
        [[ $cnt == $seen ]] ||