From: Elena Gryaznova Date: Thu, 11 Oct 2018 14:20:32 +0000 (+0300) Subject: LU-9272 tests: fix for facet_failover mgs X-Git-Tag: 2.14.57~68 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=36475ba5fc;hp=67e1017da0e5c71367336ccb77c4cb6f1fbc0e88 LU-9272 tests: fix for facet_failover mgs Have facet_failover to work for mgs facet and include failover nodes in the list of nodes to load modules. When mgs/mds are combined, assign the failover host of mds to the failover host of mgs. Assign mgsfailover_dev with mds1failover_dev when mgs/mds are combined while mounting facets, as mds1failover_dev is already defined when mgs/mds are combined. Fix start() to export mgs_dev and mgsfailover_dev for combined_mds_mgs. Do not wait recovery complete on mgs. Test-Parameters: trivial failover=true osscount=2 mdscount=2 mdtcount=1 austeroptions=-R iscsi=1 env="ONLY=121" testlist=conf-sanity Test-Parameters: testlist=conf-sanity Change-Id: Ie698814c530c8deb98aa0010f2a0fa8e261b4b69 HPE-bug-id: MRP-3374, LUS-4858, LUS-2361 Signed-off-by: Elena Gryaznova Signed-off-by: Alexander Boyko Signed-off-by: Noopur Maheshwari Signed-off-by: Vladimir Saveliev Signed-off-by: Alexander Zarochentsev Reviewed-on: https://review.whamcloud.com/26235 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Elena Gryaznova Reviewed-by: Oleg Drokin --- diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index a45468a..a95399f 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -8672,6 +8672,18 @@ test_120() { # LU-11130 } run_test 120 "cross-target rename should not create bad symlinks" +test_121(){ + stopall + start_mgsmds || error "MGS MDS Start failed" + fail mgs + stop_mds || error "Stopping MDSes failed" + #failback + start_mds + fail mgs + stop_mds || error "Stopping MDSes failed" +} +run_test 121 "failover MGS" + test_122a() { [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" [[ "$OST1_VERSION" -ge $(version_code 2.11.53) ]] || @@ -9029,10 +9041,7 @@ test_123F() { echo "Writeconf" writeconf_all echo "Remounting" - mountmgs - mountmds - mountoss - mountcli + setup_noconfig # Reapply the config from before echo "Setting configuration parameters" diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index ddfcf85..6020b10 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -817,12 +817,24 @@ load_modules_local() { } load_modules () { + local facets + local facet + local failover load_modules_local # bug 19124 # load modules on remote nodes optionally # lustre-tests have to be installed on these nodes if $LOAD_MODULES_REMOTE; then local list=$(comma_list $(remote_nodes_list)) + + # include failover nodes in case they are not in the list yet + facets=$(get_facets) + for facet in ${facets//,/ }; do + failover=$(facet_failover_host $facet) + [ -n "$list" ] && [[ ! "$list" =~ "$failover" ]] && + list="$list,$failover" + done + if [ -n "$list" ]; then echo "loading modules on: '$list'" do_rpc_nodes "$list" load_modules_local @@ -2048,6 +2060,9 @@ mount_facet() { local devicelabel local dm_dev=${!dev} + [[ $dev == "mgsfailover_dev" ]] && combined_mgs_mds && + dev=mds1failover_dev + module_loaded lustre || load_modules case $fstype in @@ -2140,11 +2155,17 @@ start() { eval export ${dev_alias}_dev=${device} eval export ${facet}_opt=\"$@\" + combined_mgs_mds && [[ ${dev_alias} == mds1 ]] && + eval export mgs_dev=${device} + local varname=${dev_alias}failover_dev if [ -n "${!varname}" ] ; then eval export ${dev_alias}failover_dev=${!varname} else eval export ${dev_alias}failover_dev=$device + combined_mgs_mds && [[ ${dev_alias} == mds1 ]] && + eval export mgsfailover_dev=${device} + fi local mntpt=$(facet_mntpt $facet) @@ -3377,8 +3398,11 @@ wait_recovery_complete () { fi echo affected facets: $facets - # we can use "for" here because we are waiting the slowest - for facet in ${facets//,/ }; do + facets=${facets//,/ } + # We can use "for" here because we are waiting the slowest. + # The mgs not having the recovery_status proc entry, exclude it + # from the facet list. + for facet in ${facets//mgs/ }; do local var_svc=${facet}_svc local param="*.${!var_svc}.recovery_status" @@ -3677,11 +3701,14 @@ facet_failover() { if ! combined_mgs_mds && list_member ${affecteds[index]} mgs; then mount_facet mgs || error "Restart of mgs failed" + affecteds[index]=$(exclude_items_from_list \ + ${affecteds[index]} mgs) fi # FIXME; has to be changed to mount all facets concurrently - affected=$(exclude_items_from_list ${affecteds[index]} mgs) - echo mount facets: ${affecteds[index]} - mount_facets ${affecteds[index]} + if [ -n "${affecteds[index]}" ]; then + echo mount facets: ${affecteds[index]} + mount_facets ${affecteds[index]} + fi if $GSS_SK; then do_nodes $(comma_list $(all_nodes)) \ "keyctl show | grep lustre | cut -c1-11 | @@ -3965,7 +3992,14 @@ facet_host() { elif [ "${facet:0:3}" == "mdt" -o \ "${facet:0:3}" == "mds" -o \ "${facet:0:3}" == "mgs" ]; then - eval export ${facet}_HOST=${mds_HOST} + local temp + if [ "${facet}" == "mgsfailover" ] && + [ -n "$mds1failover_HOST" ]; then + temp=$mds1failover_HOST + else + temp=${mds_HOST} + fi + eval export ${facet}_HOST=$temp fi fi echo -n ${!varname} @@ -3981,6 +4015,13 @@ facet_failover_host() { return fi + if combined_mgs_mds && [ $facet == "mgs" ] && + [ -z $mds1failover_HOST ]; then + temp=mds1failover_HOST + echo ${!temp} + return + fi + if [ "${facet:0:3}" == "mdt" -o "${facet:0:3}" == "mds" -o \ "${facet:0:3}" == "mgs" ]; then @@ -4038,8 +4079,6 @@ change_active() { local facetlist=$1 local facet - facetlist=$(exclude_items_from_list $facetlist mgs) - for facet in ${facetlist//,/ }; do local failover=${facet}failover local host=`facet_host $failover` @@ -5156,7 +5195,14 @@ init_facet_vars () { local varname=${facet}failover_HOST if [ -z "${!varname}" ]; then - eval export $varname=$(facet_host $facet) + local temp + if combined_mgs_mds && [ $facet == "mgs" ] && + [ -n "$mds1failover_HOST" ]; then + temp=$mds1failover_HOST + else + temp=$(facet_host $facet) + fi + eval export $varname=$temp fi varname=${facet}_HOST @@ -5189,12 +5235,12 @@ init_facets_vars () { if ! remote_mds_nodsh; then for num in $(seq $MDSCOUNT); do - DEVNAME=`mdsdevname $num` + DEVNAME=$(mdsdevname $num) init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS done fi - combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS + init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS if ! remote_ost_nodsh; then for num in $(seq $OSTCOUNT); do