From 8b5254fbabfeeaceb82f664ee06d289cdc7b13ff Mon Sep 17 00:00:00 2001 From: Elena Gryaznova Date: Fri, 27 Sep 2024 15:13:27 +0300 Subject: [PATCH] LU-11073 tests: enable DNE in recovery-mds-scale In recovery-mds-scale.sh, recovery-random-scale.sh, recovery-double-scale.sh add environmental variables RECOVERY_SCALE_ENABLE_REMOTE_DIRS, RECOVERY_SCALE_ENABLE_STRIPED_DIRS to enable or disable the corresponding operations. RECOVERY_SCALE_ENABLE_REMOTE_DIRS defaults to true if the MDS version is at least 2.5, while RECOVERY_SCALE_ENABLE_STRIPED_DIRS defaults to true if the MDS version is at least 2.8. Patch also: simplifies the run_*.sh logic: it removes the never used BREAK_ON_ERROR and useless CONTINUE parameters; adds the possibility to set the directory striping via client_load_SETSTRIPEPARAMS; ignores failures caused by ENOSPC error: instead of adding the complex free space calculation logic for various layouts let's allow the loads to fail with ENOSPC. This does not affect fofb testing the recovery scale tests are for. forces lustre-rsync-test test_4 to run iozone on not striped directory with offset=0 Test-Parameters: trivial env=SLOW=no,FAILURE_MODE=HARD,RACER_ENABLE_REMOTE_DIRS=true,RACER_ENABLE_STRIPED_DIRS=true clientcount=4 mdtcount=1 mdscount=2 osscount=2 austeroptions=-R failover=true iscsi=1 testlist=recovery-mds-scale,recovery-oss-scale,recovery-random-scale,lustre-rsync-test,racer Signed-off-by: Elena Gryaznova Signed-off-by: Andriy Skulysh HPE-bug-id: LUS-5973, LUS-6386, LUS-7540, LUS-9399 Reviewed-by: Vladimir Saveliev Reviewed-by: Alexander Boyko Reviewed-by: Alexander Zarochentsev Change-Id: Icd42ae9b4e5ac403ba76a9e3909616977dbd6a72 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/32626 Tested-by: Maloo Tested-by: jenkins Reviewed-by: Shaun Tancheff Reviewed-by: Oleg Drokin --- lustre/tests/functions.sh | 43 +++++++++++++++++++++++++++++++++-- lustre/tests/lustre-rsync-test.sh | 3 ++- lustre/tests/parallel-scale.sh | 3 +++ lustre/tests/racer.sh | 9 +++----- lustre/tests/recovery-double-scale.sh | 5 ++++ lustre/tests/recovery-mds-scale.sh | 3 +++ lustre/tests/recovery-random-scale.sh | 12 ++++++++-- lustre/tests/run_IOR.sh | 29 +++++++++++++++-------- lustre/tests/run_dbench.sh | 24 ++++++++++--------- lustre/tests/run_dd.sh | 23 ++++++++++--------- lustre/tests/run_iozone.sh | 28 +++++++++++------------ lustre/tests/run_tar.sh | 29 ++++++++++------------- lustre/tests/test-framework.sh | 31 ++++++++++++++++++++++--- 13 files changed, 164 insertions(+), 78 deletions(-) diff --git a/lustre/tests/functions.sh b/lustre/tests/functions.sh index 7d0ecf9..1e8d9ad 100644 --- a/lustre/tests/functions.sh +++ b/lustre/tests/functions.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Simple function used by run_*.sh scripts +# functions used by other scripts assert_env() { local failed="" @@ -669,7 +669,7 @@ run_ior() { print_opts IOR ior_THREADS ior_DURATION MACHINEFILE - test_mkdir -p $testdir + client_load_mkdir $testdir # mpi_run uses mpiuser chmod 0777 $testdir @@ -1399,3 +1399,42 @@ run_xdd() { rm -rf $testdir } + +client_load_mkdir () { + local dir=$1 + local parent=$(dirname $dir) + + local mdtcount=$($LFS df $parent 2> /dev/null | grep -c MDT) + if [ $mdtcount -le 1 ] || ! is_lustre ${parent}; then + mkdir $dir || return 1 + return 0 + else + mdt_idx=$((RANDOM % mdtcount)) + if $RECOVERY_SCALE_ENABLE_STRIPED_DIRS; then + # stripe_count in range [1,mdtcount] + # $LFS mkdir treats stripe_count 0 and 1 the same + stripe_count_opt="-c$((RANDOM % mdtcount + 1))" + else + stripe_count_opt="" + fi + fi + + if $RECOVERY_SCALE_ENABLE_REMOTE_DIRS || + $RECOVERY_SCALE_ENABLE_STRIPED_DIRS; then + $LFS mkdir -i$mdt_idx $stripe_count_opt $dir || + return 1 + else + mkdir $dir || return 1 + fi + $LFS getdirstripe $dir || return 1 + + if [ -n "$client_load_SETSTRIPEPARAMS" ]; then + $LFS setstripe $client_load_SETSTRIPEPARAMS $dir || + return 1 + fi + $LFS getstripe $dir || return 1 +} + +enospc_detected () { + grep "No space left on device" $1 | grep -qv grep +} diff --git a/lustre/tests/lustre-rsync-test.sh b/lustre/tests/lustre-rsync-test.sh index 9e5720d..1ba4237 100755 --- a/lustre/tests/lustre-rsync-test.sh +++ b/lustre/tests/lustre-rsync-test.sh @@ -455,7 +455,8 @@ test_4() { init_changelog END_RUN_FILE=${DIR}/$tdir/run LOAD_PID_FILE=${DIR}/$tdir/pid \ - MOUNT=${DIR}/$tdir run_iozone.sh & + MOUNT=${DIR}/$tdir RECOVERY_SCALE_ENABLE_REMOTE_DIRS=false \ + RECOVERY_SCALE_ENABLE_STRIPED_DIRS=false run_iozone.sh & sleep 30 child_pid=$(pgrep iozone) stop_procs $child_pid diff --git a/lustre/tests/parallel-scale.sh b/lustre/tests/parallel-scale.sh index f126748..8ba305c 100644 --- a/lustre/tests/parallel-scale.sh +++ b/lustre/tests/parallel-scale.sh @@ -5,6 +5,9 @@ LUSTRE=${LUSTRE:-$(dirname $0)/..} init_test_env "$@" init_logging +init_stripe_dir_params RECOVERY_SCALE_ENABLE_REMOTE_DIRS \ + RECOVERY_SCALE_ENABLE_STRIPED_DIRS + ALWAYS_EXCEPT="$PARALLEL_SCALE_EXCEPT " # bug number for skipped test: LU-9429 ALWAYS_EXCEPT+=" parallel_grouplock " diff --git a/lustre/tests/racer.sh b/lustre/tests/racer.sh index 9f1d3c9..724f976 100755 --- a/lustre/tests/racer.sh +++ b/lustre/tests/racer.sh @@ -50,11 +50,10 @@ if $RACER_FAILOVER; then echo Victim facets "${victims[@]}" fi +init_stripe_dir_params RACER_ENABLE_REMOTE_DIRS \ + RACER_ENABLE_STRIPED_DIRS + if ((MDSCOUNT > 1)); then - (( $MDS1_VERSION >= $(version_code 2.5.0) )) && - RACER_ENABLE_REMOTE_DIRS=${RACER_ENABLE_REMOTE_DIRS:-true} - (( $MDS1_VERSION >= $(version_code 2.8.0) )) && - RACER_ENABLE_STRIPED_DIRS=${RACER_ENABLE_STRIPED_DIRS:-true} (( $MDS1_VERSION >= $(version_code 2.13.57) )) && RACER_ENABLE_MIGRATION=${RACER_ENABLE_MIGRATION:-true} (( $MDS1_VERSION >= $(version_code 2.15.55.45) )) && @@ -80,8 +79,6 @@ fi RACER_ENABLE_FALLOCATE=false check_set_fallocate || RACER_ENABLE_FALLOCATE=false -RACER_ENABLE_REMOTE_DIRS=${RACER_ENABLE_REMOTE_DIRS:-false} -RACER_ENABLE_STRIPED_DIRS=${RACER_ENABLE_STRIPED_DIRS:-false} RACER_ENABLE_MIGRATION=${RACER_ENABLE_MIGRATION:-false} RACER_ENABLE_SNAPSHOT=${RACER_ENABLE_SNAPSHOT:-true} RACER_ENABLE_FILE_MIGRATE=${RACER_ENABLE_FILE_MIGRATE:-true} diff --git a/lustre/tests/recovery-double-scale.sh b/lustre/tests/recovery-double-scale.sh index 021431d..1398493 100644 --- a/lustre/tests/recovery-double-scale.sh +++ b/lustre/tests/recovery-double-scale.sh @@ -49,6 +49,9 @@ FAILOVER_PERIOD=${FAILOVER_PERIOD:-$((60 * 5))} # 5 minutes END_RUN_FILE=${END_RUN_FILE:-$SHARED_DIRECTORY/end_run_file} LOAD_PID_FILE=${LOAD_PID_FILE:-$TMP/client-load.pid} +init_stripe_dir_params RECOVERY_SCALE_ENABLE_REMOTE_DIRS \ + RECOVERY_SCALE_ENABLE_STRIPED_DIRS + reboot_recover_node () { # item var contains a pair of clients if nodetype=clients # I would prefer to have a list here @@ -214,7 +217,9 @@ check_timeout || exit 1 # -- remove hostname from clients list zconf_umount $HOSTNAME $MOUNT NODES_TO_USE=${NODES_TO_USE:-$CLIENTS} +log "Using NODES_TO_USE: $NODES_TO_USE and CLIENTS: $CLIENTS" NODES_TO_USE=$(exclude_items_from_list $NODES_TO_USE $HOSTNAME) +log "Using remote NODES_TO_USE: $NODES_TO_USE HOSTNAME=$HOSTNAME" check_progs_installed $NODES_TO_USE "${CLIENT_LOADS[@]}" diff --git a/lustre/tests/recovery-mds-scale.sh b/lustre/tests/recovery-mds-scale.sh index 7b68a14..a4efa1c 100644 --- a/lustre/tests/recovery-mds-scale.sh +++ b/lustre/tests/recovery-mds-scale.sh @@ -35,6 +35,9 @@ fi ERRORS_OK="" # No application failures should occur during this test. +init_stripe_dir_params RECOVERY_SCALE_ENABLE_REMOTE_DIRS \ + RECOVERY_SCALE_ENABLE_STRIPED_DIRS + check_and_setup_lustre rm -rf $DIR/[Rdfs][0-9]* diff --git a/lustre/tests/recovery-random-scale.sh b/lustre/tests/recovery-random-scale.sh index b07e420..8b98937 100644 --- a/lustre/tests/recovery-random-scale.sh +++ b/lustre/tests/recovery-random-scale.sh @@ -45,6 +45,9 @@ fi # but not for other clients. ERRORS_OK="yes" +init_stripe_dir_params RECOVERY_SCALE_ENABLE_REMOTE_DIRS \ + RECOVERY_SCALE_ENABLE_STRIPED_DIRS + numfailovers () { local facet local var @@ -152,8 +155,13 @@ test_fail_client_mds() { if [ -e $END_RUN_FILE ]; then local end_run_node read end_run_node < $END_RUN_FILE - [[ $end_run_node = $fail_client ]] && - rm -f $END_RUN_FILE || exit 13 + if [[ $end_run_node = $fail_client ]]; then + rm -f $END_RUN_FILE + else + echo "failure is expected on FAIL CLIENT \ + $fail_client, not on $end_run_node" + exit 13 + fi fi restart_client_loads $fail_client $ERRORS_OK || exit $? diff --git a/lustre/tests/run_IOR.sh b/lustre/tests/run_IOR.sh index 98379f5..c4ccc1a 100755 --- a/lustre/tests/run_IOR.sh +++ b/lustre/tests/run_IOR.sh @@ -9,6 +9,11 @@ TESTNAME=${TESTNAME:-""} LOG=$TESTLOG_PREFIX.$(basename $0 .sh)_stdout.$(hostname -s).log DEBUGLOG=$(echo $LOG | sed 's/\(.*\)stdout/\1debug/') +error () { + echo "$@" + exit 17 +} + mkdir -p ${LOG%/*} rm -f $LOG $DEBUGLOG @@ -37,33 +42,37 @@ echo $$ >$LOAD_PID_FILE TESTDIR=${TESTDIR:-$MOUNT/d0.ior-$(hostname)} -CONTINUE=true -while [ ! -e "$END_RUN_FILE" ] && $CONTINUE; do +while [ ! -e "$END_RUN_FILE" ]; do echoerr "$(date +'%F %H:%M:%S'): IOR run starting" - mkdir -p $TESTDIR + rm -rf $TESTDIR + client_load_mkdir $TESTDIR + if [ $? -ne 0 ]; then + echoerr "$(date +'%F %H:%M:%S'): failed to create $TESTDIR" + echo $(hostname) >> $END_RUN_FILE + break + fi + # need this only if TESTDIR is not default chmod -R 777 $TESTDIR sync - run_ior fpp $TESTDIR 1>$LOG & + run_ior fpp $TESTDIR 1>>$LOG & load_pid=$! wait $load_pid if [ ${PIPESTATUS[0]} -eq 0 ]; then echoerr "$(date +'%F %H:%M:%S'): IOR succeeded" cd $TMP - rm -rf $TESTDIR - echoerr "$(date +'%F %H:%M:%S'): IOR run finished" else + enospc_detected $DEBUGLOG && + echoerr "$(date +'%F %H:%M:%S'): IOR ENOSPC, ignored" && + continue + echoerr "$(date +'%F %H:%M:%S'): IOR failed" if [ -z "$ERRORS_OK" ]; then echo $(hostname) >> $END_RUN_FILE fi - if [ $BREAK_ON_ERROR ]; then - # break - CONTINUE=false - fi fi done diff --git a/lustre/tests/run_dbench.sh b/lustre/tests/run_dbench.sh index 1872967..53e6ea6 100755 --- a/lustre/tests/run_dbench.sh +++ b/lustre/tests/run_dbench.sh @@ -26,12 +26,16 @@ echo $$ >$LOAD_PID_FILE TESTDIR=$MOUNT/d0.dbench-$(hostname) -CONTINUE=true - -while [ ! -e "$END_RUN_FILE" ] && $CONTINUE; do +while [ ! -e "$END_RUN_FILE" ]; do echoerr "$(date +'%F %H:%M:%S'): dbench run starting" - mkdir -p $TESTDIR + rm -rf $TESTDIR + client_load_mkdir $TESTDIR + if [ $? -ne 0 ]; then + echoerr "$(date +'%F %H:%M:%S'): failed to create $TESTDIR" + echo $(hostname) >> $END_RUN_FILE + break + fi sync rundbench -D $TESTDIR 2 1>$LOG & @@ -41,18 +45,16 @@ while [ ! -e "$END_RUN_FILE" ] && $CONTINUE; do if [ ${PIPESTATUS[0]} -eq 0 ]; then echoerr "$(date +'%F %H:%M:%S'): dbench succeeded" cd $TMP - rm -rf $TESTDIR - echoerr "$(date +'%F %H:%M:%S'): dbench run finished" else + enospc_detected $DEBUGLOG && + echoerr "$(date +'%F %H:%M:%S'):"\ + "dbench ENOSPC, ignored" && + continue + echoerr "$(date +'%F %H:%M:%S'): dbench failed" if [ -z "$ERRORS_OK" ]; then echo $(hostname) >> $END_RUN_FILE fi - - if [ $BREAK_ON_ERROR ]; then - # break - CONTINUE=false - fi fi done diff --git a/lustre/tests/run_dd.sh b/lustre/tests/run_dd.sh index 1ec14ec..e369285 100755 --- a/lustre/tests/run_dd.sh +++ b/lustre/tests/run_dd.sh @@ -26,18 +26,22 @@ echo $$ >$LOAD_PID_FILE TESTDIR=$MOUNT/d0.dd-$(hostname) -CONTINUE=true -while [ ! -e "$END_RUN_FILE" ] && $CONTINUE; do +while [ ! -e "$END_RUN_FILE" ]; do echoerr "$(date +'%F %H:%M:%S'): dd run starting" - mkdir -p $TESTDIR - $LFS setstripe -c -1 $TESTDIR + rm -rf $TESTDIR + client_load_mkdir $TESTDIR + if [ $? -ne 0 ]; then + echoerr "$(date +'%F %H:%M:%S'): failed to create $TESTDIR" + echo $(hostname) >> $END_RUN_FILE + break + fi cd $TESTDIR sync # suppress dd xfer stat to workaround buggy coreutils/gettext # combination in RHEL5 and OEL5, see BZ 21264 FREE_SPACE=$(df -P $TESTDIR | awk '/:/ { print $4 }') - BLKS=$((FREE_SPACE * 9 / 40 / CLIENT_COUNT)) + BLKS=$((FREE_SPACE / 4 / CLIENT_COUNT)) echoerr "Total free disk space is $FREE_SPACE, 4k blocks to dd is $BLKS" df $TESTDIR || true @@ -46,17 +50,14 @@ while [ ! -e "$END_RUN_FILE" ] && $CONTINUE; do if [ $? -eq 0 ]; then echoerr "$(date +'%F %H:%M:%S'): dd succeeded" cd $TMP - rm -rf $TESTDIR - echoerr "$(date +'%F %H:%M:%S'): dd run finished" else + enospc_detected $DEBUGLOG && + echoerr "$(date +'%F %H:%M:%S'): dd ENOSPC, ignored" && + continue echoerr "$(date +'%F %H:%M:%S'): dd failed" if [ -z "$ERRORS_OK" ]; then echo $(hostname) >> $END_RUN_FILE fi - if [ $BREAK_ON_ERROR ]; then - # break - CONTINUE=false - fi fi done diff --git a/lustre/tests/run_iozone.sh b/lustre/tests/run_iozone.sh index 6bd7dd4..4f56958 100755 --- a/lustre/tests/run_iozone.sh +++ b/lustre/tests/run_iozone.sh @@ -26,10 +26,15 @@ echo $$ >$LOAD_PID_FILE TESTDIR=$MOUNT/d0.iozone-$(hostname) -CONTINUE=true -while [ ! -e "$END_RUN_FILE" ] && $CONTINUE; do +while [ ! -e "$END_RUN_FILE" ]; do echoerr "$(date +'%F %H:%M:%S'): iozone run starting" - mkdir -p $TESTDIR + rm -rf $TESTDIR + client_load_mkdir $TESTDIR + if [ $? -ne 0 ]; then + echoerr "$(date +'%F %H:%M:%S'): failed to create $TESTDIR" + echo $(hostname) >> $END_RUN_FILE + break + fi cd $TESTDIR sync @@ -40,23 +45,16 @@ while [ ! -e "$END_RUN_FILE" ] && $CONTINUE; do if [ ${PIPESTATUS[0]} -eq 0 ]; then echoerr "$(date +'%F %H:%M:%S'): iozone succeeded" cd $TMP - rm -rf $TESTDIR - if [ -d $TESTDIR ]; then - echoerr "$(date +'%F %H:%M:%S'): failed to remove \ - $TESTDIR" - echo $(hostname) >> $END_RUN_FILE - CONTINUE=false - fi - echoerr "$(date +'%F %H:%M:%S'): iozone run finished" else + enospc_detected $DEBUGLOG && + echoerr "$(date +'%F %H:%M:%S'):"\ + "iozone ENOSPC, ignored" && + continue + echoerr "$(date +'%F %H:%M:%S'): iozone failed" if [ -z "$ERRORS_OK" ]; then echo $(hostname) >> $END_RUN_FILE fi - if [ $BREAK_ON_ERROR ]; then - # break - CONTINUE=false - fi fi done diff --git a/lustre/tests/run_tar.sh b/lustre/tests/run_tar.sh index 9748949..19d333c 100755 --- a/lustre/tests/run_tar.sh +++ b/lustre/tests/run_tar.sh @@ -31,24 +31,21 @@ do_tar() { return ${PIPESTATUS[1]} } -CONTINUE=true -while [ ! -e "$END_RUN_FILE" ] && $CONTINUE; do +while [ ! -e "$END_RUN_FILE" ]; do echoerr "$(date +'%F %H:%M:%S'): tar run starting" - mkdir -p $TESTDIR + rm -rf $TESTDIR + client_load_mkdir $TESTDIR + if [ $? -ne 0 ]; then + echoerr "$(date +'%F %H:%M:%S'): failed to create $TESTDIR" + echo $(hostname) >> $END_RUN_FILE + break + fi cd $TESTDIR sync - USAGE=$(du -s /etc | awk '{print $1}') $LCTL set_param llite.*.lazystatfs=0 df $TESTDIR || true sleep 2 - FREE_SPACE=$(df $TESTDIR | awk '/:/ { print $4 }') - AVAIL=$((FREE_SPACE * 9 / 10 / CLIENT_COUNT)) - if [ $AVAIL -lt $USAGE ]; then - echoerr "no enough free disk space: need $USAGE, avail $AVAIL" - echo $(hostname) >> $END_RUN_FILE - break - fi do_tar & wait $! RC=$? @@ -60,17 +57,15 @@ while [ ! -e "$END_RUN_FILE" ] && $CONTINUE; do if [ $RC -eq 0 ]; then echoerr "$(date +'%F %H:%M:%S'): tar succeeded" cd $TMP - rm -rf $TESTDIR & wait $! - echoerr "$(date +'%F %H:%M:%S'): tar run finished" else + enospc_detected $DEBUGLOG && + echoerr "$(date +'%F %H:%M:%S'): tar ENOSPC, ignored" && + continue + echoerr "$(date +'%F %H:%M:%S'): tar failed" if [ -z "$ERRORS_OK" ]; then echo $(hostname) >> $END_RUN_FILE fi - if [ $BREAK_ON_ERROR ]; then - # break - CONTINUE=false - fi fi done diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index d56c0ee..0d498e9 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -3252,7 +3252,6 @@ start_client_load() { eval export ${var}=$load do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \ - BREAK_ON_ERROR=$BREAK_ON_ERROR \ END_RUN_FILE=$END_RUN_FILE \ LOAD_PID_FILE=$LOAD_PID_FILE \ TESTLOG_PREFIX=$TESTLOG_PREFIX \ @@ -3260,9 +3259,12 @@ start_client_load() { DBENCH_LIB=$DBENCH_LIB \ DBENCH_SRC=$DBENCH_SRC \ CLIENT_COUNT=$((CLIENTCOUNT - 1)) \ + RECOVERY_SCALE_ENABLE_REMOTE_DIRS=$RECOVERY_SCALE_ENABLE_REMOTE_DIRS \ + RECOVERY_SCALE_ENABLE_STRIPED_DIRS=$RECOVERY_SCALE_ENABLE_STRIPED_DIRS \ LFS=$LFS \ LCTL=$LCTL \ FSNAME=$FSNAME \ + MPI_USER=$MPI_USER \ MPIRUN=$MPIRUN \ MPIRUN_OPTIONS=\\\"$MPIRUN_OPTIONS\\\" \ MACHINEFILE_OPTION=\\\"$MACHINEFILE_OPTION\\\" \ @@ -3805,7 +3807,7 @@ wait_for_host() { for host in ${hostlist//,/ }; do check_network "$host" 900 done - while ! do_nodes $hostlist hostname > /dev/null; do sleep 5; done + while ! do_nodes $hostlist hostname; do sleep 5; done } wait_for_facet() { @@ -4176,6 +4178,9 @@ facet_failover() { local host=$(facet_active_host $facet) hostlist=$(expand_list $hostlist $host) + local fhost=$(facet_host $facet) + local ffhost=$(facet_failover_host $facet) + echo "facet: $facet facet_host: $fhost facet_failover_host: $ffhost" if [ $(facet_host $facet) = \ $(facet_failover_host $facet) ]; then waithostlist=$(expand_list $waithostlist $host) @@ -4186,7 +4191,7 @@ facet_failover() { for host in ${hostlist//,/ }; do reboot_node $host done - echo "$(date +'%H:%M:%S (%s)') $hostlist rebooted" + echo "$(date +'%H:%M:%S (%s)') $hostlist rebooted; waithostlist: $waithostlist" # We need to wait the rebooted hosts in case if # facet_HOST == facetfailover_HOST if ! [ -z "$waithostlist" ]; then @@ -11289,6 +11294,26 @@ save_layout_restore_at_exit() { stack_trap "restore_layout $dir $layout" EXIT } +init_stripe_dir_params() { + local varremote=$1 + local varstriped=$2 + + if ((MDSCOUNT > 1 && + $MDS1_VERSION >= + $(version_code 2.8.0))); then + eval $varremote=${!varremote:-true} + eval $varstriped=${!varstriped:-true} + elif ((MDSCOUNT > 1 && + $MDS1_VERSION >= + $(version_code 2.5.0))); then + eval $varremote=${!varremote:-true} + eval $varstriped=${!varstriped:-false} + fi + + eval $varremote=${!varremote:-false} + eval $varstriped=${!varstriped:-false} +} + verify_yaml_layout() { local src=$1 local dst=$2 -- 1.8.3.1