LU-14272 tests: different mpirun options for different users

Patch adds the ability to use different mpirun options for
different mpi users, for example we use:
  --wdir=/home/quota15_1 for user quota15_1
and
  --wdir=/home/quota15_2 for user quota15_2.

Per mpi user option can be specified after ':' in ha_mpi_users:
Example:
  ha_mpi_users="quota15_1:--wdir=/home/quota15_1
                quota15_2:--wdir=/home/quota15_2
                quota15_3
                quota15_4:--wdir=/home/quota15_4"
Several options for user  can be set as:
  ha_mpi_users="user1:<option11>
                user1:<option12>
                user2:<option21>"
MPIRUN_OPTIONS will be applied for all users.

Patch adds the ability to execute the command before loads start.
Example: ha_precmd="for i in 1 2 3 4; do
                lfs quota -v -u quota15_$i $ha_test_dir; done"

Test-Parameters: trivial
Signed-off-by: Elena Gryaznova <elena.gryaznova@hpe.com>
HPE-bug-id: LUS-9069
Reviewed-by: Vladimir Saveliev <vladimir.saveliev@hpe.com>
Reviewed-by: Sergey Cheremencev <sergey.cheremencev@hpe.com>
Change-Id: I35feadcc856dbc3d6d6d5f2720ab49c6c111161b
Reviewed-on: https://review.whamcloud.com/41072
Tested-by: jenkins <devops@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/ha.sh b/lustre/tests/ha.sh
index c26689d..d18a26b 100755 (executable)
--- a/lustre/tests/ha.sh
+++ b/lustre/tests/ha.sh
@@ -220,6 +220,18 @@ declare     ha_mpi_instances=${ha_mpi_instances:-1}
 declare     ha_mpi_loads=${ha_mpi_loads="ior simul mdtest"}
 declare -a  ha_mpi_load_tags=($ha_mpi_loads)
 declare -a  ha_mpiusers=(${ha_mpi_users="mpiuser"})
 declare     ha_mpi_loads=${ha_mpi_loads="ior simul mdtest"}
 declare -a  ha_mpi_load_tags=($ha_mpi_loads)
 declare -a  ha_mpiusers=(${ha_mpi_users="mpiuser"})

+declare -a  ha_users
+declare -A  ha_mpiopts
+
+for ((i=0; i<${#ha_mpiusers[@]}; i++)); do
+       u=${ha_mpiusers[i]%%:*}
+       o=""
+       # user gets empty option if ha_mpi_users does not specify it explicitly
+       [[ ${ha_mpiusers[i]} =~ : ]] && o=${ha_mpiusers[i]##*:}
+       ha_users[i]=$u
+       ha_mpiopts[$u]+=" $o"
+done
+ha_users=(${!ha_mpiopts[@]})

 
 declare     ha_ior_params=${IORP:-'" -b $ior_blockSize -t 2m -w -W -T 1"'}
 declare     ha_simul_params=${SIMULP:-'" -n 10"'}
 
 declare     ha_ior_params=${IORP:-'" -b $ior_blockSize -t 2m -w -W -T 1"'}
 declare     ha_simul_params=${SIMULP:-'" -n 10"'}


@@ -337,8 +349,8 @@ ha_on()
        # -S is to be used here to track the
        # remote command return values
        #
        # -S is to be used here to track the
        # remote command return values
        #

-       pdsh -S -w $nodes PATH=/usr/local/sbin:/usr/local/bin:/sbin:\
-/bin:/usr/sbin:/usr/bin "$@" ||
+       pdsh -S -w $nodes "PATH=/usr/local/sbin:/usr/local/bin:/sbin:\
+/bin:/usr/sbin:/usr/bin; $@" ||

                rc=$?
        return $rc
 }
                rc=$?
        return $rc
 }


@@ -429,6 +441,7 @@ ha_repeat_mpi_load()
        local stripeparams=$6
        local mpiuser=$7
        local mustpass=$8
        local stripeparams=$6
        local mpiuser=$7
        local mustpass=$8

+       local mpirunoptions=$9

        local tag=${ha_mpi_load_tags[$load]}
        local cmd=${ha_mpi_load_cmds[$tag]}
        local dir=$ha_test_dir/$client-$tag
        local tag=${ha_mpi_load_tags[$load]}
        local cmd=${ha_mpi_load_cmds[$tag]}
        local dir=$ha_test_dir/$client-$tag


@@ -442,11 +455,12 @@ ha_repeat_mpi_load()
        cmd=${cmd//"{params}"/$parameter}
 
        [[ -n "$ha_postcmd" ]] && ha_postcmd=${ha_postcmd//"{}"/$dir}
        cmd=${cmd//"{params}"/$parameter}
 
        [[ -n "$ha_postcmd" ]] && ha_postcmd=${ha_postcmd//"{}"/$dir}

+       [[ -n "$ha_precmd" ]] && ha_precmd=${ha_precmd//"{}"/$dir}

        ha_info "Starting $tag"
 
        machines="-machinefile $machines"
        while [ ! -e "$ha_stop_file" ] && ((rc == 0)); do
        ha_info "Starting $tag"
 
        machines="-machinefile $machines"
        while [ ! -e "$ha_stop_file" ] && ((rc == 0)); do

-               ha_info "$client Starts: $cmd" 2>&1 |  tee -a $log
+               ha_info "$client Starts: $mpiuser: $cmd" 2>&1 |  tee -a $log

                {
                local mdt_index
                if $ha_mdt_index_random && [ $ha_mdt_index -ne 0 ]; then
                {
                local mdt_index
                if $ha_mdt_index_random && [ $ha_mdt_index -ne 0 ]; then


@@ -454,22 +468,24 @@ ha_repeat_mpi_load()
                else
                        mdt_index=$ha_mdt_index
                fi
                else
                        mdt_index=$ha_mdt_index
                fi

+               [[ -n "$ha_precmd" ]] && ha_info "$ha_precmd" &&
+                       ha_on $client "$ha_precmd" >>"$log" 2>&1

                ha_on $client $LFS mkdir -i$mdt_index -c$ha_dir_stripe_count "$dir" &&
                ha_on $client $LFS getdirstripe "$dir" &&
                ha_on $client $LFS setstripe $stripeparams $dir &&
                ha_on $client $LFS getstripe $dir &&
                ha_on $client chmod a+xwr $dir &&
                ha_on $client $LFS mkdir -i$mdt_index -c$ha_dir_stripe_count "$dir" &&
                ha_on $client $LFS getdirstripe "$dir" &&
                ha_on $client $LFS setstripe $stripeparams $dir &&
                ha_on $client $LFS getstripe $dir &&
                ha_on $client chmod a+xwr $dir &&

-               ha_on $client "su $mpiuser sh -c \" $mpirun $ha_mpirun_options \
+               ha_on $client "su $mpiuser sh -c \" $mpirun $mpirunoptions \

                        -np $((${#ha_clients[@]} * mpi_threads_per_client )) \
                        $machines $cmd \" " || rc=$?
                [[ -n "$ha_postcmd" ]] && ha_info "$ha_postcmd" &&
                        -np $((${#ha_clients[@]} * mpi_threads_per_client )) \
                        $machines $cmd \" " || rc=$?
                [[ -n "$ha_postcmd" ]] && ha_info "$ha_postcmd" &&

-                       ha_on $client $ha_postcmd >>"$log" 2>&1
+                       ha_on $client "$ha_postcmd" >>"$log" 2>&1

                (( ((rc == 0)) && (( mustpass != 0 )) )) ||
                (( ((rc != 0)) && (( mustpass == 0 )) )) &&
                        ha_on $client rm -rf "$dir";
                } >>"$log" 2>&1 || rc=$?
 
                (( ((rc == 0)) && (( mustpass != 0 )) )) ||
                (( ((rc != 0)) && (( mustpass == 0 )) )) &&
                        ha_on $client rm -rf "$dir";
                } >>"$log" 2>&1 || rc=$?
 

-               ha_info rc=$rc mustpass=$mustpass
+               ha_info $client: rc=$rc mustpass=$mustpass

 
                # mustpass=0 means that failure is expected
                if (( rc !=0 )); then
 
                # mustpass=0 means that failure is expected
                if (( rc !=0 )); then


@@ -510,6 +526,7 @@ ha_start_mpi_loads()
        local m
        local -a mach
        local mpiuser
        local m
        local -a mach
        local mpiuser

+       local nmpi

 
        # ha_mpi_instances defines the number of
        # clients start mpi loads; should be <= ${#ha_clients[@]}
 
        # ha_mpi_instances defines the number of
        # clients start mpi loads; should be <= ${#ha_clients[@]}


@@ -543,7 +560,8 @@ ha_start_mpi_loads()
 
        for ((n = 0; n < $inst; n++)); do
                client=${ha_clients[n]}
 
        for ((n = 0; n < $inst; n++)); do
                client=${ha_clients[n]}

-               mpiuser=${ha_mpiusers[$((n % ${#ha_mpiusers[@]}))]}
+               nmpi=$((n % ${#ha_users[@]}))
+               mpiuser=${ha_users[nmpi]}

                for ((load = 0; load < ${#ha_mpi_load_tags[@]}; load++)); do
                        tag=${ha_mpi_load_tags[$load]}
                        status=$ha_status_file_prefix-$tag-$client
                for ((load = 0; load < ${#ha_mpi_load_tags[@]}; load++)); do
                        tag=${ha_mpi_load_tags[$load]}
                        status=$ha_status_file_prefix-$tag-$client


@@ -562,7 +580,8 @@ ha_start_mpi_loads()
                        [[ $ha_ninstmustfail == 0 ]] ||
                                mustpass=$(( n % ha_ninstmustfail ))
                        ha_repeat_mpi_load $client $load $status "$parameter" \
                        [[ $ha_ninstmustfail == 0 ]] ||
                                mustpass=$(( n % ha_ninstmustfail ))
                        ha_repeat_mpi_load $client $load $status "$parameter" \

-                               $machines "$stripe" "$mpiuser" "$mustpass" &
+                               $machines "$stripe" "$mpiuser" "$mustpass" \
+                               "${ha_mpiopts[$mpiuser]} $ha_mpirun_options" &

                                ha_status_files+=("$status")
                done
        done
                                ha_status_files+=("$status")
                done
        done