From 1129afd34896c9a9778e9467318cde2bb1676e24 Mon Sep 17 00:00:00 2001 From: Elena Gryaznova Date: Tue, 22 Dec 2020 18:00:08 +0300 Subject: [PATCH] LU-14272 tests: different mpirun options for different users Patch adds the ability to use different mpirun options for different mpi users, for example we use: --wdir=/home/quota15_1 for user quota15_1 and --wdir=/home/quota15_2 for user quota15_2. Per mpi user option can be specified after ':' in ha_mpi_users: Example: ha_mpi_users="quota15_1:--wdir=/home/quota15_1 quota15_2:--wdir=/home/quota15_2 quota15_3 quota15_4:--wdir=/home/quota15_4" Several options for user can be set as: ha_mpi_users="user1: user1: user2:" MPIRUN_OPTIONS will be applied for all users. Patch adds the ability to execute the command before loads start. Example: ha_precmd="for i in 1 2 3 4; do lfs quota -v -u quota15_$i $ha_test_dir; done" Test-Parameters: trivial Signed-off-by: Elena Gryaznova HPE-bug-id: LUS-9069 Reviewed-by: Vladimir Saveliev Reviewed-by: Sergey Cheremencev Change-Id: I35feadcc856dbc3d6d6d5f2720ab49c6c111161b Reviewed-on: https://review.whamcloud.com/41072 Tested-by: jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/tests/ha.sh | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/lustre/tests/ha.sh b/lustre/tests/ha.sh index c26689d..d18a26b 100755 --- a/lustre/tests/ha.sh +++ b/lustre/tests/ha.sh @@ -220,6 +220,18 @@ declare ha_mpi_instances=${ha_mpi_instances:-1} declare ha_mpi_loads=${ha_mpi_loads="ior simul mdtest"} declare -a ha_mpi_load_tags=($ha_mpi_loads) declare -a ha_mpiusers=(${ha_mpi_users="mpiuser"}) +declare -a ha_users +declare -A ha_mpiopts + +for ((i=0; i<${#ha_mpiusers[@]}; i++)); do + u=${ha_mpiusers[i]%%:*} + o="" + # user gets empty option if ha_mpi_users does not specify it explicitly + [[ ${ha_mpiusers[i]} =~ : ]] && o=${ha_mpiusers[i]##*:} + ha_users[i]=$u + ha_mpiopts[$u]+=" $o" +done +ha_users=(${!ha_mpiopts[@]}) declare ha_ior_params=${IORP:-'" -b $ior_blockSize -t 2m -w -W -T 1"'} declare ha_simul_params=${SIMULP:-'" -n 10"'} @@ -337,8 +349,8 @@ ha_on() # -S is to be used here to track the # remote command return values # - pdsh -S -w $nodes PATH=/usr/local/sbin:/usr/local/bin:/sbin:\ -/bin:/usr/sbin:/usr/bin "$@" || + pdsh -S -w $nodes "PATH=/usr/local/sbin:/usr/local/bin:/sbin:\ +/bin:/usr/sbin:/usr/bin; $@" || rc=$? return $rc } @@ -429,6 +441,7 @@ ha_repeat_mpi_load() local stripeparams=$6 local mpiuser=$7 local mustpass=$8 + local mpirunoptions=$9 local tag=${ha_mpi_load_tags[$load]} local cmd=${ha_mpi_load_cmds[$tag]} local dir=$ha_test_dir/$client-$tag @@ -442,11 +455,12 @@ ha_repeat_mpi_load() cmd=${cmd//"{params}"/$parameter} [[ -n "$ha_postcmd" ]] && ha_postcmd=${ha_postcmd//"{}"/$dir} + [[ -n "$ha_precmd" ]] && ha_precmd=${ha_precmd//"{}"/$dir} ha_info "Starting $tag" machines="-machinefile $machines" while [ ! -e "$ha_stop_file" ] && ((rc == 0)); do - ha_info "$client Starts: $cmd" 2>&1 | tee -a $log + ha_info "$client Starts: $mpiuser: $cmd" 2>&1 | tee -a $log { local mdt_index if $ha_mdt_index_random && [ $ha_mdt_index -ne 0 ]; then @@ -454,22 +468,24 @@ ha_repeat_mpi_load() else mdt_index=$ha_mdt_index fi + [[ -n "$ha_precmd" ]] && ha_info "$ha_precmd" && + ha_on $client "$ha_precmd" >>"$log" 2>&1 ha_on $client $LFS mkdir -i$mdt_index -c$ha_dir_stripe_count "$dir" && ha_on $client $LFS getdirstripe "$dir" && ha_on $client $LFS setstripe $stripeparams $dir && ha_on $client $LFS getstripe $dir && ha_on $client chmod a+xwr $dir && - ha_on $client "su $mpiuser sh -c \" $mpirun $ha_mpirun_options \ + ha_on $client "su $mpiuser sh -c \" $mpirun $mpirunoptions \ -np $((${#ha_clients[@]} * mpi_threads_per_client )) \ $machines $cmd \" " || rc=$? [[ -n "$ha_postcmd" ]] && ha_info "$ha_postcmd" && - ha_on $client $ha_postcmd >>"$log" 2>&1 + ha_on $client "$ha_postcmd" >>"$log" 2>&1 (( ((rc == 0)) && (( mustpass != 0 )) )) || (( ((rc != 0)) && (( mustpass == 0 )) )) && ha_on $client rm -rf "$dir"; } >>"$log" 2>&1 || rc=$? - ha_info rc=$rc mustpass=$mustpass + ha_info $client: rc=$rc mustpass=$mustpass # mustpass=0 means that failure is expected if (( rc !=0 )); then @@ -510,6 +526,7 @@ ha_start_mpi_loads() local m local -a mach local mpiuser + local nmpi # ha_mpi_instances defines the number of # clients start mpi loads; should be <= ${#ha_clients[@]} @@ -543,7 +560,8 @@ ha_start_mpi_loads() for ((n = 0; n < $inst; n++)); do client=${ha_clients[n]} - mpiuser=${ha_mpiusers[$((n % ${#ha_mpiusers[@]}))]} + nmpi=$((n % ${#ha_users[@]})) + mpiuser=${ha_users[nmpi]} for ((load = 0; load < ${#ha_mpi_load_tags[@]}; load++)); do tag=${ha_mpi_load_tags[$load]} status=$ha_status_file_prefix-$tag-$client @@ -562,7 +580,8 @@ ha_start_mpi_loads() [[ $ha_ninstmustfail == 0 ]] || mustpass=$(( n % ha_ninstmustfail )) ha_repeat_mpi_load $client $load $status "$parameter" \ - $machines "$stripe" "$mpiuser" "$mustpass" & + $machines "$stripe" "$mpiuser" "$mustpass" \ + "${ha_mpiopts[$mpiuser]} $ha_mpirun_options" & ha_status_files+=("$status") done done -- 1.8.3.1