lustre/tests/racer.sh

   1 #!/bin/bash
   2 # SPDX-License-Identifier: GPL-2.0
   3
   4 #
   5 # This file is part of Lustre, http://www.lustre.org/
   6 #
   7 # lustre/tests/racer.sh
   8 #
   9 # Launcher racer script which runs individual racer tests under
  10 # lustre/tests/racer/
  11 #
  12 set -e
  13
  14 ONLY=${ONLY:-"$*"}
  15
  16 LUSTRE=${LUSTRE:-$(dirname $0)/..}
  17 . $LUSTRE/tests/test-framework.sh
  18 init_test_env "$@"
  19 init_logging
  20
  21 ALWAYS_EXCEPT="$RACER_EXCEPT "
  22
  23 build_test_filter
  24
  25 racer=$LUSTRE/tests/racer/racer.sh
  26 echo racer: $racer with $MDSCOUNT MDTs
  27
  28 if [ "$SLOW" = "no" ]; then
  29         DURATION=${DURATION:-300}
  30 else
  31         DURATION=${DURATION:-900}
  32 fi
  33 MOUNT_2=${MOUNT_2:-"yes"}
  34
  35 check_and_setup_lustre
  36
  37 CLIENTS=${CLIENTS:-$HOSTNAME}
  38 RACERDIRS=${RACERDIRS:-"$DIR $DIR2"}
  39 echo RACERDIRS=$RACERDIRS
  40
  41 RACER_FAILOVER=${RACER_FAILOVER:-false}
  42 FAIL_TARGETS=${FAIL_TARGETS:-"MDS OST"}
  43 RACER_FAILOVER_PERIOD=${RACER_FAILOVER_PERIOD:-60}
  44
  45 if $RACER_FAILOVER; then
  46         declare -a  victims
  47         for target in $FAIL_TARGETS; do
  48                 victims=(${victims[@]} $(get_facets $target))
  49         done
  50         echo Victim facets "${victims[@]}"
  51 fi
  52
  53 init_stripe_dir_params RACER_ENABLE_REMOTE_DIRS \
  54         RACER_ENABLE_STRIPED_DIRS
  55
  56 if ((MDSCOUNT > 1)); then
  57         (( $MDS1_VERSION >= $(version_code 2.13.57) )) &&
  58                 RACER_ENABLE_MIGRATION=${RACER_ENABLE_MIGRATION:-true}
  59         (( $MDS1_VERSION >= $(version_code 2.15.55.45) )) &&
  60                 RACER_MIGRATE_STRIPE_MAX=$MDSCOUNT
  61 fi
  62
  63 [[ "$MDS1_VERSION" -lt $(version_code 2.9.54) || $mds1_FSTYPE != zfs ]] &&
  64         RACER_ENABLE_SNAPSHOT=false
  65
  66 (( "$MDS1_VERSION" <= $(version_code 2.9.55) )) &&
  67         RACER_ENABLE_PFL=false
  68
  69 (( "$MDS1_VERSION" <= $(version_code 2.10.53) )) &&
  70         RACER_ENABLE_DOM=false
  71
  72 (( "$MDS1_VERSION" < $(version_code 2.10.55) )) &&
  73         RACER_ENABLE_FLR=false
  74
  75 (( $MDS1_VERSION < $(version_code 2.12.0) )) &&
  76         RACER_ENABLE_SEL=false
  77
  78 [[ $OST1_VERSION -lt $(version_code 2.15.59) || $ost1_FSTYPE != ldiskfs ]] &&
  79         RACER_ENABLE_FALLOCATE=false
  80 check_set_fallocate || RACER_ENABLE_FALLOCATE=false
  81
  82 RACER_ENABLE_MIGRATION=${RACER_ENABLE_MIGRATION:-false}
  83 RACER_ENABLE_SNAPSHOT=${RACER_ENABLE_SNAPSHOT:-true}
  84 RACER_ENABLE_FILE_MIGRATE=${RACER_ENABLE_FILE_MIGRATE:-true}
  85 RACER_ENABLE_PFL=${RACER_ENABLE_PFL:-true}
  86 RACER_ENABLE_DOM=${RACER_ENABLE_DOM:-true}
  87 RACER_ENABLE_FLR=${RACER_ENABLE_FLR:-true}
  88 RACER_ENABLE_SEL=${RACER_ENABLE_SEL:-true}
  89 # set false, LU-14988
  90 RACER_ENABLE_OVERSTRIPE=${RACER_ENABLE_OVERSTRIPE:-false}
  91 RACER_LOV_MAX_STRIPECOUNT=${RACER_LOV_MAX_STRIPECOUNT:-$LOV_MAX_STRIPE_COUNT}
  92 RACER_EXTRA_LAYOUT=${RACER_EXTRA_LAYOUT:-""}
  93 RACER_MIGRATE_STRIPE_MAX=${RACER_MIGRATE_STRIPE_MAX:-1}
  94 RACER_ENABLE_FALLOCATE=${RACER_ENABLE_FALLOCATE:-true}
  95
  96 fail_random_facet () {
  97         local facets=${victims[@]}
  98         facets=${facets// /,}
  99
 100         sleep $RACER_FAILOVER_PERIOD
 101         while [ ! -f $racer_done ]; do
 102                 local facet=$(get_random_entry $facets)
 103                 facet_failover $facet
 104                 sleep $RACER_FAILOVER_PERIOD
 105         done
 106 }
 107
 108 # run racer
 109 test_1() {
 110         local rrc=0
 111         local rc=0
 112         local clients=$CLIENTS
 113         local RDIRS
 114         local i
 115         local racer_done=$TMP/racer_done
 116
 117         rm -f $racer_done
 118
 119         for d in ${RACERDIRS}; do
 120                 is_mounted $d || continue
 121
 122                 RDIRS="$RDIRS $d/racer"
 123                 mkdir -p $d/racer
 124                 if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
 125                         $LFS setstripe $d/racer $RACER_EXTRA_LAYOUT ||
 126                         error "setstripe $RACER_EXTRA_LAYOUT failed"
 127                 fi
 128                 if (( MDSCOUNT >= 2 )); then
 129                         for ((i = 0; i < MDSCOUNT; i++)); do
 130                                 RDIRS="$RDIRS $d/racer$i"
 131                                 if [[ ! -e $d/racer$i ]]; then
 132                                         $LFS mkdir -i $i $RACER_MKDIR_OPTS $d/racer$i ||
 133                                                 error "lfs mkdir $i failed"
 134                                 fi
 135                                 if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
 136                                         $LFS setstripe $d/racer$i \
 137                                                 $RACER_EXTRA_LAYOUT ||
 138                                         error "setstripe \
 139                                                 $RACER_EXTRA_LAYOUT failed"
 140                                 fi
 141                         done
 142                 fi
 143         done
 144
 145         local rpids=""
 146         for rdir in $RDIRS; do
 147                 do_nodes $clients "DURATION=$DURATION \
 148                         MDSCOUNT=$MDSCOUNT OSTCOUNT=$OSTCOUNT\
 149                         RACER_ENABLE_REMOTE_DIRS=$RACER_ENABLE_REMOTE_DIRS \
 150                         RACER_ENABLE_STRIPED_DIRS=$RACER_ENABLE_STRIPED_DIRS \
 151                         RACER_ENABLE_MIGRATION=$RACER_ENABLE_MIGRATION \
 152                         RACER_ENABLE_FILE_MIGRATE=$RACER_ENABLE_FILE_MIGRATE \
 153                         RACER_ENABLE_PFL=$RACER_ENABLE_PFL \
 154                         RACER_ENABLE_DOM=$RACER_ENABLE_DOM \
 155                         RACER_ENABLE_FLR=$RACER_ENABLE_FLR \
 156                         RACER_MAX_CLEANUP_WAIT=$RACER_MAX_CLEANUP_WAIT \
 157                         RACER_ENABLE_SEL=$RACER_ENABLE_SEL \
 158                         LOV_MAX_STRIPE_COUNT=$LOV_MAX_STRIPE_COUNT \
 159                         RACER_ENABLE_OVERSTRIPE=$RACER_ENABLE_OVERSTRIPE \
 160                         RACER_LOV_MAX_STRIPECOUNT=$RACER_LOV_MAX_STRIPECOUNT \
 161                         RACER_EXTRA=$RACER_EXTRA \
 162                         RACER_EXTRA_LAYOUT=\\\"$RACER_EXTRA_LAYOUT\\\" \
 163                         RACER_MIGRATE_STRIPE_MAX=$RACER_MIGRATE_STRIPE_MAX \
 164                         RACER_PROGS=$RACER_PROGS \
 165                         NUM_THREADS=$NUM_THREADS \
 166                         MAX_FILES=$MAX_FILES \
 167                         LFS=$LFS \
 168                         LCTL=$LCTL \
 169                         $racer $rdir $NUM_RACER_THREADS" &
 170                 pid=$!
 171                 rpids="$rpids $pid"
 172         done
 173
 174         local failpid=""
 175         if $RACER_FAILOVER; then
 176                 fail_random_facet &
 177                 failpid=$!
 178                 echo racers failpid: $failpid
 179         fi
 180
 181         local lss_pids=""
 182         if $RACER_ENABLE_SNAPSHOT; then
 183                 lss_gen_conf
 184
 185                 $LUSTRE/tests/racer/lss_create.sh &
 186                 pid=$!
 187                 lss_pids="$lss_pids $pid"
 188
 189                 $LUSTRE/tests/racer/lss_destroy.sh &
 190                 pid=$!
 191                 lss_pids="$lss_pids $pid"
 192         fi
 193
 194         echo racers pids: $rpids
 195         for pid in $rpids; do
 196                 wait $pid
 197                 rc=$?
 198                 echo "pid=$pid rc=$rc"
 199                 if [ $rc != 0 ]; then
 200                     rrc=$((rrc + 1))
 201                 fi
 202         done
 203
 204         if $RACER_FAILOVER; then
 205                 touch $racer_done
 206                 wait $failpid
 207                 rrc=$((rrc + $?))
 208         fi
 209
 210         if $RACER_ENABLE_SNAPSHOT; then
 211                 killall -q lss_create.sh
 212                 killall -q lss_destroy.sh
 213
 214                 for pid in $lss_pids; do
 215                         wait $pid
 216                 done
 217
 218                 lss_cleanup
 219         fi
 220
 221         return $rrc
 222 }
 223 run_test 1 "racer on clients: ${CLIENTS:-$(hostname)} DURATION=$DURATION"
 224
 225 # racer rename stress test
 226 test_2() {
 227         local rrc=0
 228         local rc=0
 229         local clients=$CLIENTS
 230         local RDIRS
 231         local i
 232         local racer_done=$TMP/racer_done
 233
 234         (( MDSCOUNT > 1 )) || skip "need at least 2 MDTs"
 235
 236         rm -f $racer_done
 237
 238         for d in ${RACERDIRS}; do
 239                 is_mounted $d || continue
 240                 mkdir -p $d
 241
 242                 for ((i = 0; i < $MDSCOUNT; i++)); do
 243                         RDIRS+=" $d/racer$i"
 244                         [[  -d "$d/racer$i" ]] && continue
 245                         $LFS mkdir $RACER_MKDIR_OPTS $d/racer$i ||
 246                                 error "mkdir $d/racer$i failed"
 247                         if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
 248                                 $LFS setstripe $d/racer$i $RACER_EXTRA_LAYOUT ||
 249                                         error "extra $RACER_EXTRA_LAYOUT failed"
 250                         fi
 251                 done
 252         done
 253
 254         local rpids=""
 255         local progs="dir_create+dir_remote+file_rename+file_rename+file_create+file_rm"
 256         for rdir in $RDIRS; do
 257                 echo "starting on $clients:$rdir with: ${progs//+/ }"
 258                 do_nodes $clients "DURATION=$DURATION \
 259                         MDSCOUNT=$MDSCOUNT OSTCOUNT=$OSTCOUNT\
 260                         RACER_MAX_MB=0 \
 261                         RACER_ENABLE_FLR=false \
 262                         RACER_ENABLE_DOM=false \
 263                         RACER_ENABLE_SEL=false \
 264                         RACER_ENABLE_MIGRATION=false \
 265                         RACER_MAX_CLEANUP_WAIT=$RACER_MAX_CLEANUP_WAIT \
 266                         RACER_EXTRA=\\\"$RACER_EXTRA\\\" \
 267                         RACER_EXTRA_LAYOUT=\\\"$RACER_EXTRA_LAYOUT\\\" \
 268                         RACER_PROGS="$progs" \
 269                         NUM_THREADS=$NUM_THREADS \
 270                         MAX_FILES=$MAX_FILES \
 271                         LFS=$LFS \
 272                         LCTL=$LCTL \
 273                         $racer $rdir $NUM_RACER_THREADS" &
 274                 pid=$!
 275                 rpids="$rpids $pid"
 276         done
 277
 278         echo racers pids: $rpids
 279         for pid in $rpids; do
 280                 wait $pid
 281                 rc=$?
 282                 echo "pid=$pid rc=$rc"
 283                 if [ $rc != 0 ]; then
 284                     rrc=$((rrc + 1))
 285                 fi
 286         done
 287
 288         return $rrc
 289 }
 290 run_test 2 "racer rename: ${CLIENTS:-$(hostname)} DURATION=$DURATION"
 291
 292 complete_test $SECONDS
 293 FSCK_ALWAYS=${FSCK_ALWAYS:-"yes"} check_and_cleanup_lustre
 294
 295 exit_status