lustre/tests/racer.sh

   1 #!/bin/bash
   2 # SPDX-License-Identifier: GPL-2.0
   3
   4 #
   5 # This file is part of Lustre, http://www.lustre.org/
   6 #
   7 # lustre/tests/racer.sh
   8 #
   9 # Launcher racer script which runs individual racer tests under
  10 # lustre/tests/racer/
  11 #
  12 set -e
  13
  14 ONLY=${ONLY:-"$*"}
  15
  16 LUSTRE=${LUSTRE:-$(dirname $0)/..}
  17 . $LUSTRE/tests/test-framework.sh
  18 init_test_env "$@"
  19 init_logging
  20
  21 build_test_filter
  22
  23 racer=$LUSTRE/tests/racer/racer.sh
  24 echo racer: $racer with $MDSCOUNT MDTs
  25
  26 if [ "$SLOW" = "no" ]; then
  27         DURATION=${DURATION:-300}
  28 else
  29         DURATION=${DURATION:-900}
  30 fi
  31 MOUNT_2=${MOUNT_2:-"yes"}
  32
  33 check_and_setup_lustre
  34
  35 CLIENTS=${CLIENTS:-$HOSTNAME}
  36 RACERDIRS=${RACERDIRS:-"$DIR $DIR2"}
  37 echo RACERDIRS=$RACERDIRS
  38
  39 RACER_FAILOVER=${RACER_FAILOVER:-false}
  40 FAIL_TARGETS=${FAIL_TARGETS:-"MDS OST"}
  41 RACER_FAILOVER_PERIOD=${RACER_FAILOVER_PERIOD:-60}
  42
  43 if $RACER_FAILOVER; then
  44         declare -a  victims
  45         for target in $FAIL_TARGETS; do
  46                 victims=(${victims[@]} $(get_facets $target))
  47         done
  48         echo Victim facets "${victims[@]}"
  49 fi
  50
  51 if ((MDSCOUNT > 1)); then
  52         (( $MDS1_VERSION >= $(version_code 2.5.0) )) &&
  53                 RACER_ENABLE_REMOTE_DIRS=${RACER_ENABLE_REMOTE_DIRS:-true}
  54         (( $MDS1_VERSION >= $(version_code 2.8.0) )) &&
  55                 RACER_ENABLE_STRIPED_DIRS=${RACER_ENABLE_STRIPED_DIRS:-true}
  56         (( $MDS1_VERSION >= $(version_code 2.13.57) )) &&
  57                 RACER_ENABLE_MIGRATION=${RACER_ENABLE_MIGRATION:-true}
  58         (( $MDS1_VERSION >= $(version_code 2.15.55.45) )) &&
  59                 RACER_MIGRATE_STRIPE_MAX=$MDSCOUNT
  60 fi
  61
  62 [[ "$MDS1_VERSION" -lt $(version_code 2.9.54) || $mgs_FSTYPE != zfs ]] &&
  63         RACER_ENABLE_SNAPSHOT=false
  64
  65 (( "$MDS1_VERSION" <= $(version_code 2.9.55) )) &&
  66         RACER_ENABLE_PFL=false
  67
  68 (( "$MDS1_VERSION" <= $(version_code 2.10.53) )) &&
  69         RACER_ENABLE_DOM=false
  70
  71 (( "$MDS1_VERSION" < $(version_code 2.10.55) )) &&
  72         RACER_ENABLE_FLR=false
  73
  74 (( $MDS1_VERSION < $(version_code 2.12.0) )) &&
  75         RACER_ENABLE_SEL=false
  76
  77 [[ $OST1_VERSION -lt $(version_code 2.15.59) || $ost1_FSTYPE != ldiskfs ]] &&
  78         RACER_ENABLE_FALLOCATE=false
  79 check_set_fallocate || RACER_ENABLE_FALLOCATE=false
  80
  81 RACER_ENABLE_REMOTE_DIRS=${RACER_ENABLE_REMOTE_DIRS:-false}
  82 RACER_ENABLE_STRIPED_DIRS=${RACER_ENABLE_STRIPED_DIRS:-false}
  83 RACER_ENABLE_MIGRATION=${RACER_ENABLE_MIGRATION:-false}
  84 RACER_ENABLE_SNAPSHOT=${RACER_ENABLE_SNAPSHOT:-true}
  85 RACER_ENABLE_FILE_MIGRATE=${RACER_ENABLE_FILE_MIGRATE:-true}
  86 RACER_ENABLE_PFL=${RACER_ENABLE_PFL:-true}
  87 RACER_ENABLE_DOM=${RACER_ENABLE_DOM:-true}
  88 RACER_ENABLE_FLR=${RACER_ENABLE_FLR:-true}
  89 RACER_ENABLE_SEL=${RACER_ENABLE_SEL:-true}
  90 # set false, LU-14988
  91 RACER_ENABLE_OVERSTRIPE=${RACER_ENABLE_OVERSTRIPE:-false}
  92 RACER_LOV_MAX_STRIPECOUNT=${RACER_LOV_MAX_STRIPECOUNT:-$LOV_MAX_STRIPE_COUNT}
  93 RACER_EXTRA_LAYOUT=${RACER_EXTRA_LAYOUT:-""}
  94 RACER_MIGRATE_STRIPE_MAX=${RACER_MIGRATE_STRIPE_MAX:-1}
  95 RACER_ENABLE_FALLOCATE=${RACER_ENABLE_FALLOCATE:-true}
  96
  97 fail_random_facet () {
  98         local facets=${victims[@]}
  99         facets=${facets// /,}
 100
 101         sleep $RACER_FAILOVER_PERIOD
 102         while [ ! -f $racer_done ]; do
 103                 local facet=$(get_random_entry $facets)
 104                 facet_failover $facet
 105                 sleep $RACER_FAILOVER_PERIOD
 106         done
 107 }
 108
 109 # run racer
 110 test_1() {
 111         local rrc=0
 112         local rc=0
 113         local clients=$CLIENTS
 114         local RDIRS
 115         local i
 116         local racer_done=$TMP/racer_done
 117
 118         rm -f $racer_done
 119
 120         for d in ${RACERDIRS}; do
 121                 is_mounted $d || continue
 122
 123                 RDIRS="$RDIRS $d/racer"
 124                 mkdir -p $d/racer
 125                 if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
 126                         $LFS setstripe $d/racer $RACER_EXTRA_LAYOUT ||
 127                         error "setstripe $RACER_EXTRA_LAYOUT failed"
 128                 fi
 129                 if (( MDSCOUNT >= 2 )); then
 130                         for ((i = 0; i < MDSCOUNT; i++)); do
 131                                 RDIRS="$RDIRS $d/racer$i"
 132                                 if [[ ! -e $d/racer$i ]]; then
 133                                         $LFS mkdir -i $i $RACER_MKDIR_OPTS $d/racer$i ||
 134                                                 error "lfs mkdir $i failed"
 135                                 fi
 136                                 if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
 137                                         $LFS setstripe $d/racer$i \
 138                                                 $RACER_EXTRA_LAYOUT ||
 139                                         error "setstripe \
 140                                                 $RACER_EXTRA_LAYOUT failed"
 141                                 fi
 142                         done
 143                 fi
 144         done
 145
 146         local rpids=""
 147         for rdir in $RDIRS; do
 148                 do_nodes $clients "DURATION=$DURATION \
 149                         MDSCOUNT=$MDSCOUNT OSTCOUNT=$OSTCOUNT\
 150                         RACER_ENABLE_REMOTE_DIRS=$RACER_ENABLE_REMOTE_DIRS \
 151                         RACER_ENABLE_STRIPED_DIRS=$RACER_ENABLE_STRIPED_DIRS \
 152                         RACER_ENABLE_MIGRATION=$RACER_ENABLE_MIGRATION \
 153                         RACER_ENABLE_FILE_MIGRATE=$RACER_ENABLE_FILE_MIGRATE \
 154                         RACER_ENABLE_PFL=$RACER_ENABLE_PFL \
 155                         RACER_ENABLE_DOM=$RACER_ENABLE_DOM \
 156                         RACER_ENABLE_FLR=$RACER_ENABLE_FLR \
 157                         RACER_MAX_CLEANUP_WAIT=$RACER_MAX_CLEANUP_WAIT \
 158                         RACER_ENABLE_SEL=$RACER_ENABLE_SEL \
 159                         LOV_MAX_STRIPE_COUNT=$LOV_MAX_STRIPE_COUNT \
 160                         RACER_ENABLE_OVERSTRIPE=$RACER_ENABLE_OVERSTRIPE \
 161                         RACER_LOV_MAX_STRIPECOUNT=$RACER_LOV_MAX_STRIPECOUNT \
 162                         RACER_EXTRA=$RACER_EXTRA \
 163                         RACER_EXTRA_LAYOUT=\\\"$RACER_EXTRA_LAYOUT\\\" \
 164                         RACER_MIGRATE_STRIPE_MAX=$RACER_MIGRATE_STRIPE_MAX \
 165                         RACER_PROGS=$RACER_PROGS \
 166                         NUM_THREADS=$NUM_THREADS \
 167                         MAX_FILES=$MAX_FILES \
 168                         LFS=$LFS \
 169                         LCTL=$LCTL \
 170                         $racer $rdir $NUM_RACER_THREADS" &
 171                 pid=$!
 172                 rpids="$rpids $pid"
 173         done
 174
 175         local failpid=""
 176         if $RACER_FAILOVER; then
 177                 fail_random_facet &
 178                 failpid=$!
 179                 echo racers failpid: $failpid
 180         fi
 181
 182         local lss_pids=""
 183         if $RACER_ENABLE_SNAPSHOT; then
 184                 lss_gen_conf
 185
 186                 $LUSTRE/tests/racer/lss_create.sh &
 187                 pid=$!
 188                 lss_pids="$lss_pids $pid"
 189
 190                 $LUSTRE/tests/racer/lss_destroy.sh &
 191                 pid=$!
 192                 lss_pids="$lss_pids $pid"
 193         fi
 194
 195         echo racers pids: $rpids
 196         for pid in $rpids; do
 197                 wait $pid
 198                 rc=$?
 199                 echo "pid=$pid rc=$rc"
 200                 if [ $rc != 0 ]; then
 201                     rrc=$((rrc + 1))
 202                 fi
 203         done
 204
 205         if $RACER_FAILOVER; then
 206                 touch $racer_done
 207                 wait $failpid
 208                 rrc=$((rrc + $?))
 209         fi
 210
 211         if $RACER_ENABLE_SNAPSHOT; then
 212                 killall -q lss_create.sh
 213                 killall -q lss_destroy.sh
 214
 215                 for pid in $lss_pids; do
 216                         wait $pid
 217                 done
 218
 219                 lss_cleanup
 220         fi
 221
 222         return $rrc
 223 }
 224 run_test 1 "racer on clients: ${CLIENTS:-$(hostname)} DURATION=$DURATION"
 225
 226 # racer rename stress test
 227 test_2() {
 228         local rrc=0
 229         local rc=0
 230         local clients=$CLIENTS
 231         local RDIRS
 232         local i
 233         local racer_done=$TMP/racer_done
 234
 235         (( MDSCOUNT > 1 )) || skip "need at least 2 MDTs"
 236
 237         rm -f $racer_done
 238
 239         for d in ${RACERDIRS}; do
 240                 is_mounted $d || continue
 241                 mkdir -p $d
 242
 243                 for ((i = 0; i < $MDSCOUNT; i++)); do
 244                         RDIRS+=" $d/racer$i"
 245                         [[  -d "$d/racer$i" ]] && continue
 246                         $LFS mkdir $RACER_MKDIR_OPTS $d/racer$i ||
 247                                 error "mkdir $d/racer$i failed"
 248                         if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
 249                                 $LFS setstripe $d/racer$i $RACER_EXTRA_LAYOUT ||
 250                                         error "extra $RACER_EXTRA_LAYOUT failed"
 251                         fi
 252                 done
 253         done
 254
 255         local rpids=""
 256         local progs="dir_create+dir_remote+file_rename+file_rename+file_create+file_rm"
 257         for rdir in $RDIRS; do
 258                 echo "starting on $clients:$rdir with: ${progs//+/ }"
 259                 do_nodes $clients "DURATION=$DURATION \
 260                         MDSCOUNT=$MDSCOUNT OSTCOUNT=$OSTCOUNT\
 261                         RACER_MAX_MB=0 \
 262                         RACER_ENABLE_FLR=false \
 263                         RACER_ENABLE_DOM=false \
 264                         RACER_ENABLE_SEL=false \
 265                         RACER_ENABLE_MIGRATION=false \
 266                         RACER_MAX_CLEANUP_WAIT=$RACER_MAX_CLEANUP_WAIT \
 267                         RACER_EXTRA=\\\"$RACER_EXTRA\\\" \
 268                         RACER_EXTRA_LAYOUT=\\\"$RACER_EXTRA_LAYOUT\\\" \
 269                         RACER_PROGS="$progs" \
 270                         NUM_THREADS=$NUM_THREADS \
 271                         MAX_FILES=$MAX_FILES \
 272                         LFS=$LFS \
 273                         LCTL=$LCTL \
 274                         $racer $rdir $NUM_RACER_THREADS" &
 275                 pid=$!
 276                 rpids="$rpids $pid"
 277         done
 278
 279         echo racers pids: $rpids
 280         for pid in $rpids; do
 281                 wait $pid
 282                 rc=$?
 283                 echo "pid=$pid rc=$rc"
 284                 if [ $rc != 0 ]; then
 285                     rrc=$((rrc + 1))
 286                 fi
 287         done
 288
 289         return $rrc
 290 }
 291 run_test 2 "racer rename: ${CLIENTS:-$(hostname)} DURATION=$DURATION"
 292
 293 complete_test $SECONDS
 294 FSCK_ALWAYS=${FSCK_ALWAYS:-"yes"} check_and_cleanup_lustre
 295
 296 exit_status