lustre/tests/racer.sh

   1 #!/bin/bash
   2 set -e
   3
   4 ONLY=${ONLY:-"$*"}
   5
   6 LUSTRE=${LUSTRE:-$(dirname $0)/..}
   7 . $LUSTRE/tests/test-framework.sh
   8 init_test_env "$@"
   9 init_logging
  10
  11 build_test_filter
  12
  13 racer=$LUSTRE/tests/racer/racer.sh
  14 echo racer: $racer with $MDSCOUNT MDTs
  15
  16 if [ "$SLOW" = "no" ]; then
  17         DURATION=${DURATION:-300}
  18 else
  19         DURATION=${DURATION:-900}
  20 fi
  21 MOUNT_2=${MOUNT_2:-"yes"}
  22
  23 check_and_setup_lustre
  24
  25 CLIENTS=${CLIENTS:-$HOSTNAME}
  26 RACERDIRS=${RACERDIRS:-"$DIR $DIR2"}
  27 echo RACERDIRS=$RACERDIRS
  28
  29 RACER_FAILOVER=${RACER_FAILOVER:-false}
  30 FAIL_TARGETS=${FAIL_TARGETS:-"MDS OST"}
  31 RACER_FAILOVER_PERIOD=${RACER_FAILOVER_PERIOD:-60}
  32
  33 if $RACER_FAILOVER; then
  34         declare -a  victims
  35         for target in $FAIL_TARGETS; do
  36                 victims=(${victims[@]} $(get_facets $target))
  37         done
  38         echo Victim facets "${victims[@]}"
  39 fi
  40
  41 if ((MDSCOUNT > 1)); then
  42         (( $MDS1_VERSION >= $(version_code 2.5.0) )) &&
  43                 RACER_ENABLE_REMOTE_DIRS=${RACER_ENABLE_REMOTE_DIRS:-true}
  44         (( $MDS1_VERSION >= $(version_code 2.8.0) )) &&
  45                 RACER_ENABLE_STRIPED_DIRS=${RACER_ENABLE_STRIPED_DIRS:-true}
  46         (( $MDS1_VERSION >= $(version_code 2.13.57) )) &&
  47                 RACER_ENABLE_MIGRATION=${RACER_ENABLE_MIGRATION:-true}
  48         (( $MDS1_VERSION >= $(version_code 2.15.55.45) )) &&
  49                 RACER_MIGRATE_STRIPE_MAX=$MDSCOUNT
  50 fi
  51
  52 [[ "$MDS1_VERSION" -lt $(version_code 2.9.54) || $mgs_FSTYPE != zfs ]] &&
  53         RACER_ENABLE_SNAPSHOT=false
  54
  55 (( "$MDS1_VERSION" <= $(version_code 2.9.55) )) &&
  56         RACER_ENABLE_PFL=false
  57
  58 (( "$MDS1_VERSION" <= $(version_code 2.10.53) )) &&
  59         RACER_ENABLE_DOM=false
  60
  61 (( "$MDS1_VERSION" < $(version_code 2.10.55) )) &&
  62         RACER_ENABLE_FLR=false
  63
  64 (( $MDS1_VERSION < $(version_code 2.12.0) )) &&
  65         RACER_ENABLE_SEL=false
  66
  67 RACER_ENABLE_REMOTE_DIRS=${RACER_ENABLE_REMOTE_DIRS:-false}
  68 RACER_ENABLE_STRIPED_DIRS=${RACER_ENABLE_STRIPED_DIRS:-false}
  69 RACER_ENABLE_MIGRATION=${RACER_ENABLE_MIGRATION:-false}
  70 RACER_ENABLE_SNAPSHOT=${RACER_ENABLE_SNAPSHOT:-true}
  71 RACER_ENABLE_FILE_MIGRATE=${RACER_ENABLE_FILE_MIGRATE:-true}
  72 RACER_ENABLE_PFL=${RACER_ENABLE_PFL:-true}
  73 RACER_ENABLE_DOM=${RACER_ENABLE_DOM:-true}
  74 RACER_ENABLE_FLR=${RACER_ENABLE_FLR:-true}
  75 RACER_ENABLE_SEL=${RACER_ENABLE_SEL:-true}
  76 # set false, LU-14988
  77 RACER_ENABLE_OVERSTRIPE=${RACER_ENABLE_OVERSTRIPE:-false}
  78 RACER_LOV_MAX_STRIPECOUNT=${RACER_LOV_MAX_STRIPECOUNT:-$LOV_MAX_STRIPE_COUNT}
  79 RACER_EXTRA_LAYOUT=${RACER_EXTRA_LAYOUT:-""}
  80 RACER_MIGRATE_STRIPE_MAX=${RACER_MIGRATE_STRIPE_MAX:-1}
  81
  82 fail_random_facet () {
  83         local facets=${victims[@]}
  84         facets=${facets// /,}
  85
  86         sleep $RACER_FAILOVER_PERIOD
  87         while [ ! -f $racer_done ]; do
  88                 local facet=$(get_random_entry $facets)
  89                 facet_failover $facet
  90                 sleep $RACER_FAILOVER_PERIOD
  91         done
  92 }
  93
  94 # run racer
  95 test_1() {
  96         local rrc=0
  97         local rc=0
  98         local clients=$CLIENTS
  99         local RDIRS
 100         local i
 101         local racer_done=$TMP/racer_done
 102
 103         rm -f $racer_done
 104
 105         for d in ${RACERDIRS}; do
 106                 is_mounted $d || continue
 107
 108                 RDIRS="$RDIRS $d/racer"
 109                 mkdir -p $d/racer
 110                 if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
 111                         $LFS setstripe $d/racer $RACER_EXTRA_LAYOUT ||
 112                         error "setstripe $RACER_EXTRA_LAYOUT failed"
 113                 fi
 114                 if [ $MDSCOUNT -ge 2 ]; then
 115                         for i in $(seq $((MDSCOUNT - 1))); do
 116                                 RDIRS="$RDIRS $d/racer$i"
 117                                 if [ ! -e $d/racer$i ]; then
 118                                         $LFS mkdir -i $i $d/racer$i ||
 119                                                 error "lfs mkdir $i failed"
 120                                 fi
 121                                 if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
 122                                         $LFS setstripe $d/racer$i \
 123                                                 $RACER_EXTRA_LAYOUT ||
 124                                         error "setstripe \
 125                                                 $RACER_EXTRA_LAYOUT failed"
 126                                 fi
 127                         done
 128                 fi
 129         done
 130
 131         local rpids=""
 132         for rdir in $RDIRS; do
 133                 do_nodes $clients "DURATION=$DURATION \
 134                         MDSCOUNT=$MDSCOUNT OSTCOUNT=$OSTCOUNT\
 135                         RACER_ENABLE_REMOTE_DIRS=$RACER_ENABLE_REMOTE_DIRS \
 136                         RACER_ENABLE_STRIPED_DIRS=$RACER_ENABLE_STRIPED_DIRS \
 137                         RACER_ENABLE_MIGRATION=$RACER_ENABLE_MIGRATION \
 138                         RACER_ENABLE_FILE_MIGRATE=$RACER_ENABLE_FILE_MIGRATE \
 139                         RACER_ENABLE_PFL=$RACER_ENABLE_PFL \
 140                         RACER_ENABLE_DOM=$RACER_ENABLE_DOM \
 141                         RACER_ENABLE_FLR=$RACER_ENABLE_FLR \
 142                         RACER_MAX_CLEANUP_WAIT=$RACER_MAX_CLEANUP_WAIT \
 143                         RACER_ENABLE_SEL=$RACER_ENABLE_SEL \
 144                         LOV_MAX_STRIPE_COUNT=$LOV_MAX_STRIPE_COUNT \
 145                         RACER_ENABLE_OVERSTRIPE=$RACER_ENABLE_OVERSTRIPE \
 146                         RACER_LOV_MAX_STRIPECOUNT=$RACER_LOV_MAX_STRIPECOUNT \
 147                         RACER_EXTRA=$RACER_EXTRA \
 148                         RACER_EXTRA_LAYOUT=\\\"$RACER_EXTRA_LAYOUT\\\" \
 149                         RACER_MIGRATE_STRIPE_MAX=$RACER_MIGRATE_STRIPE_MAX \
 150                         RACER_PROGS=$RACER_PROGS \
 151                         NUM_THREADS=$NUM_THREADS \
 152                         MAX_FILES=$MAX_FILES \
 153                         LFS=$LFS \
 154                         LCTL=$LCTL \
 155                         $racer $rdir $NUM_RACER_THREADS" &
 156                 pid=$!
 157                 rpids="$rpids $pid"
 158         done
 159
 160         local failpid=""
 161         if $RACER_FAILOVER; then
 162                 fail_random_facet &
 163                 failpid=$!
 164                 echo racers failpid: $failpid
 165         fi
 166
 167         local lss_pids=""
 168         if $RACER_ENABLE_SNAPSHOT; then
 169                 lss_gen_conf
 170
 171                 $LUSTRE/tests/racer/lss_create.sh &
 172                 pid=$!
 173                 lss_pids="$lss_pids $pid"
 174
 175                 $LUSTRE/tests/racer/lss_destroy.sh &
 176                 pid=$!
 177                 lss_pids="$lss_pids $pid"
 178         fi
 179
 180         echo racers pids: $rpids
 181         for pid in $rpids; do
 182                 wait $pid
 183                 rc=$?
 184                 echo "pid=$pid rc=$rc"
 185                 if [ $rc != 0 ]; then
 186                     rrc=$((rrc + 1))
 187                 fi
 188         done
 189
 190         if $RACER_FAILOVER; then
 191                 touch $racer_done
 192                 wait $failpid
 193                 rrc=$((rrc + $?))
 194         fi
 195
 196         if $RACER_ENABLE_SNAPSHOT; then
 197                 killall -q lss_create.sh
 198                 killall -q lss_destroy.sh
 199
 200                 for pid in $lss_pids; do
 201                         wait $pid
 202                 done
 203
 204                 lss_cleanup
 205         fi
 206
 207         return $rrc
 208 }
 209 run_test 1 "racer on clients: ${CLIENTS:-$(hostname)} DURATION=$DURATION"
 210
 211 complete_test $SECONDS
 212 check_and_cleanup_lustre
 213 exit_status