lustre/tests/racer.sh

   1 #!/bin/bash
   2 #set -vx
   3 set -e
   4
   5 ONLY=${ONLY:-"$*"}
   6 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
   7 . $LUSTRE/tests/test-framework.sh
   8 init_test_env $@
   9 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
  10 init_logging
  11
  12 racer=$LUSTRE/tests/racer/racer.sh
  13 echo racer: $racer with $MDSCOUNT MDTs
  14
  15 if [ "$SLOW" = "no" ]; then
  16     DURATION=${DURATION:-300}
  17 else
  18     DURATION=${DURATION:-900}
  19 fi
  20 MOUNT_2=${MOUNT_2:-"yes"}
  21
  22 build_test_filter
  23 check_and_setup_lustre
  24
  25 CLIENTS=${CLIENTS:-$HOSTNAME}
  26 RACERDIRS=${RACERDIRS:-"$DIR $DIR2"}
  27 echo RACERDIRS=$RACERDIRS
  28
  29 RACER_FAILOVER=${RACER_FAILOVER:-false}
  30 FAIL_TARGETS=${FAIL_TARGETS:-"MDS OST"}
  31 RACER_FAILOVER_PERIOD=${RACER_FAILOVER_PERIOD:-60}
  32
  33 if $RACER_FAILOVER; then
  34         declare -a  victims
  35         for target in $FAIL_TARGETS; do
  36                 victims=(${victims[@]} $(get_facets $target))
  37         done
  38         echo Victim facets ${victims[@]}
  39 fi
  40
  41 #LU-4684
  42 RACER_ENABLE_MIGRATION=false
  43
  44 if ((MDSCOUNT > 1 && "$MDS1_VERSION" >= $(version_code 2.8.0))); then
  45         RACER_ENABLE_REMOTE_DIRS=${RACER_ENABLE_REMOTE_DIRS:-true}
  46         RACER_ENABLE_STRIPED_DIRS=${RACER_ENABLE_STRIPED_DIRS:-true}
  47         RACER_ENABLE_MIGRATION=${RACER_ENABLE_MIGRATION:-true}
  48 elif ((MDSCOUNT > 1 && "$MDS1_VERSION" >= $(version_code 2.5.0))); then
  49         RACER_ENABLE_REMOTE_DIRS=${RACER_ENABLE_REMOTE_DIRS:-true}
  50 fi
  51
  52 [[ "$MDS1_VERSION" -lt $(version_code 2.9.54) ||
  53    $(facet_fstype mgs) != zfs ]] && RACER_ENABLE_SNAPSHOT=false
  54
  55 [[ "$MDS1_VERSION" -le $(version_code 2.9.55) ]] &&
  56         RACER_ENABLE_PFL=false
  57
  58 [[ "$MDS1_VERSION" -le $(version_code 2.10.53) ]] &&
  59         RACER_ENABLE_DOM=false
  60
  61 [[ "$MDS1_VERSION" -lt $(version_code 2.10.55) ]] &&
  62         RACER_ENABLE_FLR=false
  63
  64 [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.12.0) ]] &&
  65         RACER_ENABLE_SEL=false
  66
  67 RACER_ENABLE_REMOTE_DIRS=${RACER_ENABLE_REMOTE_DIRS:-false}
  68 RACER_ENABLE_STRIPED_DIRS=${RACER_ENABLE_STRIPED_DIRS:-false}
  69 RACER_ENABLE_MIGRATION=${RACER_ENABLE_MIGRATION:-false}
  70 RACER_ENABLE_SNAPSHOT=${RACER_ENABLE_SNAPSHOT:-true}
  71 RACER_ENABLE_PFL=${RACER_ENABLE_PFL:-true}
  72 RACER_ENABLE_DOM=${RACER_ENABLE_DOM:-true}
  73 RACER_ENABLE_FLR=${RACER_ENABLE_FLR:-true}
  74 RACER_ENABLE_SEL=${RACER_ENABLE_SEL:-true}
  75
  76 fail_random_facet () {
  77         local facets=${victims[@]}
  78         facets=${facets// /,}
  79
  80         sleep $RACER_FAILOVER_PERIOD
  81         while [ ! -f $racer_done ]; do
  82                 local facet=$(get_random_entry $facets)
  83                 facet_failover $facet
  84                 sleep $RACER_FAILOVER_PERIOD
  85         done
  86 }
  87
  88 # run racer
  89 test_1() {
  90         local rrc=0
  91         local rc=0
  92         local clients=$CLIENTS
  93         local RDIRS
  94         local i
  95         local racer_done=$TMP/racer_done
  96
  97         rm -f $racer_done
  98
  99         for d in ${RACERDIRS}; do
 100                 is_mounted $d || continue
 101
 102                 RDIRS="$RDIRS $d/racer"
 103                 mkdir -p $d/racer
 104         #       lfs setstripe $d/racer -c -1
 105                 if [ $MDSCOUNT -ge 2 ]; then
 106                         for i in $(seq $((MDSCOUNT - 1))); do
 107                                 RDIRS="$RDIRS $d/racer$i"
 108                                 if [ ! -e $d/racer$i ]; then
 109                                         $LFS mkdir -i $i $d/racer$i ||
 110                                                 error "lfs mkdir $i failed"
 111                                 fi
 112                         done
 113                 fi
 114         done
 115
 116         local rpids=""
 117         for rdir in $RDIRS; do
 118                 do_nodes $clients "DURATION=$DURATION \
 119                         MDSCOUNT=$MDSCOUNT OSTCOUNT=$OSTCOUNT\
 120                         RACER_ENABLE_REMOTE_DIRS=$RACER_ENABLE_REMOTE_DIRS \
 121                         RACER_ENABLE_STRIPED_DIRS=$RACER_ENABLE_STRIPED_DIRS \
 122                         RACER_ENABLE_MIGRATION=$RACER_ENABLE_MIGRATION \
 123                         RACER_ENABLE_PFL=$RACER_ENABLE_PFL \
 124                         RACER_ENABLE_DOM=$RACER_ENABLE_DOM \
 125                         RACER_ENABLE_FLR=$RACER_ENABLE_FLR \
 126                         RACER_MAX_CLEANUP_WAIT=$RACER_MAX_CLEANUP_WAIT \
 127                         RACER_ENABLE_SEL=$RACER_ENABLE_SEL \
 128                         LFS=$LFS \
 129                         LCTL=$LCTL \
 130                         $racer $rdir $NUM_RACER_THREADS" &
 131                 pid=$!
 132                 rpids="$rpids $pid"
 133         done
 134
 135         local failpid=""
 136         if $RACER_FAILOVER; then
 137                 fail_random_facet &
 138                 failpid=$!
 139                 echo racers failpid: $failpid
 140         fi
 141
 142         local lss_pids=""
 143         if $RACER_ENABLE_SNAPSHOT; then
 144                 lss_gen_conf
 145
 146                 $LUSTRE/tests/racer/lss_create.sh &
 147                 pid=$!
 148                 lss_pids="$lss_pids $pid"
 149
 150                 $LUSTRE/tests/racer/lss_destroy.sh &
 151                 pid=$!
 152                 lss_pids="$lss_pids $pid"
 153         fi
 154
 155         echo racers pids: $rpids
 156         for pid in $rpids; do
 157                 wait $pid
 158                 rc=$?
 159                 echo "pid=$pid rc=$rc"
 160                 if [ $rc != 0 ]; then
 161                     rrc=$((rrc + 1))
 162                 fi
 163         done
 164
 165         if $RACER_FAILOVER; then
 166                 touch $racer_done
 167                 wait $failpid
 168                 rrc=$((rrc + $?))
 169         fi
 170
 171         if $RACER_ENABLE_SNAPSHOT; then
 172                 killall -q lss_create.sh
 173                 killall -q lss_destroy.sh
 174
 175                 for pid in $lss_pids; do
 176                         wait $pid
 177                 done
 178
 179                 lss_cleanup
 180         fi
 181
 182         return $rrc
 183 }
 184 run_test 1 "racer on clients: ${CLIENTS:-$(hostname)} DURATION=$DURATION"
 185
 186 complete $SECONDS
 187 check_and_cleanup_lustre
 188 exit_status