lustre/tests/large-scale.sh

   1 #!/bin/bash
   2 # -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
   3 # vim:shiftwidth=4:softtabstop=4:tabstop=4:
   4
   5 set -e
   6
   7 # bug number:
   8 ALWAYS_EXCEPT="$LARGE_SCALE_EXCEPT"
   9
  10 SAVE_PWD=$PWD
  11 PTLDEBUG=${PTLDEBUG:--1}
  12 LUSTRE=${LUSTRE:-`dirname $0`/..}
  13 SETUP=${SETUP:-""}
  14 CLEANUP=${CLEANUP:-""}
  15 . $LUSTRE/tests/test-framework.sh
  16
  17 init_test_env $@
  18
  19 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
  20 init_logging
  21
  22 remote_mds_nodsh && log "SKIP: remote MDS with nodsh" && exit 0
  23
  24 [ -z "$CLIENTS" ] && skip_env "$TESTSUITE: Need two or more clients" && exit 0
  25 [ $CLIENTCOUNT -lt 2 ] &&
  26         skip_env "$TESTSUITE: Need 2+ clients, have only $CLIENTCOUNT" && exit 0
  27
  28 #
  29 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
  30
  31 MOUNT_2=""
  32 build_test_filter
  33
  34 check_and_setup_lustre
  35 rm -rf $DIR/[df][0-9]*
  36
  37 get_mpiuser_id $MPI_USER
  38 MPI_RUNAS=${MPI_RUNAS:-"runas -u $MPI_USER_UID -g $MPI_USER_GID"}
  39 $GSS_KRB5 && refresh_krb5_tgt $MPI_USER_UID $MPI_USER_GID $MPI_RUNAS
  40
  41 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
  42
  43 test_3a() {
  44     assert_env CLIENTS MDSRATE MPIRUN
  45
  46     local -a nodes=(${CLIENTS//,/ })
  47
  48     # INCREMENT is a number of clients
  49     # a half of clients by default
  50     increment=${INCREMENT:-$(( CLIENTCOUNT / 2 ))}
  51
  52     machinefile=${MACHINEFILE:-$TMP/$TESTSUITE.machines}
  53     local LOG=$TMP/${TESTSUITE}_$tfile
  54
  55     local var=${SINGLEMDS}_svc
  56     local procfile="*.${!var}.recovery_status"
  57     local iters=${ITERS:-3}
  58     local nfiles=${NFILES:-50000}
  59     local nthreads=${THREADS_PER_CLIENT:-3}
  60
  61     local IFree=$(inodes_available)
  62     [ $IFree -gt $nfiles ] || nfiles=$IFree
  63
  64     local dir=$DIR/d0.$TESTNAME
  65     mkdir -p $dir
  66     chmod 0777 $dir
  67
  68     local pid
  69     local list
  70     local -a res
  71
  72     local num=$increment
  73
  74         while [ $num -le $CLIENTCOUNT ]; do
  75                 list=$(comma_list ${nodes[@]:0:$num})
  76
  77                 generate_machine_file $list $machinefile ||
  78                         { error "can not generate machinefile"; exit 1; }
  79
  80                 for i in $(seq $iters); do
  81                         mdsrate_cleanup $num $machinefile $nfiles $dir 'f%%d' \
  82                                 --ignore
  83
  84                         COMMAND="${MDSRATE} --create --nfiles $nfiles --dir
  85                                  $dir --filefmt 'f%%d'"
  86                         mpi_run ${MACHINEFILE_OPTION} $machinefile \
  87                                 -np $((num * nthreads)) ${COMMAND} | tee ${LOG}&
  88
  89                         pid=$!
  90                         echo "pid=$pid"
  91
  92                         # 2 threads 100000 creates 117 secs
  93                         sleep 20
  94
  95                         log "$i : Starting failover on $SINGLEMDS"
  96                         facet_failover $SINGLEMDS
  97                         if ! wait_recovery_complete $SINGLEMDS \
  98                              $((TIMEOUT * 10)); then
  99                                 echo "$SINGLEMDS recovery is not completed!"
 100                                 kill -9 $pid
 101                                 exit 7
 102                         fi
 103
 104                         duration=$(do_facet $SINGLEMDS lctl get_param -n \
 105                                 $procfile | grep recovery_duration)
 106
 107                         res=( "${res[@]}" "$num" )
 108                         res=( "${res[@]}" "$duration" )
 109                         echo "RECOVERY TIME: NFILES=$nfiles number of clients: $num $duration"
 110                         wait $pid
 111                 done
 112                 num=$((num + increment))
 113         done
 114
 115     mdsrate_cleanup $num $machinefile $nfiles $dir 'f%%d' --ignore
 116
 117     i=0
 118     while [ $i -lt ${#res[@]} ]; do
 119         echo "RECOVERY TIME: NFILES=$nfiles number of clients: ${res[i]}  ${res[i+1]}"
 120         i=$((i+2))
 121     done
 122 }
 123
 124 run_test 3a "recovery time, $CLIENTCOUNT clients"
 125
 126 complete $SECONDS
 127 check_and_cleanup_lustre
 128 exit_status