2 # -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
3 # vim:shiftwidth=4:softtabstop=4:tabstop=4:
8 ALWAYS_EXCEPT="$LARGE_SCALE_EXCEPT"
11 PTLDEBUG=${PTLDEBUG:--1}
12 LUSTRE=${LUSTRE:-`dirname $0`/..}
14 CLEANUP=${CLEANUP:-""}
15 . $LUSTRE/tests/test-framework.sh
19 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
22 remote_mds_nodsh && log "SKIP: remote MDS with nodsh" && exit 0
24 [ -z "$CLIENTS" ] && skip_env "$TESTSUITE: Need two or more clients" && exit 0
25 [ $CLIENTCOUNT -lt 2 ] &&
26 skip_env "$TESTSUITE: Need 2+ clients, have only $CLIENTCOUNT" && exit 0
29 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
34 check_and_setup_lustre
35 rm -rf $DIR/[df][0-9]*
37 get_mpiuser_id $MPI_USER
38 MPI_RUNAS=${MPI_RUNAS:-"runas -u $MPI_USER_UID -g $MPI_USER_GID"}
39 $GSS_KRB5 && refresh_krb5_tgt $MPI_USER_UID $MPI_USER_GID $MPI_RUNAS
41 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
44 assert_env CLIENTS MDSRATE MPIRUN
46 local -a nodes=(${CLIENTS//,/ })
48 # INCREMENT is a number of clients
49 # a half of clients by default
50 increment=${INCREMENT:-$(( CLIENTCOUNT / 2 ))}
52 machinefile=${MACHINEFILE:-$TMP/$TESTSUITE.machines}
53 local LOG=$TMP/${TESTSUITE}_$tfile
55 local var=${SINGLEMDS}_svc
56 local procfile="*.${!var}.recovery_status"
57 local iters=${ITERS:-3}
58 local nfiles=${NFILES:-50000}
59 local nthreads=${THREADS_PER_CLIENT:-3}
61 local IFree=$(inodes_available)
62 [ $IFree -gt $nfiles ] || nfiles=$IFree
64 local dir=$DIR/d0.$TESTNAME
74 while [ $num -le $CLIENTCOUNT ]; do
75 list=$(comma_list ${nodes[@]:0:$num})
77 generate_machine_file $list $machinefile ||
78 { error "can not generate machinefile"; exit 1; }
80 for i in $(seq $iters); do
81 mdsrate_cleanup $num $machinefile $nfiles $dir 'f%%d' --ignore
83 COMMAND="${MDSRATE} --create --nfiles $nfiles --dir $dir --filefmt 'f%%d'"
84 mpi_run -np $((num * nthreads)) -machinefile $machinefile ${COMMAND} | tee ${LOG} &
89 # 2 threads 100000 creates 117 secs
92 log "$i : Starting failover on $SINGLEMDS"
93 facet_failover $SINGLEMDS
94 if ! wait_recovery_complete $SINGLEMDS $((TIMEOUT * 10)); then
95 echo "$SINGLEMDS recovery is not completed!"
100 duration=$(do_facet $SINGLEMDS lctl get_param -n $procfile | grep recovery_duration)
102 res=( "${res[@]}" "$num" )
103 res=( "${res[@]}" "$duration" )
104 echo "RECOVERY TIME: NFILES=$nfiles number of clients: $num $duration"
108 num=$((num + increment))
111 mdsrate_cleanup $num $machinefile $nfiles $dir 'f%%d' --ignore
114 while [ $i -lt ${#res[@]} ]; do
115 echo "RECOVERY TIME: NFILES=$nfiles number of clients: ${res[i]} ${res[i+1]}"
120 run_test 3a "recovery time, $CLIENTCOUNT clients"
123 check_and_cleanup_lustre