From 37c38cc7b4a46ef3fed13b22ac708cfd6fce20c8 Mon Sep 17 00:00:00 2001 From: grev Date: Thu, 26 Mar 2009 14:06:26 +0000 Subject: [PATCH] b=17195 i=Tappro new LARGE_SCALE testsuite test_3a only --- lustre/tests/Makefile.am | 2 +- lustre/tests/acceptance-small.sh | 9 ++- lustre/tests/large-scale.sh | 117 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 lustre/tests/large-scale.sh diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index a3711d8..5c4252d 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -11,7 +11,7 @@ noinst_SCRIPTS += sanity.sh rundbench acceptance-small.sh compile.sh noinst_SCRIPTS += conf-sanity.sh insanity.sh lfscktest.sh oos.sh oos2.sh noinst_SCRIPTS += llog-test.sh recovery-small.sh replay-dual.sh sanity-quota.sh noinst_SCRIPTS += replay-ost-single.sh replay-single.sh run-llog.sh sanityN.sh -noinst_SCRIPTS += runracer replay-vbr.sh +noinst_SCRIPTS += large-scale.sh runracer replay-vbr.sh noinst_SCRIPTS += performance-sanity.sh mdsrate-create-small.sh noinst_SCRIPTS += mdsrate-create-large.sh mdsrate-lookup-1dir.sh noinst_SCRIPTS += mdsrate-stat-small.sh mdsrate-stat-large.sh diff --git a/lustre/tests/acceptance-small.sh b/lustre/tests/acceptance-small.sh index 4c4029e..26442a4 100755 --- a/lustre/tests/acceptance-small.sh +++ b/lustre/tests/acceptance-small.sh @@ -23,7 +23,7 @@ fi [ "$DEBUG_OFF" ] || DEBUG_OFF="eval lctl set_param debug=\"$DEBUG_LVL\"" [ "$DEBUG_ON" ] || DEBUG_ON="eval lctl set_param debug=0x33f0484" -export TESTSUITE_LIST="RUNTESTS SANITY DBENCH BONNIE IOZONE FSX SANITYN LFSCK LIBLUSTRE RACER REPLAY_SINGLE CONF_SANITY RECOVERY_SMALL REPLAY_OST_SINGLE REPLAY_DUAL INSANITY SANITY_QUOTA SANITY_SEC SANITY_GSS PERFORMANCE_SANITY RECOVERY_MDS_SCALE RECOVERY_DOUBLE_SCALE RECOVERY_RANDOM_SCALE" +export TESTSUITE_LIST="RUNTESTS SANITY DBENCH BONNIE IOZONE FSX SANITYN LFSCK LIBLUSTRE RACER REPLAY_SINGLE CONF_SANITY RECOVERY_SMALL REPLAY_OST_SINGLE REPLAY_DUAL INSANITY SANITY_QUOTA SANITY_SEC SANITY_GSS PERFORMANCE_SANITY LARGE_SCALE RECOVERY_MDS_SCALE RECOVERY_DOUBLE_SCALE RECOVERY_RANDOM_SCALE" if [ "$ACC_SM_ONLY" ]; then for O in $TESTSUITE_LIST; do @@ -434,6 +434,13 @@ if [ "$PERFORMANCE_SANITY" != "no" ]; then PERFORMANCE_SANITY="done" fi +[ "$LARGE_SCALE" != "no" ] && skip_remmds large-scale && LARGE_SCALE=no && MSKIPPED=1 +if [ "$LARGE_SCALE" != "no" ]; then + title large-scale + bash large-scale.sh + LARGE_SCALE="done" +fi + [ "$RECOVERY_MDS_SCALE" != "no" ] && skip_remmds recovery-mds-scale && RECOVERY_MDS_SCALE=no && MSKIPPED=1 [ "$RECOVERY_MDS_SCALE" != "no" ] && skip_remost recovery-mds-scale && RECOVERY_MDS_SCALE=no && OSKIPPED=1 if [ "$RECOVERY_MDS_SCALE" != "no" ]; then diff --git a/lustre/tests/large-scale.sh b/lustre/tests/large-scale.sh new file mode 100644 index 0000000..c3ecd82 --- /dev/null +++ b/lustre/tests/large-scale.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +set -e + +# bug number: +ALWAYS_EXCEPT="$LARGE_SCALE_EXCEPT" + +SAVE_PWD=$PWD +PTLDEBUG=${PTLDEBUG:--1} +LUSTRE=${LUSTRE:-`dirname $0`/..} +SETUP=${SETUP:-""} +CLEANUP=${CLEANUP:-""} +. $LUSTRE/tests/test-framework.sh + +init_test_env $@ + +. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} + +remote_mds_nodsh && log "SKIP: remote MDS with nodsh" && exit 0 + +[ -n "$CLIENTS" ] || { skip "$0: Need two or more clients" && exit 0; } +[ $CLIENTCOUNT -ge 2 ] || \ + { skip "$0: Need two or more clients, have $CLIENTCOUNT" && exit 0; } + +# +[ "$SLOW" = "no" ] && EXCEPT_SLOW="" + +MOUNT_2="" +build_test_filter + +check_and_setup_lustre +rm -rf $DIR/[df][0-9]* + +[ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE + +test_3a() { + assert_env CLIENTS MDSRATE MPIRUN + + local -a nodes=(${CLIENTS//,/ }) + + # INCREMENT is a number of clients + # a half of clients by default + increment=${INCREMENT:-$(( CLIENTCOUNT / 2 ))} + + machinefile=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines} + local LOG=$TMP/${TESTSUITE}_$tfile + + local var=${SINGLEMDS}_svc + local procfile="*.${!var}.recovery_status" + local iters=${ITERS:-3} + local nfiles=${NFILES:-50000} + local nthreads=${THREADS_PER_CLIENT:-3} + + local IFree=$(inodes_available) + [ $IFree -gt $nfiles ] || nfiles=$IFree + + local dir=$DIR/$tdir + mkdir -p $dir + chmod 0777 $dir + + local pid + local list + local -a res + + local num=$increment + + while [ $num -le $CLIENTCOUNT ]; do + list=$(comma_list ${nodes[@]:0:$num}) + + generate_machine_file $list $machinefile || + { error "can not generate machinefile"; exit 1; } + + for i in $(seq $iters); do + mdsrate_cleanup $num $machinefile $nfiles $dir 'f%%d' --ignore + + COMMAND="${MDSRATE} --create --nfiles $nfiles --dir $dir --filefmt 'f%%d'" + mpi_run -np $((num * nthreads)) -machinefile $machinefile ${COMMAND} | tee ${LOG} & + + pid=$! + echo "pid=$pid" + + # 2 threads 100000 creates 117 secs + sleep 20 + + log "$i : Starting failover on $SINGLEMDS" + facet_failover $SINGLEMDS + if ! wait_recovery_complete $SINGLEMDS $((TIMEOUT * 10)); then + echo "$SINGLEMDS recovery is not completed!" + kill -9 $pid + exit 7 + fi + + duration=$(do_facet $SINGLEMDS lctl get_param -n $procfile | grep recovery_duration) + + res=( "${res[@]}" "$num" ) + res=( "${res[@]}" "$duration" ) + echo "RECOVERY TIME: NFILES=$nfiles number of clients: $num $duration" + wait $pid + + done + num=$((num + increment)) + done + + mdsrate_cleanup $num $machinefile $nfiles $dir 'f%%d' --ignore + + i=0 + while [ $i -lt ${#res[@]} ]; do + echo "RECOVERY TIME: NFILES=$nfiles number of clients: ${res[i]} ${res[i+1]}" + i=$((i+2)) + done +} + +run_test 3a "recovery time, $CLIENTCOUNT clients" + +equals_msg `basename $0`: test complete, cleaning up +check_and_cleanup_lustre +[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true -- 1.8.3.1