X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Facceptance-small.sh;h=855ba29219841b67cc0b6c02bfac8781cc18ff0a;hp=bb9da5acb380946fbd192e76d1e55888e21b22c3;hb=1efcc0d66298588ff757106441a647867c2fed9e;hpb=d037b8032573a02ce6d67134296ce45c0471f359 diff --git a/lustre/tests/acceptance-small.sh b/lustre/tests/acceptance-small.sh index bb9da5a..855ba29 100755 --- a/lustre/tests/acceptance-small.sh +++ b/lustre/tests/acceptance-small.sh @@ -4,8 +4,6 @@ #set -vx set -e -PATH=`dirname $0`/../utils:$PATH - [ -z "$CONFIG" -a "$NAME" ] && CONFIGS=$NAME [ "$CONFIGS" ] || CONFIGS="local" #"local lov" [ "$MAX_THREADS" ] || MAX_THREADS=20 @@ -22,16 +20,17 @@ fi [ "$TMP" ] || TMP=/tmp [ "$COUNT" ] || COUNT=1000 [ "$DEBUG_LVL" ] || DEBUG_LVL=0 -[ "$DEBUG_OFF" ] || DEBUG_OFF="eval sysctl -w lnet.debug=\"$DEBUG_LVL\"" -[ "$DEBUG_ON" ] || DEBUG_ON="eval sysctl -w lnet.debug=0x33f0484" +[ "$DEBUG_OFF" ] || DEBUG_OFF="eval lctl set_param debug=\"$DEBUG_LVL\"" +[ "$DEBUG_ON" ] || DEBUG_ON="eval lctl set_param debug=0x33f0484" -export TESTSUITE_LIST="RUNTESTS SANITY DBENCH BONNIE IOZONE FSX SANITYN LFSCK LIBLUSTRE REPLAY_SINGLE CONF_SANITY RECOVERY_SMALL REPLAY_OST_SINGLE REPLAY_DUAL INSANITY SANITY_QUOTA SANITY_SEC" +export TESTSUITE_LIST="RUNTESTS SANITY DBENCH BONNIE IOZONE FSX SANITYN LFSCK LIBLUSTRE RACER REPLAY_SINGLE CONF_SANITY RECOVERY_SMALL REPLAY_OST_SINGLE REPLAY_DUAL REPLAY_VBR INSANITY SANITY_QUOTA SANITY_SEC SANITY_GSS PERFORMANCE_SANITY LARGE_SCALE RECOVERY_MDS_SCALE RECOVERY_DOUBLE_SCALE RECOVERY_RANDOM_SCALE PARALLEL_SCALE" if [ "$ACC_SM_ONLY" ]; then for O in $TESTSUITE_LIST; do export ${O}="no" done for O in $ACC_SM_ONLY; do + O=`echo ${O%.sh} | tr "-" "_"` O=`echo $O | tr "[:lower:]" "[:upper:]"` export ${O}="yes" done @@ -43,13 +42,15 @@ LIBLUSTRETESTS=${LIBLUSTRETESTS:-../liblustre/tests} STARTTIME=`date +%s` RANTEST="" -LUSTRE=${LUSTRE:-`dirname $0`/..} +LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} . $LUSTRE/tests/test-framework.sh init_test_env $@ if $GSS; then # liblustre doesn't support GSS export LIBLUSTRE=no +else + export SANITY_GSS="no" fi SETUP=${SETUP:-setupall} @@ -57,25 +58,69 @@ FORMAT=${FORMAT:-formatall} CLEANUP=${CLEANUP:-stopall} setup_if_needed() { - mount | grep $MOUNT && return - $FORMAT && $SETUP + local MOUNTED=$(mounted_lustre_filesystems) + if $(echo $MOUNTED | grep -w -q $MOUNT); then + check_config $MOUNT + init_facets_vars + init_param_vars + return + fi + + echo "Lustre is not mounted, trying to do setup SETUP=$SETUP ... " + [ "$REFORMAT" ] && $FORMAT + $SETUP + + MOUNTED=$(mounted_lustre_filesystems) + if ! $(echo $MOUNTED | grep -w -q $MOUNT); then + echo "Lustre is not mounted after setup! SETUP=$SETUP" + exit 1 + fi } title() { + # update titlebar if stdin is attaached to an xterm + if ${UPDATE_TITLEBAR:-false}; then + if tty -s; then + case $TERM in + xterm*) + echo -ne "\033]2; acceptance-small: $* \007" >&0 + ;; + esac + fi + fi log "-----============= acceptance-small: "$*" ============----- `date`" RANTEST=${RANTEST}$*", " } +skip_remost() +{ + remote_ost_nodsh && log "SKIP: $1: remote OST with nodsh" && return 0 + return 1 +} + +skip_remmds() +{ + remote_mds_nodsh && log "SKIP: $1: remote MDS with nodsh" && return 0 + return 1 +} + for NAME in $CONFIGS; do export NAME MOUNT START CLEAN . $LUSTRE/tests/cfg/$NAME.sh + + if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mds.ko -a \ + ! -f `dirname $0`/../mds/mds.ko ]; then + export CLIENTMODSONLY=true + fi - assert_env mds_HOST MDS_MKFS_OPTS MDSDEV + assert_env mds_HOST MDS_MKFS_OPTS assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT assert_env FSNAME MOUNT MOUNT2 setup_if_needed + MSKIPPED=0 + OSKIPPED=0 if [ "$RUNTESTS" != "no" ]; then title runtests bash runtests @@ -104,19 +149,19 @@ for NAME in $CONFIGS; do $DEBUG_OFF myUID=$RUNAS_ID myRUNAS=$RUNAS - FAIL_ON_ERROR=false check_runas_id_ret $myUID $myRUNAS || { myRUNAS="" && myUID=$UID; } + FAIL_ON_ERROR=false check_runas_id_ret $myUID $myUID $myRUNAS || { myRUNAS="" && myUID=$UID; } chown $myUID:$myUID $DBENCHDIR duration="" [ "$SLOW" = "no" ] && duration=" -t 120" if [ "$SLOW" != "no" -o $DB_THREADS -eq 1 ]; then - DIR=$DBENCHDIR $myRUNAS bash rundbench 1 $duration + $myRUNAS bash rundbench -D $DBENCHDIR 1 $duration || error "dbench failed!" $DEBUG_ON $CLEANUP $SETUP fi if [ $DB_THREADS -gt 1 ]; then $DEBUG_OFF - DIR=$DBENCHDIR $myRUNAS bash rundbench $DB_THREADS $duration + $myRUNAS bash rundbench -D $DBENCHDIR $DB_THREADS $duration $DEBUG_ON $CLEANUP $SETUP @@ -132,14 +177,14 @@ for NAME in $CONFIGS; do mkdir -p $BONDIR $LFS setstripe -c -1 $BONDIR sync - MIN=`cat /proc/fs/lustre/osc/*/kbytesavail | sort -n | head -n1` + MIN=`lctl get_param -n osc.*.kbytesavail | sort -n | head -n1` SPACE=$(( OSTCOUNT * MIN )) [ $SPACE -lt $SIZE ] && SIZE=$((SPACE * 3 / 4)) log "min OST has ${MIN}kB available, using ${SIZE}kB file size" $DEBUG_OFF myUID=$RUNAS_ID myRUNAS=$RUNAS - FAIL_ON_ERROR=false check_runas_id_ret $myUID $myRUNAS || { myRUNAS="" && myUID=$UID; } + FAIL_ON_ERROR=false check_runas_id_ret $myUID $myUID $myRUNAS || { myRUNAS="" && myUID=$UID; } chown $myUID:$myUID $BONDIR $myRUNAS bonnie++ -f -r 0 -s$((SIZE / 1024)) -n 10 -u$myUID:$myUID -d$BONDIR $DEBUG_ON @@ -158,19 +203,23 @@ for NAME in $CONFIGS; do mkdir -p $IOZDIR $LFS setstripe -c -1 $IOZDIR sync - MIN=`cat /proc/fs/lustre/osc/*/kbytesavail | sort -n | head -n1` + MIN=`lctl get_param -n osc.*.kbytesavail | sort -n | head -n1` SPACE=$(( OSTCOUNT * MIN )) [ $SPACE -lt $SIZE ] && SIZE=$((SPACE * 3 / 4)) log "min OST has ${MIN}kB available, using ${SIZE}kB file size" - IOZONE_OPTS="-i 0 -i 1 -i 2 -e -+d -r $RSIZE -s $SIZE" + IOZONE_OPTS="-i 0 -i 1 -i 2 -e -+d -r $RSIZE" IOZFILE="$IOZDIR/iozone" + IOZLOG=$TMP/iozone.log # $SPACE was calculated with all OSTs $DEBUG_OFF myUID=$RUNAS_ID myRUNAS=$RUNAS - FAIL_ON_ERROR=false check_runas_id_ret $myUID $myRUNAS || { myRUNAS="" && myUID=$UID; } + FAIL_ON_ERROR=false check_runas_id_ret $myUID $myUID $myRUNAS || { myRUNAS="" && myUID=$UID; } chown $myUID:$myUID $IOZDIR - $myRUNAS iozone $IOZONE_OPTS -f $IOZFILE + $myRUNAS iozone $IOZONE_OPTS -s $SIZE -f $IOZFILE 2>&1 | tee $IOZLOG + tail -1 $IOZLOG | grep -q complete || \ + { error "iozone (1) failed" && false; } + rm -f $IOZLOG $DEBUG_ON $CLEANUP $SETUP @@ -179,14 +228,17 @@ for NAME in $CONFIGS; do if [ -z "$O_DIRECT" ]; then touch $MOUNT/f.iozone if ! ./directio write $MOUNT/f.iozone 0 1; then + log "SKIP iozone DIRECT IO test" O_DIRECT=no fi rm -f $MOUNT/f.iozone fi if [ "$O_DIRECT" != "no" -a "$IOZONE_DIR" != "no" ]; then $DEBUG_OFF - # cd TMP to have write permission for tmp file iozone writes - ( cd $TMP && $myRUNAS iozone -I $IOZONE_OPTS $IOZFILE.odir ) + $myRUNAS iozone -I $IOZONE_OPTS -s $SIZE -f $IOZFILE.odir 2>&1 | tee $IOZLOG + tail -1 $IOZLOG | grep -q complete || \ + { error "iozone (2) failed" && false; } + rm -f $IOZLOG $DEBUG_ON $CLEANUP $SETUP @@ -197,15 +249,18 @@ for NAME in $CONFIGS; do [ $THREADS -lt $IOZ_THREADS ] && IOZ_THREADS=$THREADS IOZVER=`iozone -v | awk '/Revision:/ {print $3}' | tr -d .` if [ "$IOZ_THREADS" -gt 1 -a "$IOZVER" -ge 3145 ]; then - $LFS setstripe -c 1 $IOZDIR + $LFS setstripe -c -1 $IOZDIR $DEBUG_OFF THREAD=1 - IOZFILE="-F " + IOZFILE=" " while [ $THREAD -le $IOZ_THREADS ]; do IOZFILE="$IOZFILE $IOZDIR/iozone.$THREAD" THREAD=$((THREAD + 1)) done - $myRUNAS iozone $IOZONE_OPTS -t $IOZ_THREADS $IOZFILE + $myRUNAS iozone $IOZONE_OPTS -s $((SIZE / IOZ_THREADS)) -t $IOZ_THREADS -F $IOZFILE 2>&1 | tee $IOZLOG + tail -1 $IOZLOG | grep -q complete || \ + { error "iozone (3) failed" && false; } + rm -f $IOZLOG $DEBUG_ON $CLEANUP $SETUP @@ -218,10 +273,15 @@ for NAME in $CONFIGS; do if [ "$FSX" != "no" ]; then title fsx + FSX_SIZE=$((RAMKB / 2)) SPACE=`df -P $MOUNT | tail -n 1 | awk '{ print $4 }'` - [ $SPACE -lt $SIZE ] && SIZE=$((SPACE * 3 / 4)) + [ $SPACE -lt $FSX_SIZE ] && FSX_SIZE=$((SPACE * 3 / 4)) $DEBUG_OFF - ./fsx -c 50 -p 1000 -P $TMP -l $SIZE \ + FSX_SEED=${FSX_SEED:-$RANDOM} + rm -f $MOUNT/fsxfile + $LFS setstripe -c -1 $MOUNT/fsxfile + echo Using FSX_SEED=$FSX_SEED FSX_SIZE=$FSX_SIZE COUNT=$COUNT + ./fsx -c 50 -p 1000 -S $FSX_SEED -P $TMP -l $FSX_SIZE \ -N $(($COUNT * 100)) $MOUNT/fsxfile $DEBUG_ON $CLEANUP @@ -237,7 +297,7 @@ for NAME in $CONFIGS; do mount_client $MOUNT2 #echo "can't mount2 for '$NAME', skipping sanityN.sh" START=: CLEAN=: bash sanityN.sh - umount $MOUNT2 + [ "$(mount | grep $MOUNT2)" ] && umount $MOUNT2 $DEBUG_ON $CLEANUP @@ -245,100 +305,180 @@ for NAME in $CONFIGS; do SANITYN="done" fi - if [ "$LFSCK" != "no" -a -x /usr/sbin/lfsck ]; then + [ "$LFSCK" != "no" ] && remote_mds && log "Remote MDS, skipping LFSCK test" && LFSCK=no && MSKIPPED=1 + [ "$LFSCK" != "no" ] && remote_ost && log "Remote OST, skipping LFSCK test" && LFSCK=no && OSKIPPED=1 + if [ "$LFSCK" != "no" ]; then title lfsck - E2VER=`e2fsck -V 2>&1 | head -n 1 | cut -d' ' -f 2` - if grep -q obdfilter /proc/fs/lustre/devices; then - if [ `echo $E2VER | cut -d. -f2` -ge 39 ] && \ - [ "`echo $E2VER | grep cfs`" ]; then - bash lfscktest.sh - else - e2fsck -V - echo "e2fsck does not support lfsck, skipping" - fi + if [ -x /usr/sbin/lfsck ]; then + bash lfscktest.sh else - echo "remote OST, skipping test" + log "$($E2FSCK -V)" + log "SKIP: $E2FSCK does not support lfsck" fi LFSCK="done" fi + [ "$NETTYPE" = "tcp" -o "$NETTYPE" = "ptl" ] || LIBLUSTRE=no # bug 15660 + if [ "$LIBLUSTRE" != "no" ] && ! check_versions ; then + skip liblustre version mismatch: cli $CLIVER, mds $MDSVER, ost $OSTVER + LIBLUSTRE=no # bug 17696 + fi if [ "$LIBLUSTRE" != "no" ]; then - title liblustre + title liblustre assert_env MGSNID MOUNT2 - $CLEANUP - unload_modules - # Liblustre needs accept=all, noacl - [ -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf - [ -f /etc/modprobe.d/Lustre ] && MODPROBECONF=/etc/modprobe.d/Lustre - - LNETOPTS="$(awk '/^options lnet/ { print $0}' $MODPROBECONF | \ - sed 's/^options lnet //g') accept=all" \ - MDS_MOUNT_OPTS=$(echo $MDS_MOUNT_OPTS | sed 's/^[ \t]*//;s/[ \t]*$//') \ - MDS_MOUNT_OPTS="${MDS_MOUNT_OPTS},noacl" \ - MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS/#,/-o } \ - $SETUP export LIBLUSTRE_MOUNT_POINT=$MOUNT2 + export LIBLUSTRE_MOUNT_RETRY=5 export LIBLUSTRE_MOUNT_TARGET=$MGSNID:/$FSNAME - export LIBLUSTRE_TIMEOUT=`cat /proc/sys/lustre/timeout` - #export LIBLUSTRE_DEBUG_MASK=`cat /proc/sys/lnet/debug` + export LIBLUSTRE_TIMEOUT=`lctl get_param -n timeout` + #export LIBLUSTRE_DEBUG_MASK=`lctl get_param -n debug` if [ -x $LIBLUSTRETESTS/sanity ]; then mkdir -p $MOUNT2 - echo $LIBLUSTRETESTS/sanity --target=$LIBLUSTRE_MOUNT_TARGET - $LIBLUSTRETESTS/sanity --target=$LIBLUSTRE_MOUNT_TARGET + if [ "$LIBLUSTRE_EXCEPT" ]; then + LIBLUSTRE_OPT="$LIBLUSTRE_OPT \ + $(echo ' '$LIBLUSTRE_EXCEPT | sed -re 's/\s+/ -e /g')" + fi + echo $LIBLUSTRETESTS/sanity --target=$LIBLUSTRE_MOUNT_TARGET $LIBLUSTRE_OPT + $LIBLUSTRETESTS/sanity --target=$LIBLUSTRE_MOUNT_TARGET $LIBLUSTRE_OPT fi $CLEANUP #$SETUP LIBLUSTRE="done" fi - $CLEANUP + [ "$RACER" != "no" ] && [ -n "$CLIENTS" -a "$PDSH" = "no_dsh" ] && log "Remote client with no_dsh" && RACER=no + if [ "$RACER" != "no" ]; then + title racer + setup_if_needed + DURATION=${DURATION:-900} + [ "$SLOW" = "no" ] && DURATION=300 + RACERCLIENTS=$HOSTNAME + [ ! -z ${CLIENTS} ] && RACERCLIENTS=$CLIENTS + log "racer on clients: $RACERCLIENTS DURATION=$DURATION RACERDIRS=$RACERDIRS" + CLIENTS=${RACERCLIENTS} DURATION=$DURATION bash runracer $RACERDIRS + $CLEANUP + $SETUP + RACER="done" + fi done +[ "$REPLAY_SINGLE" != "no" ] && skip_remmds replay-single && REPLAY_SINGLE=no && MSKIPPED=1 if [ "$REPLAY_SINGLE" != "no" ]; then title replay-single bash replay-single.sh REPLAY_SINGLE="done" fi +[ "$CONF_SANITY" != "no" ] && skip_remmds conf-sanity && CONF_SANITY=no && MSKIPPED=1 +[ "$CONF_SANITY" != "no" ] && skip_remost conf-sanity && CONF_SANITY=no && OSKIPPED=1 if [ "$CONF_SANITY" != "no" ]; then title conf-sanity bash conf-sanity.sh CONF_SANITY="done" fi +[ "$RECOVERY_SMALL" != "no" ] && skip_remmds recover-small && RECOVERY_SMALL=no && MSKIPPED=1 if [ "$RECOVERY_SMALL" != "no" ]; then title recovery-small bash recovery-small.sh RECOVERY_SMALL="done" fi +[ "$REPLAY_OST_SINGLE" != "no" ] && skip_remost replay-ost-single && REPLAY_OST_SINGLE=no && OSKIPPED=1 if [ "$REPLAY_OST_SINGLE" != "no" ]; then title replay-ost-single bash replay-ost-single.sh REPLAY_OST_SINGLE="done" fi +[ "$REPLAY_DUAL" != "no" ] && skip_remost replay-dual && REPLAY_DUAL=no && OSKIPPED=1 if [ "$REPLAY_DUAL" != "no" ]; then title replay-dual bash replay-dual.sh REPLAY_DUAL="done" fi +[ "$REPLAY_VBR" != "no" ] && skip_remmds replay-vbr && REPLAY_VBR=no && MSKIPPED=1 +if [ "$REPLAY_VBR" != "no" ]; then + title replay-vbr + bash replay-vbr.sh + REPLAY_VBR="done" +fi + +[ "$INSANITY" != "no" ] && skip_remmds insanity && INSANITY=no && MSKIPPED=1 +[ "$INSANITY" != "no" ] && skip_remost insanity && INSANITY=no && OSKIPPED=1 if [ "$INSANITY" != "no" ]; then title insanity bash insanity.sh -r INSANITY="done" fi +[ "$SANITY_QUOTA" != "no" ] && skip_remmds sanity-quota && SANITY_QUOTA=no && MSKIPPED=1 +[ "$SANITY_QUOTA" != "no" ] && skip_remost sanity-quota && SANITY_QUOTA=no && OSKIPPED=1 if [ "$SANITY_QUOTA" != "no" ]; then title sanity-quota bash sanity-quota.sh SANITY_QUOTA="done" fi +[ "$SANITY_SEC" != "no" ] && skip_remmds sanity-sec && SANITY_SEC=no && MSKIPPED=1 +[ "$SANITY_SEC" != "no" ] && skip_remost sanity-sec && SANITY_SEC=no && OSKIPPED=1 if [ "$SANITY_SEC" != "no" ]; then title sanity-sec bash sanity-sec.sh + SANITY_SEC="done" +fi + +[ "$SANITY_GSS" != "no" ] && skip_remmds sanity-gss && SANITY_GSS=no && MSKIPPED=1 +if [ "$SANITY_GSS" != "no" ]; then + title sanity-gss + bash sanity-gss.sh + SANITY_GSS="done" +fi + +[ "$SLOW" = no ] && PERFORMANCE_SANITY="no" +[ -x "$MDSRATE" ] || PERFORMANCE_SANITY="no" +which mpirun > /dev/null 2>&1 || PERFORMANCE_SANITY="no" +if [ "$PERFORMANCE_SANITY" != "no" ]; then + title performance-sanity + bash performance-sanity.sh + PERFORMANCE_SANITY="done" +fi + +[ "$LARGE_SCALE" != "no" ] && skip_remmds large-scale && LARGE_SCALE=no && MSKIPPED=1 +if [ "$LARGE_SCALE" != "no" ]; then + title large-scale + bash large-scale.sh + LARGE_SCALE="done" +fi + +[ "$RECOVERY_MDS_SCALE" != "no" ] && skip_remmds recovery-mds-scale && RECOVERY_MDS_SCALE=no && MSKIPPED=1 +[ "$RECOVERY_MDS_SCALE" != "no" ] && skip_remost recovery-mds-scale && RECOVERY_MDS_SCALE=no && OSKIPPED=1 +if [ "$RECOVERY_MDS_SCALE" != "no" ]; then + title recovery-mds-scale + bash recovery-mds-scale.sh + RECOVERY_MDS_SCALE="done" +fi + +[ "$RECOVERY_DOUBLE_SCALE" != "no" ] && skip_remmds recovery-double-scale && RECOVERY_DOUBLE_SCALE=no && MSKIPPED=1 +[ "$RECOVERY_DOUBLE_SCALE" != "no" ] && skip_remost recovery-double-scale && RECOVERY_DOUBLE_SCALE=no && OSKIPPED=1 +if [ "$RECOVERY_DOUBLE_SCALE" != "no" ]; then + title recovery-double-scale + bash recovery-double-scale.sh + RECOVERY_DOUBLE_SCALE="done" +fi + +[ "$RECOVERY_RANDOM_SCALE" != "no" ] && skip_remmds recovery-random-scale && RECOVERY_RANDOM_SCALE=no && MSKIPPED=1 +if [ "$RECOVERY_RANDOM_SCALE" != "no" ]; then + title recovery-random-scale + bash recovery-random-scale.sh + RECOVERY_RANDOM_SCALE="done" +fi + +which mpirun > /dev/null 2>&1 || PARALLEL_SCALE="no" +if [ "$PARALLEL_SCALE" != "no" ]; then + title parallel-scale + bash parallel-scale.sh + PARALLEL_SCALE="done" fi RC=$? @@ -346,4 +486,6 @@ title FINISHED echo "Finished at `date` in $((`date +%s` - $STARTTIME))s" echo "Tests ran: $RANTEST" print_summary +[ "$MSKIPPED" = 1 ] && log "FAIL: remote MDS tests skipped" && RC=1 +[ "$OSKIPPED" = 1 ] && log "FAIL: remote OST tests skipped" && RC=1 echo "$0: completed with rc $RC" && exit $RC