X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=b7216d25095334581485b72d22503749c2885c31;hp=bd45247ddf55cf01380d15754ae2663808d00497;hb=50bbf65fec92adbcedf1706eb881e2a156874d0d;hpb=16ddb77519b26ea8df18b5abd84afb03d08c47dc diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index bd45247..b7216d2 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1,14 +1,14 @@ #!/bin/bash -# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4: trap 'print_summary && touch $TF_FAIL && \ echo "test-framework exiting on error"' ERR set -e #set -x +export EJOURNAL=${EJOURNAL:-""} export REFORMAT=${REFORMAT:-""} export WRITECONF=${WRITECONF:-""} -export VERBOSE=false +export VERBOSE=${VERBOSE:-false} export CATASTROPHE=${CATASTROPHE:-/proc/sys/lnet/catastrophe} export GSS=false export GSS_KRB5=false @@ -16,6 +16,10 @@ export GSS_PIPEFS=false export IDENTITY_UPCALL=default export QUOTA_AUTO=1 +# LOAD_LLOOP: LU-409: only load llite_lloop module if kernel < 2.6.32 or +# LOAD_LLOOP is true. LOAD_LLOOP is false by default. +export LOAD_LLOOP=${LOAD_LLOOP:-false} + #export PDSH="pdsh -S -Rssh -w" # function used by scripts run on remote nodes @@ -33,8 +37,13 @@ if [ -f "$EXCEPT_LIST_FILE" ]; then . $EXCEPT_LIST_FILE fi -[ -z "$MODPROBECONF" -a -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf -[ -z "$MODPROBECONF" -a -f /etc/modprobe.d/Lustre ] && MODPROBECONF=/etc/modprobe.d/Lustre +# check config files for options in decreasing order of preference +[ -z "$MODPROBECONF" -a -f /etc/modprobe.d/lustre.conf ] && + MODPROBECONF=/etc/modprobe.d/lustre.conf +[ -z "$MODPROBECONF" -a -f /etc/modprobe.d/Lustre ] && + MODPROBECONF=/etc/modprobe.d/Lustre +[ -z "$MODPROBECONF" -a -f /etc/modprobe.conf ] && + MODPROBECONF=/etc/modprobe.conf assert_DIR () { local failed="" @@ -58,44 +67,50 @@ usage() { print_summary () { trap 0 [ "$TESTSUITE" == "lfsck" ] && return 0 - [ -n "$ONLY" ] && echo "WARNING: ONLY is set to ${ONLY}." - local form="%-13s %-17s %s\n" - printf "$form" "status" "script" "skipped tests E(xcluded) S(low)" + [ -n "$ONLY" ] && echo "WARNING: ONLY is set to $(echo $ONLY)" + local details + local form="%-13s %-17s %-9s %s %s\n" + printf "$form" "status" "script" "Total(sec)" "E(xcluded) S(low)" echo "------------------------------------------------------------------------------------" for O in $DEFAULT_SUITES; do - local skipped="" - local slow="" O=$(echo $O | tr "-" "_" | tr "[:lower:]" "[:upper:]") - local o=$(echo $O | tr "[:upper:]" "[:lower:]") - o=${o//_/-} + [ "${!O}" = "no" ] && continue || true + local o=$(echo $O | tr "[:upper:]_" "[:lower:]-") local log=${TMP}/${o}.log - [ -f $log ] && skipped=$(grep excluded $log | awk '{ printf " %s", $3 }' | sed 's/test_//g') - [ -f $log ] && slow=$(grep SLOW $log | awk '{ printf " %s", $3 }' | sed 's/test_//g') - [ "${!O}" = "done" ] && \ - printf "$form" "Done" "$O" "E=$skipped" && \ - [ -n "$slow" ] && printf "$form" "-" "-" "S=$slow" - + if is_sanity_benchmark $o; then + log=${TMP}/sanity-benchmark.log + fi + local slow= + local skipped= + local total= + local status=Unfinished + if [ -f $log ]; then + skipped=$(grep excluded $log | awk '{ printf " %s", $3 }' | sed 's/test_//g') + slow=$(egrep "^PASS|^FAIL" $log | tr -d "("| sed s/s\)$//g | sort -nr -k 3 | head -5 | awk '{ print $2":"$3"s" }') + total=$(grep duration $log | awk '{ print $2}') + if [ "${!O}" = "done" ]; then + status=Done + fi + if $DDETAILS; then + local durations=$(egrep "^PASS|^FAIL" $log | tr -d "("| sed s/s\)$//g | awk '{ print $2":"$3"|" }') + details=$(printf "%s\n%s %s %s\n" "$details" "DDETAILS" "$O" "$(echo $durations)") + fi + fi + printf "$form" $status "$O" "${total}" "E=$skipped" + printf "$form" "-" "-" "-" "S=$(echo $slow)" done for O in $DEFAULT_SUITES; do O=$(echo $O | tr "-" "_" | tr "[:lower:]" "[:upper:]") if [ "${!O}" = "no" ]; then - # FIXME. - # only for those tests suits which are run directly from acc-sm script: - # bonnie, iozone, etc. - if [ -f "$TESTSUITELOG" ] && grep FAIL $TESTSUITELOG | grep -q ' '$O ; then - printf "$form" "UNFINISHED" "$O" "" - else - printf "$form" "Skipped" "$O" "" - fi + printf "$form" "Skipped" "$O" "" fi done - for O in $DEFAULT_SUITES; do - O=$(echo $O | tr "-" "_" | tr "[:lower:]" "[:upper:]") - [ "${!O}" = "done" -o "${!O}" = "no" ] || \ - printf "$form" "UNFINISHED" "$O" "" - done + # print the detailed tests durations if DDETAILS=true + if $DDETAILS; then + echo "$details" + fi } init_test_env() { @@ -111,41 +126,37 @@ init_test_env() { export DUMPE2FS=${DUMPE2FS:-dumpe2fs} export E2FSCK=${E2FSCK:-e2fsck} export LFSCK_BIN=${LFSCK_BIN:-lfsck} - export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check filesystem after each test suit - export SKIP_LFSCK=${SKIP_LFSCK:-"yes"} # bug 13698, change to "no" when fixed - export SHARED_DIRECTORY=${SHARED_DIRECTORY:-"/tmp"} + + export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite export FSCK_MAX_ERR=4 # File system errors left uncorrected - if [ "$SKIP_LFSCK" == "no" ]; then - if [ ! -x `which $LFSCK_BIN` ]; then - log "$($E2FSCK -V)" - error_exit "$E2FSCK does not support lfsck" - fi - export MDSDB=${MDSDB:-$SHARED_DIRECTORY/mdsdb} - export OSTDB=${OSTDB:-$SHARED_DIRECTORY/ostdb} - export MDSDB_OPT="--mdsdb $MDSDB" - export OSTDB_OPT="--ostdb $OSTDB-\$ostidx" - fi + export ZFS=${ZFS:-zfs} + export ZPOOL=${ZPOOL:-zpool} + export ZDB=${ZDB:-zdb} + #[ -d /r ] && export ROOT=${ROOT:-/r} export TMP=${TMP:-$ROOT/tmp} export TESTSUITELOG=${TMP}/${TESTSUITE}.log - if [[ -z $LOGDIRSET ]]; then - export LOGDIR=${LOGDIR:-${TMP}/test_logs/}/$(date +%s) - export LOGDIRSET=true - fi - export HOSTNAME=${HOSTNAME:-`hostname`} + export LOGDIR=${LOGDIR:-${TMP}/test_logs/$(date +%s)} + export TESTLOG_PREFIX=$LOGDIR/$TESTSUITE + + export HOSTNAME=${HOSTNAME:-$(hostname -s)} if ! echo $PATH | grep -q $LUSTRE/utils; then - export PATH=$PATH:$LUSTRE/utils + export PATH=$LUSTRE/utils:$PATH fi if ! echo $PATH | grep -q $LUSTRE/utils/gss; then - export PATH=$PATH:$LUSTRE/utils/gss + export PATH=$LUSTRE/utils/gss:$PATH fi if ! echo $PATH | grep -q $LUSTRE/tests; then export PATH=$LUSTRE/tests:$PATH fi + if ! echo $PATH | grep -q $LUSTRE/../lustre-iokit/sgpdd-survey; then + export PATH=$LUSTRE/../lustre-iokit/sgpdd-survey:$PATH + fi export LST=${LST:-"$LUSTRE/../lnet/utils/lst"} [ ! -f "$LST" ] && export LST=$(which lst) - export SGPDDSURVEY=${SGPDDSURVEY:-$(which sgpdd-survey)} + export SGPDDSURVEY=${SGPDDSURVEY:-"$LUSTRE/../lustre-iokit/sgpdd-survey/sgpdd-survey")} + [ ! -f "$SGPDDSURVEY" ] && export SGPDDSURVEY=$(which sgpdd-survey) # Ubuntu, at least, has a truncate command in /usr/bin # so fully path our truncate command. export TRUNCATE=${TRUNCATE:-$LUSTRE/tests/truncate} @@ -155,13 +166,17 @@ init_test_env() { export PATH=$LUSTRE/tests/racer:$PATH: fi if ! echo $PATH | grep -q $LUSTRE/tests/mpi; then - export PATH=$PATH:$LUSTRE/tests/mpi + export PATH=$LUSTRE/tests/mpi:$PATH fi export RSYNC_RSH=${RSYNC_RSH:-rsh} + export LCTL=${LCTL:-"$LUSTRE/utils/lctl"} [ ! -f "$LCTL" ] && export LCTL=$(which lctl) export LFS=${LFS:-"$LUSTRE/utils/lfs"} [ ! -f "$LFS" ] && export LFS=$(which lfs) + SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"} + GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"} + export L_GETIDENTITY=${L_GETIDENTITY:-"$LUSTRE/utils/l_getidentity"} if [ ! -f "$L_GETIDENTITY" ]; then if `which l_getidentity > /dev/null 2>&1`; then @@ -178,8 +193,11 @@ init_test_env() { [ ! -f "$TUNEFS" ] && export TUNEFS=$(which tunefs.lustre) export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} " export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod} - [ ! -f "$LUSTRE_RMMOD" ] && export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null) - export FSTYPE=${FSTYPE:-"ldiskfs"} + [ ! -f "$LUSTRE_RMMOD" ] && + export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null) + export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate} + [ ! -f "$LFS_MIGRATE" ] && + export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null) export NAME=${NAME:-local} export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"} [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && \ @@ -225,6 +243,8 @@ init_test_env() { export CLIENTMODSONLY=yes fi + export SHUTDOWN_ATTEMPTS=${SHUTDOWN_ATTEMPTS:-3} + # command line while getopts "rvwf:" opt $*; do @@ -240,18 +260,35 @@ init_test_env() { shift $((OPTIND - 1)) ONLY=${ONLY:-$*} + # print the durations of each test if "true" + DDETAILS=${DDETAILS:-false} [ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true rm -f $TMP/*active } -case `uname -r` in -2.4.*) EXT=".o"; USE_QUOTA=no; [ ! "$CLIENTONLY" ] && FSTYPE=ext3;; - *) EXT=".ko"; USE_QUOTA=yes;; -esac +check_cpt_number() { + local ncpts + + ncpts=$(lctl get_param -n cpu_partition_table 2>/dev/null| wc -l) + if [ $ncpts -eq 0 ]; then + echo "1" + else + echo $ncpts + fi +} +version_code() { + # split arguments like "1.8.6-wc3" into "1", "8", "6", "wc3" + eval set -- $(tr "[:punct:]" " " <<< $*) + + echo -n $((($1 << 16) | ($2 << 8) | $3)) +} + +export LINUX_VERSION=$(uname -r | sed -e "s/[-.]/ /3" -e "s/ .*//") +export LINUX_VERSION_CODE=$(version_code ${LINUX_VERSION//\./ }) module_loaded () { - /sbin/lsmod | grep -q $1 + /sbin/lsmod | grep -q "^\<$1\>" } # Load a module on the system where this is running. @@ -280,7 +317,7 @@ load_module() { eval set -- \$$optvar if [ $# -eq 0 -a -n "$MODPROBECONF" ]; then # Nothing in $MODOPTS_; try modprobe.conf - set -- $(grep "^options\\s*\<${module}\>" $MODPROBECONF) + set -- $(grep -P "^options\\s+${BASE}" $MODPROBECONF) # Get rid of "options $module" (($# > 0)) && shift 2 @@ -316,15 +353,76 @@ load_module() { fi } -load_modules_local() { - if [ -n "$MODPROBE" ]; then - # use modprobe - echo "Using modprobe to load modules" +llite_lloop_enabled() { + local n1=$(uname -r | cut -d. -f1) + local n2=$(uname -r | cut -d. -f2) + local n3=$(uname -r | cut -d- -f1 | cut -d. -f3) + + # load the llite_lloop module for < 2.6.32 kernels + if [[ $n1 -lt 2 ]] || [[ $n1 -eq 2 && $n2 -lt 6 ]] || \ + [[ $n1 -eq 2 && $n2 -eq 6 && $n3 -lt 32 ]] || \ + $LOAD_LLOOP; then return 0 fi + return 1 +} + +load_modules_local() { + if [ "$USE_OFD" == yes ]; then + if module_loaded obdfilter; then + if ! $LUSTRE_RMMOD ldiskfs; then + echo "$HOSTNAME may still be using obdfilter.ko" + return 1 + fi + fi + else + if module_loaded ofd; then + if ! $LUSTRE_RMMOD ldiskfs; then + echo "$HOSTNAME may still be using ofd.ko" + return 1 + fi + fi + fi + + if [ -n "$MODPROBE" ]; then + # use modprobe + echo "Using modprobe to load modules" + return 0 + fi + + echo Loading modules from $LUSTRE + + local ncpus + + if [ -f /sys/devices/system/cpu/online ]; then + ncpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/online) + 1)) + echo "detected $ncpus online CPUs by sysfs" + else + ncpus=$(getconf _NPROCESSORS_CONF 2>/dev/null) + local rc=$? + if [ $rc -eq 0 ]; then + echo "detected $ncpus online CPUs by getconf" + else + echo "Can't detect number of CPUs" + ncpus=1 + fi + fi + + local ncpts=0 + # if there is only one CPU core, libcfs can only create one partition + # if there is more than 4 CPU cores, libcfs should create multiple CPU + # partitions. So we just force libcfs to create 2 partitions for + # system with 2 or 4 cores + if [ $ncpus -le 4 ] && [ $ncpus -gt 1 ]; then + # force to enable multiple CPU partitions + echo "Force libcfs to create 2 CPU partitions" + ncpts=2 + else + echo "libcfs will create CPU partition based on online CPUs" + fi + + load_module ../libcfs/libcfs/libcfs cpu_npartitions=$ncpts - echo Loading modules from $LUSTRE - load_module ../libcfs/libcfs/libcfs [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG" [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }" load_module ../lnet/lnet/lnet @@ -334,7 +432,6 @@ load_modules_local() { load_module obdclass/obdclass load_module ptlrpc/ptlrpc load_module ptlrpc/gss/ptlrpc_gss - [ "$USE_QUOTA" = "yes" -a "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS load_module fld/fld load_module fid/fid load_module lmv/lmv @@ -342,42 +439,66 @@ load_modules_local() { load_module osc/osc load_module lov/lov load_module mgc/mgc + load_module obdecho/obdecho if ! client_only; then - grep -q crc16 /proc/kallsyms || { modprobe crc16 2>/dev/null || true; } - grep -q jbd /proc/kallsyms || { modprobe jbd 2>/dev/null || true; } - [ "$FSTYPE" = "ldiskfs" ] && load_module ../ldiskfs/ldiskfs/ldiskfs - load_module mgs/mgs - load_module mds/mds - load_module mdd/mdd - load_module mdt/mdt - load_module lvfs/fsfilt_$FSTYPE - load_module cmm/cmm - load_module osd-ldiskfs/osd_ldiskfs - load_module ost/ost - load_module obdfilter/obdfilter + SYMLIST=/proc/kallsyms + grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; } + grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; } + grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; } + [ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS + if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then + load_module osd-zfs/osd_zfs + fi + load_module mgs/mgs + load_module mds/mds + load_module mdd/mdd + if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then + # + # This block shall be moved up beside osd-zfs as soon + # as osd-ldiskfs stops using mdd symbols. + # + grep -q exportfs_decode_fh $SYMLIST || + { modprobe exportfs 2> /dev/null || true; } + load_module ../ldiskfs/ldiskfs/ldiskfs + load_module lvfs/fsfilt_ldiskfs + load_module osd-ldiskfs/osd_ldiskfs + fi + load_module mdt/mdt + load_module cmm/cmm + load_module ost/ost + if [ "$USE_OFD" == yes ]; then + load_module ofd/ofd + else + load_module obdfilter/obdfilter + fi fi + load_module llite/lustre - load_module llite/llite_lloop + llite_lloop_enabled && load_module llite/llite_lloop [ -d /r ] && OGDB=${OGDB:-"/r/tmp"} OGDB=${OGDB:-$TMP} rm -f $OGDB/ogdb-$HOSTNAME $LCTL modules > $OGDB/ogdb-$HOSTNAME # 'mount' doesn't look in $PATH, just sbin - [ -f $LUSTRE/utils/mount.lustre ] && cp $LUSTRE/utils/mount.lustre /sbin/. || true + if [ -f $LUSTRE/utils/mount.lustre ] && \ + ! grep -qe "/sbin/mount\.lustre " /proc/mounts; then + [ ! -f /sbin/mount.lustre ] && touch /sbin/mount.lustre + mount --bind $LUSTRE/utils/mount.lustre /sbin/mount.lustre || true + fi } load_modules () { - load_modules_local - # bug 19124 - # load modules on remote nodes optionally - # lustre-tests have to be installed on these nodes - if $LOAD_MODULES_REMOTE ; then - local list=$(comma_list $(remote_nodes_list)) - echo loading modules on $list - do_rpc_nodes $list load_modules - fi + load_modules_local + # bug 19124 + # load modules on remote nodes optionally + # lustre-tests have to be installed on these nodes + if $LOAD_MODULES_REMOTE ; then + local list=$(comma_list $(remote_nodes_list)) + echo loading modules on $list + do_rpc_nodes $list load_modules_local + fi } check_mem_leak () { @@ -394,17 +515,23 @@ check_mem_leak () { } unload_modules() { - wait_exit_ST client # bug 12845 + wait_exit_ST client # bug 12845 - $LUSTRE_RMMOD $FSTYPE || return 2 + $LUSTRE_RMMOD ldiskfs || return 2 - if $LOAD_MODULES_REMOTE ; then - local list=$(comma_list $(remote_nodes_list)) - if [ ! -z $list ]; then - echo unloading modules on $list - do_rpc_nodes $list $LUSTRE_RMMOD $FSTYPE - do_rpc_nodes $list check_mem_leak - fi + if $LOAD_MODULES_REMOTE ; then + local list=$(comma_list $(remote_nodes_list)) + if [ ! -z $list ]; then + echo unloading modules on $list + do_rpc_nodes $list $LUSTRE_RMMOD ldiskfs + do_rpc_nodes $list check_mem_leak + fi + fi + + if grep -qe "/sbin/mount\.lustre" /proc/mounts; then + umount /sbin/mount.lustre || true + [ -w /sbin/mount.lustre -a ! -s /sbin/mount.lustre ] && \ + rm -f /sbin/mount.lustre || true fi check_mem_leak || return 254 @@ -525,50 +652,242 @@ cleanup_gss() { fi } +facet_type() { + local facet=$1 + + echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9]\+//' | + tr '[:lower:]' '[:upper:]' +} + +facet_number() { + local facet=$1 + + if [ $facet == mgs ]; then + return 1 + fi + + echo -n $facet | sed -e 's/^fs[0-9]\+//' | sed -e 's/^[a-z]\+//' +} + +facet_fstype() { + local facet=$1 + local var + + var=${facet}_FSTYPE + if [ -n "${!var}" ]; then + echo -n ${!var} + return + fi + + var=$(facet_type $facet)FSTYPE + if [ -n "${!var}" ]; then + echo -n ${!var} + return + fi + + if [ -n "$FSTYPE" ]; then + echo -n $FSTYPE + return + fi + + return 1 +} + +node_fstypes() { + local node=$1 + local fstypes + local fstype + local facets=$(get_facets) + local facet + + for facet in ${facets//,/ }; do + if [ $node == $(facet_host $facet) ] || + [ $node == "$(facet_failover_host $facet)" ]; then + fstype=$(facet_fstype $facet) + if [[ $fstypes != *$fstype* ]]; then + fstypes+="${fstypes:+,}$fstype" + fi + fi + done + echo -n $fstypes +} + +devicelabel() { + local facet=$1 + local dev=$2 + local label + local fstype=$(facet_fstype $facet) + + case $fstype in + ldiskfs) + label=$(do_facet ${facet} "$E2LABEL ${dev} 2>/dev/null");; + zfs) + label=$(do_facet ${facet} "$ZFS get -H -o value lustre:svname \ + ${dev} 2>/dev/null");; + *) + error "unknown fstype!";; + esac + + echo -n $label +} + mdsdevlabel() { - local num=$1 - local device=`mdsdevname $num` - local label=`do_facet mds$num "e2label ${device}" | grep -v "CMD: "` - echo -n $label + local num=$1 + local device=$(mdsdevname $num) + local label=$(devicelabel mds$num ${device} | grep -v "CMD: ") + echo -n $label } ostdevlabel() { - local num=$1 - local device=`ostdevname $num` - local label=`do_facet ost$num "e2label ${device}" | grep -v "CMD: "` - echo -n $label + local num=$1 + local device=$(ostdevname $num) + local label=$(devicelabel ost$num ${device} | grep -v "CMD: ") + echo -n $label +} + +# +# This and set_obdfilter_param() shall be used to access OSD parameters +# once existed under "obdfilter": +# +# mntdev +# stats +# read_cache_enable +# writethrough_cache_enable +# +get_obdfilter_param() { + local nodes=$1 + local device=${2:-$FSNAME-OST*} + local name=$3 + + do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \ + osd-*.$device.$name 2>&1" | grep -v 'Found no match' +} + +set_obdfilter_param() { + local nodes=$1 + local device=${2:-$FSNAME-OST*} + local name=$3 + local value=$4 + + do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \ + osd-*.$device.$name=$value 2>&1" | grep -v 'Found no match' +} + +set_debug_size () { + local dz=${1:-$DEBUG_SIZE} + + if [ -f /sys/devices/system/cpu/possible ]; then + local cpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/possible)+1)) + else + local cpus=$(getconf _NPROCESSORS_CONF) + fi + + # bug 19944, adjust size to be -gt num_possible_cpus() + # promise 2MB for every cpu at least + if [ -n "$cpus" ] && [ $((cpus * 2)) -gt $dz ]; then + dz=$((cpus * 2)) + fi + lctl set_param debug_mb=$dz +} + +set_default_debug () { + local debug=${1:-"$PTLDEBUG"} + local subsys=${2:-"$SUBSYSTEM"} + local debug_size=${3:-$DEBUG_SIZE} + + [ -n "$debug" ] && lctl set_param debug="$debug" >/dev/null + [ -n "$subsys" ] && lctl set_param subsystem_debug="${subsys# }" >/dev/null + + [ -n "$debug_size" ] && set_debug_size $debug_size > /dev/null +} + +set_default_debug_nodes () { + local nodes=$1 + + if [[ ,$nodes, = *,$HOSTNAME,* ]]; then + nodes=$(exclude_items_from_list "$nodes" "$HOSTNAME") + set_default_debug + fi + + [[ -n $nodes ]] && do_rpc_nodes $nodes set_default_debug \ + \\\"$PTLDEBUG\\\" \\\"$SUBSYSTEM\\\" $DEBUG_SIZE || true +} + +set_default_debug_facet () { + local facet=$1 + local node=$(facet_active_host $facet) + [ -z "$node" ] && echo "No host defined for facet $facet" && exit 1 + + set_default_debug_nodes $node } # Facet functions mount_facets () { - local facets=${1:-$(get_facets)} - local facet + local facets=${1:-$(get_facets)} + local facet - for facet in ${facets//,/ }; do - mount_facet $facet || error "Restart of $facet failed!" - done + for facet in ${facets//,/ }; do + mount_facet $facet + local RC=$? + [ $RC -eq 0 ] && continue + + if [ "$TESTSUITE.$TESTNAME" = "replay-dual.test_0a" ]; then + skip "Restart of $facet failed!." && touch $LU482_FAILED + else + error "Restart of $facet failed!" + fi + return $RC + done } -mount_facet() { - local facet=$1 - shift - local dev=$(facet_active $facet)_dev - local opt=${facet}_opt - local mntpt=$(facet_mntpt $facet) +# +# Add argument "arg" (e.g., "loop") to the comma-separated list +# of arguments for option "opt" (e.g., "-o") on command +# line "opts" (e.g., "-o flock"). +# +csa_add() { + local opts=$1 + local opt=$2 + local arg=$3 + local opt_pattern="\([[:space:]]\+\|^\)$opt" + + if echo "$opts" | grep -q $opt_pattern; then + opts=$(echo "$opts" | sed -e \ + "s/$opt_pattern[[:space:]]*[^[:space:]]\+/&,$arg/") + else + opts+="${opts:+ }$opt $arg" + fi + echo -n "$opts" +} - echo "Starting ${facet}: ${!opt} $@ ${!dev} $mntpt" - do_facet ${facet} mount -t lustre ${!opt} $@ ${!dev} $mntpt - RC=${PIPESTATUS[0]} - if [ $RC -ne 0 ]; then - echo "mount -t lustre $@ ${!dev} $mntpt" - echo "Start of ${!dev} on ${facet} failed ${RC}" +mount_facet() { + local facet=$1 + shift + local dev=$(facet_active $facet)_dev + local opt=${facet}_opt + local mntpt=$(facet_mntpt $facet) + local opts="${!opt} $@" + + if [ $(facet_fstype $facet) == ldiskfs ] && + ! do_facet $facet test -b ${!dev}; then + opts=$(csa_add "$opts" -o loop) + fi + + echo "Starting ${facet}: $opts ${!dev} $mntpt" + # for testing LU-482 error handling in mount_facets() and test_0a() + if [ -f $TMP/test-lu482-trigger ]; then + RC=2 + else + do_facet ${facet} "mkdir -p $mntpt; mount -t lustre $opts \ + ${!dev} $mntpt" + RC=${PIPESTATUS[0]} + fi + if [ $RC -ne 0 ]; then + echo "Start of ${!dev} on ${facet} failed ${RC}" else - do_facet ${facet} "lctl set_param debug=\\\"$PTLDEBUG\\\"; \ - lctl set_param subsystem_debug=\\\"${SUBSYSTEM# }\\\"; \ - lctl set_param debug_mb=${DEBUG_SIZE}; \ - sync" + set_default_debug_facet $facet - label=$(do_facet ${facet} "$E2LABEL ${!dev}") + label=$(devicelabel ${facet} ${!dev}) [ -z "$label" ] && echo no label for ${!dev} && exit 1 eval export ${facet}_svc=${label} echo Started ${label} @@ -600,6 +919,39 @@ start() { return $RC } +# +# When a ZFS OSD is made read-only by replay_barrier(), its pool is "freezed". +# Because stopping corresponding target may not clear this in-memory state, we +# need to zap the pool from memory by exporting and reimporting the pool. +# +# Although the uberblocks are not updated when a pool is freezed, transactions +# are still written to the disks. Modified blocks may be cached in memory when +# tests try reading them back. The export-and-reimport process also evicts any +# cached pool data from memory to provide the correct "data loss" semantics. +# +refresh_disk() { + local facet=$1 + local fstype=$(facet_fstype $facet) + local _dev + local dev + local poolname + + if [ "${fstype}" == "zfs" ]; then + _dev=$(facet_active $facet)_dev + dev=${!_dev} # expand _dev to its value, e.g. ${mds1_dev} + poolname="${dev%%/*}" # poolname is string before "/" + + if [ "${poolname}" == "" ]; then + echo "invalid dataset name: $dev" + return + fi + do_facet $facet "cp /etc/zfs/zpool.cache /tmp/zpool.cache.back" + do_facet $facet "$ZPOOL export ${poolname}" + do_facet $facet "$ZPOOL import -f -c /tmp/zpool.cache.back \ + ${poolname}" + fi +} + stop() { local running local facet=$1 @@ -610,7 +962,7 @@ stop() { local mntpt=$(facet_mntpt $facet) running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts") || true if [ ${running} -ne 0 ]; then - echo "Stopping $mntpt (opts:$@)" + echo "Stopping $mntpt (opts:$@) on $HOST" do_facet ${facet} umount -d $@ $mntpt fi @@ -662,6 +1014,11 @@ restore_quota_type () { setup_quota(){ local mntpt=$1 + if [ "$USE_OFD" = "yes" ]; then + $LFS quotacheck $mntpt || error "quotacheck failed" + return + fi + # We need save the original quota_type params, and restore them after testing # Suppose that quota type the same on mds and ost @@ -705,13 +1062,10 @@ setup_quota(){ } zconf_mount() { - local OPTIONS local client=$1 local mnt=$2 - # Only supply -o to mount if we have options - if [ -n "$MOUNTOPT" ]; then - OPTIONS="-o $MOUNTOPT" - fi + local OPTIONS=${3:-$MOUNTOPT} + local device=$MGSNID:/$FSNAME if [ -z "$mnt" -o -z "$FSNAME" ]; then echo Bad zconf mount command: opt=$OPTIONS dev=$device mnt=$mnt @@ -722,9 +1076,7 @@ zconf_mount() { do_node $client mkdir -p $mnt do_node $client mount -t lustre $OPTIONS $device $mnt || return 1 - do_node $client "lctl set_param debug=\\\"$PTLDEBUG\\\"; - lctl set_param subsystem_debug=\\\"${SUBSYSTEM# }\\\"; - lctl set_param debug_mb=${DEBUG_SIZE}" + set_default_debug_nodes $client return 0 } @@ -769,7 +1121,7 @@ sanity_mount_check_nodes () { local rc=0 for mnt in $mnts ; do do_nodes $nodes "running=\\\$(grep -c $mnt' ' /proc/mounts); -mpts=\\\$(mount | grep -w -c $mnt); +mpts=\\\$(mount | grep -c $mnt' '); if [ \\\$running -ne \\\$mpts ]; then echo \\\$(hostname) env are INSANE!; exit 1; @@ -820,10 +1172,6 @@ zconf_mount_clients() { local mnt=$2 local OPTIONS=${3:-$MOUNTOPT} - # Only supply -o to mount if we have options - if [ "$OPTIONS" ]; then - OPTIONS="-o $OPTIONS" - fi local device=$MGSNID:/$FSNAME if [ -z "$mnt" -o -z "$FSNAME" ]; then echo Bad zconf mount command: opt=$OPTIONS dev=$device mnt=$mnt @@ -843,11 +1191,9 @@ fi; exit \\\$rc" || return ${PIPESTATUS[0]} echo "Started clients $clients: " - do_nodes $clients "mount | grep -w $mnt" + do_nodes $clients "mount | grep $mnt' '" - do_nodes $clients "lctl set_param debug=\\\"$PTLDEBUG\\\"; - lctl set_param subsystem_debug=\\\"${SUBSYSTEM# }\\\"; - lctl set_param debug_mb=${DEBUG_SIZE};" + set_default_debug_nodes $clients return 0 } @@ -863,18 +1209,16 @@ zconf_umount_clients() { do_nodes $clients "running=\\\$(grep -c $mnt' ' /proc/mounts); if [ \\\$running -ne 0 ] ; then echo Stopping client \\\$(hostname) $mnt opts:$force; -lsof -t $mnt || need_kill=no; +lsof $mnt || need_kill=no; if [ "x$force" != "x" -a "x\\\$need_kill" != "xno" ]; then pids=\\\$(lsof -t $mnt | sort -u); if [ -n \\\"\\\$pids\\\" ]; then kill -9 \\\$pids; fi fi; -busy=\\\$(umount $force $mnt 2>&1 | grep -c "busy"); -if [ \\\$busy -ne 0 ] ; then +while umount $force $mnt 2>&1 | grep -q "busy"; do echo "$mnt is still busy, wait one second" && sleep 1; - umount $force $mnt; -fi +done; fi" } @@ -886,12 +1230,12 @@ shutdown_node () { shutdown_node_hard () { local host=$1 - local attempts=3 + local attempts=$SHUTDOWN_ATTEMPTS for i in $(seq $attempts) ; do shutdown_node $host sleep 1 - ping -w 3 -c 1 $host > /dev/null 2>&1 || return 0 + wait_for_function --quiet "! ping -w 3 -c 1 $host" 5 1 && return 0 echo "waiting for $host to fail attempts=$attempts" [ $i -lt $attempts ] || \ { echo "$host still pingable after power down! attempts=$attempts" && return 1; } @@ -972,12 +1316,13 @@ remount_facet() { } reboot_facet() { - local facet=$1 - if [ "$FAILURE_MODE" = HARD ]; then - reboot_node $(facet_active_host $facet) - else - sleep 10 - fi + local facet=$1 + if [ "$FAILURE_MODE" = HARD ]; then + reboot_node $(facet_active_host $facet) + else + refresh_disk ${facet} + sleep 10 + fi } boot_node() { @@ -988,41 +1333,63 @@ boot_node() { fi } -# recovery-scale functions +facets_hosts () { + local facets=$1 + local hosts + + for facet in ${facets//,/ }; do + hosts=$(expand_list $hosts $(facet_host $facet) ) + done + + echo $hosts +} + +_check_progs_installed () { + local progs=$@ + local rc=0 + + for prog in $progs; do + if ! [ "$(which $prog)" -o "${!prog}" ]; then + echo $prog missing on $(hostname) + rc=1 + fi + done + return $rc +} + check_progs_installed () { - local clients=$1 + local nodes=$1 shift - local progs=$@ - do_nodes $clients "PATH=:$PATH; status=true; -for prog in $progs; do - if ! [ \\\"\\\$(which \\\$prog)\\\" -o \\\"\\\${!prog}\\\" ]; then - echo \\\$prog missing on \\\$(hostname); - status=false; - fi -done; -eval \\\$status" + do_rpc_nodes $nodes _check_progs_installed $@ } -client_var_name() { - echo __$(echo $1 | tr '-' 'X') +# recovery-scale functions +node_var_name() { + echo __$(echo $1 | tr '-' '_' | tr '.' '_') } start_client_load() { local client=$1 local load=$2 - local var=$(client_var_name $client)_load + local var=$(node_var_name $client)_load eval export ${var}=$load do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \ - BREAK_ON_ERROR=$BREAK_ON_ERROR \ - END_RUN_FILE=$END_RUN_FILE \ - LOAD_PID_FILE=$LOAD_PID_FILE \ - TESTSUITELOG=$TESTSUITELOG \ - run_${load}.sh" & - CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $!" +BREAK_ON_ERROR=$BREAK_ON_ERROR \ +END_RUN_FILE=$END_RUN_FILE \ +LOAD_PID_FILE=$LOAD_PID_FILE \ +TESTLOG_PREFIX=$TESTLOG_PREFIX \ +TESTNAME=$TESTNAME \ +DBENCH_LIB=$DBENCH_LIB \ +DBENCH_SRC=$DBENCH_SRC \ +run_${load}.sh" & + local ppid=$! log "Started client load: ${load} on $client" + # get the children process IDs + local pids=$(ps --ppid $ppid -o pid= | xargs) + CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $ppid $pids" return 0 } @@ -1039,14 +1406,14 @@ start_client_loads () { sleep 2 } -# only for remote client +# only for remote client check_client_load () { local client=$1 - local var=$(client_var_name $client)_load + local var=$(node_var_name $client)_load local TESTLOAD=run_${!var}.sh ps auxww | grep -v grep | grep $client | grep -q "$TESTLOAD" || return 1 - + # bug 18914: try to connect several times not only when # check ps, but while check_catastrophe also local tries=3 @@ -1116,7 +1483,7 @@ restart_client_loads () { check_client_load $client rc=${PIPESTATUS[0]} if [ "$rc" != 0 -a "$expectedfail" ]; then - local var=$(client_var_name $client)_load + local var=$(node_var_name $client)_load start_client_load $client ${!var} echo "Restarted client load ${!var}: on $client. Checking ..." check_client_load $client @@ -1124,7 +1491,7 @@ restart_client_loads () { if [ "$rc" != 0 ]; then log "Client load failed to restart on node $client, rc=$rc" # failure one client load means test fail - # we do not need to check other + # we do not need to check other return $rc fi else @@ -1132,6 +1499,70 @@ restart_client_loads () { fi done } + +# Start vmstat and save its process ID in a file. +start_vmstat() { + local nodes=$1 + local pid_file=$2 + + [ -z "$nodes" -o -z "$pid_file" ] && return 0 + + do_nodes $nodes \ + "vmstat 1 > $TESTLOG_PREFIX.$TESTNAME.vmstat.\\\$(hostname -s).log \ + 2>/dev/null $pid_file" +} + +# Display the nodes on which client loads failed. +print_end_run_file() { + local file=$1 + local node + + [ -s $file ] || return 0 + + echo "Found the END_RUN_FILE file: $file" + cat $file + + # A client load will stop if it finds the END_RUN_FILE file. + # That does not mean the client load actually failed though. + # The first node in END_RUN_FILE is the one we are interested in. + read node < $file + + if [ -n "$node" ]; then + local var=$(node_var_name $node)_load + + local prefix=$TESTLOG_PREFIX + [ -n "$TESTNAME" ] && prefix=$prefix.$TESTNAME + local stdout_log=$prefix.run_${!var}_stdout.$node.log + local debug_log=$(echo $stdout_log | sed 's/\(.*\)stdout/\1debug/') + + echo "Client load ${!var} failed on node $node:" + echo "$stdout_log" + echo "$debug_log" + fi +} + +# Stop the process which had its PID saved in a file. +stop_process() { + local nodes=$1 + local pid_file=$2 + + [ -z "$nodes" -o -z "$pid_file" ] && return 0 + + do_nodes $nodes "test -f $pid_file && + { kill -s TERM \\\$(cat $pid_file); rm -f $pid_file; }" || true +} + +# Stop all client loads. +stop_client_loads() { + local nodes=${1:-$CLIENTS} + local pid_file=$2 + + # stop the client loads + stop_process $nodes $pid_file + + # clean up the processes that started them + [ -n "$CLIENT_LOAD_PIDS" ] && kill -9 $CLIENT_LOAD_PIDS 2>/dev/null || true +} # End recovery-scale functions # verify that lustre actually cleaned up properly @@ -1166,19 +1597,22 @@ wait_update () { local RESULT local WAIT=0 - local sleep=5 + local sleep=1 + local print=10 while [ true ]; do RESULT=$(do_node $node "$TEST") if [ "$RESULT" == "$FINAL" ]; then - echo "Updated after $WAIT sec: wanted '$FINAL' got '$RESULT'" + [ -z "$RESULT" -o $WAIT -le $sleep ] || + echo "Updated after ${WAIT}s: wanted '$FINAL' got '$RESULT'" return 0 fi [ $WAIT -ge $MAX ] && break - echo "Waiting $((MAX - WAIT)) secs for update" + [ $((WAIT % print)) -eq 0 ] && + echo "Waiting $((MAX - WAIT)) secs for update" WAIT=$((WAIT + sleep)) sleep $sleep done - echo "Update not seen after $MAX sec: wanted '$FINAL' got '$RESULT'" + echo "Update not seen after ${MAX}s: wanted '$FINAL' got '$RESULT'" return 3 } @@ -1187,6 +1621,11 @@ wait_update_facet () { wait_update $(facet_active_host $facet) "$@" } +sync_all_data() { + do_node $(osts_nodes) "lctl set_param -n osd*.*OS*.force_sync 1" 2>&1 | + grep -v 'Found no match' +} + wait_delete_completed () { local TOTALPREV=`lctl get_param -n osc.*.kbytesavail | \ awk 'BEGIN{total=0}; {total+=$1}; END{print total}'` @@ -1459,8 +1898,13 @@ obd_name() { replay_barrier() { local facet=$1 - do_facet $facet sync + do_facet $facet "sync; sync; sync" df $MOUNT + + # make sure there will be no seq change + local clients=${CLIENTS:-$HOSTNAME} + do_nodes $clients "f=${MOUNT}/fsa-\\\$(hostname); mcreate \\\$f; rm \\\$f" + local svc=${facet}_svc do_facet $facet $LCTL --device %${!svc} notransno do_facet $facet $LCTL --device %${!svc} readonly @@ -1470,7 +1914,7 @@ replay_barrier() { replay_barrier_nodf() { local facet=$1 echo running=${running} - do_facet $facet sync + do_facet $facet "sync; sync; sync" local svc=${facet}_svc echo Replay barrier on ${!svc} do_facet $facet $LCTL --device %${!svc} notransno @@ -1510,12 +1954,14 @@ fail_nodf() { } fail_abort() { - local facet=$1 - stop $facet - change_active $facet - mount_facet $facet -o abort_recovery - clients_up || echo "first df failed: $?" - clients_up || error "post-failover df: $?" + local facet=$1 + stop $facet + refresh_disk ${facet} + change_active $facet + wait_for_facet $facet + mount_facet $facet -o abort_recovery + clients_up || echo "first df failed: $?" + clients_up || error "post-failover df: $?" } do_lmc() { @@ -1523,6 +1969,18 @@ do_lmc() { exit 1 } +host_nids_address() { + local nodes=$1 + local kind=$2 + + if [ -n "$kind" ]; then + nids=$(do_nodes $nodes "$LCTL list_nids | grep $kind | cut -f 1 -d '@'") + else + nids=$(do_nodes $nodes "$LCTL list_nids all | cut -f 1 -d '@'") + fi + echo $nids +} + h2name_or_ip() { if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else echo $1"@$2" @@ -1563,6 +2021,95 @@ h2o2ib() { } declare -fx h2o2ib +# This enables variables in cfg/"setup".sh files to support the pdsh HOSTLIST +# expressions format. As a bonus we can then just pass in those variables +# to pdsh. What this function does is take a HOSTLIST type string and +# expand it into a space deliminated list for us. +hostlist_expand() { + local hostlist=$1 + local offset=$2 + local myList + local item + local list + + [ -z "$hostlist" ] && return + + # Translate the case of [..],..,[..] to [..] .. [..] + list="${hostlist/],/] }" + front=${list%%[*} + [[ "$front" == *,* ]] && { + new="${list%,*} " + old="${list%,*}," + list=${list/${old}/${new}} + } + + for item in $list; do + # Test if we have any []'s at all + if [ "$item" != "${item/\[/}" ]; then { + # Expand the [*] into list + name=${item%%[*} + back=${item#*]} + + if [ "$name" != "$item" ]; then + group=${item#$name[*} + group=${group%%]*} + + for range in ${group//,/ }; do + begin=${range%-*} + end=${range#*-} + + # Number of leading zeros + padlen=${#begin} + padlen2=${#end} + end=$(echo $end | sed 's/0*//') + [[ -z "$end" ]] && end=0 + [[ $padlen2 -gt $padlen ]] && { + [[ $padlen2 -eq ${#end} ]] && padlen2=0 + padlen=$padlen2 + } + begin=$(echo $begin | sed 's/0*//') + [ -z $begin ] && begin=0 + + for num in $(seq -f "%0${padlen}g" $begin $end); do + value="${name#*,}${num}${back}" + [ "$value" != "${value/\[/}" ] && { + value=$(hostlist_expand "$value") + } + myList="$myList $value" + done + done + fi + } else { + myList="$myList $item" + } fi + done + myList=${myList//,/ } + myList=${myList:1} # Remove first character which is a space + + # Filter any duplicates without sorting + list="$myList " + myList="${list%% *}" + + while [[ "$list" != ${myList##* } ]]; do + list=${list//${list%% *} /} + myList="$myList ${list%% *}" + done + myList="${myList%* }"; + + # We can select an object at a offset in the list + [ $# -eq 2 ] && { + cnt=0 + for item in $myList; do + let cnt=cnt+1 + [ $cnt -eq $offset ] && { + myList=$item + } + done + [ $(get_node_count $myList) -ne 1 ] && myList="" + } + echo $myList +} + facet_host() { local facet=$1 @@ -1576,6 +2123,25 @@ facet_host() { echo -n ${!varname} } +facet_failover_host() { + local facet=$1 + local var + + var=${facet}failover_HOST + if [ -n "${!var}" ]; then + echo ${!var} + return + fi + + if [[ $facet == ost* ]]; then + var=ostfailover_HOST + if [ -n "${!var}" ]; then + echo ${!var} + return + fi + fi +} + facet_active() { local facet=$1 local activevar=${facet}active @@ -1622,6 +2188,8 @@ change_active() { # save the active host for this facet local activevar=${facet}active echo "$activevar=${!activevar}" > $TMP/$activevar + [[ $facet = mds1 ]] && combined_mgs_mds && \ + echo "mgsactive=${!activevar}" > $TMP/mgsactive local TO=`facet_active_host $facet` echo "Failover $facet to $TO" done @@ -1646,7 +2214,7 @@ do_node() { fi if $VERBOSE; then echo "CMD: $HOST $@" >&2 - $myPDSH $HOST $LCTL mark "$@" > /dev/null 2>&1 || : + $myPDSH $HOST "$LCTL mark \"$@\"" > /dev/null 2>&1 || : fi if [ "$myPDSH" = "rsh" ]; then @@ -1683,13 +2251,34 @@ single_local_node () { # Outputs environment variable assignments that should be passed to remote nodes get_env_vars() { - local var - local value + local var + local value + local facets=$(get_facets) + local facet - for var in ${!MODOPTS_*}; do - value=${!var} - echo "${var}=\"$value\"" - done + for var in ${!MODOPTS_*}; do + value=${!var} + echo -n " ${var}=\"$value\"" + done + + echo -n " USE_OFD=$USE_OFD" + + for facet in ${facets//,/ }; do + var=${facet}_FSTYPE + if [ -n "${!var}" ]; then + echo -n " $var=${!var}" + fi + done + + for var in MGSFSTYPE MDSFSTYPE OSTFSTYPE; do + if [ -n "${!var}" ]; then + echo -n " $var=${!var}" + fi + done + + if [ -n "$FSTYPE" ]; then + echo -n " FSTYPE=$FSTYPE" + fi } do_nodes() { @@ -1718,9 +2307,10 @@ do_nodes() { [ -z "$myPDSH" -o "$myPDSH" = "no_dsh" -o "$myPDSH" = "rsh" ] && \ echo "cannot run remote command on $rnodes with $myPDSH" && return 128 + export FANOUT=$(get_node_count "${rnodes//,/ }") if $VERBOSE; then echo "CMD: $rnodes $@" >&2 - $myPDSH $rnodes $LCTL mark "$@" > /dev/null 2>&1 || : + $myPDSH $rnodes "$LCTL mark \"$@\"" > /dev/null 2>&1 || : fi # do not replace anything from pdsh output if -N is used @@ -1741,6 +2331,25 @@ do_facet() { do_node $HOST "$@" } +# Function: do_facet_random_file $FACET $FILE $SIZE +# Creates FILE with random content on the given FACET of given SIZE + +do_facet_random_file() { + local facet="$1" + local fpath="$2" + local fsize="$3" + local cmd="dd if=/dev/urandom of='$fpath' bs=$fsize count=1" + do_facet $facet "$cmd 2>/dev/null" +} + +do_facet_create_file() { + local facet="$1" + local fpath="$2" + local fsize="$3" + local cmd="dd if=/dev/zero of='$fpath' bs=$fsize count=1" + do_facet $facet "$cmd 2>/dev/null" +} + do_nodesv() { do_nodes --verbose "$@" } @@ -1751,27 +2360,132 @@ add() { # make sure its not already running stop ${facet} -f rm -f $TMP/${facet}active + [[ $facet = mds1 ]] && combined_mgs_mds && rm -f $TMP/mgsactive do_facet ${facet} $MKFS $* } ostdevname() { num=$1 DEVNAME=OSTDEV$num - #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num - eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}} + + local fstype=$(facet_fstype ost$num) + + case $fstype in + ldiskfs ) + #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num + eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};; + zfs ) + #dataset name is independent of vdev device names + eval DEVPTR=${FSNAME}-ost${num}/ost${num};; + * ) + error "unknown fstype!";; + esac + echo -n $DEVPTR } +ostvdevname() { + num=$1 + DEVNAME=OSTDEV$num + + local fstype=$(facet_fstype ost$num) + + case $fstype in + ldiskfs ) + # vdevs are not supported by ldiskfs + eval VDEVPTR="";; + zfs ) + #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num + eval VDEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};; + * ) + error "unknown fstype!";; + esac + + echo -n $VDEVPTR +} + mdsdevname() { num=$1 DEVNAME=MDSDEV$num - #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num - eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}} + + local fstype=$(facet_fstype mds$num) + + case $fstype in + ldiskfs ) + #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num + eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};; + zfs ) + #dataset name is independent of vdev device names + eval DEVPTR=${FSNAME}-mdt${num}/mdt${num};; + * ) + error "unknown fstype!";; + esac + + echo -n $DEVPTR +} + +mdsvdevname() { + num=$1 + DEVNAME=MDSDEV$num + + local fstype=$(facet_fstype mds$num) + + case $fstype in + ldiskfs ) + # vdevs are not supported by ldiskfs + eval VDEVPTR="";; + zfs ) + #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num + eval VDEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};; + * ) + error "unknown fstype!";; + esac + + echo -n $VDEVPTR +} + +mgsdevname() { + DEVNAME=MGSDEV + + local fstype=$(facet_fstype mds$num) + + case $fstype in + ldiskfs ) + #if $MGSDEV isn't defined, default is $MDSDEV1 + eval DEVPTR=${!DEVNAME:=${MDSDEV1}};; + zfs ) + #dataset name is independent of vdev device names + eval DEVPTR=${FSNAME}-mgs/mgs;; + * ) + error "unknown fstype!";; + esac + echo -n $DEVPTR } +mgsvdevname() { + DEVNAME=MGSDEV + + local fstype=$(facet_fstype mds$num) + + case $fstype in + ldiskfs ) + # vdevs are not supported by ldiskfs + eval VDEVPTR="";; + zfs ) + #if $MGSDEV isn't defined, default is $MGSDEV1 + eval VDEVPTR=${!DEVNAME:=${MDSDEV1}};; + * ) + error "unknown fstype!";; + esac + + echo -n $VDEVPTR +} + facet_mntpt () { local facet=$1 + [[ $facet = mgs ]] && combined_mgs_mds && facet="mds1" + local var=${facet}_MOUNT eval mntpt=${!var:-${MOUNT%/*}/$facet} @@ -1803,6 +2517,7 @@ stopall() { stop mds$num -f rm -f ${TMP}/mds${num}active done + combined_mgs_mds && rm -f $TMP/mgsactive for num in `seq $OSTCOUNT`; do stop ost$num -f @@ -1839,88 +2554,123 @@ combined_mgs_mds () { [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]] } -mkfs_opts () { - local facet=$1 +lower() { + echo -n "$1" | tr '[:upper:]' '[:lower:]' +} - local tgt=$(echo $facet | tr -d [:digit:] | tr "[:lower:]" "[:upper:]") - local optvar=${tgt}_MKFS_OPTS - local opt=${!optvar} +upper() { + echo -n "$1" | tr '[:lower:]' '[:upper:]' +} - # FIXME: ! combo mgs/mds + mgsfailover is not supported yet - [[ $facet = mgs ]] && echo $opt && return +mkfs_opts() { + local facet=$1 + local type=$(facet_type $facet) + local index=$(($(facet_number $facet) - 1)) + local fstype=$(facet_fstype $facet) + local opts + local fs_mkfs_opts + local var - # 1. - # --failnode options - local var=${facet}failover_HOST - if [ x"${!var}" != x ] && [ x"${!var}" != x$(facet_host $facet) ] ; then - local failnode=$(h2$NETTYPE ${!var}) - failnode="--failnode=$failnode" - # options does not contain - # or contains wrong --failnode= - if [[ $opt != *${failnode}* ]]; then - opt=$(echo $opt | sed 's/--failnode=.* / /') - opt="$opt $failnode" - fi - fi + if [ $type == MGS ] && combined_mgs_mds; then + return 1 + fi - # 2. - # --mgsnode options - # no additional mkfs mds "--mgsnode" option for this configuration - if [[ $facet = mds ]] && combined_mgs_mds; then - echo $opt - return - fi + if [ $type == MGS ] || ( [ $type == MDS ] && combined_mgs_mds ); then + opts="--mgs" + else + opts="--mgsnode=$MGSNID" + fi - # additional mkfs "--mgsnode" - local mgsnode="--mgsnode=$MGSNID" - opt=${opt//$mgsnode } - for nid in ${MGSNID//:/ }; do - local mgsnode="--mgsnode=$nid" - # options does not contain - # --mgsnode=$nid - if [[ $opt != *${mgsnode}" "* ]]; then - opt="$opt --mgsnode=$nid" - fi - done + if [ $type != MGS ]; then + opts+=" --fsname=$FSNAME --$(lower ${type/MDS/MDT}) --index=$index" + fi - echo $opt -} + var=${facet}failover_HOST + if [ -n "${!var}" ] && [ ${!var} != $(facet_host $facet) ]; then + opts+=" --failnode=$(h2$NETTYPE ${!var})" + fi -formatall() { - if [ "$IAMDIR" == "yes" ]; then - MDS_MKFS_OPTS="$MDS_MKFS_OPTS --iam-dir" - fi + opts+=${TIMEOUT:+" --param=sys.timeout=$TIMEOUT"} + opts+=${LDLM_TIMEOUT:+" --param=sys.ldlm_timeout=$LDLM_TIMEOUT"} - [ "$FSTYPE" ] && FSTYPE_OPT="--backfstype $FSTYPE" + if [ $type == MDS ]; then + opts+=${SECLEVEL:+" --param=mdt.sec_level"} + opts+=${MDSCAPA:+" --param-mdt.capa=$MDSCAPA"} + opts+=${STRIPE_BYTES:+" --param=lov.stripesize=$STRIPE_BYTES"} + opts+=${STRIPES_PER_OBJ:+" --param=lov.stripecount=$STRIPES_PER_OBJ"} + opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"} - stopall - # We need ldiskfs here, may as well load them all - load_modules - [ "$CLIENTONLY" ] && return - echo Formatting mgs, mds, osts - if ! combined_mgs_mds ; then - add mgs $(mkfs_opts mgs) $FSTYPE_OPT --reformat $MGSDEV || exit 10 - fi + if [ $fstype == ldiskfs ]; then + opts+=${IAMDIR:+" --iam-dir"} - for num in `seq $MDSCOUNT`; do - echo "Format mds$num: $(mdsdevname $num)" - if $VERBOSE; then - add mds$num $(mkfs_opts mds) $FSTYPE_OPT --reformat $(mdsdevname $num) || exit 10 - else - add mds$num $(mkfs_opts mds) $FSTYPE_OPT --reformat $(mdsdevname $num) > /dev/null || exit 10 - fi - done + fs_mkfs_opts+=${MDSJOURNALSIZE:+" -J size=$MDSJOURNALSIZE"} + fs_mkfs_opts+=${MDSISIZE:+" -i $MDSISIZE"} + fi + fi - # the ost-s could have different OST_MKFS_OPTS - # because of different failnode-s - for num in `seq $OSTCOUNT`; do - echo "Format ost$num: $(ostdevname $num)" - if $VERBOSE; then - add ost$num $(mkfs_opts ost${num}) $FSTYPE_OPT --reformat `ostdevname $num` || exit 10 - else - add ost$num $(mkfs_opts ost${num}) $FSTYPE_OPT --reformat `ostdevname $num` > /dev/null || exit 10 - fi - done + if [ $type == OST ]; then + opts+=${SECLEVEL:+" --param=ost.sec_level"} + opts+=${OSSCAPA:+" --param=ost.capa=$OSSCAPA"} + + if [ $fstype == ldiskfs ]; then + fs_mkfs_opts+=${OSTJOURNALSIZE:+" -J size=$OSTJOURNALSIZE"} + fi + fi + + opts+=" --backfstype=$fstype" + + var=${type}SIZE + if [ -n "${!var}" ]; then + opts+=" --device-size=${!var}" + fi + + var=$(upper $fstype)_MKFS_OPTS + fs_mkfs_opts+=${!var:+" ${!var}"} + + var=${type}_FS_MKFS_OPTS + fs_mkfs_opts+=${!var:+" ${!var}"} + + if [ -n "${fs_mkfs_opts## }" ]; then + opts+=" --mkfsoptions=\\\"${fs_mkfs_opts## }\\\"" + fi + + var=${type}OPT + opts+=${!var:+" ${!var}"} + + echo -n "$opts" +} + +formatall() { + local quiet + + if ! $VERBOSE; then + quiet=yes + fi + + stopall + # We need ldiskfs here, may as well load them all + load_modules + [ "$CLIENTONLY" ] && return + echo Formatting mgs, mds, osts + if ! combined_mgs_mds ; then + echo "Format mgs: $(mgsdevname)" + add mgs $(mkfs_opts mgs) --reformat $(mgsdevname) \ + $(mgsvdevname) ${quiet:+>/dev/null} || exit 10 + fi + + for num in `seq $MDSCOUNT`; do + echo "Format mds$num: $(mdsdevname $num)" + add mds$num $(mkfs_opts mds$num) --reformat \ + $(mdsdevname $num) $(mdsvdevname $num) \ + ${quiet:+>/dev/null} || exit 10 + done + + for num in `seq $OSTCOUNT`; do + echo "Format ost$num: $(ostdevname $num)" + add ost$num $(mkfs_opts ost$num) --reformat \ + $(ostdevname $num) $(ostvdevname ${num}) \ + ${quiet:+>/dev/null} || exit 10 + done } mount_client() { @@ -2000,7 +2750,7 @@ setupall() { echo $WRITECONF | grep -q "writeconf" && \ writeconf_all if ! combined_mgs_mds ; then - start mgs $MGSDEV $MGS_MOUNT_OPTS + start mgs $(mgsdevname) $MGS_MOUNT_OPTS fi for num in `seq $MDSCOUNT`; do @@ -2043,6 +2793,7 @@ setupall() { [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE mount_client $MOUNT [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT + clients_up if [ "$MOUNT_2" ]; then mount_client $MOUNT2 @@ -2080,7 +2831,7 @@ init_facet_vars () { eval export ${facet}_opt=\"$@\" local dev=${facet}_dev - local label=$(do_facet ${facet} "$E2LABEL ${!dev}") + local label=$(devicelabel ${facet} ${!dev}) [ -z "$label" ] && echo no label for ${!dev} && exit 1 eval export ${facet}_svc=${label} @@ -2120,7 +2871,7 @@ init_facets_vars () { done fi - combined_mgs_mds || init_facet_vars mgs $MGSDEV $MGS_MOUNT_OPTS + combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS remote_ost_nodsh && return @@ -2150,12 +2901,6 @@ osc_ensure_active () { } init_param_vars () { - if ! remote_ost_nodsh && ! remote_mds_nodsh; then - export MDSVER=$(do_facet $SINGLEMDS "lctl get_param version" | cut -d. -f1,2) - export OSTVER=$(do_facet ost1 "lctl get_param version" | cut -d. -f1,2) - export CLIVER=$(lctl get_param version | cut -d. -f 1,2) - fi - remote_mds_nodsh || TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout") @@ -2164,7 +2909,7 @@ init_param_vars () { osc_ensure_active $SINGLEMDS $TIMEOUT osc_ensure_active client $TIMEOUT - if [ $QUOTA_AUTO -ne 0 ]; then + if [ $QUOTA_AUTO -ne 0 ]; then if [ "$ENABLE_QUOTA" ]; then echo "enable quota as required" setup_quota $MOUNT || return 2 @@ -2186,6 +2931,9 @@ nfs_client_mode () { # FIXME: remove hostname when 19215 fixed do_nodes $clients "echo \\\$(hostname); grep ' '$MOUNT' ' /proc/mounts" declare -a nfsexport=(`grep ' '$MOUNT' ' /proc/mounts | awk '{print $1}' | awk -F: '{print $1 " " $2}'`) + if [[ ${#nfsexport[@]} -eq 0 ]]; then + error_exit NFSCLIENT=$NFSCLIENT mode, but no NFS export found! + fi do_nodes ${nfsexport[0]} "echo \\\$(hostname); df -T ${nfsexport[1]}" return fi @@ -2332,18 +3080,17 @@ check_and_setup_lustre() { init_facets_vars init_param_vars - do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=\\\"$PTLDEBUG\\\"; - lctl set_param subsystem_debug=\\\"${SUBSYSTEM# }\\\"; - lctl set_param debug_mb=${DEBUG_SIZE}; - sync" + set_default_debug_nodes $(comma_list $(nodes_list)) fi - init_gss - set_flavor_all $SEC + init_gss + if $GSS; then + set_flavor_all $SEC + fi - if [ "$ONLY" == "setup" ]; then - exit 0 - fi + if [ "$ONLY" == "setup" ]; then + exit 0 + fi } restore_mount () { @@ -2373,26 +3120,24 @@ cleanup_and_setup_lustre() { # Get all of the server target devices from a given server node and type. get_mnt_devs() { - local node=$1 - local type=$2 - local obd_type - local devs - local dev - - case $type in - mdt) obd_type="osd" ;; - ost) obd_type="obdfilter" ;; # needs to be fixed when OST also uses an OSD - *) echo "invalid server type" && return 1 ;; - esac - - devs=$(do_node $node "lctl get_param -n $obd_type*.*.mntdev") - for dev in $devs; do - case $dev in - *loop*) do_node $node "losetup $dev" | \ - sed -e "s/.*(//" -e "s/).*//" ;; - *) echo $dev ;; - esac - done + local node=$1 + local type=$2 + local devs + local dev + + if [ "$type" == ost ]; then + devs=$(get_obdfilter_param $node "" mntdev) + else + devs=$(do_node $node \ + "lctl get_param -n osd-*.$FSNAME-M*.mntdev") + fi + for dev in $devs; do + case $dev in + *loop*) do_node $node "losetup $dev" | \ + sed -e "s/.*(//" -e "s/).*//" ;; + *) echo $dev ;; + esac + done } # Get all of the server target devices. @@ -2414,11 +3159,10 @@ get_svr_devs() { run_e2fsck() { local node=$1 local target_dev=$2 - local ostidx=$3 - local ostdb_opt=$4 + local extra_opts=$3 df > /dev/null # update statfs data on disk - local cmd="$E2FSCK -d -v -f -n $MDSDB_OPT $ostdb_opt $target_dev" + local cmd="$E2FSCK -d -v -t -t -f -n $extra_opts $target_dev" echo $cmd local rc=0 do_node $node $cmd || rc=$? @@ -2427,34 +3171,38 @@ run_e2fsck() { return 0 } +# verify a directory is shared among nodes. +check_shared_dir() { + local dir=$1 + + [ -z "$dir" ] && return 1 + do_rpc_nodes $(comma_list $(nodes_list)) check_logdir $dir + check_write_access $dir || return 1 + return 0 +} + # Run e2fsck on MDT and OST(s) to generate databases used for lfsck. generate_db() { local i local ostidx local dev - local tmp_file - [ $MDSCOUNT -eq 1 ] || error "CMD is not supported" - tmp_file=$(mktemp -p $SHARED_DIRECTORY || - error "fail to create file in $SHARED_DIRECTORY") + check_shared_dir $SHARED_DIRECTORY || + error "$SHARED_DIRECTORY isn't a shared directory" - # make sure everything gets to the backing store - local list=$(comma_list $CLIENTS $(facet_host $SINGLEMDS) $(osts_nodes)) - do_nodes $list "sync; sleep 2; sync" + export MDSDB=$SHARED_DIRECTORY/mdsdb + export OSTDB=$SHARED_DIRECTORY/ostdb - do_nodes $list ls $tmp_file || \ - error "$SHARED_DIRECTORY is not a shared directory" - rm $tmp_file + [ $MDSCOUNT -eq 1 ] || error "CMD is not supported" - run_e2fsck $(mdts_nodes) $MDTDEV + run_e2fsck $(mdts_nodes) $MDTDEV "--mdsdb $MDSDB" i=0 ostidx=0 OSTDB_LIST="" for node in $(osts_nodes); do for dev in ${OSTDEVS[i]}; do - local ostdb_opt=`eval echo $OSTDB_OPT` - run_e2fsck $node $dev $ostidx "$ostdb_opt" + run_e2fsck $node $dev "--mdsdb $MDSDB --ostdb $OSTDB-$ostidx" OSTDB_LIST="$OSTDB_LIST $OSTDB-$ostidx" ostidx=$((ostidx + 1)) done @@ -2476,18 +3224,15 @@ run_lfsck() { } check_and_cleanup_lustre() { - if [ "$LFSCK_ALWAYS" = "yes" ]; then + if [ "$LFSCK_ALWAYS" = "yes" -a "$TESTSUITE" != "lfsck" ]; then get_svr_devs generate_db - if [ "$SKIP_LFSCK" == "no" ]; then - run_lfsck - else - echo "skip lfsck" - fi + run_lfsck fi if is_mounted $MOUNT; then - [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* + [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* || + error "remove sub-test dirs failed" [ "$ENABLE_QUOTA" ] && restore_quota_type || true fi @@ -2508,28 +3253,45 @@ check_and_cleanup_lustre() { ####### # General functions -check_network() { - local NETWORK=0 - local WAIT=0 - local MAX=$2 - while [ $NETWORK -eq 0 ]; do - if ping -c 1 -w 3 $1 > /dev/null; then - NETWORK=1 - else - WAIT=$((WAIT + 5)) - echo "waiting for $1, $((MAX - WAIT)) secs left" - sleep 5 - fi - if [ $WAIT -gt $MAX ]; then - echo "Network not available" - exit 1 - fi +wait_for_function () { + local quiet="" + + # suppress fn both stderr and stdout + if [ "$1" = "--quiet" ]; then + shift + quiet=" > /dev/null 2>&1" + + fi + + local fn=$1 + local max=${2:-900} + local sleep=${3:-5} + + local wait=0 + + while true; do + + eval $fn $quiet && return 0 + + wait=$((wait + sleep)) + [ $wait -lt $max ] || return 1 + echo waiting $fn, $((max - wait)) secs left ... + sleep $sleep done } -check_port() { - while( !($DSH2 $1 "netstat -tna | grep -q $2") ) ; do - sleep 9 - done + +check_network() { + local host=$1 + local max=$2 + local sleep=${3:-5} + + echo `date +"%H:%M:%S (%s)"` waiting for $host network $max secs ... + if ! wait_for_function --quiet "ping -c 1 -w 3 $host" $max $sleep ; then + echo "Network not available!" + exit 1 + fi + + echo `date +"%H:%M:%S (%s)"` network interface is UP } no_dsh() { @@ -2631,15 +3393,22 @@ at_is_enabled() { fi } -at_max_get() { +at_get() { local facet=$1 + local at=$2 - # suppose that all ost-s has the same at_max set - if [ $facet == "ost" ]; then - do_facet ost1 "lctl get_param -n at_max" - else - do_facet $facet "lctl get_param -n at_max" - fi + # suppose that all ost-s have the same $at value set + [ $facet != "ost" ] || facet=ost1 + + do_facet $facet "lctl get_param -n $at" +} + +at_max_get() { + at_get $1 at_max +} + +at_min_get() { + at_get $1 at_min } at_max_set() { @@ -2647,20 +3416,17 @@ at_max_set() { shift local facet + local hosts for facet in $@; do if [ $facet == "ost" ]; then - for i in `seq $OSTCOUNT`; do - do_facet ost$i "lctl set_param at_max=$at_max" - - done + facet=$(get_facets OST) elif [ $facet == "mds" ]; then - for i in `seq $MDSCOUNT`; do - do_facet mds$i "lctl set_param at_max=$at_max" - done - else - do_facet $facet "lctl set_param at_max=$at_max" + facet=$(get_facets MDS) fi + hosts=$(expand_list $hosts $(facets_hosts $facet)) done + + do_nodes $hosts lctl set_param at_max=$at_max } ################################## @@ -2705,11 +3471,14 @@ pause_bulk() { drop_ldlm_cancel() { #define OBD_FAIL_LDLM_CANCEL 0x304 - RC=0 - do_facet client lctl set_param fail_loc=0x304 - do_facet client "$@" || RC=$? - do_facet client lctl set_param fail_loc=0 - return $RC + local RC=0 + local list=$(comma_list $(mdts_nodes) $(osts_nodes)) + do_nodes $list lctl set_param fail_loc=0x304 + + do_facet client "$@" || RC=$? + + do_nodes $list lctl set_param fail_loc=0 + return $RC } drop_bl_callback() { @@ -2816,6 +3585,17 @@ stop_full_debug_logging() { debugrestore } +# prints bash call stack +log_trace_dump() { + echo " Trace dump:" + for (( i=1; i < ${#BASH_LINENO[*]} ; i++ )) ; do + local s=${BASH_SOURCE[$i]} + local l=${BASH_LINENO[$i-1]} + local f=${FUNCNAME[$i]} + echo " = $s:$l:$f()" + done +} + ################################## # Test interface ################################## @@ -2831,7 +3611,9 @@ error_noexit() { fi log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ " + log_trace_dump + mkdir -p $LOGDIR # We need to dump the logs on all nodes if $dump; then gather_logs $(comma_list $(nodes_list)) @@ -2842,6 +3624,14 @@ error_noexit() { echo "$@" > $LOGDIR/err } +exit_status () { + local status=0 + local log=$TESTSUITELOG + + [ -f "$log" ] && grep -q FAIL $log && status=1 + exit $status +} + error() { error_noexit "$@" exit 1 @@ -2867,6 +3657,8 @@ skip_env () { skip () { echo log " SKIP: ${TESTSUITE} ${TESTNAME} $@" + [ "$ALWAYS_SKIPPED" ] && \ + skip_logged ${TESTNAME} "$@" || true [ "$TESTSUITELOG" ] && \ echo "${TESTSUITE}: SKIP: $TESTNAME $@" >> $TESTSUITELOG || true } @@ -2882,9 +3674,12 @@ build_test_filter() { log "excepting tests: `echo $EXCEPT $ALWAYS_EXCEPT`" [ "$EXCEPT_SLOW" ] && \ log "skipping tests SLOW=no: `echo $EXCEPT_SLOW`" - for E in $EXCEPT $ALWAYS_EXCEPT; do + for E in $EXCEPT; do eval EXCEPT_${E}=true done + for E in $ALWAYS_EXCEPT; do + eval EXCEPT_ALWAYS_${E}=true + done for E in $EXCEPT_SLOW; do eval EXCEPT_SLOW_${E}=true done @@ -2903,6 +3698,7 @@ basetest() { # print a newline if the last test was skipped export LAST_SKIPPED= +export ALWAYS_SKIPPED= # # Main entry into test-framework. This is called with the name and # description of a test. The name is used to find the function to run @@ -2944,6 +3740,20 @@ run_test() { TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)" return 0 fi + testname=EXCEPT_ALWAYS_$1 + if [ ${!testname}x != x ]; then + LAST_SKIPPED="y" + ALWAYS_SKIPPED="y" + TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1" + return 0 + fi + testname=EXCEPT_ALWAYS_$base + if [ ${!testname}x != x ]; then + LAST_SKIPPED="y" + ALWAYS_SKIPPED="y" + TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1 (base $base)" + return 0 + fi testname=EXCEPT_SLOW_$1 if [ ${!testname}x != x ]; then LAST_SKIPPED="y" @@ -2958,6 +3768,7 @@ run_test() { fi LAST_SKIPPED= + ALWAYS_SKIPPED= run_one_logged $1 "$2" return $? @@ -2992,6 +3803,12 @@ trace() { return 1 } +complete () { + equals_msg $1 test complete, duration $2 sec + [ -f "$TESTSUITELOG" ] && egrep .FAIL $TESTSUITELOG || true + echo duration $2 >>$TESTSUITELOG +} + pass() { # Set TEST_STATUS here; will be used for logging the result if [ -f $LOGDIR/err ]; then @@ -2999,12 +3816,15 @@ pass() { else TEST_STATUS="PASS" fi - echo $TEST_STATUS " " $@ + echo "$TEST_STATUS $@" 2>&1 | tee -a $TESTSUITELOG } check_mds() { - FFREE=$(do_node $SINGLEMDS lctl get_param -n osd.*MDT*.filesfree | awk 'BEGIN{avail=0}; {avail+=$1}; END{print avail}') - FTOTAL=$(do_node $SINGLEMDS lctl get_param -n osd.*MDT*.filestotal | awk 'BEGIN{avail=0}; {avail+=$1}; END{print avail}') + local FFREE=$(do_node $SINGLEMDS \ + lctl get_param -n osd*.*MDT*.filesfree | calc_sum) + local FTOTAL=$(do_node $SINGLEMDS \ + lctl get_param -n osd*.*MDT*.filestotal | calc_sum) + [ $FFREE -ge $FTOTAL ] && error "files free $FFREE > total $FTOTAL" || true } @@ -3039,7 +3859,7 @@ banner() { run_one() { local testnum=$1 local message=$2 - tfile=f${testnum} + tfile=f.${TESTSUITE}.${testnum} export tdir=d0.${TESTSUITE}/d${base} export TESTNAME=test_$testnum local SAVE_UMASK=`umask` @@ -3065,41 +3885,48 @@ run_one() { # - test result is saved to data file # run_one_logged() { - local BEFORE=`date +%s` - local TEST_ERROR - local name=${TESTSUITE}.test_${1}.test_log.$(hostname).log - local test_log=$LOGDIR/$name - rm -rf $LOGDIR/err + local BEFORE=`date +%s` + local TEST_ERROR + local name=${TESTSUITE}.test_${1}.test_log.$(hostname -s).log + local test_log=$LOGDIR/$name + rm -rf $LOGDIR/err + local SAVE_UMASK=`umask` + umask 0022 - echo - log_sub_test_begin test_${1} - (run_one $1 "$2") 2>&1 | tee $test_log - local RC=${PIPESTATUS[0]} + echo + log_sub_test_begin test_${1} + (run_one $1 "$2") 2>&1 | tee -i $test_log + local RC=${PIPESTATUS[0]} - [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \ - echo "test_$1 returned $RC" | tee $LOGDIR/err + [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \ + echo "test_$1 returned $RC" | tee $LOGDIR/err - duration=$((`date +%s` - $BEFORE)) - pass "(${duration}s)" - [ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err) - log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR" + duration=$((`date +%s` - $BEFORE)) + pass "$1" "(${duration}s)" + [ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err) + log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR" - if [ -f $LOGDIR/err ]; then - $FAIL_ON_ERROR && exit $RC - fi + if [ -f $LOGDIR/err ]; then + $FAIL_ON_ERROR && exit $RC + fi - return 0 + umask $SAVE_UMASK + + return 0 +} + +# +# Print information of skipped tests to result.yml +# +skip_logged(){ + log_sub_test_begin $1 + log_sub_test_end "SKIP" "0" "0" "\"$2\"" } canonical_path() { (cd `dirname $1`; echo $PWD/`basename $1`) } -sync_clients() { - [ -d $DIR1 ] && cd $DIR1 && sync; sleep 1; sync - [ -d $DIR2 ] && cd $DIR2 && sync; sleep 1; sync - cd $SAVE_PWD -} check_grant() { export base=`basetest $1` @@ -3109,39 +3936,34 @@ check_grant() { [ ${!testname}x == x ] && return 0 echo -n "checking grant......" - cd $SAVE_PWD - # write some data to sync client lost_grant - rm -f $DIR1/${tfile}_check_grant_* 2>&1 - for i in `seq $OSTCOUNT`; do - $LFS setstripe $DIR1/${tfile}_check_grant_$i -i $(($i -1)) -c 1 - dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \ - count=1 > /dev/null 2>&1 - done - # sync all the data and make sure no pending data on server - sync_clients - - #get client grant and server grant - client_grant=0 - for d in `lctl get_param -n osc.*.cur_grant_bytes`; do - client_grant=$((client_grant + $d)) - done - server_grant=0 - for d in `lctl get_param -n obdfilter.*.tot_granted`; do - server_grant=$((server_grant + $d)) - done - # cleanup the check_grant file - for i in `seq $OSTCOUNT`; do - rm $DIR1/${tfile}_check_grant_$i - done + local clients=$CLIENTS + [ -z $clients ] && clients=$(hostname) + + # sync all the data and make sure no pending data on server + do_nodes $clients sync + + # get client grant + client_grant=`do_nodes $clients \ + "$LCTL get_param -n osc.${FSNAME}-*.cur_*grant_bytes" | \ + awk '{total += $1} END{print total}'` + + # get server grant + server_grant=`do_nodes $(comma_list $(osts_nodes)) \ + "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" | + awk '{total += $1} END{print total}'` + + # check whether client grant == server grant + if [ $client_grant -ne $server_grant ]; then + echo "failed: client:${client_grant} server: ${server_grant}." + do_nodes $(comma_list $(osts_nodes)) \ + "$LCTL get_param obdfilter.${FSNAME}-OST*.tot*" + do_nodes $clients "$LCTL get_param osc.${FSNAME}-*.cur_*_bytes" + return 1 + else + echo "pass: client:${client_grant} server: ${server_grant}" + fi - #check whether client grant == server grant - if [ $client_grant != $server_grant ]; then - echo "failed: client:${client_grant} server: ${server_grant}" - return 1 - else - echo "pass" - fi } ######################## @@ -3159,7 +3981,22 @@ osc_to_ost() ostuuid_from_index() { - $LFS osts $2 | awk '/^'$1'/ { print $2 }' + $LFS osts $2 | sed -ne "/^$1: /s/.* \(.*\) .*$/\1/p" +} + +ostname_from_index() { + local uuid=$(ostuuid_from_index $1) + echo ${uuid/_UUID/} +} + +index_from_ostuuid() +{ + $LFS osts $2 | sed -ne "/${1}/s/\(.*\): .* .*$/\1/p" +} + +mdtuuid_from_index() +{ + $LFS mdts $2 | sed -ne "/^$1: /s/.* \(.*\) .*$/\1/p" } remote_node () { @@ -3288,19 +4125,20 @@ remote_nodes_list () { init_clients_lists () { # Sanity check: exclude the local client from RCLIENTS - local rclients=$(echo " $RCLIENTS " | sed -re "s/\s+$HOSTNAME\s+/ /g") + local clients=$(hostlist_expand "$RCLIENTS") + local rclients=$(exclude_items_from_list "$clients" $HOSTNAME) # Sanity check: exclude the dup entries - rclients=$(for i in $rclients; do echo $i; done | sort -u) + RCLIENTS=$(for i in ${rclients//,/ }; do echo $i; done | sort -u) - local clients="$SINGLECLIENT $HOSTNAME $rclients" + clients="$SINGLECLIENT $HOSTNAME $RCLIENTS" # Sanity check: exclude the dup entries from CLIENTS # for those configs which has SINGLCLIENT set to local client clients=$(for i in $clients; do echo $i; done | sort -u) - CLIENTS=`comma_list $clients` - local -a remoteclients=($rclients) + CLIENTS=$(comma_list $clients) + local -a remoteclients=($RCLIENTS) for ((i=0; $i<${#remoteclients[@]}; i++)); do varname=CLIENT$((i + 2)) eval $varname=${remoteclients[i]} @@ -3331,7 +4169,8 @@ is_patchless () } check_versions () { - [ "$MDSVER" = "$CLIVER" -a "$OSTVER" = "$CLIVER" ] + [ "$(lustre_version_code client)" = "$(lustre_version_code $SINGLEMDS)" -a \ + "$(lustre_version_code client)" = "$(lustre_version_code ost1)" ] } get_node_count() { @@ -3443,7 +4282,7 @@ fi" # Run multiop in the background, but wait for it to print # "PAUSING" to its stdout before returning from this function. multiop_bg_pause() { - MULTIOP_PROG=${MULTIOP_PROG:-multiop} + MULTIOP_PROG=${MULTIOP_PROG:-$MULTIOP} FILE=$1 ARGS=$2 @@ -3488,7 +4327,8 @@ inodes_available () { } mdsrate_inodes_available () { - echo $(($(inodes_available) - 1)) + local min_inodes=$(inodes_available) + echo $((min_inodes * 99 / 100)) } # reset llite stat counters @@ -3553,25 +4393,6 @@ exit \\\$rc;" fi } -# $1 node -# $2 file -# $3 $RUNAS -get_stripe_info() { - local tmp_file - - stripe_size=0 - stripe_count=0 - stripe_index=0 - tmp_file=$(mktemp) - - do_facet $1 $3 lfs getstripe -v $2 > $tmp_file - - stripe_size=`awk '$1 ~ /size/ {print $2}' $tmp_file` - stripe_count=`awk '$1 ~ /count/ {print $2}' $tmp_file` - stripe_index=`awk '$1 ~ /stripe_offset/ {print $2}' $tmp_file` - rm -f $tmp_file -} - # CMD: determine mds index where directory inode presents get_mds_dir () { local dir=$1 @@ -3630,26 +4451,17 @@ convert_facet2label() { } get_clientosc_proc_path() { - local ost=$1 - - echo "${1}-osc-*" + echo "${1}-osc-[^M]*" } get_lustre_version () { local facet=${1:-"$SINGLEMDS"} - do_facet $facet $LCTL get_param -n version | awk '/^lustre:/ {print $2}' -} - -get_mds_version_major () { - local facet=${1:-"$SINGLEMDS"} - local version=$(get_lustre_version $facet) - echo $version | awk -F. '{print $1}' + do_facet $facet $LCTL get_param -n version | awk '/^lustre:/ {print $2}' } -get_mds_version_minor () { +lustre_version_code() { local facet=${1:-"$SINGLEMDS"} - local version=$(get_lustre_version $facet) - echo $version | awk -F. '{print $2}' + version_code $(get_lustre_version $1) } # If the 2.0 MDS was mounted on 1.8 device, then the OSC and LOV names @@ -3658,10 +4470,8 @@ get_mds_version_minor () { # mdt osc: fsname-OSTXXXX-osc mds_on_old_device() { local mds=${1:-"$SINGLEMDS"} - local major=$(get_mds_version_major $mds) - local minor=$(get_mds_version_minor $mds) - if [ $major -ge 2 ] || [ $major -eq 1 -a $minor -gt 8 ]; then + if [ $(lustre_version_code $mds) -gt $(version_code 1.9.0) ]; then do_facet $mds "lctl list_param osc.$FSNAME-OST*-osc \ > /dev/null 2>&1" && return 0 fi @@ -3676,9 +4486,8 @@ get_mdtosc_proc_path() { local mdt_label=$(convert_facet2label $mds_facet) local mdt_index=$(echo $mdt_label | sed -e 's/^.*-//') - local major=$(get_mds_version_major $mds_facet) - local minor=$(get_mds_version_minor $mds_facet) - if [ $major -le 1 -a $minor -le 8 ] || mds_on_old_device $mds_facet; then + if [ $(lustre_version_code $mds_facet) -le $(version_code 1.8.0) ] || + mds_on_old_device $mds_facet; then echo "${ost_label}-osc" else echo "${ost_label}-osc-${mdt_index}" @@ -3702,12 +4511,19 @@ get_osc_import_name() { _wait_import_state () { local expected=$1 local CONN_PROC=$2 - local maxtime=${3:-max_recovery_time} + local maxtime=${3:-$(max_recovery_time)} local CONN_STATE local i=0 CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2) while [ "${CONN_STATE}" != "${expected}" ]; do + if [ "${expected}" == "DISCONN" ]; then + # for disconn we can check after proc entry is removed + [ "x${CONN_STATE}" == "x" ] && return 0 + # with AT enabled, we can have connect request timeout near of + # reconnect timeout and test can't see real disconnect + [ "${CONN_STATE}" == "CONNECTING" ] && return 0 + fi [ $i -ge $maxtime ] && \ error "can't put import for $CONN_PROC into ${expected} state after $i sec, have ${CONN_STATE}" && \ return 1 @@ -3723,43 +4539,75 @@ _wait_import_state () { wait_import_state() { local state=$1 local params=$2 - local maxtime=${3:-max_recovery_time} + local maxtime=${3:-$(max_recovery_time)} local param for param in ${params//,/ }; do _wait_import_state $state $param $maxtime || return done } + +# One client request could be timed out because server was not ready +# when request was sent by client. +# The request timeout calculation details : +# ptl_send_rpc () +# /* We give the server rq_timeout secs to process the req, and +# add the network latency for our local timeout. */ +# request->rq_deadline = request->rq_sent + request->rq_timeout + +# ptlrpc_at_get_net_latency(request) ; +# +# ptlrpc_connect_import () +# request->rq_timeout = INITIAL_CONNECT_TIMEOUT +# +# init_imp_at () -> +# -> at_init(&at->iat_net_latency, 0, 0) -> iat_net_latency=0 +# ptlrpc_at_get_net_latency(request) -> +# at_get (max (iat_net_latency=0, at_min)) = at_min +# +# i.e.: +# request->rq_timeout + ptlrpc_at_get_net_latency(request) = +# INITIAL_CONNECT_TIMEOUT + at_min +# +# We will use obd_timeout instead of INITIAL_CONNECT_TIMEOUT +# because we can not get this value in runtime, +# the value depends on configure options, and it is not stored in /proc. +# obd_support.h: +# #define CONNECTION_SWITCH_MIN 5U +# #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/20) + +request_timeout () { + local facet=$1 + + # request->rq_timeout = INITIAL_CONNECT_TIMEOUT + local init_connect_timeout=$TIMEOUT + [[ $init_connect_timeout -ge 5 ]] || init_connect_timeout=5 + + local at_min=$(at_get $facet at_min) + + echo $(( init_connect_timeout + at_min )) +} + wait_osc_import_state() { local facet=$1 local ost_facet=$2 local expected=$3 local ost=$(get_osc_import_name $facet $ost_facet) - local CONN_PROC - local CONN_STATE - local i=0 - CONN_PROC="osc.${ost}.ost_server_uuid" - CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2) - while [ "${CONN_STATE}" != "${expected}" ]; do - if [ "${expected}" == "DISCONN" ]; then - # for disconn we can check after proc entry is removed - [ "x${CONN_STATE}" == "x" ] && return 0 - # with AT we can have connect request timeout ~ reconnect timeout - # and test can't see real disconnect - [ "${CONN_STATE}" == "CONNECTING" ] && return 0 - fi - # disconnect rpc should be wait not more obd_timeout - [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \ - error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1 - sleep 1 - CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2) - i=$(($i + 1)) - done + local param="osc.${ost}.ost_server_uuid" + + # 1. wait the deadline of client 1st request (it could be skipped) + # 2. wait the deadline of client 2nd request + local maxtime=$(( 2 * $(request_timeout $facet))) + + if ! do_rpc_nodes $(facet_host $facet) \ + _wait_import_state $expected $param $maxtime; then + error "import is not in ${expected} state" + return 1 + fi - log "${ost_facet} now in ${CONN_STATE} state" return 0 } + get_clientmdc_proc_path() { echo "${1}-mdc-*" } @@ -3802,20 +4650,30 @@ wait_clients_import_state () { } oos_full() { - local -a AVAILA - local -a GRANTA - local OSCFULL=1 - AVAILA=($(do_nodes $(comma_list $(osts_nodes)) \ - $LCTL get_param obdfilter.*.kbytesavail)) - GRANTA=($(do_nodes $(comma_list $(osts_nodes)) \ - $LCTL get_param -n obdfilter.*.tot_granted)) - for ((i=0; i<${#AVAILA[@]}; i++)); do - local -a AVAIL1=(${AVAILA[$i]//=/ }) - GRANT=$((${GRANTA[$i]}/1024)) - echo -n $(echo ${AVAIL1[0]} | cut -d"." -f2) avl=${AVAIL1[1]} grnt=$GRANT diff=$((AVAIL1[1] - GRANT)) - [ $((AVAIL1[1] - GRANT)) -lt 400 ] && OSCFULL=0 && echo " FULL" || echo - done - return $OSCFULL + local -a AVAILA + local -a GRANTA + local -a TOTALA + local OSCFULL=1 + AVAILA=($(do_nodes $(comma_list $(osts_nodes)) \ + $LCTL get_param obdfilter.*.kbytesavail)) + GRANTA=($(do_nodes $(comma_list $(osts_nodes)) \ + $LCTL get_param -n obdfilter.*.tot_granted)) + TOTALA=($(do_nodes $(comma_list $(osts_nodes)) \ + $LCTL get_param -n obdfilter.*.kbytestotal)) + for ((i=0; i<${#AVAILA[@]}; i++)); do + local -a AVAIL1=(${AVAILA[$i]//=/ }) + local -a TOTAL=(${TOTALA[$i]//=/ }) + GRANT=$((${GRANTA[$i]}/1024)) + # allow 1% of total space in bavail because of delayed + # allocation with ZFS which might release some free space after + # txg commit. For small devices, we set a mininum of 8MB + local LIMIT=$((${TOTAL} / 100 + 8000)) + echo -n $(echo ${AVAIL1[0]} | cut -d"." -f2) avl=${AVAIL1[1]} \ + grnt=$GRANT diff=$((AVAIL1[1] - GRANT)) limit=${LIMIT} + [ $((AVAIL1[1] - GRANT)) -lt $LIMIT ] && OSCFULL=0 && \ + echo " FULL" || echo + done + return $OSCFULL } pool_list () { @@ -3904,7 +4762,7 @@ destroy_pools () { echo destroy the created pools: ${!listvar} for poolname in ${!listvar//,/ }; do - destroy_pool $fsname.$poolname + destroy_pool $fsname.$poolname done } @@ -3919,38 +4777,34 @@ gather_logs () { local ts=$(date +%s) local docp=true + + if [[ ! -f "$YAML_LOG" ]]; then + # init_logging is not performed before gather_logs, + # so the $LOGDIR needs to be checked here + check_shared_dir $LOGDIR && touch $LOGDIR/shared + fi + [ -f $LOGDIR/shared ] && docp=false - + # dump lustre logs, dmesg - prefix="$LOGDIR/${TESTSUITE}.${TESTNAME}" + prefix="$TESTLOG_PREFIX.$TESTNAME" suffix="$ts.log" echo "Dumping lctl log to ${prefix}.*.${suffix}" if [ "$CLIENTONLY" -o "$PDSH" == "no_dsh" ]; then echo "Dumping logs only on local client." - $LCTL dk > ${prefix}.debug_log.$(hostname).${suffix} - dmesg > ${prefix}.dmesg.$(hostname).${suffix} + $LCTL dk > ${prefix}.debug_log.$(hostname -s).${suffix} + dmesg > ${prefix}.dmesg.$(hostname -s).${suffix} return fi do_nodesv $list \ - "$LCTL dk > ${prefix}.debug_log.\\\$(hostname).${suffix}; - dmesg > ${prefix}.dmesg.\\\$(hostname).${suffix}" + "$LCTL dk > ${prefix}.debug_log.\\\$(hostname -s).${suffix}; + dmesg > ${prefix}.dmesg.\\\$(hostname -s).${suffix}" if [ ! -f $LOGDIR/shared ]; then do_nodes $list rsync -az "${prefix}.*.${suffix}" $HOSTNAME:$LOGDIR - fi - - local archive=$LOGDIR/${TESTSUITE}-$ts.tar.bz2 - tar -jcf $archive $LOGDIR/*$ts* $LOGDIR/*${TESTSUITE}* - - echo $archive -} - -cleanup_logs () { - local list=${1:-$(comma_list $(nodes_list))} - - [ -n ${TESTSUITE} ] && do_nodes $list "rm -f $TMP/*${TESTSUITE}*" || true + fi } do_ls () { @@ -3987,7 +4841,7 @@ do_ls () { max_recovery_time () { local init_connect_timeout=$(( TIMEOUT / 20 )) - [[ $init_connect_timeout > 5 ]] || init_connect_timeout=5 + [[ $init_connect_timeout -ge 5 ]] || init_connect_timeout=5 local service_time=$(( $(at_max_get client) + $(( 2 * $(( 25 + 1 + init_connect_timeout)) )) )) @@ -4226,7 +5080,7 @@ wait_flavor() echo -n "checking $dir..." res=$(do_check_flavor $dir $flavor) echo "found $res/$expect $flavor connections" - [ $res -eq $expect ] && return 0 + [ $res -ge $expect ] && return 0 sleep 4 done @@ -4295,18 +5149,23 @@ check_logdir() { # Not found. Create local logdir mkdir -p $dir else - touch $dir/node.$(hostname).yml + touch $dir/check_file.$(hostname -s) fi return 0 } check_write_access() { local dir=$1 + local node + local file + for node in $(nodes_list); do - if [ ! -f "$dir/node.${node}.yml" ]; then + file=$dir/check_file.$(short_hostname $node) + if [[ ! -f "$file" ]]; then # Logdir not accessible/writable from this node. return 1 fi + rm -f $file || return 1 done return 0 } @@ -4315,20 +5174,26 @@ init_logging() { if [[ -n $YAML_LOG ]]; then return fi + local SAVE_UMASK=`umask` + umask 0000 + export YAML_LOG=${LOGDIR}/results.yml mkdir -p $LOGDIR init_clients_lists - do_rpc_nodes $(comma_list $(nodes_list)) check_logdir $LOGDIR - if check_write_access $LOGDIR; then - touch $LOGDIR/shared - echo "Logging to shared log directory: $LOGDIR" - else - echo "Logging to local directory: $LOGDIR" + if [ ! -f $YAML_LOG ]; then # If the yaml log already exists then we will just append to it + if check_shared_dir $LOGDIR; then + touch $LOGDIR/shared + echo "Logging to shared log directory: $LOGDIR" + else + echo "Logging to local directory: $LOGDIR" + fi + + yml_nodes_file $LOGDIR >> $YAML_LOG + yml_results_file >> $YAML_LOG fi - yml_nodes_file $LOGDIR - yml_results_file >> $YAML_LOG + umask $SAVE_UMASK } log_test() { @@ -4340,16 +5205,17 @@ log_test_status() { } log_sub_test_begin() { - yml_log_sub_test_begin $@ >> $YAML_LOG + yml_log_sub_test_begin "$@" >> $YAML_LOG } log_sub_test_end() { - yml_log_sub_test_end $@ >> $YAML_LOG + yml_log_sub_test_end "$@" >> $YAML_LOG } run_llverdev() { local dev=$1 + local llverdev_opts=$2 local devname=$(basename $1) local size=$(grep "$devname"$ /proc/partitions | awk '{print $3}') # loop devices aren't in /proc/partitions @@ -4359,39 +5225,65 @@ run_llverdev() local partial_arg="" # Run in partial (fast) mode if the size - # of a partition > 10 GB - [ $size -gt 10 ] && partial_arg="-p" + # of a partition > 1 GB + [ $size -gt 1 ] && partial_arg="-p" - llverdev --force $partial_arg $dev + llverdev --force $partial_arg $llverdev_opts $dev } -remove_mdt_files() { - local facet=$1 - local mdtdev=$2 - shift 2 - local files="$@" - local mntpt=${MOUNT%/*}/$facet +run_llverfs() +{ + local dir=$1 + local llverfs_opts=$2 + local partial_arg="" + local size=$(df -B G $dir |tail -n 1 |awk '{print $2}' |sed 's/G//') #GB - echo "removing files from $mdtdev on $facet: $files" - mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $? - rc=0; - for f in $files; do - rm $mntpt/ROOT/$f || { rc=$?; break; } - done - umount -f $mntpt || return $? - return $rc + # Run in partial (fast) mode if the size + # of a partition > 1 GB + [ $size -gt 1 ] && partial_arg="-p" + + llverfs $partial_arg $llverfs_opts $dir } -duplicate_mdt_files() { - local facet=$1 - local mdtdev=$2 - shift 2 - local files="$@" - local mntpt=${MOUNT%/*}/$facet +remove_mdt_files() { + local facet=$1 + local mdtdev=$2 + shift 2 + local files="$@" + local mntpt=$(facet_mntpt $facet) + local opts=$MDS_MOUNT_OPTS + + echo "removing files from $mdtdev on $facet: $files" + if [ $(facet_fstype $facet) == ldiskfs ] && + ! do_facet $facet test -b ${!dev}; then + opts=$(csa_add "$opts" -o loop) + fi + mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt || + return $? + rc=0; + for f in $files; do + rm $mntpt/ROOT/$f || { rc=$?; break; } + done + umount -f $mntpt || return $? + return $rc +} - echo "duplicating files on $mdtdev on $facet: $files" - mkdir -p $mntpt || return $? - mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $? +duplicate_mdt_files() { + local facet=$1 + local mdtdev=$2 + shift 2 + local files="$@" + local mntpt=$(facet_mntpt $facet) + local opts=$MDS_MOUNT_OPTS + + echo "duplicating files on $mdtdev on $facet: $files" + mkdir -p $mntpt || return $? + if [ $(facet_fstype $facet) == ldiskfs ] && + ! do_facet $facet test -b ${!dev}; then + opts=$(csa_add "$opts" -o loop) + fi + mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt || + return $? do_umount() { trap 0 @@ -4440,3 +5332,169 @@ else echo \\\$(basename \\\$dv); fi;" } + +is_sanity_benchmark() { + local benchmarks="dbench bonnie iozone fsx" + local suite=$1 + for b in $benchmarks; do + if [ "$b" == "$suite" ]; then + return 0 + fi + done + return 1 +} + +min_ost_size () { + $LCTL get_param -n osc.*.kbytesavail | sort -n | head -n1 +} + +# Get the block size of the filesystem. +get_block_size() { + local facet=$1 + local device=$2 + local size + + size=$(do_facet $facet "$DUMPE2FS -h $device 2>&1" | + awk '/^Block size:/ {print $3}') + echo $size +} + +# Check whether the "large_xattr" feature is enabled or not. +large_xattr_enabled() { + local mds_dev=$(mdsdevname ${SINGLEMDS//mds/}) + + do_facet $SINGLEMDS "$DUMPE2FS -h $mds_dev 2>&1 | grep -q large_xattr" + return ${PIPESTATUS[0]} +} + +# Get the maximum xattr size supported by the filesystem. +max_xattr_size() { + local size + + if large_xattr_enabled; then + # include/linux/limits.h: #define XATTR_SIZE_MAX 65536 + size=65536 + else + local mds_dev=$(mdsdevname ${SINGLEMDS//mds/}) + local block_size=$(get_block_size $SINGLEMDS $mds_dev) + + # maximum xattr size = size of block - size of header - + # size of 1 entry - 4 null bytes + size=$((block_size - 32 - 32 - 4)) + fi + + echo $size +} + +# Dump the value of the named xattr from a file. +get_xattr_value() { + local xattr_name=$1 + local file=$2 + + echo "$(getfattr -n $xattr_name --absolute-names --only-values $file)" +} + +# Generate a string with size of $size bytes. +generate_string() { + local size=${1:-1024} # in bytes + + echo "$(head -c $size < /dev/zero | tr '\0' y)" +} + +reformat_external_journal() { + if [ ! -z ${EJOURNAL} ]; then + local rcmd="do_facet ${SINGLEMDS}" + + echo "reformat external journal on ${SINGLEMDS}:${EJOURNAL}" + ${rcmd} mke2fs -O journal_dev ${EJOURNAL} || return 1 + fi +} + +# MDT file-level backup/restore +mds_backup_restore() { + local devname=$(mdsdevname ${SINGLEMDS//mds/}) + local mntpt=$(facet_mntpt brpt) + local rcmd="do_facet ${SINGLEMDS}" + local metaea=${TMP}/backup_restore.ea + local metadata=${TMP}/backup_restore.tgz + local opts=${MDS_MOUNT_OPTS} + + if ! ${rcmd} test -b ${devname}; then + opts=$(csa_add "$opts" -o loop) + fi + + echo "file-level backup/restore on ${SINGLEMDS}:${devname}" + + # step 1: build mount point + ${rcmd} mkdir -p $mntpt + # step 2: cleanup old backup + ${rcmd} rm -f $metaea $metadata + # step 3: mount dev + ${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 1 + # step 4: backup metaea + echo "backup EA" + ${rcmd} "cd $mntpt && getfattr -R -d -m '.*' -P . > $metaea && cd -" || + return 2 + # step 5: backup metadata + echo "backup data" + ${rcmd} tar zcf $metadata -C $mntpt/ . > /dev/null 2>&1 || return 3 + # step 6: umount + ${rcmd} umount -d $mntpt || return 4 + # step 7: reformat external journal if needed + reformat_external_journal || return 5 + # step 8: reformat dev + echo "reformat new device" + add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS}) --backfstype ldiskfs \ + --reformat $devname > /dev/null || return 6 + # step 9: mount dev + ${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 7 + # step 10: restore metadata + echo "restore data" + ${rcmd} tar zxfp $metadata -C $mntpt > /dev/null 2>&1 || return 8 + # step 11: restore metaea + echo "restore EA" + ${rcmd} "cd $mntpt && setfattr --restore=$metaea && cd - " || return 9 + # step 12: remove recovery logs + echo "remove recovery logs" + ${rcmd} rm -fv $mntpt/OBJECTS/* $mntpt/CATALOGS + # step 13: umount dev + ${rcmd} umount -d $mntpt || return 10 + # step 14: cleanup tmp backup + ${rcmd} rm -f $metaea $metadata +} + +# remove OI files +mds_remove_ois() { + local devname=$(mdsdevname ${SINGLEMDS//mds/}) + local mntpt=$(facet_mntpt brpt) + local rcmd="do_facet ${SINGLEMDS}" + local idx=$1 + local opts=${MDS_MOUNT_OPTS} + + if ! ${rcmd} test -b ${devname}; then + opts=$(csa_add "$opts" -o loop) + fi + + echo "remove OI files: idx=${idx}" + + # step 1: build mount point + ${rcmd} mkdir -p $mntpt + # step 2: mount dev + ${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 1 + if [ -z $idx ]; then + # step 3: remove all OI files + ${rcmd} rm -fv $mntpt/oi.16* + elif [ $idx -lt 2 ]; then + ${rcmd} rm -fv $mntpt/oi.16.${idx} + else + local i + + # others, rm oi.16.[idx, idx * idx, idx ** ...] + for ((i=${idx}; i<64; i=$((i * idx)))); do + ${rcmd} rm -fv $mntpt/oi.16.${i} + done + fi + # step 4: umount + ${rcmd} umount -d $mntpt || return 2 + # OI files will be recreated when mounted as lustre next time. +}