X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=7776af0dd783952ad8268c7a4d42a984fbda043a;hp=27d06068bf48da5a46b7e92c056bc26aaaab97b7;hb=2dc19f20ba9fcc1bcac6ae7ee5169ce10caab882;hpb=af8129c6039ecf3e23b722270ddbb786248a5164;ds=sidebyside diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 27d0606..7776af0 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1,12 +1,11 @@ #!/bin/bash -trap 'print_summary && touch $TF_FAIL && \ - echo "test-framework exiting on error"' ERR +trap 'print_summary && print_stack_trace | tee $TF_FAIL && \ + echo "$TESTSUITE: FAIL: test-framework exiting on error"' ERR set -e #set -x export LANG=en_US -export EJOURNAL=${EJOURNAL:-""} export REFORMAT=${REFORMAT:-""} export WRITECONF=${WRITECONF:-""} export VERBOSE=${VERBOSE:-false} @@ -18,12 +17,17 @@ export QUOTA_AUTO=1 # specify environment variable containing batch job name for server statistics export JOBID_VAR=${JOBID_VAR:-"procname_uid"} # or "existing" or "disable" -# LOAD_LLOOP: LU-409: only load llite_lloop module if kernel < 2.6.32 or -# LOAD_LLOOP is true. LOAD_LLOOP is false by default. -export LOAD_LLOOP=${LOAD_LLOOP:-false} - #export PDSH="pdsh -S -Rssh -w" export MOUNT_CMD=${MOUNT_CMD:-"mount -t lustre"} +export UMOUNT=${UMOUNT:-"umount -d"} + +export LSNAPSHOT_CONF="/etc/ldev.conf" +export LSNAPSHOT_LOG="/var/log/lsnapshot.log" + +# sles12 umount has a issue with -d option +[ -e /etc/SuSE-release ] && grep -w VERSION /etc/SuSE-release | grep -wq 12 && { + export UMOUNT="umount" +} # function used by scripts run on remote nodes LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} @@ -50,16 +54,25 @@ fi [ -z "$MODPROBECONF" -a -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf +sanitize_parameters() { + for i in DIR DIR1 DIR2 MOUNT MOUNT1 MOUNT2 + do + local path=${!i} + if [ -d "$path" ]; then + eval export $i=$(echo $path | sed -r 's/\/+$//g') + fi + done +} assert_DIR () { - local failed="" - [[ $DIR/ = $MOUNT/* ]] || \ - { failed=1 && echo "DIR=$DIR not in $MOUNT. Aborting."; } - [[ $DIR1/ = $MOUNT1/* ]] || \ - { failed=1 && echo "DIR1=$DIR1 not in $MOUNT1. Aborting."; } - [[ $DIR2/ = $MOUNT2/* ]] || \ - { failed=1 && echo "DIR2=$DIR2 not in $MOUNT2. Aborting"; } + local failed="" + [[ $DIR/ = $MOUNT/* ]] || + { failed=1 && echo "DIR=$DIR not in $MOUNT. Aborting."; } + [[ $DIR1/ = $MOUNT1/* ]] || + { failed=1 && echo "DIR1=$DIR1 not in $MOUNT1. Aborting."; } + [[ $DIR2/ = $MOUNT2/* ]] || + { failed=1 && echo "DIR2=$DIR2 not in $MOUNT2. Aborting"; } - [ -n "$failed" ] && exit 99 || true + [ -n "$failed" ] && exit 99 || true } usage() { @@ -71,7 +84,7 @@ usage() { print_summary () { trap 0 - [ "$TESTSUITE" == "lfsck" ] && return 0 + [ -z "$DEFAULT_SUITES"] && return 0 [ -n "$ONLY" ] && echo "WARNING: ONLY is set to $(echo $ONLY)" local details local form="%-13s %-17s %-9s %s %s\n" @@ -129,6 +142,8 @@ init_test_env() { export TEST_FAILED=false export FAIL_ON_SKIP_ENV=${FAIL_ON_SKIP_ENV:-false} export RPC_MODE=${RPC_MODE:-false} + export DO_CLEANUP=${DO_CLEANUP:-true} + export KEEP_ZPOOL=${KEEP_ZPOOL:-false} export MKE2FS=$MKE2FS if [ -z "$MKE2FS" ]; then @@ -193,9 +208,8 @@ init_test_env() { fi fi - export LFSCK_BIN=${LFSCK_BIN:-lfsck} - export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite - export FSCK_MAX_ERR=4 # File system errors left uncorrected + export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after test suite + export FSCK_MAX_ERR=4 # File system errors left uncorrected export ZFS=${ZFS:-zfs} export ZPOOL=${ZPOOL:-zpool} @@ -229,6 +243,7 @@ init_test_env() { # Ubuntu, at least, has a truncate command in /usr/bin # so fully path our truncate command. export TRUNCATE=${TRUNCATE:-$LUSTRE/tests/truncate} + export FSX=${FSX:-$LUSTRE/tests/fsx} export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mpi/mdsrate"} [ ! -f "$MDSRATE" ] && export MDSRATE=$(which mdsrate 2> /dev/null) if ! echo $PATH | grep -q $LUSTRE/tests/racer; then @@ -256,6 +271,8 @@ init_test_env() { fi export LL_DECODE_FILTER_FID=${LL_DECODE_FILTER_FID:-"$LUSTRE/utils/ll_decode_filter_fid"} [ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID="ll_decode_filter_fid" + export LL_DECODE_LINKEA=${LL_DECODE_LINKEA:-"$LUSTRE/utils/ll_decode_linkea"} + [ ! -f "$LL_DECODE_LINKEA" ] && export LL_DECODE_LINKEA="ll_decode_linkea" export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"} [ ! -f "$MKFS" ] && export MKFS="mkfs.lustre" export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"} @@ -267,6 +284,9 @@ init_test_env() { export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate} [ ! -f "$LFS_MIGRATE" ] && export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null) + export LR_READER=${LR_READER:-"$LUSTRE/utils/lr_reader"} + [ ! -f "$LR_READER" ] && export LR_READER=$(which lr_reader 2> /dev/null) + [ -z "$LR_READER" ] && export LR_READER="/usr/sbin/lr_reader" export NAME=${NAME:-local} export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"} [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && \ @@ -277,6 +297,8 @@ init_test_env() { export DIR2 export SAVE_PWD=${SAVE_PWD:-$LUSTRE/tests} export AT_MAX_PATH + export LDEV=${LDEV:-"$LUSTRE/scripts/ldev"} + [ ! -f "$LDEV" ] && export LDEV=$(which ldev 2> /dev/null) if [ "$ACCEPTOR_PORT" ]; then export PORT_OPT="--port $ACCEPTOR_PORT" @@ -317,20 +339,20 @@ init_test_env() { export SHUTDOWN_ATTEMPTS=${SHUTDOWN_ATTEMPTS:-3} export OSD_TRACK_DECLARES_LBUG=${OSD_TRACK_DECLARES_LBUG:-"yes"} - # command line + # command line - while getopts "rvwf:" opt $*; do - case $opt in - f) CONFIG=$OPTARG;; - r) REFORMAT=--reformat;; - v) VERBOSE=true;; - w) WRITECONF=writeconf;; - \?) usage;; - esac - done + while getopts "rvwf:" opt $*; do + case $opt in + f) CONFIG=$OPTARG;; + r) REFORMAT=yes;; + v) VERBOSE=true;; + w) WRITECONF=writeconf;; + \?) usage;; + esac + done - shift $((OPTIND - 1)) - ONLY=${ONLY:-$*} + shift $((OPTIND - 1)) + ONLY=${ONLY:-$*} # print the durations of each test if "true" DDETAILS=${DDETAILS:-false} @@ -338,6 +360,8 @@ init_test_env() { if ! $RPC_MODE; then rm -f $TMP/*active fi + + export TF_FAIL=${TF_FAIL:-$TMP/tf.fail} } check_cpt_number() { @@ -372,30 +396,41 @@ export LINUX_VERSION_CODE=$(version_code ${LINUX_VERSION//\./ }) # # All Lustre versions support "lctl get_param" to report the version of the # code running in the kernel (what our tests are interested in), but it -# doesn't work without modules loaded. If that fails, use "lctl version" -# instead, which is easy to parse and works without the kernel modules, -# but was only added in 2.6.50. If that also fails, fall back to calling -# "lctl lustre_build_version" which prints either (or both) the userspace -# and kernel build versions, but is deprecated and should eventually be -# removed. +# doesn't work without modules loaded. After 2.9.53 and in upstream kernels +# the "version" parameter doesn't include "lustre: " at the beginning. +# If that fails, call "lctl lustre_build_version" which prints either (or both) +# the userspace and kernel build versions, but until 2.8.55 required root +# access to get the Lustre kernel version. If that also fails, fall back to +# using "lctl --version", which is easy to parse and works without the kernel +# modules, but was only added in 2.6.50 and only prints the lctl tool version, +# not the module version, though they are usually the same. # -# output: prints version string to stdout in dotted-decimal format +# Various commands and their output format for different Lustre versions: +# lctl get_param version: 2.9.55 +# lctl get_param version: lustre: 2.8.53 +# lctl get_param version: lustre: 2.6.52 +# kernel: patchless_client +# build: v2_6_92_0-2.6.32-431.el6_lustre.x86_64 +# lctl lustre_build_version: Lustre version: 2.8.53_27_gae67fc01 +# lctl lustre_build_version: error: lustre_build_version: Permission denied +# (as non-root user) lctl version: v2_6_92_0-2.6.32-431.el6.x86_64 +# lctl lustre_build_version: Lustre version: 2.5.3-2.6.32.26-175.fc12.x86_64 +# lctl version: 2.5.3-2.6.32..26-175fc12.x86_64 +# lctl --version: lctl 2.6.50 +# +# output: prints version string to stdout in (up to 4) dotted-decimal values lustre_build_version() { local facet=${1:-client} + local ver + + local ver=$(do_facet $facet "$LCTL get_param -n version 2>/dev/null || + $LCTL lustre_build_version 2>/dev/null || + $LCTL --version 2>/dev/null | cut -d' ' -f2") + local lver=$(egrep -i "lustre: |version: " <<<$ver | head -n 1) + [ -n "$lver" ] && ver="$lver" - # lustre: 2.6.52 - # kernel: patchless_client - # build: v2_6_92_0-gadb3ee4-2.6.32-431.29.2.el6_lustre.x86_64 - local VER=$(do_facet $facet $LCTL get_param -n version 2> /dev/null | - awk '/lustre: / { print $2 }') - # lctl 2.6.50 - [ -z "$VER" ] && VER=$(do_facet $facet $LCTL --version 2>/dev/null | - awk '{ print $2 }') - # Lustre version: 2.5.3-gfcfd782-CHANGED-2.6.32.26-175.fc12.x86_64 - # lctl version: 2.5.3-gfcfd782-CHANGED-2.6.32.26-175.fc12.x86_64 - [ -z "$VER" ] && VER=$(do_facet $facet $LCTL lustre_build_version | - awk '/version:/ { print $3; exit; }') - sed -e 's/^v//' -e 's/-.*//' -e 's/_/./g' <<<$VER + sed -e 's/.*: //' -e 's/^v//' -e 's/-.*//' -e 's/_/./g' <<<$ver | + cut -d. -f1-4 } # Report the Lustre numeric build version code for the supplied facet. @@ -407,6 +442,33 @@ module_loaded () { /sbin/lsmod | grep -q "^\<$1\>" } +PRLFS=false +lustre_insmod() { + local module=$1 + shift + local args="$@" + local msg + local rc=0 + + if ! $PRLFS; then + msg="$(insmod $module $args 2>&1)" && return 0 || rc=$? + fi + + # parallels can't load modules directly from prlfs, use /tmp instead + if $PRLFS || [[ "$(stat -f -c%t $module)" == "7c7c6673" ]]; then + local target="$(mktemp)" + + cp "$module" "$target" + insmod $target $args + rc=$? + [[ $rc == 0 ]] && PRLFS=true + rm -f $target + else + echo "$msg" + fi + return $rc +} + # Load a module on the system where this is running. # # usage: load_module module_name [module arguments for insmod/modprobe] @@ -459,10 +521,10 @@ load_module() { # we're passing options on the command-line. if [[ "$BASE" == "lnet_selftest" ]] && [[ -f ${LUSTRE}/../lnet/selftest/${module}${EXT} ]]; then - insmod ${LUSTRE}/../lnet/selftest/${module}${EXT} + lustre_insmod ${LUSTRE}/../lnet/selftest/${module}${EXT} elif [[ -f ${LUSTRE}/${module}${EXT} ]]; then [[ "$BASE" != "ptlrpc_gss" ]] || modprobe sunrpc - insmod ${LUSTRE}/${module}${EXT} "$@" + lustre_insmod ${LUSTRE}/${module}${EXT} "$@" else # must be testing a "make install" or "rpm" installation # note failed to load ptlrpc_gss is considered not fatal @@ -475,20 +537,6 @@ load_module() { fi } -llite_lloop_enabled() { - local n1=$(uname -r | cut -d. -f1) - local n2=$(uname -r | cut -d. -f2) - local n3=$(uname -r | cut -d- -f1 | cut -d. -f3) - - # load the llite_lloop module for < 2.6.32 kernels - if [[ $n1 -lt 2 ]] || [[ $n1 -eq 2 && $n2 -lt 6 ]] || \ - [[ $n1 -eq 2 && $n2 -eq 6 && $n3 -lt 32 ]] || \ - $LOAD_LLOOP; then - return 0 - fi - return 1 -} - load_modules_local() { if [ -n "$MODPROBE" ]; then # use modprobe @@ -527,18 +575,24 @@ load_modules_local() { fi load_module ../libcfs/libcfs/libcfs - - [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG" - [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }" - load_module ../lnet/lnet/lnet - case $NETTYPE in - o2ib) - LNETLND="o2iblnd/ko2iblnd" - ;; - *) - ;; - esac - LNETLND=${LNETLND:-"socklnd/ksocklnd"} + # Prevent local MODOPTS_LIBCFS being passed as part of environment + # variable to remote nodes + unset MODOPTS_LIBCFS + + set_default_debug + load_module ../lnet/lnet/lnet + + LNDPATH=${LNDPATH:-"../lnet/klnds"} + if [ -z "$LNETLND" ]; then + case $NETTYPE in + o2ib*) LNETLND="o2iblnd/ko2iblnd" ;; + tcp*) LNETLND="socklnd/ksocklnd" ;; + *) local lnd="${NETTYPE%%[0-9]}lnd" + [ -f "$LNDPATH/$lnd/k$lnd.ko" ] && + LNETLND="$lnd/k$lnd" || + LNETLND="socklnd/ksocklnd" + esac + fi load_module ../lnet/klnds/$LNETLND load_module obdclass/obdclass load_module ptlrpc/ptlrpc @@ -551,20 +605,24 @@ load_modules_local() { load_module lov/lov load_module mgc/mgc load_module obdecho/obdecho - if ! client_only; then - SYMLIST=/proc/kallsyms - grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; } - grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; } - grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; } + if ! client_only; then + SYMLIST=/proc/kallsyms + grep -q crc16 $SYMLIST || + { modprobe crc16 2>/dev/null || true; } + grep -q -w jbd2 $SYMLIST || + { modprobe jbd2 2>/dev/null || true; } load_module lfsck/lfsck - [ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS + [ "$LQUOTA" != "no" ] && + load_module quota/lquota $LQUOTAOPTS if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then - modprobe zfs + lsmod | grep zfs >&/dev/null || modprobe zfs load_module osd-zfs/osd_zfs fi if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then grep -q exportfs_decode_fh $SYMLIST || { modprobe exportfs 2> /dev/null || true; } + grep -q -w mbcache $SYMLIST || + { modprobe mbcache 2>/dev/null || true; } load_module ../ldiskfs/ldiskfs load_module osd-ldiskfs/osd_ldiskfs fi @@ -576,10 +634,9 @@ load_modules_local() { load_module osp/osp load_module ofd/ofd load_module osp/osp - fi + fi load_module llite/lustre - llite_lloop_enabled && load_module llite/llite_lloop [ -d /r ] && OGDB=${OGDB:-"/r/tmp"} OGDB=${OGDB:-$TMP} rm -f $OGDB/ogdb-$HOSTNAME @@ -588,7 +645,7 @@ load_modules_local() { # 'mount' doesn't look in $PATH, just sbin local mount_lustre=$LUSTRE/utils/mount.lustre if [ -f $mount_lustre ]; then - local sbin_mount=/sbin/mount.lustre + local sbin_mount=$(readlink -f /sbin)/mount.lustre if grep -qw "$sbin_mount" /proc/mounts; then cmp -s $mount_lustre $sbin_mount || umount $sbin_mount fi @@ -652,7 +709,7 @@ unload_modules() { fi fi - local sbin_mount=/sbin/mount.lustre + local sbin_mount=$(readlink -f /sbin)/mount.lustre if grep -qe "$sbin_mount " /proc/mounts; then umount $sbin_mount || true [ -s $sbin_mount ] && ! grep -q "STUB MARK" $sbin_mount || @@ -677,6 +734,18 @@ fs_log_size() { echo -n $size } +fs_inode_ksize() { + local facet=${1:-$SINGLEMDS} + local fstype=$(facet_fstype $facet) + local size=0 + case $fstype in + ldiskfs) size=4;; # ~4KB per inode + zfs) size=11;; # 10 to 11KB per inode + esac + + echo -n $size +} + check_gss_daemon_nodes() { local list=$1 dname=$2 @@ -808,7 +877,7 @@ facet_type() { facet_number() { local facet=$1 - if [ $facet == mgs ]; then + if [ $facet == mgs ] || [ $facet == client ]; then return 1 fi @@ -955,6 +1024,20 @@ facet_vdevice() { echo -n $device } +running_in_vm() { + local virt=$(virt-what 2> /dev/null) + + [ $? -eq 0 ] && [ -n "$virt" ] && { echo $virt; return; } + + virt=$(dmidecode -s system-product-name | awk '{print $1}') + + case $virt in + VMware|KVM|VirtualBox|Parallels) + echo $virt | tr '[A-Z]' '[a-z]' ;; + *) ;; + esac +} + # # Re-read the partition table on failover partner host. # After a ZFS storage pool is created on a shared device, the partition table @@ -993,6 +1076,17 @@ zpool_name() { } # +# +# Get ZFS local fsname. +# +zfs_local_fsname() { + local facet=$1 + local lfsname=$(basename $(facet_device $facet)) + + echo -n $lfsname +} + +# # Create ZFS storage pool. # create_zpool() { @@ -1002,7 +1096,8 @@ create_zpool() { shift 3 local opts=${@:-"-o cachefile=none"} - do_facet $facet "$ZPOOL list -H $poolname >/dev/null 2>&1 || + do_facet $facet "lsmod | grep zfs >&/dev/null || modprobe zfs; + $ZPOOL list -H $poolname >/dev/null 2>&1 || $ZPOOL create -f $opts $poolname $vdev" } @@ -1068,12 +1163,27 @@ import_zpool() { if [[ -n "$poolname" ]]; then opts+=" -d $(dirname $(facet_vdevice $facet))" - do_facet $facet "$ZPOOL list -H $poolname >/dev/null 2>&1 || + do_facet $facet "lsmod | grep zfs >&/dev/null || modprobe zfs; + $ZPOOL list -H $poolname >/dev/null 2>&1 || $ZPOOL import -f $opts $poolname" fi } # +# Reimport ZFS storage pool with new name +# +reimport_zpool() { + local facet=$1 + local newpool=$2 + local opts="-o cachefile=none" + local poolname=$(zpool_name $facet) + + opts+=" -d $(dirname $(facet_vdevice $facet))" + do_facet $facet "$ZPOOL export $poolname; + $ZPOOL import $opts $poolname $newpool" +} + +# # Set the "cachefile=none" property on ZFS storage pool so that the pool # is not automatically imported on system startup. # @@ -1107,7 +1217,7 @@ get_osd_param() { local name=$3 do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \ - osd-*.$device.$name 2>&1" | grep -v 'Found no match' + osd-*.$device.$name 2>&1" | grep -v 'error:' } set_osd_param() { @@ -1117,7 +1227,7 @@ set_osd_param() { local value=$4 do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \ - osd-*.$device.$name=$value 2>&1" | grep -v 'Found no match' + osd-*.$device.$name=$value 2>&1" | grep -v 'error:' } set_debug_size () { @@ -1126,7 +1236,7 @@ set_debug_size () { if [ -f /sys/devices/system/cpu/possible ]; then local cpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/possible)+1)) else - local cpus=$(getconf _NPROCESSORS_CONF) + local cpus=$(getconf _NPROCESSORS_CONF 2>/dev/null) fi # bug 19944, adjust size to be -gt num_possible_cpus() @@ -1168,6 +1278,15 @@ set_default_debug_facet () { set_default_debug_nodes $node } +set_hostid () { + local hostid=${1:-$(hostid)} + + if [ ! -s /etc/hostid ]; then + printf $(echo -n $hostid | + sed 's/\(..\)\(..\)\(..\)\(..\)/\\x\4\\x\3\\x\2\\x\1/') >/etc/hostid + fi +} + # Facet functions mount_facets () { local facets=${1:-$(get_facets)} @@ -1214,6 +1333,10 @@ mount_facet() { local opt=${facet}_opt local mntpt=$(facet_mntpt $facet) local opts="${!opt} $@" + local fstype=$(facet_fstype $facet) + local devicelabel + + module_loaded lustre || load_modules if [ $(facet_fstype $facet) == ldiskfs ] && ! do_facet $facet test -b ${!dev}; then @@ -1225,6 +1348,16 @@ mount_facet() { import_zpool $facet || return ${PIPESTATUS[0]} fi + case $fstype in + ldiskfs) + devicelabel=$(do_facet ${facet} "$E2LABEL ${!dev}");; + zfs) + devicelabel=$(do_facet ${facet} "$ZFS get -H -o value \ + lustre:svname ${!dev}");; + *) + error "unknown fstype!";; + esac + echo "Starting ${facet}: $opts ${!dev} $mntpt" # for testing LU-482 error handling in mount_facets() and test_0a() if [ -f $TMP/test-lu482-trigger ]; then @@ -1234,40 +1367,74 @@ mount_facet() { ${!dev} $mntpt" RC=${PIPESTATUS[0]} fi + if [ $RC -ne 0 ]; then echo "Start of ${!dev} on ${facet} failed ${RC}" - else - set_default_debug_facet $facet + return $RC + fi + + health=$(do_facet ${facet} "$LCTL get_param -n health_check") + if [[ "$health" != "healthy" ]]; then + error "$facet is in a unhealthy state" + fi + + set_default_debug_facet $facet if [[ $facet == mds* ]]; then do_facet $facet \ - lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 \ - 2>/dev/null + lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 2>/dev/null fi - label=$(devicelabel ${facet} ${!dev}) - [ -z "$label" ] && echo no label for ${!dev} && exit 1 - eval export ${facet}_svc=${label} - echo Started ${label} - fi - return $RC + if [[ $opts =~ .*nosvc.* ]]; then + echo "Start ${!dev} without service" + else + + case $fstype in + ldiskfs) + wait_update_facet ${facet} "$E2LABEL ${!dev} \ + 2>/dev/null | grep -E ':[a-zA-Z]{3}[0-9]{4}'" \ + "" || error "${!dev} failed to initialize!";; + zfs) + wait_update_facet ${facet} "$ZFS get -H -o value \ + lustre:svname ${!dev} 2>/dev/null | \ + grep -E ':[a-zA-Z]{3}[0-9]{4}'" "" || + error "${!dev} failed to initialize!";; + + *) + error "unknown fstype!";; + esac + fi + + # commit the device label change to disk + if [[ $devicelabel =~ (:[a-zA-Z]{3}[0-9]{4}) ]]; then + echo "Commit the device label on ${!dev}" + do_facet $facet "sync; sleep 1; sync" + fi + + + label=$(devicelabel ${facet} ${!dev}) + [ -z "$label" ] && echo no label for ${!dev} && exit 1 + eval export ${facet}_svc=${label} + echo Started ${label} + + return $RC } # start facet device options start() { - local facet=$1 - shift - local device=$1 - shift - eval export ${facet}_dev=${device} - eval export ${facet}_opt=\"$@\" + local facet=$1 + shift + local device=$1 + shift + eval export ${facet}_dev=${device} + eval export ${facet}_opt=\"$@\" - local varname=${facet}failover_dev - if [ -n "${!varname}" ] ; then - eval export ${facet}failover_dev=${!varname} - else - eval export ${facet}failover_dev=$device - fi + local varname=${facet}failover_dev + if [ -n "${!varname}" ] ; then + eval export ${facet}failover_dev=${!varname} + else + eval export ${facet}failover_dev=$device + fi local mntpt=$(facet_mntpt $facet) do_facet ${facet} mkdir -p $mntpt @@ -1295,7 +1462,7 @@ stop() { running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts") || true if [ ${running} -ne 0 ]; then echo "Stopping $mntpt (opts:$@) on $HOST" - do_facet ${facet} umount -d $@ $mntpt + do_facet ${facet} $UMOUNT $@ $mntpt fi # umount should block, but we should wait for unrelated obd's @@ -1304,7 +1471,7 @@ stop() { if [[ $(facet_fstype $facet) == zfs ]]; then # export ZFS storage pool - export_zpool $facet + [ "$KEEP_ZPOOL" = "true" ] || export_zpool $facet fi } @@ -1348,70 +1515,6 @@ quota_type() { return $rc } -# XXX This function is kept for interoperability with old server (< 2.3.50), -# it should be removed whenever we drop the interoperability for such -# server. -restore_quota_old() { - local mntpt=${1:-$MOUNT} - local quota_type=$(quota_type $FSNAME | grep MDT | cut -d "=" -f2) - if [ ! "$old_QUOTA_TYPE" ] || - [ "$quota_type" = "$old_QUOTA_TYPE" ]; then - return - fi - quota_save_version $old_QUOTA_TYPE -} - -# XXX This function is kept for interoperability with old server (< 2.3.50), -# it should be removed whenever we drop the interoperability for such -# server. -setup_quota_old(){ - local mntpt=$1 - - # no quota enforcement for now and accounting works out of the box - return - - # We need save the original quota_type params, and restore them after testing - - # Suppose that quota type the same on mds and ost - local quota_type=$(quota_type | grep MDT | cut -d "=" -f2) - [ ${PIPESTATUS[0]} -eq 0 ] || error "quota_type failed!" - echo "[HOST:$HOSTNAME] [old_quota_type:$quota_type] [new_quota_type:$QUOTA_TYPE]" - if [ "$quota_type" != "$QUOTA_TYPE" ]; then - export old_QUOTA_TYPE=$quota_type - quota_save_version $QUOTA_TYPE - else - qtype=$(tr -c -d "ug" <<< $QUOTA_TYPE) - $LFS quotacheck -$qtype $mntpt || error "quotacheck has failed for $type" - fi - - local quota_usrs=$QUOTA_USERS - - # get_filesystem_size - local disksz=$(lfs_df $mntpt | grep "summary" | awk '{print $2}') - local blk_soft=$((disksz + 1024)) - local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over - - local Inodes=$(lfs_df -i $mntpt | grep "summary" | awk '{print $2}') - local i_soft=$Inodes - local i_hard=$((i_soft + i_soft / 20)) - - echo "Total disk size: $disksz block-softlimit: $blk_soft block-hardlimit: - $blk_hard inode-softlimit: $i_soft inode-hardlimit: $i_hard" - - local cmd - for usr in $quota_usrs; do - echo "Setting up quota on $HOSTNAME:$mntpt for $usr..." - for type in u g; do - cmd="$LFS setquota -$type $usr -b $blk_soft -B $blk_hard -i $i_soft -I $i_hard $mntpt" - echo "+ $cmd" - eval $cmd || error "$cmd FAILED!" - done - # display the quota status - echo "Quota settings for $usr : " - $LFS quota -v -u $usr $mntpt || true - done -} - # get mdt quota type mdt_quota_type() { local varsvc=${SINGLEMDS}_svc @@ -1429,11 +1532,6 @@ ost_quota_type() { # restore old quota type settings restore_quota() { - if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then - restore_quota_old - return - fi - if [ "$old_MDT_QUOTA_TYPE" ]; then do_facet mgs $LCTL conf_param \ $FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE @@ -1468,12 +1566,19 @@ mdt_free_inodes() { echo $free_inodes } -setup_quota(){ - if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then - setup_quota_old $1 - return - fi +# +# Get the OST device status from 'lfs df' with a given OST index. +# +ost_dev_status() { + local ost_idx=$1 + local mnt_pnt=${2:-$MOUNT} + local ost_uuid + + ost_uuid=$(ostuuid_from_index $ost_idx $mnt_pnt) + lfs_df $mnt_pnt | awk '/'$ost_uuid'/ { print $7 }' +} +setup_quota(){ local mntpt=$1 # save old quota type & set new quota type @@ -1522,32 +1627,44 @@ setup_quota(){ } zconf_mount() { - local client=$1 - local mnt=$2 - local opts=${3:-$MOUNT_OPTS} - opts=${opts:+-o $opts} - local flags=${4:-$MOUNT_FLAGS} - - local device=$MGSNID:/$FSNAME - if [ -z "$mnt" -o -z "$FSNAME" ]; then - echo Bad zconf mount command: opt=$flags $opts dev=$device mnt=$mnt - exit 1 - fi - - echo "Starting client: $client: $flags $opts $device $mnt" - do_node $client mkdir -p $mnt - do_node $client $MOUNT_CMD $flags $opts $device $mnt || return 1 - - set_default_debug_nodes $client + local client=$1 + local mnt=$2 + local opts=${3:-$MOUNT_OPTS} + opts=${opts:+-o $opts} + local flags=${4:-$MOUNT_FLAGS} + + local device=$MGSNID:/$FSNAME$FILESET + if [ -z "$mnt" -o -z "$FSNAME" ]; then + echo "Bad mount command: opt=$flags $opts dev=$device " \ + "mnt=$mnt" + exit 1 + fi + + echo "Starting client: $client: $flags $opts $device $mnt" + do_node $client mkdir -p $mnt + if [ -n "$FILESET" -a -z "$SKIP_FILESET" ];then + do_node $client $MOUNT_CMD $flags $opts $MGSNID:/$FSNAME \ + $mnt || return 1 + #disable FILESET if not supported + do_nodes $client lctl get_param -n \ + mdc.$FSNAME-MDT0000*.import | grep -q subtree || + device=$MGSNID:/$FSNAME + do_node $client mkdir -p $mnt/$FILESET + do_node $client "! grep -q $mnt' ' /proc/mounts || + umount $mnt" + fi + do_node $client $MOUNT_CMD $flags $opts $device $mnt || return 1 + + set_default_debug_nodes $client - return 0 + return 0 } zconf_umount() { local client=$1 local mnt=$2 local force - local busy + local busy local need_kill [ "$3" ] && force=-f @@ -1588,13 +1705,13 @@ if [ \\\$running -ne \\\$mpts ]; then echo \\\$(hostname) env are INSANE!; exit 1; fi" - [ $? -eq 0 ] || rc=1 + [ $? -eq 0 ] || rc=1 done return $rc } sanity_mount_check_servers () { - [ "$CLIENTONLY" ] && + [ -n "$CLIENTONLY" ] && { echo "CLIENTONLY mode, skip mount_check_servers"; return 0; } || true echo Checking servers environments @@ -1630,36 +1747,60 @@ sanity_mount_check () { # mount clients if not mouted zconf_mount_clients() { - local clients=$1 - local mnt=$2 - local opts=${3:-$MOUNT_OPTS} - opts=${opts:+-o $opts} - local flags=${4:-$MOUNT_FLAGS} - - local device=$MGSNID:/$FSNAME - if [ -z "$mnt" -o -z "$FSNAME" ]; then - echo Bad zconf mount command: opt=$flags $opts dev=$device mnt=$mnt - exit 1 - fi - - echo "Starting client $clients: $flags $opts $device $mnt" - - do_nodes $clients " -running=\\\$(mount | grep -c $mnt' '); -rc=0; -if [ \\\$running -eq 0 ] ; then - mkdir -p $mnt; - $MOUNT_CMD $flags $opts $device $mnt; - rc=\\\$?; -fi; -exit \\\$rc" || return ${PIPESTATUS[0]} - - echo "Started clients $clients: " - do_nodes $clients "mount | grep $mnt' '" - - set_default_debug_nodes $clients + local clients=$1 + local mnt=$2 + local opts=${3:-$MOUNT_OPTS} + opts=${opts:+-o $opts} + local flags=${4:-$MOUNT_FLAGS} + + local device=$MGSNID:/$FSNAME$FILESET + if [ -z "$mnt" -o -z "$FSNAME" ]; then + echo "Bad conf mount command: opt=$flags $opts dev=$device " \ + "mnt=$mnt" + exit 1 + fi + + echo "Starting client $clients: $flags $opts $device $mnt" + if [ -n "$FILESET" -a ! -n "$SKIP_FILESET" ]; then + do_nodes $clients "! grep -q $mnt' ' /proc/mounts || + umount $mnt" + do_nodes $clients $MOUNT_CMD $flags $opts $MGSNID:/$FSNAME \ + $mnt || return 1 + #disable FILESET if not supported + do_nodes $clients lctl get_param -n \ + mdc.$FSNAME-MDT0000*.import | grep -q subtree || + device=$MGSNID:/$FSNAME + do_nodes $clients mkdir -p $mnt/$FILESET + do_nodes $clients "! grep -q $mnt' ' /proc/mounts || + umount $mnt" + fi + + do_nodes $clients " + running=\\\$(mount | grep -c $mnt' '); + rc=0; + if [ \\\$running -eq 0 ] ; then + mkdir -p $mnt; + $MOUNT_CMD $flags $opts $device $mnt; + rc=\\\$?; + else + lustre_mnt_count=\\\$(mount | grep $mnt' ' | \ + grep 'type lustre' | wc -l); + if [ \\\$running -ne \\\$lustre_mnt_count ] ; then + echo zconf_mount_clients FAILED: \ + mount count \\\$running, not matching \ + with mount count of 'type lustre' \ + \\\$lustre_mnt_count; + rc=1; + fi; + fi; + exit \\\$rc" || return ${PIPESTATUS[0]} + + echo "Started clients $clients: " + do_nodes $clients "mount | grep $mnt' '" + + set_default_debug_nodes $clients - return 0 + return 0 } zconf_umount_clients() { @@ -1702,7 +1843,7 @@ shutdown_node_hard () { wait_for_function --quiet "! ping -w 3 -c 1 $host" 5 1 && return 0 echo "waiting for $host to fail attempts=$attempts" [ $i -lt $attempts ] || \ - { echo "$host still pingable after power down! attempts=$attempts" && return 1; } + { echo "$host still pingable after power down! attempts=$attempts" && return 1; } done } @@ -1833,29 +1974,31 @@ node_var_name() { } start_client_load() { - local client=$1 - local load=$2 - local var=$(node_var_name $client)_load - eval export ${var}=$load - - do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \ -BREAK_ON_ERROR=$BREAK_ON_ERROR \ -END_RUN_FILE=$END_RUN_FILE \ -LOAD_PID_FILE=$LOAD_PID_FILE \ -TESTLOG_PREFIX=$TESTLOG_PREFIX \ -TESTNAME=$TESTNAME \ -DBENCH_LIB=$DBENCH_LIB \ -DBENCH_SRC=$DBENCH_SRC \ -CLIENT_COUNT=$((CLIENTCOUNT - 1)) \ -LFS=$LFS \ -run_${load}.sh" & - local ppid=$! - log "Started client load: ${load} on $client" - - # get the children process IDs - local pids=$(ps --ppid $ppid -o pid= | xargs) - CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $ppid $pids" - return 0 + local client=$1 + local load=$2 + local var=$(node_var_name $client)_load + eval export ${var}=$load + + do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \ + BREAK_ON_ERROR=$BREAK_ON_ERROR \ + END_RUN_FILE=$END_RUN_FILE \ + LOAD_PID_FILE=$LOAD_PID_FILE \ + TESTLOG_PREFIX=$TESTLOG_PREFIX \ + TESTNAME=$TESTNAME \ + DBENCH_LIB=$DBENCH_LIB \ + DBENCH_SRC=$DBENCH_SRC \ + CLIENT_COUNT=$((CLIENTCOUNT - 1)) \ + LFS=$LFS \ + LCTL=$LCTL \ + FSNAME=$FSNAME \ + run_${load}.sh" & + local ppid=$! + log "Started client load: ${load} on $client" + + # get the children process IDs + local pids=$(ps --ppid $ppid -o pid= | xargs) + CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $ppid $pids" + return 0 } start_client_loads () { @@ -1873,55 +2016,59 @@ start_client_loads () { # only for remote client check_client_load () { - local client=$1 - local var=$(node_var_name $client)_load - local TESTLOAD=run_${!var}.sh - - ps auxww | grep -v grep | grep $client | grep -q "$TESTLOAD" || return 1 - - # bug 18914: try to connect several times not only when - # check ps, but while check_catastrophe also - local tries=3 - local RC=254 - while [ $RC = 254 -a $tries -gt 0 ]; do - let tries=$tries-1 - # assume success - RC=0 - if ! check_catastrophe $client; then - RC=${PIPESTATUS[0]} - if [ $RC -eq 254 ]; then - # FIXME: not sure how long we shuold sleep here - sleep 10 - continue - fi - echo "check catastrophe failed: RC=$RC " - return $RC - fi - done - # We can continue try to connect if RC=254 - # Just print the warning about this - if [ $RC = 254 ]; then - echo "got a return status of $RC from do_node while checking catastrophe on $client" - fi - - # see if the load is still on the client - tries=3 - RC=254 - while [ $RC = 254 -a $tries -gt 0 ]; do - let tries=$tries-1 - # assume success - RC=0 - if ! do_node $client "ps auxwww | grep -v grep | grep -q $TESTLOAD"; then - RC=${PIPESTATUS[0]} - sleep 30 - fi - done - if [ $RC = 254 ]; then - echo "got a return status of $RC from do_node while checking (catastrophe and 'ps') the client load on $client" - # see if we can diagnose a bit why this is - fi + local client=$1 + local var=$(node_var_name $client)_load + local testload=run_${!var}.sh + + ps auxww | grep -v grep | grep $client | grep -q $testload || return 1 + + # bug 18914: try to connect several times not only when + # check ps, but while check_node_health also + + local tries=3 + local RC=254 + while [ $RC = 254 -a $tries -gt 0 ]; do + let tries=$tries-1 + # assume success + RC=0 + if ! check_node_health $client; then + RC=${PIPESTATUS[0]} + if [ $RC -eq 254 ]; then + # FIXME: not sure how long we shuold sleep here + sleep 10 + continue + fi + echo "check node health failed: RC=$RC " + return $RC + fi + done + # We can continue try to connect if RC=254 + # Just print the warning about this + if [ $RC = 254 ]; then + echo "got a return status of $RC from do_node while checking " \ + "node health on $client" + fi + + # see if the load is still on the client + tries=3 + RC=254 + while [ $RC = 254 -a $tries -gt 0 ]; do + let tries=$tries-1 + # assume success + RC=0 + if ! do_node $client \ + "ps auxwww | grep -v grep | grep -q $testload"; then + RC=${PIPESTATUS[0]} + sleep 30 + fi + done + if [ $RC = 254 ]; then + echo "got a return status of $RC from do_node while checking " \ + "(node health and 'ps') the client load on $client" + # see if we can diagnose a bit why this is + fi - return $RC + return $RC } check_client_loads () { local clients=${1//,/ } @@ -2101,19 +2248,31 @@ wait_update () { } wait_update_facet() { + local verbose= + [ "$1" = "--verbose" ] && verbose="$1" && shift + local facet=$1 shift - wait_update $(facet_active_host $facet) "$@" + wait_update $verbose $(facet_active_host $facet) "$@" } sync_all_data() { do_nodes $(comma_list $(mdts_nodes)) \ - "lctl set_param -n osd*.*MDT*.force_sync 1" + "lctl set_param -n osd*.*MDT*.force_sync=1" do_nodes $(comma_list $(osts_nodes)) \ - "lctl set_param -n osd*.*OS*.force_sync 1" 2>&1 | + "lctl set_param -n osd*.*OS*.force_sync=1" 2>&1 | grep -v 'Found no match' } +wait_zfs_commit() { + # the occupied disk space will be released + # only after DMUs are committed + if [[ $(facet_fstype $1) == zfs ]]; then + echo "sleep $2 for ZFS OSD" + sleep $2 + fi +} + wait_delete_completed_mds() { local MAX_WAIT=${1:-20} # for ZFS, waiting more time for DMUs to be committed @@ -2134,6 +2293,7 @@ wait_delete_completed_mds() { mds2sync="$mds2sync $node" done if [ -z "$mds2sync" ]; then + wait_zfs_commit $SINGLEMDS $ZFS_WAIT return fi mds2sync=$(comma_list $mds2sync) @@ -2151,16 +2311,7 @@ wait_delete_completed_mds() { "$LCTL get_param -n osc.*MDT*.sync_*" | calc_sum) #echo "$node: $changes changes on all" if [[ $changes -eq 0 ]]; then - etime=$(date +%s) - #echo "delete took $((etime - stime)) seconds" - - # the occupied disk space will be released - # only after DMUs are committed - if [[ $(facet_fstype $SINGLEMDS) == zfs ]]; then - echo "sleep $ZFS_WAIT for ZFS OSD" - sleep $ZFS_WAIT - fi - + wait_zfs_commit $SINGLEMDS $ZFS_WAIT return fi sleep 1 @@ -2282,6 +2433,19 @@ wait_mds_ost_sync () { return 1 } +# Wait OSTs to be active on both client and MDT side. +wait_osts_up() { + local cmd="$LCTL get_param -n lov.$FSNAME-clilov-*.target_obd | + awk 'BEGIN {c = 0} /ACTIVE/{c += 1} END {printf \\\"%d\\\", c}'" + wait_update $HOSTNAME "eval $cmd" $OSTCOUNT || + error "wait_update OSTs up on client failed" + + cmd="$LCTL get_param osp.$FSNAME-OST*-MDT0000.prealloc_last_id | + awk '/=[1-9][0-9]/ { c += 1 } END { printf \\\"%d\\\", c }'" + wait_update_facet $SINGLEMDS "eval $cmd" $OSTCOUNT || + error "wait_update OSTs up on MDT0000 failed" +} + wait_destroy_complete () { echo "Waiting for local destroys to complete" # MAX value shouldn't be big as this mean server responsiveness @@ -2367,25 +2531,27 @@ wait_remote_prog () { return $rc } +lfs_df_check() { + local clients=${1:-$CLIENTS} + + if [ -z "$clients" ]; then + $LFS df $MOUNT + else + $PDSH $clients "$LFS df $MOUNT" > /dev/null + fi +} + + clients_up() { - # not every config has many clients - sleep 1 - if [ ! -z "$CLIENTS" ]; then - $PDSH $CLIENTS "stat -f $MOUNT" > /dev/null - else - stat -f $MOUNT > /dev/null - fi + # not every config has many clients + sleep 1 + lfs_df_check } client_up() { - local client=$1 - # usually checked on particular client or locally - sleep 1 - if [ ! -z "$client" ]; then - $PDSH $client "stat -f $MOUNT" > /dev/null - else - stat -f $MOUNT > /dev/null - fi + # usually checked on particular client or locally + sleep 1 + lfs_df_check $1 } client_evicted() { @@ -2393,16 +2559,17 @@ client_evicted() { } client_reconnect_try() { - uname -n >> $MOUNT/recon - if [ -z "$CLIENTS" ]; then - df $MOUNT; uname -n >> $MOUNT/recon - else - do_nodes $CLIENTS "df $MOUNT; uname -n >> $MOUNT/recon" > /dev/null - fi - echo Connected clients: - cat $MOUNT/recon - ls -l $MOUNT/recon > /dev/null - rm $MOUNT/recon + local f=$MOUNT/recon + + uname -n >> $f + if [ -z "$CLIENTS" ]; then + $LFS df $MOUNT; uname -n >> $f + else + do_nodes $CLIENTS "$LFS df $MOUNT; uname -n >> $f" > /dev/null + fi + echo "Connected clients: $(cat $f)" + ls -l $f > /dev/null + rm $f } client_reconnect() { @@ -2426,6 +2593,13 @@ affected_facets () { } facet_failover() { + local E2FSCK_ON_MDT0=false + if [ "$1" == "--fsck" ]; then + shift + [ $(facet_fstype $SINGLEMDS) == ldiskfs ] && + E2FSCK_ON_MDT0=true + fi + local facets=$1 local sleep_time=$2 local -a affecteds @@ -2459,6 +2633,9 @@ facet_failover() { shutdown_facet $facet done + $E2FSCK_ON_MDT0 && (run_e2fsck $(facet_active_host $SINGLEMDS) \ + $(mdsdevname 1) "-n" || error "Running e2fsck") + for ((index=0; index<$total; index++)); do facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1) echo reboot facets: ${affecteds[index]} @@ -2487,7 +2664,7 @@ obd_name() { replay_barrier() { local facet=$1 do_facet $facet "sync; sync; sync" - df $MOUNT + $LFS df $MOUNT # make sure there will be no seq change local clients=${CLIENTS:-$HOSTNAME} @@ -2572,7 +2749,7 @@ fail() { facet_failover $* || error "failover: $?" wait_clients_import_state "$clients" "$facets" FULL - clients_up || error "post-failover df: $?" + clients_up || error "post-failover stat: $?" } fail_nodf() { @@ -2586,25 +2763,15 @@ fail_abort() { change_active $facet wait_for_facet $facet mount_facet $facet -o abort_recovery - clients_up || echo "first df failed: $?" - clients_up || error "post-failover df: $?" -} - -do_lmc() { - echo There is no lmc. This is mountconf, baby. - exit 1 + clients_up || echo "first stat failed: $?" + clients_up || error "post-failover stat: $?" } host_nids_address() { - local nodes=$1 - local kind=$2 + local nodes=$1 + local net=${2:-"."} - if [ -n "$kind" ]; then - nids=$(do_nodes $nodes "$LCTL list_nids | grep $kind | cut -f 1 -d '@'") - else - nids=$(do_nodes $nodes "$LCTL list_nids all | cut -f 1 -d '@'") - fi - echo $nids + do_nodes $nodes "$LCTL list_nids | grep $net | cut -f 1 -d @" } h2name_or_ip() { @@ -2613,40 +2780,34 @@ h2name_or_ip() { fi } -h2ptl() { - if [ "$1" = "'*'" ]; then echo \'*\'; else - ID=`xtprocadmin -n $1 2>/dev/null | egrep -v 'NID' | \ - awk '{print $1}'` - if [ -z "$ID" ]; then - echo "Could not get a ptl id for $1..." - exit 1 - fi - echo $ID"@ptl" +h2nettype() { + if [[ -n "$NETTYPE" ]]; then + h2name_or_ip "$1" "$NETTYPE" + else + h2name_or_ip "$1" "$2" fi } -declare -fx h2ptl +declare -fx h2nettype +# Wrapper function to print the deprecation warning h2tcp() { - h2name_or_ip "$1" "tcp" -} -declare -fx h2tcp - -h2elan() { - if [ "$1" = "'*'" ]; then echo \'*\'; else - if type __h2elan >/dev/null 2>&1; then - ID=$(__h2elan $1) - else - ID=`echo $1 | sed 's/[^0-9]*//g'` - fi - echo $ID"@elan" + echo "h2tcp: deprecated, use h2nettype instead" 1>&2 + if [[ -n "$NETTYPE" ]]; then + h2nettype "$@" + else + h2nettype "$1" "tcp" fi } -declare -fx h2elan +# Wrapper function to print the deprecation warning h2o2ib() { - h2name_or_ip "$1" "o2ib" + echo "h2o2ib: deprecated, use h2nettype instead" 1>&2 + if [[ -n "$NETTYPE" ]]; then + h2nettype "$@" + else + h2nettype "$1" "o2ib" + fi } -declare -fx h2o2ib # This enables variables in cfg/"setup".sh files to support the pdsh HOSTLIST # expressions format. As a bonus we can then just pass in those variables @@ -2682,6 +2843,8 @@ hostlist_expand() { group=${group%%]*} for range in ${group//,/ }; do + local order + begin=${range%-*} end=${range#*-} @@ -2697,7 +2860,13 @@ hostlist_expand() { begin=$(echo $begin | sed 's/0*//') [ -z $begin ] && begin=0 - for num in $(seq -f "%0${padlen}g" $begin $end); do + if [ ! -z "${begin##[!0-9]*}" ]; then + order=$(seq -f "%0${padlen}g" $begin $end) + else + order=$(eval echo {$begin..$end}); + fi + + for num in $order; do value="${name#*,}${num}${back}" [ "$value" != "${value/\[/}" ] && { value=$(hostlist_expand "$value") @@ -2718,8 +2887,10 @@ hostlist_expand() { myList="${list%% *}" while [[ "$list" != ${myList##* } ]]; do - list=${list//${list%% *} /} - myList="$myList ${list%% *}" + local tlist=" $list" + list=${tlist// ${list%% *} / } + list=${list:1} + myList="$myList ${list%% *}" done myList="${myList%* }"; @@ -2928,6 +3099,12 @@ get_env_vars() { fi done + for var in VERBOSE; do + if [ -n "${!var}" ]; then + echo -n " $var=${!var}" + fi + done + if [ -n "$FSTYPE" ]; then echo -n " FSTYPE=$FSTYPE" fi @@ -2981,6 +3158,13 @@ do_nodes() { return ${PIPESTATUS[0]} } +## +# Execute commands on a single service's host +# +# The \a facet (service) may be on a local or remote node, which is +# determined at the time the command is run. +# +# usage: do_facet $facet command [arg ...] do_facet() { local facet=$1 shift @@ -3181,7 +3365,7 @@ facet_mntpt () { [[ $facet = mgs ]] && combined_mgs_mds && facet="mds1" local var=${facet}_MOUNT - eval mntpt=${!var:-${MOUNT%/*}/$facet} + eval mntpt=${!var:-${MOUNT}-$facet} echo -n $mntpt } @@ -3189,7 +3373,7 @@ facet_mntpt () { mount_ldiskfs() { local facet=$1 local dev=$(facet_device $facet) - local mnt=$(facet_mntpt $facet) + local mnt=${2:-$(facet_mntpt $facet)} local opts if ! do_facet $facet test -b $dev; then @@ -3201,9 +3385,9 @@ mount_ldiskfs() { unmount_ldiskfs() { local facet=$1 local dev=$(facet_device $facet) - local mnt=$(facet_mntpt $facet) + local mnt=${2:-$(facet_mntpt $facet)} - do_facet $facet umount -d $mnt + do_facet $facet $UMOUNT $mnt } var_name() { @@ -3213,7 +3397,7 @@ var_name() { mount_zfs() { local facet=$1 local ds=$(facet_device $facet) - local mnt=$(facet_mntpt $facet) + local mnt=${2:-$(facet_mntpt $facet)} local canmnt local mntpt @@ -3236,7 +3420,7 @@ mount_zfs() { unmount_zfs() { local facet=$1 local ds=$(facet_device $facet) - local mnt=$(facet_mntpt $facet) + local mnt=${2:-$(facet_mntpt $facet)} local var_mntpt=mz_$(var_name ${facet}_$ds)_mountpoint local var_canmnt=mz_$(var_name ${facet}_$ds)_canmount local mntpt=${!var_mntpt} @@ -3252,16 +3436,18 @@ unmount_zfs() { mount_fstype() { local facet=$1 + local mnt=$2 local fstype=$(facet_fstype $facet) - mount_$fstype $facet + mount_$fstype $facet $mnt } unmount_fstype() { local facet=$1 + local mnt=$2 local fstype=$(facet_fstype $facet) - unmount_$fstype $facet + unmount_$fstype $facet $mnt } ######## @@ -3281,7 +3467,8 @@ stopall() { zconf_umount_clients $clients $MOUNT "$*" || true [ -n "$MOUNT2" ] && zconf_umount_clients $clients $MOUNT2 "$*" || true - [ "$CLIENTONLY" ] && return + [ -n "$CLIENTONLY" ] && return + # The add fn does rm ${facet}active file, this would be enough # if we use do_facet only after the facet added, but # currently we use do_facet mds in local.sh @@ -3304,16 +3491,19 @@ stopall() { } cleanup_echo_devs () { - local devs=$($LCTL dl | grep echo | awk '{print $4}') + trap 0 + local dev + local devs=$($LCTL dl | grep echo | awk '{print $4}') - for dev in $devs; do - $LCTL --device $dev cleanup - $LCTL --device $dev detach - done + for dev in $devs; do + $LCTL --device $dev cleanup + $LCTL --device $dev detach + done } cleanupall() { nfs_client_mode && return + cifs_client_mode && return stopall $* cleanup_echo_devs @@ -3362,14 +3552,13 @@ mkfs_opts() { var=${facet}failover_HOST if [ -n "${!var}" ] && [ ${!var} != $(facet_host $facet) ]; then - opts+=" --failnode=$(h2$NETTYPE ${!var})" + opts+=" --failnode=$(h2nettype ${!var})" fi opts+=${TIMEOUT:+" --param=sys.timeout=$TIMEOUT"} opts+=${LDLM_TIMEOUT:+" --param=sys.ldlm_timeout=$LDLM_TIMEOUT"} if [ $type == MDS ]; then - opts+=${SECLEVEL:+" --param=mdt.sec_level"} opts+=${MDSCAPA:+" --param-mdt.capa=$MDSCAPA"} opts+=${STRIPE_BYTES:+" --param=lov.stripesize=$STRIPE_BYTES"} opts+=${STRIPES_PER_OBJ:+" --param=lov.stripecount=$STRIPES_PER_OBJ"} @@ -3382,20 +3571,26 @@ mkfs_opts() { fs_mkfs_opts+="-O large_xattr" fi - fs_mkfs_opts+=${MDSJOURNALSIZE:+" -J size=$MDSJOURNALSIZE"} - if [ ! -z $EJOURNAL ]; then - fs_mkfs_opts+=${MDSJOURNALSIZE:+" device=$EJOURNAL"} + var=${facet}_JRN + if [ -n "${!var}" ]; then + fs_mkfs_opts+=" -J device=${!var}" + else + fs_mkfs_opts+=${MDSJOURNALSIZE:+" -J size=$MDSJOURNALSIZE"} fi fs_mkfs_opts+=${MDSISIZE:+" -i $MDSISIZE"} fi fi if [ $type == OST ]; then - opts+=${SECLEVEL:+" --param=ost.sec_level"} opts+=${OSSCAPA:+" --param=ost.capa=$OSSCAPA"} if [ $fstype == ldiskfs ]; then - fs_mkfs_opts+=${OSTJOURNALSIZE:+" -J size=$OSTJOURNALSIZE"} + var=${facet}_JRN + if [ -n "${!var}" ]; then + fs_mkfs_opts+=" -J device=${!var}" + else + fs_mkfs_opts+=${OSTJOURNALSIZE:+" -J size=$OSTJOURNALSIZE"} + fi fi fi @@ -3422,6 +3617,17 @@ mkfs_opts() { echo -n "$opts" } +mountfs_opts() { + local facet=$1 + local type=$(facet_type $facet) + local var=${type}_MOUNT_FS_OPTS + local opts="" + if [ -n "${!var}" ]; then + opts+=" --mountfsoptions=${!var}" + fi + echo -n "$opts" +} + check_ost_indices() { local index_count=${#OST_INDICES[@]} [[ $index_count -eq 0 || $OSTCOUNT -le $index_count ]] && return 0 @@ -3440,39 +3646,110 @@ check_ost_indices() { done } -formatall() { +__touch_device() +{ + local facet_type=$1 # mgs || mds || ost + local facet_num=$2 + local facet=${1}${2} + local device + + case "$(facet_fstype $facet)" in + ldiskfs) + device=$(${facet_type}devname $facet_num) + ;; + zfs) + device=$(${facet_type}vdevname $facet_num) + ;; + *) + error "Unhandled filesystem type" + ;; + esac + + do_facet $facet "[ -e \"$device\" ]" && return + + # Note: the following check only works with absolute paths + [[ ! "$device" =~ ^/dev/ ]] || [[ "$device" =~ ^/dev/shm/ ]] || + error "$facet: device '$device' does not exist" + + do_facet $facet "touch \"${device}\"" +} + +format_mgs() { + local quiet + + if ! $VERBOSE; then + quiet=yes + fi + echo "Format mgs: $(mgsdevname)" + reformat_external_journal mgs + + # touch "device" in case it is a loopback file for testing and needs to + # be created. mkfs.lustre doesn't do this to avoid accidentally writing + # to non-existent files in /dev if the admin made a typo during setup + __touch_device mgs + + add mgs $(mkfs_opts mgs $(mgsdevname)) $(mountfs_opts mgs) --reformat \ + $(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} || exit 10 +} + +format_mdt() { + local num=$1 local quiet if ! $VERBOSE; then quiet=yes fi + echo "Format mds$num: $(mdsdevname $num)" + reformat_external_journal mds$num + + __touch_device mds $num + + add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \ + $(mountfs_opts mds$num) --reformat $(mdsdevname $num) \ + $(mdsvdevname $num) ${quiet:+>/dev/null} || exit 10 +} + +format_ost() { + local num=$1 + + if ! $VERBOSE; then + quiet=yes + fi + echo "Format ost$num: $(ostdevname $num)" + reformat_external_journal ost$num + + __touch_device ost $num + + add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \ + $(mountfs_opts ost$num) --reformat $(ostdevname $num) \ + $(ostvdevname ${num}) ${quiet:+>/dev/null} || exit 10 +} +formatall() { stopall + # Set hostid for ZFS/SPL zpool import protection + # (Assumes MDS version is also OSS version) + if [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.8.54) ]; + then + do_rpc_nodes "$(comma_list $(remote_nodes_list))" set_hostid + fi + # We need ldiskfs here, may as well load them all load_modules - [ "$CLIENTONLY" ] && return + [ -n "$CLIENTONLY" ] && return echo Formatting mgs, mds, osts if ! combined_mgs_mds ; then - echo "Format mgs: $(mgsdevname)" - add mgs $(mkfs_opts mgs $(mgsdevname)) --reformat \ - $(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} || - exit 10 + format_mgs fi for num in $(seq $MDSCOUNT); do - echo "Format mds$num: $(mdsdevname $num)" - add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \ - --reformat $(mdsdevname $num) $(mdsvdevname $num) \ - ${quiet:+>/dev/null} || exit 10 + format_mdt $num done export OST_INDICES=($(hostlist_expand "$OST_INDEX_LIST")) check_ost_indices for num in $(seq $OSTCOUNT); do - echo "Format ost$num: $(ostdevname $num)" - add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \ - --reformat $(ostdevname $num) $(ostvdevname ${num}) \ - ${quiet:+>/dev/null} || exit 10 + format_ost $num done } @@ -3549,20 +3826,21 @@ writeconf_all () { } setupall() { - nfs_client_mode && return + local arg1=$1 - sanity_mount_check || - error "environments are insane!" + nfs_client_mode && return + cifs_client_mode && return - load_modules + sanity_mount_check || error "environments are insane!" + + load_modules - if [ -z "$CLIENTONLY" ]; then - echo Setup mgs, mdt, osts - echo $WRITECONF | grep -q "writeconf" && \ - writeconf_all - if ! combined_mgs_mds ; then + if [ -z "$CLIENTONLY" ]; then + echo Setup mgs, mdt, osts + echo $WRITECONF | grep -q "writeconf" && writeconf_all + if ! combined_mgs_mds ; then start mgs $(mgsdevname) $MGS_MOUNT_OPTS - fi + fi for num in `seq $MDSCOUNT`; do DEVNAME=$(mdsdevname $num) @@ -3602,6 +3880,11 @@ setupall() { fi [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE + + if [ ! -z $arg1 ]; then + [ "$arg1" = "server_only" ] && return + fi + mount_client $MOUNT [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT clients_up @@ -3631,7 +3914,7 @@ mounted_lustre_filesystems() { } init_facet_vars () { - [ "$CLIENTONLY" ] && return 0 + [ -n "$CLIENTONLY" ] && return 0 local facet=$1 shift local device=$1 @@ -3756,14 +4039,17 @@ set_conf_param_and_check() { do_facet mgs "$LCTL conf_param $PARAM='$FINAL'" || error "conf_param $PARAM failed" - wait_update $(facet_host $myfacet) "$TEST" "$FINAL" || + wait_update_facet $myfacet "$TEST" "$FINAL" || error "check $PARAM failed!" } init_param_vars () { - remote_mds_nodsh || - TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout") + TIMEOUT=$(lctl get_param -n timeout) + TIMEOUT=${TIMEOUT:-20} + remote_mds_nodsh && log "Using TIMEOUT=$TIMEOUT" && return 0 + + TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout") log "Using TIMEOUT=$TIMEOUT" osc_ensure_active $SINGLEMDS $TIMEOUT @@ -3815,18 +4101,23 @@ nfs_client_mode () { return 1 } +cifs_client_mode () { + [ x$CIFSCLIENT = xyes ] && + echo "CIFSCLIENT=$CIFSCLIENT mode: setup, cleanup, check config skipped" +} + check_config_client () { local mntpt=$1 local mounted=$(mount | grep " $mntpt ") - if [ "$CLIENTONLY" ]; then + if [ -n "$CLIENTONLY" ]; then # bug 18021 # CLIENTONLY should not depend on *_HOST settings local mgc=$($LCTL device_list | awk '/MGC/ {print $4}') # in theory someone could create a new, # client-only config file that assumed lustre was already # configured and didn't set the MGSNID. If MGSNID is not set, - # then we should use the mgs nid currently being used + # then we should use the mgs nid currently being used # as the default value. bug 18021 [[ x$MGSNID = x ]] && MGSNID=${mgc//MGC/} @@ -3841,20 +4132,10 @@ check_config_client () { return 0 fi - local myMGS_host=$mgs_HOST - if [ "$NETTYPE" = "ptl" ]; then - myMGS_host=$(h2ptl $mgs_HOST | sed -e s/@ptl//) - fi - echo Checking config lustre mounted on $mntpt local mgshost=$(mount | grep " $mntpt " | awk -F@ '{print $1}') mgshost=$(echo $mgshost | awk -F: '{print $1}') -# if [ "$mgshost" != "$myMGS_host" ]; then -# log "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE -# Please use correct config or set mds_HOST correctly!" -# fi - } check_config_clients () { @@ -3862,6 +4143,7 @@ check_config_clients () { local mntpt=$1 nfs_client_mode && return + cifs_client_mode && return do_rpc_nodes "$clients" check_config_client $mntpt @@ -3907,20 +4189,22 @@ is_empty_fs() { } check_and_setup_lustre() { - nfs_client_mode && return + sanitize_parameters + nfs_client_mode && return + cifs_client_mode && return - local MOUNTED=$(mounted_lustre_filesystems) - - local do_check=true - # 1. - # both MOUNT and MOUNT2 are not mounted - if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then - [ "$REFORMAT" ] && formatall - # setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set) - setupall - is_mounted $MOUNT || error "NAME=$NAME not mounted" - export I_MOUNTED=yes - do_check=false + local MOUNTED=$(mounted_lustre_filesystems) + + local do_check=true + # 1. + # both MOUNT and MOUNT2 are not mounted + if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then + [ "$REFORMAT" = "yes" ] && formatall + # setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set) + setupall + is_mounted $MOUNT || error "NAME=$NAME not mounted" + export I_MOUNTED=yes + do_check=false # 2. # MOUNT2 is mounted elif is_mounted $MOUNT2; then @@ -3943,7 +4227,7 @@ check_and_setup_lustre() { restore_mount $MOUNT2 export I_MOUNTED2=yes fi - fi + fi # 5. # MOUNT is mounted MOUNT2 is not mounted @@ -3964,7 +4248,7 @@ check_and_setup_lustre() { set_default_debug_nodes $(comma_list $(nodes_list)) fi - if [ $(lower $OSD_TRACK_DECLARES_LBUG) == 'yes' ] ; then + if [ -z "$CLIENTONLY" -a $(lower $OSD_TRACK_DECLARES_LBUG) == 'yes' ]; then local facets="" [ "$(facet_fstype ost1)" = "ldiskfs" ] && facets="$(get_facets OST)" @@ -3984,12 +4268,14 @@ check_and_setup_lustre() { set_flavor_all $SEC fi - #Enable remote MDT create for testing - for num in $(seq $MDSCOUNT); do - do_facet mds$num \ - lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 \ - 2>/dev/null - done + if [ -z "$CLIENTONLY" ]; then + # Enable remote MDT create for testing + for num in $(seq $MDSCOUNT); do + do_facet mds$num \ + lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 \ + 2>/dev/null + done + fi if [ "$ONLY" == "setup" ]; then exit 0 @@ -4120,88 +4406,31 @@ check_shared_dir() { return 0 } -# Run e2fsck on MDT and OST(s) to generate databases used for lfsck. -generate_db() { - local i - local ostidx - local dev - local node - - [[ $(lustre_version_code $SINGLEMDS) -ne $(version_code 2.2.0) ]] || - { skip "Lustre 2.2.0 lacks the patch for LU-1255"; exit 0; } - - check_shared_dir $SHARED_DIRECTORY || - error "$SHARED_DIRECTORY isn't a shared directory" - - export MDSDB=$SHARED_DIRECTORY/mdsdb - export OSTDB=$SHARED_DIRECTORY/ostdb - - # DNE is not supported, so when running e2fsck on a DNE filesystem, - # we only pass master MDS parameters. - run_e2fsck $MDTNODE $MDTDEV "-n --mdsdb $MDSDB" - - i=0 - ostidx=0 - OSTDB_LIST="" - for node in $(osts_nodes); do - for dev in ${OSTDEVS[i]}; do - run_e2fsck $node $dev "-n --mdsdb $MDSDB --ostdb $OSTDB-$ostidx" - OSTDB_LIST="$OSTDB_LIST $OSTDB-$ostidx" - ostidx=$((ostidx + 1)) - done - i=$((i + 1)) - done -} - -# Run lfsck on server node if lfsck can't be found on client (LU-2571) -run_lfsck_remote() { - local cmd="$LFSCK_BIN -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT" - local client=$1 - local mounted=true - local rc=0 - - #Check if lustre is already mounted - do_rpc_nodes $client is_mounted $MOUNT || mounted=false - if ! $mounted; then - zconf_mount $client $MOUNT || - error "failed to mount Lustre on $client" - fi - #Run lfsck - echo $cmd - do_node $client $cmd || rc=$? - #Umount if necessary - if ! $mounted; then - zconf_umount $client $MOUNT || - error "failed to unmount Lustre on $client" - fi - - [ $rc -le $FSCK_MAX_ERR ] || - error "$cmd returned $rc, should be <= $FSCK_MAX_ERR" - echo "lfsck finished with rc=$rc" - - return $rc -} - run_lfsck() { - local facets="client $SINGLEMDS" - local found=false - local facet - local node - local rc=0 - - for facet in $facets; do - node=$(facet_active_host $facet) - if check_progs_installed $node $LFSCK_BIN; then - found=true - break - fi + do_nodes $(comma_list $(mdts_nodes) $(osts_nodes)) \ + $LCTL set_param printk=+lfsck + do_facet $SINGLEMDS "$LCTL lfsck_start -M $FSNAME-MDT0000 -r -A -t all" + + for k in $(seq $MDSCOUNT); do + # wait up to 10+1 minutes for LFSCK to complete + wait_update_facet --verbose mds${k} "$LCTL get_param -n \ + mdd.$(facet_svc mds${k}).lfsck_layout | + awk '/^status/ { print \\\$2 }'" "completed" 600 || + error "MDS${k} layout isn't the expected 'completed'" + wait_update_facet --verbose mds${k} "$LCTL get_param -n \ + mdd.$(facet_svc mds${k}).lfsck_namespace | + awk '/^status/ { print \\\$2 }'" "completed" 60 || + error "MDS${k} namespace isn't the expected 'completed'" done - ! $found && error "None of \"$facets\" supports lfsck" - - run_lfsck_remote $node || rc=$? - - rm -rvf $MDSDB* $OSTDB* || true - return $rc + local rep_mdt=$(do_nodes $(comma_list $(mdts_nodes)) \ + $LCTL get_param -n mdd.$FSNAME-*.lfsck_* | + awk '/repaired/ { print $2 }' | calc_sum) + local rep_ost=$(do_nodes $(comma_list $(osts_nodes)) \ + $LCTL get_param -n obdfilter.$FSNAME-*.lfsck_* | + awk '/repaired/ { print $2 }' | calc_sum) + local repaired=$((rep_mdt + rep_ost)) + [ $repaired -eq 0 ] || + error "lfsck repaired $rep_mdt MDT and $rep_ost OST errors" } dump_file_contents() { @@ -4259,15 +4488,18 @@ log_zfs_info() { } check_and_cleanup_lustre() { - if [ "$LFSCK_ALWAYS" = "yes" -a "$TESTSUITE" != "lfsck" ]; then - get_svr_devs - generate_db - run_lfsck - fi + if [ "$LFSCK_ALWAYS" = "yes" -a "$TESTSUITE" != "sanity-lfsck" -a \ + "$TESTSUITE" != "sanity-scrub" ]; then + run_lfsck + fi if is_mounted $MOUNT; then - [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* || - error "remove sub-test dirs failed" + if $DO_CLEANUP; then + [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* || + error "remove sub-test dirs failed" + else + echo "skip cleanup" + fi [ "$ENABLE_QUOTA" ] && restore_quota || true fi @@ -4590,37 +4822,15 @@ clear_failloc() { } set_nodes_failloc () { - do_nodes $(comma_list $1) lctl set_param fail_val=0 fail_loc=$2 + local fv=${3:-0} + do_nodes $(comma_list $1) lctl set_param fail_val=$fv fail_loc=$2 } cancel_lru_locks() { - $LCTL mark "cancel_lru_locks $1 start" - - if [ $1 != "MGC" ]; then - for d in $(lctl get_param -N ldlm.namespaces.*.lru_size | - egrep -i $1); do - $LCTL set_param -n $d=clear - done - $LCTL get_param ldlm.namespaces.*.lock_unused_count | egrep -i $1 | - grep -v '=0' - else - for d in $(find \ - /{proc,sys}/fs/lustre/ldlm/namespaces/*$1*/lru_size \ - 2> /dev/null); do - echo "clear" > $d - done - - for d in $(find \ - /{proc,sys}/fs/lustre/ldlm/namespaces/*$1*/lock_unused_count \ - 2> /dev/null); do - if [ $(cat $d) != 0 ]; then - echo "ldlm.namespaces.$(echo "$d" | - cut -f 7 -d'/').lock_unused_count=$(cat $d)" - fi - done - fi - - $LCTL mark "cancel_lru_locks $1 stop" + #$LCTL mark "cancel_lru_locks $1 start" + $LCTL set_param -n ldlm.namespaces.*$1*.lru_size=clear + $LCTL get_param ldlm.namespaces.*$1*.lock_unused_count | grep -v '=0' + #$LCTL mark "cancel_lru_locks $1 stop" } default_lru_size() @@ -4660,13 +4870,21 @@ pgcache_empty() { } debugsave() { - DEBUGSAVE="$(lctl get_param -n debug)" + DEBUGSAVE="$(lctl get_param -n debug)" + DEBUGSAVE_SERVER=$(do_facet $SINGLEMDS "$LCTL get_param -n debug") } debugrestore() { - [ -n "$DEBUGSAVE" ] && \ - do_nodes $(comma_list $(nodes_list)) "$LCTL set_param debug=\\\"${DEBUGSAVE}\\\";" - DEBUGSAVE="" + [ -n "$DEBUGSAVE" ] && + do_nodes $CLIENTS "$LCTL set_param debug=\\\"${DEBUGSAVE}\\\""|| + true + DEBUGSAVE="" + + [ -n "$DEBUGSAVE_SERVER" ] && + do_nodes $(comma_list $(all_server_nodes)) \ + "$LCTL set_param debug=\\\"${DEBUGSAVE_SERVER}\\\"" || + true + DEBUGSAVE_SERVER="" } debug_size_save() { @@ -4697,20 +4915,17 @@ stop_full_debug_logging() { # prints bash call stack print_stack_trace() { - echo " Trace dump:" - for (( i=1; i < ${#BASH_LINENO[*]} ; i++ )) ; do - local s=${BASH_SOURCE[$i]} - local l=${BASH_LINENO[$i-1]} - local f=${FUNCNAME[$i]} - echo " = $s:$l:$f()" - done -} - -################################## -# Test interface -################################## + local skip=${1:-1} + echo " Trace dump:" + for (( i=$skip; i < ${#BASH_LINENO[*]} ; i++ )) ; do + local src=${BASH_SOURCE[$i]} + local lineno=${BASH_LINENO[$i-1]} + local funcname=${FUNCNAME[$i]} + echo " = $src:$lineno:$funcname()" + done +} -error_noexit() { +report_error() { local TYPE=${TYPE:-"FAIL"} local dump=true @@ -4720,10 +4935,8 @@ error_noexit() { dump=false fi - log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ " - print_stack_trace >&2 - + (print_stack_trace 2) >&2 mkdir -p $LOGDIR # We need to dump the logs on all nodes if $dump; then @@ -4742,6 +4955,17 @@ error_noexit() { echo "$@" > $LOGDIR/err fi fi + + # cleanup the env for failed tests + reset_fail_loc +} + +################################## +# Test interface +################################## + +error_noexit() { + report_error "$@" } exit_status () { @@ -4753,12 +4977,13 @@ exit_status () { } error() { - error_noexit "$@" + report_error "$@" exit 1 } error_exit() { - error "$@" + report_error "$@" + exit 1 } # use only if we are ignoring failures for this test, bugno required. @@ -4768,15 +4993,27 @@ error_exit() { error_ignore() { local TYPE="IGNORE ($1)" shift - error_noexit "$@" + report_error "$@" } error_and_remount() { - error_noexit "$@" + report_error "$@" remount_client $MOUNT exit 1 } +# Throw an error if it's not running in vm - usually for performance +# verification +error_not_in_vm() { + local virt=$(running_in_vm) + if [[ -n "$virt" ]]; then + echo "running in VM '$virt', ignore error" + error_ignore env=$virt "$@" + else + error "$@" + fi +} + skip_env () { $FAIL_ON_SKIP_ENV && error false $@ || skip $@ } @@ -4962,17 +5199,17 @@ check_mds() { } reset_fail_loc () { - echo -n "Resetting fail_loc on all nodes..." - do_nodes $(comma_list $(nodes_list)) "lctl set_param -n fail_loc=0 \ - fail_val=0 2>/dev/null || true" - echo done. + echo -n "Resetting fail_loc on all nodes..." + do_nodes $(comma_list $(nodes_list)) "lctl set_param -n fail_loc=0 \ + fail_val=0 2>/dev/null" || true + echo done. } # -# Log a message (on all nodes) padded with "=" before and after. +# Log a message (on all nodes) padded with "=" before and after. # Also appends a timestamp and prepends the testsuite name. -# +# EQUALS="====================================================================================================" banner() { @@ -4984,6 +5221,18 @@ banner() { log "$msg== $(date +"%H:%M:%S (%s)")" } +check_dmesg_for_errors() { + local res + local errors="VFS: Busy inodes after unmount of\|\ +ldiskfs_check_descriptors: Checksum for group 0 failed\|\ +group descriptors corrupted" + + res=$(do_nodes $(comma_list $(nodes_list)) "dmesg" | grep "$errors") + [ -z "$res" ] && return 0 + echo "Kernel error detected: $res" + return 1 +} + # # Run a single test function and cleanup after it. # @@ -4999,12 +5248,17 @@ run_one() { local SAVE_UMASK=`umask` umask 0022 + if ! grep -q $DIR /proc/mounts; then + $SETUP + fi + banner "test $testnum: $message" test_${testnum} || error "test_$testnum failed with $?" cd $SAVE_PWD reset_fail_loc check_grant ${testnum} || error "check_grant $testnum failed with $?" - check_catastrophe || error "LBUG/LASSERT detected" + check_node_health + check_dmesg_for_errors || error "Error in dmesg detected" if [ "$PARALLEL" != "yes" ]; then ps auxww | grep -v grep | grep -q multiop && error "multiop still running" @@ -5013,6 +5267,7 @@ run_one() { unset tdir unset tfile umask $SAVE_UMASK + $CLEANUP return 0 } @@ -5087,8 +5342,9 @@ check_grant() { export base=$(basetest $1) [ "$CHECK_GRANT" == "no" ] && return 0 - testname=GCHECK_ONLY_${base} - [ ${!testname}x == x ] && return 0 + testnamebase=GCHECK_ONLY_${base} + testname=GCHECK_ONLY_$1 + [ ${!testnamebase}x == x -a ${!testname}x == x ] && return 0 echo -n "checking grant......" @@ -5104,17 +5360,22 @@ check_grant() { awk '{ total += $1 } END { printf("%0.0f", total) }') # get server grant + # which is tot_granted less grant_precreate server_grant=$(do_nodes $(comma_list $(osts_nodes)) \ - "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" | - awk '{ total += $1 } END { printf("%0.0f", total) }') + "$LCTL get_param "\ + "obdfilter.${FSNAME}-OST*.{tot_granted,tot_pending,grant_precreate}" | + sed 's/=/ /'| awk '/tot_granted/{ total += $2 }; + /tot_pending/{ total -= $2 }; + /grant_precreate/{ total -= $2 }; + END { printf("%0.0f", total) }') # check whether client grant == server grant if [[ $client_grant -ne $server_grant ]]; then - echo "failed: client:${client_grant} server: ${server_grant}." do_nodes $(comma_list $(osts_nodes)) \ - "$LCTL get_param obdfilter.${FSNAME}-OST*.tot*" + "$LCTL get_param obdfilter.${FSNAME}-OST*.tot*" \ + "obdfilter.${FSNAME}-OST*.grant_*" do_nodes $clients "$LCTL get_param osc.${FSNAME}-*.cur_*_bytes" - return 1 + error "failed: client:${client_grant} server: ${server_grant}." else echo "pass: client:${client_grant} server: ${server_grant}" fi @@ -5205,15 +5466,15 @@ remote_mds () remote_mds_nodsh() { - [ "$CLIENTONLY" ] && return 0 || true - remote_mds && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$mds_HOST" ] + [ -n "$CLIENTONLY" ] && return 0 || true + remote_mds && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$mds_HOST" ] } require_dsh_mds() { - remote_mds_nodsh && echo "SKIP: $TESTSUITE: remote MDS with nodsh" && \ - MSKIPPED=1 && return 1 - return 0 + remote_mds_nodsh && echo "SKIP: $TESTSUITE: remote MDS with nodsh" && + MSKIPPED=1 && return 1 + return 0 } remote_ost () @@ -5227,8 +5488,8 @@ remote_ost () remote_ost_nodsh() { - [ "$CLIENTONLY" ] && return 0 || true - remote_ost && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ] + [ -n "$CLIENTONLY" ] && return 0 || true + remote_ost && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ] } require_dsh_ost() @@ -5240,9 +5501,10 @@ require_dsh_ost() remote_mgs_nodsh() { - local MGS - MGS=$(facet_host mgs) - remote_node $MGS && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ] + [ -n "$CLIENTONLY" ] && return 0 || true + local MGS + MGS=$(facet_host mgs) + remote_node $MGS && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ] } local_mode () @@ -5403,19 +5665,14 @@ get_random_entry () { rnodes=${rnodes//,/ } local -a nodes=($rnodes) - local num=${#nodes[@]} + local num=${#nodes[@]} local i=$((RANDOM * num * 2 / 65536)) echo ${nodes[i]} } client_only () { - [ "$CLIENTONLY" ] || [ "$CLIENTMODSONLY" = yes ] -} - -is_patchless () -{ - lctl get_param version | grep -q patchless + [ -n "$CLIENTONLY" ] || [ "x$CLIENTMODSONLY" = "xyes" ] } check_versions () { @@ -5460,13 +5717,19 @@ get_stripe () { setstripe_nfsserver () { local dir=$1 + local nfsexportdir=$2 + shift + shift - local nfsserver=$(awk '"'$dir'" ~ $2 && $3 ~ "nfs" && $2 != "/" \ - { print $1 }' /proc/mounts | cut -f 1 -d : | head -n1) + local -a nfsexport=($(awk '"'$dir'" ~ $2 && $3 ~ "nfs" && $2 != "/" \ + { print $1 }' /proc/mounts | cut -f 1 -d :)) - [ -z $nfsserver ] && echo "$dir is not nfs mounted" && return 1 + # check that only one nfs mounted + [[ -z $nfsexport ]] && echo "$dir is not nfs mounted" && return 1 + (( ${#nfsexport[@]} == 1 )) || + error "several nfs mounts found for $dir: ${nfsexport[@]} !" - do_nodev $nfsserver lfs setstripe "$@" + do_nodev ${nfsexport[0]} lfs setstripe $nfsexportdir "$@" } # Check and add a test group. @@ -5636,32 +5899,22 @@ inodes_available () { } mdsrate_inodes_available () { - local min_inodes=$(inodes_available) - echo $((min_inodes * 99 / 100)) -} - -# reset llite stat counters -clear_llite_stats(){ - lctl set_param -n llite.*.stats 0 -} - -# sum llite stat items -calc_llite_stats() { - local res=$(lctl get_param -n llite.*.stats | - awk '/^'"$1"'/ {sum += $2} END { printf("%0.0f", sum) }') - echo $((res)) + local min_inodes=$(inodes_available) + echo $((min_inodes * 99 / 100)) } -# reset osc stat counters -clear_osc_stats(){ - lctl set_param -n osc.*.osc_stats 0 +# reset stat counters +clear_stats() { + local paramfile="$1" + lctl set_param -n $paramfile=0 } -# sum osc stat items -calc_osc_stats() { - local res=$(lctl get_param -n osc.*.osc_stats | - awk '/^'"$1"'/ {sum += $2} END { printf("%0.0f", sum) }') - echo $((res)) +# sum stat items +calc_stats() { + local paramfile="$1" + local stat="$2" + lctl get_param -n $paramfile | + awk '/^'$stat'/ { sum += $2 } END { printf("%0.0f", sum) }' } calc_sum () { @@ -5669,8 +5922,8 @@ calc_sum () { } calc_osc_kbytes () { - df $MOUNT > /dev/null - $LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum + df $MOUNT > /dev/null + $LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum } # save_lustre_params(comma separated facet list, parameter_mask) @@ -5702,16 +5955,21 @@ restore_lustre_params() { done } -check_catastrophe() { +check_node_health() { local nodes=${1:-$(comma_list $(nodes_list))} - do_nodes $nodes "rc=0; -val=\\\$($LCTL get_param -n catastrophe 2>&1); -if [[ \\\$? -eq 0 && \\\$val -ne 0 ]]; then - echo \\\$(hostname -s): \\\$val; - rc=\\\$val; -fi; -exit \\\$rc" + for node in ${nodes//,/ }; do + check_network "$node" 5 + if [ $? -eq 0 ]; then + do_node $node "rc=0; + val=\\\$($LCTL get_param -n catastrophe 2>&1); + if [[ \\\$? -eq 0 && \\\$val -ne 0 ]]; then + echo \\\$(hostname -s): \\\$val; + rc=\\\$val; + fi; + exit \\\$rc" || error "$node:LBUG/LASSERT detected" + fi + done } mdsrate_cleanup () { @@ -5764,19 +6022,21 @@ mds_on_old_device() { } get_mdtosc_proc_path() { - local mds_facet=$1 - local ost_label=${2:-"*OST*"} - - [ "$mds_facet" = "mds" ] && mds_facet=$SINGLEMDS - local mdt_label=$(convert_facet2label $mds_facet) - local mdt_index=$(echo $mdt_label | sed -e 's/^.*-//') - - if [ $(lustre_version_code $mds_facet) -le $(version_code 1.8.0) ] || - mds_on_old_device $mds_facet; then - echo "${ost_label}-osc" - else - echo "${ost_label}-osc-${mdt_index}" - fi + local mds_facet=$1 + local ost_label=${2:-"*OST*"} + + [ "$mds_facet" = "mds" ] && mds_facet=$SINGLEMDS + local mdt_label=$(convert_facet2label $mds_facet) + local mdt_index=$(echo $mdt_label | sed -e 's/^.*-//') + + if [ $(lustre_version_code $mds_facet) -le $(version_code 1.8.0) ] || + mds_on_old_device $mds_facet; then + echo "${ost_label}-osc" + elif [[ $ost_label = *OST* ]]; then + echo "${ost_label}-osc-${mdt_index}" + else + echo "${ost_label}-osp-${mdt_index}" + fi } get_osc_import_name() { @@ -5797,6 +6057,7 @@ _wait_import_state () { local expected=$1 local CONN_PROC=$2 local maxtime=${3:-$(max_recovery_time)} + local error_on_failure=${4:-1} local CONN_STATE local i=0 @@ -5809,9 +6070,12 @@ _wait_import_state () { # reconnect timeout and test can't see real disconnect [ "${CONN_STATE}" == "CONNECTING" ] && return 0 fi - [ $i -ge $maxtime ] && \ - error "can't put import for $CONN_PROC into ${expected} state after $i sec, have ${CONN_STATE}" && \ + if [ $i -ge $maxtime ]; then + [ $error_on_failure -ne 0 ] && \ + error "can't put import for $CONN_PROC into ${expected}" \ + "state after $i sec, have ${CONN_STATE}" return 1 + fi sleep 1 # Add uniq for multi-mount case CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2 | uniq) @@ -5826,10 +6090,11 @@ wait_import_state() { local state=$1 local params=$2 local maxtime=${3:-$(max_recovery_time)} + local error_on_failure=${4:-1} local param for param in ${params//,/ }; do - _wait_import_state $state $param $maxtime || return + _wait_import_state $state $param $maxtime $error_on_failure || return done } @@ -5885,8 +6150,8 @@ _wait_osc_import_state() { local facet=$1 local ost_facet=$2 local expected=$3 - local ost=$(get_osc_import_name $facet $ost_facet) - local param="osc.${ost}.ost_server_uuid" + local target=$(get_osc_import_name $facet $ost_facet) + local param="osc.${target}.ost_server_uuid" local params=$param local i=0 @@ -5896,9 +6161,7 @@ _wait_osc_import_state() { if [[ $facet == client* ]]; then # During setup time, the osc might not be setup, it need wait - # until list_param can return valid value. And also if there - # are mulitple osc entries we should list all of them before - # go to wait. + # until list_param can return valid value. params=$($LCTL list_param $param 2>/dev/null || true) while [ -z "$params" ]; do if [ $i -ge $maxtime ]; then @@ -5910,9 +6173,19 @@ _wait_osc_import_state() { params=$($LCTL list_param $param 2>/dev/null || true) done fi + + if [[ $ost_facet = mds* ]]; then + # no OSP connection to itself + if [[ $facet = $ost_facet ]]; then + return 0 + fi + param="osp.${target}.mdt_server_uuid" + params=$param + fi + if ! do_rpc_nodes "$(facet_active_host $facet)" \ wait_import_state $expected "$params" $maxtime; then - error "import is not in ${expected} state" + error "$facet: import is not in $expected state after $maxtime" return 1 fi @@ -5934,6 +6207,73 @@ wait_osc_import_state() { fi } +_wait_mgc_import_state() { + local facet=$1 + local expected=$2 + local error_on_failure=${3:-1} + local param="mgc.*.mgs_server_uuid" + local params=$param + local i=0 + + # 1. wait the deadline of client 1st request (it could be skipped) + # 2. wait the deadline of client 2nd request + local maxtime=$(( 2 * $(request_timeout $facet))) + + if [[ $facet == client* ]]; then + # During setup time, the osc might not be setup, it need wait + # until list_param can return valid value. And also if there + # are mulitple osc entries we should list all of them before + # go to wait. + params=$($LCTL list_param $param 2>/dev/null || true) + while [ -z "$params" ]; do + if [ $i -ge $maxtime ]; then + echo "can't get $param in $maxtime secs" + return 1 + fi + sleep 1 + i=$((i + 1)) + params=$($LCTL list_param $param 2>/dev/null || true) + done + fi + if ! do_rpc_nodes "$(facet_active_host $facet)" \ + wait_import_state $expected "$params" $maxtime \ + $error_on_failure; then + if [ $error_on_failure -ne 0 ]; then + error "import is not in ${expected} state" + fi + return 1 + fi + + return 0 +} + +wait_mgc_import_state() { + local facet=$1 + local expected=$2 + local error_on_failure=${3:-1} + local num + + if [[ $facet = mds ]]; then + for num in $(seq $MDSCOUNT); do + _wait_mgc_import_state mds$num "$expected" \ + $error_on_failure || return + done + else + _wait_mgc_import_state "$facet" "$expected" + $error_on_failure || return + fi +} + +wait_dne_interconnect() { + local num + + if [ $MDSCOUNT -gt 1 ]; then + for num in $(seq $MDSCOUNT); do + wait_osc_import_state mds mds$num FULL + done + fi +} + get_clientmdc_proc_path() { echo "${1}-mdc-*" } @@ -5989,6 +6329,47 @@ wait_clients_import_state () { fi } +wait_osp_active() { + local facet=$1 + local tgt_name=$2 + local tgt_idx=$3 + local expected=$4 + local num + + # wait until all MDTs are in the expected state + for ((num = 1; num <= $MDSCOUNT; num++)); do + local mdtosp=$(get_mdtosc_proc_path mds${num} ${tgt_name}) + local mproc + + if [ $facet = "mds" ]; then + mproc="osp.$mdtosp.active" + [ $num -eq $((tgt_idx + 1)) ] && continue + else + mproc="osc.$mdtosp.active" + fi + + echo "check $mproc" + while [ 1 ]; do + sleep 5 + local result=$(do_facet mds${num} "$LCTL get_param -n $mproc") + local max=30 + local wait=0 + + [ ${PIPESTATUS[0]} = 0 ] || error "Can't read $mproc" + if [ $result -eq $expected ]; then + echo -n "target updated after" + echo "$wait sec (got $result)" + break + fi + wait=$((wait + 5)) + if [ $wait -eq $max ]; then + error "$tgt_name: wanted $expected got $result" + fi + echo "Waiting $((max - wait)) secs for $tgt_name" + done + done +} + oos_full() { local -a AVAILA local -a GRANTA @@ -6016,100 +6397,119 @@ oos_full() { return $OSCFULL } -pool_list () { - do_facet mgs lctl pool_list $1 +list_pool() { + echo -e "$(do_facet $SINGLEMDS $LCTL pool_list $1 | sed '1d')" +} + +check_pool_not_exist() { + local fsname=${1%%.*} + local poolname=${1##$fsname.} + [[ $# -ne 1 ]] && return 0 + [[ x$poolname = x ]] && return 0 + list_pool $fsname | grep -w $1 && return 1 + return 0 } create_pool() { - local fsname=${1%%.*} - local poolname=${1##$fsname.} - - do_facet mgs lctl pool_new $1 - local RC=$? - # get param should return err unless pool is created - [[ $RC -ne 0 ]] && return $RC - - wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ - 2>/dev/null || echo foo" "" || RC=1 - if [[ $RC -eq 0 ]]; then - add_pool_to_list $1 - else - error "pool_new failed $1" - fi - return $RC + local fsname=${1%%.*} + local poolname=${1##$fsname.} + + trap "destroy_test_pools $fsname" EXIT + do_facet mgs lctl pool_new $1 + local RC=$? + # get param should return err unless pool is created + [[ $RC -ne 0 ]] && return $RC + + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$fsname-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$poolname \ + 2>/dev/null || echo foo" "" || + error "mds$mds_id: pool_new failed $1" + done + wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ + 2>/dev/null || echo foo" "" || error "pool_new failed $1" + + add_pool_to_list $1 + return $RC } add_pool_to_list () { - local fsname=${1%%.*} - local poolname=${1##$fsname.} + local fsname=${1%%.*} + local poolname=${1##$fsname.} - local listvar=${fsname}_CREATED_POOLS - eval export ${listvar}=$(expand_list ${!listvar} $poolname) + local listvar=${fsname}_CREATED_POOLS + local temp=${listvar}=$(expand_list ${!listvar} $poolname) + eval export $temp } remove_pool_from_list () { - local fsname=${1%%.*} - local poolname=${1##$fsname.} + local fsname=${1%%.*} + local poolname=${1##$fsname.} - local listvar=${fsname}_CREATED_POOLS - eval export ${listvar}=$(exclude_items_from_list ${!listvar} $poolname) + local listvar=${fsname}_CREATED_POOLS + local temp=${listvar}=$(exclude_items_from_list ${!listvar} $poolname) + eval export $temp } destroy_pool_int() { - local ost - local OSTS=$(do_facet $SINGLEMDS lctl pool_list $1 | \ - awk '$1 !~ /^Pool:/ {print $1}') - for ost in $OSTS; do - do_facet mgs lctl pool_remove $1 $ost - done - do_facet mgs lctl pool_destroy $1 + local ost + local OSTS=$(list_pool $1) + for ost in $OSTS; do + do_facet mgs lctl pool_remove $1 $ost + done + do_facet mgs lctl pool_destroy $1 } # . or destroy_pool() { - local fsname=${1%%.*} - local poolname=${1##$fsname.} + local fsname=${1%%.*} + local poolname=${1##$fsname.} - [[ x$fsname = x$poolname ]] && fsname=$FSNAME + [[ x$fsname = x$poolname ]] && fsname=$FSNAME - local RC + local RC - pool_list $fsname.$poolname || return $? + check_pool_not_exist $fsname.$poolname + [[ $? -eq 0 ]] && return 0 - destroy_pool_int $fsname.$poolname - RC=$? - [[ $RC -ne 0 ]] && return $RC + destroy_pool_int $fsname.$poolname + RC=$? + [[ $RC -ne 0 ]] && return $RC + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$fsname-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$poolname \ + 2>/dev/null || echo foo" "foo" || + error "mds$mds_id: destroy pool failed $1" + done + wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ + 2>/dev/null || echo foo" "foo" || error "destroy pool failed $1" - wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ - 2>/dev/null || echo foo" "foo" || RC=1 + remove_pool_from_list $fsname.$poolname - if [[ $RC -eq 0 ]]; then - remove_pool_from_list $fsname.$poolname - else - error "destroy pool failed $1" - fi - return $RC + return $RC } destroy_pools () { - local fsname=${1:-$FSNAME} - local poolname - local listvar=${fsname}_CREATED_POOLS - - pool_list $fsname + local fsname=${1:-$FSNAME} + local poolname + local listvar=${fsname}_CREATED_POOLS - [ x${!listvar} = x ] && return 0 + [ x${!listvar} = x ] && return 0 - echo destroy the created pools: ${!listvar} - for poolname in ${!listvar//,/ }; do - destroy_pool $fsname.$poolname - done + echo "Destroy the created pools: ${!listvar}" + for poolname in ${!listvar//,/ }; do + destroy_pool $fsname.$poolname + done } -cleanup_pools () { - local fsname=${1:-$FSNAME} - trap 0 - destroy_pools $fsname +destroy_test_pools () { + trap 0 + local fsname=${1:-$FSNAME} + destroy_pools $fsname || true } gather_logs () { @@ -6132,7 +6532,7 @@ gather_logs () { suffix="$ts.log" echo "Dumping lctl log to ${prefix}.*.${suffix}" - if [ "$CLIENTONLY" -o "$PDSH" == "no_dsh" ]; then + if [ -n "$CLIENTONLY" -o "$PDSH" == "no_dsh" ]; then echo "Dumping logs only on local client." $LCTL dk > ${prefix}.debug_log.$(hostname -s).${suffix} dmesg > ${prefix}.dmesg.$(hostname -s).${suffix} @@ -6189,6 +6589,31 @@ max_recovery_time() { echo -n $service_time } +recovery_time_min() { + local connection_switch_min=5 + local connection_switch_inc=5 + local connection_switch_max + local reconnect_delay_max + local initial_connect_timeout + local max + local timout_20 + + #connection_switch_max=min(50, max($connection_switch_min,$TIMEOUT) + (($connection_switch_min > $TIMEOUT)) && + max=$connection_switch_min || max=$TIMEOUT + (($max < 50)) && connection_switch_max=$max || connection_switch_max=50 + + #initial_connect_timeout = max(connection_switch_min, obd_timeout/20) + timeout_20=$((TIMEOUT/20)) + (($connection_switch_min > $timeout_20)) && + initial_connect_timeout=$connection_switch_min || + initial_connect_timeout=$timeout_20 + + reconnect_delay_max=$((connection_switch_max + connection_switch_inc + \ + initial_connect_timeout)) + echo $((2 * reconnect_delay_max)) +} + get_clients_mount_count () { local clients=${CLIENTS:-`hostname`} @@ -6513,29 +6938,36 @@ check_write_access() { } init_logging() { - if [[ -n $YAML_LOG ]]; then - return - fi - local SAVE_UMASK=`umask` - umask 0000 - - export YAML_LOG=${LOGDIR}/results.yml - mkdir -p $LOGDIR - init_clients_lists - - if [ ! -f $YAML_LOG ]; then # If the yaml log already exists then we will just append to it - if check_shared_dir $LOGDIR; then - touch $LOGDIR/shared - echo "Logging to shared log directory: $LOGDIR" - else - echo "Logging to local directory: $LOGDIR" - fi + [[ -n $YAML_LOG ]] && return + local save_umask=$(umask) + umask 0000 - yml_nodes_file $LOGDIR >> $YAML_LOG - yml_results_file >> $YAML_LOG - fi + export YAML_LOG=${LOGDIR}/results.yml + mkdir -p $LOGDIR + init_clients_lists + + # If the yaml log already exists then we will just append to it + if [ ! -f $YAML_LOG ]; then + if check_shared_dir $LOGDIR; then + touch $LOGDIR/shared + echo "Logging to shared log directory: $LOGDIR" + else + echo "Logging to local directory: $LOGDIR" + fi - umask $SAVE_UMASK + yml_nodes_file $LOGDIR >> $YAML_LOG + yml_results_file >> $YAML_LOG + fi + + umask $save_umask + + # If modules are not yet loaded then older "lctl lustre_build_version" + # will fail. Use lctl build version instead. + log "Client: $($LCTL lustre_build_version)" + log "MDS: $(do_facet $SINGLEMDS $LCTL lustre_build_version 2>/dev/null|| + do_facet $SINGLEMDS $LCTL --version)" + log "OSS: $(do_facet ost1 $LCTL lustre_build_version 2> /dev/null || + do_facet ost1 $LCTL --version)" } log_test() { @@ -6715,7 +7147,7 @@ is_sanity_benchmark() { } min_ost_size () { - $LCTL get_param -n osc.*.kbytesavail | sort -n | head -n1 + $LFS df | grep OST | awk '{print $4}' | sort -un | head -1 } # @@ -6737,11 +7169,10 @@ get_obd_size() { # get_page_size() { local facet=$1 - local size + local size=$(getconf PAGE_SIZE 2>/dev/null) - size=$(do_facet $facet getconf PAGE_SIZE) - [[ ${PIPESTATUS[0]} = 0 && -n "$size" ]] || size=4096 - echo -n $size + [ -z "$CLIENTONLY" ] && size=$(do_facet $facet getconf PAGE_SIZE) + echo -n ${size:-4096} } # @@ -6752,20 +7183,20 @@ get_block_count() { local device=$2 local count - count=$(do_facet $facet "$DUMPE2FS -h $device 2>&1" | + [ -z "$CLIENTONLY" ] && count=$(do_facet $facet "$DUMPE2FS -h $device 2>&1" | awk '/^Block count:/ {print $3}') - echo -n $count + echo -n ${count:-0} } # Get the block size of the filesystem. get_block_size() { - local facet=$1 - local device=$2 - local size + local facet=$1 + local device=$2 + local size - size=$(do_facet $facet "$DUMPE2FS -h $device 2>&1" | - awk '/^Block size:/ {print $3}') - echo $size + [ -z "$CLIENTONLY" ] && size=$(do_facet $facet "$DUMPE2FS -h $device 2>&1" | + awk '/^Block size:/ {print $3}') + echo -n ${size:-0} } # Check whether the "large_xattr" feature is enabled or not. @@ -6815,12 +7246,14 @@ generate_string() { reformat_external_journal() { local facet=$1 + local var - if [ ! -z ${EJOURNAL} ]; then + var=${facet}_JRN + if [ -n "${!var}" ]; then local rcmd="do_facet $facet" - echo "reformat external journal on $facet:${EJOURNAL}" - ${rcmd} mke2fs -O journal_dev ${EJOURNAL} || return 1 + echo "reformat external journal on $facet:${!var}" + ${rcmd} mke2fs -O journal_dev ${!var} || return 1 fi } @@ -6860,14 +7293,10 @@ mds_backup_restore() { echo "backup data" ${rcmd} tar zcf $metadata -C $mntpt/ . > /dev/null 2>&1 || return 3 # step 6: umount - ${rcmd} umount -d $mntpt || return 4 - # step 7: reformat external journal if needed - reformat_external_journal $facet || return 5 + ${rcmd} $UMOUNT $mntpt || return 4 # step 8: reformat dev echo "reformat new device" - add $facet $(mkfs_opts $facet ${devname}) --backfstype ldiskfs \ - --reformat ${devname} $(mdsvdevname $(facet_number $facet)) \ - > /dev/null || exit 6 + format_mdt $(facet_number $facet) # step 9: mount dev ${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 7 # step 10: restore metadata @@ -6880,7 +7309,7 @@ mds_backup_restore() { echo "remove recovery logs" ${rcmd} rm -fv $mntpt/OBJECTS/* $mntpt/CATALOGS # step 13: umount dev - ${rcmd} umount -d $mntpt || return 10 + ${rcmd} $UMOUNT $mntpt || return 10 # step 14: cleanup tmp backup ${rcmd} rm -f $metaea $metadata # step 15: reset device label - it's not virgin on @@ -6920,7 +7349,7 @@ mds_remove_ois() { done fi # step 4: umount - ${rcmd} umount -d $mntpt || return 2 + ${rcmd} $UMOUNT $mntpt || return 2 # OI files will be recreated when mounted as lustre next time. } @@ -6976,16 +7405,22 @@ test_mkdir() { fi } -# find the smallest and not in use file descriptor +# free_fd: find the smallest and not in use file descriptor [above @last_fd] +# +# If called many times, passing @last_fd will avoid repeated searching +# already-open FDs repeatedly if we know they are still in use. +# +# usage: free_fd [last_fd] free_fd() { - local max_fd=$(ulimit -n) - local fd=3 - while [[ $fd -le $max_fd && -e /proc/self/fd/$fd ]]; do - ((++fd)) - done - [ $fd -lt $max_fd ] || error "finding free file descriptor failed" - echo $fd + local max_fd=$(ulimit -n) + local fd=$((${1:-2} + 1)) + + while [[ $fd -le $max_fd && -e /proc/self/fd/$fd ]]; do + ((++fd)) + done + [ $fd -lt $max_fd ] || error "finding free file descriptor failed" + echo $fd } check_mount_and_prep() @@ -7041,7 +7476,7 @@ pool_add() { create_pool $FSNAME.$pool || { error_noexit "No pool created, result code $?"; return 1; } - [ $($LFS pool_list $FSNAME | grep -c $pool) -eq 1 ] || + [ $($LFS pool_list $FSNAME | grep -c "$FSNAME.${pool}\$") -eq 1 ] || { error_noexit "$pool not in lfs pool_list"; return 2; } } @@ -7057,6 +7492,18 @@ pool_add_targets() { local t=$(for i in $list; do printf "$FSNAME-OST%04x_UUID " $i; done) do_facet mgs $LCTL pool_add \ $FSNAME.$pool $FSNAME-OST[$first-$last/$step] + + # wait for OSTs to be added to the pool + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$pool | + sort -u | tr '\n' ' ' " "$t" || { + error_noexit "mds$mds_id: Add to pool failed" + return 3 + } + done wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool \ | sort -u | tr '\n' ' ' " "$t" || { error_noexit "Add to pool failed" @@ -7193,6 +7640,17 @@ pool_remove_first_target() { local pname="lov.$FSNAME-*.pools.$pool" local t=$($LCTL get_param -n $pname | head -1) do_facet mgs $LCTL pool_remove $FSNAME.$pool $t + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$pool | + grep $t" "" || { + error_noexit "mds$mds_id: $t not removed from" \ + "$FSNAME.$pool" + return 2 + } + done wait_update $HOSTNAME "lctl get_param -n $pname | grep $t" "" || { error_noexit "$t not removed from $FSNAME.$pool" return 1 @@ -7208,6 +7666,15 @@ pool_remove_all_targets() { do do_facet mgs $LCTL pool_remove $FSNAME.$pool $t done + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id "lctl get_param -n \ + lod.$lodname.pools.$pool" "" || { + error_noexit "mds$mds_id: Pool $pool not drained" + return 4 + } + done wait_update $HOSTNAME "lctl get_param -n $pname" "" || { error_noexit "Pool $FSNAME.$pool cannot be drained" return 1 @@ -7316,3 +7783,190 @@ check_start_ost_idx() { error "OST index of the first stripe on $file is" \ "$start_ost_idx, should be $expected" } + +killall_process () { + local clients=${1:-$(hostname)} + local name=$2 + local signal=$3 + local rc=0 + + do_nodes $clients "killall $signal $name" +} + +lsnapshot_create() +{ + do_facet mgs "$LCTL snapshot_create -F $FSNAME $*" +} + +lsnapshot_destroy() +{ + do_facet mgs "$LCTL snapshot_destroy -F $FSNAME $*" +} + +lsnapshot_modify() +{ + do_facet mgs "$LCTL snapshot_modify -F $FSNAME $*" +} + +lsnapshot_list() +{ + do_facet mgs "$LCTL snapshot_list -F $FSNAME $*" +} + +lsnapshot_mount() +{ + do_facet mgs "$LCTL snapshot_mount -F $FSNAME $*" +} + +lsnapshot_umount() +{ + do_facet mgs "$LCTL snapshot_umount -F $FSNAME $*" +} + +lss_err() +{ + local msg=$1 + + do_facet mgs "cat $LSNAPSHOT_LOG" + error $msg +} + +lss_cleanup() +{ + echo "Cleaning test environment ..." + + # Every lsnapshot command takes exclusive lock with others, + # so can NOT destroy the snapshot during list with 'xargs'. + while true; do + local ssname=$(lsnapshot_list | grep snapshot_name | + grep lss_ | awk '{ print $2 }' | head -n 1) + [ -z "$ssname" ] && break + + lsnapshot_destroy -n $ssname -f || + lss_err "Fail to destroy $ssname by force" + done +} + +lss_gen_conf_one() +{ + local facet=$1 + local role=$2 + local idx=$3 + + local host=$(facet_active_host $facet) + local dir=$(dirname $(facet_vdevice $facet)) + local pool=$(zpool_name $facet) + local lfsname=$(zfs_local_fsname $facet) + local label=${FSNAME}-${role}$(printf '%04x' $idx) + + do_facet mgs \ + "echo '$host - $label zfs:${dir}/${pool}/${lfsname} - -' >> \ + $LSNAPSHOT_CONF" +} + +lss_gen_conf() +{ + do_facet mgs "rm -f $LSNAPSHOT_CONF" + echo "Generating $LSNAPSHOT_CONF on MGS ..." + + if ! combined_mgs_mds ; then + [ $(facet_fstype mgs) != zfs ] && + skip "Lustre snapshot 1 only works for ZFS backend" && + exit 0 + + local host=$(facet_active_host mgs) + local dir=$(dirname $(facet_vdevice mgs)) + local pool=$(zpool_name mgs) + local lfsname=$(zfs_local_fsname mgs) + + do_facet mgs \ + "echo '$host - MGS zfs:${dir}/${pool}/${lfsname} - -' \ + >> $LSNAPSHOT_CONF" || lss_err "generate lss conf (mgs)" + fi + + for num in `seq $MDSCOUNT`; do + [ $(facet_fstype mds$num) != zfs ] && + skip "Lustre snapshot 1 only works for ZFS backend" && + exit 0 + + lss_gen_conf_one mds$num MDT $((num - 1)) || + lss_err "generate lss conf (mds$num)" + done + + for num in `seq $OSTCOUNT`; do + [ $(facet_fstype ost$num) != zfs ] && + skip "Lustre snapshot 1 only works for ZFS backend" && + exit 0 + + lss_gen_conf_one ost$num OST $((num - 1)) || + lss_err "generate lss conf (ost$num)" + done + + do_facet mgs "cat $LSNAPSHOT_CONF" +} + +parse_plain_param() +{ + local line=$1 + local val=$(awk '{print $2}' <<< $line) + + if [[ $line =~ ^"lmm_stripe_count:" ]]; then + echo "-c $val" + elif [[ $line =~ ^"lmm_stripe_size:" ]]; then + echo "-S $val" + elif [[ $line =~ ^"lmm_stripe_offset:" ]]; then + echo "-i $val" + fi +} + +parse_layout_param() +{ + local mode="" + local val="" + local param="" + + while read line; do + if [[ -z $mode ]]; then + if [[ $line =~ ^"stripe_count:" ]]; then + mode="plain_dir" + elif [[ $line =~ ^"lmm_stripe_count:" ]]; then + mode="plain_file" + elif [[ $line =~ ^"lcm_layout_gen:" ]]; then + mode="pfl" + fi + fi + + if [[ $mode = "plain_dir" ]]; then + param=$(echo $line | + awk '{printf("-c %d -S %d -i %d",$2,$4,$6)}') + elif [[ $mode = "plain_file" ]]; then + val=$(parse_plain_param "$line") + [[ ! -z $val ]] && param="$param $val" + elif [[ $mode = "pfl" ]]; then + val=$(echo $line | awk '{print $2}') + if [[ $line =~ ^"lcme_extent.e_end:" ]]; then + if [[ $val = "EOF" ]]; then + param="$param -E -1" + else + param="$param -E $val" + fi + elif [[ $line =~ ^"stripe_count:" ]]; then + # pfl dir + val=$(echo $line | + awk '{printf("-c %d -S %d -i %d",$2,$4,$6)}') + param="$param $val" + else + #pfl file + val=$(parse_plain_param "$line") + [[ ! -z $val ]] && param="$param $val" + fi + fi + done + echo "$param" +} + +get_layout_param() +{ + local param=$($LFS getstripe -d $1 | parse_layout_param) + echo "$param" +}