X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=be82fec55c6ff9a1112b095a78a9139f40946b92;hp=e2ad9ce9b5d25f89044eb95431177dad87a504b9;hb=c7ed812d6a87f0ffa3c0d4c1a9289497f680ccec;hpb=8f9ab85d3c79f06c9db6b7b44368857a4714d455;ds=sidebyside diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index e2ad9ce..be82fec 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1,28 +1,43 @@ #!/bin/bash -trap 'print_summary && touch $TF_FAIL && \ - echo "test-framework exiting on error"' ERR +trap 'print_summary && print_stack_trace | tee $TF_FAIL && \ + echo "$TESTSUITE: FAIL: test-framework exiting on error"' ERR set -e -#set -x export LANG=en_US export REFORMAT=${REFORMAT:-""} export WRITECONF=${WRITECONF:-""} export VERBOSE=${VERBOSE:-false} -export GSS=false +export GSS=${GSS:-false} +export GSS_SK=${GSS_SK:-false} export GSS_KRB5=false export GSS_PIPEFS=false +export SHARED_KEY=${SHARED_KEY:-false} +export SK_PATH=${SK_PATH:-/tmp/test-framework-keys} +export SK_OM_PATH=$SK_PATH'/tmp-request-mount' +export SK_MOUNTED=${SK_MOUNTED:-false} +export SK_FLAVOR=${SK_FLAVOR:-ski} +export SK_NO_KEY=${SK_NO_KEY:-true} +export SK_UNIQUE_NM=${SK_UNIQUE_NM:-false} +export SK_S2S=${SK_S2S:-false} +export SK_S2SNM=${SK_S2SNM:-TestFrameNM} +export SK_S2SNMCLI=${SK_S2SNMCLI:-TestFrameNMCli} export IDENTITY_UPCALL=default export QUOTA_AUTO=1 # specify environment variable containing batch job name for server statistics export JOBID_VAR=${JOBID_VAR:-"procname_uid"} # or "existing" or "disable" -# LOAD_LLOOP: LU-409: only load llite_lloop module if kernel < 2.6.32 or -# LOAD_LLOOP is true. LOAD_LLOOP is false by default. -export LOAD_LLOOP=${LOAD_LLOOP:-false} - #export PDSH="pdsh -S -Rssh -w" export MOUNT_CMD=${MOUNT_CMD:-"mount -t lustre"} +export UMOUNT=${UMOUNT:-"umount -d"} + +export LSNAPSHOT_CONF="/etc/ldev.conf" +export LSNAPSHOT_LOG="/var/log/lsnapshot.log" + +# sles12 umount has a issue with -d option +[ -e /etc/SuSE-release ] && grep -w VERSION /etc/SuSE-release | grep -wq 12 && { + export UMOUNT="umount" +} # function used by scripts run on remote nodes LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} @@ -49,16 +64,25 @@ fi [ -z "$MODPROBECONF" -a -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf +sanitize_parameters() { + for i in DIR DIR1 DIR2 MOUNT MOUNT1 MOUNT2 + do + local path=${!i} + if [ -d "$path" ]; then + eval export $i=$(echo $path | sed -r 's/\/+$//g') + fi + done +} assert_DIR () { - local failed="" - [[ $DIR/ = $MOUNT/* ]] || \ - { failed=1 && echo "DIR=$DIR not in $MOUNT. Aborting."; } - [[ $DIR1/ = $MOUNT1/* ]] || \ - { failed=1 && echo "DIR1=$DIR1 not in $MOUNT1. Aborting."; } - [[ $DIR2/ = $MOUNT2/* ]] || \ - { failed=1 && echo "DIR2=$DIR2 not in $MOUNT2. Aborting"; } + local failed="" + [[ $DIR/ = $MOUNT/* ]] || + { failed=1 && echo "DIR=$DIR not in $MOUNT. Aborting."; } + [[ $DIR1/ = $MOUNT1/* ]] || + { failed=1 && echo "DIR1=$DIR1 not in $MOUNT1. Aborting."; } + [[ $DIR2/ = $MOUNT2/* ]] || + { failed=1 && echo "DIR2=$DIR2 not in $MOUNT2. Aborting"; } - [ -n "$failed" ] && exit 99 || true + [ -n "$failed" ] && exit 99 || true } usage() { @@ -71,7 +95,6 @@ usage() { print_summary () { trap 0 [ -z "$DEFAULT_SUITES"] && return 0 - [ "$TESTSUITE" == "lfsck" ] && return 0 [ -n "$ONLY" ] && echo "WARNING: ONLY is set to $(echo $ONLY)" local details local form="%-13s %-17s %-9s %s %s\n" @@ -129,60 +152,62 @@ init_test_env() { export TEST_FAILED=false export FAIL_ON_SKIP_ENV=${FAIL_ON_SKIP_ENV:-false} export RPC_MODE=${RPC_MODE:-false} + export DO_CLEANUP=${DO_CLEANUP:-true} + export KEEP_ZPOOL=${KEEP_ZPOOL:-false} - export MKE2FS=$MKE2FS - if [ -z "$MKE2FS" ]; then - if which mkfs.ldiskfs >/dev/null 2>&1; then - export MKE2FS=mkfs.ldiskfs - else - export MKE2FS=mke2fs - fi - fi + export MKE2FS=$MKE2FS + if [ -z "$MKE2FS" ]; then + if which mkfs.ldiskfs >/dev/null 2>&1; then + export MKE2FS=mkfs.ldiskfs + else + export MKE2FS=mke2fs + fi + fi - export DEBUGFS=$DEBUGFS - if [ -z "$DEBUGFS" ]; then - if which debugfs.ldiskfs >/dev/null 2>&1; then - export DEBUGFS=debugfs.ldiskfs - else - export DEBUGFS=debugfs - fi - fi + export DEBUGFS=$DEBUGFS + if [ -z "$DEBUGFS" ]; then + if which debugfs.ldiskfs >/dev/null 2>&1; then + export DEBUGFS=debugfs.ldiskfs + else + export DEBUGFS=debugfs + fi + fi - export TUNE2FS=$TUNE2FS - if [ -z "$TUNE2FS" ]; then - if which tunefs.ldiskfs >/dev/null 2>&1; then - export TUNE2FS=tunefs.ldiskfs - else - export TUNE2FS=tune2fs - fi - fi + export TUNE2FS=$TUNE2FS + if [ -z "$TUNE2FS" ]; then + if which tunefs.ldiskfs >/dev/null 2>&1; then + export TUNE2FS=tunefs.ldiskfs + else + export TUNE2FS=tune2fs + fi + fi - export E2LABEL=$E2LABEL - if [ -z "$E2LABEL" ]; then - if which label.ldiskfs >/dev/null 2>&1; then - export E2LABEL=label.ldiskfs - else - export E2LABEL=e2label - fi - fi + export E2LABEL=$E2LABEL + if [ -z "$E2LABEL" ]; then + if which label.ldiskfs >/dev/null 2>&1; then + export E2LABEL=label.ldiskfs + else + export E2LABEL=e2label + fi + fi - export DUMPE2FS=$DUMPE2FS - if [ -z "$DUMPE2FS" ]; then - if which dumpfs.ldiskfs >/dev/null 2>&1; then - export DUMPE2FS=dumpfs.ldiskfs - else - export DUMPE2FS=dumpe2fs - fi - fi + export DUMPE2FS=$DUMPE2FS + if [ -z "$DUMPE2FS" ]; then + if which dumpfs.ldiskfs >/dev/null 2>&1; then + export DUMPE2FS=dumpfs.ldiskfs + else + export DUMPE2FS=dumpe2fs + fi + fi - export E2FSCK=$E2FSCK - if [ -z "$E2FSCK" ]; then - if which fsck.ldiskfs >/dev/null 2>&1; then - export E2FSCK=fsck.ldiskfs - else - export E2FSCK=e2fsck - fi - fi + export E2FSCK=$E2FSCK + if [ -z "$E2FSCK" ]; then + if which fsck.ldiskfs >/dev/null 2>&1; then + export E2FSCK=fsck.ldiskfs + else + export E2FSCK=e2fsck + fi + fi export RESIZE2FS=$RESIZE2FS if [ -z "$RESIZE2FS" ]; then @@ -193,124 +218,137 @@ init_test_env() { fi fi - export LFSCK_BIN=${LFSCK_BIN:-lfsck} - export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite - export FSCK_MAX_ERR=4 # File system errors left uncorrected + export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after test suite + export FSCK_MAX_ERR=4 # File system errors left uncorrected export ZFS=${ZFS:-zfs} export ZPOOL=${ZPOOL:-zpool} export ZDB=${ZDB:-zdb} export PARTPROBE=${PARTPROBE:-partprobe} - #[ -d /r ] && export ROOT=${ROOT:-/r} - export TMP=${TMP:-$ROOT/tmp} - export TESTSUITELOG=${TMP}/${TESTSUITE}.log - export LOGDIR=${LOGDIR:-${TMP}/test_logs/$(date +%s)} - export TESTLOG_PREFIX=$LOGDIR/$TESTSUITE + #[ -d /r ] && export ROOT=${ROOT:-/r} + export TMP=${TMP:-$ROOT/tmp} + export TESTSUITELOG=${TMP}/${TESTSUITE}.log + export LOGDIR=${LOGDIR:-${TMP}/test_logs/$(date +%s)} + export TESTLOG_PREFIX=$LOGDIR/$TESTSUITE - export HOSTNAME=${HOSTNAME:-$(hostname -s)} - if ! echo $PATH | grep -q $LUSTRE/utils; then - export PATH=$LUSTRE/utils:$PATH - fi - if ! echo $PATH | grep -q $LUSTRE/utils/gss; then - export PATH=$LUSTRE/utils/gss:$PATH - fi - if ! echo $PATH | grep -q $LUSTRE/tests; then - export PATH=$LUSTRE/tests:$PATH - fi - if ! echo $PATH | grep -q $LUSTRE/../lustre-iokit/sgpdd-survey; then - export PATH=$LUSTRE/../lustre-iokit/sgpdd-survey:$PATH - fi - export LST=${LST:-"$LUSTRE/../lnet/utils/lst"} - [ ! -f "$LST" ] && export LST=$(which lst) - export SGPDDSURVEY=${SGPDDSURVEY:-"$LUSTRE/../lustre-iokit/sgpdd-survey/sgpdd-survey")} - [ ! -f "$SGPDDSURVEY" ] && export SGPDDSURVEY=$(which sgpdd-survey) + export HOSTNAME=${HOSTNAME:-$(hostname -s)} + if ! echo $PATH | grep -q $LUSTRE/utils; then + export PATH=$LUSTRE/utils:$PATH + fi + if ! echo $PATH | grep -q $LUSTRE/utils/gss; then + export PATH=$LUSTRE/utils/gss:$PATH + fi + if ! echo $PATH | grep -q $LUSTRE/tests; then + export PATH=$LUSTRE/tests:$PATH + fi + if ! echo $PATH | grep -q $LUSTRE/../lustre-iokit/sgpdd-survey; then + export PATH=$LUSTRE/../lustre-iokit/sgpdd-survey:$PATH + fi + export LST=${LST:-"$LUSTRE/../lnet/utils/lst"} + [ ! -f "$LST" ] && export LST=$(which lst) + export SGPDDSURVEY=${SGPDDSURVEY:-"$LUSTRE/../lustre-iokit/sgpdd-survey/sgpdd-survey")} + [ ! -f "$SGPDDSURVEY" ] && export SGPDDSURVEY=$(which sgpdd-survey) export MCREATE=${MCREATE:-mcreate} - # Ubuntu, at least, has a truncate command in /usr/bin - # so fully path our truncate command. - export TRUNCATE=${TRUNCATE:-$LUSTRE/tests/truncate} + # Ubuntu, at least, has a truncate command in /usr/bin + # so fully path our truncate command. + export TRUNCATE=${TRUNCATE:-$LUSTRE/tests/truncate} export FSX=${FSX:-$LUSTRE/tests/fsx} - export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mpi/mdsrate"} - [ ! -f "$MDSRATE" ] && export MDSRATE=$(which mdsrate 2> /dev/null) - if ! echo $PATH | grep -q $LUSTRE/tests/racer; then - export PATH=$LUSTRE/tests/racer:$PATH: - fi - if ! echo $PATH | grep -q $LUSTRE/tests/mpi; then - export PATH=$LUSTRE/tests/mpi:$PATH - fi - export RSYNC_RSH=${RSYNC_RSH:-rsh} - - export LCTL=${LCTL:-"$LUSTRE/utils/lctl"} - [ ! -f "$LCTL" ] && export LCTL=$(which lctl) - export LFS=${LFS:-"$LUSTRE/utils/lfs"} - [ ! -f "$LFS" ] && export LFS=$(which lfs) - SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"} - GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"} - - export L_GETIDENTITY=${L_GETIDENTITY:-"$LUSTRE/utils/l_getidentity"} - if [ ! -f "$L_GETIDENTITY" ]; then - if `which l_getidentity > /dev/null 2>&1`; then - export L_GETIDENTITY=$(which l_getidentity) - else - export L_GETIDENTITY=NONE - fi - fi - export LL_DECODE_FILTER_FID=${LL_DECODE_FILTER_FID:-"$LUSTRE/utils/ll_decode_filter_fid"} - [ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID="ll_decode_filter_fid" - export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"} - [ ! -f "$MKFS" ] && export MKFS="mkfs.lustre" - export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"} - [ ! -f "$TUNEFS" ] && export TUNEFS="tunefs.lustre" - export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} " - export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod} - [ ! -f "$LUSTRE_RMMOD" ] && - export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null) - export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate} - [ ! -f "$LFS_MIGRATE" ] && - export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null) - export LR_READER=${LR_READER:-"$LUSTRE/utils/lr_reader"} - [ ! -f "$LR_READER" ] && export LR_READER=$(which lr_reader 2> /dev/null) - [ -z "$LR_READER" ] && export LR_READER="/usr/sbin/lr_reader" - export NAME=${NAME:-local} - export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"} - [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && \ - export LGSSD=$(which lgssd) - export LSVCGSSD=${LSVCGSSD:-"$LUSTRE/utils/gss/lsvcgssd"} - [ ! -f "$LSVCGSSD" ] && export LSVCGSSD=$(which lsvcgssd 2> /dev/null) - export KRB5DIR=${KRB5DIR:-"/usr/kerberos"} - export DIR2 - export SAVE_PWD=${SAVE_PWD:-$LUSTRE/tests} - export AT_MAX_PATH - - if [ "$ACCEPTOR_PORT" ]; then - export PORT_OPT="--port $ACCEPTOR_PORT" - fi + export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mpi/mdsrate"} + [ ! -f "$MDSRATE" ] && export MDSRATE=$(which mdsrate 2> /dev/null) + if ! echo $PATH | grep -q $LUSTRE/tests/racer; then + export PATH=$LUSTRE/tests/racer:$PATH: + fi + if ! echo $PATH | grep -q $LUSTRE/tests/mpi; then + export PATH=$LUSTRE/tests/mpi:$PATH + fi + export RSYNC_RSH=${RSYNC_RSH:-rsh} + + export LCTL=${LCTL:-"$LUSTRE/utils/lctl"} + [ ! -f "$LCTL" ] && export LCTL=$(which lctl) + export LFS=${LFS:-"$LUSTRE/utils/lfs"} + [ ! -f "$LFS" ] && export LFS=$(which lfs) + SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"} + GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"} + + export L_GETIDENTITY=${L_GETIDENTITY:-"$LUSTRE/utils/l_getidentity"} + if [ ! -f "$L_GETIDENTITY" ]; then + if `which l_getidentity > /dev/null 2>&1`; then + export L_GETIDENTITY=$(which l_getidentity) + else + export L_GETIDENTITY=NONE + fi + fi + export LL_DECODE_FILTER_FID=${LL_DECODE_FILTER_FID:-"$LUSTRE/utils/ll_decode_filter_fid"} + [ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID="ll_decode_filter_fid" + export LL_DECODE_LINKEA=${LL_DECODE_LINKEA:-"$LUSTRE/utils/ll_decode_linkea"} + [ ! -f "$LL_DECODE_LINKEA" ] && export LL_DECODE_LINKEA="ll_decode_linkea" + export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"} + [ ! -f "$MKFS" ] && export MKFS="mkfs.lustre" + export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"} + [ ! -f "$TUNEFS" ] && export TUNEFS="tunefs.lustre" + export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} " + export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod} + [ ! -f "$LUSTRE_RMMOD" ] && + export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null) + export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate} + [ ! -f "$LFS_MIGRATE" ] && + export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null) + export LR_READER=${LR_READER:-"$LUSTRE/utils/lr_reader"} + [ ! -f "$LR_READER" ] && + export LR_READER=$(which lr_reader 2> /dev/null) + [ -z "$LR_READER" ] && export LR_READER="/usr/sbin/lr_reader" + export NAME=${NAME:-local} + export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"} + [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && + export LGSSD=$(which lgssd) + export LSVCGSSD=${LSVCGSSD:-"$LUSTRE/utils/gss/lsvcgssd"} + [ ! -f "$LSVCGSSD" ] && export LSVCGSSD=$(which lsvcgssd 2> /dev/null) + export KRB5DIR=${KRB5DIR:-"/usr/kerberos"} + export DIR2 + export SAVE_PWD=${SAVE_PWD:-$LUSTRE/tests} + export AT_MAX_PATH + export LDEV=${LDEV:-"$LUSTRE/scripts/ldev"} + [ ! -f "$LDEV" ] && export LDEV=$(which ldev 2> /dev/null) + + if [ "$ACCEPTOR_PORT" ]; then + export PORT_OPT="--port $ACCEPTOR_PORT" + fi + + if $SHARED_KEY; then + $RPC_MODE || echo "Using GSS shared-key feature" + which lgss_sk > /dev/null 2>&1 || + error_exit "built with lgss_sk disabled! SEC=$SEC" + GSS=true + GSS_SK=true + SEC=$SK_FLAVOR + fi + + case "x$SEC" in + xkrb5*) + $RPC_MODE || echo "Using GSS/krb5 ptlrpc security flavor" + which lgss_keyring > /dev/null 2>&1 || + error_exit "built with gss disabled! SEC=$SEC" + GSS=true + GSS_KRB5=true + ;; + esac + + case "x$IDUP" in + xtrue) + IDENTITY_UPCALL=true + ;; + xfalse) + IDENTITY_UPCALL=false + ;; + esac + + export LOAD_MODULES_REMOTE=${LOAD_MODULES_REMOTE:-false} - case "x$SEC" in - xkrb5*) - echo "Using GSS/krb5 ptlrpc security flavor" - which lgss_keyring > /dev/null 2>&1 || \ - error_exit "built with gss disabled! SEC=$SEC" - GSS=true - GSS_KRB5=true - ;; - esac - - case "x$IDUP" in - xtrue) - IDENTITY_UPCALL=true - ;; - xfalse) - IDENTITY_UPCALL=false - ;; - esac - - export LOAD_MODULES_REMOTE=${LOAD_MODULES_REMOTE:-false} - - # Paths on remote nodes, if different - export RLUSTRE=${RLUSTRE:-$LUSTRE} - export RPWD=${RPWD:-$PWD} - export I_MOUNTED=${I_MOUNTED:-"no"} + # Paths on remote nodes, if different + export RLUSTRE=${RLUSTRE:-$LUSTRE} + export RPWD=${RPWD:-$PWD} + export I_MOUNTED=${I_MOUNTED:-"no"} if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \ ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \ ! -f /lib/modules/$(uname -r)/extra/kernel/fs/lustre/mdt.ko -a \ @@ -321,20 +359,20 @@ init_test_env() { export SHUTDOWN_ATTEMPTS=${SHUTDOWN_ATTEMPTS:-3} export OSD_TRACK_DECLARES_LBUG=${OSD_TRACK_DECLARES_LBUG:-"yes"} - # command line + # command line - while getopts "rvwf:" opt $*; do - case $opt in - f) CONFIG=$OPTARG;; - r) REFORMAT=--reformat;; - v) VERBOSE=true;; - w) WRITECONF=writeconf;; - \?) usage;; - esac - done + while getopts "rvwf:" opt $*; do + case $opt in + f) CONFIG=$OPTARG;; + r) REFORMAT=yes;; + v) VERBOSE=true;; + w) WRITECONF=writeconf;; + \?) usage;; + esac + done - shift $((OPTIND - 1)) - ONLY=${ONLY:-$*} + shift $((OPTIND - 1)) + ONLY=${ONLY:-$*} # print the durations of each test if "true" DDETAILS=${DDETAILS:-false} @@ -378,30 +416,39 @@ export LINUX_VERSION_CODE=$(version_code ${LINUX_VERSION//\./ }) # # All Lustre versions support "lctl get_param" to report the version of the # code running in the kernel (what our tests are interested in), but it -# doesn't work without modules loaded. If that fails, use "lctl version" -# instead, which is easy to parse and works without the kernel modules, -# but was only added in 2.6.50. If that also fails, fall back to calling -# "lctl lustre_build_version" which prints either (or both) the userspace -# and kernel build versions, but is deprecated and should eventually be -# removed. +# doesn't work without modules loaded. After 2.9.53 and in upstream kernels +# the "version" parameter doesn't include "lustre: " at the beginning. +# If that fails, call "lctl lustre_build_version" which prints either (or both) +# the userspace and kernel build versions, but until 2.8.55 required root +# access to get the Lustre kernel version. If that also fails, fall back to +# using "lctl --version", which is easy to parse and works without the kernel +# modules, but was only added in 2.6.50 and only prints the lctl tool version, +# not the module version, though they are usually the same. +# +# Various commands and their output format for different Lustre versions: +# lctl get_param version: 2.9.55 +# lctl get_param version: lustre: 2.8.53 +# lctl get_param version: lustre: 2.6.52 +# kernel: patchless_client +# build: v2_6_92_0-2.6.32-431.el6_lustre.x86_64 +# lctl lustre_build_version: Lustre version: 2.8.53_27_gae67fc01 +# lctl lustre_build_version: error: lustre_build_version: Permission denied +# (as non-root user) lctl version: v2_6_92_0-2.6.32-431.el6.x86_64 +# lctl lustre_build_version: Lustre version: 2.5.3-2.6.32.26-175.fc12.x86_64 +# lctl version: 2.5.3-2.6.32..26-175fc12.x86_64 +# lctl --version: lctl 2.6.50 # -# output: prints version string to stdout in dotted-decimal format +# output: prints version string to stdout in (up to 4) dotted-decimal values lustre_build_version() { local facet=${1:-client} + local ver=$(do_facet $facet "$LCTL get_param -n version 2>/dev/null || + $LCTL lustre_build_version 2>/dev/null || + $LCTL --version 2>/dev/null | cut -d' ' -f2") + local lver=$(egrep -i "lustre: |version: " <<<"$ver" | head -n 1) + [ -n "$lver" ] && ver="$lver" - # lustre: 2.6.52 - # kernel: patchless_client - # build: v2_6_92_0-gadb3ee4-2.6.32-431.29.2.el6_lustre.x86_64 - local VER=$(do_facet $facet $LCTL get_param -n version 2> /dev/null | - awk '/lustre: / { print $2 }') - # lctl 2.6.50 - [ -z "$VER" ] && VER=$(do_facet $facet $LCTL --version 2>/dev/null | - awk '{ print $2 }') - # Lustre version: 2.5.3-gfcfd782-CHANGED-2.6.32.26-175.fc12.x86_64 - # lctl version: 2.5.3-gfcfd782-CHANGED-2.6.32.26-175.fc12.x86_64 - [ -z "$VER" ] && VER=$(do_facet $facet $LCTL lustre_build_version | - awk '/version:/ { print $3; exit; }') - sed -e 's/^v//' -e 's/-.*//' -e 's/_/./g' <<<$VER + sed -e 's/[^:]*: //' -e 's/^v//' -e 's/[ -].*//' -e 's/_/./g' <<<$ver | + cut -d. -f1-4 } # Report the Lustre numeric build version code for the supplied facet. @@ -413,6 +460,33 @@ module_loaded () { /sbin/lsmod | grep -q "^\<$1\>" } +PRLFS=false +lustre_insmod() { + local module=$1 + shift + local args="$@" + local msg + local rc=0 + + if ! $PRLFS; then + msg="$(insmod $module $args 2>&1)" && return 0 || rc=$? + fi + + # parallels can't load modules directly from prlfs, use /tmp instead + if $PRLFS || [[ "$(stat -f -c%t $module)" == "7c7c6673" ]]; then + local target="$(mktemp)" + + cp "$module" "$target" + insmod $target $args + rc=$? + [[ $rc == 0 ]] && PRLFS=true + rm -f $target + else + echo "$msg" + fi + return $rc +} + # Load a module on the system where this is running. # # usage: load_module module_name [module arguments for insmod/modprobe] @@ -465,10 +539,10 @@ load_module() { # we're passing options on the command-line. if [[ "$BASE" == "lnet_selftest" ]] && [[ -f ${LUSTRE}/../lnet/selftest/${module}${EXT} ]]; then - insmod ${LUSTRE}/../lnet/selftest/${module}${EXT} + lustre_insmod ${LUSTRE}/../lnet/selftest/${module}${EXT} elif [[ -f ${LUSTRE}/${module}${EXT} ]]; then [[ "$BASE" != "ptlrpc_gss" ]] || modprobe sunrpc - insmod ${LUSTRE}/${module}${EXT} "$@" + lustre_insmod ${LUSTRE}/${module}${EXT} "$@" else # must be testing a "make install" or "rpm" installation # note failed to load ptlrpc_gss is considered not fatal @@ -481,20 +555,6 @@ load_module() { fi } -llite_lloop_enabled() { - local n1=$(uname -r | cut -d. -f1) - local n2=$(uname -r | cut -d. -f2) - local n3=$(uname -r | cut -d- -f1 | cut -d. -f3) - - # load the llite_lloop module for < 2.6.32 kernels - if [[ $n1 -lt 2 ]] || [[ $n1 -eq 2 && $n2 -lt 6 ]] || \ - [[ $n1 -eq 2 && $n2 -eq 6 && $n3 -lt 32 ]] || \ - $LOAD_LLOOP; then - return 0 - fi - return 1 -} - load_modules_local() { if [ -n "$MODPROBE" ]; then # use modprobe @@ -533,44 +593,54 @@ load_modules_local() { fi load_module ../libcfs/libcfs/libcfs - - [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG" - [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }" - load_module ../lnet/lnet/lnet - case $NETTYPE in - o2ib) - LNETLND="o2iblnd/ko2iblnd" - ;; - *) - ;; - esac - LNETLND=${LNETLND:-"socklnd/ksocklnd"} - load_module ../lnet/klnds/$LNETLND - load_module obdclass/obdclass - load_module ptlrpc/ptlrpc - load_module ptlrpc/gss/ptlrpc_gss - load_module fld/fld - load_module fid/fid - load_module lmv/lmv - load_module mdc/mdc - load_module osc/osc - load_module lov/lov - load_module mgc/mgc - load_module obdecho/obdecho - if ! client_only; then - SYMLIST=/proc/kallsyms - grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; } - grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; } - grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; } + # Prevent local MODOPTS_LIBCFS being passed as part of environment + # variable to remote nodes + unset MODOPTS_LIBCFS + + set_default_debug + load_module ../lnet/lnet/lnet + + LNDPATH=${LNDPATH:-"../lnet/klnds"} + if [ -z "$LNETLND" ]; then + case $NETTYPE in + o2ib*) LNETLND="o2iblnd/ko2iblnd" ;; + tcp*) LNETLND="socklnd/ksocklnd" ;; + *) local lnd="${NETTYPE%%[0-9]}lnd" + [ -f "$LNDPATH/$lnd/k$lnd.ko" ] && + LNETLND="$lnd/k$lnd" || + LNETLND="socklnd/ksocklnd" + esac + fi + load_module ../lnet/klnds/$LNETLND + load_module obdclass/obdclass + load_module ptlrpc/ptlrpc + load_module ptlrpc/gss/ptlrpc_gss + load_module fld/fld + load_module fid/fid + load_module lmv/lmv + load_module osc/osc + load_module mdc/mdc + load_module lov/lov + load_module mgc/mgc + load_module obdecho/obdecho + if ! client_only; then + SYMLIST=/proc/kallsyms + grep -q crc16 $SYMLIST || + { modprobe crc16 2>/dev/null || true; } + grep -q -w jbd2 $SYMLIST || + { modprobe jbd2 2>/dev/null || true; } load_module lfsck/lfsck - [ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS + [ "$LQUOTA" != "no" ] && + load_module quota/lquota $LQUOTAOPTS if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then - modprobe zfs + lsmod | grep zfs >&/dev/null || modprobe zfs load_module osd-zfs/osd_zfs fi if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then grep -q exportfs_decode_fh $SYMLIST || { modprobe exportfs 2> /dev/null || true; } + grep -q -w mbcache $SYMLIST || + { modprobe mbcache 2>/dev/null || true; } load_module ../ldiskfs/ldiskfs load_module osd-ldiskfs/osd_ldiskfs fi @@ -582,10 +652,9 @@ load_modules_local() { load_module osp/osp load_module ofd/ofd load_module osp/osp - fi + fi load_module llite/lustre - llite_lloop_enabled && load_module llite/llite_lloop [ -d /r ] && OGDB=${OGDB:-"/r/tmp"} OGDB=${OGDB:-$TMP} rm -f $OGDB/ogdb-$HOSTNAME @@ -677,7 +746,19 @@ fs_log_size() { local size=0 case $fstype in ldiskfs) size=50;; # largest seen is 44, leave some headroom - zfs) size=400;; # largest seen is 384 + zfs) size=512;; # largest seen is 512 + esac + + echo -n $size +} + +fs_inode_ksize() { + local facet=${1:-$SINGLEMDS} + local fstype=$(facet_fstype $facet) + local size=0 + case $fstype in + ldiskfs) size=4;; # ~4KB per inode + zfs) size=11;; # 10 to 11KB per inode esac echo -n $size @@ -713,88 +794,234 @@ send_sigint() { do_nodes $list "killall -2 $@ 2>/dev/null || true" } -# start gss daemons on all nodes, or -# "daemon" on "list" if set +# start gss daemons on all nodes, or "daemon" on "nodes" if set start_gss_daemons() { - local list=$1 - local daemon=$2 + local nodes=$1 + local daemon=$2 - if [ "$list" ] && [ "$daemon" ] ; then - echo "Starting gss daemon on nodes: $list" - do_nodes $list "$daemon" || return 8 - return 0 - fi + if [ "$nodes" ] && [ "$daemon" ] ; then + echo "Starting gss daemon on nodes: $nodes" + do_nodes $nodes "$daemon" || return 8 + return 0 + fi - local list=$(comma_list $(mdts_nodes)) - echo "Starting gss daemon on mds: $list" - do_nodes $list "$LSVCGSSD -v" || return 1 - if $GSS_PIPEFS; then - do_nodes $list "$LGSSD -v" || return 2 - fi + nodes=$(comma_list $(mdts_nodes)) + echo "Starting gss daemon on mds: $nodes" + if $GSS_SK; then + # Start all versions, in case of switching + do_nodes $nodes "$LSVCGSSD -vvv -s -m -o -z" || return 1 + else + do_nodes $nodes "$LSVCGSSD -v" || return 1 + fi + if $GSS_PIPEFS; then + do_nodes $nodes "$LGSSD -v" || return 2 + fi - list=$(comma_list $(osts_nodes)) - echo "Starting gss daemon on ost: $list" - do_nodes $list "$LSVCGSSD -v" || return 3 - # starting on clients + nodes=$(comma_list $(osts_nodes)) + echo "Starting gss daemon on ost: $nodes" + if $GSS_SK; then + # Start all versions, in case of switching + do_nodes $nodes "$LSVCGSSD -vvv -s -m -o -z" || return 3 + else + do_nodes $nodes "$LSVCGSSD -v" || return 3 + fi + # starting on clients - local clients=${CLIENTS:-`hostname`} - if $GSS_PIPEFS; then - echo "Starting $LGSSD on clients $clients " - do_nodes $clients "$LGSSD -v" || return 4 - fi + local clients=${CLIENTS:-$HOSTNAME} + if $GSS_PIPEFS; then + echo "Starting $LGSSD on clients $clients " + do_nodes $clients "$LGSSD -v" || return 4 + fi - # wait daemons entering "stable" status - sleep 5 - - # - # check daemons are running - # - list=$(comma_list $(mdts_nodes) $(osts_nodes)) - check_gss_daemon_nodes $list lsvcgssd || return 5 - if $GSS_PIPEFS; then - list=$(comma_list $(mdts_nodes)) - check_gss_daemon_nodes $list lgssd || return 6 - fi - if $GSS_PIPEFS; then - check_gss_daemon_nodes $clients lgssd || return 7 - fi + # wait daemons entering "stable" status + sleep 5 + + # + # check daemons are running + # + nodes=$(comma_list $(mdts_nodes) $(osts_nodes)) + check_gss_daemon_nodes $nodes lsvcgssd || return 5 + if $GSS_PIPEFS; then + nodes=$(comma_list $(mdts_nodes)) + check_gss_daemon_nodes $nodes lgssd || return 6 + fi + if $GSS_PIPEFS; then + check_gss_daemon_nodes $clients lgssd || return 7 + fi } stop_gss_daemons() { - local list=$(comma_list $(mdts_nodes)) + local nodes=$(comma_list $(mdts_nodes)) - send_sigint $list lsvcgssd lgssd + send_sigint $nodes lsvcgssd lgssd - list=$(comma_list $(osts_nodes)) - send_sigint $list lsvcgssd + nodes=$(comma_list $(osts_nodes)) + send_sigint $nodes lsvcgssd - list=${CLIENTS:-`hostname`} - send_sigint $list lgssd + nodes=${CLIENTS:-$HOSTNAME} + send_sigint $nodes lgssd +} + +add_sk_mntflag() { + # Add mount flags for shared key + local mt_opts=$@ + if grep -q skpath <<< "$mt_opts" ; then + mt_opts=$(echo $mt_opts | + sed -e "s#skpath=[^ ,]*#skpath=$SK_PATH#") + else + if [ -z "$mt_opts" ]; then + mt_opts="-o skpath=$SK_PATH" + else + mt_opts="$mt_opts,skpath=$SK_PATH" + fi + fi + echo -n $mt_opts } init_gss() { - if $GSS; then - if ! module_loaded ptlrpc_gss; then - load_module ptlrpc/gss/ptlrpc_gss - module_loaded ptlrpc_gss || - error_exit "init_gss : GSS=$GSS, but gss/krb5 is not supported!" - fi - if $GSS_KRB5; then - start_gss_daemons || error_exit "start gss daemon failed! rc=$?" - fi + if $SHARED_KEY; then + GSS=true + GSS_SK=true + fi + + if ! $GSS; then + return + fi + + if ! module_loaded ptlrpc_gss; then + load_module ptlrpc/gss/ptlrpc_gss + module_loaded ptlrpc_gss || + error_exit "init_gss: GSS=$GSS, but gss/krb5 missing" + fi + + if $GSS_KRB5 || $GSS_SK; then + start_gss_daemons || error_exit "start gss daemon failed! rc=$?" + fi + + if $GSS_SK && $SK_NO_KEY; then + local numclients=${1:-$CLIENTCOUNT} + local clients=${CLIENTS:-$HOSTNAME} - if [ -n "$LGSS_KEYRING_DEBUG" ]; then + # security ctx config for keyring + SK_NO_KEY=false + mkdir -p $SK_OM_PATH + mount -o bind $SK_OM_PATH /etc/request-key.d/ + local lgssc_conf_line='create lgssc * * ' + lgssc_conf_line+=$(which lgss_keyring) + lgssc_conf_line+=' %o %k %t %d %c %u %g %T %P %S' + + local lgssc_conf_file="/etc/request-key.d/lgssc.conf" + echo "$lgssc_conf_line" > $lgssc_conf_file + [ -e $lgssc_conf_file ] || + error_exit "Could not find key options in $lgssc_conf_file" + + if ! local_mode; then + do_nodes $(comma_list $(all_nodes)) "mkdir -p \ + $SK_OM_PATH" + do_nodes $(comma_list $(all_nodes)) "mount \ + -o bind $SK_OM_PATH \ + /etc/request-key.d/" + do_nodes $(comma_list $(all_nodes)) "rsync -aqv \ + $HOSTNAME:$lgssc_conf_file \ + $lgssc_conf_file >/dev/null 2>&1" + fi + + # create shared key on all nodes + mkdir -p $SK_PATH/nodemap + rm -f $SK_PATH/$FSNAME.key $SK_PATH/nodemap/c*.key \ + $SK_PATH/$FSNAME-*.key + # for nodemap testing each client may need own key, + # and S2S now requires keys as well, both for "client" + # and for "server" + if $SK_S2S; then + lgss_sk -t server -f$FSNAME -n $SK_S2SNMCLI \ + -w $SK_PATH/$FSNAME-nmclient.key \ + -d /dev/urandom >/dev/null 2>&1 + lgss_sk -t mgs,server -f$FSNAME -n $SK_S2SNM \ + -w $SK_PATH/$FSNAME-s2s-server.key \ + -d /dev/urandom >/dev/null 2>&1 + fi + # basic key create + lgss_sk -t server -f$FSNAME -w $SK_PATH/$FSNAME.key \ + -d /dev/urandom >/dev/null 2>&1 + # per-nodemap keys + for i in $(seq 0 $((numclients - 1))); do + lgss_sk -t server -f$FSNAME -n c$i \ + -w $SK_PATH/nodemap/c$i.key -d /dev/urandom \ + >/dev/null 2>&1 + done + # Distribute keys + if ! local_mode; then + do_nodes $(comma_list $(all_nodes)) "rsync -av \ + $HOSTNAME:$SK_PATH/ $SK_PATH >/dev/null 2>&1" + fi + # Set client keys to client type to generate prime P + if local_mode; then + do_nodes $(all_nodes) "lgss_sk -t client,server -m \ + $SK_PATH/$FSNAME.key >/dev/null 2>&1" + else + do_nodes $clients "lgss_sk -t client -m \ + $SK_PATH/$FSNAME.key >/dev/null 2>&1" + do_nodes $clients "find $SK_PATH/nodemap -name \*.key | \ + xargs -IX lgss_sk -t client -m X >/dev/null 2>&1" + fi + # This is required for servers as well, if S2S in use + if $SK_S2S; then + do_nodes $(comma_list $(mdts_nodes)) \ + "cp $SK_PATH/$FSNAME-s2s-server.key \ + $SK_PATH/$FSNAME-s2s-client.key; lgss_sk \ + -t client -m $SK_PATH/$FSNAME-s2s-client.key \ + >/dev/null 2>&1" + do_nodes $(comma_list $(osts_nodes)) \ + "cp $SK_PATH/$FSNAME-s2s-server.key \ + $SK_PATH/$FSNAME-s2s-client.key; lgss_sk \ + -t client -m $SK_PATH/$FSNAME-s2s-client.key \ + >/dev/null 2>&1" + do_nodes $clients "lgss_sk -t client \ + -m $SK_PATH/$FSNAME-nmclient.key \ + >/dev/null 2>&1" + fi + # mount options for servers and clients + MGS_MOUNT_OPTS=$(add_sk_mntflag $MGS_MOUNT_OPTS) + MDS_MOUNT_OPTS=$(add_sk_mntflag $MDS_MOUNT_OPTS) + OST_MOUNT_OPTS=$(add_sk_mntflag $OST_MOUNT_OPTS) + MOUNT_OPTS=$(add_sk_mntflag $MOUNT_OPTS) + SEC=$SK_FLAVOR + fi + + if [ -n "$LGSS_KEYRING_DEBUG" ]; then lctl set_param -n \ - sptlrpc.gss.lgss_keyring.debug_level=$LGSS_KEYRING_DEBUG - fi - fi + sptlrpc.gss.lgss_keyring.debug_level=$LGSS_KEYRING_DEBUG + fi } cleanup_gss() { - if $GSS; then - stop_gss_daemons - # maybe cleanup credential cache? - fi + if $GSS; then + stop_gss_daemons + # maybe cleanup credential cache? + fi +} + +cleanup_sk() { + if $GSS_SK; then + if $SK_S2S; then + do_node $(mgs_node) "$LCTL nodemap_del $SK_S2SNM" + do_node $(mgs_node) "$LCTL nodemap_del $SK_S2SNMCLI" + $RPC_MODE || echo "Sleeping for 10 sec for Nodemap.." + sleep 10 + fi + stop_gss_daemons + $RPC_MODE || echo "Cleaning up Shared Key.." + do_nodes $(comma_list $(all_nodes)) "rm -f \ + $SK_PATH/$FSNAME*.key $SK_PATH/nodemap/$FSNAME*.key" + # Remove the mount and clean up the files we added to SK_PATH + do_nodes $(comma_list $(all_nodes)) "umount \ + /etc/request-key.d/" + do_nodes $(comma_list $(all_nodes)) "rm -f \ + $SK_OM_PATH/lgssc.conf" + do_nodes $(comma_list $(all_nodes)) "rmdir $SK_OM_PATH" + SK_NO_KEY=true + fi } facet_svc() { @@ -814,7 +1041,7 @@ facet_type() { facet_number() { local facet=$1 - if [ $facet == mgs ]; then + if [ $facet == mgs ] || [ $facet == client ]; then return 1 fi @@ -961,6 +1188,20 @@ facet_vdevice() { echo -n $device } +running_in_vm() { + local virt=$(virt-what 2> /dev/null) + + [ $? -eq 0 ] && [ -n "$virt" ] && { echo $virt; return; } + + virt=$(dmidecode -s system-product-name | awk '{print $1}') + + case $virt in + VMware|KVM|VirtualBox|Parallels) + echo $virt | tr '[A-Z]' '[a-z]' ;; + *) ;; + esac +} + # # Re-read the partition table on failover partner host. # After a ZFS storage pool is created on a shared device, the partition table @@ -999,6 +1240,17 @@ zpool_name() { } # +# +# Get ZFS local fsname. +# +zfs_local_fsname() { + local facet=$1 + local lfsname=$(basename $(facet_device $facet)) + + echo -n $lfsname +} + +# # Create ZFS storage pool. # create_zpool() { @@ -1008,7 +1260,8 @@ create_zpool() { shift 3 local opts=${@:-"-o cachefile=none"} - do_facet $facet "$ZPOOL list -H $poolname >/dev/null 2>&1 || + do_facet $facet "lsmod | grep zfs >&/dev/null || modprobe zfs; + $ZPOOL list -H $poolname >/dev/null 2>&1 || $ZPOOL create -f $opts $poolname $vdev" } @@ -1074,12 +1327,27 @@ import_zpool() { if [[ -n "$poolname" ]]; then opts+=" -d $(dirname $(facet_vdevice $facet))" - do_facet $facet "$ZPOOL list -H $poolname >/dev/null 2>&1 || + do_facet $facet "lsmod | grep zfs >&/dev/null || modprobe zfs; + $ZPOOL list -H $poolname >/dev/null 2>&1 || $ZPOOL import -f $opts $poolname" fi } # +# Reimport ZFS storage pool with new name +# +reimport_zpool() { + local facet=$1 + local newpool=$2 + local opts="-o cachefile=none" + local poolname=$(zpool_name $facet) + + opts+=" -d $(dirname $(facet_vdevice $facet))" + do_facet $facet "$ZPOOL export $poolname; + $ZPOOL import $opts $poolname $newpool" +} + +# # Set the "cachefile=none" property on ZFS storage pool so that the pool # is not automatically imported on system startup. # @@ -1113,7 +1381,7 @@ get_osd_param() { local name=$3 do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \ - osd-*.$device.$name 2>&1" | grep -v 'Found no match' + osd-*.$device.$name 2>&1" | grep -v 'error:' } set_osd_param() { @@ -1123,7 +1391,7 @@ set_osd_param() { local value=$4 do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \ - osd-*.$device.$name=$value 2>&1" | grep -v 'Found no match' + osd-*.$device.$name=$value 2>&1" | grep -v 'error:' } set_debug_size () { @@ -1132,7 +1400,7 @@ set_debug_size () { if [ -f /sys/devices/system/cpu/possible ]; then local cpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/possible)+1)) else - local cpus=$(getconf _NPROCESSORS_CONF) + local cpus=$(getconf _NPROCESSORS_CONF 2>/dev/null) fi # bug 19944, adjust size to be -gt num_possible_cpus() @@ -1174,6 +1442,15 @@ set_default_debug_facet () { set_default_debug_nodes $node } +set_hostid () { + local hostid=${1:-$(hostid)} + + if [ ! -s /etc/hostid ]; then + printf $(echo -n $hostid | + sed 's/\(..\)\(..\)\(..\)\(..\)/\\x\4\\x\3\\x\2\\x\1/') >/etc/hostid + fi +} + # Facet functions mount_facets () { local facets=${1:-$(get_facets)} @@ -1220,6 +1497,10 @@ mount_facet() { local opt=${facet}_opt local mntpt=$(facet_mntpt $facet) local opts="${!opt} $@" + local fstype=$(facet_fstype $facet) + local devicelabel + + module_loaded lustre || load_modules if [ $(facet_fstype $facet) == ldiskfs ] && ! do_facet $facet test -b ${!dev}; then @@ -1231,6 +1512,16 @@ mount_facet() { import_zpool $facet || return ${PIPESTATUS[0]} fi + case $fstype in + ldiskfs) + devicelabel=$(do_facet ${facet} "$E2LABEL ${!dev}");; + zfs) + devicelabel=$(do_facet ${facet} "$ZFS get -H -o value \ + lustre:svname ${!dev}");; + *) + error "unknown fstype!";; + esac + echo "Starting ${facet}: $opts ${!dev} $mntpt" # for testing LU-482 error handling in mount_facets() and test_0a() if [ -f $TMP/test-lu482-trigger ]; then @@ -1240,40 +1531,74 @@ mount_facet() { ${!dev} $mntpt" RC=${PIPESTATUS[0]} fi + if [ $RC -ne 0 ]; then echo "Start of ${!dev} on ${facet} failed ${RC}" - else - set_default_debug_facet $facet + return $RC + fi + + health=$(do_facet ${facet} "$LCTL get_param -n health_check") + if [[ "$health" != "healthy" ]]; then + error "$facet is in a unhealthy state" + fi + + set_default_debug_facet $facet if [[ $facet == mds* ]]; then do_facet $facet \ - lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 \ - 2>/dev/null + lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 2>/dev/null fi - label=$(devicelabel ${facet} ${!dev}) - [ -z "$label" ] && echo no label for ${!dev} && exit 1 - eval export ${facet}_svc=${label} - echo Started ${label} - fi - return $RC + if [[ $opts =~ .*nosvc.* ]]; then + echo "Start ${!dev} without service" + else + + case $fstype in + ldiskfs) + wait_update_facet ${facet} "$E2LABEL ${!dev} \ + 2>/dev/null | grep -E ':[a-zA-Z]{3}[0-9]{4}'" \ + "" || error "${!dev} failed to initialize!";; + zfs) + wait_update_facet ${facet} "$ZFS get -H -o value \ + lustre:svname ${!dev} 2>/dev/null | \ + grep -E ':[a-zA-Z]{3}[0-9]{4}'" "" || + error "${!dev} failed to initialize!";; + + *) + error "unknown fstype!";; + esac + fi + + # commit the device label change to disk + if [[ $devicelabel =~ (:[a-zA-Z]{3}[0-9]{4}) ]]; then + echo "Commit the device label on ${!dev}" + do_facet $facet "sync; sleep 1; sync" + fi + + + label=$(devicelabel ${facet} ${!dev}) + [ -z "$label" ] && echo no label for ${!dev} && exit 1 + eval export ${facet}_svc=${label} + echo Started ${label} + + return $RC } # start facet device options start() { - local facet=$1 - shift - local device=$1 - shift - eval export ${facet}_dev=${device} - eval export ${facet}_opt=\"$@\" + local facet=$1 + shift + local device=$1 + shift + eval export ${facet}_dev=${device} + eval export ${facet}_opt=\"$@\" - local varname=${facet}failover_dev - if [ -n "${!varname}" ] ; then - eval export ${facet}failover_dev=${!varname} - else - eval export ${facet}failover_dev=$device - fi + local varname=${facet}failover_dev + if [ -n "${!varname}" ] ; then + eval export ${facet}failover_dev=${!varname} + else + eval export ${facet}failover_dev=$device + fi local mntpt=$(facet_mntpt $facet) do_facet ${facet} mkdir -p $mntpt @@ -1301,7 +1626,7 @@ stop() { running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts") || true if [ ${running} -ne 0 ]; then echo "Stopping $mntpt (opts:$@) on $HOST" - do_facet ${facet} umount -d $@ $mntpt + do_facet ${facet} $UMOUNT $@ $mntpt fi # umount should block, but we should wait for unrelated obd's @@ -1310,7 +1635,7 @@ stop() { if [[ $(facet_fstype $facet) == zfs ]]; then # export ZFS storage pool - export_zpool $facet + [ "$KEEP_ZPOOL" = "true" ] || export_zpool $facet fi } @@ -1354,70 +1679,6 @@ quota_type() { return $rc } -# XXX This function is kept for interoperability with old server (< 2.3.50), -# it should be removed whenever we drop the interoperability for such -# server. -restore_quota_old() { - local mntpt=${1:-$MOUNT} - local quota_type=$(quota_type $FSNAME | grep MDT | cut -d "=" -f2) - if [ ! "$old_QUOTA_TYPE" ] || - [ "$quota_type" = "$old_QUOTA_TYPE" ]; then - return - fi - quota_save_version $old_QUOTA_TYPE -} - -# XXX This function is kept for interoperability with old server (< 2.3.50), -# it should be removed whenever we drop the interoperability for such -# server. -setup_quota_old(){ - local mntpt=$1 - - # no quota enforcement for now and accounting works out of the box - return - - # We need save the original quota_type params, and restore them after testing - - # Suppose that quota type the same on mds and ost - local quota_type=$(quota_type | grep MDT | cut -d "=" -f2) - [ ${PIPESTATUS[0]} -eq 0 ] || error "quota_type failed!" - echo "[HOST:$HOSTNAME] [old_quota_type:$quota_type] [new_quota_type:$QUOTA_TYPE]" - if [ "$quota_type" != "$QUOTA_TYPE" ]; then - export old_QUOTA_TYPE=$quota_type - quota_save_version $QUOTA_TYPE - else - qtype=$(tr -c -d "ug" <<< $QUOTA_TYPE) - $LFS quotacheck -$qtype $mntpt || error "quotacheck has failed for $type" - fi - - local quota_usrs=$QUOTA_USERS - - # get_filesystem_size - local disksz=$(lfs_df $mntpt | grep "summary" | awk '{print $2}') - local blk_soft=$((disksz + 1024)) - local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over - - local Inodes=$(lfs_df -i $mntpt | grep "summary" | awk '{print $2}') - local i_soft=$Inodes - local i_hard=$((i_soft + i_soft / 20)) - - echo "Total disk size: $disksz block-softlimit: $blk_soft block-hardlimit: - $blk_hard inode-softlimit: $i_soft inode-hardlimit: $i_hard" - - local cmd - for usr in $quota_usrs; do - echo "Setting up quota on $HOSTNAME:$mntpt for $usr..." - for type in u g; do - cmd="$LFS setquota -$type $usr -b $blk_soft -B $blk_hard -i $i_soft -I $i_hard $mntpt" - echo "+ $cmd" - eval $cmd || error "$cmd FAILED!" - done - # display the quota status - echo "Quota settings for $usr : " - $LFS quota -v -u $usr $mntpt || true - done -} - # get mdt quota type mdt_quota_type() { local varsvc=${SINGLEMDS}_svc @@ -1435,11 +1696,6 @@ ost_quota_type() { # restore old quota type settings restore_quota() { - if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then - restore_quota_old - return - fi - if [ "$old_MDT_QUOTA_TYPE" ]; then do_facet mgs $LCTL conf_param \ $FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE @@ -1474,12 +1730,19 @@ mdt_free_inodes() { echo $free_inodes } +# +# Get the OST device status from 'lfs df' with a given OST index. +# +ost_dev_status() { + local ost_idx=$1 + local mnt_pnt=${2:-$MOUNT} + local ost_uuid + + ost_uuid=$(ostuuid_from_index $ost_idx $mnt_pnt) + lfs_df $mnt_pnt | awk '/'$ost_uuid'/ { print $7 }' +} + setup_quota(){ - if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then - setup_quota_old $1 - return - fi - local mntpt=$1 # save old quota type & set new quota type @@ -1528,32 +1791,57 @@ setup_quota(){ } zconf_mount() { - local client=$1 - local mnt=$2 - local opts=${3:-$MOUNT_OPTS} - opts=${opts:+-o $opts} - local flags=${4:-$MOUNT_FLAGS} - - local device=$MGSNID:/$FSNAME - if [ -z "$mnt" -o -z "$FSNAME" ]; then - echo Bad zconf mount command: opt=$flags $opts dev=$device mnt=$mnt - exit 1 - fi - - echo "Starting client: $client: $flags $opts $device $mnt" - do_node $client mkdir -p $mnt - do_node $client $MOUNT_CMD $flags $opts $device $mnt || return 1 + local client=$1 + local mnt=$2 + local opts=${3:-$MOUNT_OPTS} + opts=${opts:+-o $opts} + local flags=${4:-$MOUNT_FLAGS} + + local device=$MGSNID:/$FSNAME$FILESET + if [ -z "$mnt" -o -z "$FSNAME" ]; then + echo "Bad mount command: opt=$flags $opts dev=$device " \ + "mnt=$mnt" + exit 1 + fi + + echo "Starting client: $client: $flags $opts $device $mnt" + do_node $client mkdir -p $mnt + if [ -n "$FILESET" -a -z "$SKIP_FILESET" ];then + do_node $client $MOUNT_CMD $flags $opts $MGSNID:/$FSNAME \ + $mnt || return 1 + #disable FILESET if not supported + do_nodes $client lctl get_param -n \ + mdc.$FSNAME-MDT0000*.import | grep -q subtree || + device=$MGSNID:/$FSNAME + do_node $client mkdir -p $mnt/$FILESET + do_node $client "! grep -q $mnt' ' /proc/mounts || + umount $mnt" + fi + if $GSS_SK && ($SK_UNIQUE_NM || $SK_S2S); then + # Mount using nodemap key + local mountkey=$SK_PATH/$FSNAME-nmclient.key + if $SK_UNIQUE_NM; then + mountkey=$SK_PATH/nodemap/c0.key + fi + local prunedopts=$(echo $opts | + sed -e "s#skpath=[^,^ ]*#skpath=$mountkey#g") + do_node $client $MOUNT_CMD $flags $prunedopts $device $mnt || + return 1 + else + do_node $client $MOUNT_CMD $flags $opts $device $mnt || + return 1 + fi - set_default_debug_nodes $client + set_default_debug_nodes $client - return 0 + return 0 } zconf_umount() { local client=$1 local mnt=$2 local force - local busy + local busy local need_kill [ "$3" ] && force=-f @@ -1576,6 +1864,19 @@ zconf_umount() { fi } +# Mount the file system on the MGS +mount_mgs_client() { + do_facet mgs "mkdir -p $MOUNT" + zconf_mount $mgs_HOST $MOUNT $MOUNT_OPTS || + error "unable to mount $MOUNT on MGS" +} + +# Unmount the file system on the MGS +umount_mgs_client() { + zconf_umount $mgs_HOST $MOUNT + do_facet mgs "rm -rf $MOUNT" +} + # nodes is comma list sanity_mount_check_nodes () { local nodes=$1 @@ -1594,13 +1895,13 @@ if [ \\\$running -ne \\\$mpts ]; then echo \\\$(hostname) env are INSANE!; exit 1; fi" - [ $? -eq 0 ] || rc=1 + [ $? -eq 0 ] || rc=1 done return $rc } sanity_mount_check_servers () { - [ "$CLIENTONLY" ] && + [ -n "$CLIENTONLY" ] && { echo "CLIENTONLY mode, skip mount_check_servers"; return 0; } || true echo Checking servers environments @@ -1636,36 +1937,126 @@ sanity_mount_check () { # mount clients if not mouted zconf_mount_clients() { - local clients=$1 - local mnt=$2 - local opts=${3:-$MOUNT_OPTS} - opts=${opts:+-o $opts} - local flags=${4:-$MOUNT_FLAGS} - - local device=$MGSNID:/$FSNAME - if [ -z "$mnt" -o -z "$FSNAME" ]; then - echo Bad zconf mount command: opt=$flags $opts dev=$device mnt=$mnt - exit 1 - fi + local clients=$1 + local mnt=$2 + local opts=${3:-$MOUNT_OPTS} + opts=${opts:+-o $opts} + local flags=${4:-$MOUNT_FLAGS} + local device=$MGSNID:/$FSNAME$FILESET + if [ -z "$mnt" -o -z "$FSNAME" ]; then + echo "Bad conf mount command: opt=$flags $opts dev=$device " \ + "mnt=$mnt" + exit 1 + fi + + echo "Starting client $clients: $flags $opts $device $mnt" + if [ -n "$FILESET" -a ! -n "$SKIP_FILESET" ]; then + if $GSS_SK && ($SK_UNIQUE_NM || $SK_S2S); then + # Mount with own nodemap key + local i=0 + # Mount all server nodes first with per-NM keys + for nmclient in ${clients//,/ }; do +# do_nodes $(comma_list $(all_server_nodes)) "lgss_sk -t server -l $SK_PATH/nodemap/c$i.key -n c$i" + do_nodes $(comma_list $(all_server_nodes)) "lgss_sk -t server -l $SK_PATH/nodemap/c$i.key" + i=$((i + 1)) + done + # set perms for per-nodemap keys else permission denied + do_nodes $(comma_list $(all_nodes)) \ + "keyctl show | grep lustre | cut -c1-11 | + sed -e 's/ //g;' | + xargs -IX keyctl setperm X 0x3f3f3f3f" + local mountkey=$SK_PATH/$FSNAME-nmclient.key + i=0 + for nmclient in ${clients//,/ }; do + if $SK_UNIQUE_NM; then + mountkey=$SK_PATH/nodemap/c$i.key + fi + do_node $nmclient "! grep -q $mnt' ' \ + /proc/mounts || umount $mnt" + local prunedopts=$(add_sk_mntflag $prunedopts); + prunedopts=$(echo $prunedopts | sed -e \ + "s#skpath=[^ ^,]*#skpath=$mountkey#g") + set -x + do_nodes $(comma_list $(all_server_nodes)) \ + "keyctl show" + set +x + do_node $nmclient $MOUNT_CMD $flags \ + $prunedopts $MGSNID:/$FSNAME $mnt || + return 1 + i=$((i + 1)) + done + else + do_nodes $clients "! grep -q $mnt' ' /proc/mounts || + umount $mnt" + do_nodes $clients $MOUNT_CMD $flags $opts \ + $MGSNID:/$FSNAME $mnt || return 1 + fi + #disable FILESET if not supported + do_nodes $clients lctl get_param -n \ + mdc.$FSNAME-MDT0000*.import | grep -q subtree || + device=$MGSNID:/$FSNAME + do_nodes $clients mkdir -p $mnt/$FILESET + do_nodes $clients "! grep -q $mnt' ' /proc/mounts || + umount $mnt" + fi + + if $GSS_SK && ($SK_UNIQUE_NM || $SK_S2S); then + # Mount with nodemap key + local i=0 + local mountkey=$SK_PATH/$FSNAME-nmclient.key + for nmclient in ${clients//,/ }; do + if $SK_UNIQUE_NM; then + mountkey=$SK_PATH/nodemap/c$i.key + fi + local prunedopts=$(echo $opts | sed -e \ + "s#skpath=[^ ^,]*#skpath=$mountkey#g"); + do_node $nmclient "! grep -q $mnt' ' /proc/mounts || + umount $mnt" + do_node $nmclient " + running=\\\$(mount | grep -c $mnt' '); + rc=0; + if [ \\\$running -eq 0 ] ; then + mkdir -p $mnt; + $MOUNT_CMD $flags $prunedopts $device $mnt; + rc=\\\$?; + else + lustre_mnt_count=\\\$(mount | grep $mnt' ' | \ + grep 'type lustre' | wc -l); + if [ \\\$running -ne \\\$lustre_mnt_count ] ; then + echo zconf_mount_clients FAILED: \ + mount count \\\$running, not matching \ + with mount count of 'type lustre' \ + \\\$lustre_mnt_count; + rc=1; + fi; + fi; + exit \\\$rc" || return ${PIPESTATUS[0]} - echo "Starting client $clients: $flags $opts $device $mnt" + i=$((i + 1)) + done + else - do_nodes $clients " + local tmpopts=$opts + if $SHARED_KEY; then + tmpopts=$(add_sk_mntflag $opts) + fi + do_nodes $clients " running=\\\$(mount | grep -c $mnt' '); rc=0; if [ \\\$running -eq 0 ] ; then - mkdir -p $mnt; - $MOUNT_CMD $flags $opts $device $mnt; - rc=\\\$?; + mkdir -p $mnt; + $MOUNT_CMD $flags $tmpopts $device $mnt; + rc=\\\$?; fi; exit \\\$rc" || return ${PIPESTATUS[0]} + fi - echo "Started clients $clients: " - do_nodes $clients "mount | grep $mnt' '" + echo "Started clients $clients: " + do_nodes $clients "mount | grep $mnt' '" - set_default_debug_nodes $clients + set_default_debug_nodes $clients - return 0 + return 0 } zconf_umount_clients() { @@ -1708,7 +2099,7 @@ shutdown_node_hard () { wait_for_function --quiet "! ping -w 3 -c 1 $host" 5 1 && return 0 echo "waiting for $host to fail attempts=$attempts" [ $i -lt $attempts ] || \ - { echo "$host still pingable after power down! attempts=$attempts" && return 1; } + { echo "$host still pingable after power down! attempts=$attempts" && return 1; } done } @@ -1839,29 +2230,31 @@ node_var_name() { } start_client_load() { - local client=$1 - local load=$2 - local var=$(node_var_name $client)_load - eval export ${var}=$load - - do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \ -BREAK_ON_ERROR=$BREAK_ON_ERROR \ -END_RUN_FILE=$END_RUN_FILE \ -LOAD_PID_FILE=$LOAD_PID_FILE \ -TESTLOG_PREFIX=$TESTLOG_PREFIX \ -TESTNAME=$TESTNAME \ -DBENCH_LIB=$DBENCH_LIB \ -DBENCH_SRC=$DBENCH_SRC \ -CLIENT_COUNT=$((CLIENTCOUNT - 1)) \ -LFS=$LFS \ -run_${load}.sh" & - local ppid=$! - log "Started client load: ${load} on $client" - - # get the children process IDs - local pids=$(ps --ppid $ppid -o pid= | xargs) - CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $ppid $pids" - return 0 + local client=$1 + local load=$2 + local var=$(node_var_name $client)_load + eval export ${var}=$load + + do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \ + BREAK_ON_ERROR=$BREAK_ON_ERROR \ + END_RUN_FILE=$END_RUN_FILE \ + LOAD_PID_FILE=$LOAD_PID_FILE \ + TESTLOG_PREFIX=$TESTLOG_PREFIX \ + TESTNAME=$TESTNAME \ + DBENCH_LIB=$DBENCH_LIB \ + DBENCH_SRC=$DBENCH_SRC \ + CLIENT_COUNT=$((CLIENTCOUNT - 1)) \ + LFS=$LFS \ + LCTL=$LCTL \ + FSNAME=$FSNAME \ + run_${load}.sh" & + local ppid=$! + log "Started client load: ${load} on $client" + + # get the children process IDs + local pids=$(ps --ppid $ppid -o pid= | xargs) + CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $ppid $pids" + return 0 } start_client_loads () { @@ -1879,55 +2272,59 @@ start_client_loads () { # only for remote client check_client_load () { - local client=$1 - local var=$(node_var_name $client)_load - local TESTLOAD=run_${!var}.sh - - ps auxww | grep -v grep | grep $client | grep -q "$TESTLOAD" || return 1 - - # bug 18914: try to connect several times not only when - # check ps, but while check_catastrophe also - local tries=3 - local RC=254 - while [ $RC = 254 -a $tries -gt 0 ]; do - let tries=$tries-1 - # assume success - RC=0 - if ! check_catastrophe $client; then - RC=${PIPESTATUS[0]} - if [ $RC -eq 254 ]; then - # FIXME: not sure how long we shuold sleep here - sleep 10 - continue - fi - echo "check catastrophe failed: RC=$RC " - return $RC - fi - done - # We can continue try to connect if RC=254 - # Just print the warning about this - if [ $RC = 254 ]; then - echo "got a return status of $RC from do_node while checking catastrophe on $client" - fi - - # see if the load is still on the client - tries=3 - RC=254 - while [ $RC = 254 -a $tries -gt 0 ]; do - let tries=$tries-1 - # assume success - RC=0 - if ! do_node $client "ps auxwww | grep -v grep | grep -q $TESTLOAD"; then - RC=${PIPESTATUS[0]} - sleep 30 - fi - done - if [ $RC = 254 ]; then - echo "got a return status of $RC from do_node while checking (catastrophe and 'ps') the client load on $client" - # see if we can diagnose a bit why this is - fi + local client=$1 + local var=$(node_var_name $client)_load + local testload=run_${!var}.sh + + ps auxww | grep -v grep | grep $client | grep -q $testload || return 1 + + # bug 18914: try to connect several times not only when + # check ps, but while check_node_health also + + local tries=3 + local RC=254 + while [ $RC = 254 -a $tries -gt 0 ]; do + let tries=$tries-1 + # assume success + RC=0 + if ! check_node_health $client; then + RC=${PIPESTATUS[0]} + if [ $RC -eq 254 ]; then + # FIXME: not sure how long we shuold sleep here + sleep 10 + continue + fi + echo "check node health failed: RC=$RC " + return $RC + fi + done + # We can continue try to connect if RC=254 + # Just print the warning about this + if [ $RC = 254 ]; then + echo "got a return status of $RC from do_node while checking " \ + "node health on $client" + fi + + # see if the load is still on the client + tries=3 + RC=254 + while [ $RC = 254 -a $tries -gt 0 ]; do + let tries=$tries-1 + # assume success + RC=0 + if ! do_node $client \ + "ps auxwww | grep -v grep | grep -q $testload"; then + RC=${PIPESTATUS[0]} + sleep 30 + fi + done + if [ $RC = 254 ]; then + echo "got a return status of $RC from do_node while checking " \ + "(node health and 'ps') the client load on $client" + # see if we can diagnose a bit why this is + fi - return $RC + return $RC } check_client_loads () { local clients=${1//,/ } @@ -2107,19 +2504,31 @@ wait_update () { } wait_update_facet() { + local verbose= + [ "$1" = "--verbose" ] && verbose="$1" && shift + local facet=$1 shift - wait_update $(facet_active_host $facet) "$@" + wait_update $verbose $(facet_active_host $facet) "$@" } sync_all_data() { do_nodes $(comma_list $(mdts_nodes)) \ - "lctl set_param -n osd*.*MDT*.force_sync 1" + "lctl set_param -n os[cd]*.*MDT*.force_sync=1" do_nodes $(comma_list $(osts_nodes)) \ - "lctl set_param -n osd*.*OS*.force_sync 1" 2>&1 | + "lctl set_param -n osd*.*OS*.force_sync=1" 2>&1 | grep -v 'Found no match' } +wait_zfs_commit() { + # the occupied disk space will be released + # only after DMUs are committed + if [[ $(facet_fstype $1) == zfs ]]; then + echo "sleep $2 for ZFS OSD" + sleep $2 + fi +} + wait_delete_completed_mds() { local MAX_WAIT=${1:-20} # for ZFS, waiting more time for DMUs to be committed @@ -2140,12 +2549,13 @@ wait_delete_completed_mds() { mds2sync="$mds2sync $node" done if [ -z "$mds2sync" ]; then + wait_zfs_commit $SINGLEMDS $ZFS_WAIT return fi mds2sync=$(comma_list $mds2sync) # sync MDS transactions - do_nodes $mds2sync "$LCTL set_param -n osd*.*MD*.force_sync 1" + do_nodes $mds2sync "$LCTL set_param -n os[cd]*.*MD*.force_sync 1" # wait till all changes are sent and commmitted by OSTs # for ldiskfs space is released upon execution, but DMU @@ -2157,16 +2567,7 @@ wait_delete_completed_mds() { "$LCTL get_param -n osc.*MDT*.sync_*" | calc_sum) #echo "$node: $changes changes on all" if [[ $changes -eq 0 ]]; then - etime=$(date +%s) - #echo "delete took $((etime - stime)) seconds" - - # the occupied disk space will be released - # only after DMUs are committed - if [[ $(facet_fstype $SINGLEMDS) == zfs ]]; then - echo "sleep $ZFS_WAIT for ZFS OSD" - sleep $ZFS_WAIT - fi - + wait_zfs_commit $SINGLEMDS $ZFS_WAIT return fi sleep 1 @@ -2288,6 +2689,19 @@ wait_mds_ost_sync () { return 1 } +# Wait OSTs to be active on both client and MDT side. +wait_osts_up() { + local cmd="$LCTL get_param -n lov.$FSNAME-clilov-*.target_obd | + awk 'BEGIN {c = 0} /ACTIVE/{c += 1} END {printf \\\"%d\\\", c}'" + wait_update $HOSTNAME "eval $cmd" $OSTCOUNT || + error "wait_update OSTs up on client failed" + + cmd="$LCTL get_param osp.$FSNAME-OST*-MDT0000.prealloc_last_id | + awk '/=[1-9][0-9]/ { c += 1 } END { printf \\\"%d\\\", c }'" + wait_update_facet $SINGLEMDS "eval $cmd" $OSTCOUNT || + error "wait_update OSTs up on MDT0000 failed" +} + wait_destroy_complete () { echo "Waiting for local destroys to complete" # MAX value shouldn't be big as this mean server responsiveness @@ -2373,25 +2787,26 @@ wait_remote_prog () { return $rc } +lfs_df_check() { + local clients=${1:-$CLIENTS} + + if [ -z "$clients" ]; then + $LFS df $MOUNT + else + $PDSH $clients "$LFS df $MOUNT" > /dev/null + fi +} + clients_up() { - # not every config has many clients - sleep 1 - if [ ! -z "$CLIENTS" ]; then - $PDSH $CLIENTS "stat -f $MOUNT" > /dev/null - else - stat -f $MOUNT > /dev/null - fi + # not every config has many clients + sleep 1 + lfs_df_check } client_up() { - local client=$1 - # usually checked on particular client or locally - sleep 1 - if [ ! -z "$client" ]; then - $PDSH $client "stat -f $MOUNT" > /dev/null - else - stat -f $MOUNT > /dev/null - fi + # usually checked on particular client or locally + sleep 1 + lfs_df_check $1 } client_evicted() { @@ -2399,16 +2814,17 @@ client_evicted() { } client_reconnect_try() { - uname -n >> $MOUNT/recon - if [ -z "$CLIENTS" ]; then - df $MOUNT; uname -n >> $MOUNT/recon - else - do_nodes $CLIENTS "df $MOUNT; uname -n >> $MOUNT/recon" > /dev/null - fi - echo Connected clients: - cat $MOUNT/recon - ls -l $MOUNT/recon > /dev/null - rm $MOUNT/recon + local f=$MOUNT/recon + + uname -n >> $f + if [ -z "$CLIENTS" ]; then + $LFS df $MOUNT; uname -n >> $f + else + do_nodes $CLIENTS "$LFS df $MOUNT; uname -n >> $f" > /dev/null + fi + echo "Connected clients: $(cat $f)" + ls -l $f > /dev/null + rm $f } client_reconnect() { @@ -2432,6 +2848,13 @@ affected_facets () { } facet_failover() { + local E2FSCK_ON_MDT0=false + if [ "$1" == "--fsck" ]; then + shift + [ $(facet_fstype $SINGLEMDS) == ldiskfs ] && + E2FSCK_ON_MDT0=true + fi + local facets=$1 local sleep_time=$2 local -a affecteds @@ -2465,6 +2888,9 @@ facet_failover() { shutdown_facet $facet done + $E2FSCK_ON_MDT0 && (run_e2fsck $(facet_active_host $SINGLEMDS) \ + $(mdsdevname 1) "-n" || error "Running e2fsck") + for ((index=0; index<$total; index++)); do facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1) echo reboot facets: ${affecteds[index]} @@ -2493,7 +2919,7 @@ obd_name() { replay_barrier() { local facet=$1 do_facet $facet "sync; sync; sync" - df $MOUNT + $LFS df $MOUNT # make sure there will be no seq change local clients=${CLIENTS:-$HOSTNAME} @@ -2578,7 +3004,7 @@ fail() { facet_failover $* || error "failover: $?" wait_clients_import_state "$clients" "$facets" FULL - clients_up || error "post-failover df: $?" + clients_up || error "post-failover stat: $?" } fail_nodf() { @@ -2592,25 +3018,15 @@ fail_abort() { change_active $facet wait_for_facet $facet mount_facet $facet -o abort_recovery - clients_up || echo "first df failed: $?" - clients_up || error "post-failover df: $?" -} - -do_lmc() { - echo There is no lmc. This is mountconf, baby. - exit 1 + clients_up || echo "first stat failed: $?" + clients_up || error "post-failover stat: $?" } host_nids_address() { - local nodes=$1 - local kind=$2 + local nodes=$1 + local net=${2:-"."} - if [ -n "$kind" ]; then - nids=$(do_nodes $nodes "$LCTL list_nids | grep $kind | cut -f 1 -d '@'") - else - nids=$(do_nodes $nodes "$LCTL list_nids all | cut -f 1 -d '@'") - fi - echo $nids + do_nodes $nodes "$LCTL list_nids | grep $net | cut -f 1 -d @" } h2name_or_ip() { @@ -2619,40 +3035,34 @@ h2name_or_ip() { fi } -h2ptl() { - if [ "$1" = "'*'" ]; then echo \'*\'; else - ID=`xtprocadmin -n $1 2>/dev/null | egrep -v 'NID' | \ - awk '{print $1}'` - if [ -z "$ID" ]; then - echo "Could not get a ptl id for $1..." - exit 1 - fi - echo $ID"@ptl" +h2nettype() { + if [[ -n "$NETTYPE" ]]; then + h2name_or_ip "$1" "$NETTYPE" + else + h2name_or_ip "$1" "$2" fi } -declare -fx h2ptl +declare -fx h2nettype +# Wrapper function to print the deprecation warning h2tcp() { - h2name_or_ip "$1" "tcp" -} -declare -fx h2tcp - -h2elan() { - if [ "$1" = "'*'" ]; then echo \'*\'; else - if type __h2elan >/dev/null 2>&1; then - ID=$(__h2elan $1) - else - ID=`echo $1 | sed 's/[^0-9]*//g'` - fi - echo $ID"@elan" + echo "h2tcp: deprecated, use h2nettype instead" 1>&2 + if [[ -n "$NETTYPE" ]]; then + h2nettype "$@" + else + h2nettype "$1" "tcp" fi } -declare -fx h2elan +# Wrapper function to print the deprecation warning h2o2ib() { - h2name_or_ip "$1" "o2ib" + echo "h2o2ib: deprecated, use h2nettype instead" 1>&2 + if [[ -n "$NETTYPE" ]]; then + h2nettype "$@" + else + h2nettype "$1" "o2ib" + fi } -declare -fx h2o2ib # This enables variables in cfg/"setup".sh files to support the pdsh HOSTLIST # expressions format. As a bonus we can then just pass in those variables @@ -2688,6 +3098,8 @@ hostlist_expand() { group=${group%%]*} for range in ${group//,/ }; do + local order + begin=${range%-*} end=${range#*-} @@ -2703,7 +3115,13 @@ hostlist_expand() { begin=$(echo $begin | sed 's/0*//') [ -z $begin ] && begin=0 - for num in $(seq -f "%0${padlen}g" $begin $end); do + if [ ! -z "${begin##[!0-9]*}" ]; then + order=$(seq -f "%0${padlen}g" $begin $end) + else + order=$(eval echo {$begin..$end}); + fi + + for num in $order; do value="${name#*,}${num}${back}" [ "$value" != "${value/\[/}" ] && { value=$(hostlist_expand "$value") @@ -2724,8 +3142,10 @@ hostlist_expand() { myList="${list%% *}" while [[ "$list" != ${myList##* } ]]; do - list=${list//${list%% *} /} - myList="$myList ${list%% *}" + local tlist=" $list" + list=${tlist// ${list%% *} / } + list=${list:1} + myList="$myList ${list%% *}" done myList="${myList%* }"; @@ -3200,7 +3620,7 @@ facet_mntpt () { [[ $facet = mgs ]] && combined_mgs_mds && facet="mds1" local var=${facet}_MOUNT - eval mntpt=${!var:-${MOUNT%/*}/$facet} + eval mntpt=${!var:-${MOUNT}-$facet} echo -n $mntpt } @@ -3208,7 +3628,7 @@ facet_mntpt () { mount_ldiskfs() { local facet=$1 local dev=$(facet_device $facet) - local mnt=$(facet_mntpt $facet) + local mnt=${2:-$(facet_mntpt $facet)} local opts if ! do_facet $facet test -b $dev; then @@ -3220,9 +3640,9 @@ mount_ldiskfs() { unmount_ldiskfs() { local facet=$1 local dev=$(facet_device $facet) - local mnt=$(facet_mntpt $facet) + local mnt=${2:-$(facet_mntpt $facet)} - do_facet $facet umount -d $mnt + do_facet $facet $UMOUNT $mnt } var_name() { @@ -3232,7 +3652,7 @@ var_name() { mount_zfs() { local facet=$1 local ds=$(facet_device $facet) - local mnt=$(facet_mntpt $facet) + local mnt=${2:-$(facet_mntpt $facet)} local canmnt local mntpt @@ -3255,7 +3675,7 @@ mount_zfs() { unmount_zfs() { local facet=$1 local ds=$(facet_device $facet) - local mnt=$(facet_mntpt $facet) + local mnt=${2:-$(facet_mntpt $facet)} local var_mntpt=mz_$(var_name ${facet}_$ds)_mountpoint local var_canmnt=mz_$(var_name ${facet}_$ds)_canmount local mntpt=${!var_mntpt} @@ -3271,16 +3691,18 @@ unmount_zfs() { mount_fstype() { local facet=$1 + local mnt=$2 local fstype=$(facet_fstype $facet) - mount_$fstype $facet + mount_$fstype $facet $mnt } unmount_fstype() { local facet=$1 + local mnt=$2 local fstype=$(facet_fstype $facet) - unmount_$fstype $facet + unmount_$fstype $facet $mnt } ######## @@ -3300,7 +3722,8 @@ stopall() { zconf_umount_clients $clients $MOUNT "$*" || true [ -n "$MOUNT2" ] && zconf_umount_clients $clients $MOUNT2 "$*" || true - [ "$CLIENTONLY" ] && return + [ -n "$CLIENTONLY" ] && return + # The add fn does rm ${facet}active file, this would be enough # if we use do_facet only after the facet added, but # currently we use do_facet mds in local.sh @@ -3323,23 +3746,26 @@ stopall() { } cleanup_echo_devs () { - local devs=$($LCTL dl | grep echo | awk '{print $4}') + trap 0 + local dev + local devs=$($LCTL dl | grep echo | awk '{print $4}') - for dev in $devs; do - $LCTL --device $dev cleanup - $LCTL --device $dev detach - done + for dev in $devs; do + $LCTL --device $dev cleanup + $LCTL --device $dev detach + done } cleanupall() { - nfs_client_mode && return + nfs_client_mode && return cifs_client_mode && return - stopall $* - cleanup_echo_devs + stopall $* + cleanup_echo_devs - unload_modules - cleanup_gss + unload_modules + cleanup_sk + cleanup_gss } combined_mgs_mds () { @@ -3382,14 +3808,13 @@ mkfs_opts() { var=${facet}failover_HOST if [ -n "${!var}" ] && [ ${!var} != $(facet_host $facet) ]; then - opts+=" --failnode=$(h2$NETTYPE ${!var})" + opts+=" --failnode=$(h2nettype ${!var})" fi opts+=${TIMEOUT:+" --param=sys.timeout=$TIMEOUT"} opts+=${LDLM_TIMEOUT:+" --param=sys.ldlm_timeout=$LDLM_TIMEOUT"} if [ $type == MDS ]; then - opts+=${SECLEVEL:+" --param=mdt.sec_level"} opts+=${MDSCAPA:+" --param-mdt.capa=$MDSCAPA"} opts+=${STRIPE_BYTES:+" --param=lov.stripesize=$STRIPE_BYTES"} opts+=${STRIPES_PER_OBJ:+" --param=lov.stripecount=$STRIPES_PER_OBJ"} @@ -3413,7 +3838,6 @@ mkfs_opts() { fi if [ $type == OST ]; then - opts+=${SECLEVEL:+" --param=ost.sec_level"} opts+=${OSSCAPA:+" --param=ost.capa=$OSSCAPA"} if [ $fstype == ldiskfs ]; then @@ -3449,6 +3873,17 @@ mkfs_opts() { echo -n "$opts" } +mountfs_opts() { + local facet=$1 + local type=$(facet_type $facet) + local var=${type}_MOUNT_FS_OPTS + local opts="" + if [ -n "${!var}" ]; then + opts+=" --mountfsoptions=${!var}" + fi + echo -n "$opts" +} + check_ost_indices() { local index_count=${#OST_INDICES[@]} [[ $index_count -eq 0 || $OSTCOUNT -le $index_count ]] && return 0 @@ -3467,6 +3902,34 @@ check_ost_indices() { done } +__touch_device() +{ + local facet_type=$1 # mgs || mds || ost + local facet_num=$2 + local facet=${1}${2} + local device + + case "$(facet_fstype $facet)" in + ldiskfs) + device=$(${facet_type}devname $facet_num) + ;; + zfs) + device=$(${facet_type}vdevname $facet_num) + ;; + *) + error "Unhandled filesystem type" + ;; + esac + + do_facet $facet "[ -e \"$device\" ]" && return + + # Note: the following check only works with absolute paths + [[ ! "$device" =~ ^/dev/ ]] || [[ "$device" =~ ^/dev/shm/ ]] || + error "$facet: device '$device' does not exist" + + do_facet $facet "touch \"${device}\"" +} + format_mgs() { local quiet @@ -3475,7 +3938,13 @@ format_mgs() { fi echo "Format mgs: $(mgsdevname)" reformat_external_journal mgs - add mgs $(mkfs_opts mgs $(mgsdevname)) --reformat \ + + # touch "device" in case it is a loopback file for testing and needs to + # be created. mkfs.lustre doesn't do this to avoid accidentally writing + # to non-existent files in /dev if the admin made a typo during setup + __touch_device mgs + + add mgs $(mkfs_opts mgs $(mgsdevname)) $(mountfs_opts mgs) --reformat \ $(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} || exit 10 } @@ -3488,9 +3957,12 @@ format_mdt() { fi echo "Format mds$num: $(mdsdevname $num)" reformat_external_journal mds$num + + __touch_device mds $num + add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \ - --reformat $(mdsdevname $num) $(mdsvdevname $num) \ - ${quiet:+>/dev/null} || exit 10 + $(mountfs_opts mds$num) --reformat $(mdsdevname $num) \ + $(mdsvdevname $num) ${quiet:+>/dev/null} || exit 10 } format_ost() { @@ -3501,16 +3973,26 @@ format_ost() { fi echo "Format ost$num: $(ostdevname $num)" reformat_external_journal ost$num + + __touch_device ost $num + add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \ - --reformat $(ostdevname $num) $(ostvdevname ${num}) \ - ${quiet:+>/dev/null} || exit 10 + $(mountfs_opts ost$num) --reformat $(ostdevname $num) \ + $(ostvdevname ${num}) ${quiet:+>/dev/null} || exit 10 } formatall() { - stopall + stopall -f + # Set hostid for ZFS/SPL zpool import protection + # (Assumes MDS version is also OSS version) + if [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.8.54) ]; + then + do_rpc_nodes "$(comma_list $(remote_nodes_list))" set_hostid + fi + # We need ldiskfs here, may as well load them all load_modules - [ "$CLIENTONLY" ] && return + [ -n "$CLIENTONLY" ] && return echo Formatting mgs, mds, osts if ! combined_mgs_mds ; then format_mgs @@ -3528,11 +4010,11 @@ formatall() { } mount_client() { - grep " $1 " /proc/mounts || zconf_mount $HOSTNAME $* + grep " $1 " /proc/mounts || zconf_mount $HOSTNAME $* } umount_client() { - grep " $1 " /proc/mounts && zconf_umount `hostname` $* + grep " $1 " /proc/mounts && zconf_umount $HOSTNAME $* } # return value: @@ -3568,8 +4050,8 @@ switch_identity() { remount_client() { - zconf_umount `hostname` $1 || error "umount failed" - zconf_mount `hostname` $1 || error "mount failed" + zconf_umount $HOSTNAME $1 || error "umount failed" + zconf_mount $HOSTNAME $1 || error "mount failed" } writeconf_facet() { @@ -3599,83 +4081,152 @@ writeconf_all () { return $rc } -setupall() { - nfs_client_mode && return - cifs_client_mode && return +mountmgs() { + if ! combined_mgs_mds ; then + start mgs $(mgsdevname) $MGS_MOUNT_OPTS + fi +} - sanity_mount_check || - error "environments are insane!" +mountmds() { + for num in $(seq $MDSCOUNT); do + DEVNAME=$(mdsdevname $num) + start mds$num $DEVNAME $MDS_MOUNT_OPTS + + # We started mds, now we should set failover variables properly. + # Set mds${num}failover_HOST if unset (the default + # failnode). + local varname=mds${num}failover_HOST + if [ -z "${!varname}" ]; then + eval mds${num}failover_HOST=$(facet_host mds$num) + fi - load_modules + if [ $IDENTITY_UPCALL != "default" ]; then + switch_identity $num $IDENTITY_UPCALL + fi + done +} - if [ -z "$CLIENTONLY" ]; then - echo Setup mgs, mdt, osts - echo $WRITECONF | grep -q "writeconf" && \ - writeconf_all - if ! combined_mgs_mds ; then - start mgs $(mgsdevname) $MGS_MOUNT_OPTS - fi +mountoss() { + for num in $(seq $OSTCOUNT); do + DEVNAME=$(ostdevname $num) + start ost$num $DEVNAME $OST_MOUNT_OPTS + + # We started ost$num, now we should set ost${num}failover + # variable properly. Set ost${num}failover_HOST if it is not + # set (the default failnode). + varname=ost${num}failover_HOST + if [ -z "${!varname}" ]; then + eval ost${num}failover_HOST=$(facet_host ost${num}) + fi - for num in `seq $MDSCOUNT`; do - DEVNAME=$(mdsdevname $num) - start mds$num $DEVNAME $MDS_MOUNT_OPTS + done +} - # We started mds, now we should set failover variables properly. - # Set mds${num}failover_HOST if it is not set (the default failnode). - local varname=mds${num}failover_HOST - if [ -z "${!varname}" ]; then - eval mds${num}failover_HOST=$(facet_host mds$num) - fi +mountcli() { + [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE + if [ ! -z $arg1 ]; then + [ "$arg1" = "server_only" ] && return + fi + mount_client $MOUNT + [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT + clients_up - if [ $IDENTITY_UPCALL != "default" ]; then - switch_identity $num $IDENTITY_UPCALL - fi - done - for num in `seq $OSTCOUNT`; do - DEVNAME=$(ostdevname $num) - start ost$num $DEVNAME $OST_MOUNT_OPTS - - # We started ost$num, now we should set ost${num}failover variable properly. - # Set ost${num}failover_HOST if it is not set (the default failnode). - varname=ost${num}failover_HOST - if [ -z "${!varname}" ]; then - eval ost${num}failover_HOST=$(facet_host ost${num}) - fi + if [ "$MOUNT_2" ]; then + mount_client $MOUNT2 + [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT2 + fi +} - done - fi +sk_nodemap_setup() { + local sk_map_name=${1:-$SK_S2SNM} + local sk_map_nodes=${2:-$HOSTNAME} + do_node $(mgs_node) "$LCTL nodemap_add $sk_map_name" + for servernode in $sk_map_nodes; do + local nids=$(do_nodes $servernode "$LCTL list_nids") + for nid in $nids; do + do_node $(mgs_node) "$LCTL nodemap_add_range --name \ + $sk_map_name --range $nid" + done + done +} - init_gss +setupall() { + local arg1=$1 - # wait a while to allow sptlrpc configuration be propogated to targets, - # only needed when mounting new target devices. - if $GSS; then - sleep 10 - fi + nfs_client_mode && return + cifs_client_mode && return - [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE - mount_client $MOUNT - [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT - clients_up + sanity_mount_check || error "environments are insane!" - if [ "$MOUNT_2" ]; then - mount_client $MOUNT2 - [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT2 - fi + load_modules + + init_gss - init_param_vars + if [ -z "$CLIENTONLY" ]; then + echo Setup mgs, mdt, osts + echo $WRITECONF | grep -q "writeconf" && writeconf_all - # by remounting mdt before ost, initial connect from mdt to ost might - # timeout because ost is not ready yet. wait some time to its fully - # recovery. initial obd_connect timeout is 5s; in GSS case it's preceeded - # by a context negotiation rpc with $TIMEOUT. - # FIXME better by monitoring import status. - if $GSS; then - set_flavor_all $SEC - sleep $((TIMEOUT + 5)) - else - sleep 5 - fi + if $SK_MOUNTED; then + echo "Shared Key file system already mounted" + else + mountmgs + mountmds + mountoss + if $SHARED_KEY; then + export SK_MOUNTED=true + fi + fi + if $GSS_SK; then + echo "GSS_SK: setting kernel keyring perms" + do_nodes $(comma_list $(all_nodes)) \ + "keyctl show | grep lustre | cut -c1-11 | + sed -e 's/ //g;' | + xargs -IX keyctl setperm X 0x3f3f3f3f" + + if $SK_S2S; then + # Need to start one nodemap for servers, + # and one for clients. + sk_nodemap_setup $SK_S2SNM \ + $(comma_list $(all_server_nodes)) + mountcli + sk_nodemap_setup $SK_S2SNMCLI \ + ${CLIENTS:-$HOSTNAME} + echo "Nodemap set up for SK S2S, remounting." + stopall + mountmgs + mountmds + mountoss + fi + fi + fi + + # wait a while to allow sptlrpc configuration be propogated to targets, + # only needed when mounting new target devices. + if $GSS; then + sleep 10 + fi + + mountcli + init_param_vars + + # by remounting mdt before ost, initial connect from mdt to ost might + # timeout because ost is not ready yet. wait some time to its fully + # recovery. initial obd_connect timeout is 5s; in GSS case it's + # preceeded by a context negotiation rpc with $TIMEOUT. + # FIXME better by monitoring import status. + if $GSS; then + if $GSS_SK; then + set_rule $FSNAME any cli2mdt $SK_FLAVOR + set_rule $FSNAME any cli2ost $SK_FLAVOR + wait_flavor cli2mdt $SK_FLAVOR + wait_flavor cli2ost $SK_FLAVOR + else + set_flavor_all $SEC + fi + sleep $((TIMEOUT + 5)) + else + sleep 5 + fi } mounted_lustre_filesystems() { @@ -3683,7 +4234,7 @@ mounted_lustre_filesystems() { } init_facet_vars () { - [ "$CLIENTONLY" ] && return 0 + [ -n "$CLIENTONLY" ] && return 0 local facet=$1 shift local device=$1 @@ -3808,14 +4359,17 @@ set_conf_param_and_check() { do_facet mgs "$LCTL conf_param $PARAM='$FINAL'" || error "conf_param $PARAM failed" - wait_update $(facet_host $myfacet) "$TEST" "$FINAL" || + wait_update_facet $myfacet "$TEST" "$FINAL" || error "check $PARAM failed!" } init_param_vars () { - remote_mds_nodsh || - TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout") + TIMEOUT=$(lctl get_param -n timeout) + TIMEOUT=${TIMEOUT:-20} + + remote_mds_nodsh && log "Using TIMEOUT=$TIMEOUT" && return 0 + TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout") log "Using TIMEOUT=$TIMEOUT" osc_ensure_active $SINGLEMDS $TIMEOUT @@ -3876,14 +4430,14 @@ check_config_client () { local mntpt=$1 local mounted=$(mount | grep " $mntpt ") - if [ "$CLIENTONLY" ]; then + if [ -n "$CLIENTONLY" ]; then # bug 18021 # CLIENTONLY should not depend on *_HOST settings local mgc=$($LCTL device_list | awk '/MGC/ {print $4}') # in theory someone could create a new, # client-only config file that assumed lustre was already # configured and didn't set the MGSNID. If MGSNID is not set, - # then we should use the mgs nid currently being used + # then we should use the mgs nid currently being used # as the default value. bug 18021 [[ x$MGSNID = x ]] && MGSNID=${mgc//MGC/} @@ -3898,20 +4452,10 @@ check_config_client () { return 0 fi - local myMGS_host=$mgs_HOST - if [ "$NETTYPE" = "ptl" ]; then - myMGS_host=$(h2ptl $mgs_HOST | sed -e s/@ptl//) - fi - echo Checking config lustre mounted on $mntpt local mgshost=$(mount | grep " $mntpt " | awk -F@ '{print $1}') mgshost=$(echo $mgshost | awk -F: '{print $1}') -# if [ "$mgshost" != "$myMGS_host" ]; then -# log "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE -# Please use correct config or set mds_HOST correctly!" -# fi - } check_config_clients () { @@ -3965,21 +4509,22 @@ is_empty_fs() { } check_and_setup_lustre() { - nfs_client_mode && return + sanitize_parameters + nfs_client_mode && return cifs_client_mode && return - local MOUNTED=$(mounted_lustre_filesystems) - - local do_check=true - # 1. - # both MOUNT and MOUNT2 are not mounted - if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then - [ "$REFORMAT" ] && formatall - # setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set) - setupall - is_mounted $MOUNT || error "NAME=$NAME not mounted" - export I_MOUNTED=yes - do_check=false + local MOUNTED=$(mounted_lustre_filesystems) + + local do_check=true + # 1. + # both MOUNT and MOUNT2 are not mounted + if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then + [ "$REFORMAT" = "yes" ] && formatall + # setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set) + setupall + is_mounted $MOUNT || error "NAME=$NAME not mounted" + export I_MOUNTED=yes + do_check=false # 2. # MOUNT2 is mounted elif is_mounted $MOUNT2; then @@ -4002,7 +4547,7 @@ check_and_setup_lustre() { restore_mount $MOUNT2 export I_MOUNTED2=yes fi - fi + fi # 5. # MOUNT is mounted MOUNT2 is not mounted @@ -4023,7 +4568,7 @@ check_and_setup_lustre() { set_default_debug_nodes $(comma_list $(nodes_list)) fi - if [ $(lower $OSD_TRACK_DECLARES_LBUG) == 'yes' ] ; then + if [ -z "$CLIENTONLY" -a $(lower $OSD_TRACK_DECLARES_LBUG) == 'yes' ]; then local facets="" [ "$(facet_fstype ost1)" = "ldiskfs" ] && facets="$(get_facets OST)" @@ -4039,16 +4584,20 @@ check_and_setup_lustre() { fi init_gss - if $GSS; then + if $GSS_SK; then + set_flavor_all null + elif $GSS; then set_flavor_all $SEC fi - #Enable remote MDT create for testing - for num in $(seq $MDSCOUNT); do - do_facet mds$num \ - lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 \ - 2>/dev/null - done + if [ -z "$CLIENTONLY" ]; then + # Enable remote MDT create for testing + for num in $(seq $MDSCOUNT); do + do_facet mds$num \ + lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 \ + 2>/dev/null + done + fi if [ "$ONLY" == "setup" ]; then exit 0 @@ -4179,88 +4728,31 @@ check_shared_dir() { return 0 } -# Run e2fsck on MDT and OST(s) to generate databases used for lfsck. -generate_db() { - local i - local ostidx - local dev - local node - - [[ $(lustre_version_code $SINGLEMDS) -ne $(version_code 2.2.0) ]] || - { skip "Lustre 2.2.0 lacks the patch for LU-1255"; exit 0; } - - check_shared_dir $SHARED_DIRECTORY || - error "$SHARED_DIRECTORY isn't a shared directory" - - export MDSDB=$SHARED_DIRECTORY/mdsdb - export OSTDB=$SHARED_DIRECTORY/ostdb - - # DNE is not supported, so when running e2fsck on a DNE filesystem, - # we only pass master MDS parameters. - run_e2fsck $MDTNODE $MDTDEV "-n --mdsdb $MDSDB" - - i=0 - ostidx=0 - OSTDB_LIST="" - for node in $(osts_nodes); do - for dev in ${OSTDEVS[i]}; do - run_e2fsck $node $dev "-n --mdsdb $MDSDB --ostdb $OSTDB-$ostidx" - OSTDB_LIST="$OSTDB_LIST $OSTDB-$ostidx" - ostidx=$((ostidx + 1)) - done - i=$((i + 1)) - done -} - -# Run lfsck on server node if lfsck can't be found on client (LU-2571) -run_lfsck_remote() { - local cmd="$LFSCK_BIN -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT" - local client=$1 - local mounted=true - local rc=0 - - #Check if lustre is already mounted - do_rpc_nodes $client is_mounted $MOUNT || mounted=false - if ! $mounted; then - zconf_mount $client $MOUNT || - error "failed to mount Lustre on $client" - fi - #Run lfsck - echo $cmd - do_node $client $cmd || rc=$? - #Umount if necessary - if ! $mounted; then - zconf_umount $client $MOUNT || - error "failed to unmount Lustre on $client" - fi - - [ $rc -le $FSCK_MAX_ERR ] || - error "$cmd returned $rc, should be <= $FSCK_MAX_ERR" - echo "lfsck finished with rc=$rc" - - return $rc -} - run_lfsck() { - local facets="client $SINGLEMDS" - local found=false - local facet - local node - local rc=0 - - for facet in $facets; do - node=$(facet_active_host $facet) - if check_progs_installed $node $LFSCK_BIN; then - found=true - break - fi + do_nodes $(comma_list $(mdts_nodes) $(osts_nodes)) \ + $LCTL set_param printk=+lfsck + do_facet $SINGLEMDS "$LCTL lfsck_start -M $FSNAME-MDT0000 -r -A -t all" + + for k in $(seq $MDSCOUNT); do + # wait up to 10+1 minutes for LFSCK to complete + wait_update_facet --verbose mds${k} "$LCTL get_param -n \ + mdd.$(facet_svc mds${k}).lfsck_layout | + awk '/^status/ { print \\\$2 }'" "completed" 600 || + error "MDS${k} layout isn't the expected 'completed'" + wait_update_facet --verbose mds${k} "$LCTL get_param -n \ + mdd.$(facet_svc mds${k}).lfsck_namespace | + awk '/^status/ { print \\\$2 }'" "completed" 60 || + error "MDS${k} namespace isn't the expected 'completed'" done - ! $found && error "None of \"$facets\" supports lfsck" - - run_lfsck_remote $node || rc=$? - - rm -rvf $MDSDB* $OSTDB* || true - return $rc + local rep_mdt=$(do_nodes $(comma_list $(mdts_nodes)) \ + $LCTL get_param -n mdd.$FSNAME-*.lfsck_* | + awk '/repaired/ { print $2 }' | calc_sum) + local rep_ost=$(do_nodes $(comma_list $(osts_nodes)) \ + $LCTL get_param -n obdfilter.$FSNAME-*.lfsck_* | + awk '/repaired/ { print $2 }' | calc_sum) + local repaired=$((rep_mdt + rep_ost)) + [ $repaired -eq 0 ] || + error "lfsck repaired $rep_mdt MDT and $rep_ost OST errors" } dump_file_contents() { @@ -4318,15 +4810,18 @@ log_zfs_info() { } check_and_cleanup_lustre() { - if [ "$LFSCK_ALWAYS" = "yes" -a "$TESTSUITE" != "lfsck" ]; then - get_svr_devs - generate_db - run_lfsck - fi + if [ "$LFSCK_ALWAYS" = "yes" -a "$TESTSUITE" != "sanity-lfsck" -a \ + "$TESTSUITE" != "sanity-scrub" ]; then + run_lfsck + fi if is_mounted $MOUNT; then - [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* || - error "remove sub-test dirs failed" + if $DO_CLEANUP; then + [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* || + error "remove sub-test dirs failed" + else + echo "skip cleanup" + fi [ "$ENABLE_QUOTA" ] && restore_quota || true fi @@ -4649,37 +5144,15 @@ clear_failloc() { } set_nodes_failloc () { - do_nodes $(comma_list $1) lctl set_param fail_val=0 fail_loc=$2 + local fv=${3:-0} + do_nodes $(comma_list $1) lctl set_param fail_val=$fv fail_loc=$2 } cancel_lru_locks() { - $LCTL mark "cancel_lru_locks $1 start" - - if [ $1 != "MGC" ]; then - for d in $(lctl get_param -N ldlm.namespaces.*.lru_size | - egrep -i $1); do - $LCTL set_param -n $d=clear - done - $LCTL get_param ldlm.namespaces.*.lock_unused_count | egrep -i $1 | - grep -v '=0' - else - for d in $(find \ - /{proc,sys}/fs/lustre/ldlm/namespaces/*$1*/lru_size \ - 2> /dev/null); do - echo "clear" > $d - done - - for d in $(find \ - /{proc,sys}/fs/lustre/ldlm/namespaces/*$1*/lock_unused_count \ - 2> /dev/null); do - if [ $(cat $d) != 0 ]; then - echo "ldlm.namespaces.$(echo "$d" | - cut -f 7 -d'/').lock_unused_count=$(cat $d)" - fi - done - fi - - $LCTL mark "cancel_lru_locks $1 stop" + #$LCTL mark "cancel_lru_locks $1 start" + $LCTL set_param -n ldlm.namespaces.*$1*.lru_size=clear + $LCTL get_param ldlm.namespaces.*$1*.lock_unused_count | grep -v '=0' + #$LCTL mark "cancel_lru_locks $1 stop" } default_lru_size() @@ -4719,13 +5192,21 @@ pgcache_empty() { } debugsave() { - DEBUGSAVE="$(lctl get_param -n debug)" + DEBUGSAVE="$(lctl get_param -n debug)" + DEBUGSAVE_SERVER=$(do_facet $SINGLEMDS "$LCTL get_param -n debug") } debugrestore() { - [ -n "$DEBUGSAVE" ] && \ - do_nodes $(comma_list $(nodes_list)) "$LCTL set_param debug=\\\"${DEBUGSAVE}\\\";" - DEBUGSAVE="" + [ -n "$DEBUGSAVE" ] && + do_nodes $CLIENTS "$LCTL set_param debug=\\\"${DEBUGSAVE}\\\""|| + true + DEBUGSAVE="" + + [ -n "$DEBUGSAVE_SERVER" ] && + do_nodes $(comma_list $(all_server_nodes)) \ + "$LCTL set_param debug=\\\"${DEBUGSAVE_SERVER}\\\"" || + true + DEBUGSAVE_SERVER="" } debug_size_save() { @@ -4756,20 +5237,17 @@ stop_full_debug_logging() { # prints bash call stack print_stack_trace() { + local skip=${1:-1} echo " Trace dump:" - for (( i=1; i < ${#BASH_LINENO[*]} ; i++ )) ; do - local s=${BASH_SOURCE[$i]} - local l=${BASH_LINENO[$i-1]} - local f=${FUNCNAME[$i]} - echo " = $s:$l:$f()" + for (( i=$skip; i < ${#BASH_LINENO[*]} ; i++ )) ; do + local src=${BASH_SOURCE[$i]} + local lineno=${BASH_LINENO[$i-1]} + local funcname=${FUNCNAME[$i]} + echo " = $src:$lineno:$funcname()" done } -################################## -# Test interface -################################## - -error_noexit() { +report_error() { local TYPE=${TYPE:-"FAIL"} local dump=true @@ -4779,10 +5257,8 @@ error_noexit() { dump=false fi - log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ " - print_stack_trace >&2 - + (print_stack_trace 2) >&2 mkdir -p $LOGDIR # We need to dump the logs on all nodes if $dump; then @@ -4801,6 +5277,17 @@ error_noexit() { echo "$@" > $LOGDIR/err fi fi + + # cleanup the env for failed tests + reset_fail_loc +} + +################################## +# Test interface +################################## + +error_noexit() { + report_error "$@" } exit_status () { @@ -4812,12 +5299,13 @@ exit_status () { } error() { - error_noexit "$@" + report_error "$@" exit 1 } error_exit() { - error "$@" + report_error "$@" + exit 1 } # use only if we are ignoring failures for this test, bugno required. @@ -4827,15 +5315,27 @@ error_exit() { error_ignore() { local TYPE="IGNORE ($1)" shift - error_noexit "$@" + report_error "$@" } error_and_remount() { - error_noexit "$@" + report_error "$@" remount_client $MOUNT exit 1 } +# Throw an error if it's not running in vm - usually for performance +# verification +error_not_in_vm() { + local virt=$(running_in_vm) + if [[ -n "$virt" ]]; then + echo "running in VM '$virt', ignore error" + error_ignore env=$virt "$@" + else + error "$@" + fi +} + skip_env () { $FAIL_ON_SKIP_ENV && error false $@ || skip $@ } @@ -5021,17 +5521,17 @@ check_mds() { } reset_fail_loc () { - echo -n "Resetting fail_loc on all nodes..." - do_nodes $(comma_list $(nodes_list)) "lctl set_param -n fail_loc=0 \ - fail_val=0 2>/dev/null || true" - echo done. + echo -n "Resetting fail_loc on all nodes..." + do_nodes $(comma_list $(nodes_list)) "lctl set_param -n fail_loc=0 \ + fail_val=0 2>/dev/null" || true + echo done. } # -# Log a message (on all nodes) padded with "=" before and after. +# Log a message (on all nodes) padded with "=" before and after. # Also appends a timestamp and prepends the testsuite name. -# +# EQUALS="====================================================================================================" banner() { @@ -5043,6 +5543,18 @@ banner() { log "$msg== $(date +"%H:%M:%S (%s)")" } +check_dmesg_for_errors() { + local res + local errors="VFS: Busy inodes after unmount of\|\ +ldiskfs_check_descriptors: Checksum for group 0 failed\|\ +group descriptors corrupted" + + res=$(do_nodes $(comma_list $(nodes_list)) "dmesg" | grep "$errors") + [ -z "$res" ] && return 0 + echo "Kernel error detected: $res" + return 1 +} + # # Run a single test function and cleanup after it. # @@ -5058,12 +5570,17 @@ run_one() { local SAVE_UMASK=`umask` umask 0022 + if ! grep -q $DIR /proc/mounts; then + $SETUP + fi + banner "test $testnum: $message" test_${testnum} || error "test_$testnum failed with $?" cd $SAVE_PWD reset_fail_loc check_grant ${testnum} || error "check_grant $testnum failed with $?" - check_catastrophe || error "LBUG/LASSERT detected" + check_node_health + check_dmesg_for_errors || error "Error in dmesg detected" if [ "$PARALLEL" != "yes" ]; then ps auxww | grep -v grep | grep -q multiop && error "multiop still running" @@ -5072,6 +5589,7 @@ run_one() { unset tdir unset tfile umask $SAVE_UMASK + $CLEANUP return 0 } @@ -5146,8 +5664,9 @@ check_grant() { export base=$(basetest $1) [ "$CHECK_GRANT" == "no" ] && return 0 - testname=GCHECK_ONLY_${base} - [ ${!testname}x == x ] && return 0 + testnamebase=GCHECK_ONLY_${base} + testname=GCHECK_ONLY_$1 + [ ${!testnamebase}x == x -a ${!testname}x == x ] && return 0 echo -n "checking grant......" @@ -5163,17 +5682,22 @@ check_grant() { awk '{ total += $1 } END { printf("%0.0f", total) }') # get server grant + # which is tot_granted less grant_precreate server_grant=$(do_nodes $(comma_list $(osts_nodes)) \ - "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" | - awk '{ total += $1 } END { printf("%0.0f", total) }') + "$LCTL get_param "\ + "obdfilter.${FSNAME}-OST*.{tot_granted,tot_pending,grant_precreate}" | + sed 's/=/ /'| awk '/tot_granted/{ total += $2 }; + /tot_pending/{ total -= $2 }; + /grant_precreate/{ total -= $2 }; + END { printf("%0.0f", total) }') # check whether client grant == server grant if [[ $client_grant -ne $server_grant ]]; then - echo "failed: client:${client_grant} server: ${server_grant}." do_nodes $(comma_list $(osts_nodes)) \ - "$LCTL get_param obdfilter.${FSNAME}-OST*.tot*" + "$LCTL get_param obdfilter.${FSNAME}-OST*.tot*" \ + "obdfilter.${FSNAME}-OST*.grant_*" do_nodes $clients "$LCTL get_param osc.${FSNAME}-*.cur_*_bytes" - return 1 + error "failed: client:${client_grant} server: ${server_grant}." else echo "pass: client:${client_grant} server: ${server_grant}" fi @@ -5264,15 +5788,15 @@ remote_mds () remote_mds_nodsh() { - [ "$CLIENTONLY" ] && return 0 || true - remote_mds && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$mds_HOST" ] + [ -n "$CLIENTONLY" ] && return 0 || true + remote_mds && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$mds_HOST" ] } require_dsh_mds() { - remote_mds_nodsh && echo "SKIP: $TESTSUITE: remote MDS with nodsh" && \ - MSKIPPED=1 && return 1 - return 0 + remote_mds_nodsh && echo "SKIP: $TESTSUITE: remote MDS with nodsh" && + MSKIPPED=1 && return 1 + return 0 } remote_ost () @@ -5286,8 +5810,8 @@ remote_ost () remote_ost_nodsh() { - [ "$CLIENTONLY" ] && return 0 || true - remote_ost && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ] + [ -n "$CLIENTONLY" ] && return 0 || true + remote_ost && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ] } require_dsh_ost() @@ -5299,10 +5823,10 @@ require_dsh_ost() remote_mgs_nodsh() { - [ "$CLIENTONLY" ] && return 0 || true - local MGS - MGS=$(facet_host mgs) - remote_node $MGS && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ] + [ -n "$CLIENTONLY" ] && return 0 || true + local MGS + MGS=$(facet_host mgs) + remote_node $MGS && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ] } local_mode () @@ -5331,6 +5855,11 @@ facets_nodes () { echo -n $nodes_sort } +# Get name of the active MGS node. +mgs_node () { + echo -n $(facets_nodes $(get_facets MGS)) +} + # Get all of the active MDS nodes. mdts_nodes () { echo -n $(facets_nodes $(get_facets MDS)) @@ -5372,7 +5901,7 @@ remote_nodes_list () { all_mdts_nodes () { local host local failover_host - local nodes + local nodes="${mds_HOST} ${mdsfailover_HOST}" local nodes_sort local i @@ -5390,7 +5919,7 @@ all_mdts_nodes () { all_osts_nodes () { local host local failover_host - local nodes + local nodes="${ost_HOST} ${ostfailover_HOST}" local nodes_sort local i @@ -5463,19 +5992,14 @@ get_random_entry () { rnodes=${rnodes//,/ } local -a nodes=($rnodes) - local num=${#nodes[@]} + local num=${#nodes[@]} local i=$((RANDOM * num * 2 / 65536)) echo ${nodes[i]} } client_only () { - [ "$CLIENTONLY" ] || [ "$CLIENTMODSONLY" = yes ] -} - -is_patchless () -{ - lctl get_param version | grep -q patchless + [ -n "$CLIENTONLY" ] || [ "x$CLIENTMODSONLY" = "xyes" ] } check_versions () { @@ -5520,13 +6044,19 @@ get_stripe () { setstripe_nfsserver () { local dir=$1 + local nfsexportdir=$2 + shift + shift - local nfsserver=$(awk '"'$dir'" ~ $2 && $3 ~ "nfs" && $2 != "/" \ - { print $1 }' /proc/mounts | cut -f 1 -d : | head -n1) + local -a nfsexport=($(awk '"'$dir'" ~ $2 && $3 ~ "nfs" && $2 != "/" \ + { print $1 }' /proc/mounts | cut -f 1 -d :)) - [ -z $nfsserver ] && echo "$dir is not nfs mounted" && return 1 + # check that only one nfs mounted + [[ -z $nfsexport ]] && echo "$dir is not nfs mounted" && return 1 + (( ${#nfsexport[@]} == 1 )) || + error "several nfs mounts found for $dir: ${nfsexport[@]} !" - do_nodev $nfsserver lfs setstripe "$@" + do_nodev ${nfsexport[0]} lfs setstripe $nfsexportdir "$@" } # Check and add a test group. @@ -5696,32 +6226,22 @@ inodes_available () { } mdsrate_inodes_available () { - local min_inodes=$(inodes_available) - echo $((min_inodes * 99 / 100)) -} - -# reset llite stat counters -clear_llite_stats(){ - lctl set_param -n llite.*.stats 0 -} - -# sum llite stat items -calc_llite_stats() { - local res=$(lctl get_param -n llite.*.stats | - awk '/^'"$1"'/ {sum += $2} END { printf("%0.0f", sum) }') - echo $((res)) + local min_inodes=$(inodes_available) + echo $((min_inodes * 99 / 100)) } -# reset osc stat counters -clear_osc_stats(){ - lctl set_param -n osc.*.osc_stats 0 +# reset stat counters +clear_stats() { + local paramfile="$1" + lctl set_param -n $paramfile=0 } -# sum osc stat items -calc_osc_stats() { - local res=$(lctl get_param -n osc.*.osc_stats | - awk '/^'"$1"'/ {sum += $2} END { printf("%0.0f", sum) }') - echo $((res)) +# sum stat items +calc_stats() { + local paramfile="$1" + local stat="$2" + lctl get_param -n $paramfile | + awk '/^'$stat'/ { sum += $2 } END { printf("%0.0f", sum) }' } calc_sum () { @@ -5729,8 +6249,8 @@ calc_sum () { } calc_osc_kbytes () { - df $MOUNT > /dev/null - $LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum + df $MOUNT > /dev/null + $LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum } # save_lustre_params(comma separated facet list, parameter_mask) @@ -5738,16 +6258,17 @@ calc_osc_kbytes () { save_lustre_params() { local facets=$1 local facet - local nodes - local node + local facet_svc for facet in ${facets//,/ }; do - node=$(facet_active_host $facet) - [[ *\ $node\ * = " $nodes " ]] && continue - nodes="$nodes $node" - - do_node $node "$LCTL get_param $2 | - while read s; do echo $facet \\\$s; done" + facet_svc=$(facet_svc $facet) + do_facet $facet \ + "params=\\\$($LCTL get_param $2); + [[ -z \\\"$facet_svc\\\" ]] && param= || + param=\\\$(grep $facet_svc <<< \\\"\\\$params\\\"); + [[ -z \\\$param ]] && param=\\\"\\\$params\\\"; + while read s; do echo $facet \\\$s; + done <<< \\\"\\\$param\\\"" done } @@ -5762,16 +6283,21 @@ restore_lustre_params() { done } -check_catastrophe() { +check_node_health() { local nodes=${1:-$(comma_list $(nodes_list))} - do_nodes $nodes "rc=0; -val=\\\$($LCTL get_param -n catastrophe 2>&1); -if [[ \\\$? -eq 0 && \\\$val -ne 0 ]]; then - echo \\\$(hostname -s): \\\$val; - rc=\\\$val; -fi; -exit \\\$rc" + for node in ${nodes//,/ }; do + check_network "$node" 5 + if [ $? -eq 0 ]; then + do_node $node "rc=0; + val=\\\$($LCTL get_param -n catastrophe 2>&1); + if [[ \\\$? -eq 0 && \\\$val -ne 0 ]]; then + echo \\\$(hostname -s): \\\$val; + rc=\\\$val; + fi; + exit \\\$rc" || error "$node:LBUG/LASSERT detected" + fi + done } mdsrate_cleanup () { @@ -5963,9 +6489,7 @@ _wait_osc_import_state() { if [[ $facet == client* ]]; then # During setup time, the osc might not be setup, it need wait - # until list_param can return valid value. And also if there - # are mulitple osc entries we should list all of them before - # go to wait. + # until list_param can return valid value. params=$($LCTL list_param $param 2>/dev/null || true) while [ -z "$params" ]; do if [ $i -ge $maxtime ]; then @@ -5989,7 +6513,7 @@ _wait_osc_import_state() { if ! do_rpc_nodes "$(facet_active_host $facet)" \ wait_import_state $expected "$params" $maxtime; then - error "import is not in ${expected} state" + error "$facet: import is not in $expected state after $maxtime" return 1 fi @@ -6133,6 +6657,47 @@ wait_clients_import_state () { fi } +wait_osp_active() { + local facet=$1 + local tgt_name=$2 + local tgt_idx=$3 + local expected=$4 + local num + + # wait until all MDTs are in the expected state + for ((num = 1; num <= $MDSCOUNT; num++)); do + local mdtosp=$(get_mdtosc_proc_path mds${num} ${tgt_name}) + local mproc + + if [ $facet = "mds" ]; then + mproc="osp.$mdtosp.active" + [ $num -eq $((tgt_idx + 1)) ] && continue + else + mproc="osc.$mdtosp.active" + fi + + echo "check $mproc" + while [ 1 ]; do + sleep 5 + local result=$(do_facet mds${num} "$LCTL get_param -n $mproc") + local max=30 + local wait=0 + + [ ${PIPESTATUS[0]} = 0 ] || error "Can't read $mproc" + if [ $result -eq $expected ]; then + echo -n "target updated after" + echo "$wait sec (got $result)" + break + fi + wait=$((wait + 5)) + if [ $wait -eq $max ]; then + error "$tgt_name: wanted $expected got $result" + fi + echo "Waiting $((max - wait)) secs for $tgt_name" + done + done +} + oos_full() { local -a AVAILA local -a GRANTA @@ -6160,100 +6725,119 @@ oos_full() { return $OSCFULL } -pool_list () { - do_facet mgs lctl pool_list $1 +list_pool() { + echo -e "$(do_facet $SINGLEMDS $LCTL pool_list $1 | sed '1d')" +} + +check_pool_not_exist() { + local fsname=${1%%.*} + local poolname=${1##$fsname.} + [[ $# -ne 1 ]] && return 0 + [[ x$poolname = x ]] && return 0 + list_pool $fsname | grep -w $1 && return 1 + return 0 } create_pool() { - local fsname=${1%%.*} - local poolname=${1##$fsname.} - - do_facet mgs lctl pool_new $1 - local RC=$? - # get param should return err unless pool is created - [[ $RC -ne 0 ]] && return $RC - - wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ - 2>/dev/null || echo foo" "" || RC=1 - if [[ $RC -eq 0 ]]; then - add_pool_to_list $1 - else - error "pool_new failed $1" - fi - return $RC + local fsname=${1%%.*} + local poolname=${1##$fsname.} + + trap "destroy_test_pools $fsname" EXIT + do_facet mgs lctl pool_new $1 + local RC=$? + # get param should return err unless pool is created + [[ $RC -ne 0 ]] && return $RC + + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$fsname-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$poolname \ + 2>/dev/null || echo foo" "" || + error "mds$mds_id: pool_new failed $1" + done + wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ + 2>/dev/null || echo foo" "" || error "pool_new failed $1" + + add_pool_to_list $1 + return $RC } add_pool_to_list () { - local fsname=${1%%.*} - local poolname=${1##$fsname.} + local fsname=${1%%.*} + local poolname=${1##$fsname.} - local listvar=${fsname}_CREATED_POOLS - eval export ${listvar}=$(expand_list ${!listvar} $poolname) + local listvar=${fsname}_CREATED_POOLS + local temp=${listvar}=$(expand_list ${!listvar} $poolname) + eval export $temp } remove_pool_from_list () { - local fsname=${1%%.*} - local poolname=${1##$fsname.} + local fsname=${1%%.*} + local poolname=${1##$fsname.} - local listvar=${fsname}_CREATED_POOLS - eval export ${listvar}=$(exclude_items_from_list ${!listvar} $poolname) + local listvar=${fsname}_CREATED_POOLS + local temp=${listvar}=$(exclude_items_from_list ${!listvar} $poolname) + eval export $temp } destroy_pool_int() { - local ost - local OSTS=$(do_facet $SINGLEMDS lctl pool_list $1 | \ - awk '$1 !~ /^Pool:/ {print $1}') - for ost in $OSTS; do - do_facet mgs lctl pool_remove $1 $ost - done - do_facet mgs lctl pool_destroy $1 + local ost + local OSTS=$(list_pool $1) + for ost in $OSTS; do + do_facet mgs lctl pool_remove $1 $ost + done + do_facet mgs lctl pool_destroy $1 } # . or destroy_pool() { - local fsname=${1%%.*} - local poolname=${1##$fsname.} + local fsname=${1%%.*} + local poolname=${1##$fsname.} - [[ x$fsname = x$poolname ]] && fsname=$FSNAME + [[ x$fsname = x$poolname ]] && fsname=$FSNAME - local RC + local RC - pool_list $fsname.$poolname || return $? + check_pool_not_exist $fsname.$poolname + [[ $? -eq 0 ]] && return 0 - destroy_pool_int $fsname.$poolname - RC=$? - [[ $RC -ne 0 ]] && return $RC + destroy_pool_int $fsname.$poolname + RC=$? + [[ $RC -ne 0 ]] && return $RC + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$fsname-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$poolname \ + 2>/dev/null || echo foo" "foo" || + error "mds$mds_id: destroy pool failed $1" + done + wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ + 2>/dev/null || echo foo" "foo" || error "destroy pool failed $1" - wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ - 2>/dev/null || echo foo" "foo" || RC=1 + remove_pool_from_list $fsname.$poolname - if [[ $RC -eq 0 ]]; then - remove_pool_from_list $fsname.$poolname - else - error "destroy pool failed $1" - fi - return $RC + return $RC } destroy_pools () { - local fsname=${1:-$FSNAME} - local poolname - local listvar=${fsname}_CREATED_POOLS - - pool_list $fsname + local fsname=${1:-$FSNAME} + local poolname + local listvar=${fsname}_CREATED_POOLS - [ x${!listvar} = x ] && return 0 + [ x${!listvar} = x ] && return 0 - echo destroy the created pools: ${!listvar} - for poolname in ${!listvar//,/ }; do - destroy_pool $fsname.$poolname - done + echo "Destroy the created pools: ${!listvar}" + for poolname in ${!listvar//,/ }; do + destroy_pool $fsname.$poolname + done } -cleanup_pools () { - local fsname=${1:-$FSNAME} - trap 0 - destroy_pools $fsname +destroy_test_pools () { + trap 0 + local fsname=${1:-$FSNAME} + destroy_pools $fsname || true } gather_logs () { @@ -6276,7 +6860,7 @@ gather_logs () { suffix="$ts.log" echo "Dumping lctl log to ${prefix}.*.${suffix}" - if [ "$CLIENTONLY" -o "$PDSH" == "no_dsh" ]; then + if [ -n "$CLIENTONLY" -o "$PDSH" == "no_dsh" ]; then echo "Dumping logs only on local client." $LCTL dk > ${prefix}.debug_log.$(hostname -s).${suffix} dmesg > ${prefix}.dmesg.$(hostname -s).${suffix} @@ -6286,6 +6870,7 @@ gather_logs () { do_nodesv $list \ "$LCTL dk > ${prefix}.debug_log.\\\$(hostname -s).${suffix}; dmesg > ${prefix}.dmesg.\\\$(hostname -s).${suffix}" + if [ ! -f $LOGDIR/shared ]; then do_nodes $list rsync -az "${prefix}.*.${suffix}" $HOSTNAME:$LOGDIR fi @@ -6333,16 +6918,43 @@ max_recovery_time() { echo -n $service_time } +recovery_time_min() { + local connection_switch_min=5 + local connection_switch_inc=5 + local connection_switch_max + local reconnect_delay_max + local initial_connect_timeout + local max + local timout_20 + + #connection_switch_max=min(50, max($connection_switch_min,$TIMEOUT) + (($connection_switch_min > $TIMEOUT)) && + max=$connection_switch_min || max=$TIMEOUT + (($max < 50)) && connection_switch_max=$max || connection_switch_max=50 + + #initial_connect_timeout = max(connection_switch_min, obd_timeout/20) + timeout_20=$((TIMEOUT/20)) + (($connection_switch_min > $timeout_20)) && + initial_connect_timeout=$connection_switch_min || + initial_connect_timeout=$timeout_20 + + reconnect_delay_max=$((connection_switch_max + connection_switch_inc + \ + initial_connect_timeout)) + echo $((2 * reconnect_delay_max)) +} + get_clients_mount_count () { - local clients=${CLIENTS:-`hostname`} + local clients=${CLIENTS:-$HOSTNAME} - # we need to take into account the clients mounts and - # exclude mds/ost mounts if any; - do_nodes $clients cat /proc/mounts | grep lustre | grep $MOUNT | wc -l + # we need to take into account the clients mounts and + # exclude mds/ost mounts if any; + do_nodes $clients cat /proc/mounts | grep lustre | + grep -w $MOUNT | wc -l } # gss functions PROC_CLI="srpc_info" +PROC_CON="srpc_contexts" combination() { @@ -6365,28 +6977,39 @@ combination() } calc_connection_cnt() { - local dir=$1 + local dir=$1 - # MDT->MDT = 2 * C(M, 2) - # MDT->OST = M * O - # CLI->OST = C * O - # CLI->MDT = C * M - comb_m2=$(combination $MDSCOUNT 2) + # MDT->MDT = 2 * C(M, 2) + # MDT->OST = M * O + # CLI->OST = C * O + # CLI->MDT = C * M + comb_m2=$(combination $MDSCOUNT 2) - local num_clients=$(get_clients_mount_count) + local num_clients=$(get_clients_mount_count) - local cnt_mdt2mdt=$((comb_m2 * 2)) - local cnt_mdt2ost=$((MDSCOUNT * OSTCOUNT)) - local cnt_cli2ost=$((num_clients * OSTCOUNT)) - local cnt_cli2mdt=$((num_clients * MDSCOUNT)) - local cnt_all2ost=$((cnt_mdt2ost + cnt_cli2ost)) - local cnt_all2mdt=$((cnt_mdt2mdt + cnt_cli2mdt)) - local cnt_all2all=$((cnt_mdt2ost + cnt_mdt2mdt + cnt_cli2ost + cnt_cli2mdt)) + local cnt_mdt2mdt=$((comb_m2 * 2)) + local cnt_mdt2ost=$((MDSCOUNT * OSTCOUNT)) + local cnt_cli2ost=$((num_clients * OSTCOUNT)) + local cnt_cli2mdt=$((num_clients * MDSCOUNT)) + if is_mounted $MOUNT2; then + cnt_cli2mdt=$((cnt_cli2mdt * 2)) + cnt_cli2ost=$((cnt_cli2ost * 2)) + fi + if local_mode; then + cnt_mdt2mdt=0 + cnt_mdt2ost=0 + cnt_cli2ost=2 + cnt_cli2mdt=1 + fi + local cnt_all2ost=$((cnt_mdt2ost + cnt_cli2ost)) + local cnt_all2mdt=$((cnt_mdt2mdt + cnt_cli2mdt)) + local cnt_all2all=$((cnt_mdt2ost + cnt_mdt2mdt \ + + cnt_cli2ost + cnt_cli2mdt)) - local var=cnt_$dir - local res=${!var} + local var=cnt_$dir + local res=${!var} - echo $res + echo $res } set_rule() @@ -6411,6 +7034,13 @@ set_rule() do_facet mgs "$LCTL conf_param $cmd" } +count_contexts() +{ + local output=$1 + local total_ctx=$(echo "$output" | grep -c "expire.*key.*hdl") + echo $total_ctx +} + count_flvr() { local output=$1 @@ -6452,12 +7082,22 @@ flvr_cnt_cli2mdt() local flavor=$1 local cnt - local clients=${CLIENTS:-`hostname`} + local clients=${CLIENTS:-$HOSTNAME} for c in ${clients//,/ }; do - output=`do_node $c lctl get_param -n mdc.*-MDT*-mdc-*.$PROC_CLI 2>/dev/null` - tmpcnt=`count_flvr "$output" $flavor` - cnt=$((cnt + tmpcnt)) + local output=$(do_node $c lctl get_param -n \ + mdc.*-*-mdc-*.$PROC_CLI 2>/dev/null) + local tmpcnt=$(count_flvr "$output" $flavor) + if $GSS_SK && [ $flavor != "null" ]; then + # tmpcnt=min(contexts,flavors) to ensure SK context is on + output=$(do_node $c lctl get_param -n \ + mdc.*-MDT*-mdc-*.$PROC_CON 2>/dev/null) + local outcon=$(count_contexts "$output") + if [ "$outcon" -lt "$tmpcnt" ]; then + tmpcnt=$outcon + fi + fi + cnt=$((cnt + tmpcnt)) done echo $cnt } @@ -6467,11 +7107,21 @@ flvr_cnt_cli2ost() local flavor=$1 local cnt - local clients=${CLIENTS:-`hostname`} + local clients=${CLIENTS:-$HOSTNAME} for c in ${clients//,/ }; do - output=`do_node $c lctl get_param -n osc.*OST*-osc-[^M][^D][^T]*.$PROC_CLI 2>/dev/null` - tmpcnt=`count_flvr "$output" $flavor` + local output=$(do_node $c lctl get_param -n \ + osc.*OST*-osc-[^M][^D][^T]*.$PROC_CLI 2>/dev/null) + local tmpcnt=$(count_flvr "$output" $flavor) + if $GSS_SK && [ $flavor != "null" ]; then + # tmpcnt=min(contexts,flavors) to ensure SK context is on + output=$(do_node $c lctl get_param -n \ + osc.*OST*-osc-[^M][^D][^T]*.$PROC_CON 2>/dev/null) + local outcon=$(count_contexts "$output") + if [ "$outcon" -lt "$tmpcnt" ]; then + tmpcnt=$outcon + fi + fi cnt=$((cnt + tmpcnt)) done echo $cnt @@ -6488,8 +7138,18 @@ flvr_cnt_mdt2mdt() fi for num in `seq $MDSCOUNT`; do - output=`do_facet mds$num lctl get_param -n mdc.*-MDT*-mdc[0-9]*.$PROC_CLI 2>/dev/null` - tmpcnt=`count_flvr "$output" $flavor` + local output=$(do_facet mds$num lctl get_param -n \ + osp.*-MDT*osp-MDT*.$PROC_CLI 2>/dev/null) + local tmpcnt=$(count_flvr "$output" $flavor) + if $GSS_SK && [ $flavor != "null" ]; then + # tmpcnt=min(contexts,flavors) to ensure SK context is on + output=$(do_facet mds$num lctl get_param -n \ + osp.*-MDT*osp-MDT*.$PROC_CON 2>/dev/null) + local outcon=$(count_contexts "$output") + if [ "$outcon" -lt "$tmpcnt" ]; then + tmpcnt=$outcon + fi + fi cnt=$((cnt + tmpcnt)) done echo $cnt; @@ -6504,9 +7164,18 @@ flvr_cnt_mdt2ost() for num in `seq $MDSCOUNT`; do mdtosc=$(get_mdtosc_proc_path mds$num) mdtosc=${mdtosc/-MDT*/-MDT\*} - output=$(do_facet mds$num lctl get_param -n \ - osc.$mdtosc.$PROC_CLI 2>/dev/null) - tmpcnt=`count_flvr "$output" $flavor` + local output=$(do_facet mds$num lctl get_param -n \ + osc.$mdtosc.$PROC_CLI 2>/dev/null) + local tmpcnt=$(count_flvr "$output" $flavor) + if $GSS_SK && [ $flavor != "null" ]; then + # tmpcnt=min(contexts,flavors) to ensure SK context is on + output=$(do_facet mds$num lctl get_param -n \ + osc.$mdtosc.$PROC_CON 2>/dev/null) + local outcon=$(count_contexts "$output") + if [ "$outcon" -lt "$tmpcnt" ]; then + tmpcnt=$outcon + fi + fi cnt=$((cnt + tmpcnt)) done echo $cnt; @@ -6516,7 +7185,8 @@ flvr_cnt_mgc2mgs() { local flavor=$1 - output=`do_facet client lctl get_param -n mgc.*.$PROC_CLI 2>/dev/null` + local output=$(do_facet client lctl get_param -n mgc.*.$PROC_CLI \ + 2>/dev/null) count_flvr "$output" $flavor } @@ -6555,75 +7225,109 @@ do_check_flavor() wait_flavor() { - local dir=$1 # from to - local flavor=$2 # flavor expected - local expect=${3:-$(calc_connection_cnt $dir)} # number expected - - local res=0 - - for ((i=0;i<20;i++)); do - echo -n "checking $dir..." - res=$(do_check_flavor $dir $flavor) - echo "found $res/$expect $flavor connections" - [ $res -ge $expect ] && return 0 - sleep 4 - done + local dir=$1 # from to + local flavor=$2 # flavor expected + local expect=${3:-$(calc_connection_cnt $dir)} # number expected + local WAITFLAVOR_MAX=20 # how many retries before abort? + + local res=0 + for ((i = 0; i < $WAITFLAVOR_MAX; i++)); do + echo -n "checking $dir..." + res=$(do_check_flavor $dir $flavor) + echo "found $res/$expect $flavor connections" + [ $res -ge $expect ] && return 0 + sleep 4 + done - echo "Error checking $flavor of $dir: expect $expect, actual $res" - return 1 + echo "Error checking $flavor of $dir: expect $expect, actual $res" +# echo "Dumping additional logs for SK debug.." + do_nodes $(comma_list $(all_server_nodes)) "keyctl show" + if $dump; then + gather_logs $(comma_list $(nodes_list)) + fi + return 1 } restore_to_default_flavor() { - local proc="mgs.MGS.live.$FSNAME" + local proc="mgs.MGS.live.$FSNAME" - echo "restoring to default flavor..." + echo "restoring to default flavor..." - nrule=`do_facet mgs lctl get_param -n $proc 2>/dev/null | grep ".srpc.flavor." | wc -l` + local nrule=$(do_facet mgs lctl get_param -n $proc 2>/dev/null | + grep ".srpc.flavor" | wc -l) - # remove all existing rules if any - if [ $nrule -ne 0 ]; then - echo "$nrule existing rules" - for rule in `do_facet mgs lctl get_param -n $proc 2>/dev/null | grep ".srpc.flavor."`; do - echo "remove rule: $rule" - spec=`echo $rule | awk -F = '{print $1}'` - do_facet mgs "$LCTL conf_param -d $spec" - done - fi + # remove all existing rules if any + if [ $nrule -ne 0 ]; then + echo "$nrule existing rules" + for rule in $(do_facet mgs lctl get_param -n $proc 2>/dev/null | + grep ".srpc.flavor."); do + echo "remove rule: $rule" + spec=`echo $rule | awk -F = '{print $1}'` + do_facet mgs "$LCTL conf_param -d $spec" + done + fi - # verify no rules left - nrule=`do_facet mgs lctl get_param -n $proc 2>/dev/null | grep ".srpc.flavor." | wc -l` - [ $nrule -ne 0 ] && error "still $nrule rules left" + # verify no rules left + nrule=$(do_facet mgs lctl get_param -n $proc 2>/dev/null | + grep ".srpc.flavor." | wc -l) + [ $nrule -ne 0 ] && error "still $nrule rules left" - # wait for default flavor to be applied - # currently default flavor for all connections are 'null' - wait_flavor all2all null - echo "now at default flavor settings" + # wait for default flavor to be applied + if $GSS_SK; then + if $SK_S2S; then + set_rule $FSNAME any any $SK_FLAVOR + wait_flavor all2all $SK_FLAVOR + else + set_rule $FSNAME any cli2mdt $SK_FLAVOR + set_rule $FSNAME any cli2ost $SK_FLAVOR + wait_flavor cli2mdt $SK_FLAVOR + wait_flavor cli2ost $SK_FLAVOR + fi + echo "GSS_SK now at default flavor: $SK_FLAVOR" + else + wait_flavor all2all null + fi } set_flavor_all() { - local flavor=${1:-null} + local flavor=${1:-null} - echo "setting all flavor to $flavor" + echo "setting all flavor to $flavor" - # FIXME need parameter to this fn - # and remove global vars - local cnt_all2all=$(calc_connection_cnt all2all) + # FIXME need parameter to this fn + # and remove global vars + local cnt_all2all=$(calc_connection_cnt all2all) - local res=$(do_check_flavor all2all $flavor) - if [ $res -eq $cnt_all2all ]; then - echo "already have total $res $flavor connections" - return - fi + local res=$(do_check_flavor all2all $flavor) + if [ $res -eq $cnt_all2all ]; then + echo "already have total $res $flavor connections" + return + fi - echo "found $res $flavor out of total $cnt_all2all connections" - restore_to_default_flavor + echo "found $res $flavor out of total $cnt_all2all connections" + restore_to_default_flavor - [[ $flavor = null ]] && return 0 + [[ $flavor = null ]] && return 0 - set_rule $FSNAME any any $flavor - wait_flavor all2all $flavor + if $GSS_SK && [ $flavor != "null" ]; then + if $SK_S2S; then + set_rule $FSNAME any any $flavor + wait_flavor all2all $flavor + else + set_rule $FSNAME any cli2mdt $flavor + set_rule $FSNAME any cli2ost $flavor + set_rule $FSNAME any mdt2ost null + set_rule $FSNAME any mdt2mdt null + wait_flavor cli2mdt $flavor + wait_flavor cli2ost $flavor + fi + echo "GSS_SK now at flavor: $flavor" + else + set_rule $FSNAME any any $flavor + wait_flavor all2all $flavor + fi } @@ -6657,29 +7361,36 @@ check_write_access() { } init_logging() { - if [[ -n $YAML_LOG ]]; then - return - fi - local SAVE_UMASK=`umask` - umask 0000 - - export YAML_LOG=${LOGDIR}/results.yml - mkdir -p $LOGDIR - init_clients_lists - - if [ ! -f $YAML_LOG ]; then # If the yaml log already exists then we will just append to it - if check_shared_dir $LOGDIR; then - touch $LOGDIR/shared - echo "Logging to shared log directory: $LOGDIR" - else - echo "Logging to local directory: $LOGDIR" - fi + [[ -n $YAML_LOG ]] && return + local save_umask=$(umask) + umask 0000 - yml_nodes_file $LOGDIR >> $YAML_LOG - yml_results_file >> $YAML_LOG - fi + export YAML_LOG=${LOGDIR}/results.yml + mkdir -p $LOGDIR + init_clients_lists + + # If the yaml log already exists then we will just append to it + if [ ! -f $YAML_LOG ]; then + if check_shared_dir $LOGDIR; then + touch $LOGDIR/shared + echo "Logging to shared log directory: $LOGDIR" + else + echo "Logging to local directory: $LOGDIR" + fi + + yml_nodes_file $LOGDIR >> $YAML_LOG + yml_results_file >> $YAML_LOG + fi + + umask $save_umask - umask $SAVE_UMASK + # If modules are not yet loaded then older "lctl lustre_build_version" + # will fail. Use lctl build version instead. + log "Client: $($LCTL lustre_build_version)" + log "MDS: $(do_facet $SINGLEMDS $LCTL lustre_build_version 2>/dev/null|| + do_facet $SINGLEMDS $LCTL --version)" + log "OSS: $(do_facet ost1 $LCTL lustre_build_version 2> /dev/null || + do_facet ost1 $LCTL --version)" } log_test() { @@ -6859,7 +7570,7 @@ is_sanity_benchmark() { } min_ost_size () { - $LCTL get_param -n osc.*.kbytesavail | sort -n | head -n1 + $LFS df | grep OST | awk '{print $4}' | sort -un | head -1 } # @@ -6881,11 +7592,10 @@ get_obd_size() { # get_page_size() { local facet=$1 - local size + local size=$(getconf PAGE_SIZE 2>/dev/null) - size=$(do_facet $facet getconf PAGE_SIZE) - [[ ${PIPESTATUS[0]} = 0 && -n "$size" ]] || size=4096 - echo -n $size + [ -z "$CLIENTONLY" ] && size=$(do_facet $facet getconf PAGE_SIZE) + echo -n ${size:-4096} } # @@ -6896,20 +7606,20 @@ get_block_count() { local device=$2 local count - count=$(do_facet $facet "$DUMPE2FS -h $device 2>&1" | + [ -z "$CLIENTONLY" ] && count=$(do_facet $facet "$DUMPE2FS -h $device 2>&1" | awk '/^Block count:/ {print $3}') - echo -n $count + echo -n ${count:-0} } # Get the block size of the filesystem. get_block_size() { - local facet=$1 - local device=$2 - local size + local facet=$1 + local device=$2 + local size - size=$(do_facet $facet "$DUMPE2FS -h $device 2>&1" | - awk '/^Block size:/ {print $3}') - echo $size + [ -z "$CLIENTONLY" ] && size=$(do_facet $facet "$DUMPE2FS -h $device 2>&1" | + awk '/^Block size:/ {print $3}') + echo -n ${size:-0} } # Check whether the "large_xattr" feature is enabled or not. @@ -7006,7 +7716,7 @@ mds_backup_restore() { echo "backup data" ${rcmd} tar zcf $metadata -C $mntpt/ . > /dev/null 2>&1 || return 3 # step 6: umount - ${rcmd} umount -d $mntpt || return 4 + ${rcmd} $UMOUNT $mntpt || return 4 # step 8: reformat dev echo "reformat new device" format_mdt $(facet_number $facet) @@ -7022,7 +7732,7 @@ mds_backup_restore() { echo "remove recovery logs" ${rcmd} rm -fv $mntpt/OBJECTS/* $mntpt/CATALOGS # step 13: umount dev - ${rcmd} umount -d $mntpt || return 10 + ${rcmd} $UMOUNT $mntpt || return 10 # step 14: cleanup tmp backup ${rcmd} rm -f $metaea $metadata # step 15: reset device label - it's not virgin on @@ -7062,7 +7772,7 @@ mds_remove_ois() { done fi # step 4: umount - ${rcmd} umount -d $mntpt || return 2 + ${rcmd} $UMOUNT $mntpt || return 2 # OI files will be recreated when mounted as lustre next time. } @@ -7099,11 +7809,14 @@ test_mkdir() { local parent=$(dirname $path) [ -d $path ] && return 0 - [ ! -d ${parent} ] && mkdir -p ${parent} + if [ ! -d ${parent} ]; then + mkdir -p ${parent} || + error "mkdir parent '$parent' failed" + fi fi if [ $MDSCOUNT -le 1 ]; then - mkdir $path + mkdir $path || error "mkdir '$path' failed" else local test_num=$(echo $testnum | sed -e 's/[^0-9]*//g') local mdt_index @@ -7114,20 +7827,27 @@ test_mkdir() { mdt_index=$stripe_index fi echo "striped dir -i$mdt_index -c$stripe_count $path" - $LFS setdirstripe -i$mdt_index -c$stripe_count $path + $LFS mkdir -i$mdt_index -c$stripe_count $path || + error "mkdir -i $mdt_index -c$stripe_count $path failed" fi } -# find the smallest and not in use file descriptor +# free_fd: find the smallest and not in use file descriptor [above @last_fd] +# +# If called many times, passing @last_fd will avoid repeated searching +# already-open FDs repeatedly if we know they are still in use. +# +# usage: free_fd [last_fd] free_fd() { - local max_fd=$(ulimit -n) - local fd=3 - while [[ $fd -le $max_fd && -e /proc/self/fd/$fd ]]; do - ((++fd)) - done - [ $fd -lt $max_fd ] || error "finding free file descriptor failed" - echo $fd + local max_fd=$(ulimit -n) + local fd=$((${1:-2} + 1)) + + while [[ $fd -le $max_fd && -e /proc/self/fd/$fd ]]; do + ((++fd)) + done + [ $fd -lt $max_fd ] || error "finding free file descriptor failed" + echo $fd } check_mount_and_prep() @@ -7199,6 +7919,18 @@ pool_add_targets() { local t=$(for i in $list; do printf "$FSNAME-OST%04x_UUID " $i; done) do_facet mgs $LCTL pool_add \ $FSNAME.$pool $FSNAME-OST[$first-$last/$step] + + # wait for OSTs to be added to the pool + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$pool | + sort -u | tr '\n' ' ' " "$t" || { + error_noexit "mds$mds_id: Add to pool failed" + return 3 + } + done wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool \ | sort -u | tr '\n' ' ' " "$t" || { error_noexit "Add to pool failed" @@ -7335,6 +8067,17 @@ pool_remove_first_target() { local pname="lov.$FSNAME-*.pools.$pool" local t=$($LCTL get_param -n $pname | head -1) do_facet mgs $LCTL pool_remove $FSNAME.$pool $t + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id \ + "lctl get_param -n lod.$lodname.pools.$pool | + grep $t" "" || { + error_noexit "mds$mds_id: $t not removed from" \ + "$FSNAME.$pool" + return 2 + } + done wait_update $HOSTNAME "lctl get_param -n $pname | grep $t" "" || { error_noexit "$t not removed from $FSNAME.$pool" return 1 @@ -7350,6 +8093,15 @@ pool_remove_all_targets() { do do_facet mgs $LCTL pool_remove $FSNAME.$pool $t done + for mds_id in $(seq $MDSCOUNT); do + local mdt_id=$((mds_id-1)) + local lodname=$FSNAME-MDT$(printf "%04x" $mdt_id)-mdtlov + wait_update_facet mds$mds_id "lctl get_param -n \ + lod.$lodname.pools.$pool" "" || { + error_noexit "mds$mds_id: Pool $pool not drained" + return 4 + } + done wait_update $HOSTNAME "lctl get_param -n $pname" "" || { error_noexit "Pool $FSNAME.$pool cannot be drained" return 1 @@ -7467,3 +8219,181 @@ killall_process () { do_nodes $clients "killall $signal $name" } + +lsnapshot_create() +{ + do_facet mgs "$LCTL snapshot_create -F $FSNAME $*" +} + +lsnapshot_destroy() +{ + do_facet mgs "$LCTL snapshot_destroy -F $FSNAME $*" +} + +lsnapshot_modify() +{ + do_facet mgs "$LCTL snapshot_modify -F $FSNAME $*" +} + +lsnapshot_list() +{ + do_facet mgs "$LCTL snapshot_list -F $FSNAME $*" +} + +lsnapshot_mount() +{ + do_facet mgs "$LCTL snapshot_mount -F $FSNAME $*" +} + +lsnapshot_umount() +{ + do_facet mgs "$LCTL snapshot_umount -F $FSNAME $*" +} + +lss_err() +{ + local msg=$1 + + do_facet mgs "cat $LSNAPSHOT_LOG" + error $msg +} + +lss_cleanup() +{ + echo "Cleaning test environment ..." + + # Every lsnapshot command takes exclusive lock with others, + # so can NOT destroy the snapshot during list with 'xargs'. + while true; do + local ssname=$(lsnapshot_list | grep snapshot_name | + grep lss_ | awk '{ print $2 }' | head -n 1) + [ -z "$ssname" ] && break + + lsnapshot_destroy -n $ssname -f || + lss_err "Fail to destroy $ssname by force" + done +} + +lss_gen_conf_one() +{ + local facet=$1 + local role=$2 + local idx=$3 + + local host=$(facet_active_host $facet) + local dir=$(dirname $(facet_vdevice $facet)) + local pool=$(zpool_name $facet) + local lfsname=$(zfs_local_fsname $facet) + local label=${FSNAME}-${role}$(printf '%04x' $idx) + + do_facet mgs \ + "echo '$host - $label zfs:${dir}/${pool}/${lfsname} - -' >> \ + $LSNAPSHOT_CONF" +} + +lss_gen_conf() +{ + do_facet mgs "rm -f $LSNAPSHOT_CONF" + echo "Generating $LSNAPSHOT_CONF on MGS ..." + + if ! combined_mgs_mds ; then + [ $(facet_fstype mgs) != zfs ] && + skip "Lustre snapshot 1 only works for ZFS backend" && + exit 0 + + local host=$(facet_active_host mgs) + local dir=$(dirname $(facet_vdevice mgs)) + local pool=$(zpool_name mgs) + local lfsname=$(zfs_local_fsname mgs) + + do_facet mgs \ + "echo '$host - MGS zfs:${dir}/${pool}/${lfsname} - -' \ + >> $LSNAPSHOT_CONF" || lss_err "generate lss conf (mgs)" + fi + + for num in `seq $MDSCOUNT`; do + [ $(facet_fstype mds$num) != zfs ] && + skip "Lustre snapshot 1 only works for ZFS backend" && + exit 0 + + lss_gen_conf_one mds$num MDT $((num - 1)) || + lss_err "generate lss conf (mds$num)" + done + + for num in `seq $OSTCOUNT`; do + [ $(facet_fstype ost$num) != zfs ] && + skip "Lustre snapshot 1 only works for ZFS backend" && + exit 0 + + lss_gen_conf_one ost$num OST $((num - 1)) || + lss_err "generate lss conf (ost$num)" + done + + do_facet mgs "cat $LSNAPSHOT_CONF" +} + +parse_plain_param() +{ + local line=$1 + local val=$(awk '{print $2}' <<< $line) + + if [[ $line =~ ^"lmm_stripe_count:" ]]; then + echo "-c $val" + elif [[ $line =~ ^"lmm_stripe_size:" ]]; then + echo "-S $val" + elif [[ $line =~ ^"lmm_stripe_offset:" ]]; then + echo "-i $val" + fi +} + +parse_layout_param() +{ + local mode="" + local val="" + local param="" + + while read line; do + if [[ -z $mode ]]; then + if [[ $line =~ ^"stripe_count:" ]]; then + mode="plain_dir" + elif [[ $line =~ ^"lmm_stripe_count:" ]]; then + mode="plain_file" + elif [[ $line =~ ^"lcm_layout_gen:" ]]; then + mode="pfl" + fi + fi + + if [[ $mode = "plain_dir" ]]; then + param=$(echo $line | + awk '{printf("-c %d -S %d -i %d",$2,$4,$6)}') + elif [[ $mode = "plain_file" ]]; then + val=$(parse_plain_param "$line") + [[ ! -z $val ]] && param="$param $val" + elif [[ $mode = "pfl" ]]; then + val=$(echo $line | awk '{print $2}') + if [[ $line =~ ^"lcme_extent.e_end:" ]]; then + if [[ $val = "EOF" ]]; then + param="$param -E -1" + else + param="$param -E $val" + fi + elif [[ $line =~ ^"stripe_count:" ]]; then + # pfl dir + val=$(echo $line | + awk '{printf("-c %d -S %d -i %d",$2,$4,$6)}') + param="$param $val" + else + #pfl file + val=$(parse_plain_param "$line") + [[ ! -z $val ]] && param="$param $val" + fi + fi + done + echo "$param" +} + +get_layout_param() +{ + local param=$($LFS getstripe -d $1 | parse_layout_param) + echo "$param" +}