3 trap 'print_summary && print_stack_trace | tee $TF_FAIL && \
4 echo "$TESTSUITE: FAIL: test-framework exiting on error"' ERR
8 export REFORMAT=${REFORMAT:-""}
9 export WRITECONF=${WRITECONF:-""}
10 export VERBOSE=${VERBOSE:-false}
11 export GSS=${GSS:-false}
12 export GSS_SK=${GSS_SK:-false}
14 export GSS_PIPEFS=false
15 export SHARED_KEY=${SHARED_KEY:-false}
16 export SK_PATH=${SK_PATH:-/tmp/test-framework-keys}
17 export SK_OM_PATH=$SK_PATH'/tmp-request-mount'
18 export SK_MOUNTED=${SK_MOUNTED:-false}
19 export SK_FLAVOR=${SK_FLAVOR:-ski}
20 export SK_NO_KEY=${SK_NO_KEY:-true}
21 export SK_UNIQUE_NM=${SK_UNIQUE_NM:-false}
22 export SK_S2S=${SK_S2S:-false}
23 export SK_S2SNM=${SK_S2SNM:-TestFrameNM}
24 export SK_S2SNMCLI=${SK_S2SNMCLI:-TestFrameNMCli}
25 export SK_SKIPFIRST=${SK_SKIPFIRST:-true}
26 export IDENTITY_UPCALL=default
28 export FLAKEY=${FLAKEY:-true}
29 # specify environment variable containing batch job name for server statistics
30 export JOBID_VAR=${JOBID_VAR:-"procname_uid"} # or "existing" or "disable"
32 #export PDSH="pdsh -S -Rssh -w"
33 export MOUNT_CMD=${MOUNT_CMD:-"mount -t lustre"}
34 export UMOUNT=${UMOUNT:-"umount -d"}
36 export LSNAPSHOT_CONF="/etc/ldev.conf"
37 export LSNAPSHOT_LOG="/var/log/lsnapshot.log"
39 # sles12 umount has a issue with -d option
40 [ -e /etc/SuSE-release ] && grep -w VERSION /etc/SuSE-release | grep -wq 12 && {
41 export UMOUNT="umount"
44 # function used by scripts run on remote nodes
45 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
46 . $LUSTRE/tests/functions.sh
47 . $LUSTRE/tests/yaml.sh
49 export LD_LIBRARY_PATH=${LUSTRE}/utils/.libs:${LUSTRE}/utils:${LD_LIBRARY_PATH}
51 LUSTRE_TESTS_CFG_DIR=${LUSTRE_TESTS_CFG_DIR:-${LUSTRE}/tests/cfg}
53 EXCEPT_LIST_FILE=${EXCEPT_LIST_FILE:-${LUSTRE_TESTS_CFG_DIR}/tests-to-skip.sh}
55 if [ -f "$EXCEPT_LIST_FILE" ]; then
56 echo "Reading test skip list from $EXCEPT_LIST_FILE"
61 # check config files for options in decreasing order of preference
62 [ -z "$MODPROBECONF" -a -f /etc/modprobe.d/lustre.conf ] &&
63 MODPROBECONF=/etc/modprobe.d/lustre.conf
64 [ -z "$MODPROBECONF" -a -f /etc/modprobe.d/Lustre ] &&
65 MODPROBECONF=/etc/modprobe.d/Lustre
66 [ -z "$MODPROBECONF" -a -f /etc/modprobe.conf ] &&
67 MODPROBECONF=/etc/modprobe.conf
69 sanitize_parameters() {
70 for i in DIR DIR1 DIR2 MOUNT MOUNT1 MOUNT2
73 if [ -d "$path" ]; then
74 eval export $i=$(echo $path | sed -r 's/\/+$//g')
80 [[ $DIR/ = $MOUNT/* ]] ||
81 { failed=1 && echo "DIR=$DIR not in $MOUNT. Aborting."; }
82 [[ $DIR1/ = $MOUNT1/* ]] ||
83 { failed=1 && echo "DIR1=$DIR1 not in $MOUNT1. Aborting."; }
84 [[ $DIR2/ = $MOUNT2/* ]] ||
85 { failed=1 && echo "DIR2=$DIR2 not in $MOUNT2. Aborting"; }
87 [ -n "$failed" ] && exit 99 || true
91 echo "usage: $0 [-r] [-f cfgfile]"
99 [ -z "$DEFAULT_SUITES"] && return 0
100 [ -n "$ONLY" ] && echo "WARNING: ONLY is set to $(echo $ONLY)"
102 local form="%-13s %-17s %-9s %s %s\n"
103 printf "$form" "status" "script" "Total(sec)" "E(xcluded) S(low)"
104 echo "------------------------------------------------------------------------------------"
105 for O in $DEFAULT_SUITES; do
106 O=$(echo $O | tr "-" "_" | tr "[:lower:]" "[:upper:]")
107 [ "${!O}" = "no" ] && continue || true
108 local o=$(echo $O | tr "[:upper:]_" "[:lower:]-")
109 local log=${TMP}/${o}.log
110 if is_sanity_benchmark $o; then
111 log=${TMP}/sanity-benchmark.log
116 local status=Unfinished
118 skipped=$(grep excluded $log | awk '{ printf " %s", $3 }' |
120 slow=$(egrep "^PASS|^FAIL" $log | tr -d "("| sed s/s\)$//g |
121 sort -nr -k 3 | head -n5 | awk '{ print $2":"$3"s" }')
122 total=$(grep duration $log | awk '{ print $2 }')
123 if [ "${!O}" = "done" ]; then
127 local durations=$(egrep "^PASS|^FAIL" $log |
128 tr -d "("| sed s/s\)$//g |
129 awk '{ print $2":"$3"|" }')
130 details=$(printf "%s\n%s %s %s\n" "$details" \
131 "DDETAILS" "$O" "$(echo $durations)")
134 printf "$form" $status "$O" "${total}" "E=$skipped"
135 printf "$form" "-" "-" "-" "S=$(echo $slow)"
138 for O in $DEFAULT_SUITES; do
139 O=$(echo $O | tr "-" "_" | tr "[:lower:]" "[:upper:]")
140 if [ "${!O}" = "no" ]; then
141 printf "$form" "Skipped" "$O" ""
145 # print the detailed tests durations if DDETAILS=true
151 # Get information about the Lustre environment. The information collected
152 # will be used in Lustre tests.
153 # usage: get_lustre_env
154 # input: No required or optional arguments
155 # output: No return values, environment variables are exported
159 export mds1_FSTYPE=${mds1_FSTYPE:-$(facet_fstype mds1)}
160 export ost1_FSTYPE=${ost1_FSTYPE:-$(facet_fstype ost1)}
162 export MGS_VERSION=$(lustre_version_code mgs)
163 export MDS1_VERSION=$(lustre_version_code mds1)
164 export OST1_VERSION=$(lustre_version_code ost1)
165 export CLIENT_VERSION=$(lustre_version_code client)
169 export LUSTRE=$(absolute_path $LUSTRE)
170 export TESTSUITE=$(basename $0 .sh)
171 export TEST_FAILED=false
172 export FAIL_ON_SKIP_ENV=${FAIL_ON_SKIP_ENV:-false}
173 export RPC_MODE=${RPC_MODE:-false}
174 export DO_CLEANUP=${DO_CLEANUP:-true}
175 export KEEP_ZPOOL=${KEEP_ZPOOL:-false}
176 export CLEANUP_DM_DEV=false
177 export PAGE_SIZE=$(get_page_size client)
179 export MKE2FS=$MKE2FS
180 if [ -z "$MKE2FS" ]; then
181 if which mkfs.ldiskfs >/dev/null 2>&1; then
182 export MKE2FS=mkfs.ldiskfs
188 export DEBUGFS=$DEBUGFS
189 if [ -z "$DEBUGFS" ]; then
190 if which debugfs.ldiskfs >/dev/null 2>&1; then
191 export DEBUGFS=debugfs.ldiskfs
193 export DEBUGFS=debugfs
197 export TUNE2FS=$TUNE2FS
198 if [ -z "$TUNE2FS" ]; then
199 if which tunefs.ldiskfs >/dev/null 2>&1; then
200 export TUNE2FS=tunefs.ldiskfs
202 export TUNE2FS=tune2fs
206 export E2LABEL=$E2LABEL
207 if [ -z "$E2LABEL" ]; then
208 if which label.ldiskfs >/dev/null 2>&1; then
209 export E2LABEL=label.ldiskfs
211 export E2LABEL=e2label
215 export DUMPE2FS=$DUMPE2FS
216 if [ -z "$DUMPE2FS" ]; then
217 if which dumpfs.ldiskfs >/dev/null 2>&1; then
218 export DUMPE2FS=dumpfs.ldiskfs
220 export DUMPE2FS=dumpe2fs
224 export E2FSCK=$E2FSCK
225 if [ -z "$E2FSCK" ]; then
226 if which fsck.ldiskfs >/dev/null 2>&1; then
227 export E2FSCK=fsck.ldiskfs
233 export RESIZE2FS=$RESIZE2FS
234 if [ -z "$RESIZE2FS" ]; then
235 if which resizefs.ldiskfs >/dev/null 2>&1; then
236 export RESIZE2FS=resizefs.ldiskfs
238 export RESIZE2FS=resize2fs
242 export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after test suite
243 export FSCK_MAX_ERR=4 # File system errors left uncorrected
245 export ZFS=${ZFS:-zfs}
246 export ZPOOL=${ZPOOL:-zpool}
247 export ZDB=${ZDB:-zdb}
248 export PARTPROBE=${PARTPROBE:-partprobe}
250 #[ -d /r ] && export ROOT=${ROOT:-/r}
251 export TMP=${TMP:-$ROOT/tmp}
252 export TESTSUITELOG=${TMP}/${TESTSUITE}.log
253 export LOGDIR=${LOGDIR:-${TMP}/test_logs/$(date +%s)}
254 export TESTLOG_PREFIX=$LOGDIR/$TESTSUITE
256 export HOSTNAME=${HOSTNAME:-$(hostname -s)}
257 if ! echo $PATH | grep -q $LUSTRE/utils; then
258 export PATH=$LUSTRE/utils:$PATH
260 if ! echo $PATH | grep -q $LUSTRE/utils/gss; then
261 export PATH=$LUSTRE/utils/gss:$PATH
263 if ! echo $PATH | grep -q $LUSTRE/tests; then
264 export PATH=$LUSTRE/tests:$PATH
266 if ! echo $PATH | grep -q $LUSTRE/../lustre-iokit/sgpdd-survey; then
267 export PATH=$LUSTRE/../lustre-iokit/sgpdd-survey:$PATH
269 export LST=${LST:-"$LUSTRE/../lnet/utils/lst"}
270 [ ! -f "$LST" ] && export LST=$(which lst)
271 export SGPDDSURVEY=${SGPDDSURVEY:-"$LUSTRE/../lustre-iokit/sgpdd-survey/sgpdd-survey")}
272 [ ! -f "$SGPDDSURVEY" ] && export SGPDDSURVEY=$(which sgpdd-survey)
273 export MCREATE=${MCREATE:-mcreate}
274 export MULTIOP=${MULTIOP:-multiop}
275 # Ubuntu, at least, has a truncate command in /usr/bin
276 # so fully path our truncate command.
277 export TRUNCATE=${TRUNCATE:-$LUSTRE/tests/truncate}
278 export FSX=${FSX:-$LUSTRE/tests/fsx}
279 export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mpi/mdsrate"}
280 [ ! -f "$MDSRATE" ] && export MDSRATE=$(which mdsrate 2> /dev/null)
281 if ! echo $PATH | grep -q $LUSTRE/tests/racer; then
282 export PATH=$LUSTRE/tests/racer:$PATH:
284 if ! echo $PATH | grep -q $LUSTRE/tests/mpi; then
285 export PATH=$LUSTRE/tests/mpi:$PATH
287 export RSYNC_RSH=${RSYNC_RSH:-rsh}
289 export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
290 [ ! -f "$LCTL" ] && export LCTL=$(which lctl)
291 export LFS=${LFS:-"$LUSTRE/utils/lfs"}
292 [ ! -f "$LFS" ] && export LFS=$(which lfs)
293 SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"}
294 GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"}
296 export PERM_CMD=${PERM_CMD:-"$LCTL conf_param"}
298 export L_GETIDENTITY=${L_GETIDENTITY:-"$LUSTRE/utils/l_getidentity"}
299 if [ ! -f "$L_GETIDENTITY" ]; then
300 if `which l_getidentity > /dev/null 2>&1`; then
301 export L_GETIDENTITY=$(which l_getidentity)
303 export L_GETIDENTITY=NONE
306 export LL_DECODE_FILTER_FID=${LL_DECODE_FILTER_FID:-"$LUSTRE/utils/ll_decode_filter_fid"}
307 [ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID="ll_decode_filter_fid"
308 export LL_DECODE_LINKEA=${LL_DECODE_LINKEA:-"$LUSTRE/utils/ll_decode_linkea"}
309 [ ! -f "$LL_DECODE_LINKEA" ] && export LL_DECODE_LINKEA="ll_decode_linkea"
310 export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"}
311 [ ! -f "$MKFS" ] && export MKFS="mkfs.lustre"
312 export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"}
313 [ ! -f "$TUNEFS" ] && export TUNEFS="tunefs.lustre"
314 export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} "
315 export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod}
316 [ ! -f "$LUSTRE_RMMOD" ] &&
317 export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null)
318 export LUSTRE_ROUTES_CONVERSION=${LUSTRE_ROUTES_CONVERSION:-$LUSTRE/scripts/lustre_routes_conversion}
319 [ ! -f "$LUSTRE_ROUTES_CONVERSION" ] &&
320 export LUSTRE_ROUTES_CONVERSION=$(which lustre_routes_conversion 2> /dev/null)
321 export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate}
322 [ ! -f "$LFS_MIGRATE" ] &&
323 export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null)
324 export LR_READER=${LR_READER:-"$LUSTRE/utils/lr_reader"}
325 [ ! -f "$LR_READER" ] &&
326 export LR_READER=$(which lr_reader 2> /dev/null)
327 [ -z "$LR_READER" ] && export LR_READER="/usr/sbin/lr_reader"
328 export LSOM_SYNC=${LSOM_SYNC:-"$LUSTRE/utils/llsom_sync"}
329 [ ! -f "$LSOM_SYNC" ] &&
330 export LSOM_SYNC=$(which llsom_sync 2> /dev/null)
331 [ -z "$LSOM_SYNC" ] && export LSOM_SYNC="/usr/sbin/llsom_sync"
332 export NAME=${NAME:-local}
333 export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"}
334 [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] &&
335 export LGSSD=$(which lgssd)
336 export LSVCGSSD=${LSVCGSSD:-"$LUSTRE/utils/gss/lsvcgssd"}
337 [ ! -f "$LSVCGSSD" ] && export LSVCGSSD=$(which lsvcgssd 2> /dev/null)
338 export KRB5DIR=${KRB5DIR:-"/usr/kerberos"}
340 export SAVE_PWD=${SAVE_PWD:-$LUSTRE/tests}
342 export LDEV=${LDEV:-"$LUSTRE/scripts/ldev"}
343 [ ! -f "$LDEV" ] && export LDEV=$(which ldev 2> /dev/null)
345 export DMSETUP=${DMSETUP:-dmsetup}
346 export DM_DEV_PATH=${DM_DEV_PATH:-/dev/mapper}
347 export LOSETUP=${LOSETUP:-losetup}
349 if [ "$ACCEPTOR_PORT" ]; then
350 export PORT_OPT="--port $ACCEPTOR_PORT"
354 $RPC_MODE || echo "Using GSS shared-key feature"
355 which lgss_sk > /dev/null 2>&1 ||
356 error_exit "built with lgss_sk disabled! SEC=$SEC"
364 $RPC_MODE || echo "Using GSS/krb5 ptlrpc security flavor"
365 which lgss_keyring > /dev/null 2>&1 ||
366 error_exit "built with gss disabled! SEC=$SEC"
377 IDENTITY_UPCALL=false
381 export LOAD_MODULES_REMOTE=${LOAD_MODULES_REMOTE:-false}
383 # Paths on remote nodes, if different
384 export RLUSTRE=${RLUSTRE:-$LUSTRE}
385 export RPWD=${RPWD:-$PWD}
386 export I_MOUNTED=${I_MOUNTED:-"no"}
387 export AUSTER_CLEANUP=${AUSTER_CLEANUP:-false}
388 if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \
389 ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \
390 ! -f /lib/modules/$(uname -r)/extra/kernel/fs/lustre/mdt.ko -a \
391 ! -f $LUSTRE/mdt/mdt.ko ]; then
392 export CLIENTMODSONLY=yes
395 export SHUTDOWN_ATTEMPTS=${SHUTDOWN_ATTEMPTS:-3}
396 export OSD_TRACK_DECLARES_LBUG=${OSD_TRACK_DECLARES_LBUG:-"yes"}
400 while getopts "rvwf:" opt $*; do
405 w) WRITECONF=writeconf;;
410 shift $((OPTIND - 1))
413 # print the durations of each test if "true"
414 DDETAILS=${DDETAILS:-false}
415 [ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true
420 export TF_FAIL=${TF_FAIL:-$TMP/tf.fail}
427 ncpts=$(do_facet $facet "lctl get_param -n " \
428 "cpu_partition_table 2>/dev/null| wc -l" || echo 1)
430 if [ $ncpts -eq 0 ]; then
437 # Return a numeric version code based on a version string. The version
438 # code is useful for comparison two version strings to see which is newer.
440 # split arguments like "1.8.6-wc3" into "1", "8", "6", "wc3"
441 eval set -- $(tr "[:punct:]" " " <<< $*)
443 echo -n "$((($1 << 16) | ($2 << 8) | $3))"
446 export LINUX_VERSION=$(uname -r | sed -e "s/\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/")
447 export LINUX_VERSION_CODE=$(version_code ${LINUX_VERSION//\./ })
449 # Report the Lustre build version string (e.g. 1.8.7.3 or 2.4.1).
451 # usage: lustre_build_version
453 # All Lustre versions support "lctl get_param" to report the version of the
454 # code running in the kernel (what our tests are interested in), but it
455 # doesn't work without modules loaded. After 2.9.53 and in upstream kernels
456 # the "version" parameter doesn't include "lustre: " at the beginning.
457 # If that fails, call "lctl lustre_build_version" which prints either (or both)
458 # the userspace and kernel build versions, but until 2.8.55 required root
459 # access to get the Lustre kernel version. If that also fails, fall back to
460 # using "lctl --version", which is easy to parse and works without the kernel
461 # modules, but was only added in 2.6.50 and only prints the lctl tool version,
462 # not the module version, though they are usually the same.
464 # Various commands and their output format for different Lustre versions:
465 # lctl get_param version: 2.9.55
466 # lctl get_param version: lustre: 2.8.53
467 # lctl get_param version: lustre: 2.6.52
468 # kernel: patchless_client
469 # build: v2_6_92_0-2.6.32-431.el6_lustre.x86_64
470 # lctl lustre_build_version: Lustre version: 2.8.53_27_gae67fc01
471 # lctl lustre_build_version: error: lustre_build_version: Permission denied
472 # (as non-root user) lctl version: v2_6_92_0-2.6.32-431.el6.x86_64
473 # lctl lustre_build_version: Lustre version: 2.5.3-2.6.32.26-175.fc12.x86_64
474 # lctl version: 2.5.3-2.6.32..26-175fc12.x86_64
475 # lctl --version: lctl 2.6.50
477 # output: prints version string to stdout in (up to 4) dotted-decimal values
478 lustre_build_version() {
479 local facet=${1:-client}
480 local ver=$(do_facet $facet "$LCTL get_param -n version 2>/dev/null ||
481 $LCTL lustre_build_version 2>/dev/null ||
482 $LCTL --version 2>/dev/null | cut -d' ' -f2")
483 local lver=$(egrep -i "lustre: |version: " <<<"$ver" | head -n 1)
484 [ -n "$lver" ] && ver="$lver"
486 sed -e 's/[^:]*: //' -e 's/^v//' -e 's/[ -].*//' -e 's/_/./g' <<<$ver |
490 # Report the Lustre numeric build version code for the supplied facet.
491 lustre_version_code() {
492 version_code $(lustre_build_version $1)
496 /sbin/lsmod | grep -q "^\<$1\>"
508 msg="$(insmod $module $args 2>&1)" && return 0 || rc=$?
511 # parallels can't load modules directly from prlfs, use /tmp instead
512 if $PRLFS || [[ "$(stat -f -c%t $module)" == "7c7c6673" ]]; then
513 local target="$(mktemp)"
515 cp "$module" "$target"
518 [[ $rc == 0 ]] && PRLFS=true
526 # Load a module on the system where this is running.
528 # usage: load_module module_name [module arguments for insmod/modprobe]
530 # If module arguments are not given but MODOPTS_<MODULE> is set, then its value
531 # will be used as the arguments. Otherwise arguments will be obtained from
532 # /etc/modprobe.conf, from /etc/modprobe.d/Lustre, or else none will be used.
535 local module=$1 # '../libcfs/libcfs/libcfs', 'obdclass/obdclass', ...
538 local base=$(basename $module $ext)
540 local -A module_is_loaded_aa
544 for mod in $(lsmod | awk '{ print $1; }'); do
545 module_is_loaded_aa[${mod//-/_}]=true
549 ${module_is_loaded_aa[${1//-/_}]:-false}
552 if module_is_loaded $base; then
556 if [[ -f $LUSTRE/$module$ext ]]; then
557 path=$LUSTRE/$module$ext
558 elif [[ "$base" == lnet_selftest ]] &&
559 [[ -f $LUSTRE/../lnet/selftest/$base$ext ]]; then
560 path=$LUSTRE/../lnet/selftest/$base$ext
565 if [[ -n "$path" ]]; then
566 # Try to load any non-Lustre modules that $module depends on.
567 for mod in $(modinfo --field=depends $path | tr ',' ' '); do
568 if ! module_is_loaded $mod; then
574 # If no module arguments were passed then get them from
575 # $MODOPTS_<MODULE>, otherwise from modprobe.conf.
576 if [ $# -eq 0 ]; then
577 # $MODOPTS_<MODULE>; we could use associative arrays, but that's
578 # not in Bash until 4.x, so we resort to eval.
579 optvar="MODOPTS_$(basename $module | tr a-z A-Z)"
580 eval set -- \$$optvar
581 if [ $# -eq 0 -a -n "$MODPROBECONF" ]; then
582 # Nothing in $MODOPTS_<MODULE>; try modprobe.conf
584 opt=$(awk -v var="^options $base" '$0 ~ var \
585 {gsub("'"options $base"'",""); print}' \
587 set -- $(echo -n $opt)
589 # Ensure we have accept=all for lnet
590 if [[ "$base" == lnet ]]; then
591 # OK, this is a bit wordy...
592 local arg accept_all_present=false
595 [[ "$arg" == accept=all ]] &&
596 accept_all_present=true
599 $accept_all_present || set -- "$@" accept=all
606 [ $# -gt 0 ] && echo "${module} options: '$*'"
608 # Note that insmod will ignore anything in modprobe.conf, which is why
609 # we're passing options on the command-line. If $path does not exist
610 # then we must be testing a "make install" or"rpm" installation. Also
611 # note that failing to load ptlrpc_gss is not considered fatal.
612 if [[ -n "$path" ]]; then
613 lustre_insmod $path "$@"
614 elif [[ "$base" == ptlrpc_gss ]]; then
615 if ! modprobe $base "$@" 2>/dev/null; then
616 echo "gss/krb5 is not supported"
623 load_modules_local() {
624 if [ -n "$MODPROBE" ]; then
626 echo "Using modprobe to load modules"
630 # Create special udev test rules on every node
631 if [ -f $LUSTRE/lustre/conf/99-lustre.rules ]; then {
632 sed -e 's|/usr/sbin/lctl|$LCTL|g' $LUSTRE/lustre/conf/99-lustre.rules > /etc/udev/rules.d/99-lustre-test.rules
634 echo "SUBSYSTEM==\"lustre\", ACTION==\"change\", ENV{PARAM}==\"?*\", RUN+=\"$LCTL set_param '\$env{PARAM}=\$env{SETTING}'\"" > /etc/udev/rules.d/99-lustre-test.rules
636 udevadm control --reload-rules
639 echo Loading modules from $LUSTRE
643 if [ -f /sys/devices/system/cpu/online ]; then
644 ncpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/online) + 1))
645 echo "detected $ncpus online CPUs by sysfs"
647 ncpus=$(getconf _NPROCESSORS_CONF 2>/dev/null)
649 if [ $rc -eq 0 ]; then
650 echo "detected $ncpus online CPUs by getconf"
652 echo "Can't detect number of CPUs"
657 # if there is only one CPU core, libcfs can only create one partition
658 # if there is more than 4 CPU cores, libcfs should create multiple CPU
659 # partitions. So we just force libcfs to create 2 partitions for
660 # system with 2 or 4 cores
661 if [ $ncpus -le 4 ] && [ $ncpus -gt 1 ]; then
662 # force to enable multiple CPU partitions
663 echo "Force libcfs to create 2 CPU partitions"
664 MODOPTS_LIBCFS="cpu_npartitions=2 $MODOPTS_LIBCFS"
666 echo "libcfs will create CPU partition based on online CPUs"
669 load_module ../libcfs/libcfs/libcfs
670 # Prevent local MODOPTS_LIBCFS being passed as part of environment
671 # variable to remote nodes
675 load_module ../lnet/lnet/lnet
677 LNDPATH=${LNDPATH:-"../lnet/klnds"}
678 if [ -z "$LNETLND" ]; then
680 o2ib*) LNETLND="o2iblnd/ko2iblnd" ;;
681 tcp*) LNETLND="socklnd/ksocklnd" ;;
682 *) local lnd="${NETTYPE%%[0-9]}lnd"
683 [ -f "$LNDPATH/$lnd/k$lnd.ko" ] &&
684 LNETLND="$lnd/k$lnd" ||
685 LNETLND="socklnd/ksocklnd"
688 load_module ../lnet/klnds/$LNETLND
689 load_module obdclass/obdclass
690 load_module ptlrpc/ptlrpc
691 load_module ptlrpc/gss/ptlrpc_gss
699 load_module obdecho/obdecho
700 if ! client_only; then
701 load_module lfsck/lfsck
702 [ "$LQUOTA" != "no" ] &&
703 load_module quota/lquota $LQUOTAOPTS
704 if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then
705 load_module osd-zfs/osd_zfs
706 elif [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then
707 load_module ../ldiskfs/ldiskfs
708 load_module osd-ldiskfs/osd_ldiskfs
720 load_module llite/lustre
721 [ -d /r ] && OGDB=${OGDB:-"/r/tmp"}
723 rm -f $OGDB/ogdb-$HOSTNAME
724 $LCTL modules > $OGDB/ogdb-$HOSTNAME
726 # 'mount' doesn't look in $PATH, just sbin
727 local mount_lustre=$LUSTRE/utils/mount.lustre
728 if [ -f $mount_lustre ]; then
729 local sbin_mount=$(readlink -f /sbin)/mount.lustre
730 if grep -qw "$sbin_mount" /proc/mounts; then
731 cmp -s $mount_lustre $sbin_mount || umount $sbin_mount
733 if ! grep -qw "$sbin_mount" /proc/mounts; then
734 [ ! -f "$sbin_mount" ] && touch "$sbin_mount"
735 if [ ! -s "$sbin_mount" -a -w "$sbin_mount" ]; then
736 cat <<- EOF > "$sbin_mount"
739 echo "This $sbin_mount just a mountpoint." 1>&2
740 echo "It is never supposed to be run." 1>&2
741 logger -p emerg -- "using stub $sbin_mount $@"
744 chmod a+x $sbin_mount
746 mount --bind $mount_lustre $sbin_mount ||
747 error "can't bind $mount_lustre to $sbin_mount"
755 # load modules on remote nodes optionally
756 # lustre-tests have to be installed on these nodes
757 if $LOAD_MODULES_REMOTE; then
758 local list=$(comma_list $(remote_nodes_list))
759 if [ -n "$list" ]; then
760 echo "loading modules on: '$list'"
761 do_rpc_nodes "$list" load_modules_local
767 LEAK_LUSTRE=$(dmesg | tail -n 30 | grep "obd_memory.*leaked" || true)
768 LEAK_PORTALS=$(dmesg | tail -n 20 | grep "Portals memory leaked" || true)
769 if [ "$LEAK_LUSTRE" -o "$LEAK_PORTALS" ]; then
770 echo "$LEAK_LUSTRE" 1>&2
771 echo "$LEAK_PORTALS" 1>&2
772 mv $TMP/debug $TMP/debug-leak.`date +%s` || true
773 echo "Memory leaks detected"
774 [ -n "$IGNORE_LEAK" ] && { echo "ignoring leaks" && return 0; } || true
780 wait_exit_ST client # bug 12845
782 $LUSTRE_RMMOD ldiskfs || return 2
784 [ -f /etc/udev/rules.d/99-lustre-test.rules ] &&
785 rm /etc/udev/rules.d/99-lustre-test.rules
786 udevadm control --reload-rules
789 if $LOAD_MODULES_REMOTE; then
790 local list=$(comma_list $(remote_nodes_list))
791 if [ -n "$list" ]; then
792 echo "unloading modules on: '$list'"
793 do_rpc_nodes "$list" $LUSTRE_RMMOD ldiskfs
794 do_rpc_nodes "$list" check_mem_leak
795 do_rpc_nodes "$list" "rm /etc/udev/rules.d/99-lustre-test.rules"
796 do_rpc_nodes "$list" "udevadm control --reload-rules"
797 do_rpc_nodes "$list" "udevadm trigger"
801 local sbin_mount=$(readlink -f /sbin)/mount.lustre
802 if grep -qe "$sbin_mount " /proc/mounts; then
803 umount $sbin_mount || true
804 [ -s $sbin_mount ] && ! grep -q "STUB MARK" $sbin_mount ||
808 check_mem_leak || return 254
810 echo "modules unloaded."
815 local facet=${1:-$SINGLEMDS}
818 case $(facet_fstype $facet) in
819 ldiskfs) size=50;; # largest seen is 44, leave some headroom
820 # grant_block_size is in bytes, allow at least 2x max blocksize
821 zfs) size=$(lctl get_param osc.$FSNAME*.import |
822 awk '/grant_block_size:/ {print $2/512; exit;}')
826 echo -n $((size * MDSCOUNT))
830 local facet=${1:-$SINGLEMDS}
831 local fstype=$(facet_fstype $facet)
834 ldiskfs) size=4;; # ~4KB per inode
835 zfs) size=11;; # 10 to 11KB per inode
841 check_gss_daemon_nodes() {
845 do_nodesv $list "num=\\\$(ps -o cmd -C $dname | grep $dname | wc -l);
846 if [ \\\"\\\$num\\\" -ne 1 ]; then
847 echo \\\$num instance of $dname;
852 check_gss_daemon_facet() {
856 num=`do_facet $facet ps -o cmd -C $dname | grep $dname | wc -l`
857 if [ $num -ne 1 ]; then
858 echo "$num instance of $dname on $facet"
867 echo Stopping $@ on $list
868 do_nodes $list "killall -2 $@ 2>/dev/null || true"
871 # start gss daemons on all nodes, or "daemon" on "nodes" if set
872 start_gss_daemons() {
876 if [ "$nodes" ] && [ "$daemon" ] ; then
877 echo "Starting gss daemon on nodes: $nodes"
878 do_nodes $nodes "$daemon" || return 8
882 nodes=$(comma_list $(mdts_nodes))
883 echo "Starting gss daemon on mds: $nodes"
885 # Start all versions, in case of switching
886 do_nodes $nodes "$LSVCGSSD -vvv -s -m -o -z" || return 1
888 do_nodes $nodes "$LSVCGSSD -v" || return 1
891 do_nodes $nodes "$LGSSD -v" || return 2
894 nodes=$(comma_list $(osts_nodes))
895 echo "Starting gss daemon on ost: $nodes"
897 # Start all versions, in case of switching
898 do_nodes $nodes "$LSVCGSSD -vvv -s -m -o -z" || return 3
900 do_nodes $nodes "$LSVCGSSD -v" || return 3
902 # starting on clients
904 local clients=${CLIENTS:-$HOSTNAME}
906 echo "Starting $LGSSD on clients $clients "
907 do_nodes $clients "$LGSSD -v" || return 4
910 # wait daemons entering "stable" status
914 # check daemons are running
916 nodes=$(comma_list $(mdts_nodes) $(osts_nodes))
917 check_gss_daemon_nodes $nodes lsvcgssd || return 5
919 nodes=$(comma_list $(mdts_nodes))
920 check_gss_daemon_nodes $nodes lgssd || return 6
923 check_gss_daemon_nodes $clients lgssd || return 7
928 local nodes=$(comma_list $(mdts_nodes))
930 send_sigint $nodes lsvcgssd lgssd
932 nodes=$(comma_list $(osts_nodes))
933 send_sigint $nodes lsvcgssd
935 nodes=${CLIENTS:-$HOSTNAME}
936 send_sigint $nodes lgssd
940 # Add mount flags for shared key
942 if grep -q skpath <<< "$mt_opts" ; then
943 mt_opts=$(echo $mt_opts |
944 sed -e "s#skpath=[^ ,]*#skpath=$SK_PATH#")
946 if [ -z "$mt_opts" ]; then
947 mt_opts="-o skpath=$SK_PATH"
949 mt_opts="$mt_opts,skpath=$SK_PATH"
965 if ! module_loaded ptlrpc_gss; then
966 load_module ptlrpc/gss/ptlrpc_gss
967 module_loaded ptlrpc_gss ||
968 error_exit "init_gss: GSS=$GSS, but gss/krb5 missing"
971 if $GSS_KRB5 || $GSS_SK; then
972 start_gss_daemons || error_exit "start gss daemon failed! rc=$?"
975 if $GSS_SK && $SK_NO_KEY; then
976 local numclients=${1:-$CLIENTCOUNT}
977 local clients=${CLIENTS:-$HOSTNAME}
979 # security ctx config for keyring
982 if grep -q request-key /proc/mounts > /dev/null; then
983 echo "SSK: Request key already mounted."
985 mount -o bind $SK_OM_PATH /etc/request-key.d/
987 local lgssc_conf_line='create lgssc * * '
988 lgssc_conf_line+=$(which lgss_keyring)
989 lgssc_conf_line+=' %o %k %t %d %c %u %g %T %P %S'
991 local lgssc_conf_file="/etc/request-key.d/lgssc.conf"
992 echo "$lgssc_conf_line" > $lgssc_conf_file
993 [ -e $lgssc_conf_file ] ||
994 error_exit "Could not find key options in $lgssc_conf_file"
996 if ! local_mode; then
997 do_nodes $(comma_list $(all_nodes)) "mkdir -p \
999 do_nodes $(comma_list $(all_nodes)) "mount \
1000 -o bind $SK_OM_PATH \
1001 /etc/request-key.d/"
1002 do_nodes $(comma_list $(all_nodes)) "rsync -aqv \
1003 $HOSTNAME:$lgssc_conf_file \
1004 $lgssc_conf_file >/dev/null 2>&1"
1007 # create shared key on all nodes
1008 mkdir -p $SK_PATH/nodemap
1009 rm -f $SK_PATH/$FSNAME.key $SK_PATH/nodemap/c*.key \
1010 $SK_PATH/$FSNAME-*.key
1011 # for nodemap testing each client may need own key,
1012 # and S2S now requires keys as well, both for "client"
1015 lgss_sk -t server -f$FSNAME -n $SK_S2SNMCLI \
1016 -w $SK_PATH/$FSNAME-nmclient.key \
1017 -d /dev/urandom >/dev/null 2>&1
1018 lgss_sk -t mgs,server -f$FSNAME -n $SK_S2SNM \
1019 -w $SK_PATH/$FSNAME-s2s-server.key \
1020 -d /dev/urandom >/dev/null 2>&1
1023 lgss_sk -t server -f$FSNAME -w $SK_PATH/$FSNAME.key \
1024 -d /dev/urandom >/dev/null 2>&1
1026 for i in $(seq 0 $((numclients - 1))); do
1027 lgss_sk -t server -f$FSNAME -n c$i \
1028 -w $SK_PATH/nodemap/c$i.key -d /dev/urandom \
1032 if ! local_mode; then
1033 do_nodes $(comma_list $(all_nodes)) "rsync -av \
1034 $HOSTNAME:$SK_PATH/ $SK_PATH >/dev/null 2>&1"
1036 # Set client keys to client type to generate prime P
1038 do_nodes $(all_nodes) "lgss_sk -t client,server -m \
1039 $SK_PATH/$FSNAME.key >/dev/null 2>&1"
1041 do_nodes $clients "lgss_sk -t client -m \
1042 $SK_PATH/$FSNAME.key >/dev/null 2>&1"
1043 do_nodes $clients "find $SK_PATH/nodemap -name \*.key | \
1044 xargs -IX lgss_sk -t client -m X >/dev/null 2>&1"
1046 # This is required for servers as well, if S2S in use
1048 do_nodes $(comma_list $(mdts_nodes)) \
1049 "cp $SK_PATH/$FSNAME-s2s-server.key \
1050 $SK_PATH/$FSNAME-s2s-client.key; lgss_sk \
1051 -t client -m $SK_PATH/$FSNAME-s2s-client.key \
1053 do_nodes $(comma_list $(osts_nodes)) \
1054 "cp $SK_PATH/$FSNAME-s2s-server.key \
1055 $SK_PATH/$FSNAME-s2s-client.key; lgss_sk \
1056 -t client -m $SK_PATH/$FSNAME-s2s-client.key \
1058 do_nodes $clients "lgss_sk -t client \
1059 -m $SK_PATH/$FSNAME-nmclient.key \
1064 # mount options for servers and clients
1065 MGS_MOUNT_OPTS=$(add_sk_mntflag $MGS_MOUNT_OPTS)
1066 MDS_MOUNT_OPTS=$(add_sk_mntflag $MDS_MOUNT_OPTS)
1067 OST_MOUNT_OPTS=$(add_sk_mntflag $OST_MOUNT_OPTS)
1068 MOUNT_OPTS=$(add_sk_mntflag $MOUNT_OPTS)
1072 if [ -n "$LGSS_KEYRING_DEBUG" ]; then
1074 sptlrpc.gss.lgss_keyring.debug_level=$LGSS_KEYRING_DEBUG
1081 # maybe cleanup credential cache?
1088 do_node $(mgs_node) "$LCTL nodemap_del $SK_S2SNM"
1089 do_node $(mgs_node) "$LCTL nodemap_del $SK_S2SNMCLI"
1090 $RPC_MODE || echo "Sleeping for 10 sec for Nodemap.."
1094 $RPC_MODE || echo "Cleaning up Shared Key.."
1095 do_nodes $(comma_list $(all_nodes)) "rm -f \
1096 $SK_PATH/$FSNAME*.key $SK_PATH/nodemap/$FSNAME*.key"
1097 do_nodes $(comma_list $(all_nodes)) "keyctl show | \
1098 awk '/lustre/ { print \\\$1 }' | xargs -IX keyctl unlink X"
1099 # Remove the mount and clean up the files we added to SK_PATH
1100 do_nodes $(comma_list $(all_nodes)) "while grep -q \
1101 request-key.d /proc/mounts; do umount \
1102 /etc/request-key.d/; done"
1103 do_nodes $(comma_list $(all_nodes)) "rm -f \
1104 $SK_OM_PATH/lgssc.conf"
1105 do_nodes $(comma_list $(all_nodes)) "rmdir $SK_OM_PATH"
1112 local var=${facet}_svc
1120 echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9_]\+//' |
1121 tr '[:lower:]' '[:upper:]'
1127 if [ $facet == mgs ] || [ $facet == client ]; then
1131 echo -n $facet | sed -e 's/^fs[0-9]\+//' | sed -e 's/^[a-z]\+//'
1139 if [ -n "${!var}" ]; then
1144 var=$(facet_type $facet)FSTYPE
1145 if [ -n "${!var}" ]; then
1150 if [ -n "$FSTYPE" ]; then
1155 if [[ $facet == mgs ]] && combined_mgs_mds; then
1167 local facets=$(get_facets)
1170 for facet in ${facets//,/ }; do
1171 if [ $node == $(facet_host $facet) ] ||
1172 [ $node == "$(facet_failover_host $facet)" ]; then
1173 fstype=$(facet_fstype $facet)
1174 if [[ $fstypes != *$fstype* ]]; then
1175 fstypes+="${fstypes:+,}$fstype"
1184 local num=$(facet_number $facet)
1187 if [[ $(facet_type $facet) = OST ]]; then
1188 index=OSTINDEX${num}
1189 if [[ -n "${!index}" ]]; then
1194 index=${OST_INDICES[num - 1]}
1197 [[ -n "$index" ]] || index=$((num - 1))
1205 local fstype=$(facet_fstype $facet)
1209 label=$(do_facet ${facet} "$E2LABEL ${dev} 2>/dev/null");;
1211 label=$(do_facet ${facet} "$ZFS get -H -o value lustre:svname \
1212 ${dev} 2>/dev/null");;
1214 error "unknown fstype!";;
1222 local device=$(mdsdevname $num)
1223 local label=$(devicelabel mds$num ${device} | grep -v "CMD: ")
1229 local device=$(ostdevname $num)
1230 local label=$(devicelabel ost$num ${device} | grep -v "CMD: ")
1235 # Get the device of a facet.
1242 mgs) device=$(mgsdevname) ;;
1243 mds*) device=$(mdsdevname $(facet_number $facet)) ;;
1244 ost*) device=$(ostdevname $(facet_number $facet)) ;;
1245 fs2mds) device=$(mdsdevname 1_2) ;;
1246 fs2ost) device=$(ostdevname 1_2) ;;
1247 fs3ost) device=$(ostdevname 2_2) ;;
1255 # Get the virtual device of a facet.
1262 mgs) device=$(mgsvdevname) ;;
1263 mds*) device=$(mdsvdevname $(facet_number $facet)) ;;
1264 ost*) device=$(ostvdevname $(facet_number $facet)) ;;
1265 fs2mds) device=$(mdsvdevname 1_2) ;;
1266 fs2ost) device=$(ostvdevname 1_2) ;;
1267 fs3ost) device=$(ostvdevname 2_2) ;;
1275 local virt=$(virt-what 2> /dev/null)
1277 [ $? -eq 0 ] && [ -n "$virt" ] && { echo $virt; return; }
1279 virt=$(dmidecode -s system-product-name | awk '{print $1}')
1282 VMware|KVM|VirtualBox|Parallels)
1283 echo $virt | tr '[A-Z]' '[a-z]' ;;
1289 # Re-read the partition table on failover partner host.
1290 # After a ZFS storage pool is created on a shared device, the partition table
1291 # on the device may change. However, the operating system on the failover
1292 # host may not notice the change automatically. Without the up-to-date partition
1293 # block devices, 'zpool import ..' cannot find the labels, whose positions are
1294 # relative to partition rather than disk beginnings.
1296 # This function performs partprobe on the failover host to make it re-read the
1299 refresh_partition_table() {
1304 host=$(facet_passive_host $facet)
1305 if [[ -n "$host" ]]; then
1306 do_node $host "$PARTPROBE $device"
1311 # Get ZFS storage pool name.
1318 device=$(facet_device $facet)
1319 # poolname is string before "/"
1320 poolname="${device%%/*}"
1327 # Get ZFS local fsname.
1329 zfs_local_fsname() {
1331 local lfsname=$(basename $(facet_device $facet))
1337 # Create ZFS storage pool.
1344 local opts=${@:-"-o cachefile=none"}
1346 do_facet $facet "lsmod | grep zfs >&/dev/null || modprobe zfs;
1347 $ZPOOL list -H $poolname >/dev/null 2>&1 ||
1348 $ZPOOL create -f $opts $poolname $vdev"
1352 # Create ZFS file system.
1358 local opts=${@:-"-o mountpoint=legacy"}
1360 do_facet $facet "$ZFS list -H $dataset >/dev/null 2>&1 ||
1361 $ZFS create $opts $dataset"
1365 # Export ZFS storage pool.
1366 # Before exporting the pool, all datasets within the pool should be unmounted.
1374 poolname=$(zpool_name $facet)
1376 if [[ -n "$poolname" ]]; then
1377 do_facet $facet "! $ZPOOL list -H $poolname >/dev/null 2>&1 ||
1378 grep -q ^$poolname/ /proc/mounts ||
1379 $ZPOOL export $opts $poolname"
1384 # Destroy ZFS storage pool.
1385 # Destroy the given pool and free up any devices for other use. This command
1386 # tries to unmount any active datasets before destroying the pool.
1387 # -f Force any active datasets contained within the pool to be unmounted.
1391 local poolname=${2:-$(zpool_name $facet)}
1393 if [[ -n "$poolname" ]]; then
1394 do_facet $facet "! $ZPOOL list -H $poolname >/dev/null 2>&1 ||
1395 $ZPOOL destroy -f $poolname"
1400 # Import ZFS storage pool.
1401 # Force importing, even if the pool appears to be potentially active.
1406 local opts=${@:-"-o cachefile=none -o failmode=panic"}
1409 poolname=$(zpool_name $facet)
1411 if [[ -n "$poolname" ]]; then
1412 opts+=" -d $(dirname $(facet_vdevice $facet))"
1413 do_facet $facet "lsmod | grep zfs >&/dev/null || modprobe zfs;
1414 $ZPOOL list -H $poolname >/dev/null 2>&1 ||
1415 $ZPOOL import -f $opts $poolname"
1420 # Reimport ZFS storage pool with new name
1425 local opts="-o cachefile=none"
1426 local poolname=$(zpool_name $facet)
1428 opts+=" -d $(dirname $(facet_vdevice $facet))"
1429 do_facet $facet "$ZPOOL export $poolname;
1430 $ZPOOL import $opts $poolname $newpool"
1434 # Set the "cachefile=none" property on ZFS storage pool so that the pool
1435 # is not automatically imported on system startup.
1437 # In a failover environment, this will provide resource level fencing which
1438 # will ensure that the same ZFS storage pool will not be imported concurrently
1439 # on different nodes.
1441 disable_zpool_cache() {
1445 poolname=$(zpool_name $facet)
1447 if [[ -n "$poolname" ]]; then
1448 do_facet $facet "$ZPOOL set cachefile=none $poolname"
1453 # This and set_osd_param() shall be used to access OSD parameters
1454 # once existed under "obdfilter":
1459 # writethrough_cache_enable
1463 local device=${2:-$FSNAME-OST*}
1466 do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \
1467 osd-*.$device.$name 2>&1" | grep -v 'error:'
1472 local device=${2:-$FSNAME-OST*}
1476 do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \
1477 osd-*.$device.$name=$value 2>&1" | grep -v 'error:'
1481 local dz=${1:-$DEBUG_SIZE}
1483 if [ -f /sys/devices/system/cpu/possible ]; then
1484 local cpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/possible)+1))
1486 local cpus=$(getconf _NPROCESSORS_CONF 2>/dev/null)
1489 # bug 19944, adjust size to be -gt num_possible_cpus()
1490 # promise 2MB for every cpu at least
1491 if [ -n "$cpus" ] && [ $((cpus * 2)) -gt $dz ]; then
1494 lctl set_param debug_mb=$dz
1497 set_default_debug () {
1498 local debug=${1:-"$PTLDEBUG"}
1499 local subsys=${2:-"$SUBSYSTEM"}
1500 local debug_size=${3:-$DEBUG_SIZE}
1502 [ -n "$debug" ] && lctl set_param debug="$debug" >/dev/null
1503 [ -n "$subsys" ] && lctl set_param subsystem_debug="${subsys# }" >/dev/null
1505 [ -n "$debug_size" ] && set_debug_size $debug_size > /dev/null
1508 set_default_debug_nodes () {
1511 if [[ ,$nodes, = *,$HOSTNAME,* ]]; then
1512 nodes=$(exclude_items_from_list "$nodes" "$HOSTNAME")
1516 do_rpc_nodes "$nodes" set_default_debug \
1517 \\\"$PTLDEBUG\\\" \\\"$SUBSYSTEM\\\" $DEBUG_SIZE || true
1520 set_default_debug_facet () {
1522 local node=$(facet_active_host $facet)
1523 [ -z "$node" ] && echo "No host defined for facet $facet" && exit 1
1525 set_default_debug_nodes $node
1529 local hostid=${1:-$(hostid)}
1531 if [ ! -s /etc/hostid ]; then
1532 printf $(echo -n $hostid |
1533 sed 's/\(..\)\(..\)\(..\)\(..\)/\\x\4\\x\3\\x\2\\x\1/') >/etc/hostid
1539 local facets=${1:-$(get_facets)}
1542 for facet in ${facets//,/ }; do
1545 [ $RC -eq 0 ] && continue
1547 if [ "$TESTSUITE.$TESTNAME" = "replay-dual.test_0a" ]; then
1548 skip_noexit "Restart of $facet failed!." &&
1551 error "Restart of $facet failed!"
1558 # Add argument "arg" (e.g., "loop") to the comma-separated list
1559 # of arguments for option "opt" (e.g., "-o") on command
1560 # line "opts" (e.g., "-o flock").
1566 local opt_pattern="\([[:space:]]\+\|^\)$opt"
1568 if echo "$opts" | grep -q $opt_pattern; then
1569 opts=$(echo "$opts" | sed -e \
1570 "s/$opt_pattern[[:space:]]*[^[:space:]]\+/&,$arg/")
1572 opts+="${opts:+ }$opt $arg"
1578 # Associate loop device with a given regular file.
1579 # Return the loop device.
1581 setup_loop_device() {
1585 do_facet $facet "loop_dev=\\\$($LOSETUP -j $file | cut -d : -f 1);
1586 if [[ -z \\\$loop_dev ]]; then
1587 loop_dev=\\\$($LOSETUP -f);
1588 $LOSETUP \\\$loop_dev $file || loop_dev=;
1590 echo -n \\\$loop_dev"
1594 # Detach a loop device.
1596 cleanup_loop_device() {
1600 do_facet $facet "! $LOSETUP $loop_dev >/dev/null 2>&1 ||
1601 $LOSETUP -d $loop_dev"
1605 # Check if a given device is a block device.
1612 [[ -n "$dev" ]] || return 1
1613 do_facet $facet "test -b $dev" || return 1
1614 if [[ -n "$size" ]]; then
1615 local in=$(do_facet $facet "dd if=$dev of=/dev/null bs=1k \
1616 count=1 skip=$size 2>&1" |
1617 awk '($3 == "in") { print $1 }')
1618 [[ "$in" = "1+0" ]] || return 1
1623 # Check if a given device is a device-mapper device.
1629 [[ -n "$dev" ]] || return 1
1630 do_facet $facet "$DMSETUP status $dev >/dev/null 2>&1"
1634 # Check if a given device is a device-mapper flakey device.
1636 is_dm_flakey_dev() {
1641 [[ -n "$dev" ]] || return 1
1643 type=$(do_facet $facet "$DMSETUP status $dev 2>&1" |
1645 [[ $type = flakey ]] && return 0 || return 1
1649 # Check if device-mapper flakey device is supported by the kernel
1650 # of $facet node or not.
1652 dm_flakey_supported() {
1656 do_facet $facet "modprobe dm-flakey;
1657 $DMSETUP targets | grep -q flakey" &> /dev/null
1661 # Get the device-mapper flakey device name of a given facet.
1663 dm_facet_devname() {
1665 [[ $facet = mgs ]] && combined_mgs_mds && facet=mds1
1667 echo -n ${facet}_flakey
1671 # Get the device-mapper flakey device of a given facet.
1672 # A device created by dmsetup will appear as /dev/mapper/<device-name>.
1674 dm_facet_devpath() {
1677 echo -n $DM_DEV_PATH/$(dm_facet_devname $facet)
1681 # Set a device-mapper device with a new table.
1683 # The table has the following format:
1684 # <logical_start_sector> <num_sectors> <target_type> <target_args>
1686 # flakey <target_args> includes:
1687 # <destination_device> <offset> <up_interval> <down_interval> \
1688 # [<num_features> [<feature_arguments>]]
1690 # linear <target_args> includes:
1691 # <destination_device> <start_sector>
1693 dm_set_dev_table() {
1696 local target_type=$3
1702 read tmp num_sectors tmp real_dev tmp \
1703 <<< $(do_facet $facet "$DMSETUP table $dm_dev")
1705 case $target_type in
1707 table="0 $num_sectors flakey $real_dev 0 0 1800 1 drop_writes"
1710 table="0 $num_sectors linear $real_dev 0"
1712 *) error "invalid target type $target_type" ;;
1715 do_facet $facet "$DMSETUP suspend --nolockfs --noflush $dm_dev" ||
1716 error "failed to suspend $dm_dev"
1717 do_facet $facet "$DMSETUP load $dm_dev --table \\\"$table\\\"" ||
1718 error "failed to load $target_type table into $dm_dev"
1719 do_facet $facet "$DMSETUP resume $dm_dev" ||
1720 error "failed to resume $dm_dev"
1724 # Set a device-mapper flakey device as "read-only" by using the "drop_writes"
1725 # feature parameter.
1728 # All write I/O is silently ignored.
1729 # Read I/O is handled correctly.
1731 dm_set_dev_readonly() {
1733 local dm_dev=${2:-$(dm_facet_devpath $facet)}
1735 dm_set_dev_table $facet $dm_dev flakey
1739 # Set a device-mapper device to traditional linear mapping mode.
1741 dm_clear_dev_readonly() {
1743 local dm_dev=${2:-$(dm_facet_devpath $facet)}
1745 dm_set_dev_table $facet $dm_dev linear
1749 # Set the device of a given facet as "read-only".
1751 set_dev_readonly() {
1753 local svc=${facet}_svc
1755 if [[ $(facet_fstype $facet) = zfs ]] ||
1756 ! dm_flakey_supported $facet; then
1757 do_facet $facet $LCTL --device ${!svc} readonly
1759 dm_set_dev_readonly $facet
1764 # Get size in 512-byte sectors (BLKGETSIZE64 / 512) of a given device.
1771 num_sectors=$(do_facet $facet "blockdev --getsz $dev 2>/dev/null")
1772 [[ ${PIPESTATUS[0]} = 0 && -n "$num_sectors" ]] || num_sectors=0
1773 echo -n $num_sectors
1777 # Create a device-mapper device with a given block device or regular file (will
1778 # be associated with loop device).
1779 # Return the full path of the device-mapper device.
1783 local real_dev=$2 # destination device
1784 local dm_dev_name=${3:-$(dm_facet_devname $facet)} # device name
1785 local dm_dev=$DM_DEV_PATH/$dm_dev_name # device-mapper device
1787 # check if the device-mapper device to be created already exists
1788 if is_dm_dev $facet $dm_dev; then
1789 # if the existing device was set to "read-only", then clear it
1790 ! is_dm_flakey_dev $facet $dm_dev ||
1791 dm_clear_dev_readonly $facet $dm_dev
1797 # check if the destination device is a block device, and if not,
1798 # associate it with a loop device
1799 is_blkdev $facet $real_dev ||
1800 real_dev=$(setup_loop_device $facet $real_dev)
1801 [[ -n "$real_dev" ]] || { echo -n $real_dev; return 2; }
1803 # now create the device-mapper device
1804 local num_sectors=$(get_num_sectors $facet $real_dev)
1805 local table="0 $num_sectors linear $real_dev 0"
1808 do_facet $facet "$DMSETUP create $dm_dev_name --table \\\"$table\\\"" ||
1809 { rc=${PIPESTATUS[0]}; dm_dev=; }
1810 do_facet $facet "$DMSETUP mknodes >/dev/null 2>&1"
1817 # Map the facet name to its device variable name.
1819 facet_device_alias() {
1821 local dev_alias=$facet
1824 fs2mds) dev_alias=mds1_2 ;;
1825 fs2ost) dev_alias=ost1_2 ;;
1826 fs3ost) dev_alias=ost2_2 ;;
1834 # Save the original value of the facet device and export the new value.
1840 local active_facet=$(facet_active $facet)
1841 local dev_alias=$(facet_device_alias $active_facet)
1842 local dev_name=${dev_alias}_dev
1843 local dev=${!dev_name}
1845 if [[ $active_facet = $facet ]]; then
1846 local failover_dev=${dev_alias}failover_dev
1847 if [[ ${!failover_dev} = $dev ]]; then
1848 eval export ${failover_dev}_saved=$dev
1849 eval export ${failover_dev}=$dm_dev
1852 dev_alias=$(facet_device_alias $facet)
1853 local facet_dev=${dev_alias}_dev
1854 if [[ ${!facet_dev} = $dev ]]; then
1855 eval export ${facet_dev}_saved=$dev
1856 eval export ${facet_dev}=$dm_dev
1860 eval export ${dev_name}_saved=$dev
1861 eval export ${dev_name}=$dm_dev
1865 # Restore the saved value of the facet device.
1870 [[ $facet = mgs ]] && combined_mgs_mds && facet=mds1
1871 local dev_alias=$(facet_device_alias $facet)
1873 local saved_dev=${dev_alias}_dev_saved
1874 [[ -z ${!saved_dev} ]] ||
1875 eval export ${dev_alias}_dev=${!saved_dev}
1877 saved_dev=${dev_alias}failover_dev_saved
1878 [[ -z ${!saved_dev} ]] ||
1879 eval export ${dev_alias}failover_dev=${!saved_dev}
1883 # Remove a device-mapper device.
1884 # If the destination device is a loop device, then also detach it.
1888 local dm_dev=${2:-$(dm_facet_devpath $facet)}
1892 is_dm_dev $facet $dm_dev || return 0
1894 read major minor <<< $(do_facet $facet "$DMSETUP table $dm_dev" |
1895 awk '{ print $4 }' | awk -F: '{ print $1" "$2 }')
1897 do_facet $facet "$DMSETUP remove $dm_dev"
1898 do_facet $facet "$DMSETUP mknodes >/dev/null 2>&1"
1900 unexport_dm_dev $facet
1902 # detach a loop device
1903 [[ $major -ne 7 ]] || cleanup_loop_device $facet /dev/loop$minor
1905 # unload dm-flakey module
1906 do_facet $facet "modprobe -r dm-flakey" || true
1912 local active_facet=$(facet_active $facet)
1913 local dev_alias=$(facet_device_alias $active_facet)
1914 local dev=${dev_alias}_dev
1915 local opt=${facet}_opt
1916 local mntpt=$(facet_mntpt $facet)
1917 local opts="${!opt} $@"
1918 local fstype=$(facet_fstype $facet)
1920 local dm_dev=${!dev}
1922 module_loaded lustre || load_modules
1926 if dm_flakey_supported $facet; then
1927 dm_dev=$(dm_create_dev $facet ${!dev})
1928 [[ -n "$dm_dev" ]] || dm_dev=${!dev}
1931 is_blkdev $facet $dm_dev || opts=$(csa_add "$opts" -o loop)
1933 devicelabel=$(do_facet ${facet} "$E2LABEL $dm_dev");;
1935 # import ZFS storage pool
1936 import_zpool $facet || return ${PIPESTATUS[0]}
1938 devicelabel=$(do_facet ${facet} "$ZFS get -H -o value \
1939 lustre:svname $dm_dev");;
1941 error "unknown fstype!";;
1944 echo "Starting ${facet}: $opts $dm_dev $mntpt"
1945 # for testing LU-482 error handling in mount_facets() and test_0a()
1946 if [ -f $TMP/test-lu482-trigger ]; then
1950 "mkdir -p $mntpt; $MOUNT_CMD $opts $dm_dev $mntpt"
1954 if [ $RC -ne 0 ]; then
1955 echo "Start of $dm_dev on ${facet} failed ${RC}"
1959 health=$(do_facet ${facet} "$LCTL get_param -n health_check")
1960 if [[ "$health" != "healthy" ]]; then
1961 error "$facet is in a unhealthy state"
1964 set_default_debug_facet $facet
1966 if [[ $opts =~ .*nosvc.* ]]; then
1967 echo "Start $dm_dev without service"
1972 wait_update_facet ${facet} "$E2LABEL $dm_dev \
1973 2>/dev/null | grep -E ':[a-zA-Z]{3}[0-9]{4}'" \
1974 "" || error "$dm_dev failed to initialize!";;
1976 wait_update_facet ${facet} "$ZFS get -H -o value \
1977 lustre:svname $dm_dev 2>/dev/null | \
1978 grep -E ':[a-zA-Z]{3}[0-9]{4}'" "" ||
1979 error "$dm_dev failed to initialize!";;
1982 error "unknown fstype!";;
1986 # commit the device label change to disk
1987 if [[ $devicelabel =~ (:[a-zA-Z]{3}[0-9]{4}) ]]; then
1988 echo "Commit the device label on ${!dev}"
1989 do_facet $facet "sync; sleep 1; sync"
1993 label=$(devicelabel ${facet} $dm_dev)
1994 [ -z "$label" ] && echo no label for $dm_dev && exit 1
1995 eval export ${facet}_svc=${label}
1996 echo Started ${label}
1998 export_dm_dev $facet $dm_dev
2003 # start facet device options
2009 local dev_alias=$(facet_device_alias $facet)
2011 eval export ${dev_alias}_dev=${device}
2012 eval export ${facet}_opt=\"$@\"
2014 local varname=${dev_alias}failover_dev
2015 if [ -n "${!varname}" ] ; then
2016 eval export ${dev_alias}failover_dev=${!varname}
2018 eval export ${dev_alias}failover_dev=$device
2021 local mntpt=$(facet_mntpt $facet)
2022 do_facet ${facet} mkdir -p $mntpt
2023 eval export ${facet}_MOUNT=$mntpt
2024 mount_facet ${facet}
2034 local HOST=$(facet_active_host $facet)
2035 [[ -z $HOST ]] && echo stop: no host for $facet && return 0
2037 local mntpt=$(facet_mntpt $facet)
2038 running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts || true")
2039 if [ ${running} -ne 0 ]; then
2040 echo "Stopping $mntpt (opts:$@) on $HOST"
2041 do_facet ${facet} $UMOUNT $@ $mntpt
2044 # umount should block, but we should wait for unrelated obd's
2045 # like the MGS or MGC to also stop.
2046 wait_exit_ST ${facet} || return ${PIPESTATUS[0]}
2048 if [[ $(facet_fstype $facet) == zfs ]]; then
2049 # export ZFS storage pool
2050 [ "$KEEP_ZPOOL" = "true" ] || export_zpool $facet
2051 elif dm_flakey_supported $facet; then
2052 local host=${facet}_HOST
2053 local failover_host=${facet}failover_HOST
2054 if [[ -n ${!failover_host} && ${!failover_host} != ${!host} ]]||
2055 $CLEANUP_DM_DEV || [[ $facet = fs* ]]; then
2056 dm_cleanup_dev $facet
2061 # get mdt quota type
2063 local varsvc=${SINGLEMDS}_svc
2064 do_facet $SINGLEMDS $LCTL get_param -n \
2065 osd-$(facet_fstype $SINGLEMDS).${!varsvc}.quota_slave.enabled
2068 # get ost quota type
2070 # All OSTs should have same quota type
2071 local varsvc=ost1_svc
2072 do_facet ost1 $LCTL get_param -n \
2073 osd-$(facet_fstype ost1).${!varsvc}.quota_slave.enabled
2076 # restore old quota type settings
2078 if [ "$old_MDT_QUOTA_TYPE" ]; then
2079 if [[ $PERM_CMD == *"set_param -P"* ]]; then
2080 do_facet mgs $PERM_CMD \
2081 osd-*.$FSNAME-MDT*.quota_slave.enable = \
2084 do_facet mgs $PERM_CMD \
2085 $FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE
2088 if [ "$old_OST_QUOTA_TYPE" ]; then
2089 if [[ $PERM_CMD == *"set_param -P"* ]]; then
2090 do_facet mgs $PERM_CMD \
2091 osd-*.$FSNAME-OST*.quota_slave.enable = \
2094 do_facet mgs $LCTL conf_param \
2095 $FSNAME.quota.ost=$old_OST_QUOTA_TYPE
2100 # Handle the case when there is a space in the lfs df
2101 # "filesystem summary" line the same as when there is no space.
2102 # This will allow fixing the "lfs df" summary line in the future.
2104 $LFS df $* | sed -e 's/filesystem /filesystem_/'
2107 # Get free inodes on the MDT specified by mdt index, free indoes on
2108 # the whole filesystem will be returned when index == -1.
2114 if [ $index -eq -1 ]; then
2117 mdt_uuid=$(mdtuuid_from_index $index)
2120 free_inodes=$(lfs_df -i $MOUNT | grep $mdt_uuid | awk '{print $4}')
2125 # Get the OST device status from 'lfs df' with a given OST index.
2129 local mnt_pnt=${2:-$MOUNT}
2132 ost_uuid=$(ostuuid_from_index $ost_idx $mnt_pnt)
2133 lfs_df $mnt_pnt | awk '/'$ost_uuid'/ { print $7 }'
2139 # save old quota type & set new quota type
2140 local mdt_qtype=$(mdt_quota_type)
2141 local ost_qtype=$(ost_quota_type)
2143 echo "[HOST:$HOSTNAME] [old_mdt_qtype:$mdt_qtype]" \
2144 "[old_ost_qtype:$ost_qtype] [new_qtype:$QUOTA_TYPE]"
2146 export old_MDT_QUOTA_TYPE=$mdt_qtype
2147 export old_OST_QUOTA_TYPE=$ost_qtype
2149 if [[ $PERM_CMD == *"set_param -P"* ]]; then
2150 do_facet mgs $PERM_CMD \
2151 osd-*.$FSNAME-MDT*.quota_slave.enable=$QUOTA_TYPE
2152 do_facet mgs $PERM_CMD \
2153 osd-*.$FSNAME-OST*.quota_slave.enable=$QUOTA_TYPE
2155 do_facet mgs $PERM_CMD $FSNAME.quota.mdt=$QUOTA_TYPE ||
2156 error "set mdt quota type failed"
2157 do_facet mgs $PERM_CMD $FSNAME.quota.ost=$QUOTA_TYPE ||
2158 error "set ost quota type failed"
2161 local quota_usrs=$QUOTA_USERS
2163 # get_filesystem_size
2164 local disksz=$(lfs_df $mntpt | grep "summary" | awk '{print $2}')
2165 local blk_soft=$((disksz + 1024))
2166 local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over
2168 local inodes=$(lfs_df -i $mntpt | grep "summary" | awk '{print $2}')
2169 local i_soft=$inodes
2170 local i_hard=$((i_soft + i_soft / 20))
2172 echo "Total disk size: $disksz block-softlimit: $blk_soft" \
2173 "block-hardlimit: $blk_hard inode-softlimit: $i_soft" \
2174 "inode-hardlimit: $i_hard"
2177 for usr in $quota_usrs; do
2178 echo "Setting up quota on $HOSTNAME:$mntpt for $usr..."
2180 cmd="$LFS setquota -$type $usr -b $blk_soft"
2181 cmd="$cmd -B $blk_hard -i $i_soft -I $i_hard $mntpt"
2183 eval $cmd || error "$cmd FAILED!"
2185 # display the quota status
2186 echo "Quota settings for $usr : "
2187 $LFS quota -v -u $usr $mntpt || true
2194 local opts=${3:-$MOUNT_OPTS}
2195 opts=${opts:+-o $opts}
2196 local flags=${4:-$MOUNT_FLAGS}
2198 local device=$MGSNID:/$FSNAME$FILESET
2199 if [ -z "$mnt" -o -z "$FSNAME" ]; then
2200 echo "Bad mount command: opt=$flags $opts dev=$device " \
2205 echo "Starting client: $client: $flags $opts $device $mnt"
2206 do_node $client mkdir -p $mnt
2207 if [ -n "$FILESET" -a -z "$SKIP_FILESET" ];then
2208 do_node $client $MOUNT_CMD $flags $opts $MGSNID:/$FSNAME \
2210 #disable FILESET if not supported
2211 do_nodes $client lctl get_param -n \
2212 mdc.$FSNAME-MDT0000*.import | grep -q subtree ||
2213 device=$MGSNID:/$FSNAME
2214 do_node $client mkdir -p $mnt/$FILESET
2215 do_node $client "! grep -q $mnt' ' /proc/mounts ||
2218 if $GSS_SK && ($SK_UNIQUE_NM || $SK_S2S); then
2219 # Mount using nodemap key
2220 local mountkey=$SK_PATH/$FSNAME-nmclient.key
2221 if $SK_UNIQUE_NM; then
2222 mountkey=$SK_PATH/nodemap/c0.key
2224 local prunedopts=$(echo $opts |
2225 sed -e "s#skpath=[^,^ ]*#skpath=$mountkey#g")
2226 do_node $client $MOUNT_CMD $flags $prunedopts $device $mnt ||
2229 do_node $client $MOUNT_CMD $flags $opts $device $mnt ||
2233 set_default_debug_nodes $client
2245 [ "$3" ] && force=-f
2246 local running=$(do_node $client "grep -c $mnt' ' /proc/mounts") || true
2247 if [ $running -ne 0 ]; then
2248 echo "Stopping client $client $mnt (opts:$force)"
2249 do_node $client lsof -t $mnt || need_kill=no
2250 if [ "x$force" != "x" -a "x$need_kill" != "xno" ]; then
2251 pids=$(do_node $client lsof -t $mnt | sort -u);
2252 if [ -n $pids ]; then
2253 do_node $client kill -9 $pids || true
2257 busy=$(do_node $client "umount $force $mnt 2>&1" | grep -c "busy") || true
2258 if [ $busy -ne 0 ] ; then
2259 echo "$mnt is still busy, wait one second" && sleep 1
2260 do_node $client umount $force $mnt
2265 # Mount the file system on the MGS
2266 mount_mgs_client() {
2267 do_facet mgs "mkdir -p $MOUNT"
2268 zconf_mount $mgs_HOST $MOUNT $MOUNT_OPTS ||
2269 error "unable to mount $MOUNT on MGS"
2272 # Unmount the file system on the MGS
2273 umount_mgs_client() {
2274 zconf_umount $mgs_HOST $MOUNT
2275 do_facet mgs "rm -rf $MOUNT"
2278 # nodes is comma list
2279 sanity_mount_check_nodes () {
2285 # FIXME: assume that all cluster nodes run the same os
2286 [ "$(uname)" = Linux ] || return 0
2289 for mnt in $mnts ; do
2290 do_nodes $nodes "running=\\\$(grep -c $mnt' ' /proc/mounts);
2291 mpts=\\\$(mount | grep -c $mnt' ');
2292 if [ \\\$running -ne \\\$mpts ]; then
2293 echo \\\$(hostname) env are INSANE!;
2296 [ $? -eq 0 ] || rc=1
2301 sanity_mount_check_servers () {
2302 [ -n "$CLIENTONLY" ] &&
2303 { echo "CLIENTONLY mode, skip mount_check_servers"; return 0; } || true
2304 echo Checking servers environments
2306 # FIXME: modify get_facets to display all facets wo params
2307 local facets="$(get_facets OST),$(get_facets MDS),mgs"
2311 for facet in ${facets//,/ }; do
2312 node=$(facet_host ${facet})
2313 mntpt=$(facet_mntpt $facet)
2314 sanity_mount_check_nodes $node $mntpt ||
2315 { error "server $node environments are insane!"; return 1; }
2319 sanity_mount_check_clients () {
2320 local clients=${1:-$CLIENTS}
2321 local mntpt=${2:-$MOUNT}
2322 local mntpt2=${3:-$MOUNT2}
2324 [ -z $clients ] && clients=$(hostname)
2325 echo Checking clients $clients environments
2327 sanity_mount_check_nodes $clients $mntpt $mntpt2 ||
2328 error "clients environments are insane!"
2331 sanity_mount_check () {
2332 sanity_mount_check_servers || return 1
2333 sanity_mount_check_clients || return 2
2336 # mount clients if not mouted
2337 zconf_mount_clients() {
2340 local opts=${3:-$MOUNT_OPTS}
2341 opts=${opts:+-o $opts}
2342 local flags=${4:-$MOUNT_FLAGS}
2343 local device=$MGSNID:/$FSNAME$FILESET
2344 if [ -z "$mnt" -o -z "$FSNAME" ]; then
2345 echo "Bad conf mount command: opt=$flags $opts dev=$device " \
2350 echo "Starting client $clients: $flags $opts $device $mnt"
2351 do_nodes $clients mkdir -p $mnt
2352 if [ -n "$FILESET" -a -z "$SKIP_FILESET" ]; then
2353 if $GSS_SK && ($SK_UNIQUE_NM || $SK_S2S); then
2354 # Mount with own nodemap key
2356 # Mount all server nodes first with per-NM keys
2357 for nmclient in ${clients//,/ }; do
2358 # do_nodes $(comma_list $(all_server_nodes)) "lgss_sk -t server -l $SK_PATH/nodemap/c$i.key -n c$i"
2359 do_nodes $(comma_list $(all_server_nodes)) "lgss_sk -t server -l $SK_PATH/nodemap/c$i.key"
2362 # set perms for per-nodemap keys else permission denied
2363 do_nodes $(comma_list $(all_nodes)) \
2364 "keyctl show | grep lustre | cut -c1-11 |
2366 xargs -IX keyctl setperm X 0x3f3f3f3f"
2367 local mountkey=$SK_PATH/$FSNAME-nmclient.key
2369 for nmclient in ${clients//,/ }; do
2370 if $SK_UNIQUE_NM; then
2371 mountkey=$SK_PATH/nodemap/c$i.key
2373 do_node $nmclient "! grep -q $mnt' ' \
2374 /proc/mounts || umount $mnt"
2375 local prunedopts=$(add_sk_mntflag $prunedopts);
2376 prunedopts=$(echo $prunedopts | sed -e \
2377 "s#skpath=[^ ^,]*#skpath=$mountkey#g")
2379 do_nodes $(comma_list $(all_server_nodes)) \
2382 do_node $nmclient $MOUNT_CMD $flags \
2383 $prunedopts $MGSNID:/$FSNAME $mnt ||
2388 do_nodes $clients "! grep -q $mnt' ' /proc/mounts ||
2390 do_nodes $clients $MOUNT_CMD $flags $opts \
2391 $MGSNID:/$FSNAME $mnt || return 1
2393 #disable FILESET if not supported
2394 do_nodes $clients lctl get_param -n \
2395 mdc.$FSNAME-MDT0000*.import | grep -q subtree ||
2396 device=$MGSNID:/$FSNAME
2397 do_nodes $clients mkdir -p $mnt/$FILESET
2398 do_nodes $clients "! grep -q $mnt' ' /proc/mounts ||
2402 if $GSS_SK && ($SK_UNIQUE_NM || $SK_S2S); then
2403 # Mount with nodemap key
2405 local mountkey=$SK_PATH/$FSNAME-nmclient.key
2406 for nmclient in ${clients//,/ }; do
2407 if $SK_UNIQUE_NM; then
2408 mountkey=$SK_PATH/nodemap/c$i.key
2410 local prunedopts=$(echo $opts | sed -e \
2411 "s#skpath=[^ ^,]*#skpath=$mountkey#g");
2412 do_node $nmclient "! grep -q $mnt' ' /proc/mounts ||
2415 running=\\\$(mount | grep -c $mnt' ');
2417 if [ \\\$running -eq 0 ] ; then
2419 $MOUNT_CMD $flags $prunedopts $device $mnt;
2422 lustre_mnt_count=\\\$(mount | grep $mnt' ' | \
2423 grep 'type lustre' | wc -l);
2424 if [ \\\$running -ne \\\$lustre_mnt_count ] ; then
2425 echo zconf_mount_clients FAILED: \
2426 mount count \\\$running, not matching \
2427 with mount count of 'type lustre' \
2428 \\\$lustre_mnt_count;
2432 exit \\\$rc" || return ${PIPESTATUS[0]}
2439 if $SHARED_KEY; then
2440 tmpopts=$(add_sk_mntflag $opts)
2443 running=\\\$(mount | grep -c $mnt' ');
2445 if [ \\\$running -eq 0 ] ; then
2447 $MOUNT_CMD $flags $tmpopts $device $mnt;
2450 exit \\\$rc" || return ${PIPESTATUS[0]}
2453 echo "Started clients $clients: "
2454 do_nodes $clients "mount | grep $mnt' '"
2456 set_default_debug_nodes $clients
2461 zconf_umount_clients() {
2466 [ "$3" ] && force=-f
2468 echo "Stopping clients: $clients $mnt (opts:$force)"
2469 do_nodes $clients "running=\\\$(grep -c $mnt' ' /proc/mounts);
2470 if [ \\\$running -ne 0 ] ; then
2471 echo Stopping client \\\$(hostname) $mnt opts:$force;
2472 lsof $mnt || need_kill=no;
2473 if [ "x$force" != "x" -a "x\\\$need_kill" != "xno" ]; then
2474 pids=\\\$(lsof -t $mnt | sort -u);
2475 if [ -n \\\"\\\$pids\\\" ]; then
2479 while umount $force $mnt 2>&1 | grep -q "busy"; do
2480 echo "$mnt is still busy, wait one second" && sleep 1;
2487 echo + $POWER_DOWN $node
2491 shutdown_node_hard () {
2493 local attempts=$SHUTDOWN_ATTEMPTS
2495 for i in $(seq $attempts) ; do
2498 wait_for_function --quiet "! ping -w 3 -c 1 $host" 5 1 && return 0
2499 echo "waiting for $host to fail attempts=$attempts"
2500 [ $i -lt $attempts ] || \
2501 { echo "$host still pingable after power down! attempts=$attempts" && return 1; }
2507 local mnt=${2:-$MOUNT}
2510 if [ "$FAILURE_MODE" = HARD ]; then
2511 shutdown_node_hard $client
2513 zconf_umount_clients $client $mnt -f
2519 local facets="$(get_facets OST),$(get_facets MDS)"
2522 combined_mgs_mds || facets="$facets,mgs"
2524 for facet in ${facets//,/ }; do
2525 if [ $(facet_active_host $facet) == $host ]; then
2526 affected="$affected $facet"
2530 echo $(comma_list $affected)
2535 local host=${2:-$(facet_host $facet)}
2537 local label=$(convert_facet2label $facet)
2538 do_node $host $LCTL dl | awk '{ print $4 }' | grep -q "^$label\$"
2541 facets_up_on_host () {
2543 local facets=$(facets_on_host $host)
2546 for facet in ${facets//,/ }; do
2547 if $(facet_up $facet $host); then
2548 affected_up="$affected_up $facet"
2552 echo $(comma_list $affected_up)
2557 local affected_facet
2558 local affected_facets
2560 if [[ "$FAILURE_MODE" = HARD ]]; then
2561 if [[ $(facet_fstype $facet) = ldiskfs ]] &&
2562 dm_flakey_supported $facet; then
2563 affected_facets=$(affected_facets $facet)
2564 for affected_facet in ${affected_facets//,/ }; do
2565 unexport_dm_dev $affected_facet
2569 shutdown_node_hard $(facet_active_host $facet)
2577 echo + $POWER_UP $node
2590 if [ "$FAILURE_MODE" = HARD ]; then
2591 reboot_node $(facet_active_host $facet)
2599 if [ "$FAILURE_MODE" = HARD ]; then
2609 for facet in ${facets//,/ }; do
2610 hosts=$(expand_list $hosts $(facet_host $facet) )
2616 _check_progs_installed () {
2620 for prog in $progs; do
2621 if ! [ "$(which $prog)" -o "${!prog}" ]; then
2622 echo $prog missing on $(hostname)
2629 check_progs_installed () {
2633 do_rpc_nodes "$nodes" _check_progs_installed $@
2636 # recovery-scale functions
2638 echo __$(echo $1 | tr '-' '_' | tr '.' '_')
2641 start_client_load() {
2644 local var=$(node_var_name $client)_load
2645 eval export ${var}=$load
2647 do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \
2648 BREAK_ON_ERROR=$BREAK_ON_ERROR \
2649 END_RUN_FILE=$END_RUN_FILE \
2650 LOAD_PID_FILE=$LOAD_PID_FILE \
2651 TESTLOG_PREFIX=$TESTLOG_PREFIX \
2652 TESTNAME=$TESTNAME \
2653 DBENCH_LIB=$DBENCH_LIB \
2654 DBENCH_SRC=$DBENCH_SRC \
2655 CLIENT_COUNT=$((CLIENTCOUNT - 1)) \
2660 MPIRUN_OPTIONS=\\\"$MPIRUN_OPTIONS\\\" \
2661 MACHINEFILE_OPTION=\\\"$MACHINEFILE_OPTION\\\" \
2662 num_clients=$(get_node_count ${CLIENTS//,/ }) \
2663 ior_THREADS=$ior_THREADS ior_iteration=$ior_iteration \
2664 ior_blockSize=$ior_blockSize \
2665 ior_blockUnit=$ior_blockUnit \
2666 ior_xferSize=$ior_xferSize ior_type=$ior_type \
2667 ior_DURATION=$ior_DURATION \
2668 ior_stripe_params=\\\"$ior_stripe_params\\\" \
2669 ior_custom_params=\\\"$ior_custom_param\\\" \
2670 mpi_ior_custom_threads=$mpi_ior_custom_threads \
2673 log "Started client load: ${load} on $client"
2675 # get the children process IDs
2676 local pids=$(ps --ppid $ppid -o pid= | xargs)
2677 CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $ppid $pids"
2681 start_client_loads () {
2682 local -a clients=(${1//,/ })
2683 local numloads=${#CLIENT_LOADS[@]}
2686 for ((nodenum=0; nodenum < ${#clients[@]}; nodenum++ )); do
2687 testnum=$((nodenum % numloads))
2688 start_client_load ${clients[nodenum]} ${CLIENT_LOADS[testnum]}
2690 # bug 22169: wait the background threads to start
2694 # only for remote client
2695 check_client_load () {
2697 local var=$(node_var_name $client)_load
2698 local testload=run_${!var}.sh
2700 ps auxww | grep -v grep | grep $client | grep -q $testload || return 1
2702 # bug 18914: try to connect several times not only when
2703 # check ps, but while check_node_health also
2707 while [ $RC = 254 -a $tries -gt 0 ]; do
2711 if ! check_node_health $client; then
2713 if [ $RC -eq 254 ]; then
2714 # FIXME: not sure how long we shuold sleep here
2718 echo "check node health failed: RC=$RC "
2722 # We can continue try to connect if RC=254
2723 # Just print the warning about this
2724 if [ $RC = 254 ]; then
2725 echo "got a return status of $RC from do_node while checking " \
2726 "node health on $client"
2729 # see if the load is still on the client
2732 while [ $RC = 254 -a $tries -gt 0 ]; do
2736 if ! do_node $client \
2737 "ps auxwww | grep -v grep | grep -q $testload"; then
2742 if [ $RC = 254 ]; then
2743 echo "got a return status of $RC from do_node while checking " \
2744 "(node health and 'ps') the client load on $client"
2745 # see if we can diagnose a bit why this is
2750 check_client_loads () {
2751 local clients=${1//,/ }
2755 for client in $clients; do
2756 check_client_load $client
2758 if [ "$rc" != 0 ]; then
2759 log "Client load failed on node $client, rc=$rc"
2765 restart_client_loads () {
2766 local clients=${1//,/ }
2767 local expectedfail=${2:-""}
2771 for client in $clients; do
2772 check_client_load $client
2774 if [ "$rc" != 0 -a "$expectedfail" ]; then
2775 local var=$(node_var_name $client)_load
2776 start_client_load $client ${!var}
2777 echo "Restarted client load ${!var}: on $client. Checking ..."
2778 check_client_load $client
2780 if [ "$rc" != 0 ]; then
2781 log "Client load failed to restart on node $client, rc=$rc"
2782 # failure one client load means test fail
2783 # we do not need to check other
2792 # Start vmstat and save its process ID in a file.
2797 [ -z "$nodes" -o -z "$pid_file" ] && return 0
2800 "vmstat 1 > $TESTLOG_PREFIX.$TESTNAME.vmstat.\\\$(hostname -s).log \
2801 2>/dev/null </dev/null & echo \\\$! > $pid_file"
2804 # Display the nodes on which client loads failed.
2805 print_end_run_file() {
2809 [ -s $file ] || return 0
2811 echo "Found the END_RUN_FILE file: $file"
2814 # A client load will stop if it finds the END_RUN_FILE file.
2815 # That does not mean the client load actually failed though.
2816 # The first node in END_RUN_FILE is the one we are interested in.
2819 if [ -n "$node" ]; then
2820 local var=$(node_var_name $node)_load
2822 local prefix=$TESTLOG_PREFIX
2823 [ -n "$TESTNAME" ] && prefix=$prefix.$TESTNAME
2824 local stdout_log=$prefix.run_${!var}_stdout.$node.log
2825 local debug_log=$(echo $stdout_log | sed 's/\(.*\)stdout/\1debug/')
2827 echo "Client load ${!var} failed on node $node:"
2833 # Stop the process which had its PID saved in a file.
2838 [ -z "$nodes" -o -z "$pid_file" ] && return 0
2840 do_nodes $nodes "test -f $pid_file &&
2841 { kill -s TERM \\\$(cat $pid_file); rm -f $pid_file; }" || true
2844 # Stop all client loads.
2845 stop_client_loads() {
2846 local nodes=${1:-$CLIENTS}
2849 # stop the client loads
2850 stop_process $nodes $pid_file
2852 # clean up the processes that started them
2853 [ -n "$CLIENT_LOAD_PIDS" ] && kill -9 $CLIENT_LOAD_PIDS 2>/dev/null || true
2855 # End recovery-scale functions
2857 # verify that lustre actually cleaned up properly
2859 VAR=$(lctl get_param -n catastrophe 2>&1)
2860 if [ $? = 0 ] ; then
2861 if [ $VAR != 0 ]; then
2862 error "LBUG/LASSERT detected"
2865 BUSY=$(dmesg | grep -i destruct || true)
2866 if [ -n "$BUSY" ]; then
2868 [ -e $TMP/debug ] && mv $TMP/debug $TMP/debug-busy.$(date +%s)
2872 check_mem_leak || exit 204
2874 [[ $($LCTL dl 2>/dev/null | wc -l) -gt 0 ]] && $LCTL dl &&
2875 echo "$TESTSUITE: lustre didn't clean up..." 1>&2 &&
2878 if module_loaded lnet || module_loaded libcfs; then
2879 echo "$TESTSUITE: modules still loaded..." 1>&2
2888 if [[ "$1" == "--verbose" ]]; then
2903 PREV_RESULT=$(do_node $node "$TEST")
2905 RESULT=$(do_node $node "$TEST")
2906 if [[ "$RESULT" == "$FINAL" ]]; then
2907 [[ -z "$RESULT" || $WAIT -le $sleep ]] ||
2908 echo "Updated after ${WAIT}s: wanted '$FINAL'"\
2912 if [[ $verbose && "$RESULT" != "$PREV_RESULT" ]]; then
2913 echo "Changed after ${WAIT}s: from '$PREV_RESULT'"\
2917 [[ $WAIT -ge $MAX ]] && break
2918 [[ $((WAIT % print)) -eq 0 ]] &&
2919 echo "Waiting $((MAX - WAIT)) secs for update"
2920 WAIT=$((WAIT + sleep))
2923 echo "Update not seen after ${MAX}s: wanted '$FINAL' got '$RESULT'"
2927 wait_update_facet() {
2929 [ "$1" = "--verbose" ] && verbose="$1" && shift
2933 wait_update $verbose $(facet_active_host $facet) "$@"
2937 do_nodes $(comma_list $(mdts_nodes)) \
2938 "lctl set_param -n os[cd]*.*MDT*.force_sync=1"
2939 do_nodes $(comma_list $(osts_nodes)) \
2940 "lctl set_param -n osd*.*OS*.force_sync=1" 2>&1 |
2941 grep -v 'Found no match'
2945 local zfs_wait=${2:-5}
2947 # the occupied disk space will be released
2948 # only after TXGs are committed
2949 if [[ $(facet_fstype $1) == zfs ]]; then
2950 echo "sleep $zfs_wait for ZFS $(facet_fstype $1)"
2955 wait_delete_completed_mds() {
2956 local max_wait=${1:-20}
2958 local stime=$(date +%s)
2963 # find MDS with pending deletions
2964 for node in $(mdts_nodes); do
2965 changes=$(do_node $node "$LCTL get_param -n osc.*MDT*.sync_*" \
2966 2>/dev/null | calc_sum)
2967 if [[ $changes -eq 0 ]]; then
2970 mds2sync="$mds2sync $node"
2972 if [ -z "$mds2sync" ]; then
2973 wait_zfs_commit $SINGLEMDS
2976 mds2sync=$(comma_list $mds2sync)
2978 # sync MDS transactions
2979 do_nodes $mds2sync "$LCTL set_param -n os[cd]*.*MD*.force_sync 1"
2981 # wait till all changes are sent and commmitted by OSTs
2982 # for ldiskfs space is released upon execution, but DMU
2983 # do this upon commit
2986 while [[ $WAIT -ne $max_wait ]]; do
2987 changes=$(do_nodes $mds2sync \
2988 "$LCTL get_param -n osc.*MDT*.sync_*" | calc_sum)
2989 #echo "$node: $changes changes on all"
2990 if [[ $changes -eq 0 ]]; then
2991 wait_zfs_commit $SINGLEMDS
2993 # the occupied disk space will be released
2994 # only after TXGs are committed
2995 wait_zfs_commit ost1
3003 echo "Delete is not completed in $((etime - stime)) seconds"
3004 do_nodes $mds2sync "$LCTL get_param osc.*MDT*.sync_*"
3011 # we can use "for" here because we are waiting the slowest
3012 for host in ${hostlist//,/ }; do
3013 check_network "$host" 900
3015 while ! do_nodes $hostlist hostname > /dev/null; do sleep 5; done
3022 for facet in ${facetlist//,/ }; do
3023 hostlist=$(expand_list $hostlist $(facet_active_host $facet))
3025 wait_for_host $hostlist
3028 _wait_recovery_complete () {
3031 # Use default policy if $2 is not passed by caller.
3032 local MAX=${2:-$(max_recovery_time)}
3037 while [ $WAIT -lt $MAX ]; do
3038 STATUS=$(lctl get_param -n $param | grep status)
3040 [[ $STATUS = "status: COMPLETE" || $STATUS = "status: INACTIVE" ]] && return 0
3043 echo "Waiting $((MAX - WAIT)) secs for $param recovery done. $STATUS"
3045 echo "$param recovery not done in $MAX sec. $STATUS"
3049 wait_recovery_complete () {
3052 # with an assumption that at_max is the same on all nodes
3053 local MAX=${2:-$(max_recovery_time)}
3056 if [ "$FAILURE_MODE" = HARD ]; then
3057 facets=$(facets_on_host $(facet_active_host $facet))
3059 echo affected facets: $facets
3061 # we can use "for" here because we are waiting the slowest
3062 for facet in ${facets//,/ }; do
3063 local var_svc=${facet}_svc
3064 local param="*.${!var_svc}.recovery_status"
3066 local host=$(facet_active_host $facet)
3067 do_rpc_nodes "$host" _wait_recovery_complete $param $MAX
3071 wait_mds_ost_sync () {
3072 # just because recovery is done doesn't mean we've finished
3073 # orphan cleanup. Wait for llogs to get synchronized.
3074 echo "Waiting for orphan cleanup..."
3075 # MAX value includes time needed for MDS-OST reconnection
3076 local MAX=$(( TIMEOUT * 2 ))
3077 local WAIT_TIMEOUT=${1:-$MAX}
3080 local list=$(comma_list $(mdts_nodes))
3081 local cmd="$LCTL get_param -n osp.*osc*.old_sync_processed"
3082 if ! do_facet $SINGLEMDS \
3083 "$LCTL list_param osp.*osc*.old_sync_processed 2> /dev/null"
3085 # old way, use mds_sync
3087 list=$(comma_list $(osts_nodes))
3088 cmd="$LCTL get_param -n obdfilter.*.mds_sync"
3091 echo "wait $WAIT_TIMEOUT secs maximumly for $list mds-ost sync done."
3092 while [ $WAIT -lt $WAIT_TIMEOUT ]; do
3093 local -a sync=($(do_nodes $list "$cmd"))
3096 for ((i=0; i<${#sync[@]}; i++)); do
3098 [ ${sync[$i]} -eq 1 ] && continue
3100 [ ${sync[$i]} -eq 0 ] && continue
3102 # there is a not finished MDS-OST synchronization
3106 sleep 2 # increase waiting time and cover statfs cache
3107 [ ${con} -eq 1 ] && return 0
3108 echo "Waiting $WAIT secs for $list $i mds-ost sync done."
3112 # show which nodes are not finished.
3113 cmd=$(echo $cmd | sed 's/-n//')
3114 do_nodes $list "$cmd"
3115 echo "$facet recovery node $i not done in $WAIT_TIMEOUT sec. $STATUS"
3119 # Wait OSTs to be active on both client and MDT side.
3121 local cmd="$LCTL get_param -n lov.$FSNAME-clilov-*.target_obd |
3122 awk 'BEGIN {c = 0} /ACTIVE/{c += 1} END {printf \\\"%d\\\", c}'"
3123 wait_update $HOSTNAME "eval $cmd" $OSTCOUNT ||
3124 error "wait_update OSTs up on client failed"
3126 cmd="$LCTL get_param osp.$FSNAME-OST*-MDT0000.prealloc_last_id |
3127 awk '/=[1-9][0-9]/ { c += 1 } END { printf \\\"%d\\\", c }'"
3128 wait_update_facet $SINGLEMDS "eval $cmd" $OSTCOUNT ||
3129 error "wait_update OSTs up on MDT0000 failed"
3132 wait_destroy_complete () {
3133 echo "Waiting for local destroys to complete"
3134 # MAX value shouldn't be big as this mean server responsiveness
3135 # never increase this just to make test pass but investigate
3136 # why it takes so long time
3139 while [ $WAIT -lt $MAX ]; do
3140 local -a RPCs=($($LCTL get_param -n osc.*.destroys_in_flight))
3144 for ((i=0; i<${#RPCs[@]}; i++)); do
3145 [ ${RPCs[$i]} -eq 0 ] && continue
3146 # there are still some destroy RPCs in flight
3151 [ ${con} -eq 1 ] && return 0 # done waiting
3152 echo "Waiting ${WAIT}s for local destroys to complete"
3155 echo "Local destroys weren't done in $MAX sec."
3159 wait_delete_completed() {
3160 wait_delete_completed_mds $1 || return $?
3161 wait_destroy_complete || return $?
3170 # conf-sanity 31 takes a long time cleanup
3171 while [ $WAIT -lt 300 ]; do
3172 running=$(do_facet ${facet} "lsmod | grep lnet > /dev/null &&
3173 lctl dl | grep ' ST ' || true")
3174 [ -z "${running}" ] && return 0
3175 echo "waited $WAIT for${running}"
3176 [ $INTERVAL -lt 64 ] && INTERVAL=$((INTERVAL + INTERVAL))
3178 WAIT=$((WAIT + INTERVAL))
3180 echo "service didn't stop after $WAIT seconds. Still running:"
3185 wait_remote_prog () {
3191 [ "$PDSH" = "no_dsh" ] && return 0
3193 while [ $WAIT -lt $2 ]; do
3194 running=$(ps uax | grep "$PDSH.*$prog.*$MOUNT" | grep -v grep) || true
3195 [ -z "${running}" ] && return 0 || true
3196 echo "waited $WAIT for: "
3198 [ $INTERVAL -lt 60 ] && INTERVAL=$((INTERVAL + INTERVAL))
3200 WAIT=$((WAIT + INTERVAL))
3202 local pids=$(ps uax | grep "$PDSH.*$prog.*$MOUNT" | grep -v grep | awk '{print $2}')
3203 [ -z "$pids" ] && return 0
3204 echo "$PDSH processes still exists after $WAIT seconds. Still running: $pids"
3205 # FIXME: not portable
3206 for pid in $pids; do
3207 cat /proc/${pid}/status || true
3208 cat /proc/${pid}/wchan || true
3210 kill -9 $pid || true
3219 local clients=${1:-$CLIENTS}
3221 if [ -z "$clients" ]; then
3224 $PDSH $clients "$LFS df $MOUNT" > /dev/null
3229 # not every config has many clients
3235 # usually checked on particular client or locally
3244 client_reconnect_try() {
3245 local f=$MOUNT/recon
3248 if [ -z "$CLIENTS" ]; then
3249 $LFS df $MOUNT; uname -n >> $f
3251 do_nodes $CLIENTS "$LFS df $MOUNT; uname -n >> $f" > /dev/null
3253 echo "Connected clients: $(cat $f)"
3254 ls -l $f > /dev/null
3258 client_reconnect() {
3259 # one client_reconnect_try call does not always do the job...
3261 client_reconnect_try && break
3266 affected_facets () {
3269 local host=$(facet_active_host $facet)
3270 local affected=$facet
3272 if [ "$FAILURE_MODE" = HARD ]; then
3273 affected=$(facets_up_on_host $host)
3279 local E2FSCK_ON_MDT0=false
3280 if [ "$1" == "--fsck" ]; then
3282 [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&