+ [ "$FSTYPE" ] && FSTYPE_OPT="--backfstype $FSTYPE"
+
+ if [ ! -z $SEC ]; then
+ MDS_MKFS_OPTS="$MDS_MKFS_OPTS --param srpc.flavor.default=$SEC"
+ MDSn_MKFS_OPTS="$MDSn_MKFS_OPTS --param srpc.flavor.default=$SEC"
+ OST_MKFS_OPTS="$OST_MKFS_OPTS --param srpc.flavor.default=$SEC"
+ fi
+
+ stopall
+ # We need ldiskfs here, may as well load them all
+ load_modules
+ [ "$CLIENTONLY" ] && return
+ echo "Formatting mdts, osts"
+ for num in `seq $MDSCOUNT`; do
+ echo "Format mds$num: $(mdsdevname $num)"
+ if $VERBOSE; then
+ add mds$num `mdsmkfsopts $num` $FSTYPE_OPT --reformat `mdsdevname $num` || exit 9
+ else
+ add mds$num `mdsmkfsopts $num` $FSTYPE_OPT --reformat `mdsdevname $num` > /dev/null || exit 9
+ fi
+ done
+
+ for num in `seq $OSTCOUNT`; do
+ echo "Format ost$num: $(ostdevname $num)"
+ if $VERBOSE; then
+ add ost$num $OST_MKFS_OPTS --reformat `ostdevname $num` || exit 10
+ else
+ add ost$num $OST_MKFS_OPTS --reformat `ostdevname $num` > /dev/null || exit 10
+ fi
+ done
+}
+
+mount_client() {
+ grep " $1 " /proc/mounts || zconf_mount $HOSTNAME $*
+}
+
+umount_client() {
+ grep " $1 " /proc/mounts && zconf_umount `hostname` $*
+}
+
+# return value:
+# 0: success, the old identity set already.
+# 1: success, the old identity does not set.
+# 2: fail.
+switch_identity() {
+ local num=$1
+ local switch=$2
+ local j=`expr $num - 1`
+ local MDT="`(do_facet mds$num lctl get_param -N mdt.*MDT*$j 2>/dev/null | cut -d"." -f2 2>/dev/null) || true`"
+
+ if [ -z "$MDT" ]; then
+ return 2
+ fi
+
+ local old="`do_facet mds$num "lctl get_param -n mdt.$MDT.identity_upcall"`"
+
+ if $switch; then
+ do_facet mds$num "lctl set_param -n mdt.$MDT.identity_upcall \"$L_GETIDENTITY\""
+ else
+ do_facet mds$num "lctl set_param -n mdt.$MDT.identity_upcall \"NONE\""
+ fi
+
+ do_facet mds$num "lctl set_param -n mdt/$MDT/identity_flush \"-1\""
+
+ if [ $old = "NONE" ]; then
+ return 1
+ else
+ return 0
+ fi
+}
+
+remount_client()
+{
+ zconf_umount `hostname` $1 || error "umount failed"
+ zconf_mount `hostname` $1 || error "mount failed"
+}
+
+set_obd_timeout() {
+ local facet=$1
+ local timeout=$2
+
+ do_facet $facet lsmod | grep -q obdclass || \
+ do_facet $facet "modprobe obdclass"
+
+ do_facet $facet "lctl set_param timeout=$timeout"
+}
+
+writeconf_facet () {
+ local facet=$1
+ local dev=$2
+
+ do_facet $facet "$TUNEFS --writeconf $dev"
+}
+
+writeconf_all () {
+ for num in `seq $MDSCOUNT`; do
+ DEVNAME=$(mdsdevname $num)
+ writeconf_facet mds$num $DEVNAME
+ done
+
+ for num in `seq $OSTCOUNT`; do
+ DEVNAME=$(ostdevname $num)
+ writeconf_facet ost$num $DEVNAME
+ done
+}
+
+setupall() {
+ load_modules
+ init_gss
+ if [ -z "$CLIENTONLY" ]; then
+ echo "Setup mdts, osts"
+ echo $WRITECONF | grep -q "writeconf" && \
+ writeconf_all
+ for num in `seq $MDSCOUNT`; do
+ DEVNAME=$(mdsdevname $num)
+ set_obd_timeout mds$num $TIMEOUT
+ start mds$num $DEVNAME $MDS_MOUNT_OPTS
+
+ # We started mds, now we should set failover variables properly.
+ # Set mds${num}failover_HOST if it is not set (the default failnode).
+ local varname=mds${num}failover_HOST
+ if [ -z "${!varname}" ]; then
+ eval mds${num}failover_HOST=$(facet_host mds$num)
+ fi
+
+ if [ $IDENTITY_UPCALL != "default" ]; then
+ switch_identity $num $IDENTITY_UPCALL
+ fi
+ done
+ for num in `seq $OSTCOUNT`; do
+ DEVNAME=$(ostdevname $num)
+ set_obd_timeout ost$num $TIMEOUT
+ start ost$num $DEVNAME $OST_MOUNT_OPTS
+
+ # We started ost$num, now we should set ost${num}failover variable properly.
+ # Set ost${num}failover_HOST if it is not set (the default failnode).
+ varname=ost${num}failover_HOST
+ if [ -z "${!varname}" ]; then
+ eval ost${num}failover_HOST=$(facet_host ost${num})
+ fi
+
+ done
+ fi
+ # wait a while to allow sptlrpc configuration be propogated to targets,
+ # only needed when mounting new target devices.
+ $GSS && sleep 10
+
+ [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
+ mount_client $MOUNT
+ [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT
+
+ if [ "$MOUNT_2" ]; then
+ mount_client $MOUNT2
+ [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT2
+ fi
+
+ # by remounting mdt before ost, initial connect from mdt to ost might
+ # timeout because ost is not ready yet. wait some time to its fully
+ # recovery. initial obd_connect timeout is 5s; in GSS case it's preceeded
+ # by a context negotiation rpc with $TIMEOUT.
+ # FIXME better by monitoring import status.
+ if $GSS; then
+ sleep $((TIMEOUT + 5))
+ else
+ sleep 5
+ fi
+}
+
+mounted_lustre_filesystems() {
+ awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts
+}
+
+init_facet_vars () {
+ local facet=$1
+ shift
+ local device=$1
+
+ shift
+
+ eval export ${facet}_dev=${device}
+ eval export ${facet}_opt=\"$@\"
+
+ local dev=${facet}_dev
+ local label=$(do_facet ${facet} "e2label ${!dev}")
+ [ -z "$label" ] && echo no label for ${!dev} && exit 1
+
+ eval export ${facet}_svc=${label}
+
+ local varname=${facet}failover_HOST
+ if [ -z "${!varname}" ]; then
+ eval $varname=$(facet_host $facet)
+ fi
+}
+
+init_facets_vars () {
+ local DEVNAME
+
+ for num in `seq $MDSCOUNT`; do
+ DEVNAME=`mdsdevname $num`
+ init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
+ done
+
+ for num in `seq $OSTCOUNT`; do
+ DEVNAME=`ostdevname $num`
+ init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
+ done
+}
+
+check_config () {
+ local mntpt=$1
+
+ echo Checking config lustre mounted on $mntpt
+ local mgshost=$(mount | grep " $mntpt " | awk -F@ '{print $1}')
+ mgshost=$(echo $mgshost | awk -F: '{print $1}')
+ if [ "$mgshost" != "$mgs_HOST" ]; then
+ FAIL_ON_ERROR=true \
+ error "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST
+ Please use correct config or set mds_HOST correctly!"
+ fi
+}
+
+check_and_setup_lustre() {
+ local MOUNTED=$(mounted_lustre_filesystems)
+ if [ -z "$MOUNTED" ] || ! $(echo $MOUNTED | grep -w -q $MOUNT); then
+ [ "$REFORMAT" ] && formatall
+ setupall
+ MOUNTED=$(mounted_lustre_filesystems | head -1)
+ [ -z "$MOUNTED" ] && error "NAME=$NAME not mounted"
+ export I_MOUNTED=yes
+ else
+ check_config $MOUNT
+ init_facets_vars
+ fi
+ if [ "$ONLY" == "setup" ]; then
+ exit 0
+ fi
+}
+
+cleanup_and_setup_lustre() {
+ if [ "$ONLY" == "cleanup" -o "`mount | grep $MOUNT`" ]; then
+ lctl set_param debug=0 || true
+ cleanupall
+ if [ "$ONLY" == "cleanup" ]; then
+ exit 0
+ fi
+ fi
+ check_and_setup_lustre
+}
+
+check_and_cleanup_lustre() {
+ if [ "`mount | grep $MOUNT`" ]; then
+ [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]*
+ fi
+ if [ "$I_MOUNTED" = "yes" ]; then
+ cleanupall -f || error "cleanup failed"
+ fi
+ unset I_MOUNTED
+}
+
+#######
+# General functions
+
+check_network() {
+ local NETWORK=0
+ local WAIT=0
+ local MAX=$2
+ while [ $NETWORK -eq 0 ]; do
+ ping -c 1 -w 3 $1 > /dev/null
+ if [ $? -eq 0 ]; then
+ NETWORK=1
+ else
+ WAIT=$((WAIT + 5))
+ echo "waiting for $1, $((MAX - WAIT)) secs left"
+ sleep 5
+ fi
+ if [ $WAIT -gt $MAX ]; then
+ echo "Network not available"
+ exit 1
+ fi
+ done
+}
+check_port() {
+ while( !($DSH2 $1 "netstat -tna | grep -q $2") ) ; do
+ sleep 9
+ done
+}
+
+no_dsh() {
+ shift
+ eval $@
+}
+
+comma_list() {
+ # the sed converts spaces to commas, but leaves the last space
+ # alone, so the line doesn't end with a comma.
+ echo "$*" | tr -s " " "\n" | sort -b -u | tr "\n" " " | sed 's/ \([^$]\)/,\1/g'
+}
+
+absolute_path() {
+ (cd `dirname $1`; echo $PWD/`basename $1`)
+}
+
+##################################
+# Adaptive Timeouts funcs
+
+at_is_valid() {
+ if [ -z "$AT_MAX_PATH" ]; then
+ AT_MAX_PATH=$(do_facet $SINGLEMDS "find /sys/ -name at_max")
+ [ -z "$AT_MAX_PATH" ] && echo "missing /sys/.../at_max " && return 1
+ fi
+ return 0
+}
+
+at_is_enabled() {
+ at_is_valid || error "invalid call"
+
+ # only check mds, we assume at_max is the same on all nodes
+ local at_max=$(do_facet $SINGLEMDS "cat $AT_MAX_PATH")
+ if [ $at_max -eq 0 ]; then
+ return 1
+ else
+ return 0
+ fi
+}
+
+at_max_get() {
+ local facet=$1
+
+ at_is_valid || error "invalid call"
+
+ # suppose that all ost-s has the same at_max set
+ if [ $facet == "ost" ]; then
+ do_facet ost1 "cat $AT_MAX_PATH"
+ else
+ do_facet $facet "cat $AT_MAX_PATH"
+ fi
+}
+
+at_max_set() {
+ local at_max=$1
+ shift
+
+ at_is_valid || error "invalid call"
+
+ local facet
+ for facet in $@; do
+ if [ $facet == "ost" ]; then
+ for i in `seq $OSTCOUNT`; do
+ do_facet ost$i "echo $at_max > $AT_MAX_PATH"
+ done
+ elif [ $facet == "mds" ]; then
+ for i in `seq $MDSCOUNT`; do
+ do_facet mds$i "echo $at_max > $AT_MAX_PATH"
+ done
+ else
+ do_facet $facet "echo $at_max > $AT_MAX_PATH"
+ fi
+ done
+}
+
+##################################
+# OBD_FAIL funcs
+
+drop_request() {