init_test_env $@
-. ${CONFIG:=$LUSTRE/tests/cfg/insanity-local.sh}
+. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
ALWAYS_EXCEPT="10 $INSANITY_EXCEPT"
+if [ "$FAILURE_MODE" = "HARD" ]; then
+ mixed_ost_devs && CONFIG_EXCEPTIONS="0 2 4 5 6 8" && \
+ echo -n "Several ost services on one ost node are used with FAILURE_MODE=$FAILURE_MODE. " && \
+ echo "Except the tests: $CONFIG_EXCEPTIONS" && \
+ ALWAYS_EXCEPT="$ALWAYS_EXCEPT $CONFIG_EXCEPTIONS"
+fi
+
#
[ "$SLOW" = "no" ] && EXCEPT_SLOW=""
build_test_filter
+SINGLECLIENT=${SINGLECLIENT:-$HOSTNAME}
+LIVE_CLIENT=${LIVE_CLIENT:-$SINGLECLIENT}
+FAIL_CLIENTS=${FAIL_CLIENTS:-$RCLIENTS}
+
assert_env mds_HOST MDS_MKFS_OPTS MDSDEV
assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
assert_env LIVE_CLIENT FSNAME
# This can be a regexp, to allow more clients
-CLIENTS=${CLIENTS:-"`comma_list $LIVE_CLIENT $FAIL_CLIENTS $EXTRA_CLIENTS`"}
+CLIENTS=${CLIENTS:-"`comma_list $LIVE_CLIENT $FAIL_CLIENTS`"}
DIR=${DIR:-$MOUNT}
fail_clients() {
num=$1
+
+ log "Request clients to fail: ${num}. Num of clients to fail: ${FAIL_NUM}, already failed: $DOWN_NUM"
if [ -z "$num" ] || [ "$num" -gt $((FAIL_NUM - DOWN_NUM)) ]; then
num=$((FAIL_NUM - DOWN_NUM))
fi
if [ -z "$num" ] || [ "$num" -le 0 ]; then
+ log "No clients failed!"
return
fi
echo "Waiting for df pid: $DFPID"
wait $DFPID || { echo "df returned $?" && return 1; }
- facet_failover ost1 || return 4
- echo "Waiting for df pid: $DFPID"
- wait $DFPID || { echo "df returned $?" && return 2; }
-
- if [ $OSTCOUNT -gt 1 ]; then
- facet_failover ost2 || return 5
- echo "Waiting for df pid: $DFPID"
- wait $DFPID || { echo "df returned $?" && return 3; }
- fi
+ for i in $(seq $OSTCOUNT) ; do
+ facet_failover ost$i || return 4
+ echo "Waiting for df pid: $DFPID"
+ wait $DFPID || { echo "df returned $?" && return 3; }
+ done
return 0
}
run_test 0 "Fail all nodes, independently"
############### Fifth Failure Mode ###############
test_5() {
- [ $OSTCOUNT -lt 1 ] && skip "$OSTCOUNT < 1, not enough OSTs" && return 0
+ [ $OSTCOUNT -lt 2 ] && skip "$OSTCOUNT < 2, not enough OSTs" && return 0
echo "Fifth Failure Mode: OST/OST `date`"
init_test_env() {
export LUSTRE=`absolute_path $LUSTRE`
export TESTSUITE=`basename $0 .sh`
- export LTESTDIR=${LTESTDIR:-$LUSTRE/../ltest}
[ -d /r ] && export ROOT=${ROOT:-/r}
export TMP=${TMP:-$ROOT/tmp}
ONLY=${ONLY:-$*}
[ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true
+ rm -f $TMP/*active
}
exit 1
fi
- echo "Starting client: $client: $OPTIONS $device $mnt"
+ echo "Starting client: $client: $OPTIONS $device $mnt"
do_node $client mkdir -p $mnt
do_node $client mount -t lustre $OPTIONS $device $mnt || return 1
}
zconf_mount_clients() {
+ local OPTIONS
local clients=$1
local mnt=$2
- echo "Mounting clients: $clients"
- local client
- for client in ${clients//,/ }; do
- zconf_mount $client $mnt || true
- done
+
+ # Only supply -o to mount if we have options
+ if [ -n "$MOUNTOPT" ]; then
+ OPTIONS="-o $MOUNTOPT"
+ fi
+ local device=$MGSNID:/$FSNAME
+ if [ -z "$mnt" -o -z "$FSNAME" ]; then
+ echo Bad zconf mount command: opt=$OPTIONS dev=$device mnt=$mnt
+ exit 1
+ fi
+
+ echo "Starting client $clients: $OPTIONS $device $mnt"
+ do_nodes $clients mkdir -p $mnt
+ do_nodes $clients mount -t lustre $OPTIONS $device $mnt || return 1
+
+ do_nodes $clients "sysctl -w lnet.debug=$PTLDEBUG;
+ sysctl -w lnet.subsystem_debug=${SUBSYSTEM# };
+ sysctl -w lnet.debug_mb=${DEBUG_SIZE};"
+
+ return 0
}
zconf_umount_clients() {
local mnt=$2
[ "$3" ] && force=-f
- echo "Umounting clients: $clients"
- local client
- for client in ${clients//,/ }; do
- zconf_umount $client $mnt $force || true
- done
+ echo "Stopping clients: $clients $mnt (opts:$force)"
+ do_nodes $clients umount $force $mnt
}
shutdown_facet() {
local facet=$1
local activevar=${facet}active
- if [ -f ./${facet}active ] ; then
- source ./${facet}active
+ if [ -f $TMP/${facet}active ] ; then
+ source $TMP/${facet}active
fi
active=${!activevar}
fi
# save the active host for this facet
activevar=${facet}active
- echo "$activevar=${!activevar}" > ./$activevar
+ echo "$activevar=${!activevar}" > $TMP/$activevar
}
do_node() {
return ${PIPESTATUS[0]}
}
+do_nodes() {
+ local nodes=$1
+ shift
+
+ nodes=${nodes//,/ }
+ # split list to local and remote
+ local rnodes=$(echo " $nodes " | sed -re "s/\s+$HOSTNAME\s+/ /g")
+
+ if [ "$(get_node_count $nodes)" != "$(get_node_count $rnodes)" ]; then
+ do_node $HOSTNAME $@
+ fi
+
+ [ -z "$(echo $rnodes)" ] && return 0
+
+ # This is part from do_node
+ local myPDSH=$PDSH
+
+ rnodes=$(comma_list $rnodes)
+ [ -z "$myPDSH" -o "$myPDSH" = "no_dsh" ] && \
+ echo "cannot run remote command on $rnodes with $myPDSH" && return 128
+
+ if $VERBOSE; then
+ echo "CMD: $rnodes $@" >&2
+ $myPDSH $rnodes $LCTL mark "$@" > /dev/null 2>&1 || :
+ fi
+
+ if [ "$myPDSH" = "rsh" ]; then
+# we need this because rsh does not return exit code of an executed command
+ local command_status="$TMP/cs"
+ rsh $rnodes ":> $command_status"
+ rsh $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin;
+ cd $RPWD; sh -c \"$@\") ||
+ echo command failed >$command_status"
+ [ -n "$($myPDSH $rnodes cat $command_status)" ] && return 1 || true
+ return 0
+ fi
+ $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; sh -c \"$@\")" | sed -re "s/\w+:\s//g"
+ return ${PIPESTATUS[0]}
+}
+
do_facet() {
facet=$1
shift
shift
# make sure its not already running
stop ${facet} -f
- rm -f ${facet}active
+ rm -f $TMP/${facet}active
do_facet ${facet} $MKFS $*
}
if [ -n "$CLIENTS" ]; then
zconf_umount_clients $CLIENTS $MOUNT "$*" || true
- zconf_umount_clients $CLIENTS $MOUNT2 "$*" || true
+ [ -n "$MOUNT2" ] && { zconf_umount_clients $CLIENTS $MOUNT2 "$*" || true; }
fi
[ "$CLIENTONLY" ] && return
+ # The add fn does rm ${facet}active file, this would be enough
+ # if we use do_facet <facet> only after the facet added, but
+ # currently we use do_facet mds in local.sh
stop mds -f
+ rm -f ${TMP}/mds${num}active
+
for num in `seq $OSTCOUNT`; do
stop ost$num -f
+ rm -f $TMP/ost${num}active
done
+
return 0
}
|| do_facet mds "$TUNEFS --writeconf $MDSDEV"
set_obd_timeout mds $TIMEOUT
start mds $MDSDEV $MDS_MOUNT_OPTS
+ # We started mds, now we should set failover variable properly.
+ # Set mdsfailover_HOST if it is not set (the default failnode).
+ mdsfailover_HOST=$(facet_host mds)
+
for num in `seq $OSTCOUNT`; do
DEVNAME=`ostdevname $num`
set_obd_timeout ost$num $TIMEOUT
start ost$num $DEVNAME $OST_MOUNT_OPTS
+
+ # We started ost$num, now we should set ost${num}failover variable properly.
+ # Set ost${num}failover_HOST if it is not set (the default failnode).
+ varname=ost${num}failover_HOST
+ if [ -z "${!varname}" ]; then
+ eval ost${num}failover_HOST=$(facet_host ost${num})
+ fi
+
done
fi
[ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
done
}
+set_nodes_failloc () {
+ local nodes=$1
+ local node
+
+ for node in $nodes ; do
+ do_node $node sysctl -w lustre.fail_loc=$2
+ done
+}
+
cancel_lru_locks() {
$LCTL mark "cancel_lru_locks $1 start"
for d in `find $LPROC/ldlm/namespaces | egrep -i $1`; do
eval ONLY_${O}=true
done
[ "$EXCEPT$ALWAYS_EXCEPT" ] && \
- log "skipping tests: `echo $EXCEPT $ALWAYS_EXCEPT`"
+ log "excepting tests: `echo $EXCEPT $ALWAYS_EXCEPT`"
[ "$EXCEPT_SLOW" ] && \
log "skipping tests SLOW=no: `echo $EXCEPT_SLOW`"
for E in $EXCEPT $ALWAYS_EXCEPT; do
lctl get_param version | grep -q patchless
}
+get_node_count() {
+ local nodes="$@"
+ echo $nodes | wc -w || true
+}
+
+mixed_ost_devs () {
+ local nodes=$(osts_nodes)
+ local osscount=$(get_node_count "$nodes")
+ [ ! "$OSTCOUNT" = "$osscount" ]
+}
+
check_runas_id_ret() {
local myRC=0
local myRUNAS_ID=$1
git://git.whamcloud.com / fs / lustre-release.git / commitdiff