export LOV_MAX_STRIPE_COUNT=2000
export DELETE_OLD_POOLS=${DELETE_OLD_POOLS:-false}
export KEEP_POOLS=${KEEP_POOLS:-false}
+ export PARALLEL=${PARALLEL:-"no"}
export MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
fi
}
-load_modules_local() {
- if [ -n "$MODPROBE" ]; then
- # use modprobe
- echo "Using modprobe to load modules"
- return 0
- fi
-
- # Create special udev test rules on every node
- if [ -f $LUSTRE/lustre/conf/99-lustre.rules ]; then {
- sed -e 's|/usr/sbin/lctl|$LCTL|g' $LUSTRE/lustre/conf/99-lustre.rules > /etc/udev/rules.d/99-lustre-test.rules
- } else {
- echo "SUBSYSTEM==\"lustre\", ACTION==\"change\", ENV{PARAM}==\"?*\", RUN+=\"$LCTL set_param '\$env{PARAM}=\$env{SETTING}'\"" > /etc/udev/rules.d/99-lustre-test.rules
- } fi
- udevadm control --reload-rules
- udevadm trigger
+do_lnetctl() {
+ $LCTL mark "$LNETCTL $*"
+ echo "$LNETCTL $*"
+ $LNETCTL "$@"
+}
+load_lnet() {
# For kmemleak-enabled kernels we need clear all past state
# that obviously has nothing to do with this Lustre run
# Disable automatic memory scanning to avoid perf hit.
load_module ../libcfs/libcfs/libcfs
# Prevent local MODOPTS_LIBCFS being passed as part of environment
# variable to remote nodes
- MODOPTS_LIBCFS=$saved_opts
+ unset MODOPTS_LIBCFS
- set_default_debug
- load_module ../lnet/lnet/lnet
+ set_default_debug "neterror net nettrace malloc"
+ if [ "$1" = "config_on_load=1" ]; then
+ load_module ../lnet/lnet/lnet
+ else
+ load_module ../lnet/lnet/lnet "$@"
+ fi
LNDPATH=${LNDPATH:-"../lnet/klnds"}
if [ -z "$LNETLND" ]; then
case $NETTYPE in
- o2ib*) LNETLND="o2iblnd/ko2iblnd" ;;
- tcp*) LNETLND="socklnd/ksocklnd" ;;
- *) local lnd="${NETTYPE%%[0-9]}lnd"
+ o2ib*) LNETLND="o2iblnd/ko2iblnd" ;;
+ tcp*) LNETLND="socklnd/ksocklnd" ;;
+ *) local lnd="${NETTYPE%%[0-9]}lnd"
[ -f "$LNDPATH/$lnd/k$lnd.ko" ] &&
LNETLND="$lnd/k$lnd" ||
LNETLND="socklnd/ksocklnd"
esac
fi
load_module ../lnet/klnds/$LNETLND
+
+ if [ "$1" = "config_on_load=1" ]; then
+ do_lnetctl lnet configure --all ||
+ return $?
+ fi
+}
+
+load_modules_local() {
+ if [ -n "$MODPROBE" ]; then
+ # use modprobe
+ echo "Using modprobe to load modules"
+ return 0
+ fi
+
+ # Create special udev test rules on every node
+ if [ -f $LUSTRE/lustre/conf/99-lustre.rules ]; then {
+ sed -e 's|/usr/sbin/lctl|$LCTL|g' $LUSTRE/lustre/conf/99-lustre.rules > /etc/udev/rules.d/99-lustre-test.rules
+ } else {
+ echo "SUBSYSTEM==\"lustre\", ACTION==\"change\", ENV{PARAM}==\"?*\", RUN+=\"$LCTL set_param '\$env{PARAM}=\$env{SETTING}'\"" > /etc/udev/rules.d/99-lustre-test.rules
+ } fi
+ udevadm control --reload-rules
+ udevadm trigger
+
+ load_lnet
+
load_module obdclass/obdclass
if ! client_only; then
MODOPTS_PTLRPC=${MODOPTS_PTLRPC:-"lbug_on_grant_miscount=1"}
mount_facets () {
local facets=${1:-$(get_facets)}
local facet
+ local -a mountpids
+ local total=0
+ local ret=0
for facet in ${facets//,/ }; do
- mount_facet $facet
+ mount_facet $facet &
+ mountpids[total]=$!
+ total=$((total+1))
+ done
+ for ((index=0; index<$total; index++)); do
+ wait ${mountpids[index]}
local RC=$?
[ $RC -eq 0 ] && continue
else
error "Restart of $facet failed!"
fi
- return $RC
+ ret=$RC
done
+ return $ret
}
#
(( MDSCOUNT == 1 )) && return
# wait so that statfs data on MDT expire
- local delay=$(do_facet $SINGLEMDS lctl \
- get_param -n osp.*MDT0000*MDT0001.maxage)
+ local delay=$(do_facet mds1 $LCTL \
+ get_param -n osp.*MDT*MDT0000.maxage | sort -n | tail -1)
+
+ [ -n "$delay" ] || error "fail to get maxage"
sleep $delay
local nodes=$(comma_list $(mdts_nodes))
# initiate statfs RPC, all to all MDTs
skip=0
#check whether facet has been included in other affected facets
for ((index=0; index<$total; index++)); do
- [[ *,$facet,* == ,${affecteds[index]}, ]] && skip=1
+ [[ ,${affecteds[index]}, == *,$facet,* ]] && skip=1
done
if [ $skip -eq 0 ]; then
shutdown_facet $facet
done
- $E2FSCK_ON_MDT0 && (run_e2fsck $(facet_active_host $SINGLEMDS) \
- $(mdsdevname 1) "-n" || error "Running e2fsck")
+ echo "$(date +'%H:%M:%S (%s)') shut down"
- for ((index=0; index<$total; index++)); do
- facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
- echo reboot facets: ${affecteds[index]}
+ local hostlist
+ local waithostlist
+
+ for facet in ${facets//,/ }; do
+ local host=$(facet_active_host $facet)
+
+ hostlist=$(expand_list $hostlist $host)
+ if [ $(facet_host $facet) = \
+ $(facet_failover_host $facet) ]; then
+ waithostlist=$(expand_list $waithostlist $host)
+ fi
+ done
- reboot_facet $facet $sleep_time
+ if [ "$FAILURE_MODE" = HARD ]; then
+ for host in ${hostlist//,/ }; do
+ reboot_node $host
+ done
+ echo "$(date +'%H:%M:%S (%s)') $hostlist rebooted"
+ # We need to wait the rebooted hosts in case if
+ # facet_HOST == facetfailover_HOST
+ if ! [ -z "$waithostlist" ]; then
+ wait_for_host $waithostlist
+ if $LOAD_MODULES_REMOTE; then
+ echo "loading modules on $waithostlist"
+ do_rpc_nodes $waithostlist load_modules_local
+ fi
+ fi
+ else
+ sleep 10
+ fi
- change_active ${affecteds[index]}
+ if [[ " ${affecteds[@]} " =~ " $SINGLEMDS " ]]; then
+ change_active $SINGLEMDS
+ fi
+
+ $E2FSCK_ON_MDT0 && (run_e2fsck $(facet_active_host $SINGLEMDS) \
+ $(facet_device $SINGLEMDS) "-n" || error "Running e2fsck")
+
+ local -a mountpids
- wait_for_facet ${affecteds[index]}
+ for ((index=0; index<$total; index++)); do
+ if [[ ${affecteds[index]} != $SINGLEMDS ]]; then
+ change_active ${affecteds[index]}
+ fi
if $GSS_SK; then
init_gss
init_facets_vars_simple
affecteds[index]=$(exclude_items_from_list \
${affecteds[index]} mgs)
fi
- # FIXME; has to be changed to mount all facets concurrently
if [ -n "${affecteds[index]}" ]; then
echo mount facets: ${affecteds[index]}
- mount_facets ${affecteds[index]}
+ mount_facets ${affecteds[index]} &
+ mountpids[index]=$!
+ fi
+ done
+ for ((index=0; index<$total; index++)); do
+ if [ -n "${affecteds[index]}" ]; then
+ wait ${mountpids[index]}
fi
+
if $GSS_SK; then
do_nodes $(comma_list $(all_nodes)) \
"keyctl show | grep lustre | cut -c1-11 |
xargs -IX keyctl setperm X 0x3f3f3f3f"
fi
done
+ echo "$(date +'%H:%M:%S (%s)') targets are mounted"
+
+ if [ "$FAILURE_MODE" = HARD ]; then
+ hostlist=$(exclude_items_from_list $hostlist $waithostlist)
+ if ! [ -z "$hostlist" ]; then
+ wait_for_host $hostlist
+ if $LOAD_MODULES_REMOTE; then
+ echo "loading modules on $hostlist"
+ do_rpc_nodes $hostlist load_modules_local
+ fi
+ fi
+ fi
+
+ echo "$(date +'%H:%M:%S (%s)') facet_failover done"
}
replay_barrier() {
return ${PIPESTATUS[0]}
}
+##
+# Execute exact command line on host
+#
+# The \a host may be on a local or remote node, which is determined at
+# the time the command is run. Does careful argument quotation to
+# ensure that the exact command line is executed without any globbing,
+# substitution, or shell interpretation on the remote side. Does not
+# support --verbose or --quiet. Does not include "$host: " prefixes on
+# output. See also do_facet_vp().
+#
+# usage: do_node_vp "$host" "$command" "$arg"...
+do_node_vp() {
+ local host="$1"
+ shift
+
+ if [[ "$host" == "$HOSTNAME" ]]; then
+ sh -c "$(printf -- ' %q' "$@")"
+ return $?
+ fi
+
+ if [[ "${PDSH}" != *pdsh* || "${PDSH}" != *-S* ]]; then
+ echo "cannot run '$*' on host '${host}' with PDSH='${PDSH}'" >&2
+ return 128
+ fi
+
+ # -N Disable hostname: prefix on lines of output.
+
+ $PDSH "${host}" -N "cd $RPWD; PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; export LUSTRE=$RLUSTRE; $(printf -- ' %q' "$@")"
+}
+
single_local_node () {
[ "$1" = "$HOSTNAME" ]
}
do_node $verbose $quiet $host "$@"
}
+##
+# Execute exact command line on the host of a facet
+#
+# The \a facet (service) may be on a local or remote node, which is
+# determined at the time the command is run. Does careful argument
+# quotation to ensure that the exact command line is executed without
+# any globbing, substitution, or shell interpretation on the remote
+# side. Does not support --verbose or --quiet. Does not include
+# "$host: " prefixes on output.
+#
+# usage: do_facet_vp "$facet" "$command" "$arg"...
+do_facet_vp() {
+ local facet="$1"
+ local host=$(facet_active_host "$facet")
+ shift
+
+ if [[ -z "$host" ]]; then
+ echo "no host defined for facet ${facet}" >&2
+ exit 1
+ fi
+
+ do_node_vp "$host" "$@"
+}
+
# Function: do_facet_random_file $FACET $FILE $SIZE
# Creates FILE with random content on the given FACET of given SIZE
opts+=${LDLM_TIMEOUT:+" --param=sys.ldlm_timeout=$LDLM_TIMEOUT"}
if [ $type == MDS ]; then
- opts+=${MDSCAPA:+" --param-mdt.capa=$MDSCAPA"}
opts+=${DEF_STRIPE_SIZE:+" --param=lov.stripesize=$DEF_STRIPE_SIZE"}
opts+=${DEF_STRIPE_COUNT:+" --param=lov.stripecount=$DEF_STRIPE_COUNT"}
opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"}
if [ $fstype == ldiskfs ]; then
- fs_mkfs_opts+="-O ea_inode,large_dir"
-
var=${facet}_JRN
if [ -n "${!var}" ]; then
fs_mkfs_opts+=" -J device=${!var}"
fi
if [ $type == OST ]; then
- opts+=${OSSCAPA:+" --param=ost.capa=$OSSCAPA"}
-
if [ $fstype == ldiskfs ]; then
var=${facet}_JRN
if [ -n "${!var}" ]; then
exit 0
}
+#
+# For interop testing treate EOPNOTSUPP as success
+# and skip
+#
+skip_eopnotsupp() {
+ local retstr=$@
+
+ echo $retstr | awk -F'|' '{print $1}' |
+ grep -E unsupported\|"(Operation not supported)"
+ (( $? == 0 )) || error "$retstr"
+ skip $retstr
+}
+
+# Add a list of tests to ALWAYS_EXCEPT due to an issue.
+# Usage: always_except LU-4815 23 42q ...
+#
+function always_except() {
+ local issue="${1:-}" # single jira style issue ("LU-4815")
+ local test_num
+
+ shift
+
+ if ! [[ "$issue" =~ ^[[:upper:]]+-[[:digit:]]+$ ]]; then
+ error "always_except: invalid issue '$issue' for tests '$*'"
+ fi
+
+ for test_num in "$@"; do
+ ALWAYS_EXCEPT+=" $test_num"
+ done
+}
+
build_test_filter() {
EXCEPT="$EXCEPT $(testslist_filter)"
# Sanity check: exclude the dup entries
RCLIENTS=$(for i in ${rclients//,/ }; do echo $i; done | sort -u)
- clients="$SINGLECLIENT $HOSTNAME $RCLIENTS"
+ export CLIENT1=${CLIENT1:-$HOSTNAME}
+ export SINGLECLIENT=$CLIENT1
+
+ clients="$SINGLECLIENT $HOSTNAME $RCLIENTS"
# Sanity check: exclude the dup entries from CLIENTS
# for those configs which has SINGLCLIENT set to local client
clients=$(for i in $clients; do echo $i; done | sort -u)
- CLIENTS=$(comma_list $clients)
+ export CLIENTS=$(comma_list $clients)
local -a remoteclients=($RCLIENTS)
for ((i=0; $i<${#remoteclients[@]}; i++)); do
varname=CLIENT$((i + 2))
- eval $varname=${remoteclients[i]}
+ eval export $varname=${remoteclients[i]}
done
- CLIENTCOUNT=$((${#remoteclients[@]} + 1))
+ export CLIENTCOUNT=$((${#remoteclients[@]} + 1))
}
get_random_entry () {
$LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum
}
+free_min_max () {
+ wait_delete_completed
+ AVAIL=($(lctl get_param -n osc.*[oO][sS][cC]-[^M]*.kbytesavail))
+ echo "OST kbytes available: ${AVAIL[*]}"
+ MAXV=${AVAIL[0]}
+ MAXI=0
+ MINV=${AVAIL[0]}
+ MINI=0
+ for ((i = 0; i < ${#AVAIL[@]}; i++)); do
+ #echo OST $i: ${AVAIL[i]}kb
+ if [[ ${AVAIL[i]} -gt $MAXV ]]; then
+ MAXV=${AVAIL[i]}
+ MAXI=$i
+ fi
+ if [[ ${AVAIL[i]} -lt $MINV ]]; then
+ MINV=${AVAIL[i]}
+ MINI=$i
+ fi
+ done
+ echo "Min free space: OST $MINI: $MINV"
+ echo "Max free space: OST $MAXI: $MAXV"
+}
+
# save_lustre_params(comma separated facet list, parameter_mask)
# generate a stream of formatted strings (<facet> <param name>=<param value>)
save_lustre_params() {
# Only check/report network health if get_param isn't reported, since
# *clearly* the network is working if get_param returned something.
if (( $(grep -c catastro $health) != $(wc -w <<< ${nodes//,/ }) )); then
- for node in ${nodes//,/}; do
+ for node in ${nodes//,/ }; do
check_network $node 5
done
fi
local facets="$facet"
if [ "$FAILURE_MODE" = HARD ]; then
- facets=$(facets_on_host $(facet_active_host $facet))
+ facets=$(for f in ${facet//,/ }; do
+ facets_on_host $(facet_active_host $f) | tr "," "\n"
+ done | sort -u | paste -sd , )
fi
for facet in ${facets//,/ }; do
run_llverdev()
{
- local dev=$1
- local llverdev_opts=$2
- local devname=$(basename $1)
- local size=$(grep "$devname"$ /proc/partitions | awk '{print $3}')
- # loop devices aren't in /proc/partitions
- [ "x$size" == "x" ] && local size=$(ls -l $dev | awk '{print $5}')
+ local dev=$1; shift
+ local llverdev_opts="$*"
+ local devname=$(basename $dev)
+ local size=$(awk "/$devname$/ {print \$3}" /proc/partitions)
+ # loop devices aren't in /proc/partitions
+ [[ -z "$size" ]] && size=$(stat -c %s $dev)
- size=$(($size / 1024 / 1024)) # Gb
+ local size_gb=$((size / 1024 / 1024)) # Gb
- local partial_arg=""
- # Run in partial (fast) mode if the size
- # of a partition > 1 GB
- [ $size -gt 1 ] && partial_arg="-p"
+ local partial_arg=""
+ # Run in partial (fast) mode if the size of a partition > 1 GB
+ (( $size == 0 || $size_gb > 1 )) && partial_arg="-p"
- llverdev --force $partial_arg $llverdev_opts $dev
+ llverdev --force $partial_arg $llverdev_opts $dev
}
run_llverfs()
{
- local dir=$1
- local llverfs_opts=$2
- local use_partial_arg=$3
- local partial_arg=""
- local size=$(df -B G $dir |tail -n 1 |awk '{print $2}' |sed 's/G//') #GB
+ local dir=$1
+ local llverfs_opts=$2
+ local use_partial_arg=$3
+ local partial_arg=""
+ local size=$(df -B G $dir |tail -n 1 |awk '{print $2}' |sed 's/G//') #GB
- # Run in partial (fast) mode if the size
- # of a partition > 1 GB
- [ "x$use_partial_arg" != "xno" ] && [ $size -gt 1 ] && partial_arg="-p"
+ # Run in partial (fast) mode if the size of a partition > 1 GB
+ [ "x$use_partial_arg" != "xno" ] && [ $size -gt 1 ] && partial_arg="-p"
- llverfs $partial_arg $llverfs_opts $dir
+ llverfs $partial_arg $llverfs_opts $dir
}
run_sgpdd () {
#
function createmany() {
local count=${!#}
+ local rc
- (( count > 100 )) && {
- local saved_debug=$($LCTL get_param -n debug)
- local list=$(comma_list $(all_nodes))
-
- do_nodes $list $LCTL set_param -n debug=0
- }
+ if (( count > 100 )); then
+ debugsave
+ do_nodes $(comma_list $(all_nodes)) $LCTL set_param -n debug=0
+ fi
$LUSTRE/tests/createmany $*
- local rc=$?
- (( count > 100 )) &&
- do_nodes $list "$LCTL set_param -n debug=\\\"$saved_debug\\\""
+ rc=$?
+ debugrestore
+
return $rc
}
function unlinkmany() {
local count=${!#}
+ local rc
- (( count > 100 )) && {
- local saved_debug=$($LCTL get_param -n debug)
- local list=$(comma_list $(all_nodes))
-
- do_nodes $list $LCTL set_param -n debug=0
- }
+ if (( count > 100 )); then
+ debugsave
+ do_nodes $(comma_list $(all_nodes)) $LCTL set_param -n debug=0
+ fi
$LUSTRE/tests/unlinkmany $*
- local rc=$?
- (( count > 100 )) &&
- do_nodes $list "$LCTL set_param -n debug=\\\"$saved_debug\\\""
+ rc=$?
+ debugrestore
+
return $rc
}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff