if ! echo $PATH | grep -q $LUSTRE/tests; then
export PATH=$PATH:$LUSTRE/tests
fi
- export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mdsrate"}
+ export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mpi/mdsrate"}
[ ! -f "$MDSRATE" ] && export MDSRATE=$(which mdsrate 2> /dev/null)
if ! echo $PATH | grep -q $LUSTRE/tests/racer; then
export PATH=$PATH:$LUSTRE/tests/racer
fi
+ if ! echo $PATH | grep -q $LUSTRE/tests/mpi; then
+ export PATH=$PATH:$LUSTRE/tests/mpi
+ fi
export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
[ ! -f "$LCTL" ] && export LCTL=$(which lctl)
export LFS=${LFS:-"$LUSTRE/utils/lfs"}
mount_facet() {
local facet=$1
shift
- local dev=${facet}_dev
+ local dev=$(facet_active $facet)_dev
local opt=${facet}_opt
echo "Starting ${facet}: ${!opt} $@ ${!dev} ${MOUNT%/*}/${facet}"
do_facet ${facet} mount -t lustre ${!opt} $@ ${!dev} ${MOUNT%/*}/${facet}
shift
eval export ${facet}_dev=${device}
eval export ${facet}_opt=\"$@\"
+
+ local varname=${facet}failover_dev
+ if [ -n "${!varname}" ] ; then
+ eval export ${facet}failover_dev=${!varname}
+ else
+ eval export ${facet}failover_dev=$device
+ fi
+
do_facet ${facet} mkdir -p ${MOUNT%/*}/${facet}
mount_facet ${facet}
RC=$?
}
# save quota version (both administrative and operational quotas)
+# add an additional parameter if mountpoint is ever different from $MOUNT
quota_save_version() {
local fsname=${2:-$FSNAME}
+
+ $LFS quotaoff -ug $MOUNT # just in case
+ [ -n "$1" ] && { $LFS quotacheck -$1 $MOUNT || error "quotacheck has failed"; }
+
do_facet mgs "lctl conf_param ${fsname}-MDT*.mdd.quota_type=$1"
local varsvc
local osts=$(get_facets OST)
if [ ! "$old_QUOTA_TYPE" ] || [ "$quota_type" = "$old_QUOTA_TYPE" ]; then
return
fi
- $LFS quotaoff $mntpt
quota_save_version $old_QUOTA_TYPE
- $LFS quotacheck -ug $mntpt
}
setup_quota(){
if [ "$quota_type" != "$QUOTA_TYPE" ]; then
export old_QUOTA_TYPE=$quota_type
quota_save_version $QUOTA_TYPE
- $LFS quotacheck -ug $mntpt
fi
local quota_usrs=$QUOTA_USERS
local RESULT
local WAIT=0
local sleep=5
- while [ $WAIT -lt $MAX ]; do
+ while [ true ]; do
RESULT=$(do_node $node "$TEST")
if [ "$RESULT" == "$FINAL" ]; then
echo "Updated after $WAIT sec: wanted '$FINAL' got '$RESULT'"
return 0
fi
+ [ $WAIT -ge $MAX ] && break
echo "Waiting $((MAX - WAIT)) secs for update"
WAIT=$((WAIT + sleep))
sleep $sleep
wait_update_facet () {
local facet=$1
- wait_update $(facet_host $facet) $@
+ wait_update $(facet_active_host $facet) "$@"
}
wait_delete_completed () {
do_facet $facet sync
df $MOUNT
local svc=${facet}_svc
- do_facet $facet $LCTL --device %${!svc} readonly
do_facet $facet $LCTL --device %${!svc} notransno
+ do_facet $facet $LCTL --device %${!svc} readonly
do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
$LCTL mark "local REPLAY BARRIER on ${!svc}"
}
do_facet $facet sync
local svc=${facet}_svc
echo Replay barrier on ${!svc}
- do_facet $facet $LCTL --device %${!svc} readonly
do_facet $facet $LCTL --device %${!svc} notransno
+ do_facet $facet $LCTL --device %${!svc} readonly
do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
$LCTL mark "local REPLAY BARRIER on ${!svc}"
}
local facet=$1 echo running=${running}
local svc=${facet}_svc
echo Replay barrier on ${!svc}
- do_facet $facet $LCTL --device %${!svc} readonly
do_facet $facet $LCTL --device %${!svc} notransno
+ do_facet $facet $LCTL --device %${!svc} readonly
do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
$LCTL mark "local REPLAY BARRIER on ${!svc}"
}
change_active() {
local facet=$1
- failover=${facet}failover
+ local failover=${facet}failover
host=`facet_host $failover`
[ -z "$host" ] && return
- curactive=`facet_active $facet`
+ local curactive=`facet_active $facet`
if [ -z "${curactive}" -o "$curactive" == "$failover" ] ; then
eval export ${facet}active=$facet
else
eval export ${facet}active=$failover
fi
# save the active host for this facet
- activevar=${facet}active
+ local activevar=${facet}active
echo "$activevar=${!activevar}" > $TMP/$activevar
}
}
do_facet() {
- facet=$1
+ local facet=$1
shift
- HOST=`facet_active_host $facet`
+ local HOST=`facet_active_host $facet`
[ -z $HOST ] && echo No host defined for facet ${facet} && exit 1
do_node $HOST "$@"
}
if [ -z "${!varname}" ]; then
eval $varname=$(facet_host $facet)
fi
+
+ # ${facet}failover_dev is set in cfg file
+ varname=${facet}failover_dev
+ if [ -n "${!varname}" ] ; then
+ eval export ${facet}failover_dev=${!varname}
+ else
+ eval export ${facet}failover_dev=$device
+ fi
}
init_facets_vars () {
done
}
+mds_sanity_check () {
+ local timeout=$1
+ local period=0
+
+ while [ $period -lt $timeout ]; do
+ count=$(do_facet $SINGLEMDS "lctl dl | grep 'osc.*mdtlov_UUID' | grep ' IN ' 2>/dev/null | wc -l")
+ if [ $count -eq 0 ]; then
+ break
+ fi
+
+ echo "There are $count OST are inactive, wait $period seconds, and try again"
+ sleep 3
+ period=$((period+3))
+ done
+
+ [ $period -lt $timeout ] || log "$count OST are inactive after $timeout seconds, give up"
+}
+
+som_check() {
+ SOM_ENABLED=$(do_facet $SINGLEMDS "$LCTL get_param mdt.*.som" | awk -F= ' {print $2}' | head -n 1)
+ echo $SOM_ENABLED
+}
+
init_param_vars () {
if ! remote_ost_nodsh && ! remote_mds_nodsh; then
export MDSVER=$(do_facet $SINGLEMDS "lctl get_param version" | cut -d. -f1,2)
log "Using TIMEOUT=$TIMEOUT"
+ mds_sanity_check $TIMEOUT
+
+ if [ x"$(som_check)" = x"enabled" ]; then
+ ENABLE_QUOTA=""
+ fi
if [ "$ENABLE_QUOTA" ]; then
setup_quota $MOUNT || return 2
fi
mgshost=$(echo $mgshost | awk -F: '{print $1}')
if [ "$mgshost" != "$myMGS_host" ]; then
- FAIL_ON_ERROR=true \
- error "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE
+ log "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE
Please use correct config or set mds_HOST correctly!"
fi
##################################
# Adaptive Timeouts funcs
-at_is_valid() {
- if [ -z "$AT_MAX_PATH" ]; then
- AT_MAX_PATH=$(do_facet $SINGLEMDS "find /sys/ -name at_max")
- [ -z "$AT_MAX_PATH" ] && echo "missing /sys/.../at_max " && return 1
- fi
- return 0
-}
-
at_is_enabled() {
- at_is_valid || error "invalid call"
-
# only check mds, we assume at_max is the same on all nodes
- local at_max=$(do_facet $SINGLEMDS "cat $AT_MAX_PATH")
+ local at_max=$(do_facet $SINGLEMDS "lctl get_param -n at_max")
if [ $at_max -eq 0 ]; then
return 1
else
at_max_get() {
local facet=$1
- at_is_valid || error "invalid call"
-
# suppose that all ost-s has the same at_max set
if [ $facet == "ost" ]; then
- do_facet ost1 "cat $AT_MAX_PATH"
+ do_facet ost1 "lctl get_param -n at_max"
else
- do_facet $facet "cat $AT_MAX_PATH"
+ do_facet $facet "lctl get_param -n at_max"
fi
}
local at_max=$1
shift
- at_is_valid || error "invalid call"
-
local facet
for facet in $@; do
if [ $facet == "ost" ]; then
for i in `seq $OSTCOUNT`; do
- do_facet ost$i "echo $at_max > $AT_MAX_PATH"
+ do_facet ost$i "lctl set_param at_max=$at_max"
+
done
elif [ $facet == "mds" ]; then
for i in `seq $MDSCOUNT`; do
- do_facet mds$i "echo $at_max > $AT_MAX_PATH"
+ do_facet mds$i "lctl set_param at_max=$at_max"
done
else
- do_facet $facet "echo $at_max > $AT_MAX_PATH"
+ do_facet $facet "lctl set_param at_max=$at_max"
fi
done
}
}
set_nodes_failloc () {
- local nodes=$1
- local node
-
- for node in $nodes ; do
- do_node $node lctl set_param fail_loc=$2
- done
+ do_nodes $(comma_list $1) lctl set_param fail_loc=$2
}
cancel_lru_locks() {
ERRLOG=$TMP/lustre_${TESTSUITE}_${TESTNAME}.$(date +%s)
echo "Dumping lctl log to $ERRLOG"
# We need to dump the logs on all nodes
- local NODES=$(nodes_list)
- for NODE in $NODES; do
- do_node $NODE $LCTL dk $ERRLOG
- done
+ do_nodes $(comma_list $(nodes_list)) $NODE $LCTL dk $ERRLOG
debugrestore
[ "$TESTSUITELOG" ] && echo "$0: ${TYPE}: $TESTNAME $@" >> $TESTSUITELOG
TEST_FAILED=true
MSG=${MSG//\>/\\\>}
MSG=${MSG//\</\\\<}
MSG=${MSG//\//\\\/}
- local NODES=$(nodes_list)
- for NODE in $NODES; do
- do_node $NODE $LCTL mark "$MSG" 2> /dev/null || true
- done
+ do_nodes $(comma_list $(nodes_list)) $LCTL mark "$MSG" 2> /dev/null || true
}
trace() {
}
reset_fail_loc () {
- local myNODES=$(nodes_list)
- local NODE
-
echo -n "Resetting fail_loc on all nodes..."
- for NODE in $myNODES; do
- do_node $NODE "lctl set_param -n fail_loc=0 2>/dev/null || true"
- done
+ do_nodes $(comma_list $(nodes_list)) "lctl set_param -n fail_loc=0 2>/dev/null || true"
echo done.
}
return 0
}
+do_and_time () {
+ local cmd=$1
+ local rc
+
+ SECONDS=0
+ eval '$cmd'
+
+ [ ${PIPESTATUS[0]} -eq 0 ] || rc=1
+
+ echo $SECONDS
+ return $rc
+}
+
inodes_available () {
local IFree=$($LFS df -i $MOUNT | grep ^$FSNAME | awk '{print $4}' | sort -un | head -1) || return 1
echo $IFree
rm -f $file
sleep 1
local iused=$(lfs df -i $dir | grep MDT | awk '{print $3}')
- local oldused=($iused)
+ local -a oldused=($iused)
touch $file
sleep 1
iused=$(lfs df -i $dir | grep MDT | awk '{print $3}')
- local newused=($iused)
+ local -a newused=($iused)
local num=0
for ((i=0; i<${#newused[@]}; i++)); do
do_facet $SINGLEMDS lctl get_param -n mdd.${!var}.stale_export_age > /dev/null 2>&1
}
+########################
+convert_facet2name() {
+ case "$1" in
+ "ost" ) echo "OST0000" ;;
+ "ost1") echo "OST0000" ;;
+ "ost2") echo "OST0001" ;;
+ "ost3") echo "OST0002" ;;
+ "ost4") echo "OST0003" ;;
+ "ost5") echo "OST0004" ;;
+ *) error "unknown facet!" ;;
+ esac
+}
+
+get_clientosc_proc_path() {
+ local ost=$1
+
+ echo "{$1}-osc-*"
+}
+
+get_lustre_version () {
+ local node=${1:-"mds"}
+ do_facet $node $LCTL get_param -n version | awk '/^lustre:/ {print $2}'
+}
+
+get_mds_version_major () {
+ local version=$(get_lustre_version mds)
+ echo $version | awk -F. '{print $1}'
+}
+
+get_mds_version_minor () {
+ local version=$(get_lustre_version mds)
+ echo $version | awk -F. '{print $2}'
+}
+
+get_mdtosc_proc_path() {
+ local ost=$1
+ local major=$(get_mds_version_major)
+ local minor=$(get_mds_version_minor)
+ if [ $major -le 1 -a $minor -le 8 ] ; then
+ echo "${ost}-osc"
+ else
+ echo "${ost}-osc-MDT0000"
+ fi
+}
+
+get_osc_import_name() {
+ local node=$1
+ local ost=$2
+ local name=$(convert_facet2name $ost)
+
+ if [ "$node" == "mds" ]; then
+ get_mdtosc_proc_path $name
+ return 0
+ fi
+
+ get_clientosc_proc_path $name
+ return 0
+}
+
+wait_osc_import_state() {
+ local node=$1
+ local ost_facet=$2
+ local expected=$3
+ local ost=$(get_osc_import_name $node $ost_facet)
+ local CONN_PROC
+ local CONN_STATE
+ local i=0
+
+ CONN_PROC="osc.${FSNAME}-${ost}.ost_server_uuid"
+ CONN_STATE=$(do_facet $node lctl get_param -n $CONN_PROC | cut -f2)
+ while [ "${CONN_STATE}" != "${expected}" ]; do
+ # for disconn we can check after proc entry is removed
+ [ "x${CONN_STATE}" == "x" -a "${expected}" == "DISCONN" ] && return 0
+ # disconnect rpc should be wait not more obd_timeout
+ [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
+ error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1
+ sleep 1
+ CONN_STATE=$(do_facet $node lctl get_param -n $CONN_PROC | cut -f2)
+ i=$(($i + 1))
+ done
+
+ log "${ost_facet} now in ${CONN_STATE} state"
+ return 0
+}