[ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true
rm -f $TMP/*active
-
}
case `uname -r` in
[ -n "$ver" -a "$ver" != "3" ] && error "wrong quota version specifier"
- $LFS quotaoff -ug $MOUNT # just in case
[ -n "$type" ] && { $LFS quotacheck -$type $MOUNT || error "quotacheck has failed"; }
do_facet mgs "lctl conf_param ${fsname}-MDT*.mdd.quota_type=$spec"
local rc=0
for mnt in $mnts ; do
- do_nodes $nodes "set -x; running=\\\$(grep -c $mnt' ' /proc/mounts);
+ do_nodes $nodes "running=\\\$(grep -c $mnt' ' /proc/mounts);
mpts=\\\$(mount | grep -w -c $mnt);
if [ \\\$running -ne \\\$mpts ]; then
echo \\\$(hostname) env are INSANE!;
echo Checking servers environments
# FIXME: modify get_facets to display all facets wo params
- local facets="$(get_facets OST),$(get_facets MDS)"
+ local facets="$(get_facets OST),$(get_facets MDS),mgs"
local node
local mnt
local facet
echo "Starting client $clients: $OPTIONS $device $mnt"
- do_nodes $clients "set -x;
+ do_nodes $clients "
running=\\\$(mount | grep -c $mnt' ');
rc=0;
if [ \\\$running -eq 0 ] ; then
mkdir -p $mnt;
mount -t lustre $OPTIONS $device $mnt;
- rc=$?;
+ rc=\\\$?;
fi;
-exit $rc"
+exit \\\$rc" || return ${PIPESTATUS[0]}
echo "Started clients $clients: "
do_nodes $clients "mount | grep -w $mnt"
[ "$3" ] && force=-f
echo "Stopping clients: $clients $mnt (opts:$force)"
- do_nodes $clients "set -x; running=\\\$(grep -c $mnt' ' /proc/mounts);
+ do_nodes $clients "running=\\\$(grep -c $mnt' ' /proc/mounts);
if [ \\\$running -ne 0 ] ; then
echo Stopping client \\\$(hostname) client $mnt opts:$force;
lsof -t $mnt || need_kill=no;
local node=$1
if [ "$FAILURE_MODE" = HARD ]; then
$POWER_UP $node
+ wait_for_host $node
fi
}
shift
local progs=$@
- do_nodes $clients "set -x ; PATH=:$PATH status=true; for prog in $progs; do
- which \\\$prog || { echo \\\$prog missing on \\\$(hostname) && status=false; }
- done;
- eval \\\$status"
+ do_nodes $clients "PATH=:$PATH; status=true;
+for prog in $progs; do
+ if ! [ \\\"\\\$(which \\\$prog)\\\" -o \\\"\\\${!prog}\\\" ]; then
+ echo \\\$prog missing on \\\$(hostname);
+ status=false;
+ fi
+done;
+eval \\\$status"
}
client_var_name() {
check_client_load $client
rc=${PIPESTATUS[0]}
if [ "$rc" != 0 -a "$expectedfail" ]; then
- start_client_load $client
- echo "Restarted client load: on $client. Checking ..."
- check_client_load $client
+ local var=$(client_var_name $client)_load
+ start_client_load $client ${!var}
+ echo "Restarted client load ${!var}: on $client. Checking ..."
+ check_client_load $client
rc=${PIPESTATUS[0]}
if [ "$rc" != 0 ]; then
log "Client load failed to restart on node $client, rc=$rc"
wait_for_host() {
local host=$1
check_network "$host" 900
- while ! do_node $host "ls -d $LUSTRE " > /dev/null; do sleep 5; done
+ while ! do_node $host hostname > /dev/null; do sleep 5; done
}
wait_for() {
shutdown_facet $facet
[ -n "$sleep_time" ] && sleep $sleep_time
reboot_facet $facet
- client_df &
- DFPID=$!
- echo "df pid is $DFPID"
change_active $facet
local TO=`facet_active_host $facet`
echo "Failover $facet to $TO"
fail() {
facet_failover $* || error "failover: $?"
- df $MOUNT || error "post-failover df: $?"
+ client_df || error "post-failover df: $?"
}
fail_nodf() {
stop $facet
change_active $facet
mount_facet $facet -o abort_recovery
- df $MOUNT || echo "first df failed: $?"
+ client_df || echo "first df failed: $?"
sleep 1
- df $MOUNT || error "post-failover df: $?"
+ client_df || error "post-failover df: $?"
}
do_lmc() {
}
do_node() {
- HOST=$1
+ local verbose=false
+ # do not stripe off hostname if verbose, bug 19215
+ if [ x$1 = x--verbose ]; then
+ shift
+ verbose=true
+ fi
+
+ local HOST=$1
shift
local myPDSH=$PDSH
if [ "$HOST" = "$HOSTNAME" ]; then
[ -n "$($myPDSH $HOST cat $command_status)" ] && return 1 || true
return 0
fi
- $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; sh -c \"$@\")" | sed "s/^${HOST}: //"
+
+ if $verbose ; then
+ # print HOSTNAME for myPDSH="no_dsh"
+ if [[ $myPDSH = no_dsh ]]; then
+ $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; sh -c \"$@\")" | sed -e "s/^/${HOSTNAME}: /"
+ else
+ $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; sh -c \"$@\")"
+ fi
+ else
+ $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; sh -c \"$@\")" | sed "s/^${HOST}: //"
+ fi
return ${PIPESTATUS[0]}
}
}
do_nodes() {
+ local verbose=false
+ # do not stripe off hostname if verbose, bug 19215
+ if [ x$1 = x--verbose ]; then
+ shift
+ verbose=true
+ fi
+
local rnodes=$1
shift
if $(single_local_node $rnodes); then
- do_node $rnodes $@
+ if $verbose; then
+ do_node --verbose $rnodes $@
+ else
+ do_node $rnodes $@
+ fi
return $?
fi
$myPDSH $rnodes $LCTL mark "$@" > /dev/null 2>&1 || :
fi
- $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; sh -c \"$@\")" | sed -re "s/\w+:\s//g"
+ if $verbose ; then
+ $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; sh -c \"$@\")"
+ else
+ $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; sh -c \"$@\")" | sed -re "s/\w+:\s//g"
+ fi
return ${PIPESTATUS[0]}
}
}
cleanupall() {
+ nfs_client_mode && return
+
stopall $*
unload_modules
cleanup_gss
# We need ldiskfs here, may as well load them all
load_modules
[ "$CLIENTONLY" ] && return
- echo "Formatting mdts, osts"
+ echo Formatting mgs, mds, osts
+ if [[ $MDSDEV1 != $MGSDEV ]] || [[ $mds1_HOST != $mgs_HOST ]]; then
+ add mgs $mgs_MKFS_OPTS $FSTYPE_OPT --reformat $MGSDEV || exit 10
+ fi
+
for num in `seq $MDSCOUNT`; do
echo "Format mds$num: $(mdsdevname $num)"
if $VERBOSE; then
}
setupall() {
+ nfs_client_mode && return
+
sanity_mount_check ||
error "environments are insane!"
load_modules
init_gss
if [ -z "$CLIENTONLY" ]; then
- echo "Setup mdts, osts"
+ echo Setup mgs, mdt, osts
echo $WRITECONF | grep -q "writeconf" && \
writeconf_all
+ if [[ $mds1_HOST != $mgs_HOST ]] || [[ $MDSDEV1 != $MGSDEV ]]; then
+ start mgs $MGSDEV $mgs_MOUNT_OPTS
+ fi
+
for num in `seq $MDSCOUNT`; do
DEVNAME=$(mdsdevname $num)
start mds$num $DEVNAME $MDS_MOUNT_OPTS
done
}
-mds_sanity_check () {
- local timeout=$1
+osc_ensure_active () {
+ local facet=$1
+ local type=$2
+ local timeout=$3
local period=0
while [ $period -lt $timeout ]; do
- count=$(do_facet $SINGLEMDS "lctl dl | grep 'osc.*mdtlov_UUID' | grep ' IN ' 2>/dev/null | wc -l")
+ count=$(do_facet $facet "lctl dl | grep '${FSNAME}-OST.*-osc-${type}' | grep ' IN ' 2>/dev/null | wc -l")
if [ $count -eq 0 ]; then
break
fi
log "Using TIMEOUT=$TIMEOUT"
- mds_sanity_check $TIMEOUT
+ osc_ensure_active $SINGLEMDS M $TIMEOUT
+ osc_ensure_active client c $TIMEOUT
if [ x"$(som_check)" = x"enabled" ]; then
ENABLE_QUOTA=""
fi
}
+nfs_client_mode () {
+ if [ "$NFSCLIENT" ]; then
+ echo "NFSCLIENT mode: setup, cleanup, check config skipped"
+ local clients=$CLIENTS
+ [ -z $clients ] && clients=$(hostname)
+
+ # FIXME: remove hostname when 19215 fixed
+ do_nodes $clients "echo \\\$(hostname); grep ' '$MOUNT' ' /proc/mounts"
+ declare -a nfsexport=(`grep ' '$MOUNT' ' /proc/mounts | awk '{print $1}' | awk -F: '{print $1 " " $2}'`)
+ do_nodes ${nfsexport[0]} "echo \\\$(hostname); df -T ${nfsexport[1]}"
+ return
+ fi
+ return 1
+}
+
check_config () {
+ nfs_client_mode && return
+
local mntpt=$1
local mounted=$(mount | grep " $mntpt ")
local mgshost=$(mount | grep " $mntpt " | awk -F@ '{print $1}')
mgshost=$(echo $mgshost | awk -F: '{print $1}')
- if [ "$mgshost" != "$myMGS_host" ]; then
- error_exit "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE
- Please use correct config or set mds_HOST correctly!"
- fi
+# if [ "$mgshost" != "$myMGS_host" ]; then
+# error_exit "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE
+# Please use correct config or set mds_HOST correctly!"
+# fi
sanity_mount_check ||
error "environments are insane!"
}
check_and_setup_lustre() {
+ nfs_client_mode && return
+
local MOUNTED=$(mounted_lustre_filesystems)
if [ -z "$MOUNTED" ] || ! $(echo $MOUNTED | grep -w -q $MOUNT); then
[ "$REFORMAT" ] && formatall
check_config $MOUNT
init_facets_vars
init_param_vars
+
+ do_nodes $(comma_list $(nodes_list)) "lctl set_param debug=\\\"$PTLDEBUG\\\";
+ lctl set_param subsystem_debug=\\\"${SUBSYSTEM# }\\\";
+ lctl set_param debug_mb=${DEBUG_SIZE};
+ sync"
fi
if [ "$ONLY" == "setup" ]; then
exit 0
local WAIT=0
local MAX=$2
while [ $NETWORK -eq 0 ]; do
- ping -c 1 -w 3 $1 > /dev/null
- if [ $? -eq 0 ]; then
+ if ping -c 1 -w 3 $1 > /dev/null; then
NETWORK=1
else
WAIT=$((WAIT + 5))
return 1
fi
done
+ if [[ $MDSDEV1 != $MGSDEV ]]; then
+ stop mgs
+ fi
+
return 0
}
done
}
-_basetest() {
- echo $*
-}
-
basetest() {
- IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
+ echo ${1%%[a-z]*}
}
# print a newline if the last test was skipped
add user $myRUNAS_UID:$myRUNAS_GID on these nodes."
}
+# obtain the UID/GID for MPI_USER
+get_mpiuser_id() {
+ local mpi_user=$1
+
+ MPI_USER_UID=$(do_facet client "getent passwd $mpi_user | cut -d: -f3;
+exit \\\${PIPESTATUS[0]}") || error_exit "failed to get the UID for $mpi_user"
+
+ MPI_USER_GID=$(do_facet client "getent passwd $mpi_user | cut -d: -f4;
+exit \\\${PIPESTATUS[0]}") || error_exit "failed to get the GID for $mpi_user"
+}
+
+# obtain and cache Kerberos ticket-granting ticket
+refresh_krb5_tgt() {
+ local myRUNAS_UID=$1
+ local myRUNAS_GID=$2
+ shift 2
+ local myRUNAS=$@
+ if [ -z "$myRUNAS" ]; then
+ error_exit "myRUNAS command must be specified for refresh_krb5_tgt"
+ fi
+
+ CLIENTS=${CLIENTS:-$HOSTNAME}
+ do_nodes $CLIENTS "set -x
+if ! $myRUNAS krb5_login.sh; then
+ echo "Failed to refresh Krb5 TGT for UID/GID $myRUNAS_UID/$myRUNAS_GID."
+ exit 1
+fi"
+}
+
# Run multiop in the background, but wait for it to print
# "PAUSING" to its stdout before returning from this function.
multiop_bg_pause() {
# generate a stream of formatted strings (<node> <param name>=<param value>)
save_lustre_params() {
local s
- do_node $1 "lctl get_param $2" | while read s; do echo "$1 $s"; done
+ do_nodes --verbose $1 "lctl get_param $2 | while read s; do echo \\\$s; done"
}
# restore lustre parameters from input stream, produces by save_lustre_params
local name
local val
while IFS=" =" read node name val; do
- do_node $node "lctl set_param -n $name $val"
+ do_node ${node//:/} "lctl set_param -n $name $val"
done
}
stripe_size=`awk '$1 ~ /size/ {print $2}' $tmp_file`
stripe_count=`awk '$1 ~ /count/ {print $2}' $tmp_file`
- stripe_index=`awk '/obdidx/ {start = 1; getline; print $1; exit}' $tmp_file`
+ stripe_index=`awk '$1 ~ /stripe_offset/ {print $2}' $tmp_file`
rm -f $tmp_file
}
error "mdt-s : inodes count OLD ${oldused[@]} NEW ${newused[@]}"
}
-mpi_run () {
- local mpirun="$MPIRUN $MPIRUN_OPTIONS"
- local command="$mpirun $@"
- local mpilog=$TMP/mpi.log
- local rc
-
- if [ "$MPI_USER" != root -a $mpirun ]; then
- echo "+ chmod 0777 $MOUNT"
- chmod 0777 $MOUNT
- command="su $MPI_USER sh -c \"$command \""
- fi
-
- ls -ald $MOUNT
- echo "+ $command"
- eval $command 2>&1 > $mpilog || true
-
- rc=${PIPESTATUS[0]}
- if [ $rc -eq 0 ] && grep -q "p4_error: : [^0]" $mpilog ; then
- rc=1
- fi
- cat $mpilog
- return $rc
-}
-
mdsrate_cleanup () {
mpi_run -np $1 -machinefile $2 ${MDSRATE} --unlink --nfiles $3 --dir $4 --filefmt $5 $6
}
CONN_PROC="osc.${FSNAME}-${ost}.ost_server_uuid"
CONN_STATE=$(do_facet $node lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
while [ "${CONN_STATE}" != "${expected}" ]; do
- # for disconn we can check after proc entry is removed
- [ "x${CONN_STATE}" == "x" -a "${expected}" == "DISCONN" ] && return 0
+ if [ "${expected}" == "DISCONN" ]; then
+ # for disconn we can check after proc entry is removed
+ [ "x${CONN_STATE}" == "x" ] && return 0
+ # with AT we can have connect request timeout ~ reconnect timeout
+ # and test can't see real disconnect
+ [ "${CONN_STATE}" == "CONNECTING" ] && return 0
+ fi
# disconnect rpc should be wait not more obd_timeout
[ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1
git://git.whamcloud.com / fs / lustre-release.git / blobdiff