b=19129

[fs/lustre-release.git] / lustre / tests / test-framework.sh

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index ed2eb76..c8b801b 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -634,6 +634,62 @@ zconf_umount() {
     fi
 }
 
+# nodes is comma list
+sanity_mount_check_nodes () {
+    local nodes=$1
+    shift
+    local mnts="$@"
+    local mnt
+
+    # FIXME: assume that all cluster nodes run the same os
+    [ "$(uname)" = Linux ] || return 0
+
+    local rc=0
+    for mnt in $mnts ; do
+        do_nodes $nodes "set -x; running=\\\$(grep -c $mnt' ' /proc/mounts);
+mpts=\\\$(mount | grep -w -c $mnt);
+if [ \\\$running -ne \\\$mpts ]; then
+    echo \\\$(hostname) env are INSANE!;
+    exit 1;
+fi"
+    [ $? -eq 0 ] || rc=1 
+    done
+    return $rc
+}
+
+sanity_mount_check_servers () {
+    echo Checking servers environments
+
+    # FIXME: modify get_facets to display all facets wo params
+    local facets="$(get_facets OST),$(get_facets MDS)"
+    local node
+    local mnt
+    local facet
+    for facet in ${facets//,/ }; do
+        node=$(facet_host ${facet})
+        mnt=${MOUNT%/*}/${facet}
+        sanity_mount_check_nodes $node $mnt ||
+            { error "server $node environments are insane!"; return 1; }
+    done
+}
+
+sanity_mount_check_clients () {
+    local clients=${1:-$CLIENTS}
+    local mntpt=${2:-$MOUNT}
+    local mntpt2=${3:-$MOUNT2}
+
+    [ -z $clients ] && clients=$(hostname)
+    echo Checking clients $clients environments
+
+    sanity_mount_check_nodes $clients $mntpt $mntpt2 ||
+       error "clients environments are insane!"
+}
+
+sanity_mount_check () {
+    sanity_mount_check_servers || return 1
+    sanity_mount_check_clients || return 2
+}
+
 # mount clients if not mouted
 zconf_mount_clients() {
     local OPTIONS
@@ -652,10 +708,19 @@ zconf_mount_clients() {
     fi
 
     echo "Starting client $clients: $OPTIONS $device $mnt"
-    do_nodes $clients "mount | grep $mnt || { mkdir -p $mnt && mount -t lustre $OPTIONS $device $mnt || false; }"
+
+    do_nodes $clients "set -x;
+running=\\\$(mount | grep -c $mnt' ');
+rc=0;
+if [ \\\$running -eq 0 ] ; then
+    mkdir -p $mnt;
+    mount -t lustre $OPTIONS $device $mnt;
+    rc=$?;
+fi;
+exit $rc"
 
     echo "Started clients $clients: "
-    do_nodes $clients "mount | grep $mnt"
+    do_nodes $clients "mount | grep -w $mnt"
 
     do_nodes $clients "lctl set_param debug=$PTLDEBUG;
         lctl set_param subsystem_debug=${SUBSYSTEM# };
@@ -672,20 +737,20 @@ zconf_umount_clients() {
     [ "$3" ] && force=-f
 
     echo "Stopping clients: $clients $mnt (opts:$force)"
-    do_nodes $clients "set -x; running=\\\$(grep -c $mnt' ' /proc/mounts)
+    do_nodes $clients "set -x; running=\\\$(grep -c $mnt' ' /proc/mounts);
 if [ \\\$running -ne 0 ] ; then
-echo Stopping client \\\$(hostname) client $mnt opts:$force
-lsof -t $mnt || need_kill=no
+echo Stopping client \\\$(hostname) client $mnt opts:$force;
+lsof -t $mnt || need_kill=no;
 if [ "x$force" != "x" -a "x\\\$need_kill" != "xno" ]; then
     pids=\\\$(lsof -t $mnt | sort -u);
     if [ -n \\\"\\\$pids\\\" ]; then
-             kill -9 \\\$pids
+             kill -9 \\\$pids;
     fi
-fi
-busy=\\\$(umount $force $mnt 2>&1 | grep -c "busy")
+fi;
+busy=\\\$(umount $force $mnt 2>&1 | grep -c "busy");
 if [ \\\$busy -ne 0 ] ; then
-    echo "$mnt is still busy, wait one second" && sleep 1
-    umount $force $mnt
+    echo "$mnt is still busy, wait one second" && sleep 1;
+    umount $force $mnt;
 fi
 fi"
 }
@@ -1000,6 +1065,7 @@ wait_remote_prog () {
     local pids=$(ps  uax | grep "$PDSH.*$prog.*$MOUNT" | grep -v grep | awk '{print $2}')
     [ -z "$pids" ] && return 0
     echo "$PDSH processes still exists after $WAIT seconds.  Still running: $pids"
+    # FIXME: not portable
     for pid in $pids; do
         cat /proc/${pid}/status || true
         cat /proc/${pid}/wchan || true
@@ -1330,14 +1396,11 @@ stopall() {
         fail mds1
     fi
 
-    # assume client mount is local
-    grep " $MOUNT " /proc/mounts && zconf_umount $HOSTNAME $MOUNT $*
-    grep " $MOUNT2 " /proc/mounts && zconf_umount $HOSTNAME $MOUNT2 $*
+    local clients=$CLIENTS
+    [ -z $clients ] && clients=$(hostname)
 
-    if [ -n "$CLIENTS" ]; then
-            zconf_umount_clients $CLIENTS $MOUNT "$*" || true
-            [ -n "$MOUNT2" ] && zconf_umount_clients $CLIENTS $MOUNT2 "$*" || true
-    fi
+    zconf_umount_clients $clients $MOUNT "$*" || true
+    [ -n "$MOUNT2" ] && zconf_umount_clients $clients $MOUNT2 "$*" || true
 
     [ "$CLIENTONLY" ] && return
     # The add fn does rm ${facet}active file, this would be enough
@@ -1471,6 +1534,9 @@ writeconf_all () {
 }
 
 setupall() {
+    sanity_mount_check ||
+        error "environments are insane!"
+
     load_modules
     init_gss
     if [ -z "$CLIENTONLY" ]; then
@@ -1609,6 +1675,9 @@ check_config () {
             error "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE
                    Please use correct config or set mds_HOST correctly!"
     fi
+
+    sanity_mount_check ||
+        error "environments are insane!"
 }
 
 check_timeout () {
@@ -2546,6 +2615,8 @@ get_mds_dir () {
 mpi_run () {
     local mpirun="$MPIRUN $MPIRUN_OPTIONS"
     local command="$mpirun $@"
+    local mpilog=$TMP/mpi.log
+    local rc
 
     if [ "$MPI_USER" != root -a $mpirun ]; then
         echo "+ chmod 0777 $MOUNT"
@@ -2555,10 +2626,22 @@ mpi_run () {
 
     ls -ald $MOUNT
     echo "+ $command"
-    eval $command
+    eval $command 2>&1 > $mpilog || true
+
+    rc=${PIPESTATUS[0]}
+    if [ $rc -eq 0 ] && grep -q "p4_error: : [^0]" $mpilog ; then
+       rc=1
+    fi
+    cat $mpilog
+    return $rc
 }
 
 mdsrate_cleanup () {
     mpi_run -np $1 -machinefile $2 ${MDSRATE} --unlink --nfiles $3 --dir $4 --filefmt $5 $6
 }
 
+delayed_recovery_enabled () {
+    local var=${SINGLEMDS}_svc
+    do_facet $SINGLEMDS lctl get_param -n mdd.${!var}.stale_export_age > /dev/null 2>&1
+}
+