b=24360 fix NULL pointer deref in mds_verify_child() when ll_lookup_one_len() fails

[fs/lustre-release.git] / lustre / tests / replay-vbr.sh

diff --git a/lustre/tests/replay-vbr.sh b/lustre/tests/replay-vbr.sh
index 836b438..3cd1511 100644 (file)
--- a/lustre/tests/replay-vbr.sh
+++ b/lustre/tests/replay-vbr.sh
@@ -2,8 +2,8 @@
 
 set -e
 
-# bug number:
-ALWAYS_EXCEPT="3c 4b 4c 10 $REPLAY_VBR_EXCEPT"
+# bug number: 16356
+ALWAYS_EXCEPT="2 3c 4b 4c 10 $REPLAY_VBR_EXCEPT"
 
 SAVE_PWD=$PWD
 PTLDEBUG=${PTLDEBUG:--1}
@@ -16,9 +16,9 @@ init_test_env $@
 
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
-[ -n "$CLIENTS" ] || { skip "Need two or more clients" && exit 0; }
+[ -n "$CLIENTS" ] || { skip_env "Need two or more clients" && exit 0; }
 [ $CLIENTCOUNT -ge 2 ] || \
-    { skip "Need two or more clients, have $CLIENTCOUNT" && exit 0; }
+    { skip_env "Need two or more remote clients, have $CLIENTCOUNT" && exit 0; }
 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
 
 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
@@ -84,6 +84,12 @@ get_version() {
     do_facet mds $LCTL --device $mds_svc getobjversion $fid
 }
 
+# interop 18 <-> 20
+lustre_version=$(get_lustre_version mds)
+if [[ $lustre_version != 1.8* ]]; then
+    mds20="yes"
+fi
+
 test_0a() {
     local file=$DIR/$tfile
     local pre
@@ -100,7 +106,7 @@ test_0a() {
 run_test 0a "VBR: open and close do not change versions"
 
 test_0b() {
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     do_node $CLIENT1 mkdir -p -m 755 $DIR/$tdir
 
     replay_barrier mds
@@ -109,7 +115,7 @@ test_0b() {
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT && error "$CLIENT1 not evicted"
+    client_evicted $CLIENT1 || error "$CLIENT1 not evicted"
     if ! do_node $CLIENT1 $CHECKSTAT -a $DIR/$tdir/$tfile; then
         error "open succeeded unexpectedly"
     fi
@@ -118,7 +124,7 @@ test_0b() {
 run_test 0b "VBR: open (O_CREAT) checks version of parent"
 
 test_0c() {
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     do_node $CLIENT1 mkdir -p -m 755 $DIR/$tdir
     do_node $CLIENT1 openfile -f O_RDWR:O_CREAT -m 0644 $DIR/$tdir/$tfile
 
@@ -128,8 +134,8 @@ test_0c() {
     rmultiop_start $CLIENT1 $DIR/$tdir/$tfile o_c
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
+    client_up $CLIENT1 || error "$CLIENT1 evicted"
 
-    do_node $CLIENT1 df $MOUNT || error "$CLIENT1 evicted"
     rmultiop_stop $CLIENT1 || error "close failed"
     zconf_mount $CLIENT2 $MOUNT
 }
@@ -143,13 +149,13 @@ test_0d() {
     do_node $CLIENT1 mkfifo $DIR/$tfile
     post=$(get_version $CLIENT1 $DIR)
     if (($pre == $post)); then
-        error "version not changed: pre $pre, post $post"
+        [ -n "$mds20" ] || error "version not changed: pre $pre, post $post"
     fi
 }
 run_test 0d "VBR: create changes version of parent"
 
 test_0e() {
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     do_node $CLIENT1 mkdir -p -m 755 $DIR/$tdir
 
     replay_barrier mds
@@ -158,7 +164,7 @@ test_0e() {
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT && error "$CLIENT1 not evicted"
+    client_evicted $CLIENT1 || error "$CLIENT1 not evicted"
     if ! do_node $CLIENT1 $CHECKSTAT -a $DIR/$tdir/$tfile; then
         error "create succeeded unexpectedly"
     fi
@@ -175,13 +181,13 @@ test_0f() {
     do_node $CLIENT1 rm $DIR/$tfile
     post=$(get_version $CLIENT1 $DIR)
     if (($pre == $post)); then
-        error "version not changed: pre $pre, post $post"
+        [ -n "$mds20" ] || error "version not changed: pre $pre, post $post"
     fi
 }
 run_test 0f "VBR: unlink changes version of parent"
 
 test_0g() {
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     do_node $CLIENT1 mkdir -p -m 755 $DIR/$tdir
     do_node $CLIENT1 mcreate $DIR/$tdir/$tfile
 
@@ -191,7 +197,7 @@ test_0g() {
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT && error "$CLIENT1 not evicted"
+    client_evicted $CLIENT1 || error "$CLIENT1 not evicted"
     if do_node $CLIENT1 $CHECKSTAT -a $DIR/$tdir/$tfile; then
         error "unlink succeeded unexpectedly"
     fi
@@ -206,7 +212,7 @@ test_0h() {
 
     do_node $CLIENT1 mcreate $file
     pre=$(get_version $CLIENT1 $file)
-    do_node $CLIENT1 chown $RUNAS_ID $file
+    do_node $CLIENT1 chown $RUNAS_ID:$RUNAS_GID $file
     post=$(get_version $CLIENT1 $file)
     if (($pre == $post)); then
         error "version not changed: pre $pre, post $post"
@@ -221,7 +227,7 @@ test_0i() {
 
     do_node $CLIENT1 mcreate $file
     pre=$(get_version $CLIENT1 $file)
-    do_node $CLIENT1 chown :$RUNAS_ID $file
+    do_node $CLIENT1 chgrp $RUNAS_GID $file
     post=$(get_version $CLIENT1 $file)
     if (($pre == $post)); then
         error "version not changed: pre $pre, post $post"
@@ -232,16 +238,16 @@ run_test 0i "VBR: setattr of GID changes versions"
 test_0j() {
     local file=$DIR/$tfile
 
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     do_node $CLIENT1 mcreate $file
 
     replay_barrier mds
-    do_node $CLIENT2 chown :$RUNAS_ID $file
-    do_node $CLIENT1 chown $RUNAS_ID $file
+    do_node $CLIENT2 chgrp $RUNAS_GID $file
+    do_node $CLIENT1 chown $RUNAS_ID:$RUNAS_GID $file
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT && error "$CLIENT1 not evicted"
+    client_evicted $CLIENT1 || error "$CLIENT1 not evicted"
     if ! do_node $CLIENT1 $CHECKSTAT -u \\\#$UID $file; then
         error "setattr of UID succeeded unexpectedly"
     fi
@@ -252,16 +258,16 @@ run_test 0j "VBR: setattr of UID checks versions"
 test_0k() {
     local file=$DIR/$tfile
 
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     do_node $CLIENT1 mcreate $file
 
     replay_barrier mds
-    do_node $CLIENT2 chown $RUNAS_ID $file
-    do_node $CLIENT1 chown :$RUNAS_ID $file
+    do_node $CLIENT2 chown $RUNAS_ID:$RUNAS_GID $file
+    do_node $CLIENT1 chgrp $RUNAS_GID $file
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT && error "$CLIENT1 not evicted"
+    client_evicted $CLIENT1 || error "$CLIENT1 not evicted"
     if ! do_node $CLIENT1 $CHECKSTAT -g \\\#$UID $file; then
         error "setattr of GID succeeded unexpectedly"
     fi
@@ -287,16 +293,16 @@ run_test 0l "VBR: setattr of permission changes versions"
 test_0m() {
     local file=$DIR/$tfile
 
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     do_node $CLIENT1 openfile -f O_RDWR:O_CREAT -m 0644 $file
 
     replay_barrier mds
-    do_node $CLIENT2 chown :$RUNAS_ID $file
+    do_node $CLIENT2 chgrp $RUNAS_GID $file
     do_node $CLIENT1 chmod 666 $file
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT && error "$CLIENT1 not evicted"
+    client_evicted $CLIENT1 || error "$CLIENT1 not evicted"
     if ! do_node $CLIENT1 $CHECKSTAT -p 0644 $file; then
         error "setattr of permission succeeded unexpectedly"
     fi
@@ -336,7 +342,7 @@ test_0o() {
     local file=$DIR/$tfile
     local rc
 
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     do_node $CLIENT1 openfile -f O_RDWR:O_CREAT -m 0644 $file
 
     replay_barrier mds
@@ -345,7 +351,7 @@ test_0o() {
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT && error "$CLIENT1 not evicted"
+    client_evicted $CLIENT1 || error "$CLIENT1 not evicted"
     checkattr $CLIENT1 i $file
     rc=$?
     do_node $CLIENT1 chattr -i $file
@@ -362,8 +368,8 @@ test_0p() {
     local post
     local ad_orig
 
-    ad_orig=$(do_facet mds "$LCTL get_param mds.${mds_svc}.atime_diff")
-    do_facet mds "$LCTL set_param mds.${mds_svc}.atime_diff=0"
+    ad_orig=$(do_facet mds "$LCTL get_param *.${mds_svc}.atime_diff")
+    do_facet mds "$LCTL set_param *.${mds_svc}.atime_diff=0"
     do_node $CLIENT1 mcreate $file
     pre=$(get_version $CLIENT1 $file)
     do_node $CLIENT1 touch $file
@@ -386,7 +392,7 @@ test_0q() {
 
     do_node $CLIENT1 mcreate $file
     pre=$(get_version $CLIENT1 $file)
-    do_node $CLIENT1 truncate $file 1
+    do_node $CLIENT1 $TRUNCATE $file 1
     post=$(get_version $CLIENT1 $file)
     if (($pre != $post)); then
         error "version changed unexpectedly: pre $pre, post $post"
@@ -400,13 +406,13 @@ test_0r() {
     local mtime_post
     local mtime
 
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
-    do_facet mds "$LCTL set_param mds.${mds_svc}.atime_diff=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.atime_diff=0"
     do_node $CLIENT1 openfile -f O_RDWR:O_CREAT -m 0644 $file
 
     replay_barrier mds
     do_node $CLIENT2 chmod 666 $file
-    do_node $CLIENT1 truncate $file 1
+    do_node $CLIENT1 $TRUNCATE $file 1
     sleep 1
     mtime_pre=$(do_node $CLIENT1 stat --format=%Y $file)
     do_node $CLIENT1 touch $file
@@ -414,7 +420,7 @@ test_0r() {
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT || error "$CLIENT1 evicted"
+    client_up $CLIENT1 || error "$CLIENT1 evicted"
     if (($mtime_pre >= $mtime_post)); then
         error "time not changed: pre $mtime_pre, post $mtime_post"
     fi
@@ -446,13 +452,14 @@ test_0s() {
         error "version of source not changed: pre $pre, post $post"
     fi
     if (($tp_pre == $tp_post)); then
-        error "version of target parent not changed: pre $tp_pre, post $tp_post"
+        [ -n "$mds20" ] || \
+            error "version of target parent not changed: pre $tp_pre, post $tp_post"
     fi
 }
 run_test 0s "VBR: link changes versions of source and target parent"
 
 test_0t() {
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     do_node $CLIENT1 mcreate $DIR/$tfile
     do_node $CLIENT1 mkdir -p -m 755 $DIR/$tdir
 
@@ -462,7 +469,7 @@ test_0t() {
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT && error "$CLIENT1 not evicted"
+    client_evicted $CLIENT1 || error "$CLIENT1 not evicted"
     if ! do_node $CLIENT1 $CHECKSTAT -a $DIR/$tdir/$tfile; then
         error "link should fail"
     fi
@@ -471,7 +478,7 @@ test_0t() {
 run_test 0t "VBR: link checks version of target parent"
 
 test_0u() {
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     do_node $CLIENT1 openfile -f O_RDWR:O_CREAT -m 0644 $DIR/$tfile
     do_node $CLIENT1 mkdir -p $DIR/$tdir
 
@@ -481,7 +488,7 @@ test_0u() {
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT && error "$CLIENT1 not evicted"
+    client_evicted $CLIENT1 || error "$CLIENT1 not evicted"
     if ! do_node $CLIENT1 $CHECKSTAT -a $DIR/$tdir/$tfile; then
         error "link should fail"
     fi
@@ -503,10 +510,12 @@ test_0v() {
     sp_post=$(get_version $CLIENT1 $DIR)
     tp_post=$(get_version $CLIENT1 $DIR/$tdir)
     if (($sp_pre == $sp_post)); then
-        error "version of source parent not changed: pre $sp_pre, post $sp_post"
+        [ -n "$mds20" ] || \
+            error "version of source parent not changed: pre $sp_pre, post $sp_post"
     fi
     if (($tp_pre == $tp_post)); then
-        error "version of target parent not changed: pre $tp_pre, post $tp_post"
+        [ -n "$mds20" ] || \
+            error "version of target parent not changed: pre $tp_pre, post $tp_post"
     fi
 }
 run_test 0v "VBR: rename changes versions of source parent and target parent"
@@ -520,13 +529,14 @@ test_0w() {
     do_node $CLIENT1 mv $DIR/$tfile $DIR/$tfile-new
     post=$(get_version $CLIENT1 $DIR)
     if (($pre == $post)); then
-        error "version of parent not changed: pre $pre, post $post"
+        [ -n "$mds20" ] || \
+            error "version of parent not changed: pre $pre, post $post"
     fi
 }
 run_test 0w "VBR: rename within same dir changes version of parent"
 
 test_0x() {
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     do_node $CLIENT1 mcreate $DIR/$tfile
     do_node $CLIENT1 mkdir -p -m 755 $DIR/$tdir
 
@@ -536,7 +546,7 @@ test_0x() {
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT && error "$CLIENT1 not evicted"
+    client_evicted $CLIENT1 || error "$CLIENT1 not evicted"
     if do_node $CLIENT1 $CHECKSTAT -a $DIR/$tfile; then
         error "rename should fail"
     fi
@@ -545,7 +555,7 @@ test_0x() {
 run_test 0x "VBR: rename checks version of source parent"
 
 test_0y() {
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     do_node $CLIENT1 mcreate $DIR/$tfile
     do_node $CLIENT1 mkdir -p -m 755 $DIR/$tdir
 
@@ -555,7 +565,7 @@ test_0y() {
     zconf_umount $CLIENT2 $MOUNT
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT && error "$CLIENT1 not evicted"
+    client_evicted $CLIENT1 || error "$CLIENT1 not evicted"
     if do_node $CLIENT1 $CHECKSTAT -a $DIR/$tfile; then
         error "rename should fail"
     fi
@@ -579,7 +589,7 @@ test_1() {
 
     facet_failover mds
     # recovery shouldn't fail due to missing client 2
-    do_node $CLIENT1 df $DIR || return 1
+    client_up $CLIENT1 || return 1
 
     # All 50 files should have been replayed
     do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 2
@@ -610,7 +620,7 @@ test_2a() { # was test_2
 
     facet_failover mds
     # recovery shouldn't fail due to missing client 2
-    do_node $CLIENT1 df $DIR || return 1
+    client_up $CLIENT1 || return 1
 
     # All 50 files should have been replayed
     do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 2
@@ -636,7 +646,7 @@ test_2b() {
     local pre
     local post
 
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0"
     zconf_mount $CLIENT1 $MOUNT
     zconf_mount $CLIENT2 $MOUNT2
     zconf_mount $CLIENT2 $MOUNT1
@@ -659,16 +669,16 @@ test_2b() {
     replay_barrier mds
     do_node $CLIENT1 chmod 666 $DIR/$tfile-a            # R
     do_node $CLIENT2 chmod 666 $DIR1/$tfile-b           # R
-    do_node $CLIENT2 chown :$RUNAS_ID $DIR2/$tfile-a    # U
-    do_node $CLIENT1 chown $RUNAS_ID $DIR/$tfile-a      # J
-    do_node $CLIENT2 truncate $DIR2/$tfile-b 1          # U
-    do_node $CLIENT2 chown :$RUNAS_ID $DIR1/$tfile-b    # R
-    do_node $CLIENT1 chown $RUNAS_ID $DIR/$tfile-b      # R
+    do_node $CLIENT2 chgrp $RUNAS_GID $DIR2/$tfile-a    # U
+    do_node $CLIENT1 chown $RUNAS_ID:$RUNAS_GID $DIR/$tfile-a      # J
+    do_node $CLIENT2 $TRUNCATE $DIR2/$tfile-b 1          # U
+    do_node $CLIENT2 chgrp $RUNAS_GID $DIR1/$tfile-b    # R
+    do_node $CLIENT1 chown $RUNAS_ID:$RUNAS_GID $DIR/$tfile-b      # R
     zconf_umount $CLIENT2 $MOUNT2
     facet_failover mds
 
-    do_node $CLIENT1 df $MOUNT && error "$CLIENT1:$MOUNT not evicted"
-    do_node $CLIENT2 df $MOUNT1 || error "$CLIENT2:$MOUNT1 evicted"
+    client_evicted $CLIENT1 || error "$CLIENT1:$MOUNT not evicted"
+    client_up $CLIENT2 || error "$CLIENT2:$MOUNT1 evicted"
 
     #
     # Check the MDT epoch.  $post must be the first transaction
@@ -688,7 +698,7 @@ test_2b() {
 
     do_node $CLIENT2 $CHECKSTAT -p 0666 -u \\\#$UID -g \\\#$UID \
             $DIR1/$tfile-a || error "$DIR/$tfile-a: unexpected state"
-    do_node $CLIENT2 $CHECKSTAT -p 0666 -u \\\#$RUNAS_ID -g \\\#$RUNAS_ID \
+    do_node $CLIENT2 $CHECKSTAT -p 0666 -u \\\#$RUNAS_ID -g \\\#$RUNAS_GID \
             $DIR1/$tfile-b || error "$DIR/$tfile-b: unexpected state"
 
     zconf_umount $CLIENT2 $MOUNT1
@@ -701,7 +711,7 @@ test_3a() {
     zconf_mount_clients $CLIENT2 $DIR
 
     #make sure the time will change
-    do_facet mds "$LCTL set_param mds.${mds_svc}.atime_diff=0" || return
+    do_facet mds "$LCTL set_param *.${mds_svc}.atime_diff=0" || return
     do_node $CLIENT1 touch $DIR/$tfile
     do_node $CLIENT2 $CHECKSTAT $DIR/$tfile
     sleep 1
@@ -717,7 +727,7 @@ test_3a() {
 
     facet_failover mds
     # recovery shouldn't fail due to missing client 2
-    do_node $CLIENT1 df $DIR || return 1
+    client_up $CLIENT1 || return 1
     do_node $CLIENT1 $CHECKSTAT $DIR/$tfile && return 2
 
     zconf_mount $CLIENT2 $DIR || error "mount $CLIENT2 $DIR fail"
@@ -733,8 +743,8 @@ test_3b() {
     zconf_mount_clients $CLIENT2 $DIR
 
     #make sure the time will change
-    do_facet mds "$LCTL set_param mds.${mds_svc}.atime_diff=0" || return
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=0" || return
+    do_facet mds "$LCTL set_param *.${mds_svc}.atime_diff=0" || return
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=0" || return
     do_node $CLIENT1 touch $DIR/$tfile
     do_node $CLIENT2 $CHECKSTAT $DIR/$tfile
     sleep 1
@@ -748,7 +758,7 @@ test_3b() {
 
     facet_failover mds
     # recovery should fail due to missing client 2
-    do_node $CLIENT1 df $DIR && return 1
+    client_evicted $CLIENT1 || return 1
 
     do_node $CLIENT1 $CHECKSTAT -p 0755 $DIR/$tfile && return 2
     zconf_mount $CLIENT2 $DIR || error "mount $CLIENT2 $DIR fail"
@@ -763,13 +773,13 @@ test_3c() {
     [ "$FAILURE_MODE" = HARD ] || \
         { skip "The HARD failure is needed" && return 0; }
 
-    [ $RUNAS_ID -eq $UID ] && skip "RUNAS_ID = UID = $UID -- skipping" && return
+    [ $RUNAS_ID -eq $UID ] && skip_env "RUNAS_ID = UID = $UID -- skipping" && return
 
     zconf_mount_clients $CLIENT1 $DIR
     zconf_mount_clients $CLIENT2 $DIR
 
     # check that permission changes are synced
-    do_facet mds "$LCTL set_param mds.${mds_svc}.sync_permission=1"
+    do_facet mds "$LCTL set_param *.${mds_svc}.sync_permission=1"
 
     do_node $CLIENT1 mkdir -p $DIR/d3c/sub || error
     #chown -R $RUNAS_ID $MOUNT1/d3
@@ -792,8 +802,7 @@ test_3c() {
 
     facet_failover mds
     # recovery shouldn't fail due to missing client 2
-    do_node $CLIENT1 df $DIR || return 1
-    sleep 1
+    client_up $CLIENT1 || return 1
 
     zconf_mount $CLIENT2 $DIR || error "mount $CLIENT2 $DIR fail"
     do_node $CLIENT1 $RUNAS cat $DIR/d3c/sub/$tfile && return 6
@@ -827,7 +836,7 @@ test_4a() {
     delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
 
     remote_server $CLIENT2 || \
-        { skip "Client $CLIENT2 is on the server node" && return 0; }
+        { skip_env "Client $CLIENT2 is on the server node" && return 0; }
 
     zconf_mount_clients $CLIENT1 $DIR
     zconf_mount_clients $CLIENT2 $DIR
@@ -840,14 +849,14 @@ test_4a() {
     vbr_deactivate_client $CLIENT2
 
     facet_failover mds
-    do_node $CLIENT1 df $DIR || return 1
+    client_up $CLIENT1 || return 1
 
     # All 50 files should have been replayed
     do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 2
     do_node $CLIENT1 unlinkmany $DIR/$tfile-3- 25 || return 3
 
     vbr_activate_client $CLIENT2
-    do_node $CLIENT2 df $DIR || return 4
+    client_up $CLIENT2 || return 4
     # All 25 files from client2 should have been replayed
     do_node $CLIENT2 unlinkmany $DIR/$tdir/$tfile-2- 25 || return 5
 
@@ -860,7 +869,7 @@ test_4b() {
     delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
 
     remote_server $CLIENT2 || \
-        { skip "Client $CLIENT2 is on the server node" && return 0; }
+        { skip_env "Client $CLIENT2 is on the server node" && return 0; }
 
     zconf_mount_clients $CLIENT1 $DIR
     zconf_mount_clients $CLIENT2 $DIR
@@ -871,13 +880,13 @@ test_4b() {
     vbr_deactivate_client $CLIENT2
 
     facet_failover mds
-    do_node $CLIENT1 df $DIR || return 1
+    client_up $CLIENT1 || return 1
 
     # create another set of files
     do_node $CLIENT1 createmany -o $DIR/$tfile-3- 25
 
     vbr_activate_client $CLIENT2
-    do_node $CLIENT2 df $DIR || return 2
+    client_up $CLIENT2 || return 2
 
     # All files from should have been replayed
     do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 3
@@ -892,7 +901,7 @@ test_4c() {
     delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
 
     remote_server $CLIENT2 || \
-        { skip "Client $CLIENT2 is on the server node" && return 0; }
+        { skip_env "Client $CLIENT2 is on the server node" && return 0; }
 
     zconf_mount_clients $CLIENT1 $DIR
     zconf_mount_clients $CLIENT2 $DIR
@@ -903,13 +912,13 @@ test_4c() {
     vbr_deactivate_client $CLIENT2
 
     facet_failover mds
-    do_node $CLIENT1 df $DIR || return 1
+    client_up $CLIENT1 || return 1
 
     # create another set of files
     do_node $CLIENT1 createmany -m $DIR/$tfile-3- 25
 
     vbr_activate_client $CLIENT2
-    do_node $CLIENT2 df $DIR || return 2
+    client_up $CLIENT2 || return 2
 
     # All files from should have been replayed
     do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 3
@@ -924,7 +933,7 @@ test_5a() {
     delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
 
     remote_server $CLIENT2 || \
-        { skip "Client $CLIENT2 is on the server node" && return 0; }
+        { skip_env "Client $CLIENT2 is on the server node" && return 0; }
 
     zconf_mount_clients $CLIENT1 $DIR
     zconf_mount_clients $CLIENT2 $DIR
@@ -936,10 +945,10 @@ test_5a() {
     vbr_deactivate_client $CLIENT2
 
     facet_failover mds
-    do_node $CLIENT1 df $DIR && return 1
+    client_evicted $CLIENT1 || return 1
 
     vbr_activate_client $CLIENT2
-    do_node $CLIENT2 df $DIR || return 2
+    client_up $CLIENT2 || return 2
 
     # First 25 files should have been replayed
     do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 3
@@ -956,7 +965,7 @@ test_5b() {
     delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
 
     remote_server $CLIENT2 || \
-        { skip "Client $CLIENT2 is on the server node" && return 0; }
+        { skip_env "Client $CLIENT2 is on the server node" && return 0; }
 
     zconf_mount_clients $CLIENT1 $DIR
     zconf_mount_clients $CLIENT2 $DIR
@@ -967,14 +976,14 @@ test_5b() {
     vbr_deactivate_client $CLIENT2
 
     facet_failover mds
-    do_node $CLIENT1 df $DIR || return 1
+    client_up $CLIENT1 || return 1
     do_node $CLIENT1 $CHECKSTAT $DIR/$tfile-2-0 && error "$tfile-2-0 exists"
 
     # create another set of files
     do_node $CLIENT1 createmany -o $DIR/$tfile-3- 25
 
     vbr_activate_client $CLIENT2
-    do_node $CLIENT2 df $DIR && return 4
+    client_evicted $CLIENT2 || return 4
     # file from client2 should fail
     do_node $CLIENT2 $CHECKSTAT $DIR/$tfile-2-0 && error "$tfile-2-0 exists"
 
@@ -990,7 +999,7 @@ test_6a() {
     delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
 
     remote_server $CLIENT2 || \
-        { skip "Client $CLIENT2 is on the server node" && return 0; }
+        { skip_env "Client $CLIENT2 is on the server node" && return 0; }
 
     zconf_mount_clients $CLIENT1 $DIR
     zconf_mount_clients $CLIENT2 $DIR
@@ -1007,13 +1016,13 @@ test_6a() {
     do_node $CLIENT2 "sysctl -w lustre.fail_val=5"
 #define OBD_FAIL_PTLRPC_REPLAY        0x50e
     do_node $CLIENT2 "sysctl -w lustre.fail_loc=0x2000050e"
-    do_node $CLIENT2 df $DIR
+    client_up $CLIENT2
     # vbr_activate_client $CLIENT2
     # need way to know that client stops replays
     sleep 5
 
     facet_failover mds
-    do_node $CLIENT1 df $DIR || return 1
+    client_up $CLIENT1 || return 1
 
     # All files should have been replayed
     do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 2
@@ -1029,7 +1038,7 @@ test_7a() {
     delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
 
     remote_server $CLIENT2 || \
-        { skip "Client $CLIENT2 is on the server node" && return 0; }
+        { skip_env "Client $CLIENT2 is on the server node" && return 0; }
 
     zconf_mount_clients $CLIENT1 $DIR
     zconf_mount_clients $CLIENT2 $DIR
@@ -1043,10 +1052,10 @@ test_7a() {
 
     facet_failover mds
     vbr_activate_client $CLIENT2
-    do_node $CLIENT2 df $DIR || return 4
+    client_up $CLIENT2 || return 4
 
     facet_failover mds
-    do_node $CLIENT1 df $DIR || return 1
+    client_up $CLIENT1 || return 1
 
     # All files should have been replayed
     do_node $CLIENT1 unlinkmany $DIR/$tfile- 25 || return 2
@@ -1062,7 +1071,7 @@ test_8a() {
     delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
 
     remote_server $CLIENT2 || \
-        { skip "Client $CLIENT2 is on the server node" && return 0; }
+        { skip_env "Client $CLIENT2 is on the server node" && return 0; }
 
     zconf_mount_clients $CLIENT1 $DIR
     zconf_mount_clients $CLIENT2 $DIR
@@ -1074,10 +1083,10 @@ test_8a() {
 
     vbr_deactivate_client $CLIENT2
     facet_failover mds
-    do_node $CLIENT1 df $DIR || return 3
+    client_up $CLIENT1 || return 3
     #client1 is back and will try to open orphan
     vbr_activate_client $CLIENT2
-    do_node $CLIENT2 df $DIR || return 4
+    client_up $CLIENT2 || return 4
 
     do_node $CLIENT2 $CHECKSTAT $DIR/$tfile && error "$tfile exists"
     zconf_umount_clients $CLIENTS $DIR
@@ -1089,7 +1098,7 @@ test_8b() {
     delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
 
     remote_server $CLIENT2 || \
-        { skip "Client $CLIENT2 is on the server node" && return 0; }
+        { skip_env "Client $CLIENT2 is on the server node" && return 0; }
 
     zconf_mount_clients $CLIENT1 $DIR
     zconf_mount_clients $CLIENT2 $DIR
@@ -1100,10 +1109,10 @@ test_8b() {
 
     vbr_deactivate_client $CLIENT2
     facet_failover mds
-    do_node $CLIENT1 df $DIR || return 2
+    client_up $CLIENT1 || return 2
     #client1 is back and will try to open orphan
     vbr_activate_client $CLIENT2
-    do_node $CLIENT2 df $DIR || return 3
+    client_up $CLIENT2 || return 3
 
     rmultiop_stop $CLIENT2 || return 1
     do_node $CLIENT2 $CHECKSTAT $DIR/$tfile && error "$tfile exists"
@@ -1116,7 +1125,7 @@ test_8c() {
     delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
 
     remote_server $CLIENT2 || \
-        { skip "Client $CLIENT2 is on the server node" && return 0; }
+        { skip_env "Client $CLIENT2 is on the server node" && return 0; }
 
     zconf_mount_clients $CLIENT1 $DIR
     zconf_mount_clients $CLIENT2 $DIR
@@ -1128,10 +1137,10 @@ test_8c() {
 
     vbr_deactivate_client $CLIENT2
     facet_failover mds
-    do_node $CLIENT1 df $DIR || return 3
+    client_up $CLIENT1 || return 3
     #client1 is back and will try to open orphan
     vbr_activate_client $CLIENT2
-    do_node $CLIENT2 df $DIR || return 4
+    client_up $CLIENT2 || return 4
 
     do_node $CLIENT2 $CHECKSTAT $DIR/$tfile && error "$tfile exists"
     zconf_umount_clients $CLIENTS $DIR
@@ -1143,7 +1152,7 @@ test_8d() {
     delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
 
     remote_server $CLIENT2 || \
-        { skip "Client $CLIENT2 is on the server node" && return 0; }
+        { skip_env "Client $CLIENT2 is on the server node" && return 0; }
 
     zconf_mount_clients $CLIENT1 $DIR
     zconf_mount_clients $CLIENT2 $DIR
@@ -1157,11 +1166,11 @@ test_8d() {
 
     vbr_deactivate_client $CLIENT2
     facet_failover mds
-    do_node $CLIENT1 df $DIR || return 6
+    client_up $CLIENT1 || return 6
 
     #client1 is back and will try to open orphan
     vbr_activate_client $CLIENT2
-    do_node $CLIENT2 df $DIR || return 8
+    client_up $CLIENT2 || return 8
 
     do_node $CLIENT2 $CHECKSTAT $DIR/$tfile && error "$tfile exists"
     zconf_umount_clients $CLIENTS $DIR
@@ -1183,7 +1192,7 @@ test_8e() {
 
     zconf_umount $CLIENT1 $DIR
     facet_failover mds
-    do_node $CLIENT2 df $DIR || return 6
+    client_up $CLIENT2 || return 6
 
     do_node $CLIENT2 rm $DIR/$tfile || error "$tfile doesn't exists"
     zconf_umount_clients $CLIENTS $DIR
@@ -1205,7 +1214,7 @@ test_8f() {
 
     zconf_umount $CLIENT1 $DIR
     facet_failover mds
-    do_node $CLIENT2 df $DIR || return 6
+    client_up $CLIENT2 || return 6
 
     do_node $CLIENT2 rm $DIR/$tfile || error "$tfile doesn't exists"
     zconf_umount $CLIENT2 $DIR
@@ -1238,7 +1247,7 @@ run_test 8g "create | unlink, create shouldn't fail"
 test_10 () {
     delayed_recovery_enabled || { skip "No delayed recovery support"; return 0; }
 
-    [ -z "$DBENCH_LIB" ] && skip "DBENCH_LIB is not set" && return 0
+    [ -z "$DBENCH_LIB" ] && skip_env "DBENCH_LIB is not set" && return 0
 
     zconf_mount_clients $CLIENTS $DIR
 
@@ -1264,7 +1273,7 @@ test_10 () {
     sleep $TIMEOUT
 
     vbr_activate_client $CLIENT2
-    do_node $CLIENT2 df $DIR || return 4
+    client_up $CLIENT2 || return 4
 
     for CLIENT in ${CLIENTS//,/ }; do
         PID=`cat pid.$CLIENT`
@@ -1279,8 +1288,6 @@ run_test 10 "mds version recovery; $CLIENTCOUNT clients"
 
 [ "$CLIENTS" ] && zconf_mount_clients $CLIENTS $DIR
 
-equals_msg `basename $0`: test complete, cleaning up
-#SLEEP=$((`date +%s` - $NOW))
-#[ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
+complete $(basename $0) $SECONDS
 check_and_cleanup_lustre
-[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG && grep -q FAIL $TESTSUITELOG && exit 1 || true
+exit_status