LU-2903 tests: calculation of available space

Better debug logging for statfs path.
Account for log size in space calculation.
Sleep slightly longer to wait for good statfs because zfs doesn't
count uncommitted changes in available space.

Test-Parameters: mdsfilesystemtype=zfs ostfilesystemtype=zfs   mdtfilesystemtype=zfs testlist=replay-ost-single
Signed-off-by: Nathaniel Clark <nathaniel.l.clark@intel.com>
Change-Id: Ib4b5a7c645d6b2a630dcc729483422c8b3a095db
Reviewed-on: http://review.whamcloud.com/5662
Tested-by: Hudson
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Mike Pershin <mike.pershin@intel.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>

diff --git a/lustre/ofd/ofd_obd.c b/lustre/ofd/ofd_obd.c
index 5bd258f..99de9eb 100644 (file)
--- a/lustre/ofd/ofd_obd.c
+++ b/lustre/ofd/ofd_obd.c
@@ -1277,9 +1277,9 @@ int ofd_create(const struct lu_env *env, struct obd_export *exp,
                                              ofd_obd(ofd)->obd_self_export,
                                              &diff);
                        if (rc) {
-                               CDEBUG(D_HA, "%s: failed to acquire grant space"
-                                      "for precreate (%d)\n",
-                                      ofd_name(ofd), diff);
+                               CDEBUG(D_HA, "%s: failed to acquire grant "
+                                      "space for precreate (%d): rc = %d\n",
+                                      ofd_name(ofd), diff, rc);
                                diff = 0;
                        }
                }
@@ -1315,7 +1315,8 @@ int ofd_create(const struct lu_env *env, struct obd_export *exp,
                         * them, even if last creation failed */
                        rc = 0;
                else
-                       CERROR("unable to precreate: %d\n", rc);
+                       CERROR("%s: unable to precreate: rc = %d\n",
+                              ofd_name(ofd), rc);
 
                oa->o_id = ofd_seq_last_oid(oseq);
                oa->o_valid |= OBD_MD_FLID | OBD_MD_FLGROUP;

diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c
index 22928c4..6ae9225 100644 (file)
--- a/lustre/osd-zfs/osd_handler.c
+++ b/lustre/osd-zfs/osd_handler.c
@@ -319,7 +319,7 @@ int osd_statfs(const struct lu_env *env, struct dt_device *d,
        ENTRY;
 
        rc = udmu_objset_statfs(&osd->od_objset, osfs);
-       if (rc)
+       if (unlikely(rc))
                RETURN(rc);
        osfs->os_bavail -= min_t(obd_size,
                                 OSD_GRANT_FOR_LOCAL_OIDS / osfs->os_bsize,

diff --git a/lustre/tests/replay-ost-single.sh b/lustre/tests/replay-ost-single.sh
index 5f61e0d..e0b8204 100755 (executable)
--- a/lustre/tests/replay-ost-single.sh
+++ b/lustre/tests/replay-ost-single.sh
@@ -178,76 +178,86 @@ kbytesfree() {
 }
 
 test_6() {
-    remote_mds_nodsh && skip "remote MDS with nodsh" && return 0
-
-    f=$TDIR/$tfile
-    rm -f $f
-    sync && sleep 2 && sync  # wait for delete thread
-
-    # wait till space is returned, following
-    # (( $before > $after_dd)) test counting on that
-    wait_mds_ost_sync || return 4
-    wait_destroy_complete || return 5
-
-    before=`kbytesfree`
-    dd if=/dev/urandom bs=4096 count=1280 of=$f || return 28
-    lfs getstripe $f
-    stripe_index=$(lfs getstripe -i $f)
-
-    sync
-    sleep 2 # ensure we have a fresh statfs
-    sync
-
-    #define OBD_FAIL_MDS_REINT_NET_REP       0x119
-    do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000119"
-    after_dd=`kbytesfree`
-    log "before: $before after_dd: $after_dd"
-    (( $before > $after_dd )) || return 1
-    rm -f $f
-    fail ost$((stripe_index + 1))
-    wait_recovery_complete ost$((stripe_index + 1)) ||
-        error "OST$((stripe_index + 1)) recovery not completed"
-    $CHECKSTAT -t file $f && return 2 || true
-    sync
-    # let the delete happen
-    wait_mds_ost_sync || return 4
+       remote_mds_nodsh && skip "remote MDS with nodsh" && return 0
+
+       local f=$TDIR/$tfile
+       rm -f $f
+       sync && sleep 2 && sync  # wait for delete thread
+
+       # wait till space is returned, following
+       # (( $before > $after_dd)) test counting on that
+       wait_mds_ost_sync || return 4
+       wait_destroy_complete || return 5
+
+       local before=$(kbytesfree)
+       dd if=/dev/urandom bs=4096 count=1280 of=$f || return 28
+       lfs getstripe $f
+       local stripe_index=$(lfs getstripe -i $f)
+
+       sync
+       sleep 4 # ensure we have a fresh statfs and changes have stablalized
+       sync
+
+       #define OBD_FAIL_MDS_REINT_NET_REP       0x119
+       do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000119"
+       local after_dd=$(kbytesfree)
+       log "before: $before after_dd: $after_dd"
+       (( $before > $after_dd )) ||
+               error "space grew after dd: before:$before after_dd:$after_dd"
+       rm -f $f
+       fail ost$((stripe_index + 1))
+       wait_recovery_complete ost$((stripe_index + 1)) ||
+               error "OST$((stripe_index + 1)) recovery not completed"
+       $CHECKSTAT -t file $f && return 2 || true
+       sync
+       # let the delete happen
+       wait_mds_ost_sync || return 4
        wait_delete_completed || return 5
-    after=`kbytesfree`
-    log "before: $before after: $after"
-    (( $before <= $after + 40 )) || return 3   # take OST logs into account
+       local after=$(kbytesfree)
+       log "before: $before after: $after"
+       (( $before <= $after + $(fs_log_size) )) ||
+               error "$before > $after + logsize $(fs_log_size)"
 }
 run_test 6 "Fail OST before obd_destroy"
 
 test_7() {
-    f=$TDIR/$tfile
-    rm -f $f
-    sync && sleep 5 && sync    # wait for delete thread
-
-    # wait till space is returned, following
-    # (( $before > $after_dd)) test counting on that
-    wait_mds_ost_sync || return 4
-    wait_destroy_complete || return 5
-
-    before=`kbytesfree`
-    dd if=/dev/urandom bs=4096 count=1280 of=$f || return 4
-    sync
-    sleep 2 # ensure we have a fresh statfs
-    sync
-    after_dd=`kbytesfree`
-    log "before: $before after_dd: $after_dd"
-    (( $before > $after_dd )) || return 1
-    replay_barrier ost1
-    rm -f $f
-    fail ost1
-    wait_recovery_complete ost1 || error "OST recovery not done"
-    $CHECKSTAT -t file $f && return 2 || true
-    sync
-    # let the delete happen
-    wait_mds_ost_sync || return 4
+       local f=$TDIR/$tfile
+       rm -f $f
+       sync && sleep 5 && sync # wait for delete thread
+
+       # wait till space is returned, following
+       # (( $before > $after_dd)) test counting on that
+       wait_mds_ost_sync || return 4
+       wait_destroy_complete || return 5
+
+       local before=$(kbytesfree)
+       dd if=/dev/urandom bs=4096 count=1280 of=$f || error "dd to file failed: $?"
+
+       sync
+       local after_dd=$(kbytesfree)
+       while (( $before <= $after_dd && $i < 10 )); do
+               sync
+               sleep 1
+               let ++i
+               after_dd=$(kbytesfree)
+       done
+
+       log "before: $before after_dd: $after_dd took $i seconds"
+       (( $before > $after_dd )) ||
+               error "space grew after dd: before:$before after_dd:$after_dd"
+       replay_barrier ost1
+       rm -f $f
+       fail ost1
+       wait_recovery_complete ost1 || error "OST recovery not done"
+       $CHECKSTAT -t file $f && return 2 || true
+       sync
+       # let the delete happen
+       wait_mds_ost_sync || return 4
        wait_delete_completed || return 5
-    after=`kbytesfree`
-    log "before: $before after: $after"
-    (( $before <= $after + 40 )) || return 3   # take OST logs into account
+       local after=$(kbytesfree)
+       log "before: $before after: $after"
+       (( $before <= $after + $(fs_log_size) )) ||
+                error "$before > $after + logsize $(fs_log_size)"
 }
 run_test 7 "Fail OST before obd_destroy"
 

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 42c34d5..25c6669 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -585,6 +585,18 @@ unload_modules() {
     return 0
 }
 
+fs_log_size() {
+       local facet=${1:-$SINGLEMDS}
+       local fstype=$(facet_fstype $facet)
+       local size=0
+       case $fstype in
+               ldiskfs) size=50;; # largest seen is 44, leave some headroom
+               zfs)     size=256;;
+       esac
+
+       echo -n $size
+}
+
 check_gss_daemon_nodes() {
     local list=$1
     dname=$2