LU-4482 grant: don't use cache data in osd_statfs()

osd_statfs() shouldn't cache statfs data anymore: the statfs data
is already cached in ofd layer, put another cache in osd layer
looks redundant, and what more important is: grant mechanism relies
on dt_statfs() returning fresh statfs data, caching statfs data in
osd layer would just break grant.

Signed-off-by: Niu Yawei <yawei.niu@intel.com>
Change-Id: I89b6384cc59d77b1edb0412f24b5c8e823532170
Reviewed-on: http://review.whamcloud.com/8911
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Johann Lombardi <johann.lombardi@intel.com>
Reviewed-by: Alex Zhuravlev <alexey.zhuravlev@intel.com>

diff --git a/lustre/ofd/ofd_grant.c b/lustre/ofd/ofd_grant.c
index 0d3ec43..7fa1e4a 100644 (file)
--- a/lustre/ofd/ofd_grant.c
+++ b/lustre/ofd/ofd_grant.c
@@ -206,7 +206,8 @@ static void ofd_grant_statfs(const struct lu_env *env, struct obd_export *exp,
 
        rc = ofd_statfs_internal(env, ofd, osfs, max_age, from_cache);
        if (unlikely(rc)) {
-               *from_cache = 0;
+               if (from_cache)
+                       *from_cache = 0;
                return;
        }
 
@@ -249,10 +250,11 @@ static obd_size ofd_grant_space_left(struct obd_export *exp)
                            D_ERROR : D_CACHE;
 
                CDEBUG_LIMIT(mask, "%s: cli %s/%p left "LPU64" < tot_grant "
-                            LPU64" unstable "LPU64" pending "LPU64"\n",
+                            LPU64" unstable "LPU64" pending "LPU64" "
+                            "dirty "LPU64"\n",
                             obd->obd_name, exp->exp_client_uuid.uuid, exp,
                             left, tot_granted, unstable,
-                            ofd->ofd_tot_pending);
+                            ofd->ofd_tot_pending, ofd->ofd_tot_dirty);
                RETURN(0);
        }
 

diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c
index 2d2ddcc..3fd2e4c 100644 (file)
--- a/lustre/osd-ldiskfs/osd_handler.c
+++ b/lustre/osd-ldiskfs/osd_handler.c
@@ -1245,6 +1245,8 @@ static int osd_object_print(const struct lu_env *env, void *cookie,
                     d ? d->id_ops->id_name : "plain");
 }
 
+#define GRANT_FOR_LOCAL_OIDS 32 /* 128kB for last_rcvd, quota files, ... */
+
 /*
  * Concurrency: shouldn't matter.
  */
@@ -1269,24 +1271,28 @@ int osd_statfs(const struct lu_env *env, struct dt_device *d,
         }
 
        spin_lock(&osd->od_osfs_lock);
-       /* cache 1 second */
-       if (cfs_time_before_64(osd->od_osfs_age, cfs_time_shift_64(-1))) {
-               result = sb->s_op->statfs(sb->s_root, ksfs);
-               if (likely(result == 0)) { /* N.B. statfs can't really fail */
-                       osd->od_osfs_age = cfs_time_current_64();
-                       statfs_pack(&osd->od_statfs, ksfs);
-                       if (sb->s_flags & MS_RDONLY)
-                               sfs->os_state = OS_STATE_READONLY;
-               }
+       result = sb->s_op->statfs(sb->s_root, ksfs);
+       if (likely(result == 0)) { /* N.B. statfs can't really fail */
+               statfs_pack(sfs, ksfs);
+               if (sb->s_flags & MS_RDONLY)
+                       sfs->os_state = OS_STATE_READONLY;
        }
 
-       if (likely(result == 0))
-               *sfs = osd->od_statfs;
        spin_unlock(&osd->od_osfs_lock);
 
-        if (unlikely(env == NULL))
+       if (unlikely(env == NULL))
                 OBD_FREE_PTR(ksfs);
 
+       /* Reserve a small amount of space for local objects like last_rcvd,
+        * llog, quota files, ... */
+       if (sfs->os_bavail <= GRANT_FOR_LOCAL_OIDS) {
+               sfs->os_bavail = 0;
+       } else {
+               sfs->os_bavail -= GRANT_FOR_LOCAL_OIDS;
+               /** Take out metadata overhead for indirect blocks */
+               sfs->os_bavail -= sfs->os_bavail >> (sb->s_blocksize_bits - 3);
+       }
+
         return result;
 }
 
@@ -5686,7 +5692,6 @@ static int osd_device_init0(const struct lu_env *env,
 
        spin_lock_init(&o->od_osfs_lock);
        mutex_init(&o->od_otable_mutex);
-       o->od_osfs_age = cfs_time_shift_64(-1000);
 
        o->od_capa_hash = init_capa_hash();
        if (o->od_capa_hash == NULL)

diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h
index 6c84842..dc52885 100644 (file)
--- a/lustre/osd-ldiskfs/osd_internal.h
+++ b/lustre/osd-ldiskfs/osd_internal.h
@@ -245,11 +245,7 @@ struct osd_device {
 
         cfs_proc_dir_entry_t     *od_proc_entry;
         struct lprocfs_stats     *od_stats;
-        /*
-         * statfs optimization: we cache a bit.
-         */
-        cfs_time_t                od_osfs_age;
-        struct obd_statfs         od_statfs;
+
        spinlock_t                od_osfs_lock;
 
        int                       od_connects;

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 4b20e2b..b0220e2 100644 (file)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -4842,10 +4842,11 @@ test_62() {
 #run_test 62 "verify obd_match failure doesn't LBUG (should -EIO)"
 
 # bug 2319 - oig_wait() interrupted causes crash because of invalid waitq.
-test_63a() {   # was test_63
+# Though this test is irrelevant anymore, it helped to reveal some
+# other grant bugs (LU-4482), let's keep it.
+test_63a() {   # was test_63
        [ $PARALLEL == "yes" ] && skip "skip parallel run" && return
        MAX_DIRTY_MB=`lctl get_param -n osc.*.max_dirty_mb | head -n 1`
-       lctl set_param -n osc.*.max_dirty_mb 0
        for i in `seq 10` ; do
                dd if=/dev/zero of=$DIR/f63 bs=8k &
                sleep 5
@@ -4853,7 +4854,6 @@ test_63a() {      # was test_63
                sleep 1
        done
 
-       lctl set_param -n osc.*.max_dirty_mb $MAX_DIRTY_MB
        rm -f $DIR/f63 || true
 }
 run_test 63a "Verify oig_wait interruption does not crash ======="