From db68565b462d954dc50b0f638fea820991f54209 Mon Sep 17 00:00:00 2001 From: Niu Yawei Date: Thu, 16 Jan 2014 21:56:29 -0500 Subject: [PATCH] LU-4482 grant: don't use cache data in osd_statfs() osd_statfs() shouldn't cache statfs data anymore: the statfs data is already cached in ofd layer, put another cache in osd layer looks redundant, and what more important is: grant mechanism relies on dt_statfs() returning fresh statfs data, caching statfs data in osd layer would just break grant. Signed-off-by: Niu Yawei Change-Id: I89b6384cc59d77b1edb0412f24b5c8e823532170 Reviewed-on: http://review.whamcloud.com/8911 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Johann Lombardi Reviewed-by: Alex Zhuravlev --- lustre/ofd/ofd_grant.c | 8 +++++--- lustre/osd-ldiskfs/osd_handler.c | 31 ++++++++++++++++++------------- lustre/osd-ldiskfs/osd_internal.h | 6 +----- lustre/tests/sanity.sh | 6 +++--- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/lustre/ofd/ofd_grant.c b/lustre/ofd/ofd_grant.c index 0d3ec43..7fa1e4a 100644 --- a/lustre/ofd/ofd_grant.c +++ b/lustre/ofd/ofd_grant.c @@ -206,7 +206,8 @@ static void ofd_grant_statfs(const struct lu_env *env, struct obd_export *exp, rc = ofd_statfs_internal(env, ofd, osfs, max_age, from_cache); if (unlikely(rc)) { - *from_cache = 0; + if (from_cache) + *from_cache = 0; return; } @@ -249,10 +250,11 @@ static obd_size ofd_grant_space_left(struct obd_export *exp) D_ERROR : D_CACHE; CDEBUG_LIMIT(mask, "%s: cli %s/%p left "LPU64" < tot_grant " - LPU64" unstable "LPU64" pending "LPU64"\n", + LPU64" unstable "LPU64" pending "LPU64" " + "dirty "LPU64"\n", obd->obd_name, exp->exp_client_uuid.uuid, exp, left, tot_granted, unstable, - ofd->ofd_tot_pending); + ofd->ofd_tot_pending, ofd->ofd_tot_dirty); RETURN(0); } diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index 2d2ddcc..3fd2e4c 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -1245,6 +1245,8 @@ static int osd_object_print(const struct lu_env *env, void *cookie, d ? d->id_ops->id_name : "plain"); } +#define GRANT_FOR_LOCAL_OIDS 32 /* 128kB for last_rcvd, quota files, ... */ + /* * Concurrency: shouldn't matter. */ @@ -1269,24 +1271,28 @@ int osd_statfs(const struct lu_env *env, struct dt_device *d, } spin_lock(&osd->od_osfs_lock); - /* cache 1 second */ - if (cfs_time_before_64(osd->od_osfs_age, cfs_time_shift_64(-1))) { - result = sb->s_op->statfs(sb->s_root, ksfs); - if (likely(result == 0)) { /* N.B. statfs can't really fail */ - osd->od_osfs_age = cfs_time_current_64(); - statfs_pack(&osd->od_statfs, ksfs); - if (sb->s_flags & MS_RDONLY) - sfs->os_state = OS_STATE_READONLY; - } + result = sb->s_op->statfs(sb->s_root, ksfs); + if (likely(result == 0)) { /* N.B. statfs can't really fail */ + statfs_pack(sfs, ksfs); + if (sb->s_flags & MS_RDONLY) + sfs->os_state = OS_STATE_READONLY; } - if (likely(result == 0)) - *sfs = osd->od_statfs; spin_unlock(&osd->od_osfs_lock); - if (unlikely(env == NULL)) + if (unlikely(env == NULL)) OBD_FREE_PTR(ksfs); + /* Reserve a small amount of space for local objects like last_rcvd, + * llog, quota files, ... */ + if (sfs->os_bavail <= GRANT_FOR_LOCAL_OIDS) { + sfs->os_bavail = 0; + } else { + sfs->os_bavail -= GRANT_FOR_LOCAL_OIDS; + /** Take out metadata overhead for indirect blocks */ + sfs->os_bavail -= sfs->os_bavail >> (sb->s_blocksize_bits - 3); + } + return result; } @@ -5686,7 +5692,6 @@ static int osd_device_init0(const struct lu_env *env, spin_lock_init(&o->od_osfs_lock); mutex_init(&o->od_otable_mutex); - o->od_osfs_age = cfs_time_shift_64(-1000); o->od_capa_hash = init_capa_hash(); if (o->od_capa_hash == NULL) diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h index 6c84842..dc52885 100644 --- a/lustre/osd-ldiskfs/osd_internal.h +++ b/lustre/osd-ldiskfs/osd_internal.h @@ -245,11 +245,7 @@ struct osd_device { cfs_proc_dir_entry_t *od_proc_entry; struct lprocfs_stats *od_stats; - /* - * statfs optimization: we cache a bit. - */ - cfs_time_t od_osfs_age; - struct obd_statfs od_statfs; + spinlock_t od_osfs_lock; int od_connects; diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 4b20e2b..b0220e2 100644 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -4842,10 +4842,11 @@ test_62() { #run_test 62 "verify obd_match failure doesn't LBUG (should -EIO)" # bug 2319 - oig_wait() interrupted causes crash because of invalid waitq. -test_63a() { # was test_63 +# Though this test is irrelevant anymore, it helped to reveal some +# other grant bugs (LU-4482), let's keep it. +test_63a() { # was test_63 [ $PARALLEL == "yes" ] && skip "skip parallel run" && return MAX_DIRTY_MB=`lctl get_param -n osc.*.max_dirty_mb | head -n 1` - lctl set_param -n osc.*.max_dirty_mb 0 for i in `seq 10` ; do dd if=/dev/zero of=$DIR/f63 bs=8k & sleep 5 @@ -4853,7 +4854,6 @@ test_63a() { # was test_63 sleep 1 done - lctl set_param -n osc.*.max_dirty_mb $MAX_DIRTY_MB rm -f $DIR/f63 || true } run_test 63a "Verify oig_wait interruption does not crash =======" -- 1.8.3.1