From b917406a7f0a84442b046df34fada6f395ba6928 Mon Sep 17 00:00:00 2001 From: Andreas Dilger Date: Mon, 8 Oct 2018 16:00:08 -0600 Subject: [PATCH] LU-7770 lov: fix statfs for conf-sanity test_50b Wait for the *client* to be disconnected from the OSTs, not the MDS, to ensure that the test is actually doing what it thinks it should. In conf-sanity.sh::lazystatfs(), sleep between statfs operations to ensure we are not just getting locally-cached statfs results. Print out device status to ensure that wait_osc_import_state has actually waited long enough for OSCs to be marked disconnected. In obd_statfs() print the device name in the debug logs for clarity. Have "lfs df" print block stats from MDTs only if no OSTs connected. Checkpatch should warn about get_seconds(), not ktime_get_seconds(). Test-Parameters: mdtcount=4 testlist=conf-sanity,conf-sanity,conf-sanity envdefinitions=ONLY=50b Test-Parameters: mdtcount=4 testlist=conf-sanity,conf-sanity,conf-sanity envdefinitions=ONLY=50b Test-Parameters: mdtcount=4 testlist=conf-sanity,conf-sanity,conf-sanity envdefinitions=ONLY=50b Change-Id: Icbe68f0a133f04f89d44f74a5caaa6c523fcab07 Signed-off-by: Andreas Dilger Reviewed-on: https://review.whamcloud.com/33369 Tested-by: Jenkins Reviewed-by: Ben Evans Tested-by: Maloo Reviewed-by: Alex Zhuravlev Reviewed-by: Oleg Drokin --- contrib/scripts/spelling.txt | 2 +- lustre/include/obd_class.h | 35 +++++++++++++++++----------------- lustre/lov/lov_obd.c | 37 ++++++++++++++++++------------------ lustre/tests/conf-sanity.sh | 25 +++++++++++++----------- lustre/utils/lfs.c | 45 ++++++++++++++++++++++++++++++-------------- 5 files changed, 81 insertions(+), 63 deletions(-) diff --git a/contrib/scripts/spelling.txt b/contrib/scripts/spelling.txt index 062df9e..fd28a99 100644 --- a/contrib/scripts/spelling.txt +++ b/contrib/scripts/spelling.txt @@ -97,7 +97,7 @@ DN_MAX_BONUSLEN||DN_BONUS_SIZE(dnodesize) DN_OLD_MAX_BONUSLEN||DN_BONUS_SIZE(DNODE_MIN_SIZE) from_timer||cfs_from_timer f_dentry||f_path.dentry -get_seconds||ktime_get_real_seconds +[^_]get_seconds||ktime_get_real_seconds GETSTRIPE||LFS getstripe ldlm_appetite_t||enum ldlm_appetite ldlm_cancel_flags_t||enum ldlm_cancel_flags diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 863249d..ed410a0 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -999,8 +999,8 @@ static inline int obd_statfs_async(struct obd_export *exp, time64_t max_age, struct ptlrpc_request_set *rqset) { - int rc = 0; struct obd_device *obd; + int rc = 0; ENTRY; @@ -1013,8 +1013,8 @@ static inline int obd_statfs_async(struct obd_export *exp, RETURN(-EOPNOTSUPP); } - CDEBUG(D_SUPER, "%s: osfs %p age %lld, max_age %lld\n", - obd->obd_name, &obd->obd_osfs, obd->obd_osfs_age, max_age); + CDEBUG(D_SUPER, "%s: age %lld, max_age %lld\n", + obd->obd_name, obd->obd_osfs_age, max_age); if (obd->obd_osfs_age < max_age) { rc = OBP(obd, statfs_async)(exp, oinfo, max_age, rqset); } else { @@ -1024,8 +1024,7 @@ static inline int obd_statfs_async(struct obd_export *exp, obd->obd_osfs.os_bavail, obd->obd_osfs.os_blocks, obd->obd_osfs.os_ffree, obd->obd_osfs.os_files); spin_lock(&obd->obd_osfs_lock); - memcpy(oinfo->oi_osfs, &obd->obd_osfs, - sizeof(*oinfo->oi_osfs)); + memcpy(oinfo->oi_osfs, &obd->obd_osfs, sizeof(*oinfo->oi_osfs)); spin_unlock(&obd->obd_osfs_lock); oinfo->oi_flags |= OBD_STATFS_FROM_CACHE; if (oinfo->oi_cb_up) @@ -1040,38 +1039,38 @@ static inline int obd_statfs_async(struct obd_export *exp, */ static inline int obd_statfs(const struct lu_env *env, struct obd_export *exp, struct obd_statfs *osfs, time64_t max_age, - __u32 flags) + __u32 flags) { - int rc = 0; - struct obd_device *obd = exp->exp_obd; - ENTRY; + struct obd_device *obd = exp->exp_obd; + int rc = 0; - if (obd == NULL) - RETURN(-EINVAL); + ENTRY; + if (unlikely(obd == NULL)) + RETURN(-EINVAL); OBD_CHECK_DEV_ACTIVE(obd); - if (!obd->obd_type || !obd->obd_type->typ_dt_ops->o_statfs) { + if (unlikely(!obd->obd_type || !obd->obd_type->typ_dt_ops->o_statfs)) { CERROR("%s: no %s operation\n", obd->obd_name, __func__); RETURN(-EOPNOTSUPP); } - CDEBUG(D_SUPER, "osfs %lld, max_age %lld\n", - obd->obd_osfs_age, max_age); + CDEBUG(D_SUPER, "%s: age %lld, max_age %lld\n", + obd->obd_name, obd->obd_osfs_age, max_age); /* ignore cache if aggregated isn't expected */ if (obd->obd_osfs_age < max_age || ((obd->obd_osfs.os_state & OS_STATE_SUM) && !(flags & OBD_STATFS_SUM))) { - rc = OBP(obd, statfs)(env, exp, osfs, max_age, flags); - if (rc == 0) { + rc = OBP(obd, statfs)(env, exp, osfs, max_age, flags); + if (rc == 0) { spin_lock(&obd->obd_osfs_lock); memcpy(&obd->obd_osfs, osfs, sizeof(obd->obd_osfs)); obd->obd_osfs_age = ktime_get_seconds(); spin_unlock(&obd->obd_osfs_lock); } } else { - CDEBUG(D_SUPER, "%s: use %p cache blocks %llu/%llu" - " objects %llu/%llu\n", + CDEBUG(D_SUPER, + "%s: use %p cache blocks %llu/%llu objects %llu/%llu\n", obd->obd_name, &obd->obd_osfs, obd->obd_osfs.os_bavail, obd->obd_osfs.os_blocks, obd->obd_osfs.os_ffree, obd->obd_osfs.os_files); diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c index 71b2829..0f15c62 100644 --- a/lustre/lov/lov_obd.c +++ b/lustre/lov/lov_obd.c @@ -964,14 +964,14 @@ out_rqset: static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, void *karg, void __user *uarg) { - struct obd_device *obddev = class_exp2obd(exp); - struct lov_obd *lov = &obddev->u.lov; - int i = 0, rc = 0, count = lov->desc.ld_tgt_count; - struct obd_uuid *uuidp; - ENTRY; + struct obd_device *obd = class_exp2obd(exp); + struct lov_obd *lov = &obd->u.lov; + int i = 0, rc = 0, count = lov->desc.ld_tgt_count; + struct obd_uuid *uuidp; - switch (cmd) { - case IOC_OBD_STATFS: { + ENTRY; + switch (cmd) { + case IOC_OBD_STATFS: { struct obd_ioctl_data *data = karg; struct obd_device *osc_obd; struct obd_statfs stat_buf = {0}; @@ -980,7 +980,7 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, __u32 flags; memcpy(&index, data->ioc_inlbuf2, sizeof(index)); - if ((index >= count)) + if (index >= count) RETURN(-ENODEV); if (!lov->lov_tgts[index]) @@ -1004,12 +1004,12 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, memcpy(&flags, data->ioc_inlbuf1, sizeof(flags)); flags = flags & LL_STATFS_NODELAY ? OBD_STATFS_NODELAY : 0; - /* got statfs data */ - rc = obd_statfs(NULL, lov->lov_tgts[index]->ltd_exp, &stat_buf, + /* got statfs data */ + rc = obd_statfs(NULL, lov->lov_tgts[index]->ltd_exp, &stat_buf, ktime_get_seconds() - OBD_STATFS_CACHE_SECONDS, - flags); - if (rc) - RETURN(rc); + flags); + if (rc) + RETURN(rc); if (copy_to_user(data->ioc_pbuf1, &stat_buf, min_t(unsigned long, data->ioc_plen1, sizeof(struct obd_statfs)))) @@ -1122,12 +1122,11 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len, if (!lov->lov_tgts[i] || !lov->lov_tgts[i]->ltd_exp) continue; - /* ll_umount_begin() sets force flag but for lov, not - * osc. Let's pass it through */ - osc_obd = class_exp2obd(lov->lov_tgts[i]->ltd_exp); - osc_obd->obd_force = obddev->obd_force; - err = obd_iocontrol(cmd, lov->lov_tgts[i]->ltd_exp, - len, karg, uarg); + /* ll_umount_begin() sets force on lov, pass to osc */ + osc_obd = class_exp2obd(lov->lov_tgts[i]->ltd_exp); + osc_obd->obd_force = obd->obd_force; + err = obd_iocontrol(cmd, lov->lov_tgts[i]->ltd_exp, + len, karg, uarg); if (err) { if (lov->lov_tgts[i]->ltd_active) { CDEBUG(err == -ENOTTY ? diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 90c5cd1..a142cd6 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -3590,26 +3590,27 @@ test_49b() { # bug 17710 run_test 49b "check PARAM_SYS_LDLM_TIMEOUT option of mkfs.lustre" lazystatfs() { + # wait long enough to exceed OBD_STATFS_CACHE_SECONDS = 1 + sleep 2 # Test both statfs and lfs df and fail if either one fails multiop_bg_pause $1 f_ - RC1=$? + RC=$? PID=$! killall -USR1 multiop - [ $RC1 -ne 0 ] && log "lazystatfs multiop failed" - wait $PID || { RC1=$?; log "multiop return error "; } + [ $RC -ne 0 ] && log "lazystatfs multiop failed" + wait $PID || { RC=$?; log "multiop return error "; } + # wait long enough to exceed OBD_STATFS_CACHE_SECONDS = 1 + sleep 2 $LFS df -l & PID=$! sleep 5 - kill -s 0 $PID - RC2=$? - if [ $RC2 -eq 0 ]; then - kill -s 9 $PID - log "lazystatfs df failed" + if kill -s 0 $PID; then + RC=1 + kill -s 9 $PID + log "lazystatfs lfs df failed to complete in 5s" fi - RC=0 - [[ $RC1 -ne 0 || $RC2 -eq 0 ]] && RC=1 return $RC } @@ -3631,7 +3632,9 @@ test_50b() { # Wait for client to detect down OST stop_ost || error "Unable to stop OST1" - wait_osc_import_state mds ost DISCONN + wait_osc_import_state client ost DISCONN + $LCTL dl + log "OSCs should all be DISCONN" lazystatfs $MOUNT || error "lazystatfs should not return EIO" diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index bf5d3ce..db45030 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -4784,6 +4784,8 @@ static int mntdf(char *mntdir, char *fsname, char *pool, enum mntdf_flags flags, } for (tp = types; tp->st_name != NULL; tp++) { + bool have_ost = false; + if (!(tp->st_op & ops)) continue; @@ -4805,6 +4807,20 @@ static int mntdf(char *mntdir, char *fsname, char *pool, enum mntdf_flags flags, rc = rc2; } + /* If we have OSTs then don't report MDT block counts. + * For MDT-only filesystems the expectation is that all + * layouts have a DoM component. For filesystems with + * OSTs, files are not necessarily going to store data + * on MDTs, and MDT space is limited to a fraction of + * OST space, so don't include it in the summary. + */ + if (tp->st_op == LL_STATFS_LOV && !have_ost) { + have_ost = true; + sum.os_blocks = 0; + sum.os_bfree = 0; + sum.os_bavail = 0; + } + if (poolname && tp->st_op == LL_STATFS_LOV && llapi_search_ost(fsname, poolname, obd_uuid2str(&uuid_buf)) != 1) @@ -4828,20 +4844,21 @@ static int mntdf(char *mntdir, char *fsname, char *pool, enum mntdf_flags flags, obd_uuid2str(&uuid_buf), flags, tp->st_name, index, rc2); - if (rc2 == 0) { - if (tp->st_op == LL_STATFS_LMV) { - sum.os_ffree += stat_buf.os_ffree; - sum.os_files += stat_buf.os_files; - } else /* if (tp->st_op == LL_STATFS_LOV) */ { - sum.os_blocks += stat_buf.os_blocks * - stat_buf.os_bsize; - sum.os_bfree += stat_buf.os_bfree * - stat_buf.os_bsize; - sum.os_bavail += stat_buf.os_bavail * - stat_buf.os_bsize; - ost_ffree += stat_buf.os_ffree; - } - } + if (rc2) + continue; + + if (tp->st_op == LL_STATFS_LMV) { + sum.os_ffree += stat_buf.os_ffree; + sum.os_files += stat_buf.os_files; + } else /* if (tp->st_op == LL_STATFS_LOV) */ { + ost_ffree += stat_buf.os_ffree; + } + sum.os_blocks += stat_buf.os_blocks * + stat_buf.os_bsize; + sum.os_bfree += stat_buf.os_bfree * + stat_buf.os_bsize; + sum.os_bavail += stat_buf.os_bavail * + stat_buf.os_bsize; } } -- 1.8.3.1