From a829595add808d0fb09bab525c500d0aa6955883 Mon Sep 17 00:00:00 2001 From: Andreas Dilger Date: Sat, 2 Feb 2019 17:11:00 -0700 Subject: [PATCH] LU-11721 lod: limit statfs ffree if less than OST ffree If the OSTs report fewer total free objects than the MDTs, then use the free files count reported by the OSTs, since it represents the minimum number of files that can be created in the filesystem (creating more may be possible, but this depends on other factors). This has always been what ll_statfs_internal() reports, but the statfs aggregation via the MDT missed this step in lod_statfs(). Fix a minor defect in sanity test_418() that would let it loop forever until the test was killed due to timeout if the "df -i" and "lfs df -i" output did not converge. Fixes: b500d5193360 ("LU-10018 protocol: MDT as a statfs proxy") Fixes: 263e80f4572b ("LU-11721 tests: wait for statfs to update ...") Signed-off-by: Andreas Dilger Change-Id: Id8d7b7edfd854f1ec30bfbbb85f04b0c973ebbe5 Reviewed-on: https://review.whamcloud.com/34167 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Jian Yu Reviewed-by: Nikitas Angelinas Reviewed-by: Oleg Drokin --- lustre/include/obd_class.h | 5 ++-- lustre/llite/llite_lib.c | 59 ++++++++++++++++++++++------------------------ lustre/lmv/lmv_obd.c | 4 ++-- lustre/lod/lod_dev.c | 24 +++++++++++++------ lustre/tests/sanity.sh | 26 ++++++++++---------- 5 files changed, 63 insertions(+), 55 deletions(-) diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index f0fcedb..9ad5ec8 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -1020,8 +1020,9 @@ static inline int obd_statfs_async(struct obd_export *exp, obd = exp->exp_obd; if (!obd->obd_type || !obd->obd_type->typ_dt_ops->o_statfs) { - CERROR("%s: no %s operation\n", obd->obd_name, __func__); - RETURN(-EOPNOTSUPP); + rc = -EOPNOTSUPP; + CERROR("%s: no statfs operation: rc = %d\n", obd->obd_name, rc); + RETURN(rc); } CDEBUG(D_SUPER, "%s: age %lld, max_age %lld\n", diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index 9766890..bf17554 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -1797,57 +1797,54 @@ int ll_setattr(struct dentry *de, struct iattr *attr) int ll_statfs_internal(struct ll_sb_info *sbi, struct obd_statfs *osfs, u32 flags) { - struct obd_statfs obd_osfs; + struct obd_statfs obd_osfs = { 0 }; time64_t max_age; int rc; ENTRY; max_age = ktime_get_seconds() - OBD_STATFS_CACHE_SECONDS; - rc = obd_statfs(NULL, sbi->ll_md_exp, osfs, max_age, flags); - if (rc) { - CERROR("md_statfs fails: rc = %d\n", rc); - RETURN(rc); - } + rc = obd_statfs(NULL, sbi->ll_md_exp, osfs, max_age, flags); + if (rc) + RETURN(rc); osfs->os_type = LL_SUPER_MAGIC; CDEBUG(D_SUPER, "MDC blocks %llu/%llu objects %llu/%llu\n", - osfs->os_bavail, osfs->os_blocks, osfs->os_ffree,osfs->os_files); + osfs->os_bavail, osfs->os_blocks, osfs->os_ffree, osfs->os_files); if (osfs->os_state & OS_STATE_SUM) GOTO(out, rc); - if (sbi->ll_flags & LL_SBI_LAZYSTATFS) - flags |= OBD_STATFS_NODELAY; + if (sbi->ll_flags & LL_SBI_LAZYSTATFS) + flags |= OBD_STATFS_NODELAY; rc = obd_statfs(NULL, sbi->ll_dt_exp, &obd_osfs, max_age, flags); - if (rc) { - CERROR("obd_statfs fails: rc = %d\n", rc); - RETURN(rc); - } + if (rc) /* Possibly a filesystem with no OSTs. Report MDT totals. */ + GOTO(out, rc = 0); CDEBUG(D_SUPER, "OSC blocks %llu/%llu objects %llu/%llu\n", - obd_osfs.os_bavail, obd_osfs.os_blocks, obd_osfs.os_ffree, - obd_osfs.os_files); - - osfs->os_bsize = obd_osfs.os_bsize; - osfs->os_blocks = obd_osfs.os_blocks; - osfs->os_bfree = obd_osfs.os_bfree; - osfs->os_bavail = obd_osfs.os_bavail; - - /* If we don't have as many objects free on the OST as inodes - * on the MDS, we reduce the total number of inodes to - * compensate, so that the "inodes in use" number is correct. - */ - if (obd_osfs.os_ffree < osfs->os_ffree) { - osfs->os_files = (osfs->os_files - osfs->os_ffree) + - obd_osfs.os_ffree; - osfs->os_ffree = obd_osfs.os_ffree; - } + obd_osfs.os_bavail, obd_osfs.os_blocks, obd_osfs.os_ffree, + obd_osfs.os_files); + + osfs->os_bsize = obd_osfs.os_bsize; + osfs->os_blocks = obd_osfs.os_blocks; + osfs->os_bfree = obd_osfs.os_bfree; + osfs->os_bavail = obd_osfs.os_bavail; + + /* If we have _some_ OSTs, but don't have as many free objects on the + * OSTs as inodes on the MDTs, reduce the reported number of inodes + * to compensate, so that the "inodes in use" number is correct. + * This should be kept in sync with lod_statfs() behaviour. + */ + if (obd_osfs.os_files && obd_osfs.os_ffree < osfs->os_ffree) { + osfs->os_files = (osfs->os_files - osfs->os_ffree) + + obd_osfs.os_ffree; + osfs->os_ffree = obd_osfs.os_ffree; + } out: - RETURN(rc); + RETURN(rc); } int ll_statfs(struct dentry *de, struct kstatfs *sfs) { diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c index 19bfed4..cc0fd5d 100644 --- a/lustre/lmv/lmv_obd.c +++ b/lustre/lmv/lmv_obd.c @@ -1410,8 +1410,8 @@ static int lmv_statfs(const struct lu_env *env, struct obd_export *exp, rc = obd_statfs(env, lmv->tgts[idx]->ltd_exp, temp, max_age, flags); if (rc) { - CERROR("can't stat MDS #%d (%s), error %d\n", i, - lmv->tgts[idx]->ltd_exp->exp_obd->obd_name, + CERROR("%s: can't stat MDS #%d: rc = %d\n", + lmv->tgts[idx]->ltd_exp->exp_obd->obd_name, i, rc); GOTO(out_free_temp, rc); } diff --git a/lustre/lod/lod_dev.c b/lustre/lod/lod_dev.c index 94c373c..96e0e2a 100644 --- a/lustre/lod/lod_dev.c +++ b/lustre/lod/lod_dev.c @@ -1327,8 +1327,9 @@ static int lod_statfs(const struct lu_env *env, struct lod_ost_desc *ost; struct lod_mdt_desc *mdt; struct obd_statfs ost_sfs; + u64 ost_files = 0; + u64 ost_ffree = 0; int i, rc, bs; - bool mdtonly; rc = dt_statfs(env, dt2lod_dev(dev)->lod_child, sfs); if (rc) @@ -1360,8 +1361,6 @@ static int lod_statfs(const struct lu_env *env, * decide how to account MDT space. for simplicity let's * just fallback to pre-DoM policy if any OST is alive */ - mdtonly = true; - lod_getref(&lod->lod_ost_descs); lod_foreach_ost(lod, i) { ost = OST_TGT(lod, i); @@ -1370,17 +1369,18 @@ static int lod_statfs(const struct lu_env *env, /* ignore errors */ if (rc || ost_sfs.os_bsize == 0) continue; - if (mdtonly) { + if (!ost_files) { /* - * if only MDTs and DoM report MDT space, - * otherwise only OST space + * if only MDTs with DoM then report only MDT blocks, + * otherwise show only OST blocks, and DoM is "free" */ sfs->os_bavail = 0; sfs->os_blocks = 0; sfs->os_bfree = 0; sfs->os_granted = 0; - mdtonly = false; } + ost_files += sfs->os_files; + ost_ffree += sfs->os_ffree; ost_sfs.os_bavail += ost_sfs.os_granted; lod_statfs_sum(sfs, &ost_sfs, &bs); LASSERTF(bs == ost_sfs.os_bsize, "%d != %d\n", @@ -1389,6 +1389,16 @@ static int lod_statfs(const struct lu_env *env, lod_putref(lod, &lod->lod_ost_descs); sfs->os_state |= OS_STATE_SUM; + /* If we have _some_ OSTs, but don't have as many free objects on the + * OSTs as inodes on the MDTs, reduce the reported number of inodes + * to compensate, so that the "inodes in use" number is correct. + * This should be kept in sync with ll_statfs_internal(). + */ + if (ost_files && ost_ffree < sfs->os_ffree) { + sfs->os_files = (sfs->os_files - sfs->os_ffree) + ost_ffree; + sfs->os_ffree = ost_ffree; + } + /* a single successful statfs should be enough */ rc = 0; diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 0b7eea3..9c8714b 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -19525,40 +19525,40 @@ check_lfs_df() { local inodes local df_out local lfs_df_out - local tries=100 - local count=0 + local count local passed=false # blocks or inodes [ "$1" == "blocks" ] && inodes= || inodes="-i" - while (( count < tries )); do + for count in {1..100}; do cancel_lru_locks sync; sleep 0.2 # read the lines of interest - df_out=($(df $inodes $dir | tail -n +2)) || + df_out=($(df -P $inodes $dir | tail -n +2)) || error "df $inodes $dir | tail -n +2 failed" lfs_df_out=($($LFS df $inodes $dir | grep summary:)) || error "lfs df $inodes $dir | grep summary: failed" # skip first substrings of each output as they are different - # :/:/" for df, "filesystem_summary:" for lfs df # compare the two outputs passed=true - - for i in {0..4}; do + for i in {1..5}; do [ "${df_out[i]}" != "${lfs_df_out[i]}" ] && passed=false done $passed && break done - $passed || error "df and lfs df $1 output mismatch: " \ - "df ${inodes}: ${df_out[*]}, " \ - "lfs df ${inodes}: ${lfs_df_out[*]}" + if ! $passed; then + df -P $inodes $dir + echo + lfs df $inodes $dir + error "df and lfs df $1 output mismatch: " \ + "df ${inodes}: ${df_out[*]}, " \ + "lfs df ${inodes}: ${lfs_df_out[*]}" + fi } test_418() { -- 1.8.3.1