LU-11721 lod: limit statfs ffree if less than OST ffree

If the OSTs report fewer total free objects than the MDTs, then
use the free files count reported by the OSTs, since it represents
the minimum number of files that can be created in the filesystem
(creating more may be possible, but this depends on other factors).
This has always been what ll_statfs_internal() reports, but the
statfs aggregation via the MDT missed this step in lod_statfs().

Fix a minor defect in sanity test_418() that would let it loop
forever until the test was killed due to timeout if the "df -i"
and "lfs df -i" output did not converge.

Fixes: b500d5193360 ("LU-10018 protocol: MDT as a statfs proxy")
Fixes: 263e80f4572b ("LU-11721 tests: wait for statfs to update ...")

Lustre-change: https://review.whamcloud.com/34167
Lustre-commit: a829595add808d0fb09bab525c500d0aa6955883

Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: Id8d7b7edfd854f1ec30bfbbb85f04b0c973ebbe5
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Nikitas Angelinas <nangelinas@cray.com>
Signed-off-by: Minh Diep <mdiep@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/34453
Tested-by: Jenkins
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h
index d2dbcd7..cd3f0b5 100644 (file)
--- a/lustre/include/obd_class.h
+++ b/lustre/include/obd_class.h
@@ -1007,8 +1007,9 @@ static inline int obd_statfs_async(struct obd_export *exp,
 
        obd = exp->exp_obd;
        if (!obd->obd_type || !obd->obd_type->typ_dt_ops->o_statfs) {
-               CERROR("%s: no %s operation\n", obd->obd_name, __func__);
-               RETURN(-EOPNOTSUPP);
+               rc = -EOPNOTSUPP;
+               CERROR("%s: no statfs operation: rc = %d\n", obd->obd_name, rc);
+               RETURN(rc);
        }
 
        CDEBUG(D_SUPER, "%s: age %lld, max_age %lld\n",

diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c
index 00110f3..da3d891 100644 (file)
--- a/lustre/llite/llite_lib.c
+++ b/lustre/llite/llite_lib.c
@@ -1800,57 +1800,54 @@ int ll_setattr(struct dentry *de, struct iattr *attr)
 int ll_statfs_internal(struct ll_sb_info *sbi, struct obd_statfs *osfs,
                       u32 flags)
 {
-       struct obd_statfs obd_osfs;
+       struct obd_statfs obd_osfs = { 0 };
        time64_t max_age;
        int rc;
 
        ENTRY;
        max_age = ktime_get_seconds() - OBD_STATFS_CACHE_SECONDS;
 
-        rc = obd_statfs(NULL, sbi->ll_md_exp, osfs, max_age, flags);
-        if (rc) {
-                CERROR("md_statfs fails: rc = %d\n", rc);
-                RETURN(rc);
-        }
+       rc = obd_statfs(NULL, sbi->ll_md_exp, osfs, max_age, flags);
+       if (rc)
+               RETURN(rc);
 
        osfs->os_type = LL_SUPER_MAGIC;
 
        CDEBUG(D_SUPER, "MDC blocks %llu/%llu objects %llu/%llu\n",
-               osfs->os_bavail, osfs->os_blocks, osfs->os_ffree,osfs->os_files);
+             osfs->os_bavail, osfs->os_blocks, osfs->os_ffree, osfs->os_files);
 
        if (osfs->os_state & OS_STATE_SUM)
                GOTO(out, rc);
 
-        if (sbi->ll_flags & LL_SBI_LAZYSTATFS)
-                flags |= OBD_STATFS_NODELAY;
+       if (sbi->ll_flags & LL_SBI_LAZYSTATFS)
+               flags |= OBD_STATFS_NODELAY;
 
        rc = obd_statfs(NULL, sbi->ll_dt_exp, &obd_osfs, max_age, flags);
-        if (rc) {
-                CERROR("obd_statfs fails: rc = %d\n", rc);
-                RETURN(rc);
-        }
+       if (rc) /* Possibly a filesystem with no OSTs.  Report MDT totals. */
+               GOTO(out, rc = 0);
 
        CDEBUG(D_SUPER, "OSC blocks %llu/%llu objects %llu/%llu\n",
-               obd_osfs.os_bavail, obd_osfs.os_blocks, obd_osfs.os_ffree,
-               obd_osfs.os_files);
-
-        osfs->os_bsize = obd_osfs.os_bsize;
-        osfs->os_blocks = obd_osfs.os_blocks;
-        osfs->os_bfree = obd_osfs.os_bfree;
-        osfs->os_bavail = obd_osfs.os_bavail;
-
-        /* If we don't have as many objects free on the OST as inodes
-         * on the MDS, we reduce the total number of inodes to
-         * compensate, so that the "inodes in use" number is correct.
-         */
-        if (obd_osfs.os_ffree < osfs->os_ffree) {
-                osfs->os_files = (osfs->os_files - osfs->os_ffree) +
-                        obd_osfs.os_ffree;
-                osfs->os_ffree = obd_osfs.os_ffree;
-        }
+              obd_osfs.os_bavail, obd_osfs.os_blocks, obd_osfs.os_ffree,
+              obd_osfs.os_files);
+
+       osfs->os_bsize = obd_osfs.os_bsize;
+       osfs->os_blocks = obd_osfs.os_blocks;
+       osfs->os_bfree = obd_osfs.os_bfree;
+       osfs->os_bavail = obd_osfs.os_bavail;
+
+       /* If we have _some_ OSTs, but don't have as many free objects on the
+        * OSTs as inodes on the MDTs, reduce the reported number of inodes
+        * to compensate, so that the "inodes in use" number is correct.
+        * This should be kept in sync with lod_statfs() behaviour.
+        */
+       if (obd_osfs.os_files && obd_osfs.os_ffree < osfs->os_ffree) {
+               osfs->os_files = (osfs->os_files - osfs->os_ffree) +
+                                obd_osfs.os_ffree;
+               osfs->os_ffree = obd_osfs.os_ffree;
+       }
 
 out:
-        RETURN(rc);
+       RETURN(rc);
 }
 int ll_statfs(struct dentry *de, struct kstatfs *sfs)
 {

diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c
index 31db16d..3337fb8 100644 (file)
--- a/lustre/lmv/lmv_obd.c
+++ b/lustre/lmv/lmv_obd.c
@@ -1410,8 +1410,8 @@ static int lmv_statfs(const struct lu_env *env, struct obd_export *exp,
                rc = obd_statfs(env, lmv->tgts[idx]->ltd_exp, temp,
                                max_age, flags);
                if (rc) {
-                       CERROR("can't stat MDS #%d (%s), error %d\n", i,
-                              lmv->tgts[idx]->ltd_exp->exp_obd->obd_name,
+                       CERROR("%s: can't stat MDS #%d: rc = %d\n",
+                              lmv->tgts[idx]->ltd_exp->exp_obd->obd_name, i,
                               rc);
                        GOTO(out_free_temp, rc);
                }

diff --git a/lustre/lod/lod_dev.c b/lustre/lod/lod_dev.c
index 94c373c..96e0e2a 100644 (file)
--- a/lustre/lod/lod_dev.c
+++ b/lustre/lod/lod_dev.c
@@ -1327,8 +1327,9 @@ static int lod_statfs(const struct lu_env *env,
        struct lod_ost_desc *ost;
        struct lod_mdt_desc *mdt;
        struct obd_statfs ost_sfs;
+       u64 ost_files = 0;
+       u64 ost_ffree = 0;
        int i, rc, bs;
-       bool mdtonly;
 
        rc = dt_statfs(env, dt2lod_dev(dev)->lod_child, sfs);
        if (rc)
@@ -1360,8 +1361,6 @@ static int lod_statfs(const struct lu_env *env,
         * decide how to account MDT space. for simplicity let's
         * just fallback to pre-DoM policy if any OST is alive
         */
-       mdtonly = true;
-
        lod_getref(&lod->lod_ost_descs);
        lod_foreach_ost(lod, i) {
                ost = OST_TGT(lod, i);
@@ -1370,17 +1369,18 @@ static int lod_statfs(const struct lu_env *env,
                /* ignore errors */
                if (rc || ost_sfs.os_bsize == 0)
                        continue;
-               if (mdtonly) {
+               if (!ost_files) {
                        /*
-                        * if only MDTs and DoM report MDT space,
-                        * otherwise only OST space
+                        * if only MDTs with DoM then report only MDT blocks,
+                        * otherwise show only OST blocks, and DoM is "free"
                         */
                        sfs->os_bavail = 0;
                        sfs->os_blocks = 0;
                        sfs->os_bfree = 0;
                        sfs->os_granted = 0;
-                       mdtonly = false;
                }
+               ost_files += sfs->os_files;
+               ost_ffree += sfs->os_ffree;
                ost_sfs.os_bavail += ost_sfs.os_granted;
                lod_statfs_sum(sfs, &ost_sfs, &bs);
                LASSERTF(bs == ost_sfs.os_bsize, "%d != %d\n",
@@ -1389,6 +1389,16 @@ static int lod_statfs(const struct lu_env *env,
        lod_putref(lod, &lod->lod_ost_descs);
        sfs->os_state |= OS_STATE_SUM;
 
+       /* If we have _some_ OSTs, but don't have as many free objects on the
+        * OSTs as inodes on the MDTs, reduce the reported number of inodes
+        * to compensate, so that the "inodes in use" number is correct.
+        * This should be kept in sync with ll_statfs_internal().
+        */
+       if (ost_files && ost_ffree < sfs->os_ffree) {
+               sfs->os_files = (sfs->os_files - sfs->os_ffree) + ost_ffree;
+               sfs->os_ffree = ost_ffree;
+       }
+
        /* a single successful statfs should be enough */
        rc = 0;
 

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 3d444f5..1fbb9c3 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -19517,40 +19517,40 @@ check_lfs_df() {
        local inodes
        local df_out
        local lfs_df_out
-       local tries=100
-       local count=0
+       local count
        local passed=false
 
        # blocks or inodes
        [ "$1" == "blocks" ] && inodes= || inodes="-i"
 
-       while (( count < tries )); do
+       for count in {1..100}; do
                cancel_lru_locks
-               sync; sleep 0.2
+               sync; sleep 2
 
                # read the lines of interest
-               df_out=($(df $inodes $dir | tail -n +2)) ||
+               df_out=($(df -P $inodes $dir | tail -n +2)) ||
                        error "df $inodes $dir | tail -n +2 failed"
                lfs_df_out=($($LFS df $inodes $dir | grep summary:)) ||
                        error "lfs df $inodes $dir | grep summary: failed"
 
                # skip first substrings of each output as they are different
-               # <NID>:/<fsname for df, filesystem_summary: for lfs df
-               df_out=(${df_out[@]:1})
-               lfs_df_out=(${lfs_df_out[@]:1})
-
-               # compare the two outputs
+               # "<NID>:/<fsname>" for df, "filesystem_summary:" for lfs df
+               # compare the remaining fields of the two outputs
                passed=true
-
-               for i in {0..4}; do
+               for i in {1..5}; do
                        [ "${df_out[i]}" != "${lfs_df_out[i]}" ] && passed=false
                done
                $passed && break
        done
 
-       $passed || error "df and lfs df $1 output mismatch: "   \
-                        "df ${inodes}: ${df_out[*]}, "         \
-                        "lfs df ${inodes}: ${lfs_df_out[*]}"
+       if ! $passed; then
+               df -P $inodes $dir
+               echo
+               lfs df $inodes $dir
+               error "df and lfs df $1 output mismatch: "      \
+                     "df ${inodes}: ${df_out[*]}, "            \
+                     "lfs df ${inodes}: ${lfs_df_out[*]}"
+       fi
 }
 
 test_418() {