LU-9836 osd-ldiskfs: read directory completely

For ldiskfs backend, the return of readdir() does NOT means
the whole directory being read. Instead, it is the caller's
duty to count whether there are new items read via the last
readdir() then determine whether or not the whole directroy
has been read.

Unfortunately, some old osd-ldiskfs logic, such as OI scrub,
did not handle that properly, as to some directory, such as
lost+found, may be partly scanned. That is why some orphans
cannot be recovered.

Lustre-change: https://review.whamcloud.com/30770
Lustre-commit: 1e2cd1c7025080879a27f9ad9a3896fd3e0e8753

Signed-off-by: Fan Yong <fan.yong@intel.com>
Change-Id: Ib328643c4cdcdb14b548807ed05e8835f80bbf6a
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Lai Siyao <lai.siyao@intel.com>
Signed-off-by: Minh Diep <minh.diep@intel.com>
Reviewed-on: https://review.whamcloud.com/31019
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: John L. Hammond <john.hammond@intel.com>

diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c
index cf8e476..6877fff 100644 (file)
--- a/lustre/osd-ldiskfs/osd_handler.c
+++ b/lustre/osd-ldiskfs/osd_handler.c
@@ -801,6 +801,8 @@ struct osd_check_lmv_buf {
        struct osd_thread_info  *oclb_info;
        struct osd_device       *oclb_dev;
        struct osd_idmap_cache  *oclb_oic;
+       int                      oclb_items;
+       bool                     oclb_found;
 };
 
 /**
@@ -826,7 +828,8 @@ static int osd_stripe_dir_filldir(void *buf,
        struct osd_device *dev = oclb->oclb_dev;
        struct osd_idmap_cache *oic = oclb->oclb_oic;
        struct inode *inode;
-       int rc;
+
+       oclb->oclb_items++;
 
        if (name[0] == '.')
                return 0;
@@ -849,9 +852,10 @@ static int osd_stripe_dir_filldir(void *buf,
        oic->oic_fid = *fid;
        oic->oic_lid = *id;
        oic->oic_dev = dev;
-       rc = osd_oii_insert(dev, oic, true);
+       osd_oii_insert(dev, oic, true);
+       oclb->oclb_found = true;
 
-       return rc == 0 ? 1 : rc;
+       return 1;
 }
 
 /* When lookup item under striped directory, we need to locate the master
@@ -902,7 +906,8 @@ static int osd_check_lmv(struct osd_thread_info *oti, struct osd_device *dev,
 #endif
                .oclb_info = oti,
                .oclb_dev = dev,
-               .oclb_oic = oic
+               .oclb_oic = oic,
+               .oclb_found = false,
        };
        int rc = 0;
        ENTRY;
@@ -950,13 +955,17 @@ again:
        filp->private_data = NULL;
        set_file_inode(filp, inode);
 
+       do {
+               oclb.oclb_items = 0;
 #ifdef HAVE_DIR_CONTEXT
-       oclb.ctx.pos = filp->f_pos;
-       rc = fops->iterate(filp, &oclb.ctx);
-       filp->f_pos = oclb.ctx.pos;
+               oclb.ctx.pos = filp->f_pos;
+               rc = fops->iterate(filp, &oclb.ctx);
+               filp->f_pos = oclb.ctx.pos;
 #else
-       rc = fops->readdir(filp, &oclb, osd_stripe_dir_filldir);
+               rc = fops->readdir(filp, &oclb, osd_stripe_dir_filldir);
 #endif
+       } while (rc >= 0 && oclb.oclb_items > 0 && !oclb.oclb_found &&
+                filp->f_pos != LDISKFS_HTREE_EOF_64BIT);
        fops->release(inode, filp);
 
 out:

diff --git a/lustre/osd-ldiskfs/osd_scrub.c b/lustre/osd-ldiskfs/osd_scrub.c
index c94a3b0..3236c0a 100644 (file)
--- a/lustre/osd-ldiskfs/osd_scrub.c
+++ b/lustre/osd-ldiskfs/osd_scrub.c
@@ -1943,6 +1943,7 @@ struct osd_ios_filldir_buf {
        struct osd_thread_info  *oifb_info;
        struct osd_device       *oifb_dev;
        struct dentry           *oifb_dentry;
+       int                      oifb_items;
 };
 
 static inline struct dentry *
@@ -2104,6 +2105,8 @@ static int osd_ios_lf_fill(void *buf,
        int                         rc;
        ENTRY;
 
+       fill_buf->oifb_items++;
+
        /* skip any '.' started names */
        if (name[0] == '.')
                RETURN(0);
@@ -2172,6 +2175,8 @@ static int osd_ios_varfid_fill(void *buf,
        int                         rc;
        ENTRY;
 
+       fill_buf->oifb_items++;
+
        /* skip any '.' started names */
        if (name[0] == '.')
                RETURN(0);
@@ -2206,6 +2211,8 @@ static int osd_ios_dl_fill(void *buf,
        int                         rc       = 0;
        ENTRY;
 
+       fill_buf->oifb_items++;
+
        /* skip any '.' started names */
        if (name[0] == '.')
                RETURN(0);
@@ -2247,6 +2254,8 @@ static int osd_ios_uld_fill(void *buf,
        int                         rc       = 0;
        ENTRY;
 
+       fill_buf->oifb_items++;
+
        /* skip any non-DFID format name */
        if (name[0] != '[')
                RETURN(0);
@@ -2283,6 +2292,8 @@ static int osd_ios_root_fill(void *buf,
        int                         rc       = 0;
        ENTRY;
 
+       fill_buf->oifb_items++;
+
        /* skip any '.' started names */
        if (name[0] == '.')
                RETURN(0);
@@ -2340,13 +2351,17 @@ osd_ios_general_scan(struct osd_thread_info *info, struct osd_device *dev,
        filp->private_data = NULL;
        set_file_inode(filp, inode);
 
+       do {
+               buf.oifb_items = 0;
 #ifdef HAVE_DIR_CONTEXT
-       buf.ctx.pos = filp->f_pos;
-       rc = fops->iterate(filp, &buf.ctx);
-       filp->f_pos = buf.ctx.pos;
+               buf.ctx.pos = filp->f_pos;
+               rc = fops->iterate(filp, &buf.ctx);
+               filp->f_pos = buf.ctx.pos;
 #else
-       rc = fops->readdir(filp, &buf, filldir);
+               rc = fops->readdir(filp, &buf, filldir);
 #endif
+       } while (rc >= 0 && buf.oifb_items > 0 &&
+                filp->f_pos != LDISKFS_HTREE_EOF_64BIT);
        fops->release(inode, filp);
 
        RETURN(rc);

diff --git a/lustre/tests/sanity-scrub.sh b/lustre/tests/sanity-scrub.sh
index 87b6f82..a90a566 100644 (file)
--- a/lustre/tests/sanity-scrub.sh
+++ b/lustre/tests/sanity-scrub.sh
@@ -1106,7 +1106,7 @@ test_14() {
        do_facet ost1 $LCTL set_param fail_loc=0x196
        local count=$(precreated_ost_obj_count 0 0)
 
-       createmany -o $DIR/$tdir/f $((count + 32))
+       createmany -o $DIR/$tdir/f $((count + 1000))
        do_facet ost1 $LCTL set_param fail_loc=0
 
        umount_client $MOUNT || error "(1) Fail to stop client!"
@@ -1124,7 +1124,7 @@ test_14() {
 
        local LF_REPAIRED=$($SHOW_SCRUB_ON_OST |
                            awk '/^lf_repa[ri]*ed/ { print $2 }')
-       [ $LF_REPAIRED -gt 0 ] ||
+       [ $LF_REPAIRED -ge 1000 ] ||
                error "(6) Some entry under /lost+found should be repaired"
 
        ls -ail $DIR/$tdir > /dev/null || error "(7) ls should succeed"