LU-1816 scrub: OI scrub skips new created objects for once

For accelerating OI scrub, it will skip the objects which are new
created after the latest MDT mount and marked as I_LUSTRE_NOSCRUB.
But if the object with I_LUSTRE_NOSCRUB is used/cached in RAM for
very long time, then OI scrub has no chance to process it even if
there has been inconsistency happened until the MDT remount.

So as compromise, the OI scrub will skip the new created objects
for once only when the first accessing.

Signed-off-by: Fan Yong <yong.fan@whamcloud.com>
Change-Id: I560d7d851e3430326d24ec74154bb2a99f80b737
Reviewed-on: http://review.whamcloud.com/3848
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Mike Pershin <tappro@whamcloud.com>

diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c
index 81c00e6..db890c7 100644 (file)
--- a/lustre/osd-ldiskfs/osd_handler.c
+++ b/lustre/osd-ldiskfs/osd_handler.c
@@ -431,8 +431,6 @@ static int osd_fid_lookup(const struct lu_env *env, struct osd_object *obj,
                        verify = 1;
        }
 
-       fid_zero(&oic->oic_fid);
-
        /*
         * Objects are created as locking anchors or place holders for objects
         * yet to be created. No need to osd_oi_lookup() at here because FID
@@ -465,6 +463,7 @@ iget:
        if (IS_ERR(inode)) {
                result = PTR_ERR(inode);
                if (result == -ENOENT || result == -ESTALE) {
+                       fid_zero(&oic->oic_fid);
                        result = 0;
                } else if (result == -EREMCHG) {
 
@@ -3414,9 +3413,10 @@ static int osd_ea_lookup_rec(const struct lu_env *env, struct osd_object *obj,
                        rc = osd_ea_fid_get(env, obj, ino, fid, &oic->oic_lid);
                else
                        osd_id_gen(&oic->oic_lid, ino, OSD_OII_NOGEN);
-
-               if (rc != 0 || !fid_is_norm(fid))
+               if (rc != 0 || !fid_is_norm(fid)) {
+                       fid_zero(&oic->oic_fid);
                        GOTO(out, rc);
+               }
 
                oic->oic_fid = *fid;
                if ((scrub->os_pos_current <= ino) &&
@@ -4130,9 +4130,10 @@ static inline int osd_it_ea_rec(const struct lu_env *env,
                           it->oie_dirent->oied_name,
                           it->oie_dirent->oied_namelen,
                           it->oie_dirent->oied_type, attr);
-
-       if (!fid_is_norm(fid))
+       if (!fid_is_norm(fid)) {
+               fid_zero(&oic->oic_fid);
                RETURN(0);
+       }
 
        oic->oic_fid = *fid;
        if ((scrub->os_pos_current <= ino) &&

diff --git a/lustre/osd-ldiskfs/osd_scrub.c b/lustre/osd-ldiskfs/osd_scrub.c
index 5c9a290..65157f4 100644 (file)
--- a/lustre/osd-ldiskfs/osd_scrub.c
+++ b/lustre/osd-ldiskfs/osd_scrub.c
@@ -157,6 +157,8 @@ void osd_scrub_file_reset(struct osd_scrub *scrub, __u8 *uuid, __u64 flags)
        sf->sf_items_updated = 0;
        sf->sf_items_failed = 0;
        sf->sf_items_updated_prior = 0;
+       sf->sf_items_noscrub = 0;
+       sf->sf_items_igif = 0;
 }
 
 static int osd_scrub_file_load(struct osd_scrub *scrub)
@@ -500,6 +502,8 @@ static void osd_scrub_post(struct osd_scrub *scrub, int result)
 #define SCRUB_NEXT_WAIT        4 /* wait for free cache slot */
 #define SCRUB_NEXT_CRASH       5 /* simulate system crash during OI scrub */
 #define SCRUB_NEXT_FATAL       6 /* simulate failure during OI scrub */
+#define SCRUB_NEXT_NOSCRUB     7 /* new created object, no scrub on it */
+#define SCRUB_NEXT_IGIF        8 /* IGIF object */
 
 struct osd_iit_param {
        struct super_block *sb;
@@ -624,8 +628,14 @@ static int osd_scrub_next(struct osd_thread_info *info, struct osd_device *dev,
        if (rc != 0)
                return rc;
 
-       if (!fid_is_norm(fid) || inode->i_state & I_LUSTRE_NOSCRUB)
-               rc = SCRUB_NEXT_CONTINUE;
+       if (inode->i_state & I_LUSTRE_NOSCRUB) {
+               /* Only skip it for the first OI scrub accessing. */
+               inode->i_state &= ~I_LUSTRE_NOSCRUB;
+               rc = SCRUB_NEXT_NOSCRUB;
+       } else if (!fid_is_norm(fid)) {
+               rc = SCRUB_NEXT_IGIF;
+       }
+
        iput(inode);
        return rc;
 }
@@ -668,6 +678,8 @@ static int osd_scrub_exec(struct osd_thread_info *info, struct osd_device *dev,
 {
        struct l_wait_info       lwi    = { 0 };
        struct osd_scrub        *scrub  = &dev->od_scrub;
+       struct scrub_file       *sf     = &scrub->os_file;
+       __u64                   *items  = NULL;
        struct ptlrpc_thread    *thread = &scrub->os_thread;
        struct osd_otable_it    *it     = dev->od_otable_it;
        struct osd_otable_cache *ooc    = it ? &it->ooi_cache : NULL;
@@ -677,6 +689,20 @@ static int osd_scrub_exec(struct osd_thread_info *info, struct osd_device *dev,
                goto next;
        case SCRUB_NEXT_WAIT:
                goto wait;
+       case SCRUB_NEXT_NOSCRUB:
+               items = &sf->sf_items_noscrub;
+               break;
+       case SCRUB_NEXT_IGIF:
+               items = &sf->sf_items_igif;
+               break;
+       }
+
+       if (items != NULL) {
+               cfs_down_write(&scrub->os_rwsem);
+               scrub->os_new_checked++;
+               (*items)++;
+               cfs_up_write(&scrub->os_rwsem);
+               goto next;
        }
 
        LASSERTF(rc <= 0, "unexpected rc = %d\n", rc);
@@ -1569,9 +1595,12 @@ int osd_scrub_dump(struct osd_device *dev, char *buf, int len)
                      "updated: "LPU64"\n"
                      "failed: "LPU64"\n"
                      "prior_updated: "LPU64"\n"
+                     "noscrub: "LPU64"\n"
+                     "igif: "LPU64"\n"
                      "success_count: %u\n",
                      checked, sf->sf_items_updated, sf->sf_items_failed,
-                     sf->sf_items_updated_prior, sf->sf_success_count);
+                     sf->sf_items_updated_prior, sf->sf_items_noscrub,
+                     sf->sf_items_igif, sf->sf_success_count);
        if (rc <= 0)
                goto out;
 

diff --git a/lustre/osd-ldiskfs/osd_scrub.h b/lustre/osd-ldiskfs/osd_scrub.h
index 353c590..c3e54b3 100644 (file)
--- a/lustre/osd-ldiskfs/osd_scrub.h
+++ b/lustre/osd-ldiskfs/osd_scrub.h
@@ -140,6 +140,12 @@ struct scrub_file {
        /* How many prior objects have been updated during scanning. */
        __u64   sf_items_updated_prior;
 
+       /* How many objects marked as I_LUSTRE_NOSCRUB. */
+       __u64   sf_items_noscrub;
+
+       /* How many IGIF objects. */
+       __u64   sf_items_igif;
+
        /* How long the OI scrub has run. */
        __u32   sf_run_time;
 

diff --git a/lustre/tests/sanity-scrub.sh b/lustre/tests/sanity-scrub.sh
index 1ce2737..ea517ac 100644 (file)
--- a/lustre/tests/sanity-scrub.sh
+++ b/lustre/tests/sanity-scrub.sh
@@ -675,6 +675,46 @@ test_10b() {
 }
 run_test 10b "non-stopped OI scrub should auto restarts after MDS remount (2)"
 
+test_11() {
+       echo "stopall"
+       stopall > /dev/null
+       echo "setupall"
+       setupall > /dev/null
+
+       local tname=`date +%s`
+       rm -rf $MOUNT/$tname > /dev/null
+       mkdir $MOUNT/$tname || error "(1) Fail to mkdir $MOUNT/$tname"
+
+       createmany -o $MOUNT/$tname/f 100 || error "(2) Fail to create!"
+
+       # reset OI scrub start point by force
+       $START_SCRUB -r || error "(3) Fail to start OI scrub!"
+       sleep 3
+       local STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+       [ "$STATUS" == "completed" ] ||
+               error "(4) Expect 'completed', but got '$STATUS'"
+
+       # OI scrub should skip the new created objects for the first accessing
+       local SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
+       [ $SKIPPED -eq 101 ] ||
+               error "(5) Expect 101 objects skipped, but got $SKIPPED"
+
+       # reset OI scrub start point by force
+       $START_SCRUB -r || error "(6) Fail to start OI scrub!"
+       sleep 3
+       STATUS=$($SHOW_SCRUB | awk '/^status/ { print $2 }')
+       [ "$STATUS" == "completed" ] ||
+               error "(7) Expect 'completed', but got '$STATUS'"
+
+       # OI scrub should skip the new created object only once
+       SKIPPED=$($SHOW_SCRUB | awk '/^noscrub/ { print $2 }')
+       [ $SKIPPED -eq 0 ] ||
+               error "(8) Expect 0 objects skipped, but got $SKIPPED"
+
+       rm -rf $MOUNT/$tname > /dev/null
+}
+run_test 11 "OI scrub skips the new created objects only once"
+
 # restore the ${facet}_MKFS_OPTS variables
 for facet in MGS MDS OST; do
        opts=SAVED_${facet}_MKFS_OPTS