LU-6895 scrub: not trigger scrub if inode removed by race

When osd_consistency_check(), the target file may has just been
removed by other, so the osd_oi_lookup() will return -ENOENT to
the caller. Under such case, the osd_consistency_check() should
not trigger OI scrub.

On the other hand, if someone unlinked the file during OI scrub
adding the missed OI mapping to the OI file, the OI scrub needs
to remove the new added OI mapping.

Test-Parameters: alwaysuploadlogs \
envdefinitions=SLOW=yes,ENABLE_QUOTA=yes \
mdtfilesystemtype=ldiskfs mdsfilesystemtype=ldiskfs \
ostfilesystemtype=ldiskfs clientdistro=el7 ossdistro=el7 \
mdsdistro=el7 mdtcount=1 \
testlist=sanity-scrub,sanity-scrub,sanity-scrub,sanity-scrub

Signed-off-by: Fan Yong <fan.yong@intel.com>
Change-Id: I4703fc7f99d7b0a0f769127b5cdba5a2b992250d
Reviewed-on: http://review.whamcloud.com/16439
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Alex Zhuravlev <alexey.zhuravlev@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/osd-ldiskfs/osd_compat.c b/lustre/osd-ldiskfs/osd_compat.c
index bf317d0..2d78901 100644 (file)
--- a/lustre/osd-ldiskfs/osd_compat.c
+++ b/lustre/osd-ldiskfs/osd_compat.c
@@ -566,22 +566,19 @@ static int osd_obj_update_entry(struct osd_thread_info *info,
        rc = osd_get_lma(info, inode, dentry, lma);
        if (rc == -ENODATA) {
                rc = osd_get_idif(info, inode, dentry, oi_fid);
-               if (rc > 0) {
+               if (rc > 0 || rc == -ENODATA) {
                        oi_fid = NULL;
                        rc = 0;
                }
        }
        iput(inode);
 
-       /* If the OST-object has neither FID-in-LMA nor FID-in-ff, it is
-        * either a crashed object or a uninitialized one. Replace it. */
-       if (rc == -ENODATA || oi_fid == NULL)
-               goto update;
-
        if (rc != 0)
                GOTO(out, rc);
 
-       if (lu_fid_eq(fid, oi_fid)) {
+       /* If the OST-object has neither FID-in-LMA nor FID-in-ff, it is
+        * either a crashed object or a uninitialized one. Replace it. */
+       if (oi_fid != NULL && lu_fid_eq(fid, oi_fid)) {
                CERROR("%s: the FID "DFID" is used by two objects: "
                       "%u/%u %u/%u\n", osd_name(osd), PFID(fid),
                       oi_id->oii_ino, oi_id->oii_gen,

diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c
index d6c2f86..d62023d 100644 (file)
--- a/lustre/osd-ldiskfs/osd_handler.c
+++ b/lustre/osd-ldiskfs/osd_handler.c
@@ -270,6 +270,7 @@ struct inode *osd_iget(struct osd_thread_info *info, struct osd_device *dev,
                iput(inode);
                inode = ERR_PTR(-ENOENT);
        } else {
+               ldiskfs_clear_inode_state(inode, LDISKFS_STATE_LUSTRE_DESTROY);
                if (id->oii_gen == OSD_OII_NOGEN)
                        osd_id_gen(id, inode->i_ino, inode->i_generation);
 
@@ -374,6 +375,8 @@ static struct inode *osd_iget_check(struct osd_thread_info *info,
                goto check_oi;
        }
 
+       ldiskfs_clear_inode_state(inode, LDISKFS_STATE_LUSTRE_DESTROY);
+
 check_oi:
        if (rc != 0) {
                LASSERTF(rc == -ESTALE || rc == -ENOENT, "rc = %d\n", rc);
@@ -2523,6 +2526,7 @@ static int osd_object_destroy(const struct lu_env *env,
 
        osd_trans_exec_op(env, th, OSD_OT_DESTROY);
 
+       ldiskfs_set_inode_state(inode, LDISKFS_STATE_LUSTRE_DESTROY);
        result = osd_oi_delete(osd_oti_get(env), osd, fid, oh->ot_handle,
                               OI_CHECK_FLD);
 
@@ -4070,7 +4074,7 @@ static int osd_ea_add_rec(const struct lu_env *env, struct osd_object *pobj,
         return rc;
 }
 
-static void
+static int
 osd_consistency_check(struct osd_thread_info *oti, struct osd_device *dev,
                      struct osd_idmap_cache *oic)
 {
@@ -4082,19 +4086,39 @@ osd_consistency_check(struct osd_thread_info *oti, struct osd_device *dev,
        ENTRY;
 
        if (!fid_is_norm(fid) && !fid_is_igif(fid))
-               RETURN_EXIT;
+               RETURN(0);
 
        if (scrub->os_pos_current > id->oii_ino)
-               RETURN_EXIT;
+               RETURN(0);
 
 again:
-       rc = osd_oi_lookup(oti, dev, fid, id, OI_CHECK_FLD);
-       if (rc != 0 && rc != -ENOENT)
-               RETURN_EXIT;
+       rc = osd_oi_lookup(oti, dev, fid, id, 0);
+       if (rc == -ENOENT) {
+               struct inode *inode;
+
+               *id = oic->oic_lid;
+               inode = osd_iget(oti, dev, &oic->oic_lid);
+
+               /* The inode has been removed (by race maybe). */
+               if (IS_ERR(inode)) {
+                       rc = PTR_ERR(inode);
 
-       if (rc == 0 && osd_id_eq(id, &oic->oic_lid))
-               RETURN_EXIT;
+                       RETURN(rc == -ESTALE ? -ENOENT : rc);
+               }
+
+               iput(inode);
+               /* The OI mapping is lost. */
+               if (id->oii_gen != OSD_OII_NOGEN)
+                       goto trigger;
 
+               /* The inode may has been reused by others, we do not know,
+                * leave it to be handled by subsequent osd_fid_lookup(). */
+               RETURN(0);
+       } else if (rc != 0 || osd_id_eq(id, &oic->oic_lid)) {
+               RETURN(rc);
+       }
+
+trigger:
        if (thread_is_running(&scrub->os_thread)) {
                rc = osd_oii_insert(dev, oic, rc == -ENOENT);
                /* There is race condition between osd_oi_lookup and OI scrub.
@@ -4104,7 +4128,7 @@ again:
                if (unlikely(rc == -EAGAIN))
                        goto again;
 
-               RETURN_EXIT;
+               RETURN(0);
        }
 
        if (!dev->od_noscrub && ++once == 1) {
@@ -4118,7 +4142,7 @@ again:
                        goto again;
        }
 
-       EXIT;
+       RETURN(0);
 }
 
 static int osd_fail_fid_lookup(struct osd_thread_info *oti,
@@ -4304,16 +4328,16 @@ static int osd_ea_lookup_rec(const struct lu_env *env, struct osd_object *obj,
                        osd_id_gen(id, ino, OSD_OII_NOGEN);
                }
 
-               if (rc != 0) {
+               if (rc != 0 || osd_remote_fid(env, dev, fid)) {
                        fid_zero(&oic->oic_fid);
+
                        GOTO(out, rc);
                }
 
-               if (osd_remote_fid(env, dev, fid))
-                       GOTO(out, rc = 0);
-
                osd_add_oi_cache(osd_oti_get(env), osd_obj2dev(obj), id, fid);
-               osd_consistency_check(oti, dev, oic);
+               rc = osd_consistency_check(oti, dev, oic);
+               if (rc != 0)
+                       fid_zero(&oic->oic_fid);
        } else {
                rc = -ENOENT;
        }

diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h
index a344236..7ef1032 100644 (file)
--- a/lustre/osd-ldiskfs/osd_internal.h
+++ b/lustre/osd-ldiskfs/osd_internal.h
@@ -79,6 +79,7 @@ extern struct kmem_cache *dynlock_cachep;
 
 /* OI scrub should skip this inode. */
 #define LDISKFS_STATE_LUSTRE_NOSCRUB   31
+#define LDISKFS_STATE_LUSTRE_DESTROY   30
 
 /** Enable thandle usage statistics */
 #define OSD_THANDLE_STATS (0)

diff --git a/lustre/osd-ldiskfs/osd_oi.c b/lustre/osd-ldiskfs/osd_oi.c
index a6fed15..0a0d4e1 100644 (file)
--- a/lustre/osd-ldiskfs/osd_oi.c
+++ b/lustre/osd-ldiskfs/osd_oi.c
@@ -647,7 +647,7 @@ int osd_oi_insert(struct osd_thread_info *info, struct osd_device *osd,
                        return rc;
 
                if (unlikely(osd_id_eq(id, oi_id)))
-                       return 0;
+                       return 1;
 
                /* Check whether the mapping for oi_id is valid or not. */
                inode = osd_iget(info, osd, oi_id);

diff --git a/lustre/osd-ldiskfs/osd_scrub.c b/lustre/osd-ldiskfs/osd_scrub.c
index 567560c..d79cb2f 100644 (file)
--- a/lustre/osd-ldiskfs/osd_scrub.c
+++ b/lustre/osd-ldiskfs/osd_scrub.c
@@ -615,7 +615,8 @@ out:
        /* There may be conflict unlink during the OI scrub,
         * if happend, then remove the new added OI mapping. */
        if (ops == DTO_INDEX_INSERT && inode != NULL && !IS_ERR(inode) &&
-           unlikely(inode->i_nlink == 0))
+           unlikely(ldiskfs_test_inode_state(inode,
+                                             LDISKFS_STATE_LUSTRE_DESTROY)))
                osd_scrub_refresh_mapping(info, dev, fid, lid,
                                DTO_INDEX_DELETE, false,
                                (val == SCRUB_NEXT_OSTOBJ ||
@@ -627,13 +628,11 @@ out:
                iput(inode);
 
        if (oii != NULL) {
-               LASSERT(!list_empty(&oii->oii_list));
+               LASSERT(list_empty(&oii->oii_list));
 
-               spin_lock(&scrub->os_lock);
-               list_del_init(&oii->oii_list);
-               spin_unlock(&scrub->os_lock);
                OBD_FREE_PTR(oii);
        }
+
        RETURN(sf->sf_param & SP_FAILOUT ? rc : 0);
 }
 
@@ -1009,13 +1008,21 @@ static int osd_scrub_next(struct osd_thread_info *info, struct osd_device *dev,
                return SCRUB_NEXT_EXIT;
 
        if (!list_empty(&scrub->os_inconsistent_items)) {
-               struct osd_inconsistent_item *oii;
+               spin_lock(&scrub->os_lock);
+               if (likely(!list_empty(&scrub->os_inconsistent_items))) {
+                       struct osd_inconsistent_item *oii;
 
-               oii = list_entry(scrub->os_inconsistent_items.next,
-                                struct osd_inconsistent_item, oii_list);
-               *oic = &oii->oii_cache;
-               scrub->os_in_prior = 1;
-               return 0;
+                       oii = list_entry(scrub->os_inconsistent_items.next,
+                               struct osd_inconsistent_item, oii_list);
+                       list_del_init(&oii->oii_list);
+                       spin_unlock(&scrub->os_lock);
+
+                       *oic = &oii->oii_cache;
+                       scrub->os_in_prior = 1;
+
+                       return 0;
+               }
+               spin_unlock(&scrub->os_lock);
        }
 
        if (noslot)
@@ -1102,8 +1109,10 @@ static int osd_scrub_exec(struct osd_thread_info *info, struct osd_device *dev,
        }
 
        rc = osd_scrub_check_update(info, dev, oic, rc);
-       if (rc != 0)
+       if (rc != 0) {
+               scrub->os_in_prior = 0;
                return rc;
+       }
 
        rc = osd_scrub_checkpoint(scrub);
        if (rc != 0) {