From: Fan Yong Date: Mon, 21 Sep 2015 09:10:48 +0000 (+0800) Subject: LU-6895 scrub: not trigger scrub if inode removed by race X-Git-Tag: 2.7.62~22 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=d8612e307f28cc81df5d93e45130b9f04837ee27 LU-6895 scrub: not trigger scrub if inode removed by race When osd_consistency_check(), the target file may has just been removed by other, so the osd_oi_lookup() will return -ENOENT to the caller. Under such case, the osd_consistency_check() should not trigger OI scrub. On the other hand, if someone unlinked the file during OI scrub adding the missed OI mapping to the OI file, the OI scrub needs to remove the new added OI mapping. Test-Parameters: alwaysuploadlogs \ envdefinitions=SLOW=yes,ENABLE_QUOTA=yes \ mdtfilesystemtype=ldiskfs mdsfilesystemtype=ldiskfs \ ostfilesystemtype=ldiskfs clientdistro=el7 ossdistro=el7 \ mdsdistro=el7 mdtcount=1 \ testlist=sanity-scrub,sanity-scrub,sanity-scrub,sanity-scrub Signed-off-by: Fan Yong Change-Id: I4703fc7f99d7b0a0f769127b5cdba5a2b992250d Reviewed-on: http://review.whamcloud.com/16439 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Alex Zhuravlev Reviewed-by: Oleg Drokin --- diff --git a/lustre/osd-ldiskfs/osd_compat.c b/lustre/osd-ldiskfs/osd_compat.c index bf317d0..2d78901 100644 --- a/lustre/osd-ldiskfs/osd_compat.c +++ b/lustre/osd-ldiskfs/osd_compat.c @@ -566,22 +566,19 @@ static int osd_obj_update_entry(struct osd_thread_info *info, rc = osd_get_lma(info, inode, dentry, lma); if (rc == -ENODATA) { rc = osd_get_idif(info, inode, dentry, oi_fid); - if (rc > 0) { + if (rc > 0 || rc == -ENODATA) { oi_fid = NULL; rc = 0; } } iput(inode); - /* If the OST-object has neither FID-in-LMA nor FID-in-ff, it is - * either a crashed object or a uninitialized one. Replace it. */ - if (rc == -ENODATA || oi_fid == NULL) - goto update; - if (rc != 0) GOTO(out, rc); - if (lu_fid_eq(fid, oi_fid)) { + /* If the OST-object has neither FID-in-LMA nor FID-in-ff, it is + * either a crashed object or a uninitialized one. Replace it. */ + if (oi_fid != NULL && lu_fid_eq(fid, oi_fid)) { CERROR("%s: the FID "DFID" is used by two objects: " "%u/%u %u/%u\n", osd_name(osd), PFID(fid), oi_id->oii_ino, oi_id->oii_gen, diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index d6c2f86..d62023d 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -270,6 +270,7 @@ struct inode *osd_iget(struct osd_thread_info *info, struct osd_device *dev, iput(inode); inode = ERR_PTR(-ENOENT); } else { + ldiskfs_clear_inode_state(inode, LDISKFS_STATE_LUSTRE_DESTROY); if (id->oii_gen == OSD_OII_NOGEN) osd_id_gen(id, inode->i_ino, inode->i_generation); @@ -374,6 +375,8 @@ static struct inode *osd_iget_check(struct osd_thread_info *info, goto check_oi; } + ldiskfs_clear_inode_state(inode, LDISKFS_STATE_LUSTRE_DESTROY); + check_oi: if (rc != 0) { LASSERTF(rc == -ESTALE || rc == -ENOENT, "rc = %d\n", rc); @@ -2523,6 +2526,7 @@ static int osd_object_destroy(const struct lu_env *env, osd_trans_exec_op(env, th, OSD_OT_DESTROY); + ldiskfs_set_inode_state(inode, LDISKFS_STATE_LUSTRE_DESTROY); result = osd_oi_delete(osd_oti_get(env), osd, fid, oh->ot_handle, OI_CHECK_FLD); @@ -4070,7 +4074,7 @@ static int osd_ea_add_rec(const struct lu_env *env, struct osd_object *pobj, return rc; } -static void +static int osd_consistency_check(struct osd_thread_info *oti, struct osd_device *dev, struct osd_idmap_cache *oic) { @@ -4082,19 +4086,39 @@ osd_consistency_check(struct osd_thread_info *oti, struct osd_device *dev, ENTRY; if (!fid_is_norm(fid) && !fid_is_igif(fid)) - RETURN_EXIT; + RETURN(0); if (scrub->os_pos_current > id->oii_ino) - RETURN_EXIT; + RETURN(0); again: - rc = osd_oi_lookup(oti, dev, fid, id, OI_CHECK_FLD); - if (rc != 0 && rc != -ENOENT) - RETURN_EXIT; + rc = osd_oi_lookup(oti, dev, fid, id, 0); + if (rc == -ENOENT) { + struct inode *inode; + + *id = oic->oic_lid; + inode = osd_iget(oti, dev, &oic->oic_lid); + + /* The inode has been removed (by race maybe). */ + if (IS_ERR(inode)) { + rc = PTR_ERR(inode); - if (rc == 0 && osd_id_eq(id, &oic->oic_lid)) - RETURN_EXIT; + RETURN(rc == -ESTALE ? -ENOENT : rc); + } + + iput(inode); + /* The OI mapping is lost. */ + if (id->oii_gen != OSD_OII_NOGEN) + goto trigger; + /* The inode may has been reused by others, we do not know, + * leave it to be handled by subsequent osd_fid_lookup(). */ + RETURN(0); + } else if (rc != 0 || osd_id_eq(id, &oic->oic_lid)) { + RETURN(rc); + } + +trigger: if (thread_is_running(&scrub->os_thread)) { rc = osd_oii_insert(dev, oic, rc == -ENOENT); /* There is race condition between osd_oi_lookup and OI scrub. @@ -4104,7 +4128,7 @@ again: if (unlikely(rc == -EAGAIN)) goto again; - RETURN_EXIT; + RETURN(0); } if (!dev->od_noscrub && ++once == 1) { @@ -4118,7 +4142,7 @@ again: goto again; } - EXIT; + RETURN(0); } static int osd_fail_fid_lookup(struct osd_thread_info *oti, @@ -4304,16 +4328,16 @@ static int osd_ea_lookup_rec(const struct lu_env *env, struct osd_object *obj, osd_id_gen(id, ino, OSD_OII_NOGEN); } - if (rc != 0) { + if (rc != 0 || osd_remote_fid(env, dev, fid)) { fid_zero(&oic->oic_fid); + GOTO(out, rc); } - if (osd_remote_fid(env, dev, fid)) - GOTO(out, rc = 0); - osd_add_oi_cache(osd_oti_get(env), osd_obj2dev(obj), id, fid); - osd_consistency_check(oti, dev, oic); + rc = osd_consistency_check(oti, dev, oic); + if (rc != 0) + fid_zero(&oic->oic_fid); } else { rc = -ENOENT; } diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h index a344236..7ef1032 100644 --- a/lustre/osd-ldiskfs/osd_internal.h +++ b/lustre/osd-ldiskfs/osd_internal.h @@ -79,6 +79,7 @@ extern struct kmem_cache *dynlock_cachep; /* OI scrub should skip this inode. */ #define LDISKFS_STATE_LUSTRE_NOSCRUB 31 +#define LDISKFS_STATE_LUSTRE_DESTROY 30 /** Enable thandle usage statistics */ #define OSD_THANDLE_STATS (0) diff --git a/lustre/osd-ldiskfs/osd_oi.c b/lustre/osd-ldiskfs/osd_oi.c index a6fed15..0a0d4e1 100644 --- a/lustre/osd-ldiskfs/osd_oi.c +++ b/lustre/osd-ldiskfs/osd_oi.c @@ -647,7 +647,7 @@ int osd_oi_insert(struct osd_thread_info *info, struct osd_device *osd, return rc; if (unlikely(osd_id_eq(id, oi_id))) - return 0; + return 1; /* Check whether the mapping for oi_id is valid or not. */ inode = osd_iget(info, osd, oi_id); diff --git a/lustre/osd-ldiskfs/osd_scrub.c b/lustre/osd-ldiskfs/osd_scrub.c index 567560c..d79cb2f 100644 --- a/lustre/osd-ldiskfs/osd_scrub.c +++ b/lustre/osd-ldiskfs/osd_scrub.c @@ -615,7 +615,8 @@ out: /* There may be conflict unlink during the OI scrub, * if happend, then remove the new added OI mapping. */ if (ops == DTO_INDEX_INSERT && inode != NULL && !IS_ERR(inode) && - unlikely(inode->i_nlink == 0)) + unlikely(ldiskfs_test_inode_state(inode, + LDISKFS_STATE_LUSTRE_DESTROY))) osd_scrub_refresh_mapping(info, dev, fid, lid, DTO_INDEX_DELETE, false, (val == SCRUB_NEXT_OSTOBJ || @@ -627,13 +628,11 @@ out: iput(inode); if (oii != NULL) { - LASSERT(!list_empty(&oii->oii_list)); + LASSERT(list_empty(&oii->oii_list)); - spin_lock(&scrub->os_lock); - list_del_init(&oii->oii_list); - spin_unlock(&scrub->os_lock); OBD_FREE_PTR(oii); } + RETURN(sf->sf_param & SP_FAILOUT ? rc : 0); } @@ -1009,13 +1008,21 @@ static int osd_scrub_next(struct osd_thread_info *info, struct osd_device *dev, return SCRUB_NEXT_EXIT; if (!list_empty(&scrub->os_inconsistent_items)) { - struct osd_inconsistent_item *oii; + spin_lock(&scrub->os_lock); + if (likely(!list_empty(&scrub->os_inconsistent_items))) { + struct osd_inconsistent_item *oii; - oii = list_entry(scrub->os_inconsistent_items.next, - struct osd_inconsistent_item, oii_list); - *oic = &oii->oii_cache; - scrub->os_in_prior = 1; - return 0; + oii = list_entry(scrub->os_inconsistent_items.next, + struct osd_inconsistent_item, oii_list); + list_del_init(&oii->oii_list); + spin_unlock(&scrub->os_lock); + + *oic = &oii->oii_cache; + scrub->os_in_prior = 1; + + return 0; + } + spin_unlock(&scrub->os_lock); } if (noslot) @@ -1102,8 +1109,10 @@ static int osd_scrub_exec(struct osd_thread_info *info, struct osd_device *dev, } rc = osd_scrub_check_update(info, dev, oic, rc); - if (rc != 0) + if (rc != 0) { + scrub->os_in_prior = 0; return rc; + } rc = osd_scrub_checkpoint(scrub); if (rc != 0) {