From: Lai Siyao Date: Thu, 7 Jun 2018 11:53:14 +0000 (+0800) Subject: LU-11103 lod: add lock for lod_object layout X-Git-Tag: 2.11.54~40 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=8733d5d764531621dc3e064dfb922e49373f7946 LU-11103 lod: add lock for lod_object layout lod_object layout is loaded on demand, and it may be updated by layout split/merge. To avoid race, add ldo_layouyt_mutex to serialize layout load/free/reload. Signed-off-by: Lai Siyao Change-Id: I43c15a3b07254eadef95a14b288267904a1cd621 Reviewed-on: https://review.whamcloud.com/32589 Reviewed-by: Alex Zhuravlev Tested-by: Jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Oleg Drokin --- diff --git a/lustre/lod/lod_dev.c b/lustre/lod/lod_dev.c index 5a5eb10..d61ad2d 100644 --- a/lustre/lod/lod_dev.c +++ b/lustre/lod/lod_dev.c @@ -213,6 +213,7 @@ static struct lu_object *lod_object_alloc(const struct lu_env *env, if (lod_obj == NULL) RETURN(ERR_PTR(-ENOMEM)); + mutex_init(&lod_obj->ldo_layout_mutex); lu_obj = lod2lu_obj(lod_obj); dt_object_init(&lod_obj->ldo_obj, NULL, dev); lod_obj->ldo_obj.do_ops = &lod_obj_ops; diff --git a/lustre/lod/lod_internal.h b/lustre/lod/lod_internal.h index cb683f1..e987ca6 100644 --- a/lustre/lod/lod_internal.h +++ b/lustre/lod/lod_internal.h @@ -293,6 +293,7 @@ struct lod_mirror_entry { struct lod_object { /* common fields for both files and directories */ struct dt_object ldo_obj; + struct mutex ldo_layout_mutex; union { /* file stripe (LOV) */ struct { @@ -590,8 +591,9 @@ int lod_del_device(const struct lu_env *env, struct lod_device *lod, unsigned gen, bool for_ost); int lod_fini_tgt(const struct lu_env *env, struct lod_device *lod, struct lod_tgt_descs *ltd, bool for_ost); -int lod_load_striping_locked(const struct lu_env *env, struct lod_object *lo); -int lod_load_striping(const struct lu_env *env, struct lod_object *lo); +int lod_striping_load(const struct lu_env *env, struct lod_object *lo); +int lod_striping_reload(const struct lu_env *env, struct lod_object *lo, + const struct lu_buf *buf); int lod_get_ea(const struct lu_env *env, struct lod_object *lo, const char *name); @@ -751,7 +753,8 @@ int lod_declare_striped_create(const struct lu_env *env, struct dt_object *dt, int lod_striped_create(const struct lu_env *env, struct dt_object *dt, struct lu_attr *attr, struct dt_object_format *dof, struct thandle *th); -void lod_object_free_striping(const struct lu_env *env, struct lod_object *lo); +void lod_striping_free_nolock(const struct lu_env *env, struct lod_object *lo); +void lod_striping_free(const struct lu_env *env, struct lod_object *lo); int lod_obj_for_each_stripe(const struct lu_env *env, struct lod_object *lo, struct thandle *th, diff --git a/lustre/lod/lod_lov.c b/lustre/lod/lod_lov.c index ec6d18d..2849beb 100644 --- a/lustre/lod/lod_lov.c +++ b/lustre/lod/lod_lov.c @@ -1235,6 +1235,7 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, LASSERT(buf); LASSERT(buf->lb_buf); LASSERT(buf->lb_len); + LASSERT(mutex_is_locked(&lo->ldo_layout_mutex)); lmm = (struct lov_mds_md_v1 *)buf->lb_buf; magic = le32_to_cpu(lmm->lmm_magic); @@ -1370,7 +1371,7 @@ int lod_parse_striping(const struct lu_env *env, struct lod_object *lo, out: if (rc) - lod_object_free_striping(env, lo); + lod_striping_free_nolock(env, lo); RETURN(rc); } @@ -1390,7 +1391,7 @@ static bool lod_striping_loaded(struct lod_object *lo) return true; if (S_ISDIR(lod2lu_obj(lo)->lo_header->loh_attr)) { - if (lo->ldo_stripe != NULL || lo->ldo_dir_stripe_loaded) + if (lo->ldo_dir_stripe_loaded) return true; /* Never load LMV stripe for slaves of striped dir */ @@ -1402,37 +1403,45 @@ static bool lod_striping_loaded(struct lod_object *lo) } /** - * Initialize the object representing the stripes. + * A generic function to initialize the stripe objects. * - * Unless the stripes are initialized already, fetch LOV (for regular - * objects) or LMV (for directory objects) EA and call lod_parse_striping() - * to instantiate the objects representing the stripes. Caller should - * hold the dt_write_lock(next). + * A protected version of lod_striping_load_locked() - load the striping + * information from storage, parse that and instantiate LU objects to + * represent the stripes. The LOD object \a lo supplies a pointer to the + * next sub-object in the LU stack so we can lock it. Also use \a lo to + * return an array of references to the newly instantiated objects. * * \param[in] env execution environment for this thread - * \param[in,out] lo LOD object + * \param[in,out] lo LOD object, where striping is stored and + * which gets an array of references * * \retval 0 if parsing and object creation succeed * \retval negative error number on failure - */ -int lod_load_striping_locked(const struct lu_env *env, struct lod_object *lo) + **/ +int lod_striping_load(const struct lu_env *env, struct lod_object *lo) { - struct lod_thread_info *info = lod_env_info(env); - struct lu_buf *buf = &info->lti_buf; - struct dt_object *next = dt_object_child(&lo->ldo_obj); - int rc = 0; + struct lod_thread_info *info = lod_env_info(env); + struct dt_object *next = dt_object_child(&lo->ldo_obj); + struct lu_buf *buf = &info->lti_buf; + int rc = 0; + ENTRY; if (!dt_object_exists(next)) - GOTO(out, rc = 0); + RETURN(0); if (lod_striping_loaded(lo)) - GOTO(out, rc = 0); + RETURN(0); + + mutex_lock(&lo->ldo_layout_mutex); + if (lod_striping_loaded(lo)) + GOTO(unlock, rc = 0); if (S_ISREG(lod2lu_obj(lo)->lo_header->loh_attr)) { rc = lod_get_lov_ea(env, lo); if (rc <= 0) - GOTO(out, rc); + GOTO(unlock, rc); + /* * there is LOV EA (striping information) in this object * let's parse it and create in-core objects for the stripes @@ -1451,7 +1460,7 @@ int lod_load_striping_locked(const struct lu_env *env, struct lod_object *lo) */ if (rc == 0) lo->ldo_dir_stripe_loaded = 1; - GOTO(out, rc = rc > 0 ? -EINVAL : rc); + GOTO(unlock, rc = rc > 0 ? -EINVAL : rc); } buf->lb_buf = info->lti_ea_store; buf->lb_len = info->lti_ea_store_size; @@ -1465,7 +1474,7 @@ int lod_load_striping_locked(const struct lu_env *env, struct lod_object *lo) } if (rc < 0) - GOTO(out, rc); + GOTO(unlock, rc); } /* @@ -1476,44 +1485,26 @@ int lod_load_striping_locked(const struct lu_env *env, struct lod_object *lo) if (rc == 0) lo->ldo_dir_stripe_loaded = 1; } -out: - RETURN(rc); + EXIT; +unlock: + mutex_unlock(&lo->ldo_layout_mutex); + + return rc; } -/** - * A generic function to initialize the stripe objects. - * - * A protected version of lod_load_striping_locked() - load the striping - * information from storage, parse that and instantiate LU objects to - * represent the stripes. The LOD object \a lo supplies a pointer to the - * next sub-object in the LU stack so we can lock it. Also use \a lo to - * return an array of references to the newly instantiated objects. - * - * \param[in] env execution environment for this thread - * \param[in,out] lo LOD object, where striping is stored and - * which gets an array of references - * - * \retval 0 if parsing and object creation succeed - * \retval negative error number on failure - **/ -int lod_load_striping(const struct lu_env *env, struct lod_object *lo) +int lod_striping_reload(const struct lu_env *env, struct lod_object *lo, + const struct lu_buf *buf) { - struct dt_object *next = dt_object_child(&lo->ldo_obj); - int rc; + int rc; - if (!dt_object_exists(next)) - return 0; + ENTRY; - /* Check without locking first */ - if (lod_striping_loaded(lo)) - return 0; + mutex_lock(&lo->ldo_layout_mutex); + lod_striping_free_nolock(env, lo); + rc = lod_parse_striping(env, lo, buf); + mutex_unlock(&lo->ldo_layout_mutex); - /* currently this code is supposed to be called from declaration - * phase only, thus the object is not expected to be locked by caller */ - dt_write_lock(env, next, 0); - rc = lod_load_striping_locked(env, lo); - dt_write_unlock(env, next); - return rc; + RETURN(rc); } /** diff --git a/lustre/lod/lod_object.c b/lustre/lod/lod_object.c index 7a5c0fb..31caad5 100644 --- a/lustre/lod/lod_object.c +++ b/lustre/lod/lod_object.c @@ -949,7 +949,7 @@ static int lod_index_try(const struct lu_env *env, struct dt_object *dt, LASSERT(next->do_ops); LASSERT(next->do_ops->do_index_try); - rc = lod_load_striping_locked(env, lo); + rc = lod_striping_load(env, lo); if (rc != 0) RETURN(rc); @@ -1219,7 +1219,7 @@ static int lod_declare_attr_set(const struct lu_env *env, * is being initialized as we don't need this information till * few specific cases like destroy, chown */ - rc = lod_load_striping(env, lo); + rc = lod_striping_load(env, lo); if (rc) RETURN(rc); @@ -1316,7 +1316,7 @@ static int lod_attr_set(const struct lu_env *env, * the in-memory striping information has been freed in lod_xattr_set() * due to layout change. It has to load stripe here again. It only * changes flags of layout so declare_attr_set() is still accurate */ - rc = lod_load_striping_locked(env, lo); + rc = lod_striping_load(env, lo); if (rc) RETURN(rc); @@ -1647,6 +1647,8 @@ int lod_parse_dir_striping(const struct lu_env *env, struct lod_object *lo, int rc = 0; ENTRY; + LASSERT(mutex_is_locked(&lo->ldo_layout_mutex)); + if (le32_to_cpu(lmv1->lmv_hash_type) & LMV_HASH_FLAG_MIGRATION) RETURN(0); @@ -1705,7 +1707,7 @@ out: lo->ldo_dir_stripe_count = le32_to_cpu(lmv1->lmv_stripe_count); lo->ldo_dir_stripes_allocated = le32_to_cpu(lmv1->lmv_stripe_count); if (rc != 0) - lod_object_free_striping(env, lo); + lod_striping_free_nolock(env, lo); RETURN(rc); } @@ -2006,11 +2008,12 @@ static int lod_prep_md_striped_create(const struct lu_env *env, stripe[i] = dto; } - lo->ldo_dir_stripe_loaded = 1; lo->ldo_dir_striped = 1; lo->ldo_stripe = stripe; lo->ldo_dir_stripe_count = i; lo->ldo_dir_stripes_allocated = stripe_count; + smp_mb(); + lo->ldo_dir_stripe_loaded = 1; if (lo->ldo_dir_stripe_count == 0) GOTO(out_put, rc = -ENOSPC); @@ -2081,7 +2084,7 @@ static int lod_declare_xattr_set_lmv(const struct lu_env *env, if (rc != 0) { /* failed to create striping, let's reset * config so that others don't get confused */ - lod_object_free_striping(env, lo); + lod_striping_free(env, lo); GOTO(out, rc); } out: @@ -2137,7 +2140,7 @@ static int lod_dir_declare_xattr_set(const struct lu_env *env, RETURN(0); /* set xattr to each stripes, if needed */ - rc = lod_load_striping(env, lo); + rc = lod_striping_load(env, lo); if (rc != 0) RETURN(rc); @@ -2234,7 +2237,7 @@ static int lod_replace_parent_fid(const struct lu_env *env, LASSERT(S_ISREG(dt->do_lu.lo_header->loh_attr)); /* set xattr to each stripes, if needed */ - rc = lod_load_striping(env, lo); + rc = lod_striping_load(env, lo); if (rc != 0) RETURN(rc); @@ -2495,7 +2498,7 @@ static int lod_declare_layout_set(const struct lu_env *env, if (flags & LCME_FL_INIT) { if (changed) - lod_object_free_striping(env, lo); + lod_striping_free(env, lo); RETURN(-EINVAL); } @@ -2685,7 +2688,6 @@ static int lod_declare_modify_layout(const struct lu_env *env, { struct lod_device *d = lu2lod_dev(dt->do_lu.lo_dev); struct lod_object *lo = lod_dt_obj(dt); - struct dt_object *next = dt_object_child(&lo->ldo_obj); char *op; int rc, len = strlen(XATTR_LUSTRE_LOV); ENTRY; @@ -2699,8 +2701,7 @@ static int lod_declare_modify_layout(const struct lu_env *env, } len++; - dt_write_lock(env, next, 0); - rc = lod_load_striping_locked(env, lo); + rc = lod_striping_load(env, lo); if (rc) GOTO(unlock, rc); @@ -2725,8 +2726,7 @@ static int lod_declare_modify_layout(const struct lu_env *env, } unlock: if (rc) - lod_object_free_striping(env, lo); - dt_write_unlock(env, next); + lod_striping_free(env, lo); RETURN(rc); } @@ -2924,9 +2924,7 @@ static int lod_declare_layout_merge(const struct lu_env *env, if ((le16_to_cpu(lcm->lcm_flags) & LCM_FL_FLR_MASK) == LCM_FL_NONE) lcm->lcm_flags = cpu_to_le32(LCM_FL_RDONLY); - LASSERT(dt_write_locked(env, dt_object_child(dt))); - lod_object_free_striping(env, lo); - rc = lod_parse_striping(env, lo, buf); + rc = lod_striping_reload(env, lo, buf); if (rc) GOTO(out, rc); @@ -2953,8 +2951,7 @@ static int lod_declare_layout_split(const struct lu_env *env, lod_obj_inc_layout_gen(lo); lcm->lcm_layout_gen = cpu_to_le32(lo->ldo_layout_gen); - lod_object_free_striping(env, lo); - rc = lod_parse_striping(env, lo, mbuf); + rc = lod_striping_reload(env, lo, mbuf); if (rc) RETURN(rc); @@ -3609,7 +3606,7 @@ static int lod_generate_and_set_lovea(const struct lu_env *env, LASSERT(lo); if (lo->ldo_comp_cnt == 0) { - lod_object_free_striping(env, lo); + lod_striping_free(env, lo); rc = lod_sub_xattr_del(env, next, XATTR_NAME_LOV, th); RETURN(rc); } @@ -3736,7 +3733,7 @@ static int lod_layout_del(const struct lu_env *env, struct dt_object *dt, EXIT; out: if (rc) - lod_object_free_striping(env, lo); + lod_striping_free(env, lo); return rc; } @@ -3855,7 +3852,7 @@ static int lod_xattr_set(const struct lu_env *env, * defines striping, then create() does the work */ if (fl & LU_XATTR_REPLACE) { /* free stripes, then update disk */ - lod_object_free_striping(env, lod_dt_obj(dt)); + lod_striping_free(env, lod_dt_obj(dt)); rc = lod_sub_xattr_set(env, next, buf, name, fl, th); } else if (dt_object_remote(dt)) { @@ -3923,7 +3920,7 @@ static int lod_declare_xattr_del(const struct lu_env *env, RETURN(0); /* set xattr to each stripes, if needed */ - rc = lod_load_striping(env, lo); + rc = lod_striping_load(env, lo); if (rc != 0) RETURN(rc); @@ -3959,7 +3956,7 @@ static int lod_xattr_del(const struct lu_env *env, struct dt_object *dt, ENTRY; if (!strcmp(name, XATTR_NAME_LOV)) - lod_object_free_striping(env, lod_dt_obj(dt)); + lod_striping_free(env, lod_dt_obj(dt)); rc = lod_sub_xattr_del(env, next, name, th); if (rc != 0 || !S_ISDIR(dt->do_lu.lo_header->loh_attr)) @@ -4634,7 +4631,7 @@ out: /* failed to create striping or to set initial size, let's reset * config so that others don't get confused */ if (rc) - lod_object_free_striping(env, lo); + lod_striping_free(env, lo); RETURN(rc); } @@ -4748,7 +4745,7 @@ out: /* failed to create striping or to set initial size, let's reset * config so that others don't get confused */ if (rc) - lod_object_free_striping(env, lo); + lod_striping_free(env, lo); RETURN(rc); } @@ -4881,7 +4878,7 @@ int lod_striped_create(const struct lu_env *env, struct dt_object *dt, RETURN(0); out: - lod_object_free_striping(env, lo); + lod_striping_free(env, lo); RETURN(rc); } @@ -4956,7 +4953,7 @@ static int lod_declare_destroy(const struct lu_env *env, struct dt_object *dt, * is being initialized as we don't need this information till * few specific cases like destroy, chown */ - rc = lod_load_striping(env, lo); + rc = lod_striping_load(env, lo); if (rc) RETURN(rc); @@ -5246,7 +5243,7 @@ static int lod_object_lock(const struct lu_env *env, if (!S_ISDIR(dt->do_lu.lo_header->loh_attr)) RETURN(-ENOTDIR); - rc = lod_load_striping(env, lo); + rc = lod_striping_load(env, lo); if (rc != 0) RETURN(rc); @@ -5410,7 +5407,6 @@ static int lod_declare_update_plain(const struct lu_env *env, GOTO(out, rc = -EINVAL); } - lod_object_free_striping(env, lo); rc = lod_use_defined_striping(env, lo, buf); if (rc) GOTO(out, rc); @@ -5423,7 +5419,7 @@ static int lod_declare_update_plain(const struct lu_env *env, replay = true; } else { /* non replay path */ - rc = lod_load_striping_locked(env, lo); + rc = lod_striping_load(env, lo); if (rc) GOTO(out, rc); } @@ -5489,7 +5485,7 @@ static int lod_declare_update_plain(const struct lu_env *env, rc = lod_declare_instantiate_components(env, lo, th); out: if (rc) - lod_object_free_striping(env, lo); + lod_striping_free(env, lo); RETURN(rc); } @@ -5849,7 +5845,7 @@ static int lod_declare_update_rdonly(const struct lu_env *env, out: if (rc) - lod_object_free_striping(env, lo); + lod_striping_free(env, lo); RETURN(rc); } @@ -5977,7 +5973,7 @@ static int lod_declare_update_write_pending(const struct lu_env *env, lod_obj_inc_layout_gen(lo); out: if (rc) - lod_object_free_striping(env, lo); + lod_striping_free(env, lo); RETURN(rc); } @@ -6065,7 +6061,7 @@ static int lod_declare_update_sync_pending(const struct lu_env *env, out: if (rc) - lod_object_free_striping(env, lo); + lod_striping_free(env, lo); RETURN(rc); } @@ -6082,8 +6078,7 @@ static int lod_declare_layout_change(const struct lu_env *env, dt_object_remote(dt_object_child(dt))) RETURN(-EINVAL); - lod_write_lock(env, dt, 0); - rc = lod_load_striping_locked(env, lo); + rc = lod_striping_load(env, lo); if (rc) GOTO(out, rc); @@ -6112,7 +6107,6 @@ static int lod_declare_layout_change(const struct lu_env *env, break; } out: - dt_write_unlock(env, dt); RETURN(rc); } @@ -6331,7 +6325,7 @@ static int lod_object_init(const struct lu_env *env, struct lu_object *lo, * \param[in] env execution environment * \param[in] lo object */ -void lod_object_free_striping(const struct lu_env *env, struct lod_object *lo) +void lod_striping_free_nolock(const struct lu_env *env, struct lod_object *lo) { struct lod_layout_component *lod_comp; int i, j; @@ -6379,6 +6373,13 @@ void lod_object_free_striping(const struct lu_env *env, struct lod_object *lo) } } +void lod_striping_free(const struct lu_env *env, struct lod_object *lo) +{ + mutex_lock(&lo->ldo_layout_mutex); + lod_striping_free_nolock(env, lo); + mutex_unlock(&lo->ldo_layout_mutex); +} + /** * Implementation of lu_object_operations::loo_object_free. * @@ -6390,7 +6391,7 @@ static void lod_object_free(const struct lu_env *env, struct lu_object *o) struct lod_object *lo = lu2lod_obj(o); /* release all underlying object pinned */ - lod_object_free_striping(env, lo); + lod_striping_free(env, lo); lu_object_fini(o); OBD_SLAB_FREE_PTR(lo, lod_object_kmem); } diff --git a/lustre/lod/lod_qos.c b/lustre/lod/lod_qos.c index 797f81c..a309708 100644 --- a/lustre/lod/lod_qos.c +++ b/lustre/lod/lod_qos.c @@ -1821,17 +1821,20 @@ int lod_use_defined_striping(const struct lu_env *env, int rc = 0, i; ENTRY; + mutex_lock(&mo->ldo_layout_mutex); + lod_striping_free_nolock(env, mo); + magic = le32_to_cpu(v1->lmm_magic) & ~LOV_MAGIC_DEFINED; if (magic != LOV_MAGIC_V1 && magic != LOV_MAGIC_V3 && magic != LOV_MAGIC_COMP_V1) - RETURN(-EINVAL); + GOTO(unlock, rc = -EINVAL); if (magic == LOV_MAGIC_COMP_V1) { comp_v1 = buf->lb_buf; comp_cnt = le16_to_cpu(comp_v1->lcm_entry_count); if (comp_cnt == 0) - RETURN(-EINVAL); + GOTO(unlock, rc = -EINVAL); mirror_cnt = le16_to_cpu(comp_v1->lcm_mirror_count) + 1; mo->ldo_flr_state = le16_to_cpu(comp_v1->lcm_flags) & LCM_FL_FLR_MASK; @@ -1845,7 +1848,7 @@ int lod_use_defined_striping(const struct lu_env *env, rc = lod_alloc_comp_entries(mo, mirror_cnt, comp_cnt); if (rc) - RETURN(rc); + GOTO(unlock, rc); for (i = 0; i < comp_cnt; i++) { struct lu_extent *ext; @@ -1905,11 +1908,12 @@ int lod_use_defined_striping(const struct lu_env *env, } rc = lod_fill_mirrors(mo); - if (rc) - GOTO(out, rc); + GOTO(out, rc); out: if (rc) - lod_object_free_striping(env, mo); + lod_striping_free_nolock(env, mo); +unlock: + mutex_unlock(&mo->ldo_layout_mutex); RETURN(rc); }