From de47c7671f29b2a3a79f6a126b7e01f0b2c5991a Mon Sep 17 00:00:00 2001 From: Lai Siyao Date: Fri, 4 Dec 2020 05:07:01 +0800 Subject: [PATCH] LU-14172 lmv: optimize dir shard revalidate mdt_is_remote_object() will check whether child is directory shard if parent and child are on different MDTs, which needs to read LMV from disk, and hurt striped directory stat performance. This can be optimized, client can just set CROSS_REF flag to do a cross reference getattr, which avoids lots of checks. Signed-off-by: Lai Siyao Change-Id: Ib2d5a510b27c90a26f979f9cccfd40948e32d91a Reviewed-on: https://review.whamcloud.com/40863 Tested-by: jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Yingjin Qian --- lustre/include/obd.h | 2 +- lustre/include/obd_class.h | 3 +-- lustre/include/uapi/linux/lustre/lustre_idl.h | 7 ++++++ lustre/llite/file.c | 2 +- lustre/llite/llite_lib.c | 4 ++-- lustre/lmv/lmv_intent.c | 15 +++++++------ lustre/lmv/lmv_internal.h | 1 - lustre/lmv/lmv_obd.c | 3 +-- lustre/mdt/mdt_lib.c | 31 +-------------------------- 9 files changed, 22 insertions(+), 46 deletions(-) diff --git a/lustre/include/obd.h b/lustre/include/obd.h index d057d3e..ddc37d1 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -1183,7 +1183,7 @@ struct md_ops { int (*m_free_lustre_md)(struct obd_export *, struct lustre_md *); - int (*m_merge_attr)(struct obd_export *, const struct lu_fid *fid, + int (*m_merge_attr)(struct obd_export *, const struct lmv_stripe_md *lsm, struct cl_attr *attr, ldlm_blocking_callback); diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 0284b13..45ff5ef 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -1597,7 +1597,6 @@ static inline int md_free_lustre_md(struct obd_export *exp, } static inline int md_merge_attr(struct obd_export *exp, - const struct lu_fid *fid, const struct lmv_stripe_md *lsm, struct cl_attr *attr, ldlm_blocking_callback cb) @@ -1608,7 +1607,7 @@ static inline int md_merge_attr(struct obd_export *exp, if (rc) return rc; - return MDP(exp->exp_obd, merge_attr)(exp, fid, lsm, attr, cb); + return MDP(exp->exp_obd, merge_attr)(exp, lsm, attr, cb); } static inline int md_setxattr(struct obd_export *exp, const struct lu_fid *fid, diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h index d8fd6b2..39b59b3 100644 --- a/lustre/include/uapi/linux/lustre/lustre_idl.h +++ b/lustre/include/uapi/linux/lustre/lustre_idl.h @@ -1919,6 +1919,13 @@ struct mdt_rec_setattr { enum mds_op_bias { /* MDS_CHECK_SPLIT = 1 << 0, obsolete before 2.3.58 */ + /* used for remote object getattr/open by name: in the original + * getattr/open request, MDT found the object against name is on another + * MDT, then packed FID and LOOKUP lock in reply and returned -EREMOTE, + * and client knew it's a remote object, then set this flag in + * getattr/open request and sent to the corresponding MDT to finish + * getattr/open, which fetched attributes and UPDATE lock/opened file. + */ MDS_CROSS_REF = 1 << 1, /* MDS_VTX_BYPASS = 1 << 2, obsolete since 2.3.54 */ MDS_PERM_BYPASS = 1 << 3, diff --git a/lustre/llite/file.c b/lustre/llite/file.c index a048204..5e3debe 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -4821,7 +4821,7 @@ static int ll_merge_md_attr(struct inode *inode) RETURN(0); down_read(&lli->lli_lsm_sem); - rc = md_merge_attr(ll_i2mdexp(inode), &lli->lli_fid, lli->lli_lsm_md, + rc = md_merge_attr(ll_i2mdexp(inode), ll_i2info(inode)->lli_lsm_md, &attr, ll_md_blocking_ast); up_read(&lli->lli_lsm_sem); if (rc != 0) diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index 066f6e3..2efe350 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -1596,8 +1596,8 @@ static int ll_update_lsm_md(struct inode *inode, struct lustre_md *md) GOTO(unlock, rc = -ENOMEM); /* validate the lsm */ - rc = md_merge_attr(ll_i2mdexp(inode), &lli->lli_fid, lli->lli_lsm_md, - attr, ll_md_blocking_ast); + rc = md_merge_attr(ll_i2mdexp(inode), lli->lli_lsm_md, attr, + ll_md_blocking_ast); if (!rc) { if (md->body->mbo_valid & OBD_MD_FLNLINK) md->body->mbo_nlink = attr->cat_nlink; diff --git a/lustre/lmv/lmv_intent.c b/lustre/lmv/lmv_intent.c index 9990a5f..de2b425 100644 --- a/lustre/lmv/lmv_intent.c +++ b/lustre/lmv/lmv_intent.c @@ -152,7 +152,6 @@ out: } int lmv_revalidate_slaves(struct obd_export *exp, - const struct lu_fid *pfid, const struct lmv_stripe_md *lsm, ldlm_blocking_callback cb_blocking, int extra_lock_flags) @@ -199,11 +198,14 @@ int lmv_revalidate_slaves(struct obd_export *exp, * which is not needed here. */ memset(op_data, 0, sizeof(*op_data)); - if (exp_connect_flags2(exp) & OBD_CONNECT2_GETATTR_PFID) - op_data->op_fid1 = *pfid; - else - op_data->op_fid1 = fid; + op_data->op_fid1 = fid; op_data->op_fid2 = fid; + /* shard revalidate only needs to fetch attributes and UPDATE + * lock, which is similar to the bottom half of remote object + * getattr, set this flag so that MDT skips checking whether + * it's remote object. + */ + op_data->op_bias = MDS_CROSS_REF; tgt = lmv_tgt(lmv, lsm->lsm_md_oinfo[i].lmo_mds); if (!tgt) @@ -488,8 +490,7 @@ retry: /* If RPC happens, lsm information will be revalidated * during update_inode process (see ll_update_lsm_md) */ if (lmv_dir_striped(op_data->op_mea2)) { - rc = lmv_revalidate_slaves(exp, &op_data->op_fid2, - op_data->op_mea2, + rc = lmv_revalidate_slaves(exp, op_data->op_mea2, cb_blocking, extra_lock_flags); if (rc != 0) diff --git a/lustre/lmv/lmv_internal.h b/lustre/lmv/lmv_internal.h index d0527ba..ac8384f 100644 --- a/lustre/lmv/lmv_internal.h +++ b/lustre/lmv/lmv_internal.h @@ -53,7 +53,6 @@ int lmv_fid_alloc(const struct lu_env *env, struct obd_export *exp, struct lu_fid *fid, struct md_op_data *op_data); int lmv_revalidate_slaves(struct obd_export *exp, - const struct lu_fid *pfid, const struct lmv_stripe_md *lsm, ldlm_blocking_callback cb_blocking, int extra_lock_flags); diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c index 146f7b0..a8ee803 100644 --- a/lustre/lmv/lmv_obd.c +++ b/lustre/lmv/lmv_obd.c @@ -3560,7 +3560,6 @@ static int lmv_quotactl(struct obd_device *unused, struct obd_export *exp, } static int lmv_merge_attr(struct obd_export *exp, - const struct lu_fid *fid, const struct lmv_stripe_md *lsm, struct cl_attr *attr, ldlm_blocking_callback cb_blocking) @@ -3571,7 +3570,7 @@ static int lmv_merge_attr(struct obd_export *exp, if (!lmv_dir_striped(lsm)) return 0; - rc = lmv_revalidate_slaves(exp, fid, lsm, cb_blocking, 0); + rc = lmv_revalidate_slaves(exp, lsm, cb_blocking, 0); if (rc < 0) return rc; diff --git a/lustre/mdt/mdt_lib.c b/lustre/mdt/mdt_lib.c index 87c4a61..ce7778e 100644 --- a/lustre/mdt/mdt_lib.c +++ b/lustre/mdt/mdt_lib.c @@ -1888,29 +1888,6 @@ static int mdt_fids_different_target(struct mdt_thread_info *info, return index1 != index2; } -static bool mdt_object_is_shard(struct mdt_thread_info *info, - struct mdt_object *obj) -{ - struct lmv_mds_md_v1 *lmv = (struct lmv_mds_md_v1 *)info->mti_xattr_buf; - struct lu_buf buf; - int rc; - - if (!mdt_object_exists(obj)) - return false; - - if (!S_ISDIR(lu_object_attr(&obj->mot_obj))) - return false; - - buf.lb_buf = lmv; - buf.lb_len = sizeof(*lmv); - rc = mo_xattr_get(info->mti_env, mdt_object_child(obj), &buf, - XATTR_NAME_LMV); - if (rc < 0) - return false; - - return lmv->lmv_magic == cpu_to_le32(LMV_MAGIC_STRIPE); -} - /** * Check whether \a child is remote object on \a parent. * @@ -1943,14 +1920,8 @@ int mdt_is_remote_object(struct mdt_thread_info *info, RETURN(0); if (likely(parent != child)) { - if (mdt_object_remote(parent) ^ mdt_object_remote(child)) { - /* don't treat shard as remote object, otherwise client - * need to revalidate shards all the time. - */ - if (mdt_object_is_shard(info, child)) - RETURN(0); + if (mdt_object_remote(parent) ^ mdt_object_remote(child)) RETURN(1); - } if (!mdt_object_remote(parent) && !mdt_object_remote(child)) RETURN(0); -- 1.8.3.1