From 0860eda0544a507754b3af4aadcd651e1120ded5 Mon Sep 17 00:00:00 2001 From: wang di Date: Wed, 14 Jan 2015 14:47:53 -0800 Subject: [PATCH] LU-6088 lmv: Do not revalidate stripes with master lock Do not revalidate slave stripes while holding master lock. Otherwise if the revalidating slaves are blocked, then the master lock can not be released in time. Remove some unnecesary merging in ll_revalidate_slave(), and the attributes will be stored in each stripe, only merging them if required. Signed-off-by: wang di Change-Id: I57c43236894e2bbbf9a20b1d90c5ab2a5dc62ef1 Reviewed-on: http://review.whamcloud.com/13432 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Lai Siyao Reviewed-by: Oleg Drokin --- lustre/include/obd.h | 5 +---- lustre/include/obd_class.h | 16 +++----------- lustre/llite/file.c | 19 +++++++---------- lustre/llite/llite_internal.h | 4 ---- lustre/llite/llite_lib.c | 10 ++------- lustre/lmv/lmv_intent.c | 45 ++++------------------------------------ lustre/lmv/lmv_internal.h | 4 ++-- lustre/lmv/lmv_obd.c | 30 ++++++++++++++------------- lustre/obdclass/lprocfs_status.c | 1 - 9 files changed, 35 insertions(+), 99 deletions(-) diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 761dc19..9ec325a 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -1109,10 +1109,7 @@ struct md_ops { int (*m_merge_attr)(struct obd_export *, const struct lmv_stripe_md *lsm, - struct cl_attr *attr); - - int (*m_update_lsm_md)(struct obd_export *, struct lmv_stripe_md *lsm, - struct mdt_body *, ldlm_blocking_callback); + struct cl_attr *attr, ldlm_blocking_callback); int (*m_set_open_replay_data)(struct obd_export *, struct obd_client_handle *, diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index d9428b7..dc9bb2d 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -1659,25 +1659,15 @@ static inline int md_free_lustre_md(struct obd_export *exp, RETURN(MDP(exp->exp_obd, free_lustre_md)(exp, md)); } -static inline int md_update_lsm_md(struct obd_export *exp, - struct lmv_stripe_md *lsm, - struct mdt_body *body, - ldlm_blocking_callback cb) -{ - ENTRY; - EXP_CHECK_MD_OP(exp, update_lsm_md); - EXP_MD_COUNTER_INCREMENT(exp, update_lsm_md); - RETURN(MDP(exp->exp_obd, update_lsm_md)(exp, lsm, body, cb)); -} - static inline int md_merge_attr(struct obd_export *exp, const struct lmv_stripe_md *lsm, - struct cl_attr *attr) + struct cl_attr *attr, + ldlm_blocking_callback cb) { ENTRY; EXP_CHECK_MD_OP(exp, merge_attr); EXP_MD_COUNTER_INCREMENT(exp, merge_attr); - RETURN(MDP(exp->exp_obd, merge_attr)(exp, lsm, attr)); + RETURN(MDP(exp->exp_obd, merge_attr)(exp, lsm, attr, cb)); } static inline int md_setxattr(struct obd_export *exp, diff --git a/lustre/llite/file.c b/lustre/llite/file.c index 28bffad..230a41f 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -3277,12 +3277,13 @@ static int ll_merge_md_attr(struct inode *inode) LASSERT(ll_i2info(inode)->lli_lsm_md != NULL); rc = md_merge_attr(ll_i2mdexp(inode), ll_i2info(inode)->lli_lsm_md, - &attr); + &attr, ll_md_blocking_ast); if (rc != 0) RETURN(rc); - ll_i2info(inode)->lli_stripe_dir_size = attr.cat_size; - ll_i2info(inode)->lli_stripe_dir_nlink = attr.cat_nlink; + set_nlink(inode, attr.cat_nlink); + inode->i_blocks = attr.cat_blocks; + i_size_write(inode, attr.cat_size); ll_i2info(inode)->lli_atime = attr.cat_atime; ll_i2info(inode)->lli_mtime = attr.cat_mtime; @@ -3355,16 +3356,10 @@ int ll_getattr(struct vfsmount *mnt, struct dentry *de, struct kstat *stat) stat->mtime = inode->i_mtime; stat->ctime = inode->i_ctime; stat->blksize = 1 << inode->i_blkbits; - stat->blocks = inode->i_blocks; - if (S_ISDIR(inode->i_mode) && - ll_i2info(inode)->lli_lsm_md != NULL) { - stat->nlink = lli->lli_stripe_dir_nlink; - stat->size = lli->lli_stripe_dir_size; - } else { - stat->nlink = inode->i_nlink; - stat->size = i_size_read(inode); - } + stat->nlink = inode->i_nlink; + stat->size = i_size_read(inode); + stat->blocks = inode->i_blocks; return 0; } diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h index 6f7326b..87e404e 100644 --- a/lustre/llite/llite_internal.h +++ b/lustre/llite/llite_internal.h @@ -208,10 +208,6 @@ struct ll_inode_info { unsigned int lli_sa_generation; /* directory stripe information */ struct lmv_stripe_md *lli_lsm_md; - /* striped directory size */ - loff_t lli_stripe_dir_size; - /* striped directory nlink */ - __u64 lli_stripe_dir_nlink; }; /* for non-directory */ diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index de207ab..889e857 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -1295,10 +1295,7 @@ static int ll_init_lsm_md(struct inode *inode, struct lustre_md *md) } } - /* Here is where the lsm is being initialized(fill lmo_info) after - * client retrieve MD stripe information from MDT. */ - return md_update_lsm_md(ll_i2mdexp(inode), lsm, md->body, - ll_md_blocking_ast); + return 0; } static inline int lli_lsm_md_eq(const struct lmv_stripe_md *lsm_md1, @@ -1398,10 +1395,7 @@ static int ll_update_lsm_md(struct inode *inode, struct lustre_md *md) RETURN(-EIO); } - rc = md_update_lsm_md(ll_i2mdexp(inode), ll_i2info(inode)->lli_lsm_md, - md->body, ll_md_blocking_ast); - - RETURN(rc); + RETURN(0); } void ll_clear_inode(struct inode *inode) diff --git a/lustre/lmv/lmv_intent.c b/lustre/lmv/lmv_intent.c index 6f92f48..9d16714 100644 --- a/lustre/lmv/lmv_intent.c +++ b/lustre/lmv/lmv_intent.c @@ -152,8 +152,8 @@ out: return rc; } -int lmv_revalidate_slaves(struct obd_export *exp, struct mdt_body *mbody, - struct lmv_stripe_md *lsm, +int lmv_revalidate_slaves(struct obd_export *exp, + const struct lmv_stripe_md *lsm, ldlm_blocking_callback cb_blocking, int extra_lock_flags) { @@ -162,11 +162,6 @@ int lmv_revalidate_slaves(struct obd_export *exp, struct mdt_body *mbody, struct ptlrpc_request *req = NULL; struct mdt_body *body; struct md_op_data *op_data; - unsigned long size = 0; - unsigned long nlink = 0; - obd_time atime = 0; - obd_time ctime = 0; - obd_time mtime = 0; int i; int rc = 0; @@ -249,6 +244,7 @@ int lmv_revalidate_slaves(struct obd_export *exp, struct mdt_body *mbody, i_size_write(inode, body->mbo_size); + inode->i_blocks = body->mbo_blocks; set_nlink(inode, body->mbo_nlink); LTIME_S(inode->i_atime) = body->mbo_atime; LTIME_S(inode->i_ctime) = body->mbo_ctime; @@ -256,45 +252,12 @@ int lmv_revalidate_slaves(struct obd_export *exp, struct mdt_body *mbody, } md_set_lock_data(tgt->ltd_exp, &lockh->cookie, inode, NULL); - - size += i_size_read(inode); - - if (i != 0) - nlink += inode->i_nlink - 2; - else - nlink += inode->i_nlink; - - atime = LTIME_S(inode->i_atime) > atime ? - LTIME_S(inode->i_atime) : atime; - ctime = LTIME_S(inode->i_ctime) > ctime ? - LTIME_S(inode->i_ctime) : ctime; - mtime = LTIME_S(inode->i_mtime) > mtime ? - LTIME_S(inode->i_mtime) : mtime; - if (it.d.lustre.it_lock_mode != 0 && lockh != NULL) { ldlm_lock_decref(lockh, it.d.lustre.it_lock_mode); it.d.lustre.it_lock_mode = 0; } - - CDEBUG(D_INODE, "i %d "DFID" size %llu, nlink %u, atime " - "%lu, mtime %lu, ctime %lu.\n", i, PFID(&fid), - i_size_read(inode), inode->i_nlink, - LTIME_S(inode->i_atime), LTIME_S(inode->i_mtime), - LTIME_S(inode->i_ctime)); } - /* - * update attr of master request. - */ - CDEBUG(D_INODE, "Return refreshed attrs: size = %lu nlink %lu atime " - LPU64 "ctime "LPU64" mtime "LPU64" for " DFID"\n", size, nlink, - atime, ctime, mtime, PFID(&lsm->lsm_md_oinfo[0].lmo_fid)); - - if (mbody != NULL) { - mbody->mbo_atime = atime; - mbody->mbo_ctime = ctime; - mbody->mbo_mtime = mtime; - } cleanup: if (req != NULL) ptlrpc_req_finished(req); @@ -450,7 +413,7 @@ lmv_intent_lookup(struct obd_export *exp, struct md_op_data *op_data, /* If RPC happens, lsm information will be revalidated * during update_inode process (see ll_update_lsm_md) */ if (op_data->op_mea2 != NULL) { - rc = lmv_revalidate_slaves(exp, NULL, op_data->op_mea2, + rc = lmv_revalidate_slaves(exp, op_data->op_mea2, cb_blocking, extra_lock_flags); if (rc != 0) diff --git a/lustre/lmv/lmv_internal.h b/lustre/lmv/lmv_internal.h index 40b81d1..aa61b22 100644 --- a/lustre/lmv/lmv_internal.h +++ b/lustre/lmv/lmv_internal.h @@ -63,8 +63,8 @@ int lmv_fid_alloc(const struct lu_env *env, struct obd_export *exp, int lmv_unpack_md(struct obd_export *exp, struct lmv_stripe_md **lsmp, const union lmv_mds_md *lmm, int stripe_count); -int lmv_revalidate_slaves(struct obd_export *exp, struct mdt_body *mbody, - struct lmv_stripe_md *lsm, +int lmv_revalidate_slaves(struct obd_export *exp, + const struct lmv_stripe_md *lsm, ldlm_blocking_callback cb_blocking, int extra_lock_flags); diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c index 69a2a3f..692aaa8 100644 --- a/lustre/lmv/lmv_obd.c +++ b/lustre/lmv/lmv_obd.c @@ -3438,25 +3438,27 @@ int lmv_quotacheck(struct obd_device *unused, struct obd_export *exp, RETURN(rc); } -int lmv_update_lsm_md(struct obd_export *exp, struct lmv_stripe_md *lsm, - struct mdt_body *body, ldlm_blocking_callback cb_blocking) -{ - return lmv_revalidate_slaves(exp, body, lsm, cb_blocking, 0); -} - -int lmv_merge_attr(struct obd_export *exp, const struct lmv_stripe_md *lsm, - struct cl_attr *attr) +static int lmv_merge_attr(struct obd_export *exp, + const struct lmv_stripe_md *lsm, + struct cl_attr *attr, + ldlm_blocking_callback cb_blocking) { + int rc; int i; + rc = lmv_revalidate_slaves(exp, lsm, cb_blocking, 0); + if (rc < 0) + return rc; + for (i = 0; i < lsm->lsm_md_stripe_count; i++) { struct inode *inode = lsm->lsm_md_oinfo[i].lmo_root; - CDEBUG(D_INFO, ""DFID" size %llu, nlink %u, atime %lu ctime" - "%lu, mtime %lu.\n", PFID(&lsm->lsm_md_oinfo[i].lmo_fid), - i_size_read(inode), inode->i_nlink, - LTIME_S(inode->i_atime), LTIME_S(inode->i_ctime), - LTIME_S(inode->i_mtime)); + CDEBUG(D_INFO, ""DFID" size %llu, blocks %llu nlink %u," + " atime %lu ctime %lu, mtime %lu.\n", + PFID(&lsm->lsm_md_oinfo[i].lmo_fid), + i_size_read(inode), (unsigned long long)inode->i_blocks, + inode->i_nlink, LTIME_S(inode->i_atime), + LTIME_S(inode->i_ctime), LTIME_S(inode->i_mtime)); /* for slave stripe, it needs to subtract nlink for . and .. */ if (i != 0) @@ -3465,6 +3467,7 @@ int lmv_merge_attr(struct obd_export *exp, const struct lmv_stripe_md *lsm, attr->cat_nlink = inode->i_nlink; attr->cat_size += i_size_read(inode); + attr->cat_blocks += inode->i_blocks; if (attr->cat_atime < LTIME_S(inode->i_atime)) attr->cat_atime = LTIME_S(inode->i_atime); @@ -3523,7 +3526,6 @@ struct md_ops lmv_md_ops = { .m_lock_match = lmv_lock_match, .m_get_lustre_md = lmv_get_lustre_md, .m_free_lustre_md = lmv_free_lustre_md, - .m_update_lsm_md = lmv_update_lsm_md, .m_merge_attr = lmv_merge_attr, .m_set_open_replay_data = lmv_set_open_replay_data, .m_clear_open_replay_data = lmv_clear_open_replay_data, diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c index 2f83b07..363538b 100644 --- a/lustre/obdclass/lprocfs_status.c +++ b/lustre/obdclass/lprocfs_status.c @@ -1363,7 +1363,6 @@ void lprocfs_init_mps_stats(int num_private_stats, struct lprocfs_stats *stats) LPROCFS_MD_OP_INIT(num_private_stats, stats, init_ea_size); LPROCFS_MD_OP_INIT(num_private_stats, stats, get_lustre_md); LPROCFS_MD_OP_INIT(num_private_stats, stats, free_lustre_md); - LPROCFS_MD_OP_INIT(num_private_stats, stats, update_lsm_md); LPROCFS_MD_OP_INIT(num_private_stats, stats, merge_attr); LPROCFS_MD_OP_INIT(num_private_stats, stats, set_open_replay_data); LPROCFS_MD_OP_INIT(num_private_stats, stats, clear_open_replay_data); -- 1.8.3.1