From: Jinshan Xiong Date: Tue, 31 Dec 2013 21:49:33 +0000 (-0800) Subject: LU-3254 llite: access layout version under a lock X-Git-Tag: 2.5.58~73 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=681101c0a1eceed0caa6fc5b7e748ba868e436cf LU-3254 llite: access layout version under a lock We used to access layout version under the protection of ldlm lock, this introduces extra overhead for dlm lock matching. In this patch, lli_layout_lock is introduced to access the layout version. Also, when a layout lock is losing, we should tear down mmap of the correspoding inode to avoid stale data accessing in the future. This is part of technical verification of replication. Signed-off-by: Jinshan Xiong Change-Id: I657c6a8b27f2cdaf468dc208c43c1d92f3843705 Reviewed-on: http://review.whamcloud.com/8689 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Lai Siyao Reviewed-by: Bobi Jam Reviewed-by: Oleg Drokin --- diff --git a/lustre/llite/file.c b/lustre/llite/file.c index b92443f..9948641 100644 --- a/lustre/llite/file.c +++ b/lustre/llite/file.c @@ -3836,7 +3836,7 @@ static int ll_layout_lock_set(struct lustre_handle *lockh, ldlm_mode_t mode, if (lvb_ready) { /* layout_gen must be valid if layout lock is not * cancelled and stripe has already set */ - *gen = lli->lli_layout_gen; + *gen = ll_layout_version_get(lli); rc = 0; } GOTO(out, rc); @@ -3937,32 +3937,20 @@ int ll_layout_refresh(struct inode *inode, __u32 *gen) int rc; ENTRY; - *gen = lli->lli_layout_gen; - if (!(sbi->ll_flags & LL_SBI_LAYOUT_LOCK)) + *gen = ll_layout_version_get(lli); + if (!(sbi->ll_flags & LL_SBI_LAYOUT_LOCK) || *gen != LL_LAYOUT_GEN_NONE) RETURN(0); /* sanity checks */ LASSERT(fid_is_sane(ll_inode2fid(inode))); LASSERT(S_ISREG(inode->i_mode)); - /* mostly layout lock is caching on the local side, so try to match - * it before grabbing layout lock mutex. */ - mode = ll_take_md_lock(inode, MDS_INODELOCK_LAYOUT, &lockh, 0, - LCK_CR | LCK_CW | LCK_PR | LCK_PW); - if (mode != 0) { /* hit cached lock */ - rc = ll_layout_lock_set(&lockh, mode, inode, gen, false); - if (rc == 0) - RETURN(0); - - /* better hold lli_layout_mutex to try again otherwise - * it will have starvation problem. */ - } - /* take layout lock mutex to enqueue layout lock exclusively. */ mutex_lock(&lli->lli_layout_mutex); again: - /* try again. Maybe somebody else has done this. */ + /* mostly layout lock is caching on the local side, so try to match + * it before grabbing layout lock mutex. */ mode = ll_take_md_lock(inode, MDS_INODELOCK_LAYOUT, &lockh, 0, LCK_CR | LCK_CW | LCK_PR | LCK_PW); if (mode != 0) { /* hit cached lock */ diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h index dae4db5..702783e 100644 --- a/lustre/llite/llite_internal.h +++ b/lustre/llite/llite_internal.h @@ -291,14 +291,33 @@ struct ll_inode_info { /* mutex to request for layout lock exclusively. */ struct mutex lli_layout_mutex; - /* valid only inside LAYOUT ibits lock, protected by lli_layout_mutex */ + /* Layout version, protected by lli_layout_lock */ __u32 lli_layout_gen; + spinlock_t lli_layout_lock; struct rw_semaphore lli_xattrs_list_rwsem; struct mutex lli_xattrs_enq_lock; struct list_head lli_xattrs; /* ll_xattr_entry->xe_list */ }; +static inline __u32 ll_layout_version_get(struct ll_inode_info *lli) +{ + __u32 gen; + + spin_lock(&lli->lli_layout_lock); + gen = lli->lli_layout_gen; + spin_unlock(&lli->lli_layout_lock); + + return gen; +} + +static inline void ll_layout_version_set(struct ll_inode_info *lli, __u32 gen) +{ + spin_lock(&lli->lli_layout_lock); + lli->lli_layout_gen = gen; + spin_unlock(&lli->lli_layout_lock); +} + int ll_xattr_cache_destroy(struct inode *inode); int ll_xattr_cache_get(struct inode *inode, diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index 2edfe43..090d4c5 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -985,7 +985,8 @@ void ll_lli_init(struct ll_inode_info *lli) mutex_init(&lli->lli_och_mutex); spin_lock_init(&lli->lli_agl_lock); lli->lli_has_smd = false; - lli->lli_layout_gen = LL_LAYOUT_GEN_NONE; + spin_lock_init(&lli->lli_layout_lock); + ll_layout_version_set(lli, LL_LAYOUT_GEN_NONE); lli->lli_clob = NULL; init_rwsem(&lli->lli_xattrs_list_rwsem); diff --git a/lustre/llite/vvp_io.c b/lustre/llite/vvp_io.c index 4c7d022..36ef95e 100644 --- a/lustre/llite/vvp_io.c +++ b/lustre/llite/vvp_io.c @@ -83,7 +83,7 @@ static bool can_populate_pages(const struct lu_env *env, struct cl_io *io, case CIT_WRITE: /* don't need lock here to check lli_layout_gen as we have held * extent lock and GROUP lock has to hold to swap layout */ - if (lli->lli_layout_gen != cio->cui_layout_gen) { + if (ll_layout_version_get(lli) != cio->cui_layout_gen) { io->ci_need_restart = 1; /* this will return application a short read/write */ io->ci_continue = 0; diff --git a/lustre/llite/vvp_object.c b/lustre/llite/vvp_object.c index 92c7763..6df7eb1 100644 --- a/lustre/llite/vvp_object.c +++ b/lustre/llite/vvp_object.c @@ -129,7 +129,22 @@ int vvp_conf_set(const struct lu_env *env, struct cl_object *obj, struct ll_inode_info *lli = ll_i2info(conf->coc_inode); if (conf->coc_opc == OBJECT_CONF_INVALIDATE) { - lli->lli_layout_gen = LL_LAYOUT_GEN_NONE; + CDEBUG(D_VFSTRACE, DFID ": losing layout lock\n", + PFID(&lli->lli_fid)); + + ll_layout_version_set(lli, LL_LAYOUT_GEN_NONE); + + /* Clean up page mmap for this inode. + * The reason for us to do this is that if the page has + * already been installed into memory space, the process + * can access it without interacting with lustre, so this + * page may be stale due to layout change, and the process + * will never be notified. + * This operation is expensive but mmap processes have to pay + * a price themselves. */ + unmap_mapping_range(conf->coc_inode->i_mapping, + 0, OBD_OBJECT_EOF, 0); + return 0; } @@ -137,18 +152,18 @@ int vvp_conf_set(const struct lu_env *env, struct cl_object *obj, return 0; if (conf->u.coc_md != NULL && conf->u.coc_md->lsm != NULL) { - CDEBUG(D_VFSTRACE, "layout lock change: %u -> %u\n", - lli->lli_layout_gen, - conf->u.coc_md->lsm->lsm_layout_gen); + CDEBUG(D_VFSTRACE, DFID ": layout version change: %u -> %u\n", + PFID(&lli->lli_fid), lli->lli_layout_gen, + conf->u.coc_md->lsm->lsm_layout_gen); lli->lli_has_smd = lsm_has_objects(conf->u.coc_md->lsm); - lli->lli_layout_gen = conf->u.coc_md->lsm->lsm_layout_gen; + ll_layout_version_set(lli, conf->u.coc_md->lsm->lsm_layout_gen); } else { - CDEBUG(D_VFSTRACE, "layout lock destroyed: %u.\n", - lli->lli_layout_gen); + CDEBUG(D_VFSTRACE, DFID ": layout nuked: %u.\n", + PFID(&lli->lli_fid), lli->lli_layout_gen); lli->lli_has_smd = false; - lli->lli_layout_gen = LL_LAYOUT_GEN_EMPTY; + ll_layout_version_set(lli, LL_LAYOUT_GEN_EMPTY); } return 0; } diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh index 4b41dd6..8918f5d 100644 --- a/lustre/tests/sanityn.sh +++ b/lustre/tests/sanityn.sh @@ -2344,6 +2344,32 @@ test_51c() { } run_test 51c "layout lock: IT_LAYOUT blocked and correct layout can be returned" +test_51d() { + dd if=/dev/zero of=/$DIR1/$tfile bs=1M count=1 + cancel_lru_locks mdc + + # open should grant LAYOUT lock, mmap and read will install pages + $MULTIOP $DIR1/$tfile oO_RDWR:SMR_Uc & + local PID=$! + sleep 1 + + # rss before revoking + local br=$(grep -A 10 $tfile /proc/$PID/smaps | awk '/^Rss/{print $2}') + echo "Before revoking layout lock: $br KB mapped" + + # delete the file will revoke layout lock + rm -f $DIR2/$tfile + + # rss after revoking + local ar=$(grep -A 10 $tfile /proc/$PID/smaps | awk '/^Rss/{print $2}') + + kill -USR1 $PID + wait $PID || error + + [ $ar -eq 0 ] || error "rss before: $br, after $ar, some pages remained" +} +run_test 51d "layout lock: losing layout lock should clean up memory map region" + test_60() { [[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.3.0) ]] || { skip "Need MDS version at least 2.3.0"; return; }