LU-3254 llite: access layout version under a lock

We used to access layout version under the protection of ldlm
lock, this introduces extra overhead for dlm lock matching.

In this patch, lli_layout_lock is introduced to access the layout
version. Also, when a layout lock is losing, we should tear down
mmap of the correspoding inode to avoid stale data accessing in the
future.

This is part of technical verification of replication.

Signed-off-by: Jinshan Xiong <jinshan.xiong@intel.com>
Change-Id: I657c6a8b27f2cdaf468dc208c43c1d92f3843705
Reviewed-on: http://review.whamcloud.com/8689
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Lai Siyao <lai.siyao@intel.com>
Reviewed-by: Bobi Jam <bobijam@gmail.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/llite/file.c b/lustre/llite/file.c
index b92443f..9948641 100644 (file)
--- a/lustre/llite/file.c
+++ b/lustre/llite/file.c
@@ -3836,7 +3836,7 @@ static int ll_layout_lock_set(struct lustre_handle *lockh, ldlm_mode_t mode,
                if (lvb_ready) {
                        /* layout_gen must be valid if layout lock is not
                         * cancelled and stripe has already set */
-                       *gen = lli->lli_layout_gen;
+                       *gen = ll_layout_version_get(lli);
                        rc = 0;
                }
                GOTO(out, rc);
@@ -3937,32 +3937,20 @@ int ll_layout_refresh(struct inode *inode, __u32 *gen)
        int rc;
        ENTRY;
 
-       *gen = lli->lli_layout_gen;
-       if (!(sbi->ll_flags & LL_SBI_LAYOUT_LOCK))
+       *gen = ll_layout_version_get(lli);
+       if (!(sbi->ll_flags & LL_SBI_LAYOUT_LOCK) || *gen != LL_LAYOUT_GEN_NONE)
                RETURN(0);
 
        /* sanity checks */
        LASSERT(fid_is_sane(ll_inode2fid(inode)));
        LASSERT(S_ISREG(inode->i_mode));
 
-       /* mostly layout lock is caching on the local side, so try to match
-        * it before grabbing layout lock mutex. */
-       mode = ll_take_md_lock(inode, MDS_INODELOCK_LAYOUT, &lockh, 0,
-                              LCK_CR | LCK_CW | LCK_PR | LCK_PW);
-       if (mode != 0) { /* hit cached lock */
-               rc = ll_layout_lock_set(&lockh, mode, inode, gen, false);
-               if (rc == 0)
-                       RETURN(0);
-
-               /* better hold lli_layout_mutex to try again otherwise
-                * it will have starvation problem. */
-       }
-
        /* take layout lock mutex to enqueue layout lock exclusively. */
        mutex_lock(&lli->lli_layout_mutex);
 
 again:
-       /* try again. Maybe somebody else has done this. */
+       /* mostly layout lock is caching on the local side, so try to match
+        * it before grabbing layout lock mutex. */
        mode = ll_take_md_lock(inode, MDS_INODELOCK_LAYOUT, &lockh, 0,
                               LCK_CR | LCK_CW | LCK_PR | LCK_PW);
        if (mode != 0) { /* hit cached lock */

diff --git a/lustre/llite/llite_internal.h b/lustre/llite/llite_internal.h
index dae4db5..702783e 100644 (file)
--- a/lustre/llite/llite_internal.h
+++ b/lustre/llite/llite_internal.h
@@ -291,14 +291,33 @@ struct ll_inode_info {
 
        /* mutex to request for layout lock exclusively. */
        struct mutex                    lli_layout_mutex;
-       /* valid only inside LAYOUT ibits lock, protected by lli_layout_mutex */
+       /* Layout version, protected by lli_layout_lock */
        __u32                           lli_layout_gen;
+       spinlock_t                      lli_layout_lock;
 
        struct rw_semaphore             lli_xattrs_list_rwsem;
        struct mutex                    lli_xattrs_enq_lock;
        struct list_head                lli_xattrs; /* ll_xattr_entry->xe_list */
 };
 
+static inline __u32 ll_layout_version_get(struct ll_inode_info *lli)
+{
+       __u32 gen;
+
+       spin_lock(&lli->lli_layout_lock);
+       gen = lli->lli_layout_gen;
+       spin_unlock(&lli->lli_layout_lock);
+
+       return gen;
+}
+
+static inline void ll_layout_version_set(struct ll_inode_info *lli, __u32 gen)
+{
+       spin_lock(&lli->lli_layout_lock);
+       lli->lli_layout_gen = gen;
+       spin_unlock(&lli->lli_layout_lock);
+}
+
 int ll_xattr_cache_destroy(struct inode *inode);
 
 int ll_xattr_cache_get(struct inode *inode,

diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c
index 2edfe43..090d4c5 100644 (file)
--- a/lustre/llite/llite_lib.c
+++ b/lustre/llite/llite_lib.c
@@ -985,7 +985,8 @@ void ll_lli_init(struct ll_inode_info *lli)
        mutex_init(&lli->lli_och_mutex);
        spin_lock_init(&lli->lli_agl_lock);
        lli->lli_has_smd = false;
-       lli->lli_layout_gen = LL_LAYOUT_GEN_NONE;
+       spin_lock_init(&lli->lli_layout_lock);
+       ll_layout_version_set(lli, LL_LAYOUT_GEN_NONE);
        lli->lli_clob = NULL;
 
        init_rwsem(&lli->lli_xattrs_list_rwsem);

diff --git a/lustre/llite/vvp_io.c b/lustre/llite/vvp_io.c
index 4c7d022..36ef95e 100644 (file)
--- a/lustre/llite/vvp_io.c
+++ b/lustre/llite/vvp_io.c
@@ -83,7 +83,7 @@ static bool can_populate_pages(const struct lu_env *env, struct cl_io *io,
        case CIT_WRITE:
                /* don't need lock here to check lli_layout_gen as we have held
                 * extent lock and GROUP lock has to hold to swap layout */
-               if (lli->lli_layout_gen != cio->cui_layout_gen) {
+               if (ll_layout_version_get(lli) != cio->cui_layout_gen) {
                        io->ci_need_restart = 1;
                        /* this will return application a short read/write */
                        io->ci_continue = 0;

diff --git a/lustre/llite/vvp_object.c b/lustre/llite/vvp_object.c
index 92c7763..6df7eb1 100644 (file)
--- a/lustre/llite/vvp_object.c
+++ b/lustre/llite/vvp_object.c
@@ -129,7 +129,22 @@ int vvp_conf_set(const struct lu_env *env, struct cl_object *obj,
        struct ll_inode_info *lli = ll_i2info(conf->coc_inode);
 
        if (conf->coc_opc == OBJECT_CONF_INVALIDATE) {
-               lli->lli_layout_gen = LL_LAYOUT_GEN_NONE;
+               CDEBUG(D_VFSTRACE, DFID ": losing layout lock\n",
+                      PFID(&lli->lli_fid));
+
+               ll_layout_version_set(lli, LL_LAYOUT_GEN_NONE);
+
+               /* Clean up page mmap for this inode.
+                * The reason for us to do this is that if the page has
+                * already been installed into memory space, the process
+                * can access it without interacting with lustre, so this
+                * page may be stale due to layout change, and the process
+                * will never be notified.
+                * This operation is expensive but mmap processes have to pay
+                * a price themselves. */
+               unmap_mapping_range(conf->coc_inode->i_mapping,
+                                   0, OBD_OBJECT_EOF, 0);
+
                return 0;
        }
 
@@ -137,18 +152,18 @@ int vvp_conf_set(const struct lu_env *env, struct cl_object *obj,
                return 0;
 
        if (conf->u.coc_md != NULL && conf->u.coc_md->lsm != NULL) {
-               CDEBUG(D_VFSTRACE, "layout lock change: %u -> %u\n",
-                       lli->lli_layout_gen,
-                       conf->u.coc_md->lsm->lsm_layout_gen);
+               CDEBUG(D_VFSTRACE, DFID ": layout version change: %u -> %u\n",
+                      PFID(&lli->lli_fid), lli->lli_layout_gen,
+                      conf->u.coc_md->lsm->lsm_layout_gen);
 
                lli->lli_has_smd = lsm_has_objects(conf->u.coc_md->lsm);
-               lli->lli_layout_gen = conf->u.coc_md->lsm->lsm_layout_gen;
+               ll_layout_version_set(lli, conf->u.coc_md->lsm->lsm_layout_gen);
        } else {
-               CDEBUG(D_VFSTRACE, "layout lock destroyed: %u.\n",
-                       lli->lli_layout_gen);
+               CDEBUG(D_VFSTRACE, DFID ": layout nuked: %u.\n",
+                      PFID(&lli->lli_fid), lli->lli_layout_gen);
 
                lli->lli_has_smd = false;
-               lli->lli_layout_gen = LL_LAYOUT_GEN_EMPTY;
+               ll_layout_version_set(lli, LL_LAYOUT_GEN_EMPTY);
        }
        return 0;
 }

diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh
index 4b41dd6..8918f5d 100644 (file)
--- a/lustre/tests/sanityn.sh
+++ b/lustre/tests/sanityn.sh
@@ -2344,6 +2344,32 @@ test_51c() {
 }
 run_test 51c "layout lock: IT_LAYOUT blocked and correct layout can be returned"
 
+test_51d() {
+       dd if=/dev/zero of=/$DIR1/$tfile bs=1M count=1
+       cancel_lru_locks mdc
+
+       # open should grant LAYOUT lock, mmap and read will install pages
+       $MULTIOP $DIR1/$tfile oO_RDWR:SMR_Uc &
+       local PID=$!
+       sleep 1
+
+       # rss before revoking
+       local br=$(grep -A 10 $tfile /proc/$PID/smaps | awk '/^Rss/{print $2}')
+       echo "Before revoking layout lock: $br KB mapped"
+
+       # delete the file will revoke layout lock
+       rm -f $DIR2/$tfile
+
+       # rss after revoking
+       local ar=$(grep -A 10 $tfile /proc/$PID/smaps | awk '/^Rss/{print $2}')
+
+       kill -USR1 $PID
+       wait $PID || error
+
+       [ $ar -eq 0 ] || error "rss before: $br, after $ar, some pages remained"
+}
+run_test 51d "layout lock: losing layout lock should clean up memory map region"
+
 test_60() {
        [[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.3.0) ]] ||
        { skip "Need MDS version at least 2.3.0"; return; }