LU-16160 osc: take ldlm lock when queue sync pages

osc_queue_sync_pages() add osc_extent to osc_object's IO extent
list without taking ldlm locks, and then it calls
osc_io_unplug_async() to queue the IO work for the client.

This patch make sync page queuing take ldlm lock in the
osc_extent.

Lustre-change: https://review.whamcloud.com/48557
Lustre-commit: 67aca1fcc6bed20794832decdba590a758d67d8fp

Signed-off-by: Bobi Jam <bobijam@whamcloud.com>
Change-Id: Idefa2981e62a2a6e10d8b8a7692c0337b61b9052
Reviewed-on: https://review.whamcloud.com/48597
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Alex Zhuravlev <bzzz@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre/include/lustre_osc.h b/lustre/include/lustre_osc.h
index 2c466ce..4a30995 100644 (file)
--- a/lustre/include/lustre_osc.h
+++ b/lustre/include/lustre_osc.h
@@ -141,6 +141,7 @@ struct osc_io {
        struct osc_extent *oi_trunc;
        /** write osc_lock for this IO, used by osc_extent_find(). */
        struct osc_lock   *oi_write_osclock;
+       struct osc_lock   *oi_read_osclock;
        struct obdo        oi_oa;
        struct osc_async_cbargs {
                bool              opc_rpc_sent;
@@ -714,6 +715,8 @@ int osc_lock_enqueue_wait(const struct lu_env *env, struct osc_object *obj,
                          struct osc_lock *oscl);
 void osc_lock_set_writer(const struct lu_env *env, const struct cl_io *io,
                         struct cl_object *obj, struct osc_lock *oscl);
+void osc_lock_set_reader(const struct lu_env *env, const struct cl_io *io,
+                        struct cl_object *obj, struct osc_lock *oscl);
 int osc_lock_print(const struct lu_env *env, void *cookie,
                   lu_printer_t p, const struct cl_lock_slice *slice);
 void osc_lock_cancel(const struct lu_env *env,

diff --git a/lustre/mdc/mdc_dev.c b/lustre/mdc/mdc_dev.c
index d18238d..8a0687b 100644 (file)
--- a/lustre/mdc/mdc_dev.c
+++ b/lustre/mdc/mdc_dev.c
@@ -983,6 +983,9 @@ int mdc_lock_init(const struct lu_env *env, struct cl_object *obj,
 
        if (io->ci_type == CIT_WRITE || cl_io_is_mkwrite(io))
                osc_lock_set_writer(env, io, obj, ols);
+       else if (io->ci_type == CIT_READ ||
+                (io->ci_type == CIT_FAULT && !io->u.ci_fault.ft_mkwrite))
+               osc_lock_set_reader(env, io, obj, ols);
 
        LDLM_DEBUG_NOLOCK("lock %p, mdc lock %p, flags %llx\n",
                          lock, ols, ols->ols_flags);

diff --git a/lustre/osc/osc_cache.c b/lustre/osc/osc_cache.c
index a8fc476..7aec989 100644 (file)
--- a/lustre/osc/osc_cache.c
+++ b/lustre/osc/osc_cache.c
@@ -2609,6 +2609,7 @@ int osc_queue_sync_pages(const struct lu_env *env, struct cl_io *io,
                         struct osc_object *obj, struct list_head *list,
                         int brw_flags)
 {
+       struct osc_io *oio = osc_env_io(env);
        struct client_obd     *cli = osc_cli(obj);
        struct osc_extent     *ext;
        struct osc_async_page *oap;
@@ -2617,6 +2618,7 @@ int osc_queue_sync_pages(const struct lu_env *env, struct cl_io *io,
        bool    can_merge   = true;
        pgoff_t start      = CL_PAGE_EOF;
        pgoff_t end        = 0;
+       struct osc_lock *oscl;
        ENTRY;
 
        list_for_each_entry(oap, list, oap_pending_item) {
@@ -2656,6 +2658,11 @@ int osc_queue_sync_pages(const struct lu_env *env, struct cl_io *io,
        ext->oe_srvlock = !!(brw_flags & OBD_BRW_SRVLOCK);
        ext->oe_ndelay = !!(brw_flags & OBD_BRW_NDELAY);
        ext->oe_dio = !!(brw_flags & OBD_BRW_NOCACHE);
+       oscl = oio->oi_write_osclock ? : oio->oi_read_osclock;
+       if (oscl && oscl->ols_dlmlock != NULL) {
+               ext->oe_dlmlock = LDLM_LOCK_GET(oscl->ols_dlmlock);
+               lu_ref_add(&ext->oe_dlmlock->l_reference, "osc_extent", ext);
+       }
        if (ext->oe_dio && !ext->oe_rw) { /* direct io write */
                int grants;
                int ppc;

diff --git a/lustre/osc/osc_io.c b/lustre/osc/osc_io.c
index 0f72479..d8fdebd 100644 (file)
--- a/lustre/osc/osc_io.c
+++ b/lustre/osc/osc_io.c
@@ -455,6 +455,7 @@ void osc_io_rw_iter_fini(const struct lu_env *env,
                oio->oi_lru_reserved = 0;
        }
        oio->oi_write_osclock = NULL;
+       oio->oi_read_osclock = NULL;
 
        osc_io_iter_fini(env, ios);
 }

diff --git a/lustre/osc/osc_lock.c b/lustre/osc/osc_lock.c
index 1dbb1a5..1b33501 100644 (file)
--- a/lustre/osc/osc_lock.c
+++ b/lustre/osc/osc_lock.c
@@ -1181,6 +1181,22 @@ void osc_lock_set_writer(const struct lu_env *env, const struct cl_io *io,
 }
 EXPORT_SYMBOL(osc_lock_set_writer);
 
+void osc_lock_set_reader(const struct lu_env *env, const struct cl_io *io,
+                        struct cl_object *obj, struct osc_lock *oscl)
+{
+       struct osc_io *oio = osc_env_io(env);
+
+       if (!cl_object_same(io->ci_obj, obj))
+               return;
+
+       if (oscl->ols_glimpse || osc_lock_is_lockless(oscl))
+               return;
+
+       if (oio->oi_read_osclock == NULL)
+               oio->oi_read_osclock = oscl;
+}
+EXPORT_SYMBOL(osc_lock_set_reader);
+
 int osc_lock_init(const struct lu_env *env,
                  struct cl_object *obj, struct cl_lock *lock,
                  const struct cl_io *io)
@@ -1226,6 +1242,9 @@ int osc_lock_init(const struct lu_env *env,
 
        if (io->ci_type == CIT_WRITE || cl_io_is_mkwrite(io))
                osc_lock_set_writer(env, io, obj, oscl);
+       else if (io->ci_type == CIT_READ ||
+                (io->ci_type == CIT_FAULT && !io->u.ci_fault.ft_mkwrite))
+               osc_lock_set_reader(env, io, obj, oscl);
 
        LDLM_DEBUG_NOLOCK("lock %p, osc lock %p, flags %#llx",
                          lock, oscl, oscl->ols_flags);

diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh
index cc36220..76fe8f3 100755 (executable)
--- a/lustre/tests/sanityn.sh
+++ b/lustre/tests/sanityn.sh
@@ -585,6 +585,51 @@ test_16f() { # LU-14541
 }
 run_test 16f "rw sequential consistency vs drop_caches"
 
+test_16h() {
+       local tf=$DIR/$tdir/$tfile
+       local tf2=$DIR2/$tdir/$tfile
+       local cmd="$MMAP_CAT $tf | od -x | tail -q -n4"
+       local cmd2="$MMAP_CAT $tf2 | od -x | tail -q -n4"
+
+       test_mkdir $DIR/$tdir
+
+       # create file and populate data
+       cp /etc/passwd $tf || error "cp failed"
+
+       local size=$(stat -c %s $tf)
+       c1=$(eval $cmd)
+       c2=$(eval $cmd2)
+       if [[ "$c1" != "$c2" ]]; then
+               echo "  ------- mount 1 read --------"
+               echo $c1
+               echo "  ------- mount 2 read --------"
+               echo $c2
+               error "content mismatch"
+       fi
+
+       echo "  ------- before truncate --------"
+       echo $c1
+
+       # truncate file
+       $TRUNCATE $tf $((size / 2)) || error "truncate file"
+
+       #cancel_lru_locks
+       echo "  ------- after truncate --------"
+
+       # repeat the comparison
+       c1=$(eval $cmd)
+       c2=$(eval $cmd2)
+       if [[ "$c1" != "$c2" ]]; then
+               echo "  ------- mount 1 read --------"
+               echo $c1
+               echo "  ------- mount 2 read --------"
+               echo $c2
+               error "content mismatch after truncate"
+       fi
+       echo $c2
+}
+run_test 16h "mmap read after truncate file"
+
 test_17() { # bug 3513, 3667
        remote_ost_nodsh && skip "remote OST with nodsh" && return