From 67aca1fcc6bed20794832decdba590a758d67d8f Mon Sep 17 00:00:00 2001 From: Bobi Jam Date: Thu, 15 Sep 2022 14:46:34 +0800 Subject: [PATCH] LU-16160 osc: take ldlm lock when queue sync pages osc_queue_sync_pages() add osc_extent to osc_object's IO extent list without taking ldlm locks, and then it calls osc_io_unplug_async() to queue the IO work for the client. This patch make sync page queuing take ldlm lock in the osc_extent. Signed-off-by: Bobi Jam Change-Id: Idefa2981e62a2a6e10d8b8a7692c0337b61b9052 Reviewed-on: https://review.whamcloud.com/48557 Tested-by: jenkins Reviewed-by: Mikhail Pershin Tested-by: Maloo Reviewed-by: Alex Zhuravlev Reviewed-by: Oleg Drokin --- lustre/include/lustre_osc.h | 3 +++ lustre/mdc/mdc_dev.c | 3 +++ lustre/osc/osc_cache.c | 7 +++++++ lustre/osc/osc_io.c | 1 + lustre/osc/osc_lock.c | 19 +++++++++++++++++++ lustre/tests/sanityn.sh | 45 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 78 insertions(+) diff --git a/lustre/include/lustre_osc.h b/lustre/include/lustre_osc.h index ca0e2ff..4abb8d7 100644 --- a/lustre/include/lustre_osc.h +++ b/lustre/include/lustre_osc.h @@ -140,6 +140,7 @@ struct osc_io { struct osc_extent *oi_trunc; /** write osc_lock for this IO, used by osc_extent_find(). */ struct osc_lock *oi_write_osclock; + struct osc_lock *oi_read_osclock; struct obdo oi_oa; struct osc_async_cbargs { bool opc_rpc_sent; @@ -715,6 +716,8 @@ int osc_lock_enqueue_wait(const struct lu_env *env, struct osc_object *obj, struct osc_lock *oscl); void osc_lock_set_writer(const struct lu_env *env, const struct cl_io *io, struct cl_object *obj, struct osc_lock *oscl); +void osc_lock_set_reader(const struct lu_env *env, const struct cl_io *io, + struct cl_object *obj, struct osc_lock *oscl); int osc_lock_print(const struct lu_env *env, void *cookie, lu_printer_t p, const struct cl_lock_slice *slice); void osc_lock_cancel(const struct lu_env *env, diff --git a/lustre/mdc/mdc_dev.c b/lustre/mdc/mdc_dev.c index 563187e..4f9e3cc 100644 --- a/lustre/mdc/mdc_dev.c +++ b/lustre/mdc/mdc_dev.c @@ -989,6 +989,9 @@ int mdc_lock_init(const struct lu_env *env, struct cl_object *obj, if (io->ci_type == CIT_WRITE || cl_io_is_mkwrite(io)) osc_lock_set_writer(env, io, obj, ols); + else if (io->ci_type == CIT_READ || + (io->ci_type == CIT_FAULT && !io->u.ci_fault.ft_mkwrite)) + osc_lock_set_reader(env, io, obj, ols); LDLM_DEBUG_NOLOCK("lock %p, mdc lock %p, flags %llx\n", lock, ols, ols->ols_flags); diff --git a/lustre/osc/osc_cache.c b/lustre/osc/osc_cache.c index 0e6e50d..3721849 100644 --- a/lustre/osc/osc_cache.c +++ b/lustre/osc/osc_cache.c @@ -2582,6 +2582,7 @@ int osc_queue_sync_pages(const struct lu_env *env, struct cl_io *io, struct osc_object *obj, struct list_head *list, int brw_flags) { + struct osc_io *oio = osc_env_io(env); struct client_obd *cli = osc_cli(obj); struct osc_extent *ext; struct osc_async_page *oap; @@ -2590,6 +2591,7 @@ int osc_queue_sync_pages(const struct lu_env *env, struct cl_io *io, bool can_merge = true; pgoff_t start = CL_PAGE_EOF; pgoff_t end = 0; + struct osc_lock *oscl; ENTRY; list_for_each_entry(oap, list, oap_pending_item) { @@ -2629,6 +2631,11 @@ int osc_queue_sync_pages(const struct lu_env *env, struct cl_io *io, ext->oe_srvlock = !!(brw_flags & OBD_BRW_SRVLOCK); ext->oe_ndelay = !!(brw_flags & OBD_BRW_NDELAY); ext->oe_dio = !!(brw_flags & OBD_BRW_NOCACHE); + oscl = oio->oi_write_osclock ? : oio->oi_read_osclock; + if (oscl && oscl->ols_dlmlock != NULL) { + ext->oe_dlmlock = LDLM_LOCK_GET(oscl->ols_dlmlock); + lu_ref_add(&ext->oe_dlmlock->l_reference, "osc_extent", ext); + } if (ext->oe_dio && !ext->oe_rw) { /* direct io write */ int grants; int ppc; diff --git a/lustre/osc/osc_io.c b/lustre/osc/osc_io.c index 46a9ced..aea22bb 100644 --- a/lustre/osc/osc_io.c +++ b/lustre/osc/osc_io.c @@ -465,6 +465,7 @@ void osc_io_rw_iter_fini(const struct lu_env *env, oio->oi_lru_reserved = 0; } oio->oi_write_osclock = NULL; + oio->oi_read_osclock = NULL; osc_io_iter_fini(env, ios); } diff --git a/lustre/osc/osc_lock.c b/lustre/osc/osc_lock.c index 7230e8d..1ebd516 100644 --- a/lustre/osc/osc_lock.c +++ b/lustre/osc/osc_lock.c @@ -1191,6 +1191,22 @@ void osc_lock_set_writer(const struct lu_env *env, const struct cl_io *io, } EXPORT_SYMBOL(osc_lock_set_writer); +void osc_lock_set_reader(const struct lu_env *env, const struct cl_io *io, + struct cl_object *obj, struct osc_lock *oscl) +{ + struct osc_io *oio = osc_env_io(env); + + if (!cl_object_same(io->ci_obj, obj)) + return; + + if (oscl->ols_glimpse || osc_lock_is_lockless(oscl)) + return; + + if (oio->oi_read_osclock == NULL) + oio->oi_read_osclock = oscl; +} +EXPORT_SYMBOL(osc_lock_set_reader); + int osc_lock_init(const struct lu_env *env, struct cl_object *obj, struct cl_lock *lock, const struct cl_io *io) @@ -1236,6 +1252,9 @@ int osc_lock_init(const struct lu_env *env, if (io->ci_type == CIT_WRITE || cl_io_is_mkwrite(io)) osc_lock_set_writer(env, io, obj, oscl); + else if (io->ci_type == CIT_READ || + (io->ci_type == CIT_FAULT && !io->u.ci_fault.ft_mkwrite)) + osc_lock_set_reader(env, io, obj, oscl); LDLM_DEBUG_NOLOCK("lock %p, osc lock %p, flags %#llx", lock, oscl, oscl->ols_flags); diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh index 308072b..4a9771d 100755 --- a/lustre/tests/sanityn.sh +++ b/lustre/tests/sanityn.sh @@ -586,6 +586,51 @@ test_16f() { # LU-14541 } run_test 16f "rw sequential consistency vs drop_caches" +test_16h() { + local tf=$DIR/$tdir/$tfile + local tf2=$DIR2/$tdir/$tfile + local cmd="$MMAP_CAT $tf | od -x | tail -q -n4" + local cmd2="$MMAP_CAT $tf2 | od -x | tail -q -n4" + + test_mkdir $DIR/$tdir + + # create file and populate data + cp /etc/passwd $tf || error "cp failed" + + local size=$(stat -c %s $tf) + c1=$(eval $cmd) + c2=$(eval $cmd2) + if [[ "$c1" != "$c2" ]]; then + echo " ------- mount 1 read --------" + echo $c1 + echo " ------- mount 2 read --------" + echo $c2 + error "content mismatch" + fi + + echo " ------- before truncate --------" + echo $c1 + + # truncate file + $TRUNCATE $tf $((size / 2)) || error "truncate file" + + #cancel_lru_locks + echo " ------- after truncate --------" + + # repeat the comparison + c1=$(eval $cmd) + c2=$(eval $cmd2) + if [[ "$c1" != "$c2" ]]; then + echo " ------- mount 1 read --------" + echo $c1 + echo " ------- mount 2 read --------" + echo $c2 + error "content mismatch after truncate" + fi + echo $c2 +} +run_test 16h "mmap read after truncate file" + test_17() { # bug 3513, 3667 remote_ost_nodsh && skip "remote OST with nodsh" && return -- 1.8.3.1