From 8ed361345154138473707855b2030bad371e96dd Mon Sep 17 00:00:00 2001 From: Li Dongyang Date: Fri, 14 Dec 2018 13:16:39 +1100 Subject: [PATCH] LU-11775 obdclass: protect imp_sec using rwlock_t We've seen spinlock contention on imp_lock in sptlrpc_import_sec_ref(), introduce a new rwlock imp_sec_lock to protect imp_sec instead of using imp_lock. This patch also removes imp_sec_mutex from obd_import, which is not needed, to avoid confusion between imp_sec_lock/mutex. Test-Parameters: testlist=sanity-sec envdefinitions=SHARED_KEY=true Signed-off-by: Li Dongyang Change-Id: I3be3a6d1225888134bf7dd58e4b05b864f8415b4 Reviewed-on: https://review.whamcloud.com/33861 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Alexey Lyashkov Reviewed-by: Alexandr Boyko Reviewed-by: Sebastien Buisson Reviewed-by: Oleg Drokin --- lustre/include/lustre_import.h | 2 +- lustre/obdclass/genops.c | 2 +- lustre/ptlrpc/sec.c | 15 ++++++--------- lustre/ptlrpc/sec_config.c | 4 ++-- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/lustre/include/lustre_import.h b/lustre/include/lustre_import.h index 8f52c66..0e34d55 100644 --- a/lustre/include/lustre_import.h +++ b/lustre/include/lustre_import.h @@ -210,7 +210,7 @@ struct obd_import { * @{ */ struct ptlrpc_sec *imp_sec; - struct mutex imp_sec_mutex; + rwlock_t imp_sec_lock; time64_t imp_sec_expire; pid_t imp_sec_refpid; /** @} */ diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c index c5404d8..883e9a4 100644 --- a/lustre/obdclass/genops.c +++ b/lustre/obdclass/genops.c @@ -1326,7 +1326,7 @@ struct obd_import *class_new_import(struct obd_device *obd) imp->imp_last_success_conn = 0; imp->imp_state = LUSTRE_IMP_NEW; imp->imp_obd = class_incref(obd, "import", imp); - mutex_init(&imp->imp_sec_mutex); + rwlock_init(&imp->imp_sec_lock); init_waitqueue_head(&imp->imp_recovery_waitq); INIT_WORK(&imp->imp_zombie_work, obd_zombie_imp_cull); diff --git a/lustre/ptlrpc/sec.c b/lustre/ptlrpc/sec.c index ec05298..dba6352 100644 --- a/lustre/ptlrpc/sec.c +++ b/lustre/ptlrpc/sec.c @@ -366,13 +366,13 @@ static int import_sec_check_expire(struct obd_import *imp) { int adapt = 0; - spin_lock(&imp->imp_lock); + write_lock(&imp->imp_sec_lock); if (imp->imp_sec_expire && imp->imp_sec_expire < ktime_get_real_seconds()) { adapt = 1; imp->imp_sec_expire = 0; } - spin_unlock(&imp->imp_lock); + write_unlock(&imp->imp_sec_lock); if (!adapt) return 0; @@ -1392,9 +1392,9 @@ struct ptlrpc_sec *sptlrpc_import_sec_ref(struct obd_import *imp) { struct ptlrpc_sec *sec; - spin_lock(&imp->imp_lock); + read_lock(&imp->imp_sec_lock); sec = sptlrpc_sec_get(imp->imp_sec); - spin_unlock(&imp->imp_lock); + read_unlock(&imp->imp_sec_lock); return sec; } @@ -1407,10 +1407,10 @@ static void sptlrpc_import_sec_install(struct obd_import *imp, LASSERT_ATOMIC_POS(&sec->ps_refcount); - spin_lock(&imp->imp_lock); + write_lock(&imp->imp_sec_lock); old_sec = imp->imp_sec; imp->imp_sec = sec; - spin_unlock(&imp->imp_lock); + write_unlock(&imp->imp_sec_lock); if (old_sec) { sptlrpc_sec_kill(old_sec); @@ -1506,8 +1506,6 @@ int sptlrpc_import_sec_adapt(struct obd_import *imp, sptlrpc_flavor2name(&sf, str, sizeof(str))); } - mutex_lock(&imp->imp_sec_mutex); - newsec = sptlrpc_sec_create(imp, svc_ctx, &sf, sp); if (newsec) { sptlrpc_import_sec_install(imp, newsec); @@ -1518,7 +1516,6 @@ int sptlrpc_import_sec_adapt(struct obd_import *imp, rc = -EPERM; } - mutex_unlock(&imp->imp_sec_mutex); out: sptlrpc_sec_put(sec); RETURN(rc); diff --git a/lustre/ptlrpc/sec_config.c b/lustre/ptlrpc/sec_config.c index 7df845e..e7e08ed 100644 --- a/lustre/ptlrpc/sec_config.c +++ b/lustre/ptlrpc/sec_config.c @@ -911,11 +911,11 @@ void sptlrpc_conf_client_adapt(struct obd_device *obd) imp = obd->u.cli.cl_import; if (imp) { - spin_lock(&imp->imp_lock); + write_lock(&imp->imp_sec_lock); if (imp->imp_sec) imp->imp_sec_expire = ktime_get_real_seconds() + SEC_ADAPT_DELAY; - spin_unlock(&imp->imp_lock); + write_unlock(&imp->imp_sec_lock); } up_read(&obd->u.cli.cl_sem); -- 1.8.3.1