LU-11775 obdclass: protect imp_sec using rwlock_t

We've seen spinlock contention on imp_lock in
sptlrpc_import_sec_ref(), introduce a new rwlock
imp_sec_lock to protect imp_sec instead of using imp_lock.

This patch also removes imp_sec_mutex from obd_import,
which is not needed, to avoid confusion between
imp_sec_lock/mutex.

Test-Parameters: testlist=sanity-sec envdefinitions=SHARED_KEY=true
Signed-off-by: Li Dongyang <dongyangli@ddn.com>
Change-Id: I3be3a6d1225888134bf7dd58e4b05b864f8415b4
Reviewed-on: https://review.whamcloud.com/33861
Tested-by: Jenkins
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Alexey Lyashkov <c17817@cray.com>
Reviewed-by: Alexandr Boyko <c17825@cray.com>
Reviewed-by: Sebastien Buisson <sbuisson@ddn.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/lustre_import.h b/lustre/include/lustre_import.h
index 8f52c66..0e34d55 100644 (file)
--- a/lustre/include/lustre_import.h
+++ b/lustre/include/lustre_import.h
@@ -210,7 +210,7 @@ struct obd_import {
          * @{
          */
        struct ptlrpc_sec        *imp_sec;
-       struct mutex              imp_sec_mutex;
+       rwlock_t                  imp_sec_lock;
        time64_t                imp_sec_expire;
        pid_t                     imp_sec_refpid;
         /** @} */

diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c
index c5404d8..883e9a4 100644 (file)
--- a/lustre/obdclass/genops.c
+++ b/lustre/obdclass/genops.c
@@ -1326,7 +1326,7 @@ struct obd_import *class_new_import(struct obd_device *obd)
        imp->imp_last_success_conn = 0;
        imp->imp_state = LUSTRE_IMP_NEW;
        imp->imp_obd = class_incref(obd, "import", imp);
-       mutex_init(&imp->imp_sec_mutex);
+       rwlock_init(&imp->imp_sec_lock);
        init_waitqueue_head(&imp->imp_recovery_waitq);
        INIT_WORK(&imp->imp_zombie_work, obd_zombie_imp_cull);
 

diff --git a/lustre/ptlrpc/sec.c b/lustre/ptlrpc/sec.c
index ec05298..dba6352 100644 (file)
--- a/lustre/ptlrpc/sec.c
+++ b/lustre/ptlrpc/sec.c
@@ -366,13 +366,13 @@ static int import_sec_check_expire(struct obd_import *imp)
 {
        int adapt = 0;
 
-       spin_lock(&imp->imp_lock);
+       write_lock(&imp->imp_sec_lock);
        if (imp->imp_sec_expire &&
            imp->imp_sec_expire < ktime_get_real_seconds()) {
                adapt = 1;
                imp->imp_sec_expire = 0;
        }
-       spin_unlock(&imp->imp_lock);
+       write_unlock(&imp->imp_sec_lock);
 
        if (!adapt)
                return 0;
@@ -1392,9 +1392,9 @@ struct ptlrpc_sec *sptlrpc_import_sec_ref(struct obd_import *imp)
 {
        struct ptlrpc_sec *sec;
 
-       spin_lock(&imp->imp_lock);
+       read_lock(&imp->imp_sec_lock);
        sec = sptlrpc_sec_get(imp->imp_sec);
-       spin_unlock(&imp->imp_lock);
+       read_unlock(&imp->imp_sec_lock);
 
        return sec;
 }
@@ -1407,10 +1407,10 @@ static void sptlrpc_import_sec_install(struct obd_import *imp,
 
        LASSERT_ATOMIC_POS(&sec->ps_refcount);
 
-       spin_lock(&imp->imp_lock);
+       write_lock(&imp->imp_sec_lock);
        old_sec = imp->imp_sec;
        imp->imp_sec = sec;
-       spin_unlock(&imp->imp_lock);
+       write_unlock(&imp->imp_sec_lock);
 
        if (old_sec) {
                sptlrpc_sec_kill(old_sec);
@@ -1506,8 +1506,6 @@ int sptlrpc_import_sec_adapt(struct obd_import *imp,
                       sptlrpc_flavor2name(&sf, str, sizeof(str)));
        }
 
-       mutex_lock(&imp->imp_sec_mutex);
-
        newsec = sptlrpc_sec_create(imp, svc_ctx, &sf, sp);
        if (newsec) {
                sptlrpc_import_sec_install(imp, newsec);
@@ -1518,7 +1516,6 @@ int sptlrpc_import_sec_adapt(struct obd_import *imp,
                rc = -EPERM;
        }
 
-       mutex_unlock(&imp->imp_sec_mutex);
 out:
        sptlrpc_sec_put(sec);
        RETURN(rc);

diff --git a/lustre/ptlrpc/sec_config.c b/lustre/ptlrpc/sec_config.c
index 7df845e..e7e08ed 100644 (file)
--- a/lustre/ptlrpc/sec_config.c
+++ b/lustre/ptlrpc/sec_config.c
@@ -911,11 +911,11 @@ void sptlrpc_conf_client_adapt(struct obd_device *obd)
 
        imp = obd->u.cli.cl_import;
        if (imp) {
-               spin_lock(&imp->imp_lock);
+               write_lock(&imp->imp_sec_lock);
                if (imp->imp_sec)
                        imp->imp_sec_expire = ktime_get_real_seconds() +
                                SEC_ADAPT_DELAY;
-               spin_unlock(&imp->imp_lock);
+               write_unlock(&imp->imp_sec_lock);
        }
 
        up_read(&obd->u.cli.cl_sem);