LU-6261 gnilnd: Thread-safe optimizations.

Take advantage of improved gni threading.
Do not use gnd_cq_mutex lock for kgni versions that support thread
safe gni api.
Check if version is greater than code rev 0xb9 and use locking of
smsg and rdma on a per connection instead of the global cq lock.
Changed gnc_tx_seq and gnc_rx_seq to atomics.
Added gnc_smsg_mutex and gnc_rdma_mutex per conn to protect the lists
that the messages are placed on.

Signed-off-by: Chris Horn <hornc@cray.com>
Change-Id: Ic03f1877ab7b9632ca5517cd74f7e7fa25ba171b
Reviewed-on: http://review.whamcloud.com/15430
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: James Simmons <uja.ornl@yahoo.com>
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: James Shimek <jshimek@cray.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lnet/klnds/gnilnd/gnilnd.c b/lnet/klnds/gnilnd/gnilnd.c
index 8970f4b..4961d44 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd.c
+++ b/lnet/klnds/gnilnd/gnilnd.c
@@ -208,6 +208,8 @@ kgnilnd_create_conn(kgn_conn_t **connp, kgn_device_t *dev)
                GOTO(failed, rc = -ENOMEM);
        }
 
+       mutex_init(&conn->gnc_smsg_mutex);
+       mutex_init(&conn->gnc_rdma_mutex);
        atomic_set(&conn->gnc_refcount, 1);
        atomic_set(&conn->gnc_reaper_noop, 0);
        atomic_set(&conn->gnc_sched_noop, 0);
@@ -1615,8 +1617,8 @@ kgnilnd_get_conn_info(kgn_peer_t *peer,
 
        *device_id = conn->gnc_device->gnd_host_id;
        *peerstamp = conn->gnc_peerstamp;
-       *tx_seq = conn->gnc_tx_seq;
-       *rx_seq = conn->gnc_rx_seq;
+       *tx_seq = atomic_read(&conn->gnc_tx_seq);
+       *rx_seq = atomic_read(&conn->gnc_rx_seq);
        *fmaq_len = kgnilnd_count_list(&conn->gnc_fmaq);
        *nfma = atomic_read(&conn->gnc_nlive_fma);
        *nrdma = atomic_read(&conn->gnc_nlive_rdma);
@@ -2136,7 +2138,6 @@ kgnilnd_dev_fini(kgn_device_t *dev)
        EXIT;
 }
 
-
 int kgnilnd_base_startup(void)
 {
        struct timeval       tv;
@@ -2161,6 +2162,7 @@ int kgnilnd_base_startup(void)
 
        /* zero pointers, flags etc */
        memset(&kgnilnd_data, 0, sizeof(kgnilnd_data));
+       kgnilnd_check_kgni_version();
 
        /* CAVEAT EMPTOR: Every 'Fma' message includes the sender's NID and
         * a unique (for all time) connstamp so we can uniquely identify

diff --git a/lnet/klnds/gnilnd/gnilnd.h b/lnet/klnds/gnilnd/gnilnd.h
index 79df3c3..d82e971 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd.h
+++ b/lnet/klnds/gnilnd/gnilnd.h
@@ -718,8 +718,10 @@ typedef struct kgn_conn {
        atomic_t            gnc_sched_noop;     /* # sched triggered NOOP */
        unsigned int        gnc_timeout;        /* infer peer death if no rx for this many seconds */
        __u32               gnc_cqid;           /* my completion callback id (non-unique) */
-       __u32               gnc_tx_seq;         /* tx msg sequence number */
-       __u32               gnc_rx_seq;         /* rx msg sequence number */
+       atomic_t            gnc_tx_seq;         /* tx msg sequence number */
+       atomic_t            gnc_rx_seq;         /* rx msg sequence number */
+       struct mutex        gnc_smsg_mutex;     /* tx smsg sequence serialization */
+       struct mutex        gnc_rdma_mutex;     /* tx rdma sequence serialization */
        __u64               gnc_tx_retrans;     /* # retrans on SMSG */
        atomic_t            gnc_nlive_fma;      /* # live FMA */
        atomic_t            gnc_nq_rdma;        /* # queued (on device) RDMA */
@@ -864,6 +866,7 @@ typedef struct kgn_data {
        atomic_t                kgn_rev_copy_buff;    /* # of REV rdma buffer copies */
        struct socket          *kgn_sock;             /* for Apollo */
        unsigned long           free_pages_limit;     /* # of free pages reserve from fma block allocations */
+       int                     kgn_enable_gl_mutex;  /* kgni api mtx enable */
 } kgn_data_t;
 
 extern kgn_data_t         kgnilnd_data;
@@ -901,6 +904,38 @@ kgnilnd_thread_fini(void)
        atomic_dec(&kgnilnd_data.kgn_nthreads);
 }
 
+static inline int kgnilnd_gl_mutex_trylock(struct mutex *lock)
+{
+       if (kgnilnd_data.kgn_enable_gl_mutex)
+               return mutex_trylock(lock);
+       else
+               return 1;
+}
+
+static inline void kgnilnd_gl_mutex_lock(struct mutex *lock)
+{
+       if (kgnilnd_data.kgn_enable_gl_mutex)
+               mutex_lock(lock);
+}
+
+static inline void kgnilnd_gl_mutex_unlock(struct mutex *lock)
+{
+       if (kgnilnd_data.kgn_enable_gl_mutex)
+               mutex_unlock(lock);
+}
+
+static inline void kgnilnd_conn_mutex_lock(struct mutex *lock)
+{
+       if (!kgnilnd_data.kgn_enable_gl_mutex)
+               mutex_lock(lock);
+}
+
+static inline void kgnilnd_conn_mutex_unlock(struct mutex *lock)
+{
+       if (!kgnilnd_data.kgn_enable_gl_mutex)
+               mutex_unlock(lock);
+}
+
 /* like mutex_trylock but with a jiffies spinner. This is to allow certain
  * parts of the code to avoid a scheduler trip when the mutex is held
  *
@@ -919,6 +954,9 @@ static inline int kgnilnd_mutex_trylock(struct mutex *lock)
        int             ret;
        unsigned long   timeout;
 
+       if (!kgnilnd_data.kgn_enable_gl_mutex)
+               return 1;
+
        LASSERT(!in_interrupt());
 
        for (timeout = jiffies + 1; time_before(jiffies, timeout);) {
@@ -1936,9 +1974,6 @@ kgnilnd_conn_dgram_type2str(kgn_dgram_type_t type)
 
 #undef DO_TYPE
 
-/* API wrapper functions - include late to pick up all of the other defines */
-#include "gnilnd_api_wrap.h"
-
 /* pulls in tunables per platform and adds in nid/nic conversion
  * if RCA wasn't available at build time */
 #include "gnilnd_hss_ops.h"
@@ -1951,4 +1986,7 @@ kgnilnd_conn_dgram_type2str(kgn_dgram_type_t type)
  #error "Undefined Network Hardware Type"
 #endif
 
+/* API wrapper functions - include late to pick up all of the other defines */
+#include "gnilnd_api_wrap.h"
+
 #endif /* _GNILND_GNILND_H_ */

diff --git a/lnet/klnds/gnilnd/gnilnd_api_wrap.h b/lnet/klnds/gnilnd/gnilnd_api_wrap.h
index 7b69cb3..7f2a5e3 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd_api_wrap.h
+++ b/lnet/klnds/gnilnd/gnilnd_api_wrap.h
@@ -103,6 +103,34 @@
 #define CFS_FAIL_GNI_DGRAM_DEADLINE    0xf053
 #define CFS_FAIL_GNI_DGRAM_DROP_TX     0xf054
 
+extern uint32_t kgni_driver_version;
+
+static inline void
+kgnilnd_check_kgni_version(void)
+{
+       uint32_t *kdv;
+
+       kgnilnd_data.kgn_enable_gl_mutex = 1;
+       kdv = symbol_get(kgni_driver_version);
+       if (!kdv) {
+               LCONSOLE_INFO("Not using thread safe locking -"
+                       " no symbol kgni_driver_version\n");
+               return;
+       }
+
+       /* Thread-safe kgni implemented in minor ver 0x44/45, code rev 0xb9 */
+       if (*kdv < GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9)) {
+               symbol_put(kgni_driver_version);
+               LCONSOLE_INFO("Not using thread safe locking, gni version 0x%x,"
+                       " need >= 0x%x\n", *kdv,
+                       GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9));
+               return;
+       }
+
+       symbol_put(kgni_driver_version);
+       /* Use thread-safe locking */
+       kgnilnd_data.kgn_enable_gl_mutex = 0;
+}
 
 /* helper macros */
 extern void

diff --git a/lnet/klnds/gnilnd/gnilnd_aries.h b/lnet/klnds/gnilnd/gnilnd_aries.h
index 72ca834..e017fe3 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd_aries.h
+++ b/lnet/klnds/gnilnd/gnilnd_aries.h
@@ -53,6 +53,9 @@
 #define GNILND_RDMA_DLVR_OPTION    GNI_DLVMODE_PERFORMANCE
 #endif
 
+/* Thread-safe kgni implemented in minor ver 45, code rev 0xb9 */
+#define GNILND_KGNI_TS_MINOR_VER 0x45
+
 /* plug in our functions for use on the simulator */
 #if !defined(GNILND_USE_RCA)
 

diff --git a/lnet/klnds/gnilnd/gnilnd_cb.c b/lnet/klnds/gnilnd/gnilnd_cb.c
index 1703775..92aaf57 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd_cb.c
+++ b/lnet/klnds/gnilnd/gnilnd_cb.c
@@ -1327,7 +1327,7 @@ search_again:
         * if we are sending to the same node faster than 256000/sec.
         * To help guard against this, we OR in the tx_seq - that is 32 bits */
 
-       tx->tx_id.txe_chips = (__u32)(jiffies | conn->gnc_tx_seq);
+       tx->tx_id.txe_chips = (__u32)(jiffies | atomic_read(&conn->gnc_tx_seq));
 
        GNIDBG_TX(D_NET, tx, "set cookie/id/bits", NULL);
 
@@ -1429,7 +1429,8 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob,
         * close message.
         */
        if (atomic_read(&conn->gnc_peer->gnp_dirty_eps) != 0 && msg->gnm_type != GNILND_MSG_CLOSE) {
-               mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+               kgnilnd_conn_mutex_unlock(&conn->gnc_smsg_mutex);
+               kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
                /* Return -ETIME, we are closing the connection already so we dont want to
                 * have this tx hit the wire. The tx will be killed by the calling function.
                 * Once the EP is marked dirty the close message will be the last
@@ -1452,7 +1453,8 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob,
                libcfs_nid2str(conn->gnc_peer->gnp_nid),
                cfs_duration_sec(now - newest_last_rx),
                cfs_duration_sec(GNILND_TIMEOUTRX(timeout)));
-               mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+               kgnilnd_conn_mutex_unlock(&conn->gnc_smsg_mutex);
+               kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
                return -ETIME;
        }
 
@@ -1463,7 +1465,8 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob,
         */
        msg->gnm_connstamp = conn->gnc_my_connstamp;
        msg->gnm_payload_len = immediatenob;
-       msg->gnm_seq = conn->gnc_tx_seq;
+       kgnilnd_conn_mutex_lock(&conn->gnc_smsg_mutex);
+       msg->gnm_seq = atomic_read(&conn->gnc_tx_seq);
 
        /* always init here - kgn_checksum is a /sys module tunable
         * and can be flipped at any point, even between msg init and sending */
@@ -1495,7 +1498,7 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob,
 
        switch (rrc) {
        case GNI_RC_SUCCESS:
-               conn->gnc_tx_seq++;
+               atomic_inc(&conn->gnc_tx_seq);
                conn->gnc_last_tx = jiffies;
                /* no locking here as LIVE isn't a list */
                kgnilnd_tx_add_state_locked(tx, NULL, conn, GNILND_TX_LIVE_FMAQ, 1);
@@ -1509,7 +1512,8 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob,
 
                /* serialize with seeing CQ events for completion on this, as well as
                 * tx_seq */
-               mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+               kgnilnd_conn_mutex_unlock(&conn->gnc_smsg_mutex);
+               kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
 
                atomic_inc(&conn->gnc_device->gnd_short_ntx);
                atomic64_add(immediatenob, &conn->gnc_device->gnd_short_txbytes);
@@ -1521,8 +1525,8 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob,
                /* XXX Nic: We need to figure out how to track this
                 * - there are bound to be good reasons for it,
                 * but we want to know when it happens */
-
-               mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+               kgnilnd_conn_mutex_unlock(&conn->gnc_smsg_mutex);
+               kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
                /* We'll handle this error inline - makes the calling logic much more
                 * clean */
 
@@ -1559,7 +1563,8 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob,
                }
        default:
                /* handle bad retcode gracefully */
-               mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+               kgnilnd_conn_mutex_unlock(&conn->gnc_smsg_mutex);
+               kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
                return -EIO;
        }
 }
@@ -1574,7 +1579,7 @@ kgnilnd_sendmsg(kgn_tx_t *tx, void *immediate, unsigned int immediatenob,
        int              rc;
 
        timestamp = jiffies;
-       mutex_lock(&dev->gnd_cq_mutex);
+       kgnilnd_gl_mutex_lock(&dev->gnd_cq_mutex);
        /* delay in jiffies - we are really concerned only with things that
         * result in a schedule() or really holding this off for long times .
         * NB - mutex_lock could spin for 2 jiffies before going to sleep to wait */
@@ -1619,7 +1624,7 @@ kgnilnd_sendmsg_trylock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob
                rc = 0;
        } else {
                atomic_inc(&conn->gnc_device->gnd_fast_try);
-               rc = mutex_trylock(&conn->gnc_device->gnd_cq_mutex);
+               rc = kgnilnd_gl_mutex_trylock(&conn->gnc_device->gnd_cq_mutex);
        }
        if (!rc) {
                rc = -EAGAIN;
@@ -1983,7 +1988,8 @@ kgnilnd_rdma(kgn_tx_t *tx, int type,
        tx->tx_rdma_desc.src_cq_hndl = conn->gnc_device->gnd_snd_rdma_cqh;
 
        timestamp = jiffies;
-       mutex_lock(&conn->gnc_device->gnd_cq_mutex);
+       kgnilnd_conn_mutex_lock(&conn->gnc_rdma_mutex);
+       kgnilnd_gl_mutex_lock(&conn->gnc_device->gnd_cq_mutex);
        /* delay in jiffies - we are really concerned only with things that
         * result in a schedule() or really holding this off for long times .
         * NB - mutex_lock could spin for 2 jiffies before going to sleep to wait */
@@ -1992,7 +1998,8 @@ kgnilnd_rdma(kgn_tx_t *tx, int type,
        rrc = kgnilnd_post_rdma(conn->gnc_ephandle, &tx->tx_rdma_desc);
 
        if (rrc == GNI_RC_ERROR_RESOURCE) {
-               mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+               kgnilnd_conn_mutex_unlock(&conn->gnc_rdma_mutex);
+               kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
                kgnilnd_unmap_buffer(tx, 0);
 
                if (tx->tx_buffer_copy != NULL) {
@@ -2012,8 +2019,8 @@ kgnilnd_rdma(kgn_tx_t *tx, int type,
        kgnilnd_tx_add_state_locked(tx, conn->gnc_peer, conn, GNILND_TX_LIVE_RDMAQ, 1);
        tx->tx_qtime = jiffies;
        spin_unlock(&conn->gnc_list_lock);
-
-       mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+       kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+       kgnilnd_conn_mutex_unlock(&conn->gnc_rdma_mutex);
 
        /* XXX Nic: is this a place we should handle more errors for
         * robustness sake */
@@ -2049,14 +2056,14 @@ kgnilnd_release_msg(kgn_conn_t *conn)
        CDEBUG(D_NET, "consuming %p\n", conn);
 
        timestamp = jiffies;
-       mutex_lock(&conn->gnc_device->gnd_cq_mutex);
+       kgnilnd_gl_mutex_lock(&conn->gnc_device->gnd_cq_mutex);
        /* delay in jiffies - we are really concerned only with things that
         * result in a schedule() or really holding this off for long times .
         * NB - mutex_lock could spin for 2 jiffies before going to sleep to wait */
        conn->gnc_device->gnd_mutex_delay += (long) jiffies - timestamp;
 
        rrc = kgnilnd_smsg_release(conn->gnc_ephandle);
-       mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+       kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
 
        LASSERTF(rrc == GNI_RC_SUCCESS, "bad rrc %d\n", rrc);
        GNIDBG_SMSG_CREDS(D_NET, conn);
@@ -3165,7 +3172,7 @@ kgnilnd_check_rdma_cq(kgn_device_t *dev)
                }
 
                if (rrc == GNI_RC_NOT_DONE) {
-                       mutex_unlock(&dev->gnd_cq_mutex);
+                       kgnilnd_gl_mutex_unlock(&dev->gnd_cq_mutex);
                        CDEBUG(D_INFO, "SEND RDMA CQ %d empty processed %ld\n",
                               dev->gnd_id, num_processed);
                        return num_processed;
@@ -3182,7 +3189,7 @@ kgnilnd_check_rdma_cq(kgn_device_t *dev)
 
                rrc = kgnilnd_get_completed(dev->gnd_snd_rdma_cqh, event_data,
                                            &desc);
-               mutex_unlock(&dev->gnd_cq_mutex);
+               kgnilnd_gl_mutex_unlock(&dev->gnd_cq_mutex);
 
                /* XXX Nic: Need better error handling here... */
                LASSERTF((rrc == GNI_RC_SUCCESS) ||
@@ -3226,9 +3233,11 @@ kgnilnd_check_rdma_cq(kgn_device_t *dev)
                }
 
                /* remove from rdmaq */
+               kgnilnd_conn_mutex_lock(&conn->gnc_rdma_mutex);
                spin_lock(&conn->gnc_list_lock);
                kgnilnd_tx_del_state_locked(tx, NULL, conn, GNILND_TX_ALLOCD);
                spin_unlock(&conn->gnc_list_lock);
+               kgnilnd_conn_mutex_unlock(&conn->gnc_rdma_mutex);
 
                if (likely(desc->status == GNI_RC_SUCCESS) && rc == 0) {
                        atomic_inc(&dev->gnd_rdma_ntx);
@@ -3319,7 +3328,7 @@ kgnilnd_check_fma_send_cq(kgn_device_t *dev)
                }
 
                rrc = kgnilnd_cq_get_event(dev->gnd_snd_fma_cqh, &event_data);
-               mutex_unlock(&dev->gnd_cq_mutex);
+               kgnilnd_gl_mutex_unlock(&dev->gnd_cq_mutex);
 
                if (rrc == GNI_RC_NOT_DONE) {
                        CDEBUG(D_INFO,
@@ -3385,6 +3394,7 @@ kgnilnd_check_fma_send_cq(kgn_device_t *dev)
                }
 
                /* lock tx_list_state and tx_state */
+               kgnilnd_conn_mutex_lock(&conn->gnc_smsg_mutex);
                spin_lock(&tx->tx_conn->gnc_list_lock);
 
                GNITX_ASSERTF(tx, tx->tx_list_state == GNILND_TX_LIVE_FMAQ,
@@ -3405,6 +3415,7 @@ kgnilnd_check_fma_send_cq(kgn_device_t *dev)
                saw_reply = !(tx->tx_state & GNILND_TX_WAITING_REPLY);
 
                spin_unlock(&tx->tx_conn->gnc_list_lock);
+               kgnilnd_conn_mutex_unlock(&conn->gnc_smsg_mutex);
 
                if (queued_fma) {
                        CDEBUG(D_NET, "scheduling conn 0x%p->%s for fmaq\n",
@@ -3473,7 +3484,7 @@ kgnilnd_check_fma_rcv_cq(kgn_device_t *dev)
                        return 1;
                }
                rrc = kgnilnd_cq_get_event(dev->gnd_rcv_fma_cqh, &event_data);
-               mutex_unlock(&dev->gnd_cq_mutex);
+               kgnilnd_gl_mutex_unlock(&dev->gnd_cq_mutex);
 
                if (rrc == GNI_RC_NOT_DONE) {
                        CDEBUG(D_INFO, "SMSG RX CQ %d empty data "LPX64" "
@@ -4085,7 +4096,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn)
                RETURN_EXIT;
 
        timestamp = jiffies;
-       mutex_lock(&conn->gnc_device->gnd_cq_mutex);
+       kgnilnd_gl_mutex_lock(&conn->gnc_device->gnd_cq_mutex);
        /* delay in jiffies - we are really concerned only with things that
         * result in a schedule() or really holding this off for long times .
         * NB - mutex_lock could spin for 2 jiffies before going to sleep to wait */
@@ -4114,7 +4125,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn)
                libcfs_nid2str(conn->gnc_peer->gnp_nid),
                cfs_duration_sec(timestamp - newest_last_rx),
                cfs_duration_sec(GNILND_TIMEOUTRX(timeout)));
-               mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+               kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
                rc = -ETIME;
                kgnilnd_close_conn(conn, rc);
                RETURN_EXIT;
@@ -4123,7 +4134,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn)
        rrc = kgnilnd_smsg_getnext(conn->gnc_ephandle, &prefix);
 
        if (rrc == GNI_RC_NOT_DONE) {
-               mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+               kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
                CDEBUG(D_INFO, "SMSG RX empty conn 0x%p\n", conn);
                RETURN_EXIT;
        }
@@ -4137,7 +4148,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn)
         */
 
        if (rrc == GNI_RC_INVALID_STATE) {
-               mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+               kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
                GNIDBG_CONN(D_NETERROR | D_CONSOLE, conn, "Mailbox corruption "
                        "detected closing conn %p from peer %s\n", conn,
                        libcfs_nid2str(conn->gnc_peer->gnp_nid));
@@ -4154,7 +4165,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn)
 
        rx = kgnilnd_alloc_rx();
        if (rx == NULL) {
-               mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+               kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
                kgnilnd_release_msg(conn);
                GNIDBG_MSG(D_NETERROR, msg, "Dropping SMSG RX from 0x%p->%s, no RX memory",
                           conn, libcfs_nid2str(peer->gnp_nid));
@@ -4164,7 +4175,8 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn)
        GNIDBG_MSG(D_INFO, msg, "SMSG RX on %p", conn);
 
        timestamp = conn->gnc_last_rx;
-       last_seq = conn->gnc_rx_seq;
+       seq = last_seq = atomic_read(&conn->gnc_rx_seq);
+       atomic_inc(&conn->gnc_rx_seq);
 
        conn->gnc_last_rx = jiffies;
        /* stash first rx so we can clear out purgatory
@@ -4172,10 +4184,8 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn)
        if (conn->gnc_first_rx == 0)
                conn->gnc_first_rx = jiffies;
 
-       seq = conn->gnc_rx_seq++;
-
        /* needs to linger to protect gnc_rx_seq like we do with gnc_tx_seq */
-       mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
+       kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex);
        kgnilnd_peer_alive(conn->gnc_peer);
 
        rx->grx_msg = msg;
@@ -4299,7 +4309,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn)
                                       conn, last_seq,
                                       cfs_duration_sec(now - timestamp),
                                       cfs_duration_sec(now - conn->gnc_last_rx_cq),
-                                      conn->gnc_tx_seq,
+                                      atomic_read(&conn->gnc_tx_seq),
                                       cfs_duration_sec(now - conn->gnc_last_tx),
                                       cfs_duration_sec(now - conn->gnc_last_tx_cq),
                                       cfs_duration_sec(now - conn->gnc_last_noop_want),

diff --git a/lnet/klnds/gnilnd/gnilnd_debug.c b/lnet/klnds/gnilnd/gnilnd_debug.c
index c4f2b9b..36a9844 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd_debug.c
+++ b/lnet/klnds/gnilnd/gnilnd_debug.c
@@ -52,10 +52,10 @@ _kgnilnd_debug_conn(kgn_conn_t *conn, struct libcfs_debug_msg_data *msgdata,
                conn, conn->gnc_peer ? libcfs_nid2str(conn->gnc_peer->gnp_nid) :
                "<?>", kgnilnd_conn_state2str(conn),
                conn->gnc_cqid, conn->gnc_timeout,
-               conn->gnc_rx_seq,
+               atomic_read(&conn->gnc_rx_seq),
                cfs_duration_sec(jiffies - conn->gnc_last_rx),
                cfs_duration_sec(jiffies - conn->gnc_last_rx_cq),
-               conn->gnc_tx_seq,
+               atomic_read(&conn->gnc_tx_seq),
                cfs_duration_sec(jiffies - conn->gnc_last_tx),
                cfs_duration_sec(jiffies - conn->gnc_last_tx_cq),
                cfs_duration_sec(jiffies - conn->gnc_last_noop_want),

diff --git a/lnet/klnds/gnilnd/gnilnd_gemini.h b/lnet/klnds/gnilnd/gnilnd_gemini.h
index a74234e..79be5b3 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd_gemini.h
+++ b/lnet/klnds/gnilnd/gnilnd_gemini.h
@@ -34,4 +34,7 @@
 #define GNILND_REVERSE_RDMA        GNILND_REVERSE_NONE
 #define GNILND_RDMA_DLVR_OPTION     GNI_DLVMODE_PERFORMANCE
 
+/* Thread-safe kgni implemented in minor ver 44, code rev 0xb9 */
+#define GNILND_KGNI_TS_MINOR_VER 0x44
+
 #endif /* _GNILND_GEMINI_H */

diff --git a/lnet/klnds/gnilnd/gnilnd_proc.c b/lnet/klnds/gnilnd/gnilnd_proc.c
index 292186d..c5382fb 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd_proc.c
+++ b/lnet/klnds/gnilnd/gnilnd_proc.c
@@ -911,10 +911,10 @@ kgnilnd_conn_seq_show(struct seq_file *s, void *iter)
                kgnilnd_count_list(&conn->gnc_fmaq),
                atomic_read(&conn->gnc_nlive_fma),
                atomic_read(&conn->gnc_nlive_rdma),
-               conn->gnc_tx_seq,
+               atomic_read(&conn->gnc_tx_seq),
                jiffies_to_msecs(jiffies - conn->gnc_last_tx),
                jiffies_to_msecs(jiffies - conn->gnc_last_tx_cq),
-               conn->gnc_rx_seq,
+               atomic_read(&conn->gnc_rx_seq),
                jiffies_to_msecs(jiffies - conn->gnc_last_rx),
                jiffies_to_msecs(jiffies - conn->gnc_last_rx_cq),
                atomic_read(&conn->gnc_reaper_noop),
@@ -1043,10 +1043,10 @@ kgnilnd_proc_peer_conns_seq_show(struct seq_file *sf, void *v)
                        conn->gnc_close_recvd,
                        conn->gnc_error,
                        conn->gnc_peer_error,
-                       conn->gnc_tx_seq,
+                       atomic_read(&conn->gnc_tx_seq),
                        jiffies_to_msecs(jifs - conn->gnc_last_tx),
                        jiffies_to_msecs(jifs - conn->gnc_last_tx_cq),
-                       conn->gnc_rx_seq,
+                       atomic_read(&conn->gnc_rx_seq),
                        jiffies_to_msecs(jifs - conn->gnc_first_rx),
                        jiffies_to_msecs(jifs - conn->gnc_last_rx),
                        jiffies_to_msecs(jifs - conn->gnc_last_rx_cq),