From 31be9f94d72287d7077ffc00d392a71056f61f4d Mon Sep 17 00:00:00 2001 From: Chris Horn Date: Fri, 10 Jul 2015 11:02:07 -0400 Subject: [PATCH] LU-6261 gnilnd: Thread-safe optimizations. Take advantage of improved gni threading. Do not use gnd_cq_mutex lock for kgni versions that support thread safe gni api. Check if version is greater than code rev 0xb9 and use locking of smsg and rdma on a per connection instead of the global cq lock. Changed gnc_tx_seq and gnc_rx_seq to atomics. Added gnc_smsg_mutex and gnc_rdma_mutex per conn to protect the lists that the messages are placed on. Signed-off-by: Chris Horn Change-Id: Ic03f1877ab7b9632ca5517cd74f7e7fa25ba171b Reviewed-on: http://review.whamcloud.com/15430 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Doug Oucharek Reviewed-by: James Shimek Reviewed-by: Oleg Drokin --- lnet/klnds/gnilnd/gnilnd.c | 8 +++-- lnet/klnds/gnilnd/gnilnd.h | 48 ++++++++++++++++++++++--- lnet/klnds/gnilnd/gnilnd_api_wrap.h | 28 +++++++++++++++ lnet/klnds/gnilnd/gnilnd_aries.h | 3 ++ lnet/klnds/gnilnd/gnilnd_cb.c | 72 +++++++++++++++++++++---------------- lnet/klnds/gnilnd/gnilnd_debug.c | 4 +-- lnet/klnds/gnilnd/gnilnd_gemini.h | 3 ++ lnet/klnds/gnilnd/gnilnd_proc.c | 8 ++--- 8 files changed, 129 insertions(+), 45 deletions(-) diff --git a/lnet/klnds/gnilnd/gnilnd.c b/lnet/klnds/gnilnd/gnilnd.c index 8970f4b..4961d44 100644 --- a/lnet/klnds/gnilnd/gnilnd.c +++ b/lnet/klnds/gnilnd/gnilnd.c @@ -208,6 +208,8 @@ kgnilnd_create_conn(kgn_conn_t **connp, kgn_device_t *dev) GOTO(failed, rc = -ENOMEM); } + mutex_init(&conn->gnc_smsg_mutex); + mutex_init(&conn->gnc_rdma_mutex); atomic_set(&conn->gnc_refcount, 1); atomic_set(&conn->gnc_reaper_noop, 0); atomic_set(&conn->gnc_sched_noop, 0); @@ -1615,8 +1617,8 @@ kgnilnd_get_conn_info(kgn_peer_t *peer, *device_id = conn->gnc_device->gnd_host_id; *peerstamp = conn->gnc_peerstamp; - *tx_seq = conn->gnc_tx_seq; - *rx_seq = conn->gnc_rx_seq; + *tx_seq = atomic_read(&conn->gnc_tx_seq); + *rx_seq = atomic_read(&conn->gnc_rx_seq); *fmaq_len = kgnilnd_count_list(&conn->gnc_fmaq); *nfma = atomic_read(&conn->gnc_nlive_fma); *nrdma = atomic_read(&conn->gnc_nlive_rdma); @@ -2136,7 +2138,6 @@ kgnilnd_dev_fini(kgn_device_t *dev) EXIT; } - int kgnilnd_base_startup(void) { struct timeval tv; @@ -2161,6 +2162,7 @@ int kgnilnd_base_startup(void) /* zero pointers, flags etc */ memset(&kgnilnd_data, 0, sizeof(kgnilnd_data)); + kgnilnd_check_kgni_version(); /* CAVEAT EMPTOR: Every 'Fma' message includes the sender's NID and * a unique (for all time) connstamp so we can uniquely identify diff --git a/lnet/klnds/gnilnd/gnilnd.h b/lnet/klnds/gnilnd/gnilnd.h index 79df3c3..d82e971 100644 --- a/lnet/klnds/gnilnd/gnilnd.h +++ b/lnet/klnds/gnilnd/gnilnd.h @@ -718,8 +718,10 @@ typedef struct kgn_conn { atomic_t gnc_sched_noop; /* # sched triggered NOOP */ unsigned int gnc_timeout; /* infer peer death if no rx for this many seconds */ __u32 gnc_cqid; /* my completion callback id (non-unique) */ - __u32 gnc_tx_seq; /* tx msg sequence number */ - __u32 gnc_rx_seq; /* rx msg sequence number */ + atomic_t gnc_tx_seq; /* tx msg sequence number */ + atomic_t gnc_rx_seq; /* rx msg sequence number */ + struct mutex gnc_smsg_mutex; /* tx smsg sequence serialization */ + struct mutex gnc_rdma_mutex; /* tx rdma sequence serialization */ __u64 gnc_tx_retrans; /* # retrans on SMSG */ atomic_t gnc_nlive_fma; /* # live FMA */ atomic_t gnc_nq_rdma; /* # queued (on device) RDMA */ @@ -864,6 +866,7 @@ typedef struct kgn_data { atomic_t kgn_rev_copy_buff; /* # of REV rdma buffer copies */ struct socket *kgn_sock; /* for Apollo */ unsigned long free_pages_limit; /* # of free pages reserve from fma block allocations */ + int kgn_enable_gl_mutex; /* kgni api mtx enable */ } kgn_data_t; extern kgn_data_t kgnilnd_data; @@ -901,6 +904,38 @@ kgnilnd_thread_fini(void) atomic_dec(&kgnilnd_data.kgn_nthreads); } +static inline int kgnilnd_gl_mutex_trylock(struct mutex *lock) +{ + if (kgnilnd_data.kgn_enable_gl_mutex) + return mutex_trylock(lock); + else + return 1; +} + +static inline void kgnilnd_gl_mutex_lock(struct mutex *lock) +{ + if (kgnilnd_data.kgn_enable_gl_mutex) + mutex_lock(lock); +} + +static inline void kgnilnd_gl_mutex_unlock(struct mutex *lock) +{ + if (kgnilnd_data.kgn_enable_gl_mutex) + mutex_unlock(lock); +} + +static inline void kgnilnd_conn_mutex_lock(struct mutex *lock) +{ + if (!kgnilnd_data.kgn_enable_gl_mutex) + mutex_lock(lock); +} + +static inline void kgnilnd_conn_mutex_unlock(struct mutex *lock) +{ + if (!kgnilnd_data.kgn_enable_gl_mutex) + mutex_unlock(lock); +} + /* like mutex_trylock but with a jiffies spinner. This is to allow certain * parts of the code to avoid a scheduler trip when the mutex is held * @@ -919,6 +954,9 @@ static inline int kgnilnd_mutex_trylock(struct mutex *lock) int ret; unsigned long timeout; + if (!kgnilnd_data.kgn_enable_gl_mutex) + return 1; + LASSERT(!in_interrupt()); for (timeout = jiffies + 1; time_before(jiffies, timeout);) { @@ -1936,9 +1974,6 @@ kgnilnd_conn_dgram_type2str(kgn_dgram_type_t type) #undef DO_TYPE -/* API wrapper functions - include late to pick up all of the other defines */ -#include "gnilnd_api_wrap.h" - /* pulls in tunables per platform and adds in nid/nic conversion * if RCA wasn't available at build time */ #include "gnilnd_hss_ops.h" @@ -1951,4 +1986,7 @@ kgnilnd_conn_dgram_type2str(kgn_dgram_type_t type) #error "Undefined Network Hardware Type" #endif +/* API wrapper functions - include late to pick up all of the other defines */ +#include "gnilnd_api_wrap.h" + #endif /* _GNILND_GNILND_H_ */ diff --git a/lnet/klnds/gnilnd/gnilnd_api_wrap.h b/lnet/klnds/gnilnd/gnilnd_api_wrap.h index 7b69cb3..7f2a5e3 100644 --- a/lnet/klnds/gnilnd/gnilnd_api_wrap.h +++ b/lnet/klnds/gnilnd/gnilnd_api_wrap.h @@ -103,6 +103,34 @@ #define CFS_FAIL_GNI_DGRAM_DEADLINE 0xf053 #define CFS_FAIL_GNI_DGRAM_DROP_TX 0xf054 +extern uint32_t kgni_driver_version; + +static inline void +kgnilnd_check_kgni_version(void) +{ + uint32_t *kdv; + + kgnilnd_data.kgn_enable_gl_mutex = 1; + kdv = symbol_get(kgni_driver_version); + if (!kdv) { + LCONSOLE_INFO("Not using thread safe locking -" + " no symbol kgni_driver_version\n"); + return; + } + + /* Thread-safe kgni implemented in minor ver 0x44/45, code rev 0xb9 */ + if (*kdv < GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9)) { + symbol_put(kgni_driver_version); + LCONSOLE_INFO("Not using thread safe locking, gni version 0x%x," + " need >= 0x%x\n", *kdv, + GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9)); + return; + } + + symbol_put(kgni_driver_version); + /* Use thread-safe locking */ + kgnilnd_data.kgn_enable_gl_mutex = 0; +} /* helper macros */ extern void diff --git a/lnet/klnds/gnilnd/gnilnd_aries.h b/lnet/klnds/gnilnd/gnilnd_aries.h index 72ca834..e017fe3 100644 --- a/lnet/klnds/gnilnd/gnilnd_aries.h +++ b/lnet/klnds/gnilnd/gnilnd_aries.h @@ -53,6 +53,9 @@ #define GNILND_RDMA_DLVR_OPTION GNI_DLVMODE_PERFORMANCE #endif +/* Thread-safe kgni implemented in minor ver 45, code rev 0xb9 */ +#define GNILND_KGNI_TS_MINOR_VER 0x45 + /* plug in our functions for use on the simulator */ #if !defined(GNILND_USE_RCA) diff --git a/lnet/klnds/gnilnd/gnilnd_cb.c b/lnet/klnds/gnilnd/gnilnd_cb.c index 1703775..92aaf57 100644 --- a/lnet/klnds/gnilnd/gnilnd_cb.c +++ b/lnet/klnds/gnilnd/gnilnd_cb.c @@ -1327,7 +1327,7 @@ search_again: * if we are sending to the same node faster than 256000/sec. * To help guard against this, we OR in the tx_seq - that is 32 bits */ - tx->tx_id.txe_chips = (__u32)(jiffies | conn->gnc_tx_seq); + tx->tx_id.txe_chips = (__u32)(jiffies | atomic_read(&conn->gnc_tx_seq)); GNIDBG_TX(D_NET, tx, "set cookie/id/bits", NULL); @@ -1429,7 +1429,8 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob, * close message. */ if (atomic_read(&conn->gnc_peer->gnp_dirty_eps) != 0 && msg->gnm_type != GNILND_MSG_CLOSE) { - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_conn_mutex_unlock(&conn->gnc_smsg_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); /* Return -ETIME, we are closing the connection already so we dont want to * have this tx hit the wire. The tx will be killed by the calling function. * Once the EP is marked dirty the close message will be the last @@ -1452,7 +1453,8 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob, libcfs_nid2str(conn->gnc_peer->gnp_nid), cfs_duration_sec(now - newest_last_rx), cfs_duration_sec(GNILND_TIMEOUTRX(timeout))); - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_conn_mutex_unlock(&conn->gnc_smsg_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); return -ETIME; } @@ -1463,7 +1465,8 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob, */ msg->gnm_connstamp = conn->gnc_my_connstamp; msg->gnm_payload_len = immediatenob; - msg->gnm_seq = conn->gnc_tx_seq; + kgnilnd_conn_mutex_lock(&conn->gnc_smsg_mutex); + msg->gnm_seq = atomic_read(&conn->gnc_tx_seq); /* always init here - kgn_checksum is a /sys module tunable * and can be flipped at any point, even between msg init and sending */ @@ -1495,7 +1498,7 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob, switch (rrc) { case GNI_RC_SUCCESS: - conn->gnc_tx_seq++; + atomic_inc(&conn->gnc_tx_seq); conn->gnc_last_tx = jiffies; /* no locking here as LIVE isn't a list */ kgnilnd_tx_add_state_locked(tx, NULL, conn, GNILND_TX_LIVE_FMAQ, 1); @@ -1509,7 +1512,8 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob, /* serialize with seeing CQ events for completion on this, as well as * tx_seq */ - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_conn_mutex_unlock(&conn->gnc_smsg_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); atomic_inc(&conn->gnc_device->gnd_short_ntx); atomic64_add(immediatenob, &conn->gnc_device->gnd_short_txbytes); @@ -1521,8 +1525,8 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob, /* XXX Nic: We need to figure out how to track this * - there are bound to be good reasons for it, * but we want to know when it happens */ - - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_conn_mutex_unlock(&conn->gnc_smsg_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); /* We'll handle this error inline - makes the calling logic much more * clean */ @@ -1559,7 +1563,8 @@ kgnilnd_sendmsg_nolock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob, } default: /* handle bad retcode gracefully */ - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_conn_mutex_unlock(&conn->gnc_smsg_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); return -EIO; } } @@ -1574,7 +1579,7 @@ kgnilnd_sendmsg(kgn_tx_t *tx, void *immediate, unsigned int immediatenob, int rc; timestamp = jiffies; - mutex_lock(&dev->gnd_cq_mutex); + kgnilnd_gl_mutex_lock(&dev->gnd_cq_mutex); /* delay in jiffies - we are really concerned only with things that * result in a schedule() or really holding this off for long times . * NB - mutex_lock could spin for 2 jiffies before going to sleep to wait */ @@ -1619,7 +1624,7 @@ kgnilnd_sendmsg_trylock(kgn_tx_t *tx, void *immediate, unsigned int immediatenob rc = 0; } else { atomic_inc(&conn->gnc_device->gnd_fast_try); - rc = mutex_trylock(&conn->gnc_device->gnd_cq_mutex); + rc = kgnilnd_gl_mutex_trylock(&conn->gnc_device->gnd_cq_mutex); } if (!rc) { rc = -EAGAIN; @@ -1983,7 +1988,8 @@ kgnilnd_rdma(kgn_tx_t *tx, int type, tx->tx_rdma_desc.src_cq_hndl = conn->gnc_device->gnd_snd_rdma_cqh; timestamp = jiffies; - mutex_lock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_conn_mutex_lock(&conn->gnc_rdma_mutex); + kgnilnd_gl_mutex_lock(&conn->gnc_device->gnd_cq_mutex); /* delay in jiffies - we are really concerned only with things that * result in a schedule() or really holding this off for long times . * NB - mutex_lock could spin for 2 jiffies before going to sleep to wait */ @@ -1992,7 +1998,8 @@ kgnilnd_rdma(kgn_tx_t *tx, int type, rrc = kgnilnd_post_rdma(conn->gnc_ephandle, &tx->tx_rdma_desc); if (rrc == GNI_RC_ERROR_RESOURCE) { - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_conn_mutex_unlock(&conn->gnc_rdma_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); kgnilnd_unmap_buffer(tx, 0); if (tx->tx_buffer_copy != NULL) { @@ -2012,8 +2019,8 @@ kgnilnd_rdma(kgn_tx_t *tx, int type, kgnilnd_tx_add_state_locked(tx, conn->gnc_peer, conn, GNILND_TX_LIVE_RDMAQ, 1); tx->tx_qtime = jiffies; spin_unlock(&conn->gnc_list_lock); - - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_conn_mutex_unlock(&conn->gnc_rdma_mutex); /* XXX Nic: is this a place we should handle more errors for * robustness sake */ @@ -2049,14 +2056,14 @@ kgnilnd_release_msg(kgn_conn_t *conn) CDEBUG(D_NET, "consuming %p\n", conn); timestamp = jiffies; - mutex_lock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_gl_mutex_lock(&conn->gnc_device->gnd_cq_mutex); /* delay in jiffies - we are really concerned only with things that * result in a schedule() or really holding this off for long times . * NB - mutex_lock could spin for 2 jiffies before going to sleep to wait */ conn->gnc_device->gnd_mutex_delay += (long) jiffies - timestamp; rrc = kgnilnd_smsg_release(conn->gnc_ephandle); - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); LASSERTF(rrc == GNI_RC_SUCCESS, "bad rrc %d\n", rrc); GNIDBG_SMSG_CREDS(D_NET, conn); @@ -3165,7 +3172,7 @@ kgnilnd_check_rdma_cq(kgn_device_t *dev) } if (rrc == GNI_RC_NOT_DONE) { - mutex_unlock(&dev->gnd_cq_mutex); + kgnilnd_gl_mutex_unlock(&dev->gnd_cq_mutex); CDEBUG(D_INFO, "SEND RDMA CQ %d empty processed %ld\n", dev->gnd_id, num_processed); return num_processed; @@ -3182,7 +3189,7 @@ kgnilnd_check_rdma_cq(kgn_device_t *dev) rrc = kgnilnd_get_completed(dev->gnd_snd_rdma_cqh, event_data, &desc); - mutex_unlock(&dev->gnd_cq_mutex); + kgnilnd_gl_mutex_unlock(&dev->gnd_cq_mutex); /* XXX Nic: Need better error handling here... */ LASSERTF((rrc == GNI_RC_SUCCESS) || @@ -3226,9 +3233,11 @@ kgnilnd_check_rdma_cq(kgn_device_t *dev) } /* remove from rdmaq */ + kgnilnd_conn_mutex_lock(&conn->gnc_rdma_mutex); spin_lock(&conn->gnc_list_lock); kgnilnd_tx_del_state_locked(tx, NULL, conn, GNILND_TX_ALLOCD); spin_unlock(&conn->gnc_list_lock); + kgnilnd_conn_mutex_unlock(&conn->gnc_rdma_mutex); if (likely(desc->status == GNI_RC_SUCCESS) && rc == 0) { atomic_inc(&dev->gnd_rdma_ntx); @@ -3319,7 +3328,7 @@ kgnilnd_check_fma_send_cq(kgn_device_t *dev) } rrc = kgnilnd_cq_get_event(dev->gnd_snd_fma_cqh, &event_data); - mutex_unlock(&dev->gnd_cq_mutex); + kgnilnd_gl_mutex_unlock(&dev->gnd_cq_mutex); if (rrc == GNI_RC_NOT_DONE) { CDEBUG(D_INFO, @@ -3385,6 +3394,7 @@ kgnilnd_check_fma_send_cq(kgn_device_t *dev) } /* lock tx_list_state and tx_state */ + kgnilnd_conn_mutex_lock(&conn->gnc_smsg_mutex); spin_lock(&tx->tx_conn->gnc_list_lock); GNITX_ASSERTF(tx, tx->tx_list_state == GNILND_TX_LIVE_FMAQ, @@ -3405,6 +3415,7 @@ kgnilnd_check_fma_send_cq(kgn_device_t *dev) saw_reply = !(tx->tx_state & GNILND_TX_WAITING_REPLY); spin_unlock(&tx->tx_conn->gnc_list_lock); + kgnilnd_conn_mutex_unlock(&conn->gnc_smsg_mutex); if (queued_fma) { CDEBUG(D_NET, "scheduling conn 0x%p->%s for fmaq\n", @@ -3473,7 +3484,7 @@ kgnilnd_check_fma_rcv_cq(kgn_device_t *dev) return 1; } rrc = kgnilnd_cq_get_event(dev->gnd_rcv_fma_cqh, &event_data); - mutex_unlock(&dev->gnd_cq_mutex); + kgnilnd_gl_mutex_unlock(&dev->gnd_cq_mutex); if (rrc == GNI_RC_NOT_DONE) { CDEBUG(D_INFO, "SMSG RX CQ %d empty data "LPX64" " @@ -4085,7 +4096,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) RETURN_EXIT; timestamp = jiffies; - mutex_lock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_gl_mutex_lock(&conn->gnc_device->gnd_cq_mutex); /* delay in jiffies - we are really concerned only with things that * result in a schedule() or really holding this off for long times . * NB - mutex_lock could spin for 2 jiffies before going to sleep to wait */ @@ -4114,7 +4125,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) libcfs_nid2str(conn->gnc_peer->gnp_nid), cfs_duration_sec(timestamp - newest_last_rx), cfs_duration_sec(GNILND_TIMEOUTRX(timeout))); - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); rc = -ETIME; kgnilnd_close_conn(conn, rc); RETURN_EXIT; @@ -4123,7 +4134,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) rrc = kgnilnd_smsg_getnext(conn->gnc_ephandle, &prefix); if (rrc == GNI_RC_NOT_DONE) { - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); CDEBUG(D_INFO, "SMSG RX empty conn 0x%p\n", conn); RETURN_EXIT; } @@ -4137,7 +4148,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) */ if (rrc == GNI_RC_INVALID_STATE) { - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); GNIDBG_CONN(D_NETERROR | D_CONSOLE, conn, "Mailbox corruption " "detected closing conn %p from peer %s\n", conn, libcfs_nid2str(conn->gnc_peer->gnp_nid)); @@ -4154,7 +4165,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) rx = kgnilnd_alloc_rx(); if (rx == NULL) { - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); kgnilnd_release_msg(conn); GNIDBG_MSG(D_NETERROR, msg, "Dropping SMSG RX from 0x%p->%s, no RX memory", conn, libcfs_nid2str(peer->gnp_nid)); @@ -4164,7 +4175,8 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) GNIDBG_MSG(D_INFO, msg, "SMSG RX on %p", conn); timestamp = conn->gnc_last_rx; - last_seq = conn->gnc_rx_seq; + seq = last_seq = atomic_read(&conn->gnc_rx_seq); + atomic_inc(&conn->gnc_rx_seq); conn->gnc_last_rx = jiffies; /* stash first rx so we can clear out purgatory @@ -4172,10 +4184,8 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) if (conn->gnc_first_rx == 0) conn->gnc_first_rx = jiffies; - seq = conn->gnc_rx_seq++; - /* needs to linger to protect gnc_rx_seq like we do with gnc_tx_seq */ - mutex_unlock(&conn->gnc_device->gnd_cq_mutex); + kgnilnd_gl_mutex_unlock(&conn->gnc_device->gnd_cq_mutex); kgnilnd_peer_alive(conn->gnc_peer); rx->grx_msg = msg; @@ -4299,7 +4309,7 @@ kgnilnd_check_fma_rx(kgn_conn_t *conn) conn, last_seq, cfs_duration_sec(now - timestamp), cfs_duration_sec(now - conn->gnc_last_rx_cq), - conn->gnc_tx_seq, + atomic_read(&conn->gnc_tx_seq), cfs_duration_sec(now - conn->gnc_last_tx), cfs_duration_sec(now - conn->gnc_last_tx_cq), cfs_duration_sec(now - conn->gnc_last_noop_want), diff --git a/lnet/klnds/gnilnd/gnilnd_debug.c b/lnet/klnds/gnilnd/gnilnd_debug.c index c4f2b9b..36a9844 100644 --- a/lnet/klnds/gnilnd/gnilnd_debug.c +++ b/lnet/klnds/gnilnd/gnilnd_debug.c @@ -52,10 +52,10 @@ _kgnilnd_debug_conn(kgn_conn_t *conn, struct libcfs_debug_msg_data *msgdata, conn, conn->gnc_peer ? libcfs_nid2str(conn->gnc_peer->gnp_nid) : "", kgnilnd_conn_state2str(conn), conn->gnc_cqid, conn->gnc_timeout, - conn->gnc_rx_seq, + atomic_read(&conn->gnc_rx_seq), cfs_duration_sec(jiffies - conn->gnc_last_rx), cfs_duration_sec(jiffies - conn->gnc_last_rx_cq), - conn->gnc_tx_seq, + atomic_read(&conn->gnc_tx_seq), cfs_duration_sec(jiffies - conn->gnc_last_tx), cfs_duration_sec(jiffies - conn->gnc_last_tx_cq), cfs_duration_sec(jiffies - conn->gnc_last_noop_want), diff --git a/lnet/klnds/gnilnd/gnilnd_gemini.h b/lnet/klnds/gnilnd/gnilnd_gemini.h index a74234e..79be5b3 100644 --- a/lnet/klnds/gnilnd/gnilnd_gemini.h +++ b/lnet/klnds/gnilnd/gnilnd_gemini.h @@ -34,4 +34,7 @@ #define GNILND_REVERSE_RDMA GNILND_REVERSE_NONE #define GNILND_RDMA_DLVR_OPTION GNI_DLVMODE_PERFORMANCE +/* Thread-safe kgni implemented in minor ver 44, code rev 0xb9 */ +#define GNILND_KGNI_TS_MINOR_VER 0x44 + #endif /* _GNILND_GEMINI_H */ diff --git a/lnet/klnds/gnilnd/gnilnd_proc.c b/lnet/klnds/gnilnd/gnilnd_proc.c index 292186d..c5382fb 100644 --- a/lnet/klnds/gnilnd/gnilnd_proc.c +++ b/lnet/klnds/gnilnd/gnilnd_proc.c @@ -911,10 +911,10 @@ kgnilnd_conn_seq_show(struct seq_file *s, void *iter) kgnilnd_count_list(&conn->gnc_fmaq), atomic_read(&conn->gnc_nlive_fma), atomic_read(&conn->gnc_nlive_rdma), - conn->gnc_tx_seq, + atomic_read(&conn->gnc_tx_seq), jiffies_to_msecs(jiffies - conn->gnc_last_tx), jiffies_to_msecs(jiffies - conn->gnc_last_tx_cq), - conn->gnc_rx_seq, + atomic_read(&conn->gnc_rx_seq), jiffies_to_msecs(jiffies - conn->gnc_last_rx), jiffies_to_msecs(jiffies - conn->gnc_last_rx_cq), atomic_read(&conn->gnc_reaper_noop), @@ -1043,10 +1043,10 @@ kgnilnd_proc_peer_conns_seq_show(struct seq_file *sf, void *v) conn->gnc_close_recvd, conn->gnc_error, conn->gnc_peer_error, - conn->gnc_tx_seq, + atomic_read(&conn->gnc_tx_seq), jiffies_to_msecs(jifs - conn->gnc_last_tx), jiffies_to_msecs(jifs - conn->gnc_last_tx_cq), - conn->gnc_rx_seq, + atomic_read(&conn->gnc_rx_seq), jiffies_to_msecs(jifs - conn->gnc_first_rx), jiffies_to_msecs(jifs - conn->gnc_last_rx), jiffies_to_msecs(jifs - conn->gnc_last_rx_cq), -- 1.8.3.1