LU-11931 lnd: bring back concurrent_sends

Revert "LU-10291 lnd: remove concurrent_sends tunable"

This reverts commit 8d35d6c9bd85ed3a282aa124b672e50c02322a7d.

Signed-off-by: Amir Shehata <ashehata@whamcloud.com>
Change-Id: Icb63d7383f0d2a3cab82c1565f66670dca1f698d
Reviewed-on: https://review.whamcloud.com/34396
Reviewed-by: James Simmons <uja.ornl@yahoo.com>
Tested-by: Jenkins
Reviewed-by: Sonia Sharma <sharmaso@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index aab8890..19f327f 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -162,7 +162,9 @@ extern struct kib_tunables  kiblnd_tunables;
 #define IBLND_RECV_WRS(c)            IBLND_RX_MSGS(c)
 
 /* 2 = LNet msg + Transfer chain */
-#define IBLND_CQ_ENTRIES(c)    (IBLND_RECV_WRS(c) + kiblnd_send_wrs(c))
+#define IBLND_CQ_ENTRIES(c)    \
+       (IBLND_RECV_WRS(c) + 2 * kiblnd_concurrent_sends(c->ibc_version, \
+                                                        c->ibc_peer->ibp_ni))
 
 struct kib_hca_dev;
 
@@ -776,6 +778,26 @@ extern void kiblnd_hdev_destroy(struct kib_hca_dev *hdev);
 
 int kiblnd_msg_queue_size(int version, struct lnet_ni *ni);
 
+static inline int
+kiblnd_concurrent_sends(int version, struct lnet_ni *ni)
+{
+       struct lnet_ioctl_config_o2iblnd_tunables *tunables;
+       int concurrent_sends;
+
+       tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib;
+       concurrent_sends = tunables->lnd_concurrent_sends;
+
+       if (version == IBLND_MSG_VERSION_1) {
+               if (concurrent_sends > IBLND_MSG_QUEUE_SIZE_V1 * 2)
+                       return IBLND_MSG_QUEUE_SIZE_V1 * 2;
+
+               if (concurrent_sends < IBLND_MSG_QUEUE_SIZE_V1 / 2)
+                       return IBLND_MSG_QUEUE_SIZE_V1 / 2;
+       }
+
+       return concurrent_sends;
+}
+
 static inline void
 kiblnd_hdev_addref_locked(struct kib_hca_dev *hdev)
 {

diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index a6e06e6..d6dc96f 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -866,6 +866,7 @@ __must_hold(&conn->ibc_lock)
 {
        struct kib_msg *msg = tx->tx_msg;
        struct kib_peer_ni *peer_ni = conn->ibc_peer;
+       struct lnet_ni *ni = peer_ni->ibp_ni;
        int ver = conn->ibc_version;
        int rc;
        int done;
@@ -882,7 +883,7 @@ __must_hold(&conn->ibc_lock)
        LASSERT(conn->ibc_credits <= conn->ibc_queue_depth);
 
        if (conn->ibc_nsends_posted ==
-           conn->ibc_queue_depth) {
+           kiblnd_concurrent_sends(ver, ni)) {
                /* tx completions outstanding... */
                CDEBUG(D_NET, "%s: posted enough\n",
                       libcfs_nid2str(peer_ni->ibp_nid));
@@ -1031,7 +1032,7 @@ kiblnd_check_sends_locked(struct kib_conn *conn)
         }
 
        LASSERT(conn->ibc_nsends_posted <=
-               conn->ibc_queue_depth);
+               kiblnd_concurrent_sends(ver, ni));
         LASSERT (!IBLND_OOB_CAPABLE(ver) ||
                  conn->ibc_noops_posted <= IBLND_OOB_MSGS(ver));
         LASSERT (conn->ibc_reserved_credits >= 0);

diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index c91bfd1..2a329e5 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -108,7 +108,7 @@ MODULE_PARM_DESC(ib_mtu, "IB MTU 256/512/1024/2048/4096");
 
 static int concurrent_sends;
 module_param(concurrent_sends, int, 0444);
-MODULE_PARM_DESC(concurrent_sends, "send work-queue sizing (obsolete)");
+MODULE_PARM_DESC(concurrent_sends, "send work-queue sizing");
 
 static int use_fastreg_gaps;
 module_param(use_fastreg_gaps, int, 0444);
@@ -285,6 +285,22 @@ kiblnd_tunables_setup(struct lnet_ni *ni)
        if (tunables->lnd_peercredits_hiw >= net_tunables->lct_peer_tx_credits)
                tunables->lnd_peercredits_hiw = net_tunables->lct_peer_tx_credits - 1;
 
+       if (tunables->lnd_concurrent_sends == 0)
+                       tunables->lnd_concurrent_sends = net_tunables->lct_peer_tx_credits;
+
+       if (tunables->lnd_concurrent_sends > net_tunables->lct_peer_tx_credits * 2)
+               tunables->lnd_concurrent_sends = net_tunables->lct_peer_tx_credits * 2;
+
+       if (tunables->lnd_concurrent_sends < net_tunables->lct_peer_tx_credits / 2)
+               tunables->lnd_concurrent_sends = net_tunables->lct_peer_tx_credits / 2;
+
+       if (tunables->lnd_concurrent_sends < net_tunables->lct_peer_tx_credits) {
+               CWARN("Concurrent sends %d is lower than message "
+                     "queue size: %d, performance may drop slightly.\n",
+                     tunables->lnd_concurrent_sends,
+                     net_tunables->lct_peer_tx_credits);
+       }
+
        if (!tunables->lnd_fmr_pool_size)
                tunables->lnd_fmr_pool_size = fmr_pool_size;
        if (!tunables->lnd_fmr_flush_trigger)
@@ -307,6 +323,7 @@ kiblnd_tunables_init(void)
        default_tunables.lnd_version = CURRENT_LND_VERSION;
        default_tunables.lnd_peercredits_hiw = peer_credits_hiw;
        default_tunables.lnd_map_on_demand = map_on_demand;
+       default_tunables.lnd_concurrent_sends = concurrent_sends;
        default_tunables.lnd_fmr_pool_size = fmr_pool_size;
        default_tunables.lnd_fmr_flush_trigger = fmr_flush_trigger;
        default_tunables.lnd_fmr_cache = fmr_cache;

diff --git a/lustre/conf/ko2iblnd.conf b/lustre/conf/ko2iblnd.conf
index 3d55009..b486895 100644 (file)
--- a/lustre/conf/ko2iblnd.conf
+++ b/lustre/conf/ko2iblnd.conf
@@ -13,6 +13,6 @@
 # Lustre is first mounted.
 
 alias ko2iblnd-opa ko2iblnd
-options ko2iblnd-opa peer_credits=128 peer_credits_hiw=64 credits=1024 ntx=2048 map_on_demand=256 fmr_pool_size=2048 fmr_flush_trigger=512 fmr_cache=1 conns_per_peer=4
+options ko2iblnd-opa peer_credits=128 peer_credits_hiw=64 credits=1024 concurrent_sends=256 ntx=2048 map_on_demand=256 fmr_pool_size=2048 fmr_flush_trigger=512 fmr_cache=1 conns_per_peer=4
 
 install ko2iblnd /usr/sbin/ko2iblnd-probe