LU-10291 lnd: remove concurrent_sends tunable

Concurrent sends tunable was intended to limit the number of in-flight
transfers per connection. However queue depth does the exact same job.
So for example if the queue depth is negotiated to 16 and
concurrent_sends is set to 32, the maximum number of in-flight
transfers doesn't exceed 16. There is no need to keep concurrent_sends
around since it doesn't add any unique functionality

Test-Parameters: trivial
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Change-Id: I4dd7f5ecac5a7d10403c4f0ab0f01374a2f10206
Reviewed-on: https://review.whamcloud.com/30312
Reviewed-by: Alexey Lyashkov <c17817@cray.com>
Tested-by: Jenkins
Reviewed-by: Dmitry Eremin <dmitry.eremin@intel.com>
Reviewed-by: Doug Oucharek <dougso@me.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: James Simmons <uja.ornl@yahoo.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index 391aa7f..dfaec9e 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -167,8 +167,7 @@ extern kib_tunables_t  kiblnd_tunables;
 
 /* 2 = LNet msg + Transfer chain */
 #define IBLND_CQ_ENTRIES(c)    \
-       (IBLND_RECV_WRS(c) + 2 * kiblnd_concurrent_sends(c->ibc_version, \
-                                                        c->ibc_peer->ibp_ni))
+       (IBLND_RECV_WRS(c) + 2 * c->ibc_queue_depth)
 
 struct kib_hca_dev;
 
@@ -797,26 +796,6 @@ extern void kiblnd_hdev_destroy(kib_hca_dev_t *hdev);
 
 int kiblnd_msg_queue_size(int version, struct lnet_ni *ni);
 
-static inline int
-kiblnd_concurrent_sends(int version, struct lnet_ni *ni)
-{
-       struct lnet_ioctl_config_o2iblnd_tunables *tunables;
-       int concurrent_sends;
-
-       tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib;
-       concurrent_sends = tunables->lnd_concurrent_sends;
-
-       if (version == IBLND_MSG_VERSION_1) {
-               if (concurrent_sends > IBLND_MSG_QUEUE_SIZE_V1 * 2)
-                       return IBLND_MSG_QUEUE_SIZE_V1 * 2;
-
-               if (concurrent_sends < IBLND_MSG_QUEUE_SIZE_V1 / 2)
-                       return IBLND_MSG_QUEUE_SIZE_V1 / 2;
-       }
-
-       return concurrent_sends;
-}
-
 static inline void
 kiblnd_hdev_addref_locked(kib_hca_dev_t *hdev)
 {

diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index 2a78ef7..1ce5ff6 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -834,7 +834,6 @@ __must_hold(&conn->ibc_lock)
 {
        kib_msg_t *msg = tx->tx_msg;
        kib_peer_ni_t *peer_ni = conn->ibc_peer;
-       struct lnet_ni *ni = peer_ni->ibp_ni;
        int ver = conn->ibc_version;
        int rc;
        int done;
@@ -849,14 +848,7 @@ __must_hold(&conn->ibc_lock)
        LASSERT(conn->ibc_outstanding_credits <= conn->ibc_queue_depth);
        LASSERT(conn->ibc_credits >= 0);
        LASSERT(conn->ibc_credits <= conn->ibc_queue_depth);
-
-       if (conn->ibc_nsends_posted ==
-           kiblnd_concurrent_sends(ver, ni)) {
-                /* tx completions outstanding... */
-                CDEBUG(D_NET, "%s: posted enough\n",
-                       libcfs_nid2str(peer_ni->ibp_nid));
-                return -EAGAIN;
-        }
+       LASSERT(conn->ibc_nsends_posted <= conn->ibc_queue_depth);
 
         if (credit != 0 && conn->ibc_credits == 0) {   /* no credits */
                 CDEBUG(D_NET, "%s: no credits\n",
@@ -996,7 +988,7 @@ kiblnd_check_sends_locked(kib_conn_t *conn)
         }
 
        LASSERT(conn->ibc_nsends_posted <=
-               kiblnd_concurrent_sends(ver, ni));
+               conn->ibc_queue_depth);
         LASSERT (!IBLND_OOB_CAPABLE(ver) ||
                  conn->ibc_noops_posted <= IBLND_OOB_MSGS(ver));
         LASSERT (conn->ibc_reserved_credits >= 0);

diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index 8b02fd9..2f71667 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -108,7 +108,7 @@ MODULE_PARM_DESC(ib_mtu, "IB MTU 256/512/1024/2048/4096");
 
 static int concurrent_sends;
 module_param(concurrent_sends, int, 0444);
-MODULE_PARM_DESC(concurrent_sends, "send work-queue sizing");
+MODULE_PARM_DESC(concurrent_sends, "send work-queue sizing (obsolete)");
 
 /*
  * map_on_demand is a flag used to determine if we can use FMR or FastReg.
@@ -280,22 +280,6 @@ kiblnd_tunables_setup(struct lnet_ni *ni)
        if (tunables->lnd_peercredits_hiw >= net_tunables->lct_peer_tx_credits)
                tunables->lnd_peercredits_hiw = net_tunables->lct_peer_tx_credits - 1;
 
-       if (tunables->lnd_concurrent_sends == 0)
-                       tunables->lnd_concurrent_sends = net_tunables->lct_peer_tx_credits;
-
-       if (tunables->lnd_concurrent_sends > net_tunables->lct_peer_tx_credits * 2)
-               tunables->lnd_concurrent_sends = net_tunables->lct_peer_tx_credits * 2;
-
-       if (tunables->lnd_concurrent_sends < net_tunables->lct_peer_tx_credits / 2)
-               tunables->lnd_concurrent_sends = net_tunables->lct_peer_tx_credits / 2;
-
-       if (tunables->lnd_concurrent_sends < net_tunables->lct_peer_tx_credits) {
-               CWARN("Concurrent sends %d is lower than message "
-                     "queue size: %d, performance may drop slightly.\n",
-                     tunables->lnd_concurrent_sends,
-                     net_tunables->lct_peer_tx_credits);
-       }
-
        if (!tunables->lnd_fmr_pool_size)
                tunables->lnd_fmr_pool_size = fmr_pool_size;
        if (!tunables->lnd_fmr_flush_trigger)
@@ -318,7 +302,6 @@ kiblnd_tunables_init(void)
        default_tunables.lnd_version = CURRENT_LND_VERSION;
        default_tunables.lnd_peercredits_hiw = peer_credits_hiw;
        default_tunables.lnd_map_on_demand = map_on_demand;
-       default_tunables.lnd_concurrent_sends = concurrent_sends;
        default_tunables.lnd_fmr_pool_size = fmr_pool_size;
        default_tunables.lnd_fmr_flush_trigger = fmr_flush_trigger;
        default_tunables.lnd_fmr_cache = fmr_cache;

diff --git a/lustre/conf/ko2iblnd.conf b/lustre/conf/ko2iblnd.conf
index 2ad2932..3f0d28c 100644 (file)
--- a/lustre/conf/ko2iblnd.conf
+++ b/lustre/conf/ko2iblnd.conf
@@ -9,6 +9,6 @@
 # Lustre is first mounted.
 
 alias ko2iblnd-opa ko2iblnd
-options ko2iblnd-opa peer_credits=128 peer_credits_hiw=64 credits=1024 concurrent_sends=256 ntx=2048 map_on_demand=256 fmr_pool_size=2048 fmr_flush_trigger=512 fmr_cache=1 conns_per_peer=4
+options ko2iblnd-opa peer_credits=128 peer_credits_hiw=64 credits=1024 ntx=2048 map_on_demand=256 fmr_pool_size=2048 fmr_flush_trigger=512 fmr_cache=1 conns_per_peer=4
 
 install ko2iblnd /usr/sbin/ko2iblnd-probe