From 8d35d6c9bd85ed3a282aa124b672e50c02322a7d Mon Sep 17 00:00:00 2001 From: Amir Shehata Date: Tue, 28 Nov 2017 11:08:31 -0800 Subject: [PATCH] LU-10291 lnd: remove concurrent_sends tunable Concurrent sends tunable was intended to limit the number of in-flight transfers per connection. However queue depth does the exact same job. So for example if the queue depth is negotiated to 16 and concurrent_sends is set to 32, the maximum number of in-flight transfers doesn't exceed 16. There is no need to keep concurrent_sends around since it doesn't add any unique functionality Test-Parameters: trivial Signed-off-by: Amir Shehata Change-Id: I4dd7f5ecac5a7d10403c4f0ab0f01374a2f10206 Reviewed-on: https://review.whamcloud.com/30312 Reviewed-by: Alexey Lyashkov Tested-by: Jenkins Reviewed-by: Dmitry Eremin Reviewed-by: Doug Oucharek Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- lnet/klnds/o2iblnd/o2iblnd.h | 23 +---------------------- lnet/klnds/o2iblnd/o2iblnd_cb.c | 12 ++---------- lnet/klnds/o2iblnd/o2iblnd_modparams.c | 19 +------------------ lustre/conf/ko2iblnd.conf | 2 +- 4 files changed, 5 insertions(+), 51 deletions(-) diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index 391aa7f..dfaec9ef 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -167,8 +167,7 @@ extern kib_tunables_t kiblnd_tunables; /* 2 = LNet msg + Transfer chain */ #define IBLND_CQ_ENTRIES(c) \ - (IBLND_RECV_WRS(c) + 2 * kiblnd_concurrent_sends(c->ibc_version, \ - c->ibc_peer->ibp_ni)) + (IBLND_RECV_WRS(c) + 2 * c->ibc_queue_depth) struct kib_hca_dev; @@ -797,26 +796,6 @@ extern void kiblnd_hdev_destroy(kib_hca_dev_t *hdev); int kiblnd_msg_queue_size(int version, struct lnet_ni *ni); -static inline int -kiblnd_concurrent_sends(int version, struct lnet_ni *ni) -{ - struct lnet_ioctl_config_o2iblnd_tunables *tunables; - int concurrent_sends; - - tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib; - concurrent_sends = tunables->lnd_concurrent_sends; - - if (version == IBLND_MSG_VERSION_1) { - if (concurrent_sends > IBLND_MSG_QUEUE_SIZE_V1 * 2) - return IBLND_MSG_QUEUE_SIZE_V1 * 2; - - if (concurrent_sends < IBLND_MSG_QUEUE_SIZE_V1 / 2) - return IBLND_MSG_QUEUE_SIZE_V1 / 2; - } - - return concurrent_sends; -} - static inline void kiblnd_hdev_addref_locked(kib_hca_dev_t *hdev) { diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c index 2a78ef7..1ce5ff6 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_cb.c +++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c @@ -834,7 +834,6 @@ __must_hold(&conn->ibc_lock) { kib_msg_t *msg = tx->tx_msg; kib_peer_ni_t *peer_ni = conn->ibc_peer; - struct lnet_ni *ni = peer_ni->ibp_ni; int ver = conn->ibc_version; int rc; int done; @@ -849,14 +848,7 @@ __must_hold(&conn->ibc_lock) LASSERT(conn->ibc_outstanding_credits <= conn->ibc_queue_depth); LASSERT(conn->ibc_credits >= 0); LASSERT(conn->ibc_credits <= conn->ibc_queue_depth); - - if (conn->ibc_nsends_posted == - kiblnd_concurrent_sends(ver, ni)) { - /* tx completions outstanding... */ - CDEBUG(D_NET, "%s: posted enough\n", - libcfs_nid2str(peer_ni->ibp_nid)); - return -EAGAIN; - } + LASSERT(conn->ibc_nsends_posted <= conn->ibc_queue_depth); if (credit != 0 && conn->ibc_credits == 0) { /* no credits */ CDEBUG(D_NET, "%s: no credits\n", @@ -996,7 +988,7 @@ kiblnd_check_sends_locked(kib_conn_t *conn) } LASSERT(conn->ibc_nsends_posted <= - kiblnd_concurrent_sends(ver, ni)); + conn->ibc_queue_depth); LASSERT (!IBLND_OOB_CAPABLE(ver) || conn->ibc_noops_posted <= IBLND_OOB_MSGS(ver)); LASSERT (conn->ibc_reserved_credits >= 0); diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c index 8b02fd9..2f71667 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c +++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c @@ -108,7 +108,7 @@ MODULE_PARM_DESC(ib_mtu, "IB MTU 256/512/1024/2048/4096"); static int concurrent_sends; module_param(concurrent_sends, int, 0444); -MODULE_PARM_DESC(concurrent_sends, "send work-queue sizing"); +MODULE_PARM_DESC(concurrent_sends, "send work-queue sizing (obsolete)"); /* * map_on_demand is a flag used to determine if we can use FMR or FastReg. @@ -280,22 +280,6 @@ kiblnd_tunables_setup(struct lnet_ni *ni) if (tunables->lnd_peercredits_hiw >= net_tunables->lct_peer_tx_credits) tunables->lnd_peercredits_hiw = net_tunables->lct_peer_tx_credits - 1; - if (tunables->lnd_concurrent_sends == 0) - tunables->lnd_concurrent_sends = net_tunables->lct_peer_tx_credits; - - if (tunables->lnd_concurrent_sends > net_tunables->lct_peer_tx_credits * 2) - tunables->lnd_concurrent_sends = net_tunables->lct_peer_tx_credits * 2; - - if (tunables->lnd_concurrent_sends < net_tunables->lct_peer_tx_credits / 2) - tunables->lnd_concurrent_sends = net_tunables->lct_peer_tx_credits / 2; - - if (tunables->lnd_concurrent_sends < net_tunables->lct_peer_tx_credits) { - CWARN("Concurrent sends %d is lower than message " - "queue size: %d, performance may drop slightly.\n", - tunables->lnd_concurrent_sends, - net_tunables->lct_peer_tx_credits); - } - if (!tunables->lnd_fmr_pool_size) tunables->lnd_fmr_pool_size = fmr_pool_size; if (!tunables->lnd_fmr_flush_trigger) @@ -318,7 +302,6 @@ kiblnd_tunables_init(void) default_tunables.lnd_version = CURRENT_LND_VERSION; default_tunables.lnd_peercredits_hiw = peer_credits_hiw; default_tunables.lnd_map_on_demand = map_on_demand; - default_tunables.lnd_concurrent_sends = concurrent_sends; default_tunables.lnd_fmr_pool_size = fmr_pool_size; default_tunables.lnd_fmr_flush_trigger = fmr_flush_trigger; default_tunables.lnd_fmr_cache = fmr_cache; diff --git a/lustre/conf/ko2iblnd.conf b/lustre/conf/ko2iblnd.conf index 2ad2932..3f0d28c 100644 --- a/lustre/conf/ko2iblnd.conf +++ b/lustre/conf/ko2iblnd.conf @@ -9,6 +9,6 @@ # Lustre is first mounted. alias ko2iblnd-opa ko2iblnd -options ko2iblnd-opa peer_credits=128 peer_credits_hiw=64 credits=1024 concurrent_sends=256 ntx=2048 map_on_demand=256 fmr_pool_size=2048 fmr_flush_trigger=512 fmr_cache=1 conns_per_peer=4 +options ko2iblnd-opa peer_credits=128 peer_credits_hiw=64 credits=1024 ntx=2048 map_on_demand=256 fmr_pool_size=2048 fmr_flush_trigger=512 fmr_cache=1 conns_per_peer=4 install ko2iblnd /usr/sbin/ko2iblnd-probe -- 1.8.3.1