LU-15828 o2iblnd: reset hiw proportionally

As a result of connection negotiation, queue depth may end up
being shorter than "peer_tx_credits" tunables value. Before this
patch, the high-water mark "lnd_peercredits_hiw" would be set at
	min(current hiw, queue depth - 1).

For example, considering that hiw is allowed to only be as low as
half of peer_tx_credits, negotiating queue_depth/peer_credits down
from 32 to 8 would always result in hiw set at 7, i.e. credits would
be released as late as possible.

With this patch, if queue depth is reduced, hiw is set proportionally
relative to the level it was at before:
	hiw = (queue_depth * lnd_peercredits_hiw) / peer_tx_credits

Using the above example with queue depth initially at 32, negotiating
down to 8 would result in hiw set to 4 if "lnd_peercredits_hiw" is
initially at 16, 17, 18, 19; hiw set to 5 if "lnd_peercredits_hiw" is
initially at 20, 21, 22, 23, and so on.

Lustre-change: https://review.whamcloud.com/49497
Lustre-commit: e1944c29793d489429730a9445e243b448c3d751

Test-Parameters: trivial
Signed-off-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Change-Id: I633933d7448db1ca88d3c65de9c29e870ca2c9fb
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Chris Horn <chris.horn@hpe.com>
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/49637
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index a1bb47b..75e88b0 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -125,11 +125,6 @@ extern struct kib_tunables  kiblnd_tunables;
 #define IBLND_CREDITS_DEFAULT        8          /* default # of peer_ni credits */
 #define IBLND_CREDITS_MAX          ((typeof(((struct kib_msg *) 0)->ibm_credits)) - 1)  /* Max # of peer_ni credits */
 
-/* when eagerly to return credits */
-#define IBLND_CREDITS_HIGHWATER(t, conn) ((conn->ibc_version) == IBLND_MSG_VERSION_1 ? \
-                                       IBLND_CREDIT_HIGHWATER_V1 : \
-                       min(t->lnd_peercredits_hiw, (__u32)conn->ibc_queue_depth - 1))
-
 #ifdef HAVE_RDMA_CREATE_ID_5ARG
 # define kiblnd_rdma_create_id(ns, cb, dev, ps, qpt) \
         rdma_create_id((ns) ? (ns) : &init_net, cb, dev, ps, qpt)
@@ -971,17 +966,37 @@ kiblnd_send_keepalive(struct kib_conn *conn)
                            ktime_add_ns(conn->ibc_last_send, keepalive_ns));
 }
 
+/* when to return credits eagerly */
+static inline int
+kiblnd_credits_highwater(struct lnet_ioctl_config_o2iblnd_tunables *t,
+                        struct lnet_ioctl_config_lnd_cmn_tunables *nt,
+                        struct kib_conn *conn)
+{
+       int credits_hiw = IBLND_CREDIT_HIGHWATER_V1;
+
+       if ((conn->ibc_version) == IBLND_MSG_VERSION_1)
+               return credits_hiw;
+
+       /* if queue depth is negotiated down, calculate hiw proportionally */
+       credits_hiw = (conn->ibc_queue_depth * t->lnd_peercredits_hiw) /
+                      nt->lct_peer_tx_credits;
+
+       return credits_hiw;
+}
+
 static inline int
 kiblnd_need_noop(struct kib_conn *conn)
 {
        struct lnet_ni *ni = conn->ibc_peer->ibp_ni;
        struct lnet_ioctl_config_o2iblnd_tunables *tunables;
+       struct lnet_ioctl_config_lnd_cmn_tunables *net_tunables;
 
        LASSERT(conn->ibc_state >= IBLND_CONN_ESTABLISHED);
        tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib;
+       net_tunables = &ni->ni_net->net_tunables;
 
         if (conn->ibc_outstanding_credits <
-           IBLND_CREDITS_HIGHWATER(tunables, conn) &&
+           kiblnd_credits_highwater(tunables, net_tunables, conn) &&
             !kiblnd_send_keepalive(conn))
                 return 0; /* No need to send NOOP */