LU-12621 o2iblnd: cache max_qp_wr

When creating the device the maximum number of work requests per qp
which can be allocated is already known. Cache that internally,
and when creating the qp make sure the qp's max_send_wr does not
exceed that max. If it does then cap max_send_wr to max_qp_wr.
Recalculate the connection's queue depth based on the max_qp_wr.

Test-Parameter: nettype=o2ib
Signed-off-by: Amir Shehata <ashehata@whamcloud.com>
Change-Id: I6d9a642d03633264f5f14445a051dd14515709c1
Reviewed-on: https://review.whamcloud.com/36073
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Doug Oucharek <dougso@me.com>
Reviewed-by: Olaf Weber <olaf.weber@hpe.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index b729fa5..0d7b50a 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -734,16 +734,28 @@ static unsigned int kiblnd_send_wrs(struct kib_conn *conn)
         * One WR for the LNet message
         * And ibc_max_frags for the transfer WRs
         */
         * One WR for the LNet message
         * And ibc_max_frags for the transfer WRs
         */

-       unsigned int ret = 1 + conn->ibc_max_frags;
+       int ret;
+       int multiplier = 1 + conn->ibc_max_frags;

        enum kib_dev_caps dev_caps = conn->ibc_hdev->ibh_dev->ibd_dev_caps;
 
        /* FastReg needs two extra WRs for map and invalidate */
        if (dev_caps & IBLND_DEV_CAPS_FASTREG_ENABLED)
        enum kib_dev_caps dev_caps = conn->ibc_hdev->ibh_dev->ibd_dev_caps;
 
        /* FastReg needs two extra WRs for map and invalidate */
        if (dev_caps & IBLND_DEV_CAPS_FASTREG_ENABLED)

-               ret += 2;
+               multiplier += 2;

 
        /* account for a maximum of ibc_queue_depth in-flight transfers */
 
        /* account for a maximum of ibc_queue_depth in-flight transfers */

-       ret *= conn->ibc_queue_depth;
-       return ret;
+       ret = multiplier * conn->ibc_queue_depth;
+
+       if (ret > conn->ibc_hdev->ibh_max_qp_wr) {
+               CDEBUG(D_NET, "peer_credits %u will result in send work "
+                      "request size %d larger than maximum %d device "
+                      "can handle\n", conn->ibc_queue_depth, ret,
+                      conn->ibc_hdev->ibh_max_qp_wr);
+               conn->ibc_queue_depth =
+                       conn->ibc_hdev->ibh_max_qp_wr / multiplier;
+       }
+
+       /* don't go beyond the maximum the device can handle */
+       return min(ret, conn->ibc_hdev->ibh_max_qp_wr);

 }
 
 struct kib_conn *
 }
 
 struct kib_conn *


@@ -900,20 +912,14 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid,
        init_qp_attr->qp_type = IB_QPT_RC;
        init_qp_attr->send_cq = cq;
        init_qp_attr->recv_cq = cq;
        init_qp_attr->qp_type = IB_QPT_RC;
        init_qp_attr->send_cq = cq;
        init_qp_attr->recv_cq = cq;

+       /*
+        * kiblnd_send_wrs() can change the connection's queue depth if
+        * the maximum work requests for the device is maxed out
+        */
+       init_qp_attr->cap.max_send_wr = kiblnd_send_wrs(conn);
+       init_qp_attr->cap.max_recv_wr = IBLND_RECV_WRS(conn);

 
 

-       conn->ibc_sched = sched;
-
-       do {
-               init_qp_attr->cap.max_send_wr = kiblnd_send_wrs(conn);
-               init_qp_attr->cap.max_recv_wr = IBLND_RECV_WRS(conn);
-
-               rc = rdma_create_qp(cmid, conn->ibc_hdev->ibh_pd, init_qp_attr);
-               if (!rc || conn->ibc_queue_depth < 2)
-                       break;
-
-               conn->ibc_queue_depth--;
-       } while (rc);
-
+       rc = rdma_create_qp(cmid, conn->ibc_hdev->ibh_pd, init_qp_attr);

        if (rc) {
                CERROR("Can't create QP: %d, send_wr: %d, recv_wr: %d, "
                       "send_sge: %d, recv_sge: %d\n",
        if (rc) {
                CERROR("Can't create QP: %d, send_wr: %d, recv_wr: %d, "
                       "send_sge: %d, recv_sge: %d\n",


@@ -924,6 +930,8 @@ kiblnd_create_conn(struct kib_peer_ni *peer_ni, struct rdma_cm_id *cmid,
                goto failed_2;
        }
 
                goto failed_2;
        }
 

+       conn->ibc_sched = sched;
+

        if (conn->ibc_queue_depth != peer_ni->ibp_queue_depth)
                CWARN("peer %s - queue depth reduced from %u to %u"
                      "  to allow for qp creation\n",
        if (conn->ibc_queue_depth != peer_ni->ibp_queue_depth)
                CWARN("peer %s - queue depth reduced from %u to %u"
                      "  to allow for qp creation\n",


@@ -2562,6 +2570,7 @@ kiblnd_hdev_get_attr(struct kib_hca_dev *hdev)
 #endif
 
        hdev->ibh_mr_size = dev_attr->max_mr_size;
 #endif
 
        hdev->ibh_mr_size = dev_attr->max_mr_size;

+       hdev->ibh_max_qp_wr = dev_attr->max_qp_wr;

 
        /* Setup device Memory Registration capabilities */
 #ifdef HAVE_IB_DEVICE_OPS
 
        /* Setup device Memory Registration capabilities */
 #ifdef HAVE_IB_DEVICE_OPS

diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index d5cf7e3..269306f 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -200,6 +200,7 @@ struct kib_hca_dev {
        int                  ibh_page_size;     /* page size of current HCA */
        __u64                ibh_page_mask;     /* page mask of current HCA */
        __u64                ibh_mr_size;       /* size of MR */
        int                  ibh_page_size;     /* page size of current HCA */
        __u64                ibh_page_mask;     /* page mask of current HCA */
        __u64                ibh_mr_size;       /* size of MR */

+       int                  ibh_max_qp_wr;     /* maximum work requests size */

 #ifdef HAVE_IB_GET_DMA_MR
        struct ib_mr        *ibh_mrs;           /* global MR */
 #endif
 #ifdef HAVE_IB_GET_DMA_MR
        struct ib_mr        *ibh_mrs;           /* global MR */
 #endif