Branch HEAD

Cleanup of o2iblnd protocol V2:
. check whether ofed can support iWARP
. enable map-on-demand for (nfrags >= kib_map_on_demand)
  instead of (nfrags > kib_map_on_demand)

b=19327
i=isaac

diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4
index f73004e..301a3a3 100644 (file)
--- a/lnet/autoconf/lustre-lnet.m4
+++ b/lnet/autoconf/lustre-lnet.m4
@@ -575,6 +575,24 @@ else
                        AC_MSG_RESULT(no)
                ])
 
                        AC_MSG_RESULT(no)
                ])
 

+               LB_LINUX_TRY_COMPILE([
+                       #include <linux/version.h>
+                       #include <linux/pci.h>
+                       #if !HAVE_GFP_T
+                       typedef int gfp_t;
+                       #endif
+                       #include <rdma/ib_verbs.h>
+               ],[
+                       return RDMA_TRANSPORT_IWARP ==
+                              rdma_node_get_transport(RDMA_NODE_RNIC);
+               ],[
+                       AC_MSG_RESULT(yes)
+                       AC_DEFINE(HAVE_OFED_TRANSPORT_IWARP, 1,
+                                 [has transport iWARP])
+               ],[
+                       AC_MSG_RESULT(no)
+               ])
+

                EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save"
        fi
 fi
                EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save"
        fi
 fi

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 9549b50..9b4dec8 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1439,7 +1439,7 @@ kiblnd_find_rd_dma_mr(kib_net_t *net, kib_rdma_desc_t *rd)
         LASSERT (net->ibn_dev->ibd_mrs[0] != NULL);
 
         if (*kiblnd_tunables.kib_map_on_demand > 0 &&
         LASSERT (net->ibn_dev->ibd_mrs[0] != NULL);
 
         if (*kiblnd_tunables.kib_map_on_demand > 0 &&

-            *kiblnd_tunables.kib_map_on_demand < rd->rd_nfrags)
+            *kiblnd_tunables.kib_map_on_demand <= rd->rd_nfrags)

                 return NULL;
 
         if (net->ibn_dev->ibd_nmrs == 1)
                 return NULL;
 
         if (net->ibn_dev->ibd_nmrs == 1)


@@ -1678,7 +1678,7 @@ kiblnd_dev_get_attr(kib_dev_t *ibdev)
                 return rc;
         }
 
                 return rc;
         }
 

-#if 1
+#ifdef HAVE_OFED_TRANSPORT_IWARP

         /* XXX We can't trust this value returned by Chelsio driver, it's wrong
          * and we have reported the bug, remove these in the future when Chelsio
          * bug got fixed. */
         /* XXX We can't trust this value returned by Chelsio driver, it's wrong
          * and we have reported the bug, remove these in the future when Chelsio
          * bug got fixed. */

diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index 4e270df..c48306b 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -133,7 +133,7 @@ extern kib_tunables_t  kiblnd_tunables;
 #define IBLND_CREDIT_HIGHWATER_V1    7          /* V1 only : when eagerly to return credits */
 
 #define IBLND_CREDITS_DEFAULT        8          /* default # of peer credits */
 #define IBLND_CREDIT_HIGHWATER_V1    7          /* V1 only : when eagerly to return credits */
 
 #define IBLND_CREDITS_DEFAULT        8          /* default # of peer credits */

-#define IBLND_CREDITS_MAX            4096       /* Max # of peer credits */
+#define IBLND_CREDITS_MAX          ((typeof(((kib_msg_t*) 0)->ibm_credits)) - 1)  /* Max # of peer credits */

 
 #define IBLND_MSG_QUEUE_SIZE(v)    ((v) == IBLND_MSG_VERSION_1 ? \
                                      IBLND_MSG_QUEUE_SIZE_V1 :   \
 
 #define IBLND_MSG_QUEUE_SIZE(v)    ((v) == IBLND_MSG_VERSION_1 ? \
                                      IBLND_MSG_QUEUE_SIZE_V1 :   \

diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index 61cc0c8..6995761 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -436,6 +436,9 @@ kiblnd_tunables_init (void)
         if (*kiblnd_tunables.kib_peertxcredits > IBLND_CREDITS_MAX)
                 *kiblnd_tunables.kib_peertxcredits = IBLND_CREDITS_MAX;
 
         if (*kiblnd_tunables.kib_peertxcredits > IBLND_CREDITS_MAX)
                 *kiblnd_tunables.kib_peertxcredits = IBLND_CREDITS_MAX;
 

+        if (*kiblnd_tunables.kib_peertxcredits > *kiblnd_tunables.kib_credits)
+                *kiblnd_tunables.kib_peertxcredits = *kiblnd_tunables.kib_credits;
+

         if (*kiblnd_tunables.kib_peercredits_hiw < *kiblnd_tunables.kib_peertxcredits / 2)
                 *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peertxcredits / 2;
 
         if (*kiblnd_tunables.kib_peercredits_hiw < *kiblnd_tunables.kib_peertxcredits / 2)
                 *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peertxcredits / 2;
 


@@ -443,9 +446,12 @@ kiblnd_tunables_init (void)
                 *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peertxcredits - 1;
 
         if (*kiblnd_tunables.kib_map_on_demand < 0 ||
                 *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peertxcredits - 1;
 
         if (*kiblnd_tunables.kib_map_on_demand < 0 ||

-            *kiblnd_tunables.kib_map_on_demand >= IBLND_MAX_RDMA_FRAGS)
+            *kiblnd_tunables.kib_map_on_demand > IBLND_MAX_RDMA_FRAGS)

                 *kiblnd_tunables.kib_map_on_demand = 0; /* disable map-on-demand */
 
                 *kiblnd_tunables.kib_map_on_demand = 0; /* disable map-on-demand */
 

+        if (*kiblnd_tunables.kib_map_on_demand == 1)
+                *kiblnd_tunables.kib_map_on_demand = 2; /* don't make sense to create map if only one fragment */
+

         if (*kiblnd_tunables.kib_concurrent_sends == 0) {
                 if (*kiblnd_tunables.kib_map_on_demand > 0 &&
                     *kiblnd_tunables.kib_map_on_demand <= IBLND_MAX_RDMA_FRAGS / 8)
         if (*kiblnd_tunables.kib_concurrent_sends == 0) {
                 if (*kiblnd_tunables.kib_map_on_demand > 0 &&
                     *kiblnd_tunables.kib_map_on_demand <= IBLND_MAX_RDMA_FRAGS / 8)