Branch HEAD

Allow user to specify small value for o2iblnd::concurrent_sends

b=15983
i=isaac

diff --git a/lnet/ChangeLog b/lnet/ChangeLog
index e5bfa57..dd608fd 100644 (file)
--- a/lnet/ChangeLog
+++ b/lnet/ChangeLog
@@ -18,6 +18,17 @@ Description:
 Details    : 
 
 Severity   : major
 Details    : 
 
 Severity   : major

+Bugzilla   : 15983
+Description: workaround for OOM from o2iblnd
+Details    : OFED needs allocate big chunk of memory for QP while creating
+             connection for o2iblnd, OOM can happen if no such a contiguous
+             memory chunk.
+             QP size is decided by concurrent_sends and max_fragments of
+             o2iblnd, now we permit user to specify smaller value for
+             concurrent_sends of o2iblnd(i.e: concurrent_sends=7), which
+             will decrease memory block size required by creating QP.
+
+Severity   : major

 Bugzilla   : 15093
 Description: Support Zerocopy receive of Chelsio device
 Details    : Chelsio driver can support zerocopy for iov[1] if it's
 Bugzilla   : 15093
 Description: Support Zerocopy receive of Chelsio device
 Details    : Chelsio driver can support zerocopy for iov[1] if it's

diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index 10edb5d..91c7520 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -316,8 +316,14 @@ kiblnd_tunables_init (void)
 
         if (*kiblnd_tunables.kib_concurrent_sends > IBLND_RX_MSGS)
                 *kiblnd_tunables.kib_concurrent_sends = IBLND_RX_MSGS;
 
         if (*kiblnd_tunables.kib_concurrent_sends > IBLND_RX_MSGS)
                 *kiblnd_tunables.kib_concurrent_sends = IBLND_RX_MSGS;

-        if (*kiblnd_tunables.kib_concurrent_sends < IBLND_MSG_QUEUE_SIZE)
-                *kiblnd_tunables.kib_concurrent_sends = IBLND_MSG_QUEUE_SIZE;
+        if (*kiblnd_tunables.kib_concurrent_sends < IBLND_MSG_QUEUE_SIZE / 2)
+                *kiblnd_tunables.kib_concurrent_sends = IBLND_MSG_QUEUE_SIZE / 2;
+
+        if (*kiblnd_tunables.kib_concurrent_sends < IBLND_MSG_QUEUE_SIZE) {
+                CWARN("Concurrent sends %d is lower than message queue size: %d, "
+                      "performance may drop slightly.\n",
+                      *kiblnd_tunables.kib_concurrent_sends, IBLND_MSG_QUEUE_SIZE);
+        }

 
         return 0;
 }
 
         return 0;
 }