b=16186,i=isaac:

[fs/lustre-release.git] / lnet / klnds / o2iblnd / o2iblnd_modparams.c

diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index 45a8efb..27a6112 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -68,7 +68,11 @@ static int peer_credits_hiw = 0;
 CFS_MODULE_PARM(peer_credits_hiw, "i", int, 0444,
                 "when eagerly to return credits");
 
-static int peer_timeout = 0;
+static int peer_buffer_credits = 0;
+CFS_MODULE_PARM(peer_buffer_credits, "i", int, 0444,
+                "# per-peer router buffer credits");
+
+static int peer_timeout = 180;
 CFS_MODULE_PARM(peer_timeout, "i", int, 0444,
                 "Seconds without aliveness news to declare peer dead (<=0 to disable)");
 
@@ -102,7 +106,7 @@ CFS_MODULE_PARM(map_on_demand, "i", int, 0444,
 
 static int fmr_pool_size = 512;
 CFS_MODULE_PARM(fmr_pool_size, "i", int, 0444,
-                "size of the fmr pool (>= ntx)");
+                "size of the fmr pool (>= ntx / 4)");
 
 static int fmr_flush_trigger = 384;
 CFS_MODULE_PARM(fmr_flush_trigger, "i", int, 0444,
@@ -123,8 +127,9 @@ kib_tunables_t kiblnd_tunables = {
         .kib_keepalive              = &keepalive,
         .kib_ntx                    = &ntx,
         .kib_credits                = &credits,
-        .kib_peercredits            = &peer_credits,
+        .kib_peertxcredits          = &peer_credits,
         .kib_peercredits_hiw        = &peer_credits_hiw,
+        .kib_peerrtrcredits         = &peer_buffer_credits,
         .kib_peertimeout            = &peer_timeout,
         .kib_default_ipif           = &ipif_name,
         .kib_retry_count            = &retry_count,
@@ -150,8 +155,9 @@ enum {
         O2IBLND_TIMEOUT,
         O2IBLND_NTX,
         O2IBLND_CREDITS,
-        O2IBLND_PEER_CREDITS,
+        O2IBLND_PEER_TXCREDITS,
         O2IBLND_PEER_CREDITS_HIW,
+        O2IBLND_PEER_RTRCREDITS,
         O2IBLND_PEER_TIMEOUT,
         O2IBLND_IPIF_BASENAME,
         O2IBLND_RETRY_COUNT,
@@ -172,8 +178,9 @@ enum {
 #define O2IBLND_TIMEOUT          CTL_UNNUMBERED
 #define O2IBLND_NTX              CTL_UNNUMBERED
 #define O2IBLND_CREDITS          CTL_UNNUMBERED
-#define O2IBLND_PEER_CREDITS     CTL_UNNUMBERED
+#define O2IBLND_PEER_TXCREDITS   CTL_UNNUMBERED
 #define O2IBLND_PEER_CREDITS_HIW CTL_UNNUMBERED
+#define O2IBLND_PEER_RTRCREDITS  CTL_UNNUMBERED
 #define O2IBLND_PEER_TIMEOUT     CTL_UNNUMBERED
 #define O2IBLND_IPIF_BASENAME    CTL_UNNUMBERED
 #define O2IBLND_RETRY_COUNT      CTL_UNNUMBERED
@@ -231,7 +238,7 @@ static cfs_sysctl_table_t kiblnd_ctl_table[] = {
                 .proc_handler = &proc_dointvec
         },
         {
-                .ctl_name = O2IBLND_PEER_CREDITS,
+                .ctl_name = O2IBLND_PEER_TXCREDITS,
                 .procname = "peer_credits",
                 .data     = &peer_credits,
                 .maxlen   = sizeof(int),
@@ -247,6 +254,14 @@ static cfs_sysctl_table_t kiblnd_ctl_table[] = {
                 .proc_handler = &proc_dointvec
         },
         {
+                .ctl_name = O2IBLND_PEER_RTRCREDITS,
+                .procname = "peer_buffer_credits",
+                .data     = &peer_buffer_credits,
+                .maxlen   = sizeof(int),
+                .mode     = 0444,
+                .proc_handler = &proc_dointvec
+        },
+        {
                 .ctl_name = O2IBLND_PEER_TIMEOUT,
                 .procname = "peer_timeout",
                 .data     = &peer_timeout,
@@ -402,53 +417,52 @@ kiblnd_sysctl_fini (void)
 int
 kiblnd_tunables_init (void)
 {
-        if (*kiblnd_tunables.kib_credits > *kiblnd_tunables.kib_ntx) {
-                CERROR("Can't set credits(%d) > ntx(%d)\n",
-                       *kiblnd_tunables.kib_credits,
-                       *kiblnd_tunables.kib_ntx);
-                return -EINVAL;
-        }
-
         if (kiblnd_translate_mtu(*kiblnd_tunables.kib_ib_mtu) < 0) {
                 CERROR("Invalid ib_mtu %d, expected 256/512/1024/2048/4096\n",
                        *kiblnd_tunables.kib_ib_mtu);
                 return -EINVAL;
         }
 
-        if (*kiblnd_tunables.kib_peercredits < IBLND_CREDITS_DEFAULT)
-                *kiblnd_tunables.kib_peercredits = IBLND_CREDITS_DEFAULT;
+        if (*kiblnd_tunables.kib_peertxcredits < IBLND_CREDITS_DEFAULT)
+                *kiblnd_tunables.kib_peertxcredits = IBLND_CREDITS_DEFAULT;
 
-        if (*kiblnd_tunables.kib_peercredits > IBLND_CREDITS_MAX)
-                *kiblnd_tunables.kib_peercredits = IBLND_CREDITS_MAX;
+        if (*kiblnd_tunables.kib_peertxcredits > IBLND_CREDITS_MAX)
+                *kiblnd_tunables.kib_peertxcredits = IBLND_CREDITS_MAX;
 
-        if (*kiblnd_tunables.kib_peercredits_hiw < *kiblnd_tunables.kib_peercredits / 2)
-                *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peercredits / 2;
+        if (*kiblnd_tunables.kib_peertxcredits > *kiblnd_tunables.kib_credits)
+                *kiblnd_tunables.kib_peertxcredits = *kiblnd_tunables.kib_credits;
 
-        if (*kiblnd_tunables.kib_peercredits_hiw >= *kiblnd_tunables.kib_peercredits)
-                *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peercredits - 1;
+        if (*kiblnd_tunables.kib_peercredits_hiw < *kiblnd_tunables.kib_peertxcredits / 2)
+                *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peertxcredits / 2;
+
+        if (*kiblnd_tunables.kib_peercredits_hiw >= *kiblnd_tunables.kib_peertxcredits)
+                *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peertxcredits - 1;
 
         if (*kiblnd_tunables.kib_map_on_demand < 0 ||
-            *kiblnd_tunables.kib_map_on_demand >= IBLND_MAX_RDMA_FRAGS)
+            *kiblnd_tunables.kib_map_on_demand > IBLND_MAX_RDMA_FRAGS)
                 *kiblnd_tunables.kib_map_on_demand = 0; /* disable map-on-demand */
 
+        if (*kiblnd_tunables.kib_map_on_demand == 1)
+                *kiblnd_tunables.kib_map_on_demand = 2; /* don't make sense to create map if only one fragment */
+
         if (*kiblnd_tunables.kib_concurrent_sends == 0) {
                 if (*kiblnd_tunables.kib_map_on_demand > 0 &&
                     *kiblnd_tunables.kib_map_on_demand <= IBLND_MAX_RDMA_FRAGS / 8)
-                        *kiblnd_tunables.kib_concurrent_sends = (*kiblnd_tunables.kib_peercredits) * 2;
+                        *kiblnd_tunables.kib_concurrent_sends = (*kiblnd_tunables.kib_peertxcredits) * 2;
                 else
-                        *kiblnd_tunables.kib_concurrent_sends = (*kiblnd_tunables.kib_peercredits);
+                        *kiblnd_tunables.kib_concurrent_sends = (*kiblnd_tunables.kib_peertxcredits);
         }
 
-        if (*kiblnd_tunables.kib_concurrent_sends > *kiblnd_tunables.kib_peercredits * 2)
-                *kiblnd_tunables.kib_concurrent_sends = *kiblnd_tunables.kib_peercredits * 2;
+        if (*kiblnd_tunables.kib_concurrent_sends > *kiblnd_tunables.kib_peertxcredits * 2)
+                *kiblnd_tunables.kib_concurrent_sends = *kiblnd_tunables.kib_peertxcredits * 2;
 
-        if (*kiblnd_tunables.kib_concurrent_sends < *kiblnd_tunables.kib_peercredits / 2)
-                *kiblnd_tunables.kib_concurrent_sends = *kiblnd_tunables.kib_peercredits / 2;
+        if (*kiblnd_tunables.kib_concurrent_sends < *kiblnd_tunables.kib_peertxcredits / 2)
+                *kiblnd_tunables.kib_concurrent_sends = *kiblnd_tunables.kib_peertxcredits / 2;
 
-        if (*kiblnd_tunables.kib_concurrent_sends < *kiblnd_tunables.kib_peercredits) {
+        if (*kiblnd_tunables.kib_concurrent_sends < *kiblnd_tunables.kib_peertxcredits) {
                 CWARN("Concurrent sends %d is lower than message queue size: %d, "
                       "performance may drop slightly.\n",
-                      *kiblnd_tunables.kib_concurrent_sends, *kiblnd_tunables.kib_peercredits);
+                      *kiblnd_tunables.kib_concurrent_sends, *kiblnd_tunables.kib_peertxcredits);
         }
 
         kiblnd_sysctl_init();