LU-7955 gnilnd: Add ability to set bte_get/put_dlvr_mode

Currently the bte_dlvr_mode applies to all BTE transactions
initiated on the node. Based on testing we would like to be able to
set the mode for BTE_GET and BTE_PUT independently.

Changes
1. Add bte_get_dlvr_mode
2. Add bte_put_dlvr_mode
3. Remove old bte_dlvr_mode parameter
4. Set default for both new modes to be performance.

Signed-off-by: Chris Horn <hornc@cray.com>
Change-Id: Ifa7388b2a5b3d2f17b96f68890d3d98f082e3de6
Reviewed-on: http://review.whamcloud.com/19202
Reviewed-by: James Simmons <uja.ornl@yahoo.com>
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Chuck Fossen <chuckf@cray.com>
Reviewed-by: James Shimek <jshimek@cray.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lnet/klnds/gnilnd/gnilnd.h b/lnet/klnds/gnilnd/gnilnd.h
index 4e491a7..01d790c 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd.h
+++ b/lnet/klnds/gnilnd/gnilnd.h
@@ -459,7 +459,8 @@ typedef struct kgn_tunables {
        int              *kgn_max_immediate;    /* immediate payload breakpoint */
        int              *kgn_checksum;         /* checksum data */
        int              *kgn_checksum_dump;    /* dump raw data to D_INFO log when checksumming */
-       int              *kgn_bte_dlvr_mode;    /* BTE delivery mode mask */
+       int              *kgn_bte_put_dlvr_mode; /* BTE Put delivery mode */
+       int              *kgn_bte_get_dlvr_mode; /* BTE Get delivery mode */
        int              *kgn_bte_relaxed_ordering; /* relaxed ordering (PASSPW) on BTE transfers */
        int              *kgn_ptag;             /* PTAG for cdm_create */
        int              *kgn_pkey;             /* PKEY for cdm_create */

diff --git a/lnet/klnds/gnilnd/gnilnd_cb.c b/lnet/klnds/gnilnd/gnilnd_cb.c
index 7376443..bce8194 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd_cb.c
+++ b/lnet/klnds/gnilnd/gnilnd_cb.c
@@ -1970,8 +1970,10 @@ kgnilnd_rdma(kgn_tx_t *tx, int type,
        tx->tx_rdma_desc.remote_mem_hndl = sink->gnrd_key;
        tx->tx_rdma_desc.length = desc_nob;
        tx->tx_nob_rdma = nob;
-       if (*kgnilnd_tunables.kgn_bte_dlvr_mode)
-               tx->tx_rdma_desc.dlvr_mode = *kgnilnd_tunables.kgn_bte_dlvr_mode;
+       if (post_type == GNI_POST_RDMA_PUT && *kgnilnd_tunables.kgn_bte_put_dlvr_mode)
+               tx->tx_rdma_desc.dlvr_mode = *kgnilnd_tunables.kgn_bte_put_dlvr_mode;
+       if (post_type == GNI_POST_RDMA_GET && *kgnilnd_tunables.kgn_bte_get_dlvr_mode)
+               tx->tx_rdma_desc.dlvr_mode = *kgnilnd_tunables.kgn_bte_get_dlvr_mode;
        /* prep final completion message */
        kgnilnd_init_msg(&tx->tx_msg, type, tx->tx_msg.gnm_srcnid);
        tx->tx_msg.gnm_u.completion.gncm_cookie = cookie;

diff --git a/lnet/klnds/gnilnd/gnilnd_modparams.c b/lnet/klnds/gnilnd/gnilnd_modparams.c
index 942e5cb..5515010 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd_modparams.c
+++ b/lnet/klnds/gnilnd/gnilnd_modparams.c
@@ -78,9 +78,13 @@ static int checksum_dump = 0;
 module_param(checksum_dump, int, 0644);
 MODULE_PARM_DESC(checksum_dump, "0: None, 1: dump log on failure, 2: payload data to D_INFO log");
 
-static int bte_dlvr_mode = GNILND_RDMA_DLVR_OPTION;
-module_param(bte_dlvr_mode, int, 0644);
-MODULE_PARM_DESC(bte_dlvr_mode, "enable hashing for BTE (RDMA) transfers");
+static int bte_put_dlvr_mode = GNILND_RDMA_DLVR_OPTION;
+module_param(bte_put_dlvr_mode, int, 0644);
+MODULE_PARM_DESC(bte_put_dlvr_mode, "Modify BTE Put Routing Option");
+
+static int bte_get_dlvr_mode = GNILND_RDMA_DLVR_OPTION;
+module_param(bte_get_dlvr_mode, int, 0644);
+MODULE_PARM_DESC(bte_get_dlvr_mode, "Modify BTE Get Routing Option");
 
 static int bte_relaxed_ordering = 1;
 module_param(bte_relaxed_ordering, int, 0644);
@@ -213,7 +217,8 @@ kgn_tunables_t kgnilnd_tunables = {
        .kgn_max_immediate          = &max_immediate,
        .kgn_checksum               = &checksum,
        .kgn_checksum_dump          = &checksum_dump,
-       .kgn_bte_dlvr_mode          = &bte_dlvr_mode,
+       .kgn_bte_put_dlvr_mode      = &bte_put_dlvr_mode,
+       .kgn_bte_get_dlvr_mode      = &bte_get_dlvr_mode,
        .kgn_bte_relaxed_ordering   = &bte_relaxed_ordering,
        .kgn_ptag                   = &ptag,
        .kgn_pkey                   = &pkey,