From: Chris Horn Date: Mon, 29 Jun 2015 18:31:16 +0000 (-0500) Subject: LU-6261 gnilnd: Changes for small message rate improvment X-Git-Tag: 2.7.57~18 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=e5d5d8d09a4be22ab225bac0db3f4abc0e24b296 LU-6261 gnilnd: Changes for small message rate improvment Change number of threads on Aries service nodes to 7 for better performance of small messages. Change the max_immediate value to 8k instead of 2k. FMA has better performance up to 8k. lnet passes an niov of 256 reflecting the size of the router buffer number of pages in the buffer pool when receiving data from the ib interface when the message size is over one page. Adjust the niov to reflect the message size/offset for sends. Use the cookie defined in gni_pub.h instead of value from gnilnd.h. Signed-off-by: Chris Horn Change-Id: I84270653efcff56f06da7de7fb9674ae319800dd Reviewed-on: http://review.whamcloud.com/15431 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Doug Oucharek Reviewed-by: James Shimek Reviewed-by: Oleg Drokin --- diff --git a/lnet/klnds/gnilnd/gnilnd.c b/lnet/klnds/gnilnd/gnilnd.c index 4961d44..c911944 100644 --- a/lnet/klnds/gnilnd/gnilnd.c +++ b/lnet/klnds/gnilnd/gnilnd.c @@ -1955,7 +1955,7 @@ kgnilnd_dev_init(kgn_device_t *dev) cq_size = *kgnilnd_tunables.kgn_credits * 2 * 3; rrc = kgnilnd_cdm_create(dev->gnd_id, *kgnilnd_tunables.kgn_ptag, - GNILND_COOKIE, 0, + GNI_JOB_CREATE_COOKIE(GNI_PKEY_LND, 0), 0, &dev->gnd_domain); if (rrc != GNI_RC_SUCCESS) { CERROR("Can't create CDM %d (%d)\n", dev->gnd_id, rrc); diff --git a/lnet/klnds/gnilnd/gnilnd.h b/lnet/klnds/gnilnd/gnilnd.h index d82e971..cb9c20b 100644 --- a/lnet/klnds/gnilnd/gnilnd.h +++ b/lnet/klnds/gnilnd/gnilnd.h @@ -99,7 +99,6 @@ /* fixed constants */ #define GNILND_MAXDEVS 1 /* max # of GNI devices currently supported */ #define GNILND_MBOX_CREDITS 256 /* number of credits per mailbox */ -#define GNILND_COOKIE 0xa3579 /* cookie used by along with ptag by GNI */ #define GNILND_CONN_MAGIC 0xa100f /* magic value for verifying connection validity */ /* checksum values */ #define GNILND_CHECKSUM_OFF 0 /* checksum turned off */ @@ -120,7 +119,6 @@ #define GNILND_COMPUTE 1 /* compute image */ #define GNILND_FAST_RECONNECT 1 /* Fast Reconnect option */ #else -#define GNILND_SCHED_THREADS 3 /* default # of kgnilnd_scheduler threads */ #define GNILND_FMABLK 1024 /* default number of mboxes per fmablk */ #define GNILND_SCHED_NICE -20 /* default nice value for scheduler threads */ #define GNILND_COMPUTE 0 /* service image */ diff --git a/lnet/klnds/gnilnd/gnilnd_aries.h b/lnet/klnds/gnilnd/gnilnd_aries.h index e017fe3..ce187ed 100644 --- a/lnet/klnds/gnilnd/gnilnd_aries.h +++ b/lnet/klnds/gnilnd/gnilnd_aries.h @@ -51,6 +51,7 @@ #else #define GNILND_REVERSE_RDMA GNILND_REVERSE_GET #define GNILND_RDMA_DLVR_OPTION GNI_DLVMODE_PERFORMANCE +#define GNILND_SCHED_THREADS 7 /* scheduler threads */ #endif /* Thread-safe kgni implemented in minor ver 45, code rev 0xb9 */ diff --git a/lnet/klnds/gnilnd/gnilnd_cb.c b/lnet/klnds/gnilnd/gnilnd_cb.c index 92aaf57..7175f0d 100644 --- a/lnet/klnds/gnilnd/gnilnd_cb.c +++ b/lnet/klnds/gnilnd/gnilnd_cb.c @@ -505,8 +505,14 @@ kgnilnd_setup_immediate_buffer(kgn_tx_t *tx, unsigned int niov, struct iovec *io if (nob == 0) { tx->tx_buffer = NULL; } else if (kiov != NULL) { + + if ((niov > 0) && unlikely(niov > (nob/PAGE_SIZE))) { + niov = ((nob + offset + PAGE_SIZE - 1) / PAGE_SIZE); + } + LASSERTF(niov > 0 && niov < GNILND_MAX_IMMEDIATE/PAGE_SIZE, - "bad niov %d\n", niov); + "bad niov %d msg %p kiov %p iov %p offset %d nob%d\n", + niov, msg, kiov, iov, offset, nob); while (offset >= kiov->kiov_len) { offset -= kiov->kiov_len; diff --git a/lnet/klnds/gnilnd/gnilnd_gemini.h b/lnet/klnds/gnilnd/gnilnd_gemini.h index 79be5b3..1e7e2f6 100644 --- a/lnet/klnds/gnilnd/gnilnd_gemini.h +++ b/lnet/klnds/gnilnd/gnilnd_gemini.h @@ -34,6 +34,10 @@ #define GNILND_REVERSE_RDMA GNILND_REVERSE_NONE #define GNILND_RDMA_DLVR_OPTION GNI_DLVMODE_PERFORMANCE +#if !defined(CONFIG_CRAY_COMPUTE) +#define GNILND_SCHED_THREADS 3 /* scheduler threads */ +#endif + /* Thread-safe kgni implemented in minor ver 44, code rev 0xb9 */ #define GNILND_KGNI_TS_MINOR_VER 0x44 diff --git a/lnet/klnds/gnilnd/gnilnd_modparams.c b/lnet/klnds/gnilnd/gnilnd_modparams.c index 8bef922..7d1c39f 100644 --- a/lnet/klnds/gnilnd/gnilnd_modparams.c +++ b/lnet/klnds/gnilnd/gnilnd_modparams.c @@ -66,7 +66,7 @@ static int max_reconnect_interval = GNILND_MAX_RECONNECT_TO; CFS_MODULE_PARM(max_reconnect_interval, "i", int, 0644, "maximum connection retry interval (seconds)"); -static int max_immediate = (2<<10); +static int max_immediate = 8192; CFS_MODULE_PARM(max_immediate, "i", int, 0644, "immediate/RDMA breakpoint");