DVS on gni breaks the LNet 1M handshake of LNET_MAX_IOV.
Introduce GNILND_MAX_IOV with a 4M i/o maximum and a hint
LNET_MD_GNILND so LNet can accept the large buffer w/o complaint.
Test-Parameters: trivial
Signed-off-by: Shaun Tancheff <shaun.tancheff@hpe.com>
Change-Id: I4e78c0022fdece0d6945bbcc47e2e64d4d181dca
Reviewed-on: https://review.whamcloud.com/41373
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Chris Horn <chris.horn@hpe.com>
Reviewed-by: Alexey Lyashkov <alexey.lyashkov@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
* regardless of the value of the lnet_response_tracking param.
* - LNET_MD_NO_TRACK_RESPONSE: Disable response tracking on this MD
* regardless of the value of the lnet_response_tracking param.
+ * - LNET_MD_GNILND: Disable warning about exceeding LNET_MAX_IOV.
*
* Note:
* - LNET_MD_KIOV allows for a scatter/gather capability for memory
#define LNET_MD_TRACK_RESPONSE (1 << 10)
/** See struct lnet_md::options. */
#define LNET_MD_NO_TRACK_RESPONSE (1 << 11)
+/** See struct lnet_md::options. */
+#define LNET_MD_GNILND (1 << 12)
/** Infinite threshold on MD operations. See struct lnet_md::threshold */
#define LNET_MD_THRESH_INF (-1)
kgnilnd_data.kgn_tx_phys_cache =
kmem_cache_create("kgn_tx_phys",
- LNET_MAX_IOV * sizeof(gni_mem_segment_t),
+ GNILND_MAX_IOV * sizeof(gni_mem_segment_t),
0, 0, NULL);
if (kgnilnd_data.kgn_tx_phys_cache == NULL) {
CERROR("Can't create slab for kgn_tx_phys\n");
kgnilnd_data.kgn_cksum_npages * sizeof (struct page *));
for (i = 0; i < kgnilnd_data.kgn_cksum_npages; i++) {
- kgnilnd_data.kgn_cksum_map_pages[i] = kmalloc(LNET_MAX_IOV * sizeof (struct page *),
+ kgnilnd_data.kgn_cksum_map_pages[i] = kmalloc(GNILND_MAX_IOV * sizeof (struct page *),
GFP_KERNEL);
if (kgnilnd_data.kgn_cksum_map_pages[i] == NULL) {
CERROR("Can't allocate vmap cksum pages for cpu %d\n", i);
/* need sane upper bound to limit copy overhead */
#define GNILND_MAX_IMMEDIATE (64<<10)
+/* allow for 4M transfers over gni. Note 2.5M used by DVS */
+#define GNILND_MAX_IOV 1024
/* Max number of connections to keep in purgatory per peer */
#define GNILND_PURGATORY_MAX 5
if (tx->tx_phys != NULL) {
kmem_cache_free(kgnilnd_data.kgn_tx_phys_cache, tx->tx_phys);
CDEBUG(D_MALLOC, "slab-freed 'tx_phys': %lu at %p.\n",
- LNET_MAX_IOV * sizeof(gni_mem_segment_t), tx->tx_phys);
+ GNILND_MAX_IOV * sizeof(gni_mem_segment_t), tx->tx_phys);
}
/* Only free the buffer if we used it */
}
CDEBUG(D_MALLOC, "slab-alloced 'tx->tx_phys': %lu at %p.\n",
- LNET_MAX_IOV * sizeof(gni_mem_segment_t), tx->tx_phys);
+ GNILND_MAX_IOV * sizeof(gni_mem_segment_t), tx->tx_phys);
/* if loops changes, please change kgnilnd_cksum_kiov
* and kgnilnd_setup_immediate_buffer */
GOTO(error, rc);
}
- if ((phys - tx->tx_phys) == LNET_MAX_IOV) {
+ if ((phys - tx->tx_phys) == GNILND_MAX_IOV) {
CERROR ("payload too big (%d)\n", (int)(phys - tx->tx_phys));
rc = -EMSGSIZE;
GOTO(error, rc);
LASSERTF(nob == 0 || niov > 0,
"lntmsg %p nob %d niov %d\n", lntmsg, nob, niov);
- LASSERTF(niov <= LNET_MAX_IOV,
+ LASSERTF(niov <= GNILND_MAX_IOV,
"lntmsg %p niov %d\n", lntmsg, niov);
if (msg_vmflush)
__u16 cksum, cksum2;
__u64 mbytes;
- CFS_ALLOC_PTR_ARRAY(src, LNET_MAX_IOV);
- CFS_ALLOC_PTR_ARRAY(dest, LNET_MAX_IOV);
+ CFS_ALLOC_PTR_ARRAY(src, GNILND_MAX_IOV);
+ CFS_ALLOC_PTR_ARRAY(dest, GNILND_MAX_IOV);
if (src == NULL || dest == NULL) {
CERROR("couldn't allocate iovs\n");
GOTO(unwind, rc = -ENOMEM);
}
- for (i = 0; i < LNET_MAX_IOV; i++) {
+ for (i = 0; i < GNILND_MAX_IOV; i++) {
src[i].bv_offset = 0;
src[i].bv_len = PAGE_SIZE;
src[i].bv_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
/* add extra 2 pages - one for offset of src, 2nd to allow dest offset */
niov = (nob / PAGE_SIZE) + 2;
- if (niov > LNET_MAX_IOV) {
+ if (niov > GNILND_MAX_IOV) {
CERROR("bytes %d too large, requires niov %d > %d\n",
- nob, niov, LNET_MAX_IOV);
+ nob, niov, GNILND_MAX_IOV);
GOTO(unwind, rc = -E2BIG);
}
}
if (src != NULL)
- CFS_FREE_PTR_ARRAY(src, LNET_MAX_IOV);
+ CFS_FREE_PTR_ARRAY(src, GNILND_MAX_IOV);
if (dest != NULL)
- CFS_FREE_PTR_ARRAY(dest, LNET_MAX_IOV);
+ CFS_FREE_PTR_ARRAY(dest, GNILND_MAX_IOV);
return rc;
}
pa += plen;
i += 1;
}
+ WARN(!(lmd->md_options & LNET_MD_GNILND) && i > LNET_MAX_IOV,
+ "Max IOV exceeded: %d should be < %d\n",
+ i, LNET_MAX_IOV);
if ((umd->options & LNET_MD_MAX_SIZE) && /* max size used */
(umd->max_size < 0 ||
umd->max_size > (int)umd->length)) { /* illegal max_size */