From 2c8df2b09e1bf1c308e85346a42e3783775c1a37 Mon Sep 17 00:00:00 2001 From: liangzhen Date: Fri, 15 May 2009 10:37:53 +0000 Subject: [PATCH] Branch HEAD Cleanup of o2iblnd protocol V2: . check whether ofed can support iWARP . enable map-on-demand for (nfrags >= kib_map_on_demand) instead of (nfrags > kib_map_on_demand) b=19327 i=isaac --- lnet/autoconf/lustre-lnet.m4 | 18 ++++++++++++++++++ lnet/klnds/o2iblnd/o2iblnd.c | 4 ++-- lnet/klnds/o2iblnd/o2iblnd.h | 2 +- lnet/klnds/o2iblnd/o2iblnd_modparams.c | 8 +++++++- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 index f73004e..301a3a3 100644 --- a/lnet/autoconf/lustre-lnet.m4 +++ b/lnet/autoconf/lustre-lnet.m4 @@ -575,6 +575,24 @@ else AC_MSG_RESULT(no) ]) + LB_LINUX_TRY_COMPILE([ + #include + #include + #if !HAVE_GFP_T + typedef int gfp_t; + #endif + #include + ],[ + return RDMA_TRANSPORT_IWARP == + rdma_node_get_transport(RDMA_NODE_RNIC); + ],[ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_OFED_TRANSPORT_IWARP, 1, + [has transport iWARP]) + ],[ + AC_MSG_RESULT(no) + ]) + EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" fi fi diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index 9549b50..9b4dec8 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -1439,7 +1439,7 @@ kiblnd_find_rd_dma_mr(kib_net_t *net, kib_rdma_desc_t *rd) LASSERT (net->ibn_dev->ibd_mrs[0] != NULL); if (*kiblnd_tunables.kib_map_on_demand > 0 && - *kiblnd_tunables.kib_map_on_demand < rd->rd_nfrags) + *kiblnd_tunables.kib_map_on_demand <= rd->rd_nfrags) return NULL; if (net->ibn_dev->ibd_nmrs == 1) @@ -1678,7 +1678,7 @@ kiblnd_dev_get_attr(kib_dev_t *ibdev) return rc; } -#if 1 +#ifdef HAVE_OFED_TRANSPORT_IWARP /* XXX We can't trust this value returned by Chelsio driver, it's wrong * and we have reported the bug, remove these in the future when Chelsio * bug got fixed. */ diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index 4e270df..c48306b 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -133,7 +133,7 @@ extern kib_tunables_t kiblnd_tunables; #define IBLND_CREDIT_HIGHWATER_V1 7 /* V1 only : when eagerly to return credits */ #define IBLND_CREDITS_DEFAULT 8 /* default # of peer credits */ -#define IBLND_CREDITS_MAX 4096 /* Max # of peer credits */ +#define IBLND_CREDITS_MAX ((typeof(((kib_msg_t*) 0)->ibm_credits)) - 1) /* Max # of peer credits */ #define IBLND_MSG_QUEUE_SIZE(v) ((v) == IBLND_MSG_VERSION_1 ? \ IBLND_MSG_QUEUE_SIZE_V1 : \ diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c index 61cc0c8..6995761 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c +++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c @@ -436,6 +436,9 @@ kiblnd_tunables_init (void) if (*kiblnd_tunables.kib_peertxcredits > IBLND_CREDITS_MAX) *kiblnd_tunables.kib_peertxcredits = IBLND_CREDITS_MAX; + if (*kiblnd_tunables.kib_peertxcredits > *kiblnd_tunables.kib_credits) + *kiblnd_tunables.kib_peertxcredits = *kiblnd_tunables.kib_credits; + if (*kiblnd_tunables.kib_peercredits_hiw < *kiblnd_tunables.kib_peertxcredits / 2) *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peertxcredits / 2; @@ -443,9 +446,12 @@ kiblnd_tunables_init (void) *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peertxcredits - 1; if (*kiblnd_tunables.kib_map_on_demand < 0 || - *kiblnd_tunables.kib_map_on_demand >= IBLND_MAX_RDMA_FRAGS) + *kiblnd_tunables.kib_map_on_demand > IBLND_MAX_RDMA_FRAGS) *kiblnd_tunables.kib_map_on_demand = 0; /* disable map-on-demand */ + if (*kiblnd_tunables.kib_map_on_demand == 1) + *kiblnd_tunables.kib_map_on_demand = 2; /* don't make sense to create map if only one fragment */ + if (*kiblnd_tunables.kib_concurrent_sends == 0) { if (*kiblnd_tunables.kib_map_on_demand > 0 && *kiblnd_tunables.kib_map_on_demand <= IBLND_MAX_RDMA_FRAGS / 8) -- 1.8.3.1