From 6b8aab2fa2f39ac289aff8c6d0228bbd666ee50d Mon Sep 17 00:00:00 2001 From: adilger Date: Wed, 6 Apr 2005 23:43:55 +0000 Subject: [PATCH] Branch: b1_4 - update gmnal to use PTL_MTU, fix module refcounting, for HP b=5786 r=mdoyle (original patch) --- lnet/ChangeLog | 4 ++ lnet/klnds/gmlnd/gmlnd.h | 10 +-- lnet/klnds/gmlnd/gmlnd_api.c | 29 ++++----- lnet/klnds/gmlnd/gmlnd_cb.c | 138 ++++++++++++++++++++++------------------ lnet/klnds/gmlnd/gmlnd_comm.c | 44 ++++++------- lnet/klnds/gmlnd/gmlnd_module.c | 4 +- lnet/klnds/gmlnd/gmlnd_utils.c | 5 +- 7 files changed, 124 insertions(+), 110 deletions(-) diff --git a/lnet/ChangeLog b/lnet/ChangeLog index baa3c76..27ad0b2 100644 --- a/lnet/ChangeLog +++ b/lnet/ChangeLog @@ -1,3 +1,7 @@ +2005-04-06 Cluster File Systems, Inc. + * bugs + - update gmnal to use PTL_MTU, fix module refcounting (b=5786) + 2005-04-04 Cluster File Systems, Inc. * bugs - handle error return code in kranal_check_fma_rx() (5915,6054) diff --git a/lnet/klnds/gmlnd/gmlnd.h b/lnet/klnds/gmlnd/gmlnd.h index 604d367..bffbb0b 100644 --- a/lnet/klnds/gmlnd/gmlnd.h +++ b/lnet/klnds/gmlnd/gmlnd.h @@ -203,7 +203,7 @@ typedef struct _gmnal_rxtwe { typedef struct _gmnal_data_t { int refcnt; spinlock_t cb_lock; - spinlock_t stxd_lock; + spinlock_t stxd_lock; struct semaphore stxd_token; gmnal_stxd_t *stxd; spinlock_t rxt_stxd_lock; @@ -429,11 +429,11 @@ void gmnal_remove_rxtwe(gmnal_data_t *); /* * Small messages */ -ptl_err_t gmnal_small_rx(lib_nal_t *, void *, lib_msg_t *); -ptl_err_t gmnal_small_tx(lib_nal_t *, void *, lib_msg_t *, ptl_hdr_t *, - int, ptl_nid_t, ptl_pid_t, +ptl_err_t gmnal_small_rx(lib_nal_t *, void *, lib_msg_t *); +ptl_err_t gmnal_small_tx(lib_nal_t *, void *, lib_msg_t *, ptl_hdr_t *, + int, ptl_nid_t, ptl_pid_t, gmnal_stxd_t*, int); -void gmnal_small_tx_callback(gm_port_t *, void *, gm_status_t); +void gmnal_small_tx_callback(gm_port_t *, void *, gm_status_t); diff --git a/lnet/klnds/gmlnd/gmlnd_api.c b/lnet/klnds/gmlnd/gmlnd_api.c index bd09d78..a65272a 100644 --- a/lnet/klnds/gmlnd/gmlnd_api.c +++ b/lnet/klnds/gmlnd/gmlnd_api.c @@ -65,9 +65,11 @@ gmnal_api_shutdown(nal_t *nal) gmnal_data_t *nal_data; lib_nal_t *libnal; - if (nal->nal_refct != 0) + if (nal->nal_refct != 0) { + /* This module got the first ref */ + PORTAL_MODULE_UNUSE; return; - + } LASSERT(nal == global_nal_data->nal); libnal = (lib_nal_t *)nal->nal_data; @@ -95,11 +97,10 @@ gmnal_api_shutdown(nal_t *nal) if (nal_data->sysctl) unregister_sysctl_table (nal_data->sysctl); /* Don't free 'nal'; it's a static struct */ - PORTAL_FREE(nal_data, sizeof(gmnal_data_t)); + PORTAL_FREE(nal_data, sizeof(gmnal_data_t)); PORTAL_FREE(libnal, sizeof(lib_nal_t)); global_nal_data = NULL; - PORTAL_MODULE_UNUSE; } @@ -121,6 +122,7 @@ gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid, libnal = (lib_nal_t *)nal->nal_data; *actual_limits = libnal->libnal_ni.ni_actual_limits; } + PORTAL_MODULE_USE; return (PTL_OK); } @@ -163,7 +165,7 @@ gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid, /* - * initialise the interface, + * initialise the interface, */ CDEBUG(D_INFO, "Calling gm_init\n"); if (gm_init() != GM_SUCCESS) { @@ -175,11 +177,11 @@ gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid, CDEBUG(D_NET, "Calling gm_open with port [%d], " - "name [%s], version [%d]\n", GMNAL_GM_PORT_ID, + "name [%s], version [%d]\n", GMNAL_GM_PORT_ID, "gmnal", GM_API_VERSION); GMNAL_GM_LOCK(nal_data); - gm_status = gm_open(&nal_data->gm_port, 0, GMNAL_GM_PORT_ID, "gmnal", + gm_status = gm_open(&nal_data->gm_port, 0, GMNAL_GM_PORT_ID, "gmnal", GM_API_VERSION); GMNAL_GM_UNLOCK(nal_data); @@ -218,9 +220,8 @@ gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid, return(PTL_FAIL); } - nal_data->small_msg_size = gmnal_small_msg_size; - nal_data->small_msg_gmsize = + nal_data->small_msg_gmsize = gm_min_size_for_length(gmnal_small_msg_size); if (gmnal_alloc_srxd(nal_data) != GMNAL_STATUS_OK) { @@ -373,13 +374,10 @@ gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid, nal_data->sysctl = NULL; nal_data->sysctl = register_sysctl_table (gmnalnal_top_sysctl_table, 0); - CDEBUG(D_INFO, "gmnal_init finished\n"); - global_nal_data = nal->nal_data; - /* no unload now until shutdown */ - PORTAL_MODULE_USE; - + global_nal_data = libnal->libnal_data; + return(PTL_OK); } @@ -408,7 +406,6 @@ int gmnal_init(void) return (rc); } - /* * Called when module removed @@ -417,8 +414,8 @@ void gmnal_fini() { CDEBUG(D_TRACE, "gmnal_fini\n"); - LASSERT(global_nal_data == NULL); PtlNIFini(kgmnal_ni); ptl_unregister_nal(GMNAL); + LASSERT(global_nal_data == NULL); } diff --git a/lnet/klnds/gmlnd/gmlnd_cb.c b/lnet/klnds/gmlnd/gmlnd_cb.c index e19995c..6394c37 100644 --- a/lnet/klnds/gmlnd/gmlnd_cb.c +++ b/lnet/klnds/gmlnd/gmlnd_cb.c @@ -27,16 +27,16 @@ #include "gmnal.h" -ptl_err_t gmnal_cb_recv(lib_nal_t *libnal, void *private, lib_msg_t *cookie, - unsigned int niov, struct iovec *iov, size_t offset, +ptl_err_t gmnal_cb_recv(lib_nal_t *libnal, void *private, lib_msg_t *cookie, + unsigned int niov, struct iovec *iov, size_t offset, size_t mlen, size_t rlen) { - void *buffer = NULL; + void *buffer = NULL; gmnal_srxd_t *srxd = (gmnal_srxd_t*)private; int status = PTL_OK; CDEBUG(D_TRACE, "gmnal_cb_recv libnal [%p], private[%p], cookie[%p], " - "niov[%d], iov [%p], offset["LPSZ"], mlen["LPSZ"], rlen["LPSZ"]\n", + "niov[%d], iov [%p], offset["LPSZ"], mlen["LPSZ"], rlen["LPSZ"]\n", libnal, private, cookie, niov, iov, offset, mlen, rlen); switch(srxd->type) { @@ -45,25 +45,25 @@ ptl_err_t gmnal_cb_recv(lib_nal_t *libnal, void *private, lib_msg_t *cookie, /* HP SFS 1380: Proactively change receives to avoid a receive * side occurrence of filling pkmap_count[]. */ - buffer = srxd->buffer; + buffer = srxd->buffer; buffer += sizeof(gmnal_msghdr_t); buffer += sizeof(ptl_hdr_t); - while(niov--) { + while(niov--) { if (offset >= iov->iov_len) { offset -= iov->iov_len; } else if (offset > 0) { - CDEBUG(D_INFO, "processing [%p] base [%p] len %d, " - "offset %d, len ["LPSZ"]\n", iov, - iov->iov_base + offset, iov->iov_len, offset, - iov->iov_len - offset); + CDEBUG(D_INFO, "processing [%p] base [%p] " + "len %d, offset %d, len ["LPSZ"]\n", iov, + iov->iov_base + offset, iov->iov_len, + offset, iov->iov_len - offset); gm_bcopy(buffer, iov->iov_base + offset, iov->iov_len - offset); buffer += iov->iov_len - offset; offset = 0; } else { - CDEBUG(D_INFO, "processing [%p] len ["LPSZ"]\n", iov, - iov->iov_len); + CDEBUG(D_INFO, "processing [%p] len ["LPSZ"]\n", + iov, iov->iov_len); gm_bcopy(buffer, iov->iov_base, iov->iov_len); buffer += iov->iov_len; } @@ -76,15 +76,15 @@ ptl_err_t gmnal_cb_recv(lib_nal_t *libnal, void *private, lib_msg_t *cookie, status = gmnal_large_rx(libnal, private, cookie, niov, iov, offset, mlen, rlen); } - CDEBUG(D_INFO, "gmnal_cb_recv gmnal_return status [%d]\n", status); return(status); } -ptl_err_t gmnal_cb_recv_pages(lib_nal_t *libnal, void *private, lib_msg_t *cookie, - unsigned int kniov, ptl_kiov_t *kiov, size_t offset, - size_t mlen, size_t rlen) +ptl_err_t gmnal_cb_recv_pages(lib_nal_t *libnal, void *private, + lib_msg_t *cookie, unsigned int kniov, + ptl_kiov_t *kiov, size_t offset, size_t mlen, + size_t rlen) { gmnal_srxd_t *srxd = (gmnal_srxd_t*)private; int status = PTL_OK; @@ -97,7 +97,7 @@ ptl_err_t gmnal_cb_recv_pages(lib_nal_t *libnal, void *private, lib_msg_t *cooki libnal, private, cookie, kniov, kiov, offset, mlen, rlen); if (srxd->type == GMNAL_SMALL_MESSAGE) { - buffer = srxd->buffer; + buffer = srxd->buffer; buffer += sizeof(gmnal_msghdr_t); buffer += sizeof(ptl_hdr_t); @@ -105,53 +105,56 @@ ptl_err_t gmnal_cb_recv_pages(lib_nal_t *libnal, void *private, lib_msg_t *cooki * map each page and create an iovec for it */ while (kniov--) { - /* HP SFS 1380: Proactively change receives to avoid a receive - * side occurrence of filling pkmap_count[]. + /* HP SFS 1380: Proactively change receives to avoid a + * receive side occurrence of filling pkmap_count[]. */ - CDEBUG(D_INFO, "processing kniov [%d] [%p]\n", kniov, kiov); + CDEBUG(D_INFO, "processing kniov [%d] [%p]\n", + kniov, kiov); if (offset >= kiov->kiov_len) { offset -= kiov->kiov_len; } else { - CDEBUG(D_INFO, "kniov page [%p] len [%d] offset[%d]\n", - kiov->kiov_page, kiov->kiov_len, - kiov->kiov_offset); + CDEBUG(D_INFO, "kniov page [%p] len [%d] " + "offset[%d]\n", kiov->kiov_page, + kiov->kiov_len, kiov->kiov_offset); CDEBUG(D_INFO, "Calling kmap[%p]", kiov->kiov_page); - ptr = ((char *)kmap(kiov->kiov_page)) + kiov->kiov_offset; + ptr = ((char *)kmap(kiov->kiov_page)) + + kiov->kiov_offset; if (offset > 0) { - CDEBUG(D_INFO, "processing [%p] base [%p] len %d, " - "offset %d, len ["LPSZ"]\n", ptr, - ptr + offset, kiov->kiov_len, offset, + CDEBUG(D_INFO, "processing [%p] base " + "[%p] len %d, offset %d, len [" + LPSZ"]\n", ptr, ptr + offset, + kiov->kiov_len, offset, kiov->kiov_len - offset); gm_bcopy(buffer, ptr + offset, - kiov->kiov_len - offset); + kiov->kiov_len - offset); buffer += kiov->kiov_len - offset; offset = 0; } else { - CDEBUG(D_INFO, "processing [%p] len ["LPSZ"]\n", ptr, - kiov->kiov_len); + CDEBUG(D_INFO, "processing [%p] len [" + LPSZ"]\n", ptr, kiov->kiov_len); gm_bcopy(buffer, ptr, kiov->kiov_len); buffer += kiov->kiov_len; } kunmap(kiov->kiov_page); CDEBUG(D_INFO, "Stored in [%p]\n", ptr); - } + } kiov++; } CDEBUG(D_INFO, "calling gmnal_small_rx\n"); status = gmnal_small_rx(libnal, private, cookie); } - CDEBUG(D_INFO, "gmnal_return status [%d]\n", status); return(status); } -ptl_err_t gmnal_cb_send(lib_nal_t *libnal, void *private, lib_msg_t *cookie, - ptl_hdr_t *hdr, int type, ptl_nid_t nid, ptl_pid_t pid, - unsigned int niov, struct iovec *iov, size_t offset, size_t len) +ptl_err_t gmnal_cb_send(lib_nal_t *libnal, void *private, lib_msg_t *cookie, + ptl_hdr_t *hdr, int type, ptl_nid_t nid, ptl_pid_t pid, + unsigned int niov, struct iovec *iov, size_t offset, + size_t len) { gmnal_data_t *nal_data; @@ -159,8 +162,8 @@ ptl_err_t gmnal_cb_send(lib_nal_t *libnal, void *private, lib_msg_t *cookie, gmnal_stxd_t *stxd = NULL; - CDEBUG(D_TRACE, "gmnal_cb_send niov[%d] offset["LPSZ"] len["LPSZ"] nid["LPU64"]\n", - niov, offset, len, nid); + CDEBUG(D_TRACE, "gmnal_cb_send niov[%d] offset["LPSZ"] len["LPSZ + "] nid["LPU64"]\n", niov, offset, len, nid); nal_data = libnal->libnal_data; if (!nal_data) { CDEBUG(D_ERROR, "no nal_data\n"); @@ -168,49 +171,53 @@ ptl_err_t gmnal_cb_send(lib_nal_t *libnal, void *private, lib_msg_t *cookie, } else { CDEBUG(D_INFO, "nal_data [%p]\n", nal_data); } - + if (GMNAL_IS_SMALL_MESSAGE(nal_data, niov, iov, len)) { CDEBUG(D_INFO, "This is a small message send\n"); /* - * HP SFS 1380: With the change to gmnal_small_tx, need to get the stxd - * and do relevant setup here + * HP SFS 1380: With the change to gmnal_small_tx, need to get + * the stxd and do relevant setup here */ stxd = gmnal_get_stxd(nal_data, 1); CDEBUG(D_INFO, "stxd [%p]\n", stxd); /* Set the offset of the data to copy into the buffer */ - buffer = stxd->buffer + sizeof(gmnal_msghdr_t) + sizeof(ptl_hdr_t); + buffer = stxd->buffer +sizeof(gmnal_msghdr_t)+sizeof(ptl_hdr_t); while(niov--) { if (offset >= iov->iov_len) { offset -= iov->iov_len; } else if (offset > 0) { - CDEBUG(D_INFO, "processing iov [%p] base [%p] len ["LPSZ"] to [%p]\n", - iov, iov->iov_base + offset, iov->iov_len - offset, buffer); - gm_bcopy(iov->iov_base + offset, buffer, iov->iov_len - offset); + CDEBUG(D_INFO, "processing iov [%p] base [%p] " + "len ["LPSZ"] to [%p]\n", + iov, iov->iov_base + offset, + iov->iov_len - offset, buffer); + gm_bcopy(iov->iov_base + offset, buffer, + iov->iov_len - offset); buffer+= iov->iov_len - offset; offset = 0; } else { - CDEBUG(D_INFO, "processing iov [%p] len ["LPSZ"] to [%p]\n", - iov, iov->iov_len, buffer); + CDEBUG(D_INFO, "processing iov [%p] len ["LPSZ + "] to [%p]\n", iov, iov->iov_len,buffer); gm_bcopy(iov->iov_base, buffer, iov->iov_len); buffer+= iov->iov_len; } iov++; } - gmnal_small_tx(libnal, private, cookie, hdr, type, nid, pid, + gmnal_small_tx(libnal, private, cookie, hdr, type, nid, pid, stxd, len); } else { CDEBUG(D_ERROR, "Large message send is not supported\n"); lib_finalize(libnal, private, cookie, PTL_FAIL); return(PTL_FAIL); - gmnal_large_tx(libnal, private, cookie, hdr, type, nid, pid, + gmnal_large_tx(libnal, private, cookie, hdr, type, nid, pid, niov, iov, offset, len); } return(PTL_OK); } -ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private, lib_msg_t *cookie, - ptl_hdr_t *hdr, int type, ptl_nid_t nid, ptl_pid_t pid, - unsigned int kniov, ptl_kiov_t *kiov, size_t offset, size_t len) +ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private, + lib_msg_t *cookie, ptl_hdr_t *hdr, int type, + ptl_nid_t nid, ptl_pid_t pid, unsigned int kniov, + ptl_kiov_t *kiov, size_t offset, size_t len) { gmnal_data_t *nal_data; @@ -219,8 +226,8 @@ ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private, lib_msg_t *cooki gmnal_stxd_t *stxd = NULL; ptl_err_t status = PTL_OK; - CDEBUG(D_TRACE, "gmnal_cb_send_pages nid ["LPU64"] niov[%d] offset["LPSZ"] len["LPSZ"]\n", - nid, kniov, offset, len); + CDEBUG(D_TRACE, "gmnal_cb_send_pages nid ["LPU64"] niov[%d] offset[" + LPSZ"] len["LPSZ"]\n", nid, kniov, offset, len); nal_data = libnal->libnal_data; if (!nal_data) { CDEBUG(D_ERROR, "no nal_data\n"); @@ -230,9 +237,9 @@ ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private, lib_msg_t *cooki } /* HP SFS 1380: Need to do the gm_bcopy after the kmap so we can kunmap - * more aggressively. This is the fix for a livelock situation under load - * on ia32 that occurs when there are no more available entries in the - * pkmap_count array. Just fill the buffer and let gmnal_small_tx + * more aggressively. This is the fix for a livelock situation under + * load on ia32 that occurs when there are no more available entries in + * the pkmap_count array. Just fill the buffer and let gmnal_small_tx * put the headers in after we pass it the stxd pointer. */ stxd = gmnal_get_stxd(nal_data, 1); @@ -242,7 +249,7 @@ ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private, lib_msg_t *cooki if (GMNAL_IS_SMALL_MESSAGE(nal_data, 0, NULL, len)) { CDEBUG(D_INFO, "This is a small message send\n"); - + while(kniov--) { CDEBUG(D_INFO, "processing kniov [%d] [%p]\n", kniov, kiov); if (offset >= kiov->kiov_len) { @@ -252,16 +259,21 @@ ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private, lib_msg_t *cooki kiov->kiov_page, kiov->kiov_len, kiov->kiov_offset); - ptr = ((char *)kmap(kiov->kiov_page)) + kiov->kiov_offset; + ptr = ((char *)kmap(kiov->kiov_page)) + + kiov->kiov_offset; if (offset > 0) { - CDEBUG(D_INFO, "processing [%p] base [%p] len ["LPSZ"] to [%p]\n", - ptr, ptr + offset, kiov->kiov_len - offset, buffer); - gm_bcopy(ptr + offset, buffer, kiov->kiov_len - offset); + CDEBUG(D_INFO, "processing [%p] base " + "[%p] len ["LPSZ"] to [%p]\n", + ptr, ptr + offset, + kiov->kiov_len - offset, buffer); + gm_bcopy(ptr + offset, buffer, + kiov->kiov_len - offset); buffer+= kiov->kiov_len - offset; offset = 0; } else { - CDEBUG(D_INFO, "processing kmapped [%p] len ["LPSZ"] to [%p]\n", + CDEBUG(D_INFO, "processing kmapped [%p]" + " len ["LPSZ"] to [%p]\n", ptr, kiov->kiov_len, buffer); gm_bcopy(ptr, buffer, kiov->kiov_len); @@ -271,7 +283,7 @@ ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private, lib_msg_t *cooki } kiov++; } - status = gmnal_small_tx(libnal, private, cookie, hdr, type, nid, + status = gmnal_small_tx(libnal, private, cookie, hdr, type, nid, pid, stxd, len); } else { int i = 0; diff --git a/lnet/klnds/gmlnd/gmlnd_comm.c b/lnet/klnds/gmlnd/gmlnd_comm.c index aea2967..206d86b 100644 --- a/lnet/klnds/gmlnd/gmlnd_comm.c +++ b/lnet/klnds/gmlnd/gmlnd_comm.c @@ -115,7 +115,7 @@ int gmnal_rx_thread(void *arg) gmnal_data_t *nal_data; void *buffer; gmnal_rxtwe_t *we = NULL; - int rank; + int rank; if (!arg) { CDEBUG(D_TRACE, "NO nal_data. Exiting\n"); @@ -125,7 +125,7 @@ int gmnal_rx_thread(void *arg) nal_data = (gmnal_data_t*)arg; CDEBUG(D_TRACE, "nal_data is [%p]\n", arg); - for (rank=0; rankrxthread_pid[rank] == current->pid) break; @@ -342,7 +342,7 @@ gmnal_small_rx(lib_nal_t *libnal, void *private, lib_msg_t *cookie) srxd = (gmnal_srxd_t*)private; /* - * let portals library know receive is complete + * let portals library know receive is complete */ CDEBUG(D_PORTALS, "calling lib_finalize\n"); lib_finalize(libnal, private, cookie, PTL_OK); @@ -351,8 +351,8 @@ gmnal_small_rx(lib_nal_t *libnal, void *private, lib_msg_t *cookie) */ CDEBUG(D_NET, "calling gm_provide_receive_buffer\n"); GMNAL_GM_LOCK(nal_data); - gm_provide_receive_buffer_with_tag(nal_data->gm_port, srxd->buffer, - srxd->gmsize, GM_LOW_PRIORITY, 0); + gm_provide_receive_buffer_with_tag(nal_data->gm_port, srxd->buffer, + srxd->gmsize, GM_LOW_PRIORITY, 0); GMNAL_GM_UNLOCK(nal_data); return(PTL_OK); @@ -366,8 +366,8 @@ gmnal_small_rx(lib_nal_t *libnal, void *private, lib_msg_t *cookie) * The callback function informs when the send is complete. */ ptl_err_t -gmnal_small_tx(lib_nal_t *libnal, void *private, lib_msg_t *cookie, - ptl_hdr_t *hdr, int type, ptl_nid_t global_nid, ptl_pid_t pid, +gmnal_small_tx(lib_nal_t *libnal, void *private, lib_msg_t *cookie, + ptl_hdr_t *hdr, int type, ptl_nid_t global_nid, ptl_pid_t pid, gmnal_stxd_t *stxd, int size) { gmnal_data_t *nal_data = (gmnal_data_t*)libnal->libnal_data; @@ -379,7 +379,7 @@ gmnal_small_tx(lib_nal_t *libnal, void *private, lib_msg_t *cookie, CDEBUG(D_TRACE, "gmnal_small_tx libnal [%p] private [%p] cookie [%p] " "hdr [%p] type [%d] global_nid ["LPU64"] pid [%d] stxd [%p] " - "size [%d]\n", libnal, private, cookie, hdr, type, + "size [%d]\n", libnal, private, cookie, hdr, type, global_nid, pid, stxd, size); CDEBUG(D_INFO, "portals_hdr:: dest_nid ["LPU64"], src_nid ["LPU64"]\n", @@ -473,24 +473,24 @@ gmnal_small_tx_callback(gm_port_t *gm_port, void *context, gm_status_t status) if (status != GM_SUCCESS) { GMNAL_GM_LOCK(nal_data); gm_status = gm_node_id_to_global_id(nal_data->gm_port, - stxd->gm_target_node, &gnid); + stxd->gm_target_node,&gnid); GMNAL_GM_UNLOCK(nal_data); if (gm_status != GM_SUCCESS) { CDEBUG(D_INFO, "gm_node_id_to_global_id failed[%d]\n", gm_status); gnid = 0; } - CDEBUG(D_ERROR, "Result of send stxd [%p] is [%s] to [%u]\n", + CDEBUG(D_ERROR, "Result of send stxd [%p] is [%s] to [%u]\n", stxd, gmnal_gm_error(status), gnid); } switch(status) { - case(GM_SUCCESS): + case(GM_SUCCESS): break; - case(GM_SEND_DROPPED): + case(GM_SEND_DROPPED): /* * do a resend on the dropped ones */ @@ -1033,15 +1033,15 @@ gmnal_copyiov(int do_copy, gmnal_srxd_t *srxd, int nsiov, */ sbuf_long = (unsigned long) sbuf; remote_ptr = (gm_remote_ptr_t)sbuf_long; - gm_get(nal_data->gm_port, remote_ptr, rbuf, - rlen, GM_LOW_PRIORITY, source_node, - GMNAL_GM_PORT_ID, + gm_get(nal_data->gm_port, remote_ptr, rbuf, + rlen, GM_LOW_PRIORITY, source_node, + GMNAL_GM_PORT_ID, gmnal_remote_get_callback, ltxd); GMNAL_GM_UNLOCK(nal_data); } /* * at the end of 1 iov element - */ + */ sbuf+=rlen; slen-=rlen; riov++; @@ -1057,9 +1057,9 @@ gmnal_copyiov(int do_copy, gmnal_srxd_t *srxd, int nsiov, GMNAL_GM_LOCK(nal_data); sbuf_long = (unsigned long) sbuf; remote_ptr = (gm_remote_ptr_t)sbuf_long; - gm_get(nal_data->gm_port, remote_ptr, rbuf, - slen, GM_LOW_PRIORITY, source_node, - GMNAL_GM_PORT_ID, + gm_get(nal_data->gm_port, remote_ptr, rbuf, + slen, GM_LOW_PRIORITY, source_node, + GMNAL_GM_PORT_ID, gmnal_remote_get_callback, ltxd); GMNAL_GM_UNLOCK(nal_data); } @@ -1080,9 +1080,9 @@ gmnal_copyiov(int do_copy, gmnal_srxd_t *srxd, int nsiov, GMNAL_GM_LOCK(nal_data); sbuf_long = (unsigned long) sbuf; remote_ptr = (gm_remote_ptr_t)sbuf_long; - gm_get(nal_data->gm_port, remote_ptr, rbuf, - rlen, GM_LOW_PRIORITY, source_node, - GMNAL_GM_PORT_ID, + gm_get(nal_data->gm_port, remote_ptr, rbuf, + rlen, GM_LOW_PRIORITY, source_node, + GMNAL_GM_PORT_ID, gmnal_remote_get_callback, ltxd); GMNAL_GM_UNLOCK(nal_data); } diff --git a/lnet/klnds/gmlnd/gmlnd_module.c b/lnet/klnds/gmlnd/gmlnd_module.c index 6cd622f..3851649 100644 --- a/lnet/klnds/gmlnd/gmlnd_module.c +++ b/lnet/klnds/gmlnd/gmlnd_module.c @@ -22,7 +22,7 @@ #include "gmnal.h" -int gmnal_small_msg_size = 525312; +int gmnal_small_msg_size = sizeof(gmnal_msghdr_t) + sizeof(ptl_hdr_t) + PTL_MTU; /* * -1 indicates default value. * This is 1 thread per cpu @@ -32,7 +32,7 @@ int num_rx_threads = -1; int num_stxds = 5; int gm_port_id = 4; -int +int gmnal_cmd(struct portals_cfg *pcfg, void *private) { gmnal_data_t *nal_data = NULL; diff --git a/lnet/klnds/gmlnd/gmlnd_utils.c b/lnet/klnds/gmlnd/gmlnd_utils.c index d625019..508a48c 100644 --- a/lnet/klnds/gmlnd/gmlnd_utils.c +++ b/lnet/klnds/gmlnd/gmlnd_utils.c @@ -964,11 +964,12 @@ gmnal_get_rxtwe(gmnal_data_t *nal_data) CDEBUG(D_NET, "Getting entry to list\n"); do { - while(down_interruptible(&nal_data->rxtwe_wait) != 0); + while(down_interruptible(&nal_data->rxtwe_wait) != 0) + /* do nothing */; if (nal_data->rxthread_stop_flag == GMNAL_THREAD_STOP) { /* * time to stop - * TO DO some one free the work entries + * TO DO some one free the work entries */ return(NULL); } -- 1.8.3.1