From 5e285d95f67dd419778d68ad7e5ad5571c5dda9e Mon Sep 17 00:00:00 2001 From: eeb Date: Mon, 8 Aug 2005 12:50:12 +0000 Subject: [PATCH] * Incorporated latest GMNAL updates from HP. Compiled but not run yet. --- lnet/klnds/gmlnd/gmlnd.h | 16 ++++-- lnet/klnds/gmlnd/gmlnd_api.c | 23 ++++---- lnet/klnds/gmlnd/gmlnd_cb.c | 114 +++++++++++++++++----------------------- lnet/klnds/gmlnd/gmlnd_comm.c | 32 ++++++----- lnet/klnds/gmlnd/gmlnd_module.c | 2 +- 5 files changed, 93 insertions(+), 94 deletions(-) diff --git a/lnet/klnds/gmlnd/gmlnd.h b/lnet/klnds/gmlnd/gmlnd.h index bffbb0b..2193a21 100644 --- a/lnet/klnds/gmlnd/gmlnd.h +++ b/lnet/klnds/gmlnd/gmlnd.h @@ -47,7 +47,13 @@ #include "linux/string.h" #include "linux/stat.h" #include "linux/errno.h" +#include "linux/version.h" +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) +#include "linux/buffer_head.h" +#include "linux/fs.h" +#else #include "linux/locks.h" +#endif #include "linux/unistd.h" #include "linux/init.h" #include "linux/sem.h" @@ -166,12 +172,12 @@ typedef struct _gmnal_srxd_t { * watch alignment for ia32/64 interaction */ typedef struct _gmnal_msghdr { - int magic; - int type; - unsigned int sender_node_id; - int niov; + __s32 magic; + __s32 type; + __u32 sender_node_id; + __s32 niov; gm_remote_ptr_t stxd_remote_ptr; /* 64 bits */ - } gmnal_msghdr_t; +} WIRE_ATTR gmnal_msghdr_t; #define GMNAL_MSGHDR_SIZE sizeof(gmnal_msghdr_t) /* diff --git a/lnet/klnds/gmlnd/gmlnd_api.c b/lnet/klnds/gmlnd/gmlnd_api.c index bf182b4..a382471 100644 --- a/lnet/klnds/gmlnd/gmlnd_api.c +++ b/lnet/klnds/gmlnd/gmlnd_api.c @@ -263,16 +263,6 @@ gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid, return(PTL_FAIL); } - gmnal_start_kernel_threads(nal_data); - - while (nal_data->rxthread_flag != GMNAL_RXTHREADS_STARTED) { - gmnal_yield(1); - CDEBUG(D_INFO, "Waiting for receive thread signs of life\n"); - } - - CDEBUG(D_INFO, "receive thread seems to have started\n"); - - /* * Initialise the portals library */ @@ -346,6 +336,19 @@ gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid, return(PTL_FAIL); } + /* + * Now that we have initialised the portals library, start receive threads, + * we do this to avoid processing messages before we can parse them + */ + gmnal_start_kernel_threads(nal_data); + + while (nal_data->rxthread_flag != GMNAL_RXTHREADS_STARTED) { + gmnal_yield(1); + CDEBUG(D_INFO, "Waiting for receive thread signs of life\n"); + } + + CDEBUG(D_INFO, "receive thread seems to have started\n"); + if (libcfs_nal_cmd_register(GMNAL, &gmnal_cmd, libnal->libnal_data) != 0) { CDEBUG(D_INFO, "libcfs_nal_cmd_register failed\n"); diff --git a/lnet/klnds/gmlnd/gmlnd_cb.c b/lnet/klnds/gmlnd/gmlnd_cb.c index ddff6b9..a96f6e6 100644 --- a/lnet/klnds/gmlnd/gmlnd_cb.c +++ b/lnet/klnds/gmlnd/gmlnd_cb.c @@ -34,6 +34,8 @@ ptl_err_t gmnal_cb_recv(lib_nal_t *libnal, void *private, lib_msg_t *cookie, void *buffer = NULL; gmnal_srxd_t *srxd = (gmnal_srxd_t*)private; int status = PTL_OK; + size_t msglen = mlen; + size_t nob; CDEBUG(D_TRACE, "gmnal_cb_recv libnal [%p], private[%p], cookie[%p], " "niov[%d], iov [%p], offset["LPSZ"], mlen["LPSZ"], rlen["LPSZ"]\n", @@ -46,26 +48,22 @@ ptl_err_t gmnal_cb_recv(lib_nal_t *libnal, void *private, lib_msg_t *cookie, * side occurrence of filling pkmap_count[]. */ buffer = srxd->buffer; - buffer += sizeof(gmnal_msghdr_t); + buffer += GMNAL_MSGHDR_SIZE; buffer += sizeof(ptl_hdr_t); while(niov--) { if (offset >= iov->iov_len) { offset -= iov->iov_len; - } else if (offset > 0) { - CDEBUG(D_INFO, "processing [%p] base [%p] " - "len %d, offset %d, len ["LPSZ"]\n", iov, - iov->iov_base + offset, iov->iov_len, - offset, iov->iov_len - offset); - gm_bcopy(buffer, iov->iov_base + offset, - iov->iov_len - offset); - buffer += iov->iov_len - offset; - offset = 0; } else { - CDEBUG(D_INFO, "processing [%p] len ["LPSZ"]\n", - iov, iov->iov_len); - gm_bcopy(buffer, iov->iov_base, iov->iov_len); - buffer += iov->iov_len; + nob = MIN (iov->iov_len - offset, msglen); + CDEBUG(D_INFO, "processing iov [%p] base [%p] " + "offset [%d] len ["LPSZ"] to [%p] left " + "["LPSZ"]\n", iov, iov->iov_base, + offset, nob, buffer, msglen); + gm_bcopy(buffer, iov->iov_base + offset, nob); + buffer += nob; + msglen -= nob; + offset = 0; } iov++; } @@ -97,8 +95,11 @@ ptl_err_t gmnal_cb_recv_pages(lib_nal_t *libnal, void *private, libnal, private, cookie, kniov, kiov, offset, mlen, rlen); if (srxd->type == GMNAL_SMALL_MESSAGE) { + size_t msglen = mlen; + size_t nob; + buffer = srxd->buffer; - buffer += sizeof(gmnal_msghdr_t); + buffer += GMNAL_MSGHDR_SIZE; buffer += sizeof(ptl_hdr_t); /* @@ -114,31 +115,21 @@ ptl_err_t gmnal_cb_recv_pages(lib_nal_t *libnal, void *private, if (offset >= kiov->kiov_len) { offset -= kiov->kiov_len; } else { + nob = MIN (kiov->kiov_len - offset, msglen); CDEBUG(D_INFO, "kniov page [%p] len [%d] " "offset[%d]\n", kiov->kiov_page, kiov->kiov_len, kiov->kiov_offset); - CDEBUG(D_INFO, "Calling kmap[%p]", kiov->kiov_page); ptr = ((char *)kmap(kiov->kiov_page)) + kiov->kiov_offset; - if (offset > 0) { - CDEBUG(D_INFO, "processing [%p] base " - "[%p] len %d, offset %d, len [" - LPSZ"]\n", ptr, ptr + offset, - kiov->kiov_len, offset, - kiov->kiov_len - offset); - gm_bcopy(buffer, ptr + offset, - kiov->kiov_len - offset); - buffer += kiov->kiov_len - offset; - offset = 0; - } else { - CDEBUG(D_INFO, "processing [%p] len [" - LPSZ"]\n", ptr, kiov->kiov_len); - gm_bcopy(buffer, ptr, kiov->kiov_len); - buffer += kiov->kiov_len; - } + CDEBUG(D_INFO, "processing ptr [%p] offset [%d] " + "len ["LPSZ"] from [%p] left ["LPSZ"]\n", + ptr, offset, nob, buffer, msglen); + gm_bcopy(buffer, ptr + offset, nob); kunmap(kiov->kiov_page); - CDEBUG(D_INFO, "Stored in [%p]\n", ptr); + buffer += nob; + msglen -= nob; + offset = 0; } kiov++; } @@ -173,6 +164,9 @@ ptl_err_t gmnal_cb_send(lib_nal_t *libnal, void *private, lib_msg_t *cookie, } if (GMNAL_IS_SMALL_MESSAGE(nal_data, niov, iov, len)) { + size_t msglen = len; + size_t nob; + CDEBUG(D_INFO, "This is a small message send\n"); /* * HP SFS 1380: With the change to gmnal_small_tx, need to get @@ -181,24 +175,20 @@ ptl_err_t gmnal_cb_send(lib_nal_t *libnal, void *private, lib_msg_t *cookie, stxd = gmnal_get_stxd(nal_data, 1); CDEBUG(D_INFO, "stxd [%p]\n", stxd); /* Set the offset of the data to copy into the buffer */ - buffer = stxd->buffer +sizeof(gmnal_msghdr_t)+sizeof(ptl_hdr_t); + buffer = stxd->buffer + GMNAL_MSGHDR_SIZE + sizeof(ptl_hdr_t); while(niov--) { if (offset >= iov->iov_len) { offset -= iov->iov_len; - } else if (offset > 0) { - CDEBUG(D_INFO, "processing iov [%p] base [%p] " - "len ["LPSZ"] to [%p]\n", - iov, iov->iov_base + offset, - iov->iov_len - offset, buffer); - gm_bcopy(iov->iov_base + offset, buffer, - iov->iov_len - offset); - buffer+= iov->iov_len - offset; - offset = 0; } else { - CDEBUG(D_INFO, "processing iov [%p] len ["LPSZ - "] to [%p]\n", iov, iov->iov_len,buffer); - gm_bcopy(iov->iov_base, buffer, iov->iov_len); - buffer+= iov->iov_len; + nob = MIN (iov->iov_len - offset, msglen); + CDEBUG(D_INFO, "processing iov [%p] base [%p]" + " offset [%d] len ["LPSZ"] to [%p] left" + " ["LPSZ"]\n", iov, iov->iov_base, + offset, nob, buffer, msglen); + gm_bcopy(iov->iov_base + offset, buffer, nob); + buffer += nob; + msglen -= nob; + offset = 0; } iov++; } @@ -245,9 +235,12 @@ ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private, stxd = gmnal_get_stxd(nal_data, 1); CDEBUG(D_INFO, "stxd [%p]\n", stxd); /* Set the offset of the data to copy into the buffer */ - buffer = stxd->buffer + sizeof(gmnal_msghdr_t) + sizeof(ptl_hdr_t); + buffer = stxd->buffer + GMNAL_MSGHDR_SIZE + sizeof(ptl_hdr_t); if (GMNAL_IS_SMALL_MESSAGE(nal_data, 0, NULL, len)) { + size_t msglen = len; + size_t nob; + CDEBUG(D_INFO, "This is a small message send\n"); while(kniov--) { @@ -255,6 +248,7 @@ ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private, if (offset >= kiov->kiov_len) { offset -= kiov->kiov_len; } else { + nob = MIN (kiov->kiov_len - offset, msglen); CDEBUG(D_INFO, "kniov page [%p] len [%d] offset[%d]\n", kiov->kiov_page, kiov->kiov_len, kiov->kiov_offset); @@ -262,24 +256,14 @@ ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private, ptr = ((char *)kmap(kiov->kiov_page)) + kiov->kiov_offset; - if (offset > 0) { - CDEBUG(D_INFO, "processing [%p] base " - "[%p] len ["LPSZ"] to [%p]\n", - ptr, ptr + offset, - kiov->kiov_len - offset, buffer); - gm_bcopy(ptr + offset, buffer, - kiov->kiov_len - offset); - buffer+= kiov->kiov_len - offset; - offset = 0; - } else { - CDEBUG(D_INFO, "processing kmapped [%p]" - " len ["LPSZ"] to [%p]\n", - ptr, kiov->kiov_len, buffer); - gm_bcopy(ptr, buffer, kiov->kiov_len); - - buffer += kiov->kiov_len; - } + CDEBUG(D_INFO, "processing ptr [%p] offset [%d]" + " len ["LPSZ"] to [%p] left ["LPSZ"]\n", + ptr, offset, nob, buffer, msglen); + gm_bcopy(ptr + offset, buffer, nob); kunmap(kiov->kiov_page); + buffer += nob; + msglen -= nob; + offset = 0; } kiov++; } diff --git a/lnet/klnds/gmlnd/gmlnd_comm.c b/lnet/klnds/gmlnd/gmlnd_comm.c index 60e5d67..e669153 100644 --- a/lnet/klnds/gmlnd/gmlnd_comm.c +++ b/lnet/klnds/gmlnd/gmlnd_comm.c @@ -50,7 +50,7 @@ gmnal_ct_thread(void *arg) sprintf(current->comm, "gmnal_ct"); - daemonize(); + kportal_daemonize("gmnalctd"); nal_data->ctthread_flag = GMNAL_CTTHREAD_STARTED; @@ -112,6 +112,7 @@ gmnal_ct_thread(void *arg) */ int gmnal_rx_thread(void *arg) { + char name[16]; gmnal_data_t *nal_data; void *buffer; gmnal_rxtwe_t *we = NULL; @@ -129,9 +130,9 @@ int gmnal_rx_thread(void *arg) if (nal_data->rxthread_pid[rank] == current->pid) break; - sprintf(current->comm, "gmnal_rx_%d", rank); + snprintf(name, sizeof(name), "gmnal_rx_%d", rank); - daemonize(); + kportal_daemonize(name); /* * set 1 bit for each thread started * doesn't matter which bit @@ -257,8 +258,10 @@ gmnal_pre_receive(gmnal_data_t *nal_data, gmnal_rxtwe_t *we, int gmnal_type) rc = lib_parse(nal_data->libnal, portals_hdr, srxd); if (rc != PTL_OK) { - /* I just received garbage; take appropriate action... */ - LBUG(); + /* I just received garbage; return the srxd for use */ + CWARN("Returning srxd and discarding message, " + "lib_parse didn't like it.\n"); + return(gmnal_rx_bad(nal_data, we, srxd)); } return(GMNAL_STATUS_OK); @@ -414,7 +417,7 @@ gmnal_small_tx(lib_nal_t *libnal, void *private, lib_msg_t *cookie, msghdr->sender_node_id = nal_data->gm_global_nid; CDEBUG(D_INFO, "processing msghdr at [%p]\n", buffer); - buffer += sizeof(gmnal_msghdr_t); + buffer += GMNAL_MSGHDR_SIZE; CDEBUG(D_INFO, "processing portals hdr at [%p]\n", buffer); gm_bcopy(hdr, buffer, sizeof(ptl_hdr_t)); @@ -422,7 +425,7 @@ gmnal_small_tx(lib_nal_t *libnal, void *private, lib_msg_t *cookie, buffer += sizeof(ptl_hdr_t); CDEBUG(D_INFO, "sending\n"); - tot_size = size+sizeof(ptl_hdr_t)+sizeof(gmnal_msghdr_t); + tot_size = size+sizeof(ptl_hdr_t)+GMNAL_MSGHDR_SIZE; stxd->msg_size = tot_size; @@ -597,10 +600,11 @@ gmnal_small_tx_callback(gm_port_t *gm_port, void *context, gm_status_t status) void gmnal_resume_sending_callback(struct gm_port *gm_port, void *context, gm_status_t status) { - gmnal_data_t *nal_data; gmnal_stxd_t *stxd = (gmnal_stxd_t*)context; + gmnal_data_t *nal_data = (gmnal_data_t*)stxd->nal_data; CDEBUG(D_TRACE, "status is [%d] context is [%p]\n", status, context); gmnal_return_stxd(stxd->nal_data, stxd); + lib_finalize(nal_data->libnal, stxd, stxd->cookie, PTL_FAIL); return; } @@ -624,9 +628,11 @@ void gmnal_drop_sends_callback(struct gm_port *gm_port, void *context, } else { CERROR("send_to_peer status for stxd [%p] is " "[%d][%s]\n", stxd, status, gmnal_gm_error(status)); + /* Recycle the stxd */ + gmnal_return_stxd(nal_data, stxd); + lib_finalize(nal_data->libnal, stxd, stxd->cookie, PTL_FAIL); } - return; } @@ -694,8 +700,8 @@ gmnal_large_tx(lib_nal_t *libnal, void *private, lib_msg_t *cookie, msghdr->sender_node_id = nal_data->gm_global_nid; msghdr->stxd_remote_ptr = (gm_remote_ptr_t)stxd; msghdr->niov = niov ; - buffer += sizeof(gmnal_msghdr_t); - mlen = sizeof(gmnal_msghdr_t); + buffer += GMNAL_MSGHDR_SIZE; + mlen = GMNAL_MSGHDR_SIZE; CDEBUG(D_INFO, "mlen is [%d]\n", mlen); @@ -838,7 +844,7 @@ gmnal_large_rx(lib_nal_t *libnal, void *private, lib_msg_t *cookie, buffer = srxd->buffer; msghdr = (gmnal_msghdr_t*)buffer; - buffer += sizeof(gmnal_msghdr_t); + buffer += GMNAL_MSGHDR_SIZE; buffer += sizeof(ptl_hdr_t); /* @@ -1232,7 +1238,7 @@ gmnal_large_tx_ack(gmnal_data_t *nal_data, gmnal_srxd_t *srxd) CDEBUG(D_INFO, "processing msghdr at [%p]\n", buffer); CDEBUG(D_INFO, "sending\n"); - stxd->msg_size= sizeof(gmnal_msghdr_t); + stxd->msg_size= GMNAL_MSGHDR_SIZE; CDEBUG(D_NET, "Calling gm_send_to_peer port [%p] buffer [%p] " diff --git a/lnet/klnds/gmlnd/gmlnd_module.c b/lnet/klnds/gmlnd/gmlnd_module.c index 3dd09b3..e2549ef 100644 --- a/lnet/klnds/gmlnd/gmlnd_module.c +++ b/lnet/klnds/gmlnd/gmlnd_module.c @@ -22,7 +22,7 @@ #include "gmnal.h" -int gmnal_small_msg_size = sizeof(gmnal_msghdr_t) + sizeof(ptl_hdr_t) + PTL_MTU; +int gmnal_small_msg_size = GMNAL_MSGHDR_SIZE + sizeof(ptl_hdr_t) + PTL_MTU + 928; /* * -1 indicates default value. * This is 1 thread per cpu -- 1.8.3.1