* Incorporated latest GMNAL updates from HP.

    Compiled but not run yet.

diff --git a/lnet/klnds/gmlnd/gmlnd.h b/lnet/klnds/gmlnd/gmlnd.h
index bffbb0b..2193a21 100644 (file)
--- a/lnet/klnds/gmlnd/gmlnd.h
+++ b/lnet/klnds/gmlnd/gmlnd.h
@@ -47,7 +47,13 @@
 #include "linux/string.h"
 #include "linux/stat.h"
 #include "linux/errno.h"
+#include "linux/version.h"
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
+#include "linux/buffer_head.h"
+#include "linux/fs.h"
+#else
 #include "linux/locks.h"
+#endif
 #include "linux/unistd.h"
 #include "linux/init.h"
 #include "linux/sem.h"
@@ -166,12 +172,12 @@ typedef struct _gmnal_srxd_t {
  *     watch alignment for ia32/64 interaction
  */
 typedef struct _gmnal_msghdr {
-       int             magic;
-       int             type;
-       unsigned int    sender_node_id;
-       int             niov;
+       __s32           magic;
+       __s32           type;
+       __u32           sender_node_id;
+       __s32           niov;
        gm_remote_ptr_t stxd_remote_ptr; /* 64 bits */
-       } gmnal_msghdr_t;
+} WIRE_ATTR gmnal_msghdr_t;
 #define GMNAL_MSGHDR_SIZE      sizeof(gmnal_msghdr_t)
 
 /*

diff --git a/lnet/klnds/gmlnd/gmlnd_api.c b/lnet/klnds/gmlnd/gmlnd_api.c
index bf182b4..a382471 100644 (file)
--- a/lnet/klnds/gmlnd/gmlnd_api.c
+++ b/lnet/klnds/gmlnd/gmlnd_api.c
@@ -263,16 +263,6 @@ gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid,
                return(PTL_FAIL);
        }
 
-       gmnal_start_kernel_threads(nal_data);
-
-       while (nal_data->rxthread_flag != GMNAL_RXTHREADS_STARTED) {
-               gmnal_yield(1);
-               CDEBUG(D_INFO, "Waiting for receive thread signs of life\n");
-       }
-
-       CDEBUG(D_INFO, "receive thread seems to have started\n");
-
-
        /*
         *      Initialise the portals library
         */
@@ -346,6 +336,19 @@ gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid,
                return(PTL_FAIL);
        }
 
+       /*
+        * Now that we have initialised the portals library, start receive threads,
+        * we do this to avoid processing messages before we can parse them
+        */
+       gmnal_start_kernel_threads(nal_data);
+
+       while (nal_data->rxthread_flag != GMNAL_RXTHREADS_STARTED) {
+               gmnal_yield(1);
+               CDEBUG(D_INFO, "Waiting for receive thread signs of life\n");
+       }
+
+       CDEBUG(D_INFO, "receive thread seems to have started\n");
+
        if (libcfs_nal_cmd_register(GMNAL, &gmnal_cmd, libnal->libnal_data) != 0) {
                CDEBUG(D_INFO, "libcfs_nal_cmd_register failed\n");
 

diff --git a/lnet/klnds/gmlnd/gmlnd_cb.c b/lnet/klnds/gmlnd/gmlnd_cb.c
index ddff6b9..a96f6e6 100644 (file)
--- a/lnet/klnds/gmlnd/gmlnd_cb.c
+++ b/lnet/klnds/gmlnd/gmlnd_cb.c
@@ -34,6 +34,8 @@ ptl_err_t gmnal_cb_recv(lib_nal_t *libnal, void *private, lib_msg_t *cookie,
         void            *buffer = NULL;
        gmnal_srxd_t    *srxd = (gmnal_srxd_t*)private;
        int             status = PTL_OK;
+        size_t          msglen = mlen;
+        size_t          nob;
 
        CDEBUG(D_TRACE, "gmnal_cb_recv libnal [%p], private[%p], cookie[%p], "
               "niov[%d], iov [%p], offset["LPSZ"], mlen["LPSZ"], rlen["LPSZ"]\n",
@@ -46,26 +48,22 @@ ptl_err_t gmnal_cb_recv(lib_nal_t *libnal, void *private, lib_msg_t *cookie,
                 *  side occurrence of filling pkmap_count[].
                 */
                buffer = srxd->buffer;
-               buffer += sizeof(gmnal_msghdr_t);
+               buffer += GMNAL_MSGHDR_SIZE;
                buffer += sizeof(ptl_hdr_t);
 
                while(niov--) {
                        if (offset >= iov->iov_len) {
                                offset -= iov->iov_len;
-                       } else if (offset > 0) {
-                               CDEBUG(D_INFO, "processing [%p] base [%p] "
-                                       "len %d, offset %d, len ["LPSZ"]\n", iov,
-                                       iov->iov_base + offset, iov->iov_len,
-                                       offset, iov->iov_len - offset);
-                               gm_bcopy(buffer, iov->iov_base + offset,
-                                        iov->iov_len - offset);
-                               buffer += iov->iov_len - offset;
-                               offset = 0;
                        } else {
-                               CDEBUG(D_INFO, "processing [%p] len ["LPSZ"]\n",
-                                       iov, iov->iov_len);
-                               gm_bcopy(buffer, iov->iov_base, iov->iov_len);
-                               buffer += iov->iov_len;
+                                nob = MIN (iov->iov_len - offset, msglen);
+                                CDEBUG(D_INFO, "processing iov [%p] base [%p] "
+                                       "offset [%d] len ["LPSZ"] to [%p] left "
+                                       "["LPSZ"]\n", iov, iov->iov_base,
+                                       offset, nob, buffer, msglen);
+                                gm_bcopy(buffer, iov->iov_base + offset, nob);
+                                buffer += nob;
+                                msglen -= nob;
+                                offset = 0;
                        }
                        iov++;
                }
@@ -97,8 +95,11 @@ ptl_err_t gmnal_cb_recv_pages(lib_nal_t *libnal, void *private,
               libnal, private, cookie, kniov, kiov, offset, mlen, rlen);
 
        if (srxd->type == GMNAL_SMALL_MESSAGE) {
+                size_t          msglen = mlen;
+                size_t          nob;
+
                buffer = srxd->buffer;
-               buffer += sizeof(gmnal_msghdr_t);
+               buffer += GMNAL_MSGHDR_SIZE;
                buffer += sizeof(ptl_hdr_t);
 
                /*
@@ -114,31 +115,21 @@ ptl_err_t gmnal_cb_recv_pages(lib_nal_t *libnal, void *private,
                        if (offset >= kiov->kiov_len) {
                                offset -= kiov->kiov_len;
                        } else {
+                                nob = MIN (kiov->kiov_len - offset, msglen);
                                CDEBUG(D_INFO, "kniov page [%p] len [%d] "
                                        "offset[%d]\n", kiov->kiov_page,
                                        kiov->kiov_len, kiov->kiov_offset);
-                               CDEBUG(D_INFO, "Calling kmap[%p]", kiov->kiov_page);
                                ptr = ((char *)kmap(kiov->kiov_page)) +
                                         kiov->kiov_offset;
 
-                               if (offset > 0) {
-                                       CDEBUG(D_INFO, "processing [%p] base "
-                                               "[%p] len %d, offset %d, len ["
-                                               LPSZ"]\n", ptr, ptr + offset,
-                                               kiov->kiov_len, offset,
-                                              kiov->kiov_len - offset);
-                                       gm_bcopy(buffer, ptr + offset,
-                                                 kiov->kiov_len - offset);
-                                       buffer += kiov->kiov_len - offset;
-                                       offset = 0;
-                               } else {
-                                       CDEBUG(D_INFO, "processing [%p] len ["
-                                               LPSZ"]\n", ptr, kiov->kiov_len);
-                                       gm_bcopy(buffer, ptr, kiov->kiov_len);
-                                       buffer += kiov->kiov_len;
-                               }
+                                CDEBUG(D_INFO, "processing ptr [%p] offset [%d] "
+                                       "len ["LPSZ"] from [%p] left ["LPSZ"]\n",
+                                       ptr, offset, nob, buffer, msglen);
+                                gm_bcopy(buffer, ptr + offset, nob);
                                kunmap(kiov->kiov_page);
-                               CDEBUG(D_INFO, "Stored in [%p]\n", ptr);
+                                buffer += nob;
+                                msglen -= nob;
+                                offset = 0;
                         }
                         kiov++;
                }
@@ -173,6 +164,9 @@ ptl_err_t gmnal_cb_send(lib_nal_t *libnal, void *private, lib_msg_t *cookie,
        }
 
        if (GMNAL_IS_SMALL_MESSAGE(nal_data, niov, iov, len)) {
+                size_t msglen = len;
+                size_t nob;
+
                CDEBUG(D_INFO, "This is a small message send\n");
                /*
                 * HP SFS 1380: With the change to gmnal_small_tx, need to get
@@ -181,24 +175,20 @@ ptl_err_t gmnal_cb_send(lib_nal_t *libnal, void *private, lib_msg_t *cookie,
                stxd = gmnal_get_stxd(nal_data, 1);
                CDEBUG(D_INFO, "stxd [%p]\n", stxd);
                /* Set the offset of the data to copy into the buffer */
-               buffer = stxd->buffer +sizeof(gmnal_msghdr_t)+sizeof(ptl_hdr_t);
+               buffer = stxd->buffer + GMNAL_MSGHDR_SIZE + sizeof(ptl_hdr_t);
                while(niov--) {
                        if (offset >= iov->iov_len) {
                                offset -= iov->iov_len;
-                       } else if (offset > 0) {
-                               CDEBUG(D_INFO, "processing iov [%p] base [%p] "
-                                       "len ["LPSZ"] to [%p]\n",
-                                       iov, iov->iov_base + offset,
-                                       iov->iov_len - offset, buffer);
-                               gm_bcopy(iov->iov_base + offset, buffer,
-                                         iov->iov_len - offset);
-                               buffer+= iov->iov_len - offset;
-                               offset = 0;
                        } else {
-                               CDEBUG(D_INFO, "processing iov [%p] len ["LPSZ
-                                       "] to [%p]\n", iov, iov->iov_len,buffer);
-                               gm_bcopy(iov->iov_base, buffer, iov->iov_len);
-                               buffer+= iov->iov_len;
+                                nob = MIN (iov->iov_len - offset, msglen);
+                                CDEBUG(D_INFO, "processing iov [%p] base [%p]"
+                                      " offset [%d] len ["LPSZ"] to [%p] left"
+                                      " ["LPSZ"]\n", iov, iov->iov_base,
+                                      offset, nob, buffer, msglen);
+                                gm_bcopy(iov->iov_base + offset, buffer, nob);
+                                buffer += nob;
+                                msglen -= nob;
+                                offset = 0;
                        }
                        iov++;
                }
@@ -245,9 +235,12 @@ ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private,
        stxd = gmnal_get_stxd(nal_data, 1);
        CDEBUG(D_INFO, "stxd [%p]\n", stxd);
        /* Set the offset of the data to copy into the buffer */
-       buffer = stxd->buffer + sizeof(gmnal_msghdr_t) + sizeof(ptl_hdr_t);
+       buffer = stxd->buffer + GMNAL_MSGHDR_SIZE + sizeof(ptl_hdr_t);
 
        if (GMNAL_IS_SMALL_MESSAGE(nal_data, 0, NULL, len)) {
+                size_t msglen = len;
+                size_t nob;
+
                CDEBUG(D_INFO, "This is a small message send\n");
 
                while(kniov--) {
@@ -255,6 +248,7 @@ ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private,
                        if (offset >= kiov->kiov_len) {
                                offset -= kiov->kiov_len;
                        } else {
+                                nob = MIN (kiov->kiov_len - offset, msglen);
                                CDEBUG(D_INFO, "kniov page [%p] len [%d] offset[%d]\n",
                                       kiov->kiov_page, kiov->kiov_len, 
                                       kiov->kiov_offset);
@@ -262,24 +256,14 @@ ptl_err_t gmnal_cb_send_pages(lib_nal_t *libnal, void *private,
                                ptr = ((char *)kmap(kiov->kiov_page)) +
                                         kiov->kiov_offset;
 
-                               if (offset > 0) {
-                                       CDEBUG(D_INFO, "processing [%p] base "
-                                               "[%p] len ["LPSZ"] to [%p]\n",
-                                              ptr, ptr + offset,
-                                               kiov->kiov_len - offset, buffer);
-                                       gm_bcopy(ptr + offset, buffer,
-                                                 kiov->kiov_len - offset);
-                                       buffer+= kiov->kiov_len - offset;
-                                       offset = 0;
-                               } else {
-                                       CDEBUG(D_INFO, "processing kmapped [%p]"
-                                               " len ["LPSZ"] to [%p]\n",
-                                              ptr, kiov->kiov_len, buffer);
-                                       gm_bcopy(ptr, buffer, kiov->kiov_len);
-
-                                       buffer += kiov->kiov_len;
-                               }
+                                CDEBUG(D_INFO, "processing ptr [%p] offset [%d]"
+                                       " len ["LPSZ"] to [%p] left ["LPSZ"]\n",
+                                       ptr, offset, nob, buffer, msglen);
+                                gm_bcopy(ptr + offset, buffer, nob);
                                kunmap(kiov->kiov_page);
+                                buffer += nob;
+                                msglen -= nob;
+                                offset = 0;
                        }
                         kiov++;
                }

diff --git a/lnet/klnds/gmlnd/gmlnd_comm.c b/lnet/klnds/gmlnd/gmlnd_comm.c
index 60e5d67..e669153 100644 (file)
--- a/lnet/klnds/gmlnd/gmlnd_comm.c
+++ b/lnet/klnds/gmlnd/gmlnd_comm.c
@@ -50,7 +50,7 @@ gmnal_ct_thread(void *arg)
 
        sprintf(current->comm, "gmnal_ct");
 
-       daemonize();
+       kportal_daemonize("gmnalctd");
 
        nal_data->ctthread_flag = GMNAL_CTTHREAD_STARTED;
 
@@ -112,6 +112,7 @@ gmnal_ct_thread(void *arg)
  */
 int gmnal_rx_thread(void *arg)
 {
+        char                     name[16];
        gmnal_data_t            *nal_data;
        void                    *buffer;
        gmnal_rxtwe_t           *we = NULL;
@@ -129,9 +130,9 @@ int gmnal_rx_thread(void *arg)
                if (nal_data->rxthread_pid[rank] == current->pid)
                        break;
 
-       sprintf(current->comm, "gmnal_rx_%d", rank);
+       snprintf(name, sizeof(name), "gmnal_rx_%d", rank);
 
-       daemonize();
+       kportal_daemonize(name);
        /*
         *      set 1 bit for each thread started
         *      doesn't matter which bit
@@ -257,8 +258,10 @@ gmnal_pre_receive(gmnal_data_t *nal_data, gmnal_rxtwe_t *we, int gmnal_type)
        rc = lib_parse(nal_data->libnal, portals_hdr, srxd);
 
         if (rc != PTL_OK) {
-                /* I just received garbage; take appropriate action... */
-                LBUG();
+                /* I just received garbage; return the srxd for use */
+                CWARN("Returning srxd and discarding message, "
+                        "lib_parse didn't like it.\n");
+                return(gmnal_rx_bad(nal_data, we, srxd));
         }
 
        return(GMNAL_STATUS_OK);
@@ -414,7 +417,7 @@ gmnal_small_tx(lib_nal_t *libnal, void *private, lib_msg_t *cookie,
        msghdr->sender_node_id = nal_data->gm_global_nid;
        CDEBUG(D_INFO, "processing msghdr at [%p]\n", buffer);
 
-       buffer += sizeof(gmnal_msghdr_t);
+       buffer += GMNAL_MSGHDR_SIZE;
 
        CDEBUG(D_INFO, "processing  portals hdr at [%p]\n", buffer);
        gm_bcopy(hdr, buffer, sizeof(ptl_hdr_t));
@@ -422,7 +425,7 @@ gmnal_small_tx(lib_nal_t *libnal, void *private, lib_msg_t *cookie,
        buffer += sizeof(ptl_hdr_t);
 
        CDEBUG(D_INFO, "sending\n");
-       tot_size = size+sizeof(ptl_hdr_t)+sizeof(gmnal_msghdr_t);
+       tot_size = size+sizeof(ptl_hdr_t)+GMNAL_MSGHDR_SIZE;
        stxd->msg_size = tot_size;
 
 
@@ -597,10 +600,11 @@ gmnal_small_tx_callback(gm_port_t *gm_port, void *context, gm_status_t status)
 void gmnal_resume_sending_callback(struct gm_port *gm_port, void *context,
                                  gm_status_t status)
 {
-        gmnal_data_t    *nal_data;
         gmnal_stxd_t    *stxd = (gmnal_stxd_t*)context;
+        gmnal_data_t    *nal_data = (gmnal_data_t*)stxd->nal_data;
         CDEBUG(D_TRACE, "status is [%d] context is [%p]\n", status, context);
         gmnal_return_stxd(stxd->nal_data, stxd);
+        lib_finalize(nal_data->libnal, stxd, stxd->cookie, PTL_FAIL);
         return;
 }
 
@@ -624,9 +628,11 @@ void gmnal_drop_sends_callback(struct gm_port *gm_port, void *context,
        } else {
                CERROR("send_to_peer status for stxd [%p] is "
                       "[%d][%s]\n", stxd, status, gmnal_gm_error(status));
+                /* Recycle the stxd */
+               gmnal_return_stxd(nal_data, stxd);
+               lib_finalize(nal_data->libnal, stxd, stxd->cookie, PTL_FAIL);
        }
 
-
        return;
 }
 
@@ -694,8 +700,8 @@ gmnal_large_tx(lib_nal_t *libnal, void *private, lib_msg_t *cookie,
        msghdr->sender_node_id = nal_data->gm_global_nid;
        msghdr->stxd_remote_ptr = (gm_remote_ptr_t)stxd;
        msghdr->niov = niov ;
-       buffer += sizeof(gmnal_msghdr_t);
-       mlen = sizeof(gmnal_msghdr_t);
+       buffer += GMNAL_MSGHDR_SIZE;
+       mlen = GMNAL_MSGHDR_SIZE;
        CDEBUG(D_INFO, "mlen is [%d]\n", mlen);
 
 
@@ -838,7 +844,7 @@ gmnal_large_rx(lib_nal_t *libnal, void *private, lib_msg_t *cookie,
 
        buffer = srxd->buffer;
        msghdr = (gmnal_msghdr_t*)buffer;
-       buffer += sizeof(gmnal_msghdr_t);
+       buffer += GMNAL_MSGHDR_SIZE;
        buffer += sizeof(ptl_hdr_t);
 
        /*
@@ -1232,7 +1238,7 @@ gmnal_large_tx_ack(gmnal_data_t *nal_data, gmnal_srxd_t *srxd)
        CDEBUG(D_INFO, "processing msghdr at [%p]\n", buffer);
 
        CDEBUG(D_INFO, "sending\n");
-       stxd->msg_size= sizeof(gmnal_msghdr_t);
+       stxd->msg_size= GMNAL_MSGHDR_SIZE;
 
 
        CDEBUG(D_NET, "Calling gm_send_to_peer port [%p] buffer [%p] "

diff --git a/lnet/klnds/gmlnd/gmlnd_module.c b/lnet/klnds/gmlnd/gmlnd_module.c
index 3dd09b3..e2549ef 100644 (file)
--- a/lnet/klnds/gmlnd/gmlnd_module.c
+++ b/lnet/klnds/gmlnd/gmlnd_module.c
@@ -22,7 +22,7 @@
 #include "gmnal.h"
 
 
-int gmnal_small_msg_size = sizeof(gmnal_msghdr_t) + sizeof(ptl_hdr_t) + PTL_MTU;
+int gmnal_small_msg_size = GMNAL_MSGHDR_SIZE + sizeof(ptl_hdr_t) + PTL_MTU + 928;
 /*
  *      -1 indicates default value.
  *      This is 1 thread per cpu