* gmnal: removed spinlock round GM API calls that don't need to be serialised

    (they'd been added "blindly" previously)

*   gmnal netregression ran for 36 hours.

diff --git a/lnet/klnds/gmlnd/gmlnd.h b/lnet/klnds/gmlnd/gmlnd.h
index e2dad13..4c7b1b8 100644 (file)
--- a/lnet/klnds/gmlnd/gmlnd.h
+++ b/lnet/klnds/gmlnd/gmlnd.h
@@ -243,7 +243,6 @@ void gmnal_post_rx(gmnal_ni_t *gmnalni, gmnal_rx_t *rx);
 ptl_err_t gmnal_post_tx(gmnal_ni_t *gmnalni, gmnal_tx_t *tx, 
                         lib_msg_t *libmsg, ptl_nid_t nid, int nob);
 
-
 /* Module Parameters */
 extern  int num_rx_threads;
 extern  int num_txds;

diff --git a/lnet/klnds/gmlnd/gmlnd_api.c b/lnet/klnds/gmlnd/gmlnd_api.c
index c721ddd..18e4976 100644 (file)
--- a/lnet/klnds/gmlnd/gmlnd_api.c
+++ b/lnet/klnds/gmlnd/gmlnd_api.c
@@ -44,10 +44,8 @@ gmnal_cmd(struct portals_cfg *pcfg, void *private)
                PORTAL_ALLOC(name, pcfg->pcfg_plen1);
                copy_from_user(name, PCFG_PBUF(pcfg, 1), pcfg->pcfg_plen1);
 
-               spin_lock(&gmnalni->gmni_gm_lock);
                 gm_status = gm_host_name_to_node_id_ex(gmnalni->gmni_port, 0,
                                                        name, &nid);
-               spin_unlock(&gmnalni->gmni_gm_lock);
                 if (gm_status != GM_SUCCESS) {
                         CDEBUG(D_NET, "gm_host_name_to_node_id_ex(...host %s) "
                                "failed[%d]\n", name, gm_status);
@@ -55,10 +53,8 @@ gmnal_cmd(struct portals_cfg *pcfg, void *private)
                 }
 
                 CDEBUG(D_NET, "Local node %s id is [%d]\n", name, nid);
-               spin_lock(&gmnalni->gmni_gm_lock);
                gm_status = gm_node_id_to_global_id(gmnalni->gmni_port,
                                                    nid, &gmid);
-               spin_unlock(&gmnalni->gmni_gm_lock);
                if (gm_status != GM_SUCCESS) {
                        CDEBUG(D_NET, "gm_node_id_to_global_id failed[%d]\n",
                               gm_status);
@@ -95,20 +91,15 @@ gmnal_get_local_nid (gmnal_ni_t *gmnalni)
         gm_status_t      gm_status;
 
         /* Called before anything initialised: no need to lock */
-
-       spin_lock(&gmnalni->gmni_gm_lock);
        gm_status = gm_get_node_id(gmnalni->gmni_port, &local_gmid);
-       spin_unlock(&gmnalni->gmni_gm_lock);
        if (gm_status != GM_SUCCESS)
                return PTL_NID_ANY;
 
        CDEBUG(D_NET, "Local node id is [%u]\n", local_gmid);
         
-       spin_lock(&gmnalni->gmni_gm_lock);
        gm_status = gm_node_id_to_global_id(gmnalni->gmni_port, 
                                             local_gmid, 
                                            &global_gmid);
-       spin_unlock(&gmnalni->gmni_gm_lock);
        if (gm_status != GM_SUCCESS)
                return PTL_NID_ANY;
         
@@ -142,10 +133,8 @@ gmnal_api_shutdown(nal_t *nal)
        gmnal_stop_ctthread(gmnalni);
        gmnal_stop_rxthread(gmnalni);
 
-       spin_lock(&gmnalni->gmni_gm_lock);
        gm_close(gmnalni->gmni_port);
        gm_finalize();
-       spin_unlock(&gmnalni->gmni_gm_lock);
 
         lib_fini(libnal);
 
@@ -219,10 +208,8 @@ gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid,
               "name [%s], version [%d]\n", gm_port_id,
               "gmnal", GM_API_VERSION);
 
-       spin_lock(&gmnalni->gmni_gm_lock);
        gm_status = gm_open(&gmnalni->gmni_port, 0, gm_port_id, "gmnal",
                            GM_API_VERSION);
-       spin_unlock(&gmnalni->gmni_gm_lock);
 
         if (gm_status != GM_SUCCESS) {
                 CERROR("Can't open GM port %d: %d (%s)\n",
@@ -288,14 +275,10 @@ gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid,
        gmnal_stop_ctthread(gmnalni);
 
  failed_3:
-        spin_lock(&gmnalni->gmni_gm_lock);
         gm_close(gmnalni->gmni_port);
-        spin_unlock(&gmnalni->gmni_gm_lock);
 
  failed_2:
-        spin_lock(&gmnalni->gmni_gm_lock);
         gm_finalize();
-        spin_unlock(&gmnalni->gmni_gm_lock);
 
         /* safe to free buffers after network has been shut down */
         gmnal_free_txs(gmnalni);

diff --git a/lnet/klnds/gmlnd/gmlnd_comm.c b/lnet/klnds/gmlnd/gmlnd_comm.c
index b5213a5..2b77ea3 100644 (file)
--- a/lnet/klnds/gmlnd/gmlnd_comm.c
+++ b/lnet/klnds/gmlnd/gmlnd_comm.c
@@ -257,9 +257,11 @@ gmnal_drop_sends_callback(struct gm_port *gm_port, void *context,
         CERROR("status for tx [%p] is [%d][%s]\n", 
                tx, status, gmnal_gmstatus2str(status));
 
+        spin_lock(&gmnalni->gmni_gm_lock);
         gm_resume_sending(gmnalni->gmni_port, tx->tx_gm_priority,
                           tx->tx_gmlid, gm_port_id,
                           gmnal_resume_sending_callback, tx);
+        spin_unlock(&gmnalni->gmni_gm_lock);
 }
 
 void 
@@ -312,11 +314,8 @@ gmnal_post_tx (gmnal_ni_t *gmnalni, gmnal_tx_t *tx,
 
         LASSERT ((nid >> 32) == 0);
 
-       spin_lock(&gmnalni->gmni_gm_lock);
        gm_status = gm_global_id_to_node_id(gmnalni->gmni_port, (__u32)nid, 
                                             &tx->tx_gmlid);
-       spin_unlock(&gmnalni->gmni_gm_lock);
-
        if (gm_status != GM_SUCCESS) {
                CERROR("Failed to obtain local id\n");
                 gmnal_return_tx(gmnalni, tx);

diff --git a/lnet/klnds/gmlnd/gmlnd_utils.c b/lnet/klnds/gmlnd/gmlnd_utils.c
index 4a8ee6c..b4a1a99 100644 (file)
--- a/lnet/klnds/gmlnd/gmlnd_utils.c
+++ b/lnet/klnds/gmlnd/gmlnd_utils.c
@@ -50,10 +50,7 @@ gmnal_alloc_tx (gmnal_ni_t *gmnalni)
                 return NULL;
         }
         
-        spin_lock(&gmnalni->gmni_gm_lock);
-        buffer = gm_dma_malloc(gmnalni->gmni_port,
-                                 gmnalni->gmni_msg_size);
-        spin_unlock(&gmnalni->gmni_gm_lock);
+        buffer = gm_dma_malloc(gmnalni->gmni_port, gmnalni->gmni_msg_size);
         if (buffer == NULL) {
                 CERROR("Failed to gm_dma_malloc tx buffer size [%d]\n", 
                        gmnalni->gmni_msg_size);
@@ -82,9 +79,7 @@ gmnal_free_tx (gmnal_tx_t *tx)
                tx, tx->tx_msg, tx->tx_buffer_size);
 #if 0
         /* We free buffers after we've closed the GM port */
-        spin_lock(&gmnalni->gmni_gm_lock);
         gm_dma_free(gmnalni->gmni_port, tx->tx_msg);
-        spin_unlock(&gmnalni->gmni_gm_lock);
 #endif
         PORTAL_FREE(tx, sizeof(*tx));
 }
@@ -92,20 +87,13 @@ gmnal_free_tx (gmnal_tx_t *tx)
 int
 gmnal_alloc_txs(gmnal_ni_t *gmnalni)
 {
+       int           ntxcred = gm_num_send_tokens(gmnalni->gmni_port);
        int           ntx;
-        int           ntxcred;
         int           nrxt_tx;
         int           i;
        gmnal_tx_t   *tx;
 
-       CDEBUG(D_TRACE, "gmnal_alloc_small tx\n");
-
-       /* get total number of transmit tokens */
-       spin_lock(&gmnalni->gmni_gm_lock);
-       ntxcred = gm_num_send_tokens(gmnalni->gmni_port);
-       spin_unlock(&gmnalni->gmni_gm_lock);
-       CDEBUG(D_NET, "total number of send tokens available is [%d]\n", 
-               ntxcred);
+        CWARN("ntxcred: %d\n", ntxcred);
 
        ntx = num_txds;
         nrxt_tx = num_txds + 1;
@@ -265,19 +253,13 @@ gmnal_return_tx(gmnal_ni_t *gmnalni, gmnal_tx_t *tx)
 int
 gmnal_alloc_rxs (gmnal_ni_t *gmnalni)
 {
-        int          nrxcred;
+        int          nrxcred = gm_num_receive_tokens(gmnalni->gmni_port);
         int          nrx;
         int          i;
        gmnal_rx_t  *rxd;
        void        *rxbuffer;
 
-       CDEBUG(D_TRACE, "gmnal_alloc_small rx\n");
-
-       spin_lock(&gmnalni->gmni_gm_lock);
-       nrxcred = gm_num_receive_tokens(gmnalni->gmni_port);
-       spin_unlock(&gmnalni->gmni_gm_lock);
-       CDEBUG(D_NET, "total number of receive tokens available is [%d]\n",
-              nrxcred);
+        CWARN("nrxcred: %d\n", nrxcred);
 
        nrx = num_txds*2 + 2;
         if (nrx > nrxcred) {
@@ -288,10 +270,8 @@ gmnal_alloc_rxs (gmnal_ni_t *gmnalni)
 
        CDEBUG(D_NET, "Allocated [%d] receive tokens to small messages\n", nrx);
 
-       spin_lock(&gmnalni->gmni_gm_lock);
        gmnalni->gmni_rx_hash = gm_create_hash(gm_hash_compare_ptrs, 
-                                                  gm_hash_hash_ptr, 0, 0, nrx, 0);
-       spin_unlock(&gmnalni->gmni_gm_lock);
+                                               gm_hash_hash_ptr, 0, 0, nrx, 0);
        if (gmnalni->gmni_rx_hash == NULL) {
                 CERROR("Failed to create hash table\n");
                 return -ENOMEM;
@@ -307,10 +287,8 @@ gmnal_alloc_rxs (gmnal_ni_t *gmnalni)
                        return -ENOMEM;
                }
 
-               spin_lock(&gmnalni->gmni_gm_lock);
                rxbuffer = gm_dma_malloc(gmnalni->gmni_port, 
                                         gmnalni->gmni_msg_size);
-               spin_unlock(&gmnalni->gmni_gm_lock);
                if (rxbuffer == NULL) {
                        CERROR("Failed to gm_dma_malloc rxbuffer [%d], "
                               "size [%d]\n",i ,gmnalni->gmni_msg_size);
@@ -353,20 +331,15 @@ gmnal_free_rxs(gmnal_ni_t *gmnalni)
                       rx, rx->rx_msg, rx->rx_size);
 #if 0
                 /* We free buffers after we've shutdown the GM port */
-               spin_lock(&gmnalni->gmni_gm_lock);
                gm_dma_free(gmnalni->gmni_port, _rxd->rx_msg);
-               spin_unlock(&gmnalni->gmni_gm_lock);
 #endif
                PORTAL_FREE(rx, sizeof(*rx));
        }
 
 #if 0
         /* see above */
-        if (gmnalni->gmni_rx_hash != NULL) {
-                spin_lock(&gmnalni->gmni_gm_lock);
+        if (gmnalni->gmni_rx_hash != NULL)
                 gm_destroy_hash(gmnalni->gmni_rx_hash);
-                spin_unlock(&gmnalni->gmni_gm_lock);
-        }
 #endif
 }
 
@@ -755,9 +728,11 @@ gmnal_yield(int delay)
 int
 gmnal_enqueue_rx(gmnal_ni_t *gmnalni, gm_recv_t *recv)
 {
-        void         *ptr = gm_ntohp(recv->buffer);
+        void       *ptr = gm_ntohp(recv->buffer);
         gmnal_rx_t *rx = gm_hash_find(gmnalni->gmni_rx_hash, ptr);
 
+        /* No locking; hash is read-only */
+
        LASSERT (rx != NULL);
         LASSERT (rx->rx_msg == (gmnal_msg_t *)ptr);