From 6815097cbb06aa1a727e6bf7a8ee9e916a33ee6d Mon Sep 17 00:00:00 2001 From: eeb Date: Mon, 7 Mar 2005 11:46:21 +0000 Subject: [PATCH] * vibnal fixes 5807 - incorrect outstanding SEND accounting changed refcounting inlines to macros to improve logging cleanup unused per-work-item debug on send * lctl debug_file wasn't passing the mode argument to open(O_CREAT) fixed with restrictive (owner only) permissions * 5803 ranal initialisation fix --- lnet/klnds/ralnd/ralnd.c | 1 + lnet/klnds/viblnd/viblnd.h | 86 +++++++++++++++++++++---------------------- lnet/klnds/viblnd/viblnd_cb.c | 55 +-------------------------- lnet/utils/debug.c | 2 +- 4 files changed, 46 insertions(+), 98 deletions(-) diff --git a/lnet/klnds/ralnd/ralnd.c b/lnet/klnds/ralnd/ralnd.c index 1aa3556..0da7af4 100644 --- a/lnet/klnds/ralnd/ralnd.c +++ b/lnet/klnds/ralnd/ralnd.c @@ -2057,6 +2057,7 @@ kranal_api_startup (nal_t *nal, ptl_pid_t requested_pid, } for (i = 0; i < kranal_data.kra_ndevs; i++) { + dev = &kranal_data.kra_devices[i]; rc = kranal_thread_start(kranal_scheduler, dev); if (rc != 0) { CERROR("Can't spawn ranal scheduler[%d]: %d\n", diff --git a/lnet/klnds/viblnd/viblnd.h b/lnet/klnds/viblnd/viblnd.h index 7a5e733..1026995 100644 --- a/lnet/klnds/viblnd/viblnd.h +++ b/lnet/klnds/viblnd/viblnd.h @@ -549,50 +549,48 @@ wrq_signals_completion (vv_wr_t *wrq) return wrq->completion_notification != 0; } -static inline void -kibnal_conn_addref (kib_conn_t *conn) -{ - CDEBUG(D_NET, "++conn[%p] (%d)\n", - conn, atomic_read(&conn->ibc_refcount)); - LASSERT(atomic_read(&conn->ibc_refcount) > 0); - atomic_inc(&conn->ibc_refcount); -} - -static inline void -kibnal_conn_decref (kib_conn_t *conn) -{ - unsigned long flags; - - CDEBUG(D_NET, "--conn[%p] (%d)\n", - conn, atomic_read(&conn->ibc_refcount)); - LASSERT(atomic_read(&conn->ibc_refcount) > 0); - if (atomic_dec_and_test(&conn->ibc_refcount)) { - spin_lock_irqsave(&kibnal_data.kib_connd_lock, flags); - list_add_tail(&conn->ibc_list, &kibnal_data.kib_connd_zombies); - wake_up(&kibnal_data.kib_connd_waitq); - spin_unlock_irqrestore(&kibnal_data.kib_connd_lock, flags); - } -} - -static inline void -kibnal_peer_addref (kib_peer_t *peer) -{ - CDEBUG(D_NET, "++peer[%p] -> "LPX64" (%d)\n", - peer, peer->ibp_nid, atomic_read (&peer->ibp_refcount)); - LASSERT(atomic_read(&peer->ibp_refcount) > 0); - atomic_inc(&peer->ibp_refcount); -} - -static inline void -kibnal_peer_decref (kib_peer_t *peer) -{ - CDEBUG(D_NET, "--peer[%p] -> "LPX64" (%d)\n", - peer, peer->ibp_nid, atomic_read (&peer->ibp_refcount)); - - LASSERT(atomic_read(&peer->ibp_refcount) > 0); - if (atomic_dec_and_test (&peer->ibp_refcount)) - kibnal_destroy_peer (peer); -} +#define kibnal_conn_addref(conn) \ +do { \ + CDEBUG(D_NET, "conn[%p] (%d)++\n", \ + (conn), atomic_read(&(conn)->ibc_refcount)); \ + LASSERT(atomic_read(&(conn)->ibc_refcount) > 0); \ + atomic_inc(&(conn)->ibc_refcount); \ +} while (0) + +#define kibnal_conn_decref(conn) \ +do { \ + unsigned long flags; \ + \ + CDEBUG(D_NET, "conn[%p] (%d)--\n", \ + (conn), atomic_read(&(conn)->ibc_refcount)); \ + LASSERT(atomic_read(&(conn)->ibc_refcount) > 0); \ + if (atomic_dec_and_test(&(conn)->ibc_refcount)) { \ + spin_lock_irqsave(&kibnal_data.kib_connd_lock, flags); \ + list_add_tail(&(conn)->ibc_list, \ + &kibnal_data.kib_connd_zombies); \ + wake_up(&kibnal_data.kib_connd_waitq); \ + spin_unlock_irqrestore(&kibnal_data.kib_connd_lock, flags); \ + } \ +} while (0) + +#define kibnal_peer_addref(peer) \ +do { \ + CDEBUG(D_NET, "peer[%p] -> "LPX64" (%d)++\n", \ + (peer), (peer)->ibp_nid, \ + atomic_read (&(peer)->ibp_refcount)); \ + LASSERT(atomic_read(&(peer)->ibp_refcount) > 0); \ + atomic_inc(&(peer)->ibp_refcount); \ +} while (0) + +#define kibnal_peer_decref(peer) \ +do { \ + CDEBUG(D_NET, "peer[%p] -> "LPX64" (%d)--\n", \ + (peer), (peer)->ibp_nid, \ + atomic_read (&(peer)->ibp_refcount)); \ + LASSERT(atomic_read(&(peer)->ibp_refcount) > 0); \ + if (atomic_dec_and_test(&(peer)->ibp_refcount)) \ + kibnal_destroy_peer(peer); \ +} while (0) static inline struct list_head * kibnal_nid2peerlist (ptl_nid_t nid) diff --git a/lnet/klnds/viblnd/viblnd_cb.c b/lnet/klnds/viblnd/viblnd_cb.c index 3bb4a1d..477d190 100644 --- a/lnet/klnds/viblnd/viblnd_cb.c +++ b/lnet/klnds/viblnd/viblnd_cb.c @@ -950,7 +950,7 @@ kibnal_check_sends (kib_conn_t *conn) * stashed on ibc_active_txs, matched by an incoming PUT_ACK, * and then re-queued here. It's (just) possible that * tx_sending is non-zero if we've not done the tx_complete() from - * the first send; hence the += rather than = below. */ + * the first send; hence the ++ rather than = below. */ tx->tx_sending++; list_add (&tx->tx_list, &conn->ibc_active_txs); @@ -965,61 +965,12 @@ kibnal_check_sends (kib_conn_t *conn) vvrc = vv_return_ok; if (conn->ibc_state == IBNAL_CONN_ESTABLISHED) { tx->tx_status = 0; -#if 1 vvrc = vv_post_send_list(kibnal_data.kib_hca, conn->ibc_qp, tx->tx_nwrq, tx->tx_wrq, vv_operation_type_send_rc); rc = (vvrc == vv_return_ok) ? 0 : -EIO; -#else - /* Only post 1 item at a time for now (so we know - * exactly how many got posted successfully) */ - for (i = 0; i < tx->tx_nwrq; i++) { - switch (tx->tx_wrq[i].wr_type) { - case vv_wr_send: - CDEBUG(D_NET, "[%d]posting send [%d %x %p]%s: %x\n", - i, - tx->tx_wrq[i].scatgat_list->length, - tx->tx_wrq[i].scatgat_list->l_key, - tx->tx_wrq[i].scatgat_list->v_address, - tx->tx_wrq[i].type.send.send_qp_type.rc_type.fance_indicator ? - "(fence)":"", - tx->tx_msg->ibm_type); - break; - case vv_wr_rdma_write: - CDEBUG(D_NET, "[%d]posting PUT [%d %x %p]->[%x "LPX64"]\n", - i, - tx->tx_wrq[i].scatgat_list->length, - tx->tx_wrq[i].scatgat_list->l_key, - tx->tx_wrq[i].scatgat_list->v_address, - tx->tx_wrq[i].type.send.send_qp_type.rc_type.r_r_key, - tx->tx_wrq[i].type.send.send_qp_type.rc_type.r_addr); - break; - case vv_wr_rdma_read: - CDEBUG(D_NET, "[%d]posting GET [%d %x %p]->[%x "LPX64"]\n", - i, - tx->tx_wrq[i].scatgat_list->length, - tx->tx_wrq[i].scatgat_list->l_key, - tx->tx_wrq[i].scatgat_list->v_address, - tx->tx_wrq[i].type.send.send_qp_type.rc_type.r_r_key, - tx->tx_wrq[i].type.send.send_qp_type.rc_type.r_addr); - break; - default: - LBUG(); - } - vvrc = vv_post_send(kibnal_data.kib_hca, - conn->ibc_qp, - &tx->tx_wrq[i], - vv_operation_type_send_rc); - CDEBUG(D_NET, LPX64": post %d/%d\n", - conn->ibc_peer->ibp_nid, i, tx->tx_nwrq); - if (vvrc != vv_return_ok) { - rc = -EIO; - break; - } - } -#endif } if (rc != 0) { @@ -1081,6 +1032,7 @@ kibnal_tx_complete (kib_tx_t *tx, vv_comp_status_t vvrc) * gets to free it, which also drops its ref on 'conn'. */ tx->tx_sending--; + conn->ibc_nsends_posted--; if (failed) { tx->tx_waiting = 0; @@ -1094,9 +1046,6 @@ kibnal_tx_complete (kib_tx_t *tx, vv_comp_status_t vvrc) kibnal_conn_addref(conn); /* 1 ref for me.... */ - if (tx->tx_sending == 0) - conn->ibc_nsends_posted--; - spin_unlock(&conn->ibc_lock); if (idle) diff --git a/lnet/utils/debug.c b/lnet/utils/debug.c index 7b0ca54..f8d94ed 100644 --- a/lnet/utils/debug.c +++ b/lnet/utils/debug.c @@ -467,7 +467,7 @@ int jt_dbg_debug_file(int argc, char **argv) return 1; } if (argc > 2) { - fdout = open(argv[2], O_CREAT | O_WRONLY | O_LARGEFILE); + fdout = open(argv[2], O_CREAT | O_WRONLY | O_LARGEFILE, 0600); if (fdout == -1) { fprintf(stderr, "open(%s) failed: %s\n", argv[2], strerror(errno)); -- 1.8.3.1