X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lnet%2Flnet%2Flib-move.c;h=17a62c8dcfedfaf94bc80b0d93b4782ba4f43f2a;hp=ec03257203645485db133089899ea0de74759662;hb=88f761bc00c7fb29db4f80594ae864493bdd5071;hpb=580c1e0017296ea7a25f6f5f8aa8022f713ae762 diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index ec03257..17a62c8 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -27,7 +27,7 @@ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2012, 2013, Intel Corporation. + * Copyright (c) 2012, 2015, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -47,14 +47,12 @@ CFS_MODULE_PARM(local_nid_dist_zero, "i", int, 0444, "Reserved"); int -lnet_fail_nid (lnet_nid_t nid, unsigned int threshold) +lnet_fail_nid(lnet_nid_t nid, unsigned int threshold) { - lnet_test_peer_t *tp; - cfs_list_t *el; - cfs_list_t *next; - cfs_list_t cull; - - LASSERT (the_lnet.ln_init); + lnet_test_peer_t *tp; + struct list_head *el; + struct list_head *next; + struct list_head cull; /* NB: use lnet_net_lock(0) to serialize operations on test peers */ if (threshold != 0) { @@ -67,99 +65,98 @@ lnet_fail_nid (lnet_nid_t nid, unsigned int threshold) tp->tp_threshold = threshold; lnet_net_lock(0); - cfs_list_add_tail(&tp->tp_list, &the_lnet.ln_test_peers); + list_add_tail(&tp->tp_list, &the_lnet.ln_test_peers); lnet_net_unlock(0); return 0; } /* removing entries */ - CFS_INIT_LIST_HEAD(&cull); + INIT_LIST_HEAD(&cull); lnet_net_lock(0); - cfs_list_for_each_safe (el, next, &the_lnet.ln_test_peers) { - tp = cfs_list_entry (el, lnet_test_peer_t, tp_list); + list_for_each_safe(el, next, &the_lnet.ln_test_peers) { + tp = list_entry(el, lnet_test_peer_t, tp_list); - if (tp->tp_threshold == 0 || /* needs culling anyway */ - nid == LNET_NID_ANY || /* removing all entries */ - tp->tp_nid == nid) /* matched this one */ - { - cfs_list_del (&tp->tp_list); - cfs_list_add (&tp->tp_list, &cull); - } - } + if (tp->tp_threshold == 0 || /* needs culling anyway */ + nid == LNET_NID_ANY || /* removing all entries */ + tp->tp_nid == nid) { /* matched this one */ + list_del(&tp->tp_list); + list_add(&tp->tp_list, &cull); + } + } lnet_net_unlock(0); - while (!cfs_list_empty (&cull)) { - tp = cfs_list_entry (cull.next, lnet_test_peer_t, tp_list); + while (!list_empty(&cull)) { + tp = list_entry(cull.next, lnet_test_peer_t, tp_list); - cfs_list_del (&tp->tp_list); - LIBCFS_FREE(tp, sizeof (*tp)); - } - return 0; + list_del(&tp->tp_list); + LIBCFS_FREE(tp, sizeof(*tp)); + } + return 0; } static int fail_peer (lnet_nid_t nid, int outgoing) { - lnet_test_peer_t *tp; - cfs_list_t *el; - cfs_list_t *next; - cfs_list_t cull; - int fail = 0; + lnet_test_peer_t *tp; + struct list_head *el; + struct list_head *next; + struct list_head cull; + int fail = 0; - CFS_INIT_LIST_HEAD (&cull); + INIT_LIST_HEAD(&cull); /* NB: use lnet_net_lock(0) to serialize operations on test peers */ lnet_net_lock(0); - cfs_list_for_each_safe (el, next, &the_lnet.ln_test_peers) { - tp = cfs_list_entry (el, lnet_test_peer_t, tp_list); - - if (tp->tp_threshold == 0) { - /* zombie entry */ - if (outgoing) { - /* only cull zombies on outgoing tests, - * since we may be at interrupt priority on - * incoming messages. */ - cfs_list_del (&tp->tp_list); - cfs_list_add (&tp->tp_list, &cull); - } - continue; - } + list_for_each_safe(el, next, &the_lnet.ln_test_peers) { + tp = list_entry(el, lnet_test_peer_t, tp_list); + + if (tp->tp_threshold == 0) { + /* zombie entry */ + if (outgoing) { + /* only cull zombies on outgoing tests, + * since we may be at interrupt priority on + * incoming messages. */ + list_del(&tp->tp_list); + list_add(&tp->tp_list, &cull); + } + continue; + } - if (tp->tp_nid == LNET_NID_ANY || /* fail every peer */ - nid == tp->tp_nid) { /* fail this peer */ - fail = 1; - - if (tp->tp_threshold != LNET_MD_THRESH_INF) { - tp->tp_threshold--; - if (outgoing && - tp->tp_threshold == 0) { - /* see above */ - cfs_list_del (&tp->tp_list); - cfs_list_add (&tp->tp_list, &cull); - } - } - break; - } - } + if (tp->tp_nid == LNET_NID_ANY || /* fail every peer */ + nid == tp->tp_nid) { /* fail this peer */ + fail = 1; + + if (tp->tp_threshold != LNET_MD_THRESH_INF) { + tp->tp_threshold--; + if (outgoing && + tp->tp_threshold == 0) { + /* see above */ + list_del(&tp->tp_list); + list_add(&tp->tp_list, &cull); + } + } + break; + } + } lnet_net_unlock(0); - while (!cfs_list_empty (&cull)) { - tp = cfs_list_entry (cull.next, lnet_test_peer_t, tp_list); - cfs_list_del (&tp->tp_list); + while (!list_empty(&cull)) { + tp = list_entry(cull.next, lnet_test_peer_t, tp_list); + list_del(&tp->tp_list); - LIBCFS_FREE(tp, sizeof (*tp)); - } + LIBCFS_FREE(tp, sizeof(*tp)); + } - return (fail); + return fail; } unsigned int -lnet_iov_nob (unsigned int niov, struct iovec *iov) +lnet_iov_nob(unsigned int niov, struct kvec *iov) { unsigned int nob = 0; @@ -172,9 +169,9 @@ lnet_iov_nob (unsigned int niov, struct iovec *iov) EXPORT_SYMBOL(lnet_iov_nob); void -lnet_copy_iov2iov (unsigned int ndiov, struct iovec *diov, unsigned int doffset, - unsigned int nsiov, struct iovec *siov, unsigned int soffset, - unsigned int nob) +lnet_copy_iov2iov(unsigned int ndiov, struct kvec *diov, unsigned int doffset, + unsigned int nsiov, struct kvec *siov, unsigned int soffset, + unsigned int nob) { /* NB diov, siov are READ-ONLY */ unsigned int this_nob; @@ -231,9 +228,9 @@ lnet_copy_iov2iov (unsigned int ndiov, struct iovec *diov, unsigned int doffset, EXPORT_SYMBOL(lnet_copy_iov2iov); int -lnet_extract_iov (int dst_niov, struct iovec *dst, - int src_niov, struct iovec *src, - unsigned int offset, unsigned int len) +lnet_extract_iov(int dst_niov, struct kvec *dst, + int src_niov, struct kvec *src, + unsigned int offset, unsigned int len) { /* Initialise 'dst' to the subset of 'src' starting at 'offset', * for exactly 'len' bytes, and return the number of entries. @@ -277,50 +274,9 @@ lnet_extract_iov (int dst_niov, struct iovec *dst, } EXPORT_SYMBOL(lnet_extract_iov); -#ifndef __KERNEL__ -unsigned int -lnet_kiov_nob (unsigned int niov, lnet_kiov_t *kiov) -{ - LASSERT (0); - return (0); -} - -void -lnet_copy_kiov2kiov (unsigned int ndkiov, lnet_kiov_t *dkiov, unsigned int doffset, - unsigned int nskiov, lnet_kiov_t *skiov, unsigned int soffset, - unsigned int nob) -{ - LASSERT (0); -} - -void -lnet_copy_kiov2iov (unsigned int niov, struct iovec *iov, unsigned int iovoffset, - unsigned int nkiov, lnet_kiov_t *kiov, unsigned int kiovoffset, - unsigned int nob) -{ - LASSERT (0); -} - -void -lnet_copy_iov2kiov (unsigned int nkiov, lnet_kiov_t *kiov, unsigned int kiovoffset, - unsigned int niov, struct iovec *iov, unsigned int iovoffset, - unsigned int nob) -{ - LASSERT (0); -} - -int -lnet_extract_kiov (int dst_niov, lnet_kiov_t *dst, - int src_niov, lnet_kiov_t *src, - unsigned int offset, unsigned int len) -{ - LASSERT (0); -} - -#else /* __KERNEL__ */ unsigned int -lnet_kiov_nob (unsigned int niov, lnet_kiov_t *kiov) +lnet_kiov_nob(unsigned int niov, lnet_kiov_t *kiov) { unsigned int nob = 0; @@ -333,9 +289,9 @@ lnet_kiov_nob (unsigned int niov, lnet_kiov_t *kiov) EXPORT_SYMBOL(lnet_kiov_nob); void -lnet_copy_kiov2kiov (unsigned int ndiov, lnet_kiov_t *diov, unsigned int doffset, - unsigned int nsiov, lnet_kiov_t *siov, unsigned int soffset, - unsigned int nob) +lnet_copy_kiov2kiov(unsigned int ndiov, lnet_kiov_t *diov, unsigned int doffset, + unsigned int nsiov, lnet_kiov_t *siov, unsigned int soffset, + unsigned int nob) { /* NB diov, siov are READ-ONLY */ unsigned int this_nob; @@ -415,7 +371,7 @@ lnet_copy_kiov2kiov (unsigned int ndiov, lnet_kiov_t *diov, unsigned int doffset EXPORT_SYMBOL(lnet_copy_kiov2kiov); void -lnet_copy_kiov2iov (unsigned int niov, struct iovec *iov, unsigned int iovoffset, +lnet_copy_kiov2iov (unsigned int niov, struct kvec *iov, unsigned int iovoffset, unsigned int nkiov, lnet_kiov_t *kiov, unsigned int kiovoffset, unsigned int nob) { @@ -485,9 +441,9 @@ lnet_copy_kiov2iov (unsigned int niov, struct iovec *iov, unsigned int iovoffset EXPORT_SYMBOL(lnet_copy_kiov2iov); void -lnet_copy_iov2kiov (unsigned int nkiov, lnet_kiov_t *kiov, unsigned int kiovoffset, - unsigned int niov, struct iovec *iov, unsigned int iovoffset, - unsigned int nob) +lnet_copy_iov2kiov(unsigned int nkiov, lnet_kiov_t *kiov, unsigned int kiovoffset, + unsigned int niov, struct kvec *iov, unsigned int iovoffset, + unsigned int nob) { /* NB kiov, iov are READ-ONLY */ unsigned int this_nob; @@ -554,9 +510,9 @@ lnet_copy_iov2kiov (unsigned int nkiov, lnet_kiov_t *kiov, unsigned int kiovoffs EXPORT_SYMBOL(lnet_copy_iov2kiov); int -lnet_extract_kiov (int dst_niov, lnet_kiov_t *dst, - int src_niov, lnet_kiov_t *src, - unsigned int offset, unsigned int len) +lnet_extract_kiov(int dst_niov, lnet_kiov_t *dst, + int src_niov, lnet_kiov_t *src, + unsigned int offset, unsigned int len) { /* Initialise 'dst' to the subset of 'src' starting at 'offset', * for exactly 'len' bytes, and return the number of entries. @@ -602,14 +558,13 @@ lnet_extract_kiov (int dst_niov, lnet_kiov_t *dst, } } EXPORT_SYMBOL(lnet_extract_kiov); -#endif void lnet_ni_recv(lnet_ni_t *ni, void *private, lnet_msg_t *msg, int delayed, unsigned int offset, unsigned int mlen, unsigned int rlen) { unsigned int niov = 0; - struct iovec *iov = NULL; + struct kvec *iov = NULL; lnet_kiov_t *kiov = NULL; int rc; @@ -642,7 +597,7 @@ lnet_ni_recv(lnet_ni_t *ni, void *private, lnet_msg_t *msg, int delayed, lnet_finalize(ni, msg, rc); } -void +static void lnet_setpayloadbuffer(lnet_msg_t *msg) { lnet_libmd_t *md = msg->msg_md; @@ -663,7 +618,7 @@ lnet_setpayloadbuffer(lnet_msg_t *msg) void lnet_prep_send(lnet_msg_t *msg, int type, lnet_process_id_t target, - unsigned int offset, unsigned int len) + unsigned int offset, unsigned int len) { msg->msg_type = type; msg->msg_target = target; @@ -682,7 +637,7 @@ lnet_prep_send(lnet_msg_t *msg, int type, lnet_process_id_t target, msg->msg_hdr.payload_length = cpu_to_le32(len); } -void +static void lnet_ni_send(lnet_ni_t *ni, lnet_msg_t *msg) { void *priv = msg->msg_private; @@ -697,7 +652,7 @@ lnet_ni_send(lnet_ni_t *ni, lnet_msg_t *msg) lnet_finalize(ni, msg, rc); } -int +static int lnet_ni_eager_recv(lnet_ni_t *ni, lnet_msg_t *msg) { int rc; @@ -722,7 +677,7 @@ lnet_ni_eager_recv(lnet_ni_t *ni, lnet_msg_t *msg) } /* NB: caller shall hold a ref on 'lp' as I'd drop lnet_net_lock */ -void +static void lnet_ni_query_locked(lnet_ni_t *ni, lnet_peer_t *lp) { cfs_time_t last_alive = 0; @@ -774,7 +729,7 @@ lnet_peer_is_alive (lnet_peer_t *lp, cfs_time_t now) /* NB: returns 1 when alive, 0 when dead, negative when error; * may drop the lnet_net_lock */ -int +static int lnet_peer_alive_locked (lnet_peer_t *lp) { cfs_time_t now = cfs_time_current(); @@ -816,26 +771,30 @@ lnet_peer_alive_locked (lnet_peer_t *lp) return 0; } -int +/** + * \param msg The message to be sent. + * \param do_send True if lnet_ni_send() should be called in this function. + * lnet_send() is going to lnet_net_unlock immediately after this, so + * it sets do_send FALSE and I don't do the unlock/send/lock bit. + * + * \retval LNET_CREDIT_OK If \a msg sent or OK to send. + * \retval LNET_CREDIT_WAIT If \a msg blocked for credit. + * \retval -EHOSTUNREACH If the next hop of the message appears dead. + * \retval -ECANCELED If the MD of the message has been unlinked. + */ +static int lnet_post_send_locked(lnet_msg_t *msg, int do_send) { - /* lnet_send is going to lnet_net_unlock immediately after this, - * so it sets do_send FALSE and I don't do the unlock/send/lock bit. - * I return EAGAIN if msg blocked, EHOSTUNREACH if msg_txpeer - * appears dead, and 0 if sent or OK to send */ - struct lnet_peer *lp = msg->msg_txpeer; - struct lnet_ni *ni = lp->lp_ni; - struct lnet_tx_queue *tq; - int cpt; + lnet_peer_t *lp = msg->msg_txpeer; + lnet_ni_t *ni = lp->lp_ni; + int cpt = msg->msg_tx_cpt; + struct lnet_tx_queue *tq = ni->ni_tx_queues[cpt]; /* non-lnet_send() callers have checked before */ LASSERT(!do_send || msg->msg_tx_delayed); LASSERT(!msg->msg_receiving); LASSERT(msg->msg_tx_committed); - cpt = msg->msg_tx_cpt; - tq = ni->ni_tx_queues[cpt]; - /* NB 'lp' is always the next hop */ if ((msg->msg_target.pid & LNET_PID_USERFLAG) == 0 && lnet_peer_alive_locked(lp) == 0) { @@ -849,30 +808,44 @@ lnet_post_send_locked(lnet_msg_t *msg, int do_send) lnet_finalize(ni, msg, -EHOSTUNREACH); lnet_net_lock(cpt); - return EHOSTUNREACH; + return -EHOSTUNREACH; } - if (!msg->msg_peertxcredit) { - LASSERT ((lp->lp_txcredits < 0) == - !cfs_list_empty(&lp->lp_txq)); + if (msg->msg_md != NULL && + (msg->msg_md->md_flags & LNET_MD_FLAG_ABORTED) != 0) { + lnet_net_unlock(cpt); + + CNETERR("Aborting message for %s: LNetM[DE]Unlink() already " + "called on the MD/ME.\n", + libcfs_id2str(msg->msg_target)); + if (do_send) + lnet_finalize(ni, msg, -ECANCELED); - msg->msg_peertxcredit = 1; - lp->lp_txqnob += msg->msg_len + sizeof(lnet_hdr_t); - lp->lp_txcredits--; + lnet_net_lock(cpt); + return -ECANCELED; + } - if (lp->lp_txcredits < lp->lp_mintxcredits) - lp->lp_mintxcredits = lp->lp_txcredits; + if (!msg->msg_peertxcredit) { + LASSERT((lp->lp_txcredits < 0) == + !list_empty(&lp->lp_txq)); - if (lp->lp_txcredits < 0) { + msg->msg_peertxcredit = 1; + lp->lp_txqnob += msg->msg_len + sizeof(lnet_hdr_t); + lp->lp_txcredits--; + + if (lp->lp_txcredits < lp->lp_mintxcredits) + lp->lp_mintxcredits = lp->lp_txcredits; + + if (lp->lp_txcredits < 0) { msg->msg_tx_delayed = 1; - cfs_list_add_tail(&msg->msg_list, &lp->lp_txq); - return EAGAIN; - } - } + list_add_tail(&msg->msg_list, &lp->lp_txq); + return LNET_CREDIT_WAIT; + } + } - if (!msg->msg_txcredit) { + if (!msg->msg_txcredit) { LASSERT((tq->tq_credits < 0) == - !cfs_list_empty(&tq->tq_delayed)); + !list_empty(&tq->tq_delayed)); msg->msg_txcredit = 1; tq->tq_credits--; @@ -882,8 +855,8 @@ lnet_post_send_locked(lnet_msg_t *msg, int do_send) if (tq->tq_credits < 0) { msg->msg_tx_delayed = 1; - cfs_list_add_tail(&msg->msg_list, &tq->tq_delayed); - return EAGAIN; + list_add_tail(&msg->msg_list, &tq->tq_delayed); + return LNET_CREDIT_WAIT; } } @@ -892,12 +865,11 @@ lnet_post_send_locked(lnet_msg_t *msg, int do_send) lnet_ni_send(ni, msg); lnet_net_lock(cpt); } - return 0; + return LNET_CREDIT_OK; } -#ifdef __KERNEL__ -lnet_rtrbufpool_t * +static lnet_rtrbufpool_t * lnet_msg2bufpool(lnet_msg_t *msg) { lnet_rtrbufpool_t *rbp; @@ -917,12 +889,13 @@ lnet_msg2bufpool(lnet_msg_t *msg) return rbp; } -int +static int lnet_post_routed_recv_locked (lnet_msg_t *msg, int do_recv) { /* lnet_parse is going to lnet_net_unlock immediately after this, so it - * sets do_recv FALSE and I don't do the unlock/send/lock bit. I - * return EAGAIN if msg blocked and 0 if received or OK to receive */ + * sets do_recv FALSE and I don't do the unlock/send/lock bit. + * I return LNET_CREDIT_WAIT if msg blocked and LNET_CREDIT_OK if + * received or OK to receive */ lnet_peer_t *lp = msg->msg_rxpeer; lnet_rtrbufpool_t *rbp; lnet_rtrbuf_t *rb; @@ -937,47 +910,44 @@ lnet_post_routed_recv_locked (lnet_msg_t *msg, int do_recv) /* non-lnet_parse callers only receive delayed messages */ LASSERT(!do_recv || msg->msg_rx_delayed); - if (!msg->msg_peerrtrcredit) { - LASSERT ((lp->lp_rtrcredits < 0) == - !cfs_list_empty(&lp->lp_rtrq)); + if (!msg->msg_peerrtrcredit) { + LASSERT((lp->lp_rtrcredits < 0) == + !list_empty(&lp->lp_rtrq)); msg->msg_peerrtrcredit = 1; lp->lp_rtrcredits--; if (lp->lp_rtrcredits < lp->lp_minrtrcredits) lp->lp_minrtrcredits = lp->lp_rtrcredits; - if (lp->lp_rtrcredits < 0) { - /* must have checked eager_recv before here */ + if (lp->lp_rtrcredits < 0) { + /* must have checked eager_recv before here */ LASSERT(msg->msg_rx_ready_delay); msg->msg_rx_delayed = 1; - cfs_list_add_tail(&msg->msg_list, &lp->lp_rtrq); - return EAGAIN; - } - } - - rbp = lnet_msg2bufpool(msg); + list_add_tail(&msg->msg_list, &lp->lp_rtrq); + return LNET_CREDIT_WAIT; + } + } - if (!msg->msg_rtrcredit) { - LASSERT ((rbp->rbp_credits < 0) == - !cfs_list_empty(&rbp->rbp_msgs)); + rbp = lnet_msg2bufpool(msg); - msg->msg_rtrcredit = 1; - rbp->rbp_credits--; - if (rbp->rbp_credits < rbp->rbp_mincredits) - rbp->rbp_mincredits = rbp->rbp_credits; + if (!msg->msg_rtrcredit) { + msg->msg_rtrcredit = 1; + rbp->rbp_credits--; + if (rbp->rbp_credits < rbp->rbp_mincredits) + rbp->rbp_mincredits = rbp->rbp_credits; - if (rbp->rbp_credits < 0) { - /* must have checked eager_recv before here */ + if (rbp->rbp_credits < 0) { + /* must have checked eager_recv before here */ LASSERT(msg->msg_rx_ready_delay); msg->msg_rx_delayed = 1; - cfs_list_add_tail(&msg->msg_list, &rbp->rbp_msgs); - return EAGAIN; - } - } + list_add_tail(&msg->msg_list, &rbp->rbp_msgs); + return LNET_CREDIT_WAIT; + } + } - LASSERT (!cfs_list_empty(&rbp->rbp_bufs)); - rb = cfs_list_entry(rbp->rbp_bufs.next, lnet_rtrbuf_t, rb_list); - cfs_list_del(&rb->rb_list); + LASSERT(!list_empty(&rbp->rbp_bufs)); + rb = list_entry(rbp->rbp_bufs.next, lnet_rtrbuf_t, rb_list); + list_del(&rb->rb_list); msg->msg_niov = rbp->rbp_npages; msg->msg_kiov = &rb->rb_kiov[0]; @@ -990,9 +960,8 @@ lnet_post_routed_recv_locked (lnet_msg_t *msg, int do_recv) 0, msg->msg_len, msg->msg_len); lnet_net_lock(cpt); } - return 0; + return LNET_CREDIT_OK; } -#endif void lnet_return_tx_credits_locked(lnet_msg_t *msg) @@ -1008,13 +977,13 @@ lnet_return_tx_credits_locked(lnet_msg_t *msg) msg->msg_txcredit = 0; LASSERT((tq->tq_credits < 0) == - !cfs_list_empty(&tq->tq_delayed)); + !list_empty(&tq->tq_delayed)); tq->tq_credits++; if (tq->tq_credits <= 0) { - msg2 = cfs_list_entry(tq->tq_delayed.next, - lnet_msg_t, msg_list); - cfs_list_del(&msg2->msg_list); + msg2 = list_entry(tq->tq_delayed.next, + lnet_msg_t, msg_list); + list_del(&msg2->msg_list); LASSERT(msg2->msg_txpeer->lp_ni == ni); LASSERT(msg2->msg_tx_delayed); @@ -1028,16 +997,16 @@ lnet_return_tx_credits_locked(lnet_msg_t *msg) msg->msg_peertxcredit = 0; LASSERT((txpeer->lp_txcredits < 0) == - !cfs_list_empty(&txpeer->lp_txq)); + !list_empty(&txpeer->lp_txq)); txpeer->lp_txqnob -= msg->msg_len + sizeof(lnet_hdr_t); LASSERT (txpeer->lp_txqnob >= 0); txpeer->lp_txcredits++; if (txpeer->lp_txcredits <= 0) { - msg2 = cfs_list_entry(txpeer->lp_txq.next, + msg2 = list_entry(txpeer->lp_txq.next, lnet_msg_t, msg_list); - cfs_list_del(&msg2->msg_list); + list_del(&msg2->msg_list); LASSERT(msg2->msg_txpeer == txpeer); LASSERT(msg2->msg_tx_delayed); @@ -1053,69 +1022,119 @@ lnet_return_tx_credits_locked(lnet_msg_t *msg) } void +lnet_schedule_blocked_locked(lnet_rtrbufpool_t *rbp) +{ + lnet_msg_t *msg; + + if (list_empty(&rbp->rbp_msgs)) + return; + msg = list_entry(rbp->rbp_msgs.next, + lnet_msg_t, msg_list); + list_del(&msg->msg_list); + + (void)lnet_post_routed_recv_locked(msg, 1); +} + +void +lnet_drop_routed_msgs_locked(struct list_head *list, int cpt) +{ + lnet_msg_t *msg; + lnet_msg_t *tmp; + struct list_head drop; + + INIT_LIST_HEAD(&drop); + + list_splice_init(list, &drop); + + lnet_net_unlock(cpt); + + list_for_each_entry_safe(msg, tmp, &drop, msg_list) { + lnet_ni_recv(msg->msg_rxpeer->lp_ni, msg->msg_private, NULL, + 0, 0, 0, msg->msg_hdr.payload_length); + list_del_init(&msg->msg_list); + lnet_finalize(NULL, msg, -ECANCELED); + } + + lnet_net_lock(cpt); +} + +void lnet_return_rx_credits_locked(lnet_msg_t *msg) { lnet_peer_t *rxpeer = msg->msg_rxpeer; -#ifdef __KERNEL__ lnet_msg_t *msg2; - if (msg->msg_rtrcredit) { - /* give back global router credits */ - lnet_rtrbuf_t *rb; - lnet_rtrbufpool_t *rbp; + if (msg->msg_rtrcredit) { + /* give back global router credits */ + lnet_rtrbuf_t *rb; + lnet_rtrbufpool_t *rbp; - /* NB If a msg ever blocks for a buffer in rbp_msgs, it stays - * there until it gets one allocated, or aborts the wait - * itself */ - LASSERT (msg->msg_kiov != NULL); + /* NB If a msg ever blocks for a buffer in rbp_msgs, it stays + * there until it gets one allocated, or aborts the wait + * itself */ + LASSERT(msg->msg_kiov != NULL); - rb = cfs_list_entry(msg->msg_kiov, lnet_rtrbuf_t, rb_kiov[0]); - rbp = rb->rb_pool; - LASSERT (rbp == lnet_msg2bufpool(msg)); + rb = list_entry(msg->msg_kiov, lnet_rtrbuf_t, rb_kiov[0]); + rbp = rb->rb_pool; - msg->msg_kiov = NULL; - msg->msg_rtrcredit = 0; + msg->msg_kiov = NULL; + msg->msg_rtrcredit = 0; - LASSERT((rbp->rbp_credits < 0) == - !cfs_list_empty(&rbp->rbp_msgs)); - LASSERT((rbp->rbp_credits > 0) == - !cfs_list_empty(&rbp->rbp_bufs)); + LASSERT(rbp == lnet_msg2bufpool(msg)); - cfs_list_add(&rb->rb_list, &rbp->rbp_bufs); - rbp->rbp_credits++; - if (rbp->rbp_credits <= 0) { - msg2 = cfs_list_entry(rbp->rbp_msgs.next, - lnet_msg_t, msg_list); - cfs_list_del(&msg2->msg_list); + LASSERT((rbp->rbp_credits > 0) == + !list_empty(&rbp->rbp_bufs)); - (void) lnet_post_routed_recv_locked(msg2, 1); - } - } + /* If routing is now turned off, we just drop this buffer and + * don't bother trying to return credits. */ + if (!the_lnet.ln_routing) { + lnet_destroy_rtrbuf(rb, rbp->rbp_npages); + goto routing_off; + } - if (msg->msg_peerrtrcredit) { - /* give back peer router credits */ - msg->msg_peerrtrcredit = 0; + /* It is possible that a user has lowered the desired number of + * buffers in this pool. Make sure we never put back + * more buffers than the stated number. */ + if (unlikely(rbp->rbp_credits >= rbp->rbp_req_nbuffers)) { + /* Discard this buffer so we don't have too + * many. */ + lnet_destroy_rtrbuf(rb, rbp->rbp_npages); + rbp->rbp_nbuffers--; + } else { + list_add(&rb->rb_list, &rbp->rbp_bufs); + rbp->rbp_credits++; + if (rbp->rbp_credits <= 0) + lnet_schedule_blocked_locked(rbp); + } + } - LASSERT((rxpeer->lp_rtrcredits < 0) == - !cfs_list_empty(&rxpeer->lp_rtrq)); +routing_off: + if (msg->msg_peerrtrcredit) { + /* give back peer router credits */ + msg->msg_peerrtrcredit = 0; - rxpeer->lp_rtrcredits++; - if (rxpeer->lp_rtrcredits <= 0) { - msg2 = cfs_list_entry(rxpeer->lp_rtrq.next, - lnet_msg_t, msg_list); - cfs_list_del(&msg2->msg_list); + LASSERT((rxpeer->lp_rtrcredits < 0) == + !list_empty(&rxpeer->lp_rtrq)); - (void) lnet_post_routed_recv_locked(msg2, 1); - } - } -#else - LASSERT (!msg->msg_rtrcredit); - LASSERT (!msg->msg_peerrtrcredit); -#endif - if (rxpeer != NULL) { - msg->msg_rxpeer = NULL; - lnet_peer_decref_locked(rxpeer); - } + rxpeer->lp_rtrcredits++; + + /* drop all messages which are queued to be routed on that + * peer. */ + if (!the_lnet.ln_routing) { + lnet_drop_routed_msgs_locked(&rxpeer->lp_rtrq, + msg->msg_rx_cpt); + } else if (rxpeer->lp_rtrcredits <= 0) { + msg2 = list_entry(rxpeer->lp_rtrq.next, + lnet_msg_t, msg_list); + list_del(&msg2->msg_list); + + (void) lnet_post_routed_recv_locked(msg2, 1); + } + } + if (rxpeer != NULL) { + msg->msg_rxpeer = NULL; + lnet_peer_decref_locked(rxpeer); + } } static int @@ -1123,6 +1142,8 @@ lnet_compare_routes(lnet_route_t *r1, lnet_route_t *r2) { lnet_peer_t *p1 = r1->lr_gateway; lnet_peer_t *p2 = r2->lr_gateway; + int r1_hops = (r1->lr_hops == LNET_UNDEFINED_HOPS) ? 1 : r1->lr_hops; + int r2_hops = (r2->lr_hops == LNET_UNDEFINED_HOPS) ? 1 : r2->lr_hops; if (r1->lr_priority < r2->lr_priority) return 1; @@ -1130,10 +1151,10 @@ lnet_compare_routes(lnet_route_t *r1, lnet_route_t *r2) if (r1->lr_priority > r2->lr_priority) return -1; - if (r1->lr_hops < r2->lr_hops) + if (r1_hops < r2_hops) return 1; - if (r1->lr_hops > r2->lr_hops) + if (r1_hops > r2_hops) return -1; if (p1->lp_txqnob < p2->lp_txqnob) @@ -1158,9 +1179,9 @@ static lnet_peer_t * lnet_find_route_locked(lnet_ni_t *ni, lnet_nid_t target, lnet_nid_t rtr_nid) { lnet_remotenet_t *rnet; - lnet_route_t *rtr; - lnet_route_t *rtr_best; - lnet_route_t *rtr_last; + lnet_route_t *route; + lnet_route_t *best_route; + lnet_route_t *last_route; struct lnet_peer *lp_best; struct lnet_peer *lp; int rc; @@ -1173,13 +1194,11 @@ lnet_find_route_locked(lnet_ni_t *ni, lnet_nid_t target, lnet_nid_t rtr_nid) return NULL; lp_best = NULL; - rtr_best = rtr_last = NULL; - cfs_list_for_each_entry(rtr, &rnet->lrn_routes, lr_list) { - lp = rtr->lr_gateway; + best_route = last_route = NULL; + list_for_each_entry(route, &rnet->lrn_routes, lr_list) { + lp = route->lr_gateway; - if (!lp->lp_alive || /* gateway is down */ - ((lp->lp_ping_feats & LNET_PING_FEAT_NI_STATUS) != 0 && - rtr->lr_downis != 0)) /* NI to target is down */ + if (!lnet_is_route_alive(route)) continue; if (ni != NULL && lp->lp_ni != ni) @@ -1189,28 +1208,28 @@ lnet_find_route_locked(lnet_ni_t *ni, lnet_nid_t target, lnet_nid_t rtr_nid) return lp; if (lp_best == NULL) { - rtr_best = rtr_last = rtr; + best_route = last_route = route; lp_best = lp; continue; } /* no protection on below fields, but it's harmless */ - if (rtr_last->lr_seq - rtr->lr_seq < 0) - rtr_last = rtr; + if (last_route->lr_seq - route->lr_seq < 0) + last_route = route; - rc = lnet_compare_routes(rtr, rtr_best); + rc = lnet_compare_routes(route, best_route); if (rc < 0) continue; - rtr_best = rtr; + best_route = route; lp_best = lp; } /* set sequence number on the best router to the latest sequence + 1 * so we can round-robin all routers, it's race and inaccurate but * harmless and functional */ - if (rtr_best != NULL) - rtr_best->lr_seq = rtr_last->lr_seq + 1; + if (best_route != NULL) + best_route->lr_seq = last_route->lr_seq + 1; return lp_best; } @@ -1308,19 +1327,6 @@ lnet_send(lnet_nid_t src_nid, lnet_msg_t *msg, lnet_nid_t rtr_nid) } LASSERT (lp->lp_ni == src_ni); } else { -#ifndef __KERNEL__ - lnet_net_unlock(cpt); - - /* NB - * - once application finishes computation, check here to update - * router states before it waits for pending IO in LNetEQPoll - * - recursion breaker: router checker sends no message - * to remote networks */ - if (the_lnet.ln_rc_state == LNET_RC_STATE_RUNNING) - lnet_router_checker(); - - lnet_net_lock(cpt); -#endif /* sending to a remote network */ lp = lnet_find_route_locked(src_ni, dst_nid, rtr_nid); if (lp == NULL) { @@ -1376,7 +1382,7 @@ lnet_send(lnet_nid_t src_nid, lnet_msg_t *msg, lnet_nid_t rtr_nid) msg->msg_target_is_router = 1; msg->msg_target.nid = lp->lp_nid; - msg->msg_target.pid = LUSTRE_SRV_LNET_PID; + msg->msg_target.pid = LNET_PID_LUSTRE; } /* 'lp' is our best choice of peer */ @@ -1390,16 +1396,16 @@ lnet_send(lnet_nid_t src_nid, lnet_msg_t *msg, lnet_nid_t rtr_nid) rc = lnet_post_send_locked(msg, 0); lnet_net_unlock(cpt); - if (rc == EHOSTUNREACH) - return -EHOSTUNREACH; + if (rc < 0) + return rc; - if (rc == 0) + if (rc == LNET_CREDIT_OK) lnet_ni_send(src_ni, msg); - return 0; + return 0; /* rc == LNET_CREDIT_OK or LNET_CREDIT_WAIT */ } -static void +void lnet_drop_message(lnet_ni_t *ni, int cpt, void *private, unsigned int nob) { lnet_net_lock(cpt); @@ -1435,6 +1441,7 @@ lnet_parse_put(lnet_ni_t *ni, lnet_msg_t *msg) lnet_hdr_t *hdr = &msg->msg_hdr; struct lnet_match_info info; int rc; + bool ready_delay; /* Convert put fields to host byte order */ hdr->msg.put.match_bits = le64_to_cpu(hdr->msg.put.match_bits); @@ -1450,6 +1457,7 @@ lnet_parse_put(lnet_ni_t *ni, lnet_msg_t *msg) info.mi_mbits = hdr->msg.put.match_bits; msg->msg_rx_ready_delay = ni->ni_lnd->lnd_eager_recv == NULL; + ready_delay = msg->msg_rx_ready_delay; again: rc = lnet_ptl_match_md(&info, msg); @@ -1462,12 +1470,16 @@ lnet_parse_put(lnet_ni_t *ni, lnet_msg_t *msg) return 0; case LNET_MATCHMD_NONE: - if (msg->msg_rx_delayed) /* attached on delayed list */ + if (ready_delay) + /* no eager_recv or has already called it, should + * have been attached on delayed list */ return 0; rc = lnet_ni_eager_recv(ni, msg); - if (rc == 0) + if (rc == 0) { + ready_delay = true; goto again; + } /* fall through */ case LNET_MATCHMD_DROP: @@ -1596,7 +1608,7 @@ lnet_parse_reply(lnet_ni_t *ni, lnet_msg_t *msg) } CDEBUG(D_NET, "%s: Reply from %s of length %d/%d into md "LPX64"\n", - libcfs_nid2str(ni->ni_nid), libcfs_id2str(src), + libcfs_nid2str(ni->ni_nid), libcfs_id2str(src), mlength, rlength, hdr->msg.reply.dst_wmd.wh_object_cookie); lnet_msg_attach_md(msg, md, 0, mlength); @@ -1649,7 +1661,7 @@ lnet_parse_ack(lnet_ni_t *ni, lnet_msg_t *msg) } CDEBUG(D_NET, "%s: ACK from %s into md "LPX64"\n", - libcfs_nid2str(ni->ni_nid), libcfs_id2str(src), + libcfs_nid2str(ni->ni_nid), libcfs_id2str(src), hdr->msg.ack.dst_wmd.wh_object_cookie); lnet_msg_attach_md(msg, md, 0, 0); @@ -1662,12 +1674,19 @@ lnet_parse_ack(lnet_ni_t *ni, lnet_msg_t *msg) return 0; } -static int +/** + * \retval LNET_CREDIT_OK If \a msg is forwarded + * \retval LNET_CREDIT_WAIT If \a msg is blocked because w/o buffer + * \retval -ve error code + */ +int lnet_parse_forward_locked(lnet_ni_t *ni, lnet_msg_t *msg) { int rc = 0; -#ifdef __KERNEL__ + if (!the_lnet.ln_routing) + return -ECANCELED; + if (msg->msg_rxpeer->lp_rtrcredits <= 0 || lnet_msg2bufpool(msg)->rbp_credits <= 0) { if (ni->ni_lnd->lnd_eager_recv == NULL) { @@ -1681,9 +1700,33 @@ lnet_parse_forward_locked(lnet_ni_t *ni, lnet_msg_t *msg) if (rc == 0) rc = lnet_post_routed_recv_locked(msg, 0); -#else - LBUG(); -#endif + return rc; +} + +int +lnet_parse_local(lnet_ni_t *ni, lnet_msg_t *msg) +{ + int rc; + + switch (msg->msg_type) { + case LNET_MSG_ACK: + rc = lnet_parse_ack(ni, msg); + break; + case LNET_MSG_PUT: + rc = lnet_parse_put(ni, msg); + break; + case LNET_MSG_GET: + rc = lnet_parse_get(ni, msg, msg->msg_rdma_get); + break; + case LNET_MSG_REPLY: + rc = lnet_parse_reply(ni, msg); + break; + default: /* prevent an unused label if !kernel */ + LASSERT(0); + return -EPROTO; + } + + LASSERT(rc == 0 || rc == ENOENT); return rc; } @@ -1705,7 +1748,6 @@ lnet_msgtyp2str (int type) return (""); } } -EXPORT_SYMBOL(lnet_msgtyp2str); void lnet_print_hdr(lnet_hdr_t * hdr) @@ -1777,8 +1819,8 @@ lnet_parse(lnet_ni_t *ni, lnet_hdr_t *hdr, lnet_nid_t from_nid, int cpt; int for_me; struct lnet_msg *msg; - lnet_pid_t dest_pid; - lnet_nid_t dest_nid; + lnet_pid_t dest_pid; + lnet_nid_t dest_nid; lnet_nid_t src_nid; __u32 payload_length; __u32 type; @@ -1789,129 +1831,140 @@ lnet_parse(lnet_ni_t *ni, lnet_hdr_t *hdr, lnet_nid_t from_nid, src_nid = le64_to_cpu(hdr->src_nid); dest_nid = le64_to_cpu(hdr->dest_nid); dest_pid = le32_to_cpu(hdr->dest_pid); - payload_length = le32_to_cpu(hdr->payload_length); + payload_length = le32_to_cpu(hdr->payload_length); - for_me = (ni->ni_nid == dest_nid); + for_me = (ni->ni_nid == dest_nid); cpt = lnet_cpt_of_nid(from_nid); - switch (type) { - case LNET_MSG_ACK: - case LNET_MSG_GET: - if (payload_length > 0) { - CERROR("%s, src %s: bad %s payload %d (0 expected)\n", - libcfs_nid2str(from_nid), - libcfs_nid2str(src_nid), - lnet_msgtyp2str(type), payload_length); - return -EPROTO; - } - break; - - case LNET_MSG_PUT: - case LNET_MSG_REPLY: - if (payload_length > (__u32)(for_me ? LNET_MAX_PAYLOAD : LNET_MTU)) { - CERROR("%s, src %s: bad %s payload %d " - "(%d max expected)\n", - libcfs_nid2str(from_nid), - libcfs_nid2str(src_nid), - lnet_msgtyp2str(type), - payload_length, - for_me ? LNET_MAX_PAYLOAD : LNET_MTU); - return -EPROTO; - } - break; + switch (type) { + case LNET_MSG_ACK: + case LNET_MSG_GET: + if (payload_length > 0) { + CERROR("%s, src %s: bad %s payload %d (0 expected)\n", + libcfs_nid2str(from_nid), + libcfs_nid2str(src_nid), + lnet_msgtyp2str(type), payload_length); + return -EPROTO; + } + break; + + case LNET_MSG_PUT: + case LNET_MSG_REPLY: + if (payload_length > + (__u32)(for_me ? LNET_MAX_PAYLOAD : LNET_MTU)) { + CERROR("%s, src %s: bad %s payload %d " + "(%d max expected)\n", + libcfs_nid2str(from_nid), + libcfs_nid2str(src_nid), + lnet_msgtyp2str(type), + payload_length, + for_me ? LNET_MAX_PAYLOAD : LNET_MTU); + return -EPROTO; + } + break; - default: - CERROR("%s, src %s: Bad message type 0x%x\n", - libcfs_nid2str(from_nid), - libcfs_nid2str(src_nid), type); - return -EPROTO; - } + default: + CERROR("%s, src %s: Bad message type 0x%x\n", + libcfs_nid2str(from_nid), + libcfs_nid2str(src_nid), type); + return -EPROTO; + } if (the_lnet.ln_routing && ni->ni_last_alive != cfs_time_current_sec()) { - lnet_ni_lock(ni); - /* NB: so far here is the only place to set NI status to "up */ + lnet_ni_lock(ni); ni->ni_last_alive = cfs_time_current_sec(); if (ni->ni_status != NULL && ni->ni_status->ns_status == LNET_NI_STATUS_DOWN) ni->ni_status->ns_status = LNET_NI_STATUS_UP; lnet_ni_unlock(ni); - } + } - /* Regard a bad destination NID as a protocol error. Senders should - * know what they're doing; if they don't they're misconfigured, buggy - * or malicious so we chop them off at the knees :) */ - - if (!for_me) { - if (LNET_NIDNET(dest_nid) == LNET_NIDNET(ni->ni_nid)) { - /* should have gone direct */ - CERROR ("%s, src %s: Bad dest nid %s " - "(should have been sent direct)\n", - libcfs_nid2str(from_nid), - libcfs_nid2str(src_nid), - libcfs_nid2str(dest_nid)); - return -EPROTO; - } + /* Regard a bad destination NID as a protocol error. Senders should + * know what they're doing; if they don't they're misconfigured, buggy + * or malicious so we chop them off at the knees :) */ - if (lnet_islocalnid(dest_nid)) { - /* dest is another local NI; sender should have used - * this node's NID on its own network */ - CERROR ("%s, src %s: Bad dest nid %s " - "(it's my nid but on a different network)\n", - libcfs_nid2str(from_nid), - libcfs_nid2str(src_nid), - libcfs_nid2str(dest_nid)); - return -EPROTO; - } + if (!for_me) { + if (LNET_NIDNET(dest_nid) == LNET_NIDNET(ni->ni_nid)) { + /* should have gone direct */ + CERROR("%s, src %s: Bad dest nid %s " + "(should have been sent direct)\n", + libcfs_nid2str(from_nid), + libcfs_nid2str(src_nid), + libcfs_nid2str(dest_nid)); + return -EPROTO; + } - if (rdma_req && type == LNET_MSG_GET) { - CERROR ("%s, src %s: Bad optimized GET for %s " - "(final destination must be me)\n", - libcfs_nid2str(from_nid), - libcfs_nid2str(src_nid), - libcfs_nid2str(dest_nid)); - return -EPROTO; - } + if (lnet_islocalnid(dest_nid)) { + /* dest is another local NI; sender should have used + * this node's NID on its own network */ + CERROR("%s, src %s: Bad dest nid %s " + "(it's my nid but on a different network)\n", + libcfs_nid2str(from_nid), + libcfs_nid2str(src_nid), + libcfs_nid2str(dest_nid)); + return -EPROTO; + } - if (!the_lnet.ln_routing) { - CERROR ("%s, src %s: Dropping message for %s " - "(routing not enabled)\n", - libcfs_nid2str(from_nid), - libcfs_nid2str(src_nid), - libcfs_nid2str(dest_nid)); - goto drop; - } - } + if (rdma_req && type == LNET_MSG_GET) { + CERROR("%s, src %s: Bad optimized GET for %s " + "(final destination must be me)\n", + libcfs_nid2str(from_nid), + libcfs_nid2str(src_nid), + libcfs_nid2str(dest_nid)); + return -EPROTO; + } - /* Message looks OK; we're not going to return an error, so we MUST - * call back lnd_recv() come what may... */ + if (!the_lnet.ln_routing) { + CERROR("%s, src %s: Dropping message for %s " + "(routing not enabled)\n", + libcfs_nid2str(from_nid), + libcfs_nid2str(src_nid), + libcfs_nid2str(dest_nid)); + goto drop; + } + } - if (!cfs_list_empty (&the_lnet.ln_test_peers) && /* normally we don't */ - fail_peer (src_nid, 0)) /* shall we now? */ - { - CERROR("%s, src %s: Dropping %s to simulate failure\n", - libcfs_nid2str(from_nid), libcfs_nid2str(src_nid), - lnet_msgtyp2str(type)); - goto drop; - } + /* Message looks OK; we're not going to return an error, so we MUST + * call back lnd_recv() come what may... */ - msg = lnet_msg_alloc(); - if (msg == NULL) { - CERROR("%s, src %s: Dropping %s (out of memory)\n", - libcfs_nid2str(from_nid), libcfs_nid2str(src_nid), - lnet_msgtyp2str(type)); - goto drop; - } + if (!list_empty(&the_lnet.ln_test_peers) && /* normally we don't */ + fail_peer(src_nid, 0)) { /* shall we now? */ + CERROR("%s, src %s: Dropping %s to simulate failure\n", + libcfs_nid2str(from_nid), libcfs_nid2str(src_nid), + lnet_msgtyp2str(type)); + goto drop; + } + + if (!list_empty(&the_lnet.ln_drop_rules) && + lnet_drop_rule_match(hdr)) { + CDEBUG(D_NET, "%s, src %s, dst %s: Dropping %s to simulate" + "silent message loss\n", + libcfs_nid2str(from_nid), libcfs_nid2str(src_nid), + libcfs_nid2str(dest_nid), lnet_msgtyp2str(type)); + goto drop; + } - /* msg zeroed in lnet_msg_alloc; i.e. flags all clear, pointers NULL etc */ - msg->msg_type = type; - msg->msg_private = private; - msg->msg_receiving = 1; - msg->msg_len = msg->msg_wanted = payload_length; - msg->msg_offset = 0; - msg->msg_hdr = *hdr; + msg = lnet_msg_alloc(); + if (msg == NULL) { + CERROR("%s, src %s: Dropping %s (out of memory)\n", + libcfs_nid2str(from_nid), libcfs_nid2str(src_nid), + lnet_msgtyp2str(type)); + goto drop; + } + + /* msg zeroed in lnet_msg_alloc; i.e. flags all clear, + * pointers NULL etc */ + + msg->msg_type = type; + msg->msg_private = private; + msg->msg_receiving = 1; + msg->msg_rdma_get = rdma_req; + msg->msg_len = msg->msg_wanted = payload_length; + msg->msg_offset = 0; + msg->msg_hdr = *hdr; /* for building message event */ msg->msg_from = from_nid; if (!for_me) { @@ -1941,15 +1994,36 @@ lnet_parse(lnet_ni_t *ni, lnet_hdr_t *hdr, lnet_nid_t from_nid, goto drop; } + if (lnet_isrouter(msg->msg_rxpeer)) { + lnet_peer_set_alive(msg->msg_rxpeer); + if (avoid_asym_router_failure && + LNET_NIDNET(src_nid) != LNET_NIDNET(from_nid)) { + /* received a remote message from router, update + * remote NI status on this router. + * NB: multi-hop routed message will be ignored. + */ + lnet_router_ni_update_locked(msg->msg_rxpeer, + LNET_NIDNET(src_nid)); + } + } + lnet_msg_commit(msg, cpt); + /* message delay simulation */ + if (unlikely(!list_empty(&the_lnet.ln_delay_rules) && + lnet_delay_rule_match_locked(hdr, msg))) { + lnet_net_unlock(cpt); + return 0; + } + if (!for_me) { rc = lnet_parse_forward_locked(ni, msg); lnet_net_unlock(cpt); if (rc < 0) goto free_drop; - if (rc == 0) { + + if (rc == LNET_CREDIT_OK) { lnet_ni_recv(ni, msg->msg_private, msg, 0, 0, payload_length, payload_length); } @@ -1958,29 +2032,10 @@ lnet_parse(lnet_ni_t *ni, lnet_hdr_t *hdr, lnet_nid_t from_nid, lnet_net_unlock(cpt); - switch (type) { - case LNET_MSG_ACK: - rc = lnet_parse_ack(ni, msg); - break; - case LNET_MSG_PUT: - rc = lnet_parse_put(ni, msg); - break; - case LNET_MSG_GET: - rc = lnet_parse_get(ni, msg, rdma_req); - break; - case LNET_MSG_REPLY: - rc = lnet_parse_reply(ni, msg); - break; - default: - LASSERT(0); - rc = -EPROTO; - goto free_drop; /* prevent an unused label if !kernel */ - } - - if (rc == 0) - return 0; - - LASSERT (rc == ENOENT); + rc = lnet_parse_local(ni, msg); + if (rc != 0) + goto free_drop; + return 0; free_drop: LASSERT(msg->msg_md == NULL); @@ -1993,14 +2048,14 @@ lnet_parse(lnet_ni_t *ni, lnet_hdr_t *hdr, lnet_nid_t from_nid, EXPORT_SYMBOL(lnet_parse); void -lnet_drop_delayed_msg_list(cfs_list_t *head, char *reason) +lnet_drop_delayed_msg_list(struct list_head *head, char *reason) { - while (!cfs_list_empty(head)) { + while (!list_empty(head)) { lnet_process_id_t id = {0}; lnet_msg_t *msg; - msg = cfs_list_entry(head->next, lnet_msg_t, msg_list); - cfs_list_del(&msg->msg_list); + msg = list_entry(head->next, lnet_msg_t, msg_list); + list_del(&msg->msg_list); id.nid = msg->msg_hdr.src_nid; id.pid = msg->msg_hdr.src_pid; @@ -2035,14 +2090,14 @@ lnet_drop_delayed_msg_list(cfs_list_t *head, char *reason) } void -lnet_recv_delayed_msg_list(cfs_list_t *head) +lnet_recv_delayed_msg_list(struct list_head *head) { - while (!cfs_list_empty(head)) { + while (!list_empty(head)) { lnet_msg_t *msg; lnet_process_id_t id; - msg = cfs_list_entry(head->next, lnet_msg_t, msg_list); - cfs_list_del(&msg->msg_list); + msg = list_entry(head->next, lnet_msg_t, msg_list); + list_del(&msg->msg_list); /* md won't disappear under me, since each msg * holds a ref on it */ @@ -2121,12 +2176,10 @@ LNetPut(lnet_nid_t self, lnet_handle_md_t mdh, lnet_ack_req_t ack, int cpt; int rc; - LASSERT (the_lnet.ln_init); - LASSERT (the_lnet.ln_refcount > 0); + LASSERT(the_lnet.ln_refcount > 0); - if (!cfs_list_empty (&the_lnet.ln_test_peers) && /* normally we don't */ - fail_peer (target.nid, 1)) /* shall we now? */ - { + if (!list_empty(&the_lnet.ln_test_peers) && /* normally we don't */ + fail_peer(target.nid, 1)) { /* shall we now? */ CERROR("Dropping PUT to %s: simulated failure\n", libcfs_id2str(target)); return -EIO; @@ -2170,14 +2223,14 @@ LNetPut(lnet_nid_t self, lnet_handle_md_t mdh, lnet_ack_req_t ack, /* NB handles only looked up by creator (no flips) */ if (ack == LNET_ACK_REQ) { - msg->msg_hdr.msg.put.ack_wmd.wh_interface_cookie = + msg->msg_hdr.msg.put.ack_wmd.wh_interface_cookie = the_lnet.ln_interface_cookie; - msg->msg_hdr.msg.put.ack_wmd.wh_object_cookie = + msg->msg_hdr.msg.put.ack_wmd.wh_object_cookie = md->md_lh.lh_cookie; } else { - msg->msg_hdr.msg.put.ack_wmd.wh_interface_cookie = + msg->msg_hdr.msg.put.ack_wmd.wh_interface_cookie = LNET_WIRE_HANDLE_COOKIE_NONE; - msg->msg_hdr.msg.put.ack_wmd.wh_object_cookie = + msg->msg_hdr.msg.put.ack_wmd.wh_object_cookie = LNET_WIRE_HANDLE_COOKIE_NONE; } @@ -2215,20 +2268,20 @@ lnet_create_reply_msg (lnet_ni_t *ni, lnet_msg_t *getmsg) LASSERT(!getmsg->msg_target_is_router); LASSERT(!getmsg->msg_routing); + if (msg == NULL) { + CERROR("%s: Dropping REPLY from %s: can't allocate msg\n", + libcfs_nid2str(ni->ni_nid), libcfs_id2str(peer_id)); + goto drop; + } + cpt = lnet_cpt_of_cookie(getmd->md_lh.lh_cookie); lnet_res_lock(cpt); - LASSERT (getmd->md_refcount > 0); - - if (msg == NULL) { - CERROR ("%s: Dropping REPLY from %s: can't allocate msg\n", - libcfs_nid2str(ni->ni_nid), libcfs_id2str(peer_id)); - goto drop; - } + LASSERT(getmd->md_refcount > 0); if (getmd->md_threshold == 0) { CERROR ("%s: Dropping REPLY from %s for inactive MD %p\n", - libcfs_nid2str(ni->ni_nid), libcfs_id2str(peer_id), + libcfs_nid2str(ni->ni_nid), libcfs_id2str(peer_id), getmd); lnet_res_unlock(cpt); goto drop; @@ -2321,11 +2374,10 @@ LNetGet(lnet_nid_t self, lnet_handle_md_t mdh, int cpt; int rc; - LASSERT (the_lnet.ln_init); LASSERT (the_lnet.ln_refcount > 0); - if (!cfs_list_empty (&the_lnet.ln_test_peers) && /* normally we don't */ - fail_peer (target.nid, 1)) /* shall we now? */ + if (!list_empty(&the_lnet.ln_test_peers) && /* normally we don't */ + fail_peer(target.nid, 1)) /* shall we now? */ { CERROR("Dropping GET to %s: simulated failure\n", libcfs_id2str(target)); @@ -2354,7 +2406,6 @@ LNetGet(lnet_nid_t self, lnet_handle_md_t mdh, lnet_res_unlock(cpt); lnet_msg_free(msg); - return -ENOENT; } @@ -2380,11 +2431,11 @@ LNetGet(lnet_nid_t self, lnet_handle_md_t mdh, lnet_build_msg_event(msg, LNET_EVENT_SEND); rc = lnet_send(self, msg, LNET_NID_ANY); - if (rc < 0) { - CNETERR( "Error sending GET to %s: %d\n", - libcfs_id2str(target), rc); - lnet_finalize (NULL, msg, rc); - } + if (rc < 0) { + CNETERR("Error sending GET to %s: %d\n", + libcfs_id2str(target), rc); + lnet_finalize(NULL, msg, rc); + } /* completion will be signalled by an event */ return 0; @@ -2408,27 +2459,26 @@ EXPORT_SYMBOL(LNetGet); int LNetDist(lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp) { - cfs_list_t *e; + struct list_head *e; struct lnet_ni *ni; lnet_remotenet_t *rnet; __u32 dstnet = LNET_NIDNET(dstnid); int hops; int cpt; __u32 order = 2; - cfs_list_t *rn_list; + struct list_head *rn_list; /* if !local_nid_dist_zero, I don't return a distance of 0 ever * (when lustre sees a distance of 0, it substitutes 0@lo), so I * keep order 0 free for 0@lo and order 1 free for a local NID * match */ - LASSERT (the_lnet.ln_init); LASSERT (the_lnet.ln_refcount > 0); cpt = lnet_net_lock_current(); - cfs_list_for_each (e, &the_lnet.ln_nis) { - ni = cfs_list_entry(e, lnet_ni_t, ni_list); + list_for_each(e, &the_lnet.ln_nis) { + ni = list_entry(e, lnet_ni_t, ni_list); if (ni->ni_nid == dstnid) { if (srcnidp != NULL) @@ -2457,28 +2507,35 @@ LNetDist(lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp) } rn_list = lnet_net2rnethash(dstnet); - cfs_list_for_each(e, rn_list) { - rnet = cfs_list_entry(e, lnet_remotenet_t, lrn_list); - - if (rnet->lrn_net == dstnet) { - lnet_route_t *route; - lnet_route_t *shortest = NULL; - - LASSERT (!cfs_list_empty(&rnet->lrn_routes)); - - cfs_list_for_each_entry(route, &rnet->lrn_routes, - lr_list) { - if (shortest == NULL || - route->lr_hops < shortest->lr_hops) - shortest = route; - } + list_for_each(e, rn_list) { + rnet = list_entry(e, lnet_remotenet_t, lrn_list); + + if (rnet->lrn_net == dstnet) { + lnet_route_t *route; + lnet_route_t *shortest = NULL; + __u32 shortest_hops = LNET_UNDEFINED_HOPS; + __u32 route_hops; + + LASSERT(!list_empty(&rnet->lrn_routes)); + + list_for_each_entry(route, &rnet->lrn_routes, + lr_list) { + route_hops = route->lr_hops; + if (route_hops == LNET_UNDEFINED_HOPS) + route_hops = 1; + if (shortest == NULL || + route_hops < shortest_hops) { + shortest = route; + shortest_hops = route_hops; + } + } - LASSERT (shortest != NULL); - hops = shortest->lr_hops; - if (srcnidp != NULL) - *srcnidp = shortest->lr_gateway->lp_ni->ni_nid; - if (orderp != NULL) - *orderp = order; + LASSERT(shortest != NULL); + hops = shortest_hops; + if (srcnidp != NULL) + *srcnidp = shortest->lr_gateway->lp_ni->ni_nid; + if (orderp != NULL) + *orderp = order; lnet_net_unlock(cpt); return hops + 1; } @@ -2489,94 +2546,3 @@ LNetDist(lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp) return -EHOSTUNREACH; } EXPORT_SYMBOL(LNetDist); - -/** - * Set the number of asynchronous messages expected from a target process. - * - * This function is only meaningful for userspace callers. It's a no-op when - * called from kernel. - * - * Asynchronous messages are those that can come from a target when the - * userspace process is not waiting for IO to complete; e.g., AST callbacks - * from Lustre servers. Specifying the expected number of such messages - * allows them to be eagerly received when user process is not running in - * LNet; otherwise network errors may occur. - * - * \param id Process ID of the target process. - * \param nasync Number of asynchronous messages expected from the target. - * - * \return 0 on success, and an error code otherwise. - */ -int -LNetSetAsync(lnet_process_id_t id, int nasync) -{ -#ifdef __KERNEL__ - return 0; -#else - lnet_ni_t *ni; - lnet_remotenet_t *rnet; - cfs_list_t *tmp; - lnet_route_t *route; - lnet_nid_t *nids; - int nnids; - int maxnids = 256; - int rc = 0; - int rc2; - int cpt; - - /* Target on a local network? */ - ni = lnet_net2ni(LNET_NIDNET(id.nid)); - if (ni != NULL) { - if (ni->ni_lnd->lnd_setasync != NULL) - rc = (ni->ni_lnd->lnd_setasync)(ni, id, nasync); - lnet_ni_decref(ni); - return rc; - } - - /* Target on a remote network: apply to routers */ - again: - LIBCFS_ALLOC(nids, maxnids * sizeof(*nids)); - if (nids == NULL) - return -ENOMEM; - nnids = 0; - - /* Snapshot all the router NIDs */ - cpt = lnet_net_lock_current(); - rnet = lnet_find_net_locked(LNET_NIDNET(id.nid)); - if (rnet != NULL) { - cfs_list_for_each(tmp, &rnet->lrn_routes) { - if (nnids == maxnids) { - lnet_net_unlock(cpt); - LIBCFS_FREE(nids, maxnids * sizeof(*nids)); - maxnids *= 2; - goto again; - } - - route = cfs_list_entry(tmp, lnet_route_t, lr_list); - nids[nnids++] = route->lr_gateway->lp_nid; - } - } - lnet_net_unlock(cpt); - - /* set async on all the routers */ - while (nnids-- > 0) { - id.pid = LUSTRE_SRV_LNET_PID; - id.nid = nids[nnids]; - - ni = lnet_net2ni(LNET_NIDNET(id.nid)); - if (ni == NULL) - continue; - - if (ni->ni_lnd->lnd_setasync != NULL) { - rc2 = (ni->ni_lnd->lnd_setasync)(ni, id, nasync); - if (rc2 != 0) - rc = rc2; - } - lnet_ni_decref(ni); - } - - LIBCFS_FREE(nids, maxnids * sizeof(*nids)); - return rc; -#endif -} -EXPORT_SYMBOL(LNetSetAsync);