4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
27 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Use is subject to license terms.
30 * Copyright (c) 2012, Intel Corporation.
32 * Copyright (C) 2006 Myricom, Inc.
35 * This file is part of Lustre, http://www.lustre.org/
36 * Lustre is a trademark of Sun Microsystems, Inc.
38 * lnet/klnds/mxlnd/mxlnd.c
40 * Author: Eric Barton <eric@bartonsoftware.com>
41 * Author: Scott Atchley <atchley at myri.com>
46 mx_endpoint_addr_t MX_EPA_NULL; /* use to determine if an endpoint is NULL */
49 mxlnd_endpoint_addr_null(mx_endpoint_addr_t epa)
51 /* if memcmp() == 0, it is NULL */
52 return !(memcmp(&epa, &MX_EPA_NULL, sizeof(epa)));
56 mxlnd_ctxstate_to_str(int mxc_state)
60 return "MXLND_CTX_INIT";
62 return "MXLND_CTX_IDLE";
64 return "MXLND_CTX_PREP";
65 case MXLND_CTX_PENDING:
66 return "MXLND_CTX_PENDING";
67 case MXLND_CTX_COMPLETED:
68 return "MXLND_CTX_COMPLETED";
69 case MXLND_CTX_CANCELED:
70 return "MXLND_CTX_CANCELED";
77 mxlnd_connstatus_to_str(int mxk_status)
80 case MXLND_CONN_READY:
81 return "MXLND_CONN_READY";
83 return "MXLND_CONN_INIT";
85 return "MXLND_CONN_WAIT";
86 case MXLND_CONN_DISCONNECT:
87 return "MXLND_CONN_DISCONNECT";
89 return "MXLND_CONN_FAIL";
96 mxlnd_msgtype_to_str(int type) {
99 return "MXLND_MSG_EAGER";
100 case MXLND_MSG_CONN_REQ:
101 return "MXLND_MSG_CONN_REQ";
102 case MXLND_MSG_CONN_ACK:
103 return "MXLND_MSG_CONN_ACK";
105 return "MXLND_MSG_BYE";
107 return "MXLND_MSG_NOOP";
108 case MXLND_MSG_PUT_REQ:
109 return "MXLND_MSG_PUT_REQ";
110 case MXLND_MSG_PUT_ACK:
111 return "MXLND_MSG_PUT_ACK";
112 case MXLND_MSG_PUT_DATA:
113 return "MXLND_MSG_PUT_DATA";
114 case MXLND_MSG_GET_REQ:
115 return "MXLND_MSG_GET_REQ";
116 case MXLND_MSG_GET_DATA:
117 return "MXLND_MSG_GET_DATA";
124 mxlnd_lnetmsg_to_str(int type)
128 return "LNET_MSG_ACK";
130 return "LNET_MSG_PUT";
132 return "LNET_MSG_GET";
134 return "LNET_MSG_REPLY";
136 return "LNET_MSG_HELLO";
144 mxlnd_create_match(kmx_ctx_t *ctx, u8 error)
146 u64 type = (u64) ctx->mxc_msg_type;
147 u64 err = (u64) error;
150 mxlnd_valid_msg_type(ctx->mxc_msg_type);
151 LASSERT(ctx->mxc_cookie >> MXLND_ERROR_OFFSET == 0);
152 match = (type << MXLND_MSG_OFFSET) | (err << MXLND_ERROR_OFFSET) | ctx->mxc_cookie;
157 mxlnd_parse_match(u64 match, u8 *msg_type, u8 *error, u64 *cookie)
159 *msg_type = (u8) MXLND_MSG_TYPE(match);
160 *error = (u8) MXLND_ERROR_VAL(match);
161 *cookie = match & MXLND_MAX_COOKIE;
162 mxlnd_valid_msg_type(*msg_type);
167 mxlnd_get_idle_rx(kmx_conn_t *conn)
169 cfs_list_t *rxs = NULL;
170 kmx_ctx_t *rx = NULL;
172 LASSERT(conn != NULL);
174 rxs = &conn->mxk_rx_idle;
176 spin_lock(&conn->mxk_lock);
178 if (cfs_list_empty (rxs)) {
179 spin_unlock(&conn->mxk_lock);
183 rx = cfs_list_entry (rxs->next, kmx_ctx_t, mxc_list);
184 cfs_list_del_init(&rx->mxc_list);
185 spin_unlock(&conn->mxk_lock);
188 if (rx->mxc_get != rx->mxc_put) {
189 CNETERR("*** RX get (%llu) != put (%llu) ***\n", rx->mxc_get, rx->mxc_put);
190 CNETERR("*** incarnation= %lld ***\n", rx->mxc_incarnation);
191 CNETERR("*** deadline= %ld ***\n", rx->mxc_deadline);
192 CNETERR("*** state= %s ***\n", mxlnd_ctxstate_to_str(rx->mxc_state));
193 CNETERR("*** listed?= %d ***\n", !cfs_list_empty(&rx->mxc_list));
194 CNETERR("*** nid= 0x%llx ***\n", rx->mxc_nid);
195 CNETERR("*** peer= 0x%p ***\n", rx->mxc_peer);
196 CNETERR("*** msg_type= %s ***\n", mxlnd_msgtype_to_str(rx->mxc_msg_type));
197 CNETERR("*** cookie= 0x%llx ***\n", rx->mxc_cookie);
198 CNETERR("*** nob= %d ***\n", rx->mxc_nob);
201 LASSERT (rx->mxc_get == rx->mxc_put);
205 LASSERT (rx->mxc_state == MXLND_CTX_IDLE);
206 rx->mxc_state = MXLND_CTX_PREP;
207 rx->mxc_deadline = jiffies + MXLND_COMM_TIMEOUT;
213 mxlnd_put_idle_rx(kmx_ctx_t *rx)
215 kmx_conn_t *conn = rx->mxc_conn;
216 cfs_list_t *rxs = &conn->mxk_rx_idle;
218 LASSERT(rx->mxc_type == MXLND_REQ_RX);
223 LASSERT(rx->mxc_get == rx->mxc_put);
225 spin_lock(&conn->mxk_lock);
226 cfs_list_add(&rx->mxc_list, rxs);
227 spin_unlock(&conn->mxk_lock);
232 mxlnd_get_idle_tx(void)
234 cfs_list_t *tmp = &kmxlnd_data.kmx_tx_idle;
235 kmx_ctx_t *tx = NULL;
237 spin_lock(&kmxlnd_data.kmx_tx_idle_lock);
239 if (cfs_list_empty (&kmxlnd_data.kmx_tx_idle)) {
240 CNETERR("%d txs in use\n", kmxlnd_data.kmx_tx_used);
241 spin_unlock(&kmxlnd_data.kmx_tx_idle_lock);
245 tmp = &kmxlnd_data.kmx_tx_idle;
246 tx = cfs_list_entry (tmp->next, kmx_ctx_t, mxc_list);
247 cfs_list_del_init(&tx->mxc_list);
249 /* Allocate a new completion cookie. It might not be needed,
250 * but we've got a lock right now and we're unlikely to
252 tx->mxc_cookie = kmxlnd_data.kmx_tx_next_cookie++;
253 if (kmxlnd_data.kmx_tx_next_cookie > MXLND_MAX_COOKIE) {
254 kmxlnd_data.kmx_tx_next_cookie = 1;
256 kmxlnd_data.kmx_tx_used++;
257 spin_unlock(&kmxlnd_data.kmx_tx_idle_lock);
259 LASSERT (tx->mxc_get == tx->mxc_put);
263 LASSERT (tx->mxc_state == MXLND_CTX_IDLE);
264 LASSERT (tx->mxc_lntmsg[0] == NULL);
265 LASSERT (tx->mxc_lntmsg[1] == NULL);
267 tx->mxc_state = MXLND_CTX_PREP;
268 tx->mxc_deadline = jiffies + MXLND_COMM_TIMEOUT;
274 mxlnd_conn_disconnect(kmx_conn_t *conn, int mx_dis, int send_bye);
277 mxlnd_put_idle_tx(kmx_ctx_t *tx)
280 lnet_msg_t *lntmsg[2];
282 LASSERT(tx->mxc_type == MXLND_REQ_TX);
284 if (tx->mxc_status.code != MX_STATUS_SUCCESS || tx->mxc_errno != 0) {
285 kmx_conn_t *conn = tx->mxc_conn;
288 if (tx->mxc_errno != 0) result = tx->mxc_errno;
289 /* FIXME should we set mx_dis? */
290 mxlnd_conn_disconnect(conn, 0, 1);
293 lntmsg[0] = tx->mxc_lntmsg[0];
294 lntmsg[1] = tx->mxc_lntmsg[1];
299 LASSERT(tx->mxc_get == tx->mxc_put);
301 spin_lock(&kmxlnd_data.kmx_tx_idle_lock);
302 cfs_list_add_tail(&tx->mxc_list, &kmxlnd_data.kmx_tx_idle);
303 kmxlnd_data.kmx_tx_used--;
304 spin_unlock(&kmxlnd_data.kmx_tx_idle_lock);
306 if (lntmsg[0] != NULL)
307 lnet_finalize(kmxlnd_data.kmx_ni, lntmsg[0], result);
308 if (lntmsg[1] != NULL)
309 lnet_finalize(kmxlnd_data.kmx_ni, lntmsg[1], result);
315 mxlnd_connparams_free(kmx_connparams_t *cp)
317 LASSERT(cfs_list_empty(&cp->mxr_list));
318 MXLND_FREE(cp, sizeof(*cp));
323 mxlnd_connparams_alloc(kmx_connparams_t **cp, void *context,
324 mx_endpoint_addr_t epa, u64 match, u32 length,
325 kmx_conn_t *conn, kmx_peer_t *peer, void *data)
327 kmx_connparams_t *c = NULL;
329 MXLND_ALLOC(c, sizeof(*c));
330 if (!c) return -ENOMEM;
332 CFS_INIT_LIST_HEAD(&c->mxr_list);
333 c->mxr_context = context;
335 c->mxr_match = match;
339 c->mxr_msg = *((kmx_msg_t *) data);
346 mxlnd_set_conn_status(kmx_conn_t *conn, int status)
348 conn->mxk_status = status;
353 * mxlnd_conn_free_locked - free the conn
354 * @conn - a kmx_conn pointer
356 * The calling function should remove the conn from the conns list first
357 * then destroy it. Caller should have write-locked kmx_global_lock.
360 mxlnd_conn_free_locked(kmx_conn_t *conn)
362 int valid = !mxlnd_endpoint_addr_null(conn->mxk_epa);
363 kmx_peer_t *peer = conn->mxk_peer;
365 CDEBUG(D_NET, "freeing conn 0x%p *****\n", conn);
366 LASSERT (cfs_list_empty (&conn->mxk_tx_credit_queue) &&
367 cfs_list_empty (&conn->mxk_tx_free_queue) &&
368 cfs_list_empty (&conn->mxk_pending));
369 if (!cfs_list_empty(&conn->mxk_list)) {
370 cfs_list_del_init(&conn->mxk_list);
371 if (peer->mxp_conn == conn) {
372 peer->mxp_conn = NULL;
374 kmx_conn_t *temp = NULL;
376 mx_get_endpoint_addr_context(conn->mxk_epa,
379 mx_set_endpoint_addr_context(conn->mxk_epa,
383 /* unlink from global list and drop its ref */
384 cfs_list_del_init(&peer->mxp_list);
385 mxlnd_peer_decref(peer);
388 mxlnd_peer_decref(peer); /* drop conn's ref to peer */
389 if (conn->mxk_rx_pages) {
390 LASSERT (conn->mxk_rxs != NULL);
391 mxlnd_free_pages(conn->mxk_rx_pages);
395 kmx_ctx_t *rx = NULL;
397 for (i = 0; i < MXLND_RX_MSGS(); i++) {
398 rx = &conn->mxk_rxs[i];
399 if (rx->mxc_seg_list != NULL) {
400 LASSERT(rx->mxc_nseg > 0);
401 MXLND_FREE(rx->mxc_seg_list,
403 sizeof(*rx->mxc_seg_list));
406 MXLND_FREE(conn->mxk_rxs, MXLND_RX_MSGS() * sizeof(kmx_ctx_t));
409 MXLND_FREE(conn, sizeof (*conn));
415 mxlnd_conn_cancel_pending_rxs(kmx_conn_t *conn)
419 kmx_ctx_t *ctx = NULL;
420 kmx_ctx_t *next = NULL;
421 mx_return_t mxret = MX_SUCCESS;
426 spin_lock(&conn->mxk_lock);
427 cfs_list_for_each_entry_safe(ctx, next, &conn->mxk_pending,
429 cfs_list_del_init(&ctx->mxc_list);
430 if (ctx->mxc_type == MXLND_REQ_RX) {
432 mxret = mx_cancel(kmxlnd_data.kmx_endpt,
435 if (mxret != MX_SUCCESS) {
436 CNETERR("mx_cancel() returned %s (%d)\n", mx_strerror(mxret), mxret);
439 ctx->mxc_errno = -ECONNABORTED;
440 ctx->mxc_state = MXLND_CTX_CANCELED;
441 spin_unlock(&conn->mxk_lock);
442 spin_lock(&kmxlnd_data.kmx_conn_lock);
443 /* we may be holding the global lock,
444 * move to orphan list so that it can free it */
445 cfs_list_add_tail(&ctx->mxc_list,
446 &kmxlnd_data.kmx_orphan_msgs);
448 spin_unlock(&kmxlnd_data.kmx_conn_lock);
449 spin_lock(&conn->mxk_lock);
454 spin_unlock(&conn->mxk_lock);
461 mxlnd_cancel_queued_txs(kmx_conn_t *conn)
464 cfs_list_t *tmp = NULL;
466 spin_lock(&conn->mxk_lock);
467 while (!cfs_list_empty(&conn->mxk_tx_free_queue) ||
468 !cfs_list_empty(&conn->mxk_tx_credit_queue)) {
470 kmx_ctx_t *tx = NULL;
472 if (!cfs_list_empty(&conn->mxk_tx_free_queue)) {
473 tmp = &conn->mxk_tx_free_queue;
475 tmp = &conn->mxk_tx_credit_queue;
478 tx = cfs_list_entry(tmp->next, kmx_ctx_t, mxc_list);
479 cfs_list_del_init(&tx->mxc_list);
480 spin_unlock(&conn->mxk_lock);
481 tx->mxc_errno = -ECONNABORTED;
482 tx->mxc_state = MXLND_CTX_CANCELED;
483 /* move to orphan list and then abort */
484 spin_lock(&kmxlnd_data.kmx_conn_lock);
485 cfs_list_add_tail(&tx->mxc_list, &kmxlnd_data.kmx_orphan_msgs);
486 spin_unlock(&kmxlnd_data.kmx_conn_lock);
488 spin_lock(&conn->mxk_lock);
490 spin_unlock(&conn->mxk_lock);
496 mxlnd_send_message(mx_endpoint_addr_t epa, u8 msg_type, int error, u64 cookie)
498 u64 match = (((u64) msg_type) << MXLND_MSG_OFFSET) |
499 (((u64) error) << MXLND_ERROR_OFFSET) | cookie;
501 mx_kisend(kmxlnd_data.kmx_endpt, NULL, 0, MX_PIN_PHYSICAL,
502 epa, match, NULL, NULL);
507 * mxlnd_conn_disconnect - shutdown a connection
508 * @conn - a kmx_conn pointer
509 * @mx_dis - call mx_disconnect()
510 * @send_bye - send peer a BYE msg
512 * This function sets the status to DISCONNECT, completes queued
513 * txs with failure, calls mx_disconnect, which will complete
514 * pending txs and matched rxs with failure.
517 mxlnd_conn_disconnect(kmx_conn_t *conn, int mx_dis, int send_bye)
519 mx_endpoint_addr_t epa = conn->mxk_epa;
520 int valid = !mxlnd_endpoint_addr_null(epa);
523 spin_lock(&conn->mxk_lock);
524 if (conn->mxk_status == MXLND_CONN_DISCONNECT) {
525 spin_unlock(&conn->mxk_lock);
528 mxlnd_set_conn_status(conn, MXLND_CONN_DISCONNECT);
529 conn->mxk_timeout = 0;
530 spin_unlock(&conn->mxk_lock);
532 count = mxlnd_cancel_queued_txs(conn);
533 count += mxlnd_conn_cancel_pending_rxs(conn);
535 if (count) /* let connd call kmxlnd_abort_msgs() */
536 up(&kmxlnd_data.kmx_conn_sem);
538 if (send_bye && valid &&
539 conn->mxk_peer->mxp_nid != kmxlnd_data.kmx_ni->ni_nid) {
540 /* send a BYE to the peer */
541 CDEBUG(D_NET, "%s: sending a BYE msg to %s\n", __func__,
542 libcfs_nid2str(conn->mxk_peer->mxp_nid));
543 mxlnd_send_message(epa, MXLND_MSG_BYE, 0, 0);
544 /* wait to allow the peer to ack our message */
545 mxlnd_sleep(msecs_to_jiffies(20));
548 if (atomic_read(&kmxlnd_data.kmx_shutdown) != 1) {
549 unsigned long last_msg = 0;
551 /* notify LNET that we are giving up on this peer */
552 if (cfs_time_after(conn->mxk_last_rx, conn->mxk_last_tx))
553 last_msg = conn->mxk_last_rx;
555 last_msg = conn->mxk_last_tx;
557 lnet_notify(kmxlnd_data.kmx_ni, conn->mxk_peer->mxp_nid, 0, last_msg);
559 if (mx_dis && valid &&
560 (memcmp(&epa, &kmxlnd_data.kmx_epa, sizeof(epa) != 0)))
561 mx_disconnect(kmxlnd_data.kmx_endpt, epa);
563 mxlnd_conn_decref(conn); /* drop the owning peer's reference */
569 * mxlnd_conn_alloc - allocate and initialize a new conn struct
570 * @connp - address of a kmx_conn pointer
571 * @peer - owning kmx_peer
573 * Returns 0 on success and -ENOMEM on failure
576 mxlnd_conn_alloc_locked(kmx_conn_t **connp, kmx_peer_t *peer)
583 kmx_conn_t *conn = NULL;
584 kmx_pages_t *pages = NULL;
585 struct page *page = NULL;
586 kmx_ctx_t *rx = NULL;
588 LASSERT(peer != NULL);
590 MXLND_ALLOC(conn, sizeof (*conn));
592 CNETERR("Cannot allocate conn\n");
595 CDEBUG(D_NET, "allocated conn 0x%p for peer 0x%p\n", conn, peer);
597 memset(conn, 0, sizeof(*conn));
599 ret = mxlnd_alloc_pages(&pages, MXLND_RX_MSG_PAGES());
601 CERROR("Can't allocate rx pages\n");
602 MXLND_FREE(conn, sizeof(*conn));
605 conn->mxk_rx_pages = pages;
607 MXLND_ALLOC(conn->mxk_rxs, MXLND_RX_MSGS() * sizeof(kmx_ctx_t));
608 if (conn->mxk_rxs == NULL) {
609 CERROR("Can't allocate %d rx descriptors\n", MXLND_RX_MSGS());
610 mxlnd_free_pages(pages);
611 MXLND_FREE(conn, sizeof(*conn));
615 memset(conn->mxk_rxs, 0, MXLND_RX_MSGS() * sizeof(kmx_ctx_t));
617 conn->mxk_peer = peer;
618 CFS_INIT_LIST_HEAD(&conn->mxk_list);
619 CFS_INIT_LIST_HEAD(&conn->mxk_zombie);
620 atomic_set(&conn->mxk_refcount, 2); /* ref for owning peer
621 and one for the caller */
622 if (peer->mxp_nid == kmxlnd_data.kmx_ni->ni_nid) {
626 /* this is localhost, set the epa and status as up */
627 mxlnd_set_conn_status(conn, MXLND_CONN_READY);
628 conn->mxk_epa = kmxlnd_data.kmx_epa;
629 mx_set_endpoint_addr_context(conn->mxk_epa, (void *) conn);
630 peer->mxp_reconnect_time = 0;
631 mx_decompose_endpoint_addr(kmxlnd_data.kmx_epa, &nic_id, &ep_id);
632 peer->mxp_nic_id = nic_id;
633 peer->mxp_ep_id = ep_id;
634 conn->mxk_incarnation = kmxlnd_data.kmx_incarnation;
635 conn->mxk_timeout = 0;
637 /* conn->mxk_incarnation = 0 - will be set by peer */
638 /* conn->mxk_sid = 0 - will be set by peer */
639 mxlnd_set_conn_status(conn, MXLND_CONN_INIT);
640 /* mxk_epa - to be set after mx_iconnect() */
642 spin_lock_init(&conn->mxk_lock);
643 /* conn->mxk_timeout = 0 */
644 /* conn->mxk_last_tx = 0 */
645 /* conn->mxk_last_rx = 0 */
646 CFS_INIT_LIST_HEAD(&conn->mxk_rx_idle);
648 conn->mxk_credits = *kmxlnd_tunables.kmx_peercredits;
649 /* mxk_outstanding = 0 */
651 CFS_INIT_LIST_HEAD(&conn->mxk_tx_credit_queue);
652 CFS_INIT_LIST_HEAD(&conn->mxk_tx_free_queue);
653 /* conn->mxk_ntx_msgs = 0 */
654 /* conn->mxk_ntx_data = 0 */
655 /* conn->mxk_ntx_posted = 0 */
656 /* conn->mxk_data_posted = 0 */
657 CFS_INIT_LIST_HEAD(&conn->mxk_pending);
659 for (i = 0; i < MXLND_RX_MSGS(); i++) {
661 rx = &conn->mxk_rxs[i];
662 rx->mxc_type = MXLND_REQ_RX;
663 CFS_INIT_LIST_HEAD(&rx->mxc_list);
665 /* map mxc_msg to page */
666 page = pages->mxg_pages[ipage];
667 addr = page_address(page);
668 LASSERT(addr != NULL);
669 rx->mxc_msg = (kmx_msg_t *)(addr + offset);
670 rx->mxc_seg.segment_ptr = MX_PA_TO_U64(virt_to_phys(rx->mxc_msg));
674 rx->mxc_nid = peer->mxp_nid;
678 offset += MXLND_MSG_SIZE;
679 LASSERT (offset <= PAGE_SIZE);
681 if (offset == PAGE_SIZE) {
684 LASSERT (ipage <= MXLND_TX_MSG_PAGES());
687 cfs_list_add_tail(&rx->mxc_list, &conn->mxk_rx_idle);
692 mxlnd_peer_addref(peer); /* add a ref for this conn */
694 /* add to front of peer's conns list */
695 cfs_list_add(&conn->mxk_list, &peer->mxp_conns);
696 peer->mxp_conn = conn;
701 mxlnd_conn_alloc(kmx_conn_t **connp, kmx_peer_t *peer)
704 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
707 ret = mxlnd_conn_alloc_locked(connp, peer);
708 write_unlock(g_lock);
713 mxlnd_q_pending_ctx(kmx_ctx_t *ctx)
716 kmx_conn_t *conn = ctx->mxc_conn;
718 ctx->mxc_state = MXLND_CTX_PENDING;
720 spin_lock(&conn->mxk_lock);
721 if (conn->mxk_status >= MXLND_CONN_INIT) {
722 cfs_list_add_tail(&ctx->mxc_list, &conn->mxk_pending);
723 if (conn->mxk_timeout == 0 || ctx->mxc_deadline < conn->mxk_timeout) {
724 conn->mxk_timeout = ctx->mxc_deadline;
727 ctx->mxc_state = MXLND_CTX_COMPLETED;
730 spin_unlock(&conn->mxk_lock);
736 mxlnd_deq_pending_ctx(kmx_ctx_t *ctx)
738 LASSERT(ctx->mxc_state == MXLND_CTX_PENDING ||
739 ctx->mxc_state == MXLND_CTX_COMPLETED);
740 if (ctx->mxc_state != MXLND_CTX_PENDING &&
741 ctx->mxc_state != MXLND_CTX_COMPLETED) {
742 CNETERR("deq ctx->mxc_state = %s\n",
743 mxlnd_ctxstate_to_str(ctx->mxc_state));
745 ctx->mxc_state = MXLND_CTX_COMPLETED;
746 if (!cfs_list_empty(&ctx->mxc_list)) {
747 kmx_conn_t *conn = ctx->mxc_conn;
748 kmx_ctx_t *next = NULL;
750 LASSERT(conn != NULL);
751 spin_lock(&conn->mxk_lock);
752 cfs_list_del_init(&ctx->mxc_list);
753 conn->mxk_timeout = 0;
754 if (!cfs_list_empty(&conn->mxk_pending)) {
755 next = cfs_list_entry(conn->mxk_pending.next,
756 kmx_ctx_t, mxc_list);
757 conn->mxk_timeout = next->mxc_deadline;
759 spin_unlock(&conn->mxk_lock);
765 * mxlnd_peer_free - free the peer
766 * @peer - a kmx_peer pointer
768 * The calling function should decrement the rxs, drain the tx queues and
769 * remove the peer from the peers list first then destroy it.
772 mxlnd_peer_free(kmx_peer_t *peer)
774 CDEBUG(D_NET, "freeing peer 0x%p %s\n", peer, libcfs_nid2str(peer->mxp_nid));
776 LASSERT (atomic_read(&peer->mxp_refcount) == 0);
778 if (!cfs_list_empty(&peer->mxp_list)) {
779 /* assume we are locked */
780 cfs_list_del_init(&peer->mxp_list);
783 MXLND_FREE(peer, sizeof (*peer));
784 atomic_dec(&kmxlnd_data.kmx_npeers);
789 mxlnd_lookup_mac(u32 ip, u64 *tmp_id)
791 int ret = -EHOSTUNREACH;
792 unsigned char *haddr = NULL;
793 struct net_device *dev = NULL;
794 struct neighbour *n = NULL;
795 __be32 dst_ip = htonl(ip);
797 dev = dev_get_by_name(*kmxlnd_tunables.kmx_default_ipif);
801 haddr = (unsigned char *) tmp_id + 2; /* MAC is only 6 bytes */
803 n = neigh_lookup(&arp_tbl, &dst_ip, dev);
806 if (n->nud_state & NUD_VALID) {
807 memcpy(haddr, n->ha, dev->addr_len);
819 /* We only want the MAC address of the peer's Myricom NIC. We
820 * require that each node has the IPoMX interface (myriN) up.
821 * We will not pass any traffic over IPoMX, but it allows us
822 * to get the MAC address. */
824 mxlnd_ip2nic_id(u32 ip, u64 *nic_id, int tries)
830 cfs_socket_t *sock = NULL;
833 CDEBUG(D_NET, "try %d of %d tries\n", try, tries);
834 ret = mxlnd_lookup_mac(ip, &tmp_id);
838 /* not found, try to connect (force an arp) */
839 ret = libcfs_sock_connect(&sock, &fatal, 0, 0, ip, 987);
840 if (ret == -ECONNREFUSED) {
841 /* peer is there, get the MAC address */
842 mxlnd_lookup_mac(ip, &tmp_id);
846 } else if (ret == -EHOSTUNREACH && try < tries) {
847 /* add a little backoff */
848 CDEBUG(D_NET, "sleeping for %d jiffies\n",
853 } while (try++ < tries);
854 CDEBUG(D_NET, "done trying. ret = %d\n", ret);
858 #ifdef __LITTLE_ENDIAN
859 *nic_id = ___arch__swab64(tmp_id);
867 * mxlnd_peer_alloc - allocate and initialize a new peer struct
868 * @peerp - address of a kmx_peer pointer
869 * @nid - LNET node id
871 * Returns 0 on success and -ENOMEM on failure
874 mxlnd_peer_alloc(kmx_peer_t **peerp, lnet_nid_t nid, u32 board, u32 ep_id, u64 nic_id)
877 u32 ip = LNET_NIDADDR(nid);
878 kmx_peer_t *peer = NULL;
880 LASSERT (nid != LNET_NID_ANY && nid != 0LL);
882 MXLND_ALLOC(peer, sizeof (*peer));
884 CNETERR("Cannot allocate peer for NID 0x%llx\n",
888 CDEBUG(D_NET, "allocated peer 0x%p for NID 0x%llx\n", peer, nid);
890 memset(peer, 0, sizeof(*peer));
892 CFS_INIT_LIST_HEAD(&peer->mxp_list);
894 /* peer->mxp_ni unused - may be used for multi-rail */
895 atomic_set(&peer->mxp_refcount, 1); /* ref for kmx_peers list */
897 peer->mxp_board = board;
898 peer->mxp_ep_id = ep_id;
899 peer->mxp_nic_id = nic_id;
901 CFS_INIT_LIST_HEAD(&peer->mxp_conns);
902 ret = mxlnd_conn_alloc(&peer->mxp_conn, peer); /* adds 2nd conn ref here... */
904 mxlnd_peer_decref(peer);
907 CFS_INIT_LIST_HEAD(&peer->mxp_tx_queue);
909 if (peer->mxp_nic_id != 0ULL)
910 nic_id = peer->mxp_nic_id;
912 if (nic_id == 0ULL) {
913 ret = mxlnd_ip2nic_id(ip, &nic_id, 1);
915 peer->mxp_nic_id = nic_id;
916 mx_nic_id_to_board_number(nic_id, &peer->mxp_board);
920 peer->mxp_nic_id = nic_id; /* may be 0ULL if ip2nic_id() failed */
922 /* peer->mxp_reconnect_time = 0 */
923 /* peer->mxp_incompatible = 0 */
929 static inline kmx_peer_t *
930 mxlnd_find_peer_by_nid_locked(lnet_nid_t nid)
934 kmx_peer_t *peer = NULL;
936 hash = mxlnd_nid_to_hash(nid);
938 cfs_list_for_each_entry(peer, &kmxlnd_data.kmx_peers[hash], mxp_list) {
939 if (peer->mxp_nid == nid) {
941 mxlnd_peer_addref(peer);
945 return (found ? peer : NULL);
949 mxlnd_find_peer_by_nid(lnet_nid_t nid, int create)
953 kmx_peer_t *peer = NULL;
954 kmx_peer_t *old = NULL;
955 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
958 peer = mxlnd_find_peer_by_nid_locked(nid); /* adds peer ref */
960 if ((peer && peer->mxp_conn) || /* found peer with conn or */
961 (!peer && !create)) { /* did not find peer and do not create one */
968 /* if peer but _not_ conn */
969 if (peer && !peer->mxp_conn) {
972 if (!peer->mxp_conn) { /* check again */
973 /* create the conn */
974 ret = mxlnd_conn_alloc_locked(&peer->mxp_conn, peer);
976 /* we tried, return the peer only.
977 * the caller needs to see if the conn exists */
978 CNETERR("%s: %s could not alloc conn\n",
979 __func__, libcfs_nid2str(peer->mxp_nid));
981 /* drop extra conn ref */
982 mxlnd_conn_decref(peer->mxp_conn);
985 write_unlock(g_lock);
990 /* peer not found and we need to create one */
991 hash = mxlnd_nid_to_hash(nid);
993 /* create peer (and conn) */
994 /* adds conn ref for peer and one for this function */
995 ret = mxlnd_peer_alloc(&peer, nid, *kmxlnd_tunables.kmx_board,
996 *kmxlnd_tunables.kmx_ep_id, 0ULL);
997 if (ret != 0) /* no memory, peer is NULL */
1003 old = mxlnd_find_peer_by_nid_locked(nid);
1005 /* someone already created one */
1006 mxlnd_conn_decref(peer->mxp_conn); /* drop ref taken above.. */
1007 mxlnd_conn_decref(peer->mxp_conn); /* drop peer's ref */
1008 mxlnd_peer_decref(peer);
1011 /* no other peer, use this one */
1012 cfs_list_add_tail(&peer->mxp_list,
1013 &kmxlnd_data.kmx_peers[hash]);
1014 atomic_inc(&kmxlnd_data.kmx_npeers);
1015 mxlnd_peer_addref(peer);
1016 mxlnd_conn_decref(peer->mxp_conn); /* drop ref from peer_alloc */
1019 write_unlock(g_lock);
1025 mxlnd_tx_requires_credit(kmx_ctx_t *tx)
1027 return (tx->mxc_msg_type == MXLND_MSG_EAGER ||
1028 tx->mxc_msg_type == MXLND_MSG_GET_REQ ||
1029 tx->mxc_msg_type == MXLND_MSG_PUT_REQ ||
1030 tx->mxc_msg_type == MXLND_MSG_NOOP);
1034 * mxlnd_init_msg - set type and number of bytes
1035 * @msg - msg pointer
1036 * @type - of message
1037 * @body_nob - bytes in msg body
1040 mxlnd_init_msg(kmx_msg_t *msg, u8 type, int body_nob)
1042 msg->mxm_type = type;
1043 msg->mxm_nob = offsetof(kmx_msg_t, mxm_u) + body_nob;
1047 mxlnd_init_tx_msg (kmx_ctx_t *tx, u8 type, int body_nob, lnet_nid_t nid)
1049 int nob = offsetof (kmx_msg_t, mxm_u) + body_nob;
1050 kmx_msg_t *msg = NULL;
1052 LASSERT (tx != NULL);
1053 LASSERT (nob <= MXLND_MSG_SIZE);
1056 /* tx->mxc_peer should have already been set if we know it */
1057 tx->mxc_msg_type = type;
1059 /* tx->mxc_seg.segment_ptr is already pointing to mxc_page */
1060 tx->mxc_seg.segment_length = nob;
1061 tx->mxc_pin_type = MX_PIN_PHYSICAL;
1064 msg->mxm_type = type;
1071 mxlnd_cksum (void *ptr, int nob)
1077 sum = ((sum << 1) | (sum >> 31)) + *c++;
1079 /* ensure I don't return 0 (== no checksum) */
1080 return (sum == 0) ? 1 : sum;
1084 * mxlnd_pack_msg_locked - complete msg info
1088 mxlnd_pack_msg_locked(kmx_ctx_t *tx)
1090 kmx_msg_t *msg = tx->mxc_msg;
1092 /* type and nob should already be set in init_msg() */
1093 msg->mxm_magic = MXLND_MSG_MAGIC;
1094 msg->mxm_version = MXLND_MSG_VERSION;
1096 /* don't use mxlnd_tx_requires_credit() since we want PUT_ACK to
1097 * return credits as well */
1098 if (tx->mxc_msg_type != MXLND_MSG_CONN_REQ &&
1099 tx->mxc_msg_type != MXLND_MSG_CONN_ACK) {
1100 msg->mxm_credits = tx->mxc_conn->mxk_outstanding;
1101 tx->mxc_conn->mxk_outstanding = 0;
1103 msg->mxm_credits = 0;
1107 msg->mxm_srcnid = kmxlnd_data.kmx_ni->ni_nid;
1108 msg->mxm_srcstamp = kmxlnd_data.kmx_incarnation;
1109 msg->mxm_dstnid = tx->mxc_nid;
1110 /* if it is a new peer, the dststamp will be 0 */
1111 msg->mxm_dststamp = tx->mxc_conn->mxk_incarnation;
1113 if (*kmxlnd_tunables.kmx_cksum) {
1114 msg->mxm_cksum = mxlnd_cksum(msg, msg->mxm_nob);
1119 mxlnd_unpack_msg(kmx_msg_t *msg, int nob)
1121 const int hdr_size = offsetof(kmx_msg_t, mxm_u);
1122 __u32 msg_cksum = 0;
1126 /* 6 bytes are enough to have received magic + version */
1128 CNETERR("not enough bytes for magic + hdr: %d\n", nob);
1132 if (msg->mxm_magic == MXLND_MSG_MAGIC) {
1134 } else if (msg->mxm_magic == __swab32(MXLND_MSG_MAGIC)) {
1137 CNETERR("Bad magic: %08x\n", msg->mxm_magic);
1141 if (msg->mxm_version !=
1142 (flip ? __swab16(MXLND_MSG_VERSION) : MXLND_MSG_VERSION)) {
1143 CNETERR("Bad version: %d\n", msg->mxm_version);
1147 if (nob < hdr_size) {
1148 CNETERR("not enough for a header: %d\n", nob);
1152 msg_nob = flip ? __swab32(msg->mxm_nob) : msg->mxm_nob;
1153 if (msg_nob > nob) {
1154 CNETERR("Short message: got %d, wanted %d\n", nob, msg_nob);
1158 /* checksum must be computed with mxm_cksum zero and BEFORE anything
1160 msg_cksum = flip ? __swab32(msg->mxm_cksum) : msg->mxm_cksum;
1162 if (msg_cksum != 0 && msg_cksum != mxlnd_cksum(msg, msg_nob)) {
1163 CNETERR("Bad checksum\n");
1166 msg->mxm_cksum = msg_cksum;
1169 /* leave magic unflipped as a clue to peer endianness */
1170 __swab16s(&msg->mxm_version);
1171 CLASSERT (sizeof(msg->mxm_type) == 1);
1172 CLASSERT (sizeof(msg->mxm_credits) == 1);
1173 msg->mxm_nob = msg_nob;
1174 __swab64s(&msg->mxm_srcnid);
1175 __swab64s(&msg->mxm_srcstamp);
1176 __swab64s(&msg->mxm_dstnid);
1177 __swab64s(&msg->mxm_dststamp);
1180 if (msg->mxm_srcnid == LNET_NID_ANY) {
1181 CNETERR("Bad src nid: %s\n", libcfs_nid2str(msg->mxm_srcnid));
1185 switch (msg->mxm_type) {
1187 CNETERR("Unknown message type %x\n", msg->mxm_type);
1190 case MXLND_MSG_NOOP:
1193 case MXLND_MSG_EAGER:
1194 if (msg_nob < offsetof(kmx_msg_t, mxm_u.eager.mxem_payload[0])) {
1195 CNETERR("Short EAGER: %d(%d)\n", msg_nob,
1196 (int)offsetof(kmx_msg_t, mxm_u.eager.mxem_payload[0]));
1201 case MXLND_MSG_PUT_REQ:
1202 if (msg_nob < hdr_size + sizeof(msg->mxm_u.put_req)) {
1203 CNETERR("Short PUT_REQ: %d(%d)\n", msg_nob,
1204 (int)(hdr_size + sizeof(msg->mxm_u.put_req)));
1208 __swab64s(&msg->mxm_u.put_req.mxprm_cookie);
1211 case MXLND_MSG_PUT_ACK:
1212 if (msg_nob < hdr_size + sizeof(msg->mxm_u.put_ack)) {
1213 CNETERR("Short PUT_ACK: %d(%d)\n", msg_nob,
1214 (int)(hdr_size + sizeof(msg->mxm_u.put_ack)));
1218 __swab64s(&msg->mxm_u.put_ack.mxpam_src_cookie);
1219 __swab64s(&msg->mxm_u.put_ack.mxpam_dst_cookie);
1223 case MXLND_MSG_GET_REQ:
1224 if (msg_nob < hdr_size + sizeof(msg->mxm_u.get_req)) {
1225 CNETERR("Short GET_REQ: %d(%d)\n", msg_nob,
1226 (int)(hdr_size + sizeof(msg->mxm_u.get_req)));
1230 __swab64s(&msg->mxm_u.get_req.mxgrm_cookie);
1234 case MXLND_MSG_CONN_REQ:
1235 case MXLND_MSG_CONN_ACK:
1236 if (msg_nob < hdr_size + sizeof(msg->mxm_u.conn_req)) {
1237 CNETERR("Short connreq/ack: %d(%d)\n", msg_nob,
1238 (int)(hdr_size + sizeof(msg->mxm_u.conn_req)));
1242 __swab32s(&msg->mxm_u.conn_req.mxcrm_queue_depth);
1243 __swab32s(&msg->mxm_u.conn_req.mxcrm_eager_size);
1253 * @lntmsg - the LNET msg that this is continuing. If EAGER, then NULL.
1257 * @length - length of incoming message
1258 * @pending - add to kmx_pending (0 is NO and 1 is YES)
1260 * The caller gets the rx and sets nid, peer and conn if known.
1262 * Returns 0 on success and -1 on failure
1265 mxlnd_recv_msg(lnet_msg_t *lntmsg, kmx_ctx_t *rx, u8 msg_type, u64 cookie, u32 length)
1268 mx_return_t mxret = MX_SUCCESS;
1269 uint64_t mask = ~(MXLND_ERROR_MASK);
1271 rx->mxc_msg_type = msg_type;
1272 rx->mxc_lntmsg[0] = lntmsg; /* may be NULL if EAGER */
1273 rx->mxc_cookie = cookie;
1274 /* rx->mxc_match may already be set */
1275 /* rx->mxc_seg.segment_ptr is already set */
1276 rx->mxc_seg.segment_length = length;
1277 ret = mxlnd_q_pending_ctx(rx);
1279 /* the caller is responsible for calling conn_decref() if needed */
1282 mxret = mx_kirecv(kmxlnd_data.kmx_endpt, &rx->mxc_seg, 1, MX_PIN_PHYSICAL,
1283 cookie, mask, (void *) rx, &rx->mxc_mxreq);
1284 if (mxret != MX_SUCCESS) {
1285 mxlnd_deq_pending_ctx(rx);
1286 CNETERR("mx_kirecv() failed with %s (%d)\n",
1287 mx_strerror(mxret), (int) mxret);
1295 * mxlnd_unexpected_recv - this is the callback function that will handle
1296 * unexpected receives
1297 * @context - NULL, ignore
1298 * @source - the peer's mx_endpoint_addr_t
1299 * @match_value - the msg's bits, should be MXLND_MSG_EAGER
1300 * @length - length of incoming message
1301 * @data_if_available - used for CONN_[REQ|ACK]
1303 * If it is an eager-sized msg, we will call recv_msg() with the actual
1304 * length. If it is a large message, we will call recv_msg() with a
1305 * length of 0 bytes to drop it because we should never have a large,
1306 * unexpected message.
1308 * NOTE - The MX library blocks until this function completes. Make it as fast as
1309 * possible. DO NOT allocate memory which can block!
1311 * If we cannot get a rx or the conn is closed, drop the message on the floor
1312 * (i.e. recv 0 bytes and ignore).
1314 mx_unexp_handler_action_t
1315 mxlnd_unexpected_recv(void *context, mx_endpoint_addr_t source,
1316 uint64_t match_value, uint32_t length, void *data_if_available)
1319 kmx_ctx_t *rx = NULL;
1324 kmx_conn_t *conn = NULL;
1325 kmx_peer_t *peer = NULL;
1330 /* TODO this will change to the net struct */
1331 if (context != NULL) {
1332 CNETERR("non-NULL context\n");
1336 CDEBUG(D_NET, "bits=0x%llx length=%d\n", match_value, length);
1339 mx_decompose_endpoint_addr2(source, &nic_id, &ep_id, &sid);
1340 mxlnd_parse_match(match_value, &msg_type, &error, &cookie);
1341 read_lock(&kmxlnd_data.kmx_global_lock);
1342 mx_get_endpoint_addr_context(source, (void **) &conn);
1344 mxlnd_conn_addref(conn); /* add ref for this function */
1345 peer = conn->mxk_peer;
1347 read_unlock(&kmxlnd_data.kmx_global_lock);
1349 if (msg_type == MXLND_MSG_BYE) {
1351 CDEBUG(D_NET, "peer %s sent BYE msg\n",
1352 libcfs_nid2str(peer->mxp_nid));
1353 mxlnd_conn_disconnect(conn, 1, 0);
1354 mxlnd_conn_decref(conn); /* drop ref taken above */
1356 return MX_RECV_FINISHED;
1359 if (msg_type == MXLND_MSG_CONN_REQ) {
1360 kmx_connparams_t *cp = NULL;
1361 const int expected = offsetof(kmx_msg_t, mxm_u) +
1362 sizeof(kmx_connreq_msg_t);
1364 if (conn) mxlnd_conn_decref(conn); /* drop ref taken above */
1365 if (unlikely(length != expected || !data_if_available)) {
1366 CNETERR("received invalid CONN_REQ from %llx "
1367 "length=%d (expected %d)\n", nic_id, length, expected);
1368 mxlnd_send_message(source, MXLND_MSG_CONN_ACK, EPROTO, 0);
1369 return MX_RECV_FINISHED;
1372 ret = mxlnd_connparams_alloc(&cp, context, source, match_value, length,
1373 conn, peer, data_if_available);
1374 if (unlikely(ret != 0)) {
1375 CNETERR("unable to alloc CONN_REQ from %llx:%d\n",
1377 mxlnd_send_message(source, MXLND_MSG_CONN_ACK, ENOMEM, 0);
1378 return MX_RECV_FINISHED;
1380 spin_lock(&kmxlnd_data.kmx_conn_lock);
1381 cfs_list_add_tail(&cp->mxr_list, &kmxlnd_data.kmx_conn_reqs);
1382 spin_unlock(&kmxlnd_data.kmx_conn_lock);
1383 up(&kmxlnd_data.kmx_conn_sem);
1384 return MX_RECV_FINISHED;
1386 if (msg_type == MXLND_MSG_CONN_ACK) {
1387 kmx_connparams_t *cp = NULL;
1388 const int expected = offsetof(kmx_msg_t, mxm_u) +
1389 sizeof(kmx_connreq_msg_t);
1392 if (unlikely(error != 0)) {
1393 CNETERR("received CONN_ACK from %s with error -%d\n",
1394 libcfs_nid2str(peer->mxp_nid), (int) error);
1395 mxlnd_conn_disconnect(conn, 1, 0);
1396 } else if (unlikely(length != expected || !data_if_available)) {
1397 CNETERR("received %s CONN_ACK from %s "
1398 "length=%d (expected %d)\n",
1399 data_if_available ? "short" : "missing",
1400 libcfs_nid2str(peer->mxp_nid), length, expected);
1401 mxlnd_conn_disconnect(conn, 1, 1);
1403 /* peer is ready for messages */
1404 ret = mxlnd_connparams_alloc(&cp, context, source, match_value, length,
1405 conn, peer, data_if_available);
1406 if (unlikely(ret != 0)) {
1407 CNETERR("unable to alloc kmx_connparams_t"
1408 " from %llx:%d\n", nic_id, ep_id);
1409 mxlnd_conn_disconnect(conn, 1, 1);
1411 spin_lock(&kmxlnd_data.kmx_conn_lock);
1412 cfs_list_add_tail(&cp->mxr_list,
1413 &kmxlnd_data.kmx_conn_reqs);
1414 spin_unlock(&kmxlnd_data.kmx_conn_lock);
1415 up(&kmxlnd_data.kmx_conn_sem);
1418 mxlnd_conn_decref(conn); /* drop ref taken above */
1420 return MX_RECV_FINISHED;
1423 /* Handle unexpected messages (PUT_REQ and GET_REQ) */
1425 LASSERT(peer != NULL && conn != NULL);
1427 rx = mxlnd_get_idle_rx(conn);
1429 if (length <= MXLND_MSG_SIZE) {
1430 ret = mxlnd_recv_msg(NULL, rx, msg_type, match_value, length);
1432 CNETERR("unexpected large receive with "
1433 "match_value=0x%llx length=%d\n",
1434 match_value, length);
1435 ret = mxlnd_recv_msg(NULL, rx, msg_type, match_value, 0);
1439 /* hold conn ref until rx completes */
1440 rx->mxc_conn = conn;
1441 rx->mxc_peer = peer;
1442 rx->mxc_nid = peer->mxp_nid;
1444 CNETERR("could not post receive\n");
1445 mxlnd_put_idle_rx(rx);
1449 /* Encountered error, drop incoming message on the floor */
1450 /* We could use MX_RECV_FINISHED but posting the receive of 0 bytes
1451 * uses the standard code path and acks the sender normally */
1453 if (rx == NULL || ret != 0) {
1454 mxlnd_conn_decref(conn); /* drop ref taken above */
1456 CNETERR("no idle rxs available - dropping rx"
1457 " 0x%llx from %s\n", match_value,
1458 libcfs_nid2str(peer->mxp_nid));
1461 CNETERR("disconnected peer - dropping rx\n");
1463 seg.segment_ptr = 0ULL;
1464 seg.segment_length = 0;
1465 mx_kirecv(kmxlnd_data.kmx_endpt, &seg, 1, MX_PIN_PHYSICAL,
1466 match_value, ~0ULL, NULL, NULL);
1469 return MX_RECV_CONTINUE;
1474 mxlnd_get_peer_info(int index, lnet_nid_t *nidp, int *count)
1478 kmx_peer_t *peer = NULL;
1480 read_lock(&kmxlnd_data.kmx_global_lock);
1481 for (i = 0; i < MXLND_HASH_SIZE; i++) {
1482 cfs_list_for_each_entry(peer, &kmxlnd_data.kmx_peers[i],
1485 *nidp = peer->mxp_nid;
1486 *count = atomic_read(&peer->mxp_refcount);
1492 read_unlock(&kmxlnd_data.kmx_global_lock);
1498 mxlnd_del_peer_locked(kmx_peer_t *peer)
1500 if (peer->mxp_conn) {
1501 mxlnd_conn_disconnect(peer->mxp_conn, 1, 1);
1503 cfs_list_del_init(&peer->mxp_list); /* remove from the global list */
1504 mxlnd_peer_decref(peer); /* drop global list ref */
1510 mxlnd_del_peer(lnet_nid_t nid)
1514 kmx_peer_t *peer = NULL;
1515 kmx_peer_t *next = NULL;
1517 if (nid != LNET_NID_ANY) {
1518 peer = mxlnd_find_peer_by_nid(nid, 0); /* adds peer ref */
1520 write_lock(&kmxlnd_data.kmx_global_lock);
1521 if (nid != LNET_NID_ANY) {
1525 mxlnd_peer_decref(peer); /* and drops it */
1526 mxlnd_del_peer_locked(peer);
1528 } else { /* LNET_NID_ANY */
1529 for (i = 0; i < MXLND_HASH_SIZE; i++) {
1530 cfs_list_for_each_entry_safe(peer, next,
1531 &kmxlnd_data.kmx_peers[i],
1533 mxlnd_del_peer_locked(peer);
1537 write_unlock(&kmxlnd_data.kmx_global_lock);
1543 mxlnd_get_conn_by_idx(int index)
1546 kmx_peer_t *peer = NULL;
1547 kmx_conn_t *conn = NULL;
1549 read_lock(&kmxlnd_data.kmx_global_lock);
1550 for (i = 0; i < MXLND_HASH_SIZE; i++) {
1551 cfs_list_for_each_entry(peer, &kmxlnd_data.kmx_peers[i],
1553 cfs_list_for_each_entry(conn, &peer->mxp_conns,
1559 mxlnd_conn_addref(conn); /* add ref here, dec in ctl() */
1560 read_unlock(&kmxlnd_data.kmx_global_lock);
1565 read_unlock(&kmxlnd_data.kmx_global_lock);
1571 mxlnd_close_matching_conns_locked(kmx_peer_t *peer)
1573 kmx_conn_t *conn = NULL;
1574 kmx_conn_t *next = NULL;
1576 cfs_list_for_each_entry_safe(conn, next, &peer->mxp_conns, mxk_list)
1577 mxlnd_conn_disconnect(conn, 0, 1);
1583 mxlnd_close_matching_conns(lnet_nid_t nid)
1587 kmx_peer_t *peer = NULL;
1589 write_lock(&kmxlnd_data.kmx_global_lock);
1590 if (nid != LNET_NID_ANY) {
1591 peer = mxlnd_find_peer_by_nid_locked(nid); /* adds peer ref */
1595 mxlnd_close_matching_conns_locked(peer);
1596 mxlnd_peer_decref(peer); /* and drops it here */
1598 } else { /* LNET_NID_ANY */
1599 for (i = 0; i < MXLND_HASH_SIZE; i++) {
1600 cfs_list_for_each_entry(peer, &kmxlnd_data.kmx_peers[i], mxp_list)
1601 mxlnd_close_matching_conns_locked(peer);
1604 write_unlock(&kmxlnd_data.kmx_global_lock);
1610 * mxlnd_ctl - modify MXLND parameters
1611 * @ni - LNET interface handle
1612 * @cmd - command to change
1613 * @arg - the ioctl data
1616 mxlnd_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg)
1618 struct libcfs_ioctl_data *data = arg;
1621 LASSERT (ni == kmxlnd_data.kmx_ni);
1624 case IOC_LIBCFS_GET_PEER: {
1628 ret = mxlnd_get_peer_info(data->ioc_count, &nid, &count);
1629 data->ioc_nid = nid;
1630 data->ioc_count = count;
1633 case IOC_LIBCFS_DEL_PEER: {
1634 ret = mxlnd_del_peer(data->ioc_nid);
1637 case IOC_LIBCFS_GET_CONN: {
1638 kmx_conn_t *conn = NULL;
1640 conn = mxlnd_get_conn_by_idx(data->ioc_count);
1645 data->ioc_nid = conn->mxk_peer->mxp_nid;
1646 mxlnd_conn_decref(conn); /* dec ref taken in get_conn_by_idx() */
1650 case IOC_LIBCFS_CLOSE_CONNECTION: {
1651 ret = mxlnd_close_matching_conns(data->ioc_nid);
1655 CNETERR("unknown ctl(%d)\n", cmd);
1663 * mxlnd_peer_queue_tx_locked - add the tx to the peer's tx queue
1666 * Add the tx to the peer's msg or data queue. The caller has locked the peer.
1669 mxlnd_peer_queue_tx_locked(kmx_ctx_t *tx)
1671 u8 msg_type = tx->mxc_msg_type;
1672 kmx_conn_t *conn = tx->mxc_conn;
1674 LASSERT (msg_type != 0);
1675 LASSERT (tx->mxc_nid != 0);
1676 LASSERT (tx->mxc_peer != NULL);
1677 LASSERT (tx->mxc_conn != NULL);
1679 tx->mxc_incarnation = conn->mxk_incarnation;
1681 if (msg_type != MXLND_MSG_PUT_DATA &&
1682 msg_type != MXLND_MSG_GET_DATA) {
1684 if (mxlnd_tx_requires_credit(tx)) {
1685 cfs_list_add_tail(&tx->mxc_list,
1686 &conn->mxk_tx_credit_queue);
1687 conn->mxk_ntx_msgs++;
1688 } else if (msg_type == MXLND_MSG_CONN_REQ ||
1689 msg_type == MXLND_MSG_CONN_ACK) {
1690 /* put conn msgs at the front of the queue */
1691 cfs_list_add(&tx->mxc_list, &conn->mxk_tx_free_queue);
1693 /* PUT_ACK, PUT_NAK */
1694 cfs_list_add_tail(&tx->mxc_list,
1695 &conn->mxk_tx_free_queue);
1696 conn->mxk_ntx_msgs++;
1700 cfs_list_add_tail(&tx->mxc_list, &conn->mxk_tx_free_queue);
1701 conn->mxk_ntx_data++;
1708 * mxlnd_peer_queue_tx - add the tx to the global tx queue
1711 * Add the tx to the peer's msg or data queue
1714 mxlnd_peer_queue_tx(kmx_ctx_t *tx)
1716 LASSERT(tx->mxc_peer != NULL);
1717 LASSERT(tx->mxc_conn != NULL);
1718 spin_lock(&tx->mxc_conn->mxk_lock);
1719 mxlnd_peer_queue_tx_locked(tx);
1720 spin_unlock(&tx->mxc_conn->mxk_lock);
1726 * mxlnd_queue_tx - add the tx to the global tx queue
1729 * Add the tx to the global queue and up the tx_queue_sem
1732 mxlnd_queue_tx(kmx_ctx_t *tx)
1734 kmx_peer_t *peer = tx->mxc_peer;
1735 LASSERT (tx->mxc_nid != 0);
1738 if (peer->mxp_incompatible &&
1739 tx->mxc_msg_type != MXLND_MSG_CONN_ACK) {
1740 /* let this fail now */
1741 tx->mxc_errno = -ECONNABORTED;
1742 mxlnd_conn_decref(peer->mxp_conn);
1743 mxlnd_put_idle_tx(tx);
1746 if (tx->mxc_conn == NULL) {
1748 kmx_conn_t *conn = NULL;
1750 ret = mxlnd_conn_alloc(&conn, peer); /* adds 2nd ref for tx... */
1752 tx->mxc_errno = ret;
1753 mxlnd_put_idle_tx(tx);
1756 tx->mxc_conn = conn;
1757 mxlnd_peer_decref(peer); /* and takes it from peer */
1759 LASSERT(tx->mxc_conn != NULL);
1760 mxlnd_peer_queue_tx(tx);
1761 mxlnd_check_sends(peer);
1763 spin_lock(&kmxlnd_data.kmx_tx_queue_lock);
1764 cfs_list_add_tail(&tx->mxc_list, &kmxlnd_data.kmx_tx_queue);
1765 spin_unlock(&kmxlnd_data.kmx_tx_queue_lock);
1766 up(&kmxlnd_data.kmx_tx_queue_sem);
1773 mxlnd_setup_iov(kmx_ctx_t *ctx, u32 niov, struct iovec *iov, u32 offset, u32 nob)
1780 int first_iov_offset = 0;
1781 int first_found = 0;
1783 int last_iov_length = 0;
1784 mx_ksegment_t *seg = NULL;
1786 if (niov == 0) return 0;
1787 LASSERT(iov != NULL);
1789 for (i = 0; i < niov; i++) {
1790 sum = old_sum + (u32) iov[i].iov_len;
1791 if (!first_found && (sum > offset)) {
1793 first_iov_offset = offset - old_sum;
1795 sum = (u32) iov[i].iov_len - first_iov_offset;
1800 last_iov_length = (u32) iov[i].iov_len - (sum - nob);
1801 if (first_iov == last_iov) last_iov_length -= first_iov_offset;
1806 LASSERT(first_iov >= 0 && last_iov >= first_iov);
1807 nseg = last_iov - first_iov + 1;
1810 MXLND_ALLOC(seg, nseg * sizeof(*seg));
1812 CNETERR("MXLND_ALLOC() failed\n");
1815 memset(seg, 0, nseg * sizeof(*seg));
1816 ctx->mxc_nseg = nseg;
1818 for (i = 0; i < nseg; i++) {
1819 seg[i].segment_ptr = MX_PA_TO_U64(virt_to_phys(iov[first_iov + i].iov_base));
1820 seg[i].segment_length = (u32) iov[first_iov + i].iov_len;
1822 seg[i].segment_ptr += (u64) first_iov_offset;
1823 seg[i].segment_length -= (u32) first_iov_offset;
1825 if (i == (nseg - 1)) {
1826 seg[i].segment_length = (u32) last_iov_length;
1828 sum += seg[i].segment_length;
1830 ctx->mxc_seg_list = seg;
1831 ctx->mxc_pin_type = MX_PIN_PHYSICAL;
1832 #ifdef MX_PIN_FULLPAGES
1833 ctx->mxc_pin_type |= MX_PIN_FULLPAGES;
1835 LASSERT(nob == sum);
1840 mxlnd_setup_kiov(kmx_ctx_t *ctx, u32 niov, lnet_kiov_t *kiov, u32 offset, u32 nob)
1846 int first_kiov = -1;
1847 int first_kiov_offset = 0;
1848 int first_found = 0;
1850 int last_kiov_length = 0;
1851 mx_ksegment_t *seg = NULL;
1853 if (niov == 0) return 0;
1854 LASSERT(kiov != NULL);
1856 for (i = 0; i < niov; i++) {
1857 sum = old_sum + kiov[i].kiov_len;
1858 if (i == 0) sum -= kiov[i].kiov_offset;
1859 if (!first_found && (sum > offset)) {
1861 first_kiov_offset = offset - old_sum;
1862 if (i == 0) first_kiov_offset = kiov[i].kiov_offset;
1864 sum = kiov[i].kiov_len - first_kiov_offset;
1869 last_kiov_length = kiov[i].kiov_len - (sum - nob);
1870 if (first_kiov == last_kiov) last_kiov_length -= first_kiov_offset;
1875 LASSERT(first_kiov >= 0 && last_kiov >= first_kiov);
1876 nseg = last_kiov - first_kiov + 1;
1879 MXLND_ALLOC(seg, nseg * sizeof(*seg));
1881 CNETERR("MXLND_ALLOC() failed\n");
1884 memset(seg, 0, niov * sizeof(*seg));
1885 ctx->mxc_nseg = niov;
1887 for (i = 0; i < niov; i++) {
1888 seg[i].segment_ptr = lnet_page2phys(kiov[first_kiov + i].kiov_page);
1889 seg[i].segment_length = kiov[first_kiov + i].kiov_len;
1891 seg[i].segment_ptr += (u64) first_kiov_offset;
1892 /* we have to add back the original kiov_offset */
1893 seg[i].segment_length -= first_kiov_offset +
1894 kiov[first_kiov].kiov_offset;
1896 if (i == (nseg - 1)) {
1897 seg[i].segment_length = last_kiov_length;
1899 sum += seg[i].segment_length;
1901 ctx->mxc_seg_list = seg;
1902 ctx->mxc_pin_type = MX_PIN_PHYSICAL;
1903 #ifdef MX_PIN_FULLPAGES
1904 ctx->mxc_pin_type |= MX_PIN_FULLPAGES;
1906 LASSERT(nob == sum);
1911 mxlnd_send_nak(kmx_ctx_t *tx, lnet_nid_t nid, int type, int status, __u64 cookie)
1913 LASSERT(type == MXLND_MSG_PUT_ACK);
1914 mxlnd_init_tx_msg(tx, type, sizeof(kmx_putack_msg_t), tx->mxc_nid);
1915 tx->mxc_cookie = cookie;
1916 tx->mxc_msg->mxm_u.put_ack.mxpam_src_cookie = cookie;
1917 tx->mxc_msg->mxm_u.put_ack.mxpam_dst_cookie = ((u64) status << MXLND_ERROR_OFFSET); /* error code */
1918 tx->mxc_match = mxlnd_create_match(tx, status);
1925 * mxlnd_send_data - get tx, map [k]iov, queue tx
1932 * This setups the DATA send for PUT or GET.
1934 * On success, it queues the tx, on failure it calls lnet_finalize()
1937 mxlnd_send_data(lnet_ni_t *ni, lnet_msg_t *lntmsg, kmx_peer_t *peer, u8 msg_type, u64 cookie)
1940 lnet_process_id_t target = lntmsg->msg_target;
1941 unsigned int niov = lntmsg->msg_niov;
1942 struct iovec *iov = lntmsg->msg_iov;
1943 lnet_kiov_t *kiov = lntmsg->msg_kiov;
1944 unsigned int offset = lntmsg->msg_offset;
1945 unsigned int nob = lntmsg->msg_len;
1946 kmx_ctx_t *tx = NULL;
1948 LASSERT(lntmsg != NULL);
1949 LASSERT(peer != NULL);
1950 LASSERT(msg_type == MXLND_MSG_PUT_DATA || msg_type == MXLND_MSG_GET_DATA);
1951 LASSERT((cookie>>MXLND_ERROR_OFFSET) == 0);
1953 tx = mxlnd_get_idle_tx();
1955 CNETERR("Can't allocate %s tx for %s\n",
1956 msg_type == MXLND_MSG_PUT_DATA ? "PUT_DATA" : "GET_DATA",
1957 libcfs_nid2str(target.nid));
1960 tx->mxc_nid = target.nid;
1961 /* NOTE called when we have a ref on the conn, get one for this tx */
1962 mxlnd_conn_addref(peer->mxp_conn);
1963 tx->mxc_peer = peer;
1964 tx->mxc_conn = peer->mxp_conn;
1965 tx->mxc_msg_type = msg_type;
1966 tx->mxc_lntmsg[0] = lntmsg;
1967 tx->mxc_cookie = cookie;
1968 tx->mxc_match = mxlnd_create_match(tx, 0);
1970 /* This setups up the mx_ksegment_t to send the DATA payload */
1972 /* do not setup the segments */
1973 CNETERR("nob = 0; why didn't we use an EAGER reply "
1974 "to %s?\n", libcfs_nid2str(target.nid));
1976 } else if (kiov == NULL) {
1977 ret = mxlnd_setup_iov(tx, niov, iov, offset, nob);
1979 ret = mxlnd_setup_kiov(tx, niov, kiov, offset, nob);
1982 CNETERR("Can't setup send DATA for %s\n",
1983 libcfs_nid2str(target.nid));
1984 tx->mxc_errno = -EIO;
1991 mxlnd_conn_decref(peer->mxp_conn);
1992 mxlnd_put_idle_tx(tx);
1996 CNETERR("no tx avail\n");
1997 lnet_finalize(ni, lntmsg, -EIO);
2002 * mxlnd_recv_data - map [k]iov, post rx
2009 * This setups the DATA receive for PUT or GET.
2011 * On success, it returns 0, on failure it returns -1
2014 mxlnd_recv_data(lnet_ni_t *ni, lnet_msg_t *lntmsg, kmx_ctx_t *rx, u8 msg_type, u64 cookie)
2017 lnet_process_id_t target = lntmsg->msg_target;
2018 unsigned int niov = lntmsg->msg_niov;
2019 struct iovec *iov = lntmsg->msg_iov;
2020 lnet_kiov_t *kiov = lntmsg->msg_kiov;
2021 unsigned int offset = lntmsg->msg_offset;
2022 unsigned int nob = lntmsg->msg_len;
2023 mx_return_t mxret = MX_SUCCESS;
2024 u64 mask = ~(MXLND_ERROR_MASK);
2026 /* above assumes MXLND_MSG_PUT_DATA */
2027 if (msg_type == MXLND_MSG_GET_DATA) {
2028 niov = lntmsg->msg_md->md_niov;
2029 iov = lntmsg->msg_md->md_iov.iov;
2030 kiov = lntmsg->msg_md->md_iov.kiov;
2032 nob = lntmsg->msg_md->md_length;
2035 LASSERT(lntmsg != NULL);
2036 LASSERT(rx != NULL);
2037 LASSERT(msg_type == MXLND_MSG_PUT_DATA || msg_type == MXLND_MSG_GET_DATA);
2038 LASSERT((cookie>>MXLND_ERROR_OFFSET) == 0); /* ensure top 12 bits are 0 */
2040 rx->mxc_msg_type = msg_type;
2041 rx->mxc_state = MXLND_CTX_PENDING;
2042 rx->mxc_nid = target.nid;
2043 /* if posting a GET_DATA, we may not yet know the peer */
2044 if (rx->mxc_peer != NULL) {
2045 rx->mxc_conn = rx->mxc_peer->mxp_conn;
2047 rx->mxc_lntmsg[0] = lntmsg;
2048 rx->mxc_cookie = cookie;
2049 rx->mxc_match = mxlnd_create_match(rx, 0);
2050 /* This setups up the mx_ksegment_t to receive the DATA payload */
2052 ret = mxlnd_setup_iov(rx, niov, iov, offset, nob);
2054 ret = mxlnd_setup_kiov(rx, niov, kiov, offset, nob);
2056 if (msg_type == MXLND_MSG_GET_DATA) {
2057 rx->mxc_lntmsg[1] = lnet_create_reply_msg(kmxlnd_data.kmx_ni, lntmsg);
2058 if (rx->mxc_lntmsg[1] == NULL) {
2059 CNETERR("Can't create reply for GET -> %s\n",
2060 libcfs_nid2str(target.nid));
2065 CNETERR("Can't setup %s rx for %s\n",
2066 msg_type == MXLND_MSG_PUT_DATA ? "PUT_DATA" : "GET_DATA",
2067 libcfs_nid2str(target.nid));
2070 ret = mxlnd_q_pending_ctx(rx);
2074 CDEBUG(D_NET, "receiving %s 0x%llx\n", mxlnd_msgtype_to_str(msg_type), rx->mxc_cookie);
2075 mxret = mx_kirecv(kmxlnd_data.kmx_endpt,
2076 rx->mxc_seg_list, rx->mxc_nseg,
2077 rx->mxc_pin_type, rx->mxc_match,
2080 if (mxret != MX_SUCCESS) {
2081 if (rx->mxc_conn != NULL) {
2082 mxlnd_deq_pending_ctx(rx);
2084 CNETERR("mx_kirecv() failed with %d for %s\n",
2085 (int) mxret, libcfs_nid2str(target.nid));
2093 * mxlnd_send - the LND required send function
2098 * This must not block. Since we may not have a peer struct for the receiver,
2099 * it will append send messages on a global tx list. We will then up the
2100 * tx_queued's semaphore to notify it of the new send.
2103 mxlnd_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg)
2106 int type = lntmsg->msg_type;
2107 lnet_hdr_t *hdr = &lntmsg->msg_hdr;
2108 lnet_process_id_t target = lntmsg->msg_target;
2109 lnet_nid_t nid = target.nid;
2110 int target_is_router = lntmsg->msg_target_is_router;
2111 int routing = lntmsg->msg_routing;
2112 unsigned int payload_niov = lntmsg->msg_niov;
2113 struct iovec *payload_iov = lntmsg->msg_iov;
2114 lnet_kiov_t *payload_kiov = lntmsg->msg_kiov;
2115 unsigned int payload_offset = lntmsg->msg_offset;
2116 unsigned int payload_nob = lntmsg->msg_len;
2117 kmx_ctx_t *tx = NULL;
2118 kmx_msg_t *txmsg = NULL;
2119 kmx_ctx_t *rx = (kmx_ctx_t *) private; /* for REPLY */
2120 kmx_ctx_t *rx_data = NULL;
2121 kmx_conn_t *conn = NULL;
2123 uint32_t length = 0;
2124 kmx_peer_t *peer = NULL;
2125 rwlock_t *g_lock =&kmxlnd_data.kmx_global_lock;
2127 CDEBUG(D_NET, "sending %d bytes in %d frags to %s\n",
2128 payload_nob, payload_niov, libcfs_id2str(target));
2130 LASSERT (payload_nob == 0 || payload_niov > 0);
2131 LASSERT (payload_niov <= LNET_MAX_IOV);
2132 /* payload is either all vaddrs or all pages */
2133 LASSERT (!(payload_kiov != NULL && payload_iov != NULL));
2135 /* private is used on LNET_GET_REPLY only, NULL for all other cases */
2137 /* NOTE we may not know the peer if it is the very first PUT_REQ or GET_REQ
2138 * to a new peer, so create one if not found */
2139 peer = mxlnd_find_peer_by_nid(nid, 1); /* adds peer ref */
2140 if (peer == NULL || peer->mxp_conn == NULL) {
2141 /* we could not find it nor could we create one or
2142 * one exists but we cannot create a conn,
2143 * fail this message */
2145 /* found peer without conn, drop ref taken above */
2146 LASSERT(peer->mxp_conn == NULL);
2147 mxlnd_peer_decref(peer);
2152 /* we have a peer with a conn */
2154 if (unlikely(peer->mxp_incompatible)) {
2155 mxlnd_peer_decref(peer); /* drop ref taken above */
2158 conn = peer->mxp_conn;
2159 if (conn && conn->mxk_status != MXLND_CONN_DISCONNECT)
2160 mxlnd_conn_addref(conn);
2163 read_unlock(g_lock);
2164 mxlnd_peer_decref(peer); /* drop peer ref taken above */
2169 LASSERT(peer && conn);
2171 CDEBUG(D_NET, "%s: peer 0x%llx is 0x%p\n", __func__, nid, peer);
2175 LASSERT (payload_nob == 0);
2178 case LNET_MSG_REPLY:
2180 /* Is the payload small enough not to need DATA? */
2181 nob = offsetof(kmx_msg_t, mxm_u.eager.mxem_payload[payload_nob]);
2182 if (nob <= MXLND_MSG_SIZE)
2183 break; /* send EAGER */
2185 tx = mxlnd_get_idle_tx();
2186 if (unlikely(tx == NULL)) {
2187 CNETERR("Can't allocate %s tx for %s\n",
2188 type == LNET_MSG_PUT ? "PUT" : "REPLY",
2189 libcfs_nid2str(nid));
2190 if (conn) mxlnd_conn_decref(conn);
2194 tx->mxc_peer = peer;
2195 tx->mxc_conn = conn;
2196 /* we added a conn ref above */
2197 mxlnd_init_tx_msg (tx, MXLND_MSG_PUT_REQ, sizeof(kmx_putreq_msg_t), nid);
2198 txmsg = tx->mxc_msg;
2199 txmsg->mxm_u.put_req.mxprm_hdr = *hdr;
2200 txmsg->mxm_u.put_req.mxprm_cookie = tx->mxc_cookie;
2201 tx->mxc_match = mxlnd_create_match(tx, 0);
2203 /* we must post a receive _before_ sending the request.
2204 * we need to determine how much to receive, it will be either
2205 * a put_ack or a put_nak. The put_ack is larger, so use it. */
2207 rx = mxlnd_get_idle_rx(conn);
2208 if (unlikely(rx == NULL)) {
2209 CNETERR("Can't allocate rx for PUT_ACK for %s\n",
2210 libcfs_nid2str(nid));
2211 mxlnd_put_idle_tx(tx);
2212 if (conn) mxlnd_conn_decref(conn); /* for the ref taken above */
2216 rx->mxc_peer = peer;
2217 mxlnd_conn_addref(conn); /* for this rx */
2218 rx->mxc_conn = conn;
2219 rx->mxc_msg_type = MXLND_MSG_PUT_ACK;
2220 rx->mxc_cookie = tx->mxc_cookie;
2221 rx->mxc_match = mxlnd_create_match(rx, 0);
2223 length = offsetof(kmx_msg_t, mxm_u) + sizeof(kmx_putack_msg_t);
2224 ret = mxlnd_recv_msg(lntmsg, rx, MXLND_MSG_PUT_ACK, rx->mxc_match, length);
2225 if (unlikely(ret != 0)) {
2226 CNETERR("recv_msg() failed for PUT_ACK for %s\n",
2227 libcfs_nid2str(nid));
2228 rx->mxc_lntmsg[0] = NULL;
2229 mxlnd_put_idle_rx(rx);
2230 mxlnd_put_idle_tx(tx);
2231 mxlnd_conn_decref(conn); /* for the rx... */
2232 mxlnd_conn_decref(conn); /* and for the tx */
2233 return -EHOSTUNREACH;
2240 if (routing || target_is_router)
2241 break; /* send EAGER */
2243 /* is the REPLY message too small for DATA? */
2244 nob = offsetof(kmx_msg_t, mxm_u.eager.mxem_payload[lntmsg->msg_md->md_length]);
2245 if (nob <= MXLND_MSG_SIZE)
2246 break; /* send EAGER */
2248 /* get tx (we need the cookie) , post rx for incoming DATA,
2249 * then post GET_REQ tx */
2250 tx = mxlnd_get_idle_tx();
2251 if (unlikely(tx == NULL)) {
2252 CNETERR("Can't allocate GET tx for %s\n",
2253 libcfs_nid2str(nid));
2254 mxlnd_conn_decref(conn); /* for the ref taken above */
2257 rx_data = mxlnd_get_idle_rx(conn);
2258 if (unlikely(rx_data == NULL)) {
2259 CNETERR("Can't allocate DATA rx for %s\n",
2260 libcfs_nid2str(nid));
2261 mxlnd_put_idle_tx(tx);
2262 mxlnd_conn_decref(conn); /* for the ref taken above */
2265 rx_data->mxc_peer = peer;
2266 /* NOTE no need to lock peer before adding conn ref since we took
2267 * a conn ref for the tx (it cannot be freed between there and here ) */
2268 mxlnd_conn_addref(conn); /* for the rx_data */
2269 rx_data->mxc_conn = conn;
2271 ret = mxlnd_recv_data(ni, lntmsg, rx_data, MXLND_MSG_GET_DATA, tx->mxc_cookie);
2272 if (unlikely(ret != 0)) {
2273 CNETERR("Can't setup GET sink for %s\n",
2274 libcfs_nid2str(nid));
2275 mxlnd_put_idle_rx(rx_data);
2276 mxlnd_put_idle_tx(tx);
2277 mxlnd_conn_decref(conn); /* for the rx_data... */
2278 mxlnd_conn_decref(conn); /* and for the tx */
2282 tx->mxc_peer = peer;
2283 tx->mxc_conn = conn;
2284 /* conn ref taken above */
2285 mxlnd_init_tx_msg(tx, MXLND_MSG_GET_REQ, sizeof(kmx_getreq_msg_t), nid);
2286 txmsg = tx->mxc_msg;
2287 txmsg->mxm_u.get_req.mxgrm_hdr = *hdr;
2288 txmsg->mxm_u.get_req.mxgrm_cookie = tx->mxc_cookie;
2289 tx->mxc_match = mxlnd_create_match(tx, 0);
2296 mxlnd_conn_decref(conn); /* drop ref taken above */
2302 LASSERT (offsetof(kmx_msg_t, mxm_u.eager.mxem_payload[payload_nob])
2305 tx = mxlnd_get_idle_tx();
2306 if (unlikely(tx == NULL)) {
2307 CNETERR("Can't send %s to %s: tx descs exhausted\n",
2308 mxlnd_lnetmsg_to_str(type), libcfs_nid2str(nid));
2309 mxlnd_conn_decref(conn); /* drop ref taken above */
2313 tx->mxc_peer = peer;
2314 tx->mxc_conn = conn;
2315 /* conn ref taken above */
2316 nob = offsetof(kmx_eager_msg_t, mxem_payload[payload_nob]);
2317 mxlnd_init_tx_msg (tx, MXLND_MSG_EAGER, nob, nid);
2318 tx->mxc_match = mxlnd_create_match(tx, 0);
2320 txmsg = tx->mxc_msg;
2321 txmsg->mxm_u.eager.mxem_hdr = *hdr;
2323 if (payload_kiov != NULL)
2324 lnet_copy_kiov2flat(MXLND_MSG_SIZE, txmsg,
2325 offsetof(kmx_msg_t, mxm_u.eager.mxem_payload),
2326 payload_niov, payload_kiov, payload_offset, payload_nob);
2328 lnet_copy_iov2flat(MXLND_MSG_SIZE, txmsg,
2329 offsetof(kmx_msg_t, mxm_u.eager.mxem_payload),
2330 payload_niov, payload_iov, payload_offset, payload_nob);
2332 tx->mxc_lntmsg[0] = lntmsg; /* finalise lntmsg on completion */
2338 * mxlnd_recv - the LND required recv function
2349 * This must not block.
2352 mxlnd_recv (lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, int delayed,
2353 unsigned int niov, struct iovec *iov, lnet_kiov_t *kiov,
2354 unsigned int offset, unsigned int mlen, unsigned int rlen)
2359 kmx_ctx_t *rx = private;
2360 kmx_msg_t *rxmsg = rx->mxc_msg;
2361 lnet_nid_t nid = rx->mxc_nid;
2362 kmx_ctx_t *tx = NULL;
2363 kmx_msg_t *txmsg = NULL;
2364 kmx_peer_t *peer = rx->mxc_peer;
2365 kmx_conn_t *conn = peer->mxp_conn;
2367 int msg_type = rxmsg->mxm_type;
2372 LASSERT (mlen <= rlen);
2373 /* Either all pages or all vaddrs */
2374 LASSERT (!(kiov != NULL && iov != NULL));
2375 LASSERT (peer && conn);
2377 /* conn_addref(conn) already taken for the primary rx */
2380 case MXLND_MSG_EAGER:
2381 nob = offsetof(kmx_msg_t, mxm_u.eager.mxem_payload[rlen]);
2382 len = rx->mxc_status.xfer_length;
2383 if (unlikely(nob > len)) {
2384 CNETERR("Eager message from %s too big: %d(%d)\n",
2385 libcfs_nid2str(nid), nob, len);
2391 lnet_copy_flat2kiov(niov, kiov, offset,
2392 MXLND_MSG_SIZE, rxmsg,
2393 offsetof(kmx_msg_t, mxm_u.eager.mxem_payload),
2396 lnet_copy_flat2iov(niov, iov, offset,
2397 MXLND_MSG_SIZE, rxmsg,
2398 offsetof(kmx_msg_t, mxm_u.eager.mxem_payload),
2404 case MXLND_MSG_PUT_REQ:
2405 /* we are going to reuse the rx, store the needed info */
2406 cookie = rxmsg->mxm_u.put_req.mxprm_cookie;
2408 /* get tx, post rx, send PUT_ACK */
2410 tx = mxlnd_get_idle_tx();
2411 if (unlikely(tx == NULL)) {
2412 CNETERR("Can't allocate tx for %s\n", libcfs_nid2str(nid));
2413 /* Not replying will break the connection */
2417 if (unlikely(mlen == 0)) {
2419 tx->mxc_peer = peer;
2420 tx->mxc_conn = conn;
2421 mxlnd_send_nak(tx, nid, MXLND_MSG_PUT_ACK, 0, cookie);
2426 mxlnd_init_tx_msg(tx, MXLND_MSG_PUT_ACK, sizeof(kmx_putack_msg_t), nid);
2427 tx->mxc_peer = peer;
2428 tx->mxc_conn = conn;
2429 /* no need to lock peer first since we already have a ref */
2430 mxlnd_conn_addref(conn); /* for the tx */
2431 txmsg = tx->mxc_msg;
2432 txmsg->mxm_u.put_ack.mxpam_src_cookie = cookie;
2433 txmsg->mxm_u.put_ack.mxpam_dst_cookie = tx->mxc_cookie;
2434 tx->mxc_cookie = cookie;
2435 tx->mxc_match = mxlnd_create_match(tx, 0);
2437 /* we must post a receive _before_ sending the PUT_ACK */
2439 rx->mxc_state = MXLND_CTX_PREP;
2440 rx->mxc_peer = peer;
2441 rx->mxc_conn = conn;
2442 /* do not take another ref for this rx, it is already taken */
2443 rx->mxc_nid = peer->mxp_nid;
2444 ret = mxlnd_recv_data(ni, lntmsg, rx, MXLND_MSG_PUT_DATA,
2445 txmsg->mxm_u.put_ack.mxpam_dst_cookie);
2447 if (unlikely(ret != 0)) {
2448 /* Notify peer that it's over */
2449 CNETERR("Can't setup PUT_DATA rx for %s: %d\n",
2450 libcfs_nid2str(nid), ret);
2452 tx->mxc_state = MXLND_CTX_PREP;
2453 tx->mxc_peer = peer;
2454 tx->mxc_conn = conn;
2455 /* finalize = 0, let the PUT_ACK tx finalize this */
2456 tx->mxc_lntmsg[0] = rx->mxc_lntmsg[0];
2457 tx->mxc_lntmsg[1] = rx->mxc_lntmsg[1];
2458 /* conn ref already taken above */
2459 mxlnd_send_nak(tx, nid, MXLND_MSG_PUT_ACK, ret, cookie);
2465 /* do not return a credit until after PUT_DATA returns */
2469 case MXLND_MSG_GET_REQ:
2470 cookie = rxmsg->mxm_u.get_req.mxgrm_cookie;
2472 if (likely(lntmsg != NULL)) {
2473 mxlnd_send_data(ni, lntmsg, rx->mxc_peer, MXLND_MSG_GET_DATA,
2476 /* GET didn't match anything */
2477 /* The initiator has a rx mapped to [k]iov. We cannot send a nak.
2478 * We have to embed the error code in the match bits.
2479 * Send the error in bits 52-59 and the cookie in bits 0-51 */
2480 tx = mxlnd_get_idle_tx();
2481 if (unlikely(tx == NULL)) {
2482 CNETERR("Can't get tx for GET NAK for %s\n",
2483 libcfs_nid2str(nid));
2484 /* we can't get a tx, notify the peer that the GET failed */
2485 mxlnd_send_message(conn->mxk_epa, MXLND_MSG_GET_DATA,
2490 tx->mxc_msg_type = MXLND_MSG_GET_DATA;
2491 tx->mxc_state = MXLND_CTX_PENDING;
2493 tx->mxc_peer = peer;
2494 tx->mxc_conn = conn;
2495 /* no need to lock peer first since we already have a ref */
2496 mxlnd_conn_addref(conn); /* for this tx */
2497 tx->mxc_cookie = cookie;
2498 tx->mxc_match = mxlnd_create_match(tx, ENODATA);
2499 tx->mxc_pin_type = MX_PIN_PHYSICAL;
2502 /* finalize lntmsg after tx completes */
2510 /* we received a message, increment peer's outstanding credits */
2512 spin_lock(&conn->mxk_lock);
2513 conn->mxk_outstanding++;
2514 spin_unlock(&conn->mxk_lock);
2516 /* we are done with the rx */
2517 mxlnd_put_idle_rx(rx);
2518 mxlnd_conn_decref(conn);
2521 if (finalize == 1) lnet_finalize(kmxlnd_data.kmx_ni, lntmsg, 0);
2523 /* we received a credit, see if we can use it to send a msg */
2524 if (credit) mxlnd_check_sends(peer);
2530 mxlnd_sleep(unsigned long timeout)
2532 set_current_state(TASK_INTERRUPTIBLE);
2533 schedule_timeout(timeout);
2538 * mxlnd_tx_queued - the generic send queue thread
2539 * @arg - thread id (as a void *)
2541 * This thread moves send messages from the global tx_queue to the owning
2542 * peer's tx_[msg|data]_queue. If the peer does not exist, it creates one and adds
2543 * it to the global peer list.
2546 mxlnd_tx_queued(void *arg)
2548 long id = (long) arg;
2551 kmx_ctx_t *tx = NULL;
2552 kmx_peer_t *peer = NULL;
2553 cfs_list_t *queue = &kmxlnd_data.kmx_tx_queue;
2554 spinlock_t *tx_q_lock = &kmxlnd_data.kmx_tx_queue_lock;
2555 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
2557 while (!(atomic_read(&kmxlnd_data.kmx_shutdown))) {
2558 ret = down_interruptible(&kmxlnd_data.kmx_tx_queue_sem);
2559 if (atomic_read(&kmxlnd_data.kmx_shutdown))
2561 if (ret != 0) /* Should we check for -EINTR? */
2563 spin_lock(tx_q_lock);
2564 if (cfs_list_empty(&kmxlnd_data.kmx_tx_queue)) {
2565 spin_unlock(tx_q_lock);
2568 tx = cfs_list_entry(queue->next, kmx_ctx_t, mxc_list);
2569 cfs_list_del_init(&tx->mxc_list);
2570 spin_unlock(tx_q_lock);
2573 peer = mxlnd_find_peer_by_nid(tx->mxc_nid, 0); /* adds ref*/
2575 tx->mxc_peer = peer;
2577 if (peer->mxp_conn == NULL) {
2578 ret = mxlnd_conn_alloc_locked(&peer->mxp_conn,
2581 /* out of memory: give up, fail tx */
2582 tx->mxc_errno = -ENOMEM;
2583 mxlnd_peer_decref(peer);
2584 write_unlock(g_lock);
2585 mxlnd_put_idle_tx(tx);
2589 tx->mxc_conn = peer->mxp_conn;
2590 mxlnd_conn_addref(tx->mxc_conn); /* for this tx */
2591 mxlnd_peer_decref(peer); /* drop peer ref taken above */
2592 write_unlock(g_lock);
2598 kmx_peer_t *peer = NULL;
2599 kmx_peer_t *old = NULL;
2601 hash = mxlnd_nid_to_hash(tx->mxc_nid);
2603 LASSERT(tx->mxc_msg_type != MXLND_MSG_PUT_DATA &&
2604 tx->mxc_msg_type != MXLND_MSG_GET_DATA);
2606 /* adds conn ref for this function */
2607 ret = mxlnd_peer_alloc(&peer, tx->mxc_nid,
2608 *kmxlnd_tunables.kmx_board,
2609 *kmxlnd_tunables.kmx_ep_id, 0ULL);
2611 /* finalize message */
2612 tx->mxc_errno = ret;
2613 mxlnd_put_idle_tx(tx);
2616 tx->mxc_peer = peer;
2617 tx->mxc_conn = peer->mxp_conn;
2618 /* this tx will keep the conn ref taken in peer_alloc() */
2620 /* add peer to global peer list, but look to see
2621 * if someone already created it after we released
2624 old = mxlnd_find_peer_by_nid_locked(peer->mxp_nid);
2626 /* we have a peer ref on old */
2627 if (old->mxp_conn) {
2631 /* drop our ref taken above... */
2632 mxlnd_peer_decref(old);
2634 mxlnd_del_peer_locked(old);
2639 cfs_list_add_tail(&peer->mxp_list,
2640 &kmxlnd_data.kmx_peers[hash]);
2641 atomic_inc(&kmxlnd_data.kmx_npeers);
2644 tx->mxc_conn = old->mxp_conn;
2645 LASSERT(old->mxp_conn != NULL);
2646 mxlnd_conn_addref(old->mxp_conn);
2647 mxlnd_conn_decref(peer->mxp_conn); /* drop ref taken above.. */
2648 mxlnd_conn_decref(peer->mxp_conn); /* drop peer's ref */
2649 mxlnd_peer_decref(peer);
2651 write_unlock(g_lock);
2656 mxlnd_thread_stop(id);
2660 /* When calling this, we must not have the peer lock. */
2662 mxlnd_iconnect(kmx_peer_t *peer, u8 msg_type)
2664 mx_return_t mxret = MX_SUCCESS;
2665 mx_request_t request;
2666 kmx_conn_t *conn = peer->mxp_conn;
2667 u64 match = ((u64) msg_type) << MXLND_MSG_OFFSET;
2669 /* NOTE we are holding a conn ref every time we call this function,
2670 * we do not need to lock the peer before taking another ref */
2671 mxlnd_conn_addref(conn); /* hold until CONN_REQ or CONN_ACK completes */
2673 LASSERT(msg_type == MXLND_MSG_ICON_REQ || msg_type == MXLND_MSG_ICON_ACK);
2675 if (peer->mxp_reconnect_time == 0) {
2676 peer->mxp_reconnect_time = jiffies;
2679 if (peer->mxp_nic_id == 0ULL) {
2682 ret = mxlnd_ip2nic_id(LNET_NIDADDR(peer->mxp_nid),
2683 &peer->mxp_nic_id, MXLND_LOOKUP_COUNT);
2685 mx_nic_id_to_board_number(peer->mxp_nic_id, &peer->mxp_board);
2687 if (peer->mxp_nic_id == 0ULL && conn->mxk_status == MXLND_CONN_WAIT) {
2688 /* not mapped yet, return */
2689 spin_lock(&conn->mxk_lock);
2690 mxlnd_set_conn_status(conn, MXLND_CONN_INIT);
2691 spin_unlock(&conn->mxk_lock);
2695 if (cfs_time_after(jiffies,
2696 peer->mxp_reconnect_time + MXLND_CONNECT_TIMEOUT) &&
2697 conn->mxk_status != MXLND_CONN_DISCONNECT) {
2698 /* give up and notify LNET */
2699 CDEBUG(D_NET, "timeout trying to connect to %s\n",
2700 libcfs_nid2str(peer->mxp_nid));
2701 mxlnd_conn_disconnect(conn, 0, 0);
2702 mxlnd_conn_decref(conn);
2706 mxret = mx_iconnect(kmxlnd_data.kmx_endpt, peer->mxp_nic_id,
2707 peer->mxp_ep_id, MXLND_MSG_MAGIC, match,
2708 (void *) peer, &request);
2709 if (unlikely(mxret != MX_SUCCESS)) {
2710 spin_lock(&conn->mxk_lock);
2711 mxlnd_set_conn_status(conn, MXLND_CONN_FAIL);
2712 spin_unlock(&conn->mxk_lock);
2713 CNETERR("mx_iconnect() failed with %s (%d) to %s\n",
2714 mx_strerror(mxret), mxret, libcfs_nid2str(peer->mxp_nid));
2715 mxlnd_conn_decref(conn);
2717 mx_set_request_timeout(kmxlnd_data.kmx_endpt, request,
2718 MXLND_CONNECT_TIMEOUT/HZ*1000);
2722 #define MXLND_STATS 0
2725 mxlnd_check_sends(kmx_peer_t *peer)
2729 mx_return_t mxret = MX_SUCCESS;
2730 kmx_ctx_t *tx = NULL;
2731 kmx_conn_t *conn = NULL;
2738 static unsigned long last = 0;
2741 if (unlikely(peer == NULL)) {
2742 LASSERT(peer != NULL);
2745 write_lock(&kmxlnd_data.kmx_global_lock);
2746 conn = peer->mxp_conn;
2747 /* NOTE take a ref for the duration of this function since it is
2748 * called when there might not be any queued txs for this peer */
2750 if (conn->mxk_status == MXLND_CONN_DISCONNECT) {
2751 write_unlock(&kmxlnd_data.kmx_global_lock);
2754 mxlnd_conn_addref(conn); /* for duration of this function */
2756 write_unlock(&kmxlnd_data.kmx_global_lock);
2758 /* do not add another ref for this tx */
2761 /* we do not have any conns */
2762 CNETERR("peer %s has no conn\n", libcfs_nid2str(peer->mxp_nid));
2767 if (cfs_time_after(jiffies, last)) {
2768 last = jiffies + HZ;
2769 CDEBUG(D_NET, "status= %s credits= %d outstanding= %d ntx_msgs= %d "
2770 "ntx_posted= %d ntx_data= %d data_posted= %d\n",
2771 mxlnd_connstatus_to_str(conn->mxk_status), conn->mxk_credits,
2772 conn->mxk_outstanding, conn->mxk_ntx_msgs, conn->mxk_ntx_posted,
2773 conn->mxk_ntx_data, conn->mxk_data_posted);
2777 spin_lock(&conn->mxk_lock);
2778 ntx_posted = conn->mxk_ntx_posted;
2779 credits = conn->mxk_credits;
2781 LASSERT(ntx_posted <= *kmxlnd_tunables.kmx_peercredits);
2782 LASSERT(ntx_posted >= 0);
2784 LASSERT(credits <= *kmxlnd_tunables.kmx_peercredits);
2785 LASSERT(credits >= 0);
2787 /* check number of queued msgs, ignore data */
2788 if (conn->mxk_outstanding >= MXLND_CREDIT_HIGHWATER()) {
2789 /* check if any txs queued that could return credits... */
2790 if (cfs_list_empty(&conn->mxk_tx_credit_queue) ||
2791 conn->mxk_ntx_msgs == 0) {
2792 /* if not, send a NOOP */
2793 tx = mxlnd_get_idle_tx();
2794 if (likely(tx != NULL)) {
2795 tx->mxc_peer = peer;
2796 tx->mxc_conn = peer->mxp_conn;
2797 mxlnd_conn_addref(conn); /* for this tx */
2798 mxlnd_init_tx_msg (tx, MXLND_MSG_NOOP, 0, peer->mxp_nid);
2799 tx->mxc_match = mxlnd_create_match(tx, 0);
2800 mxlnd_peer_queue_tx_locked(tx);
2807 /* if the peer is not ready, try to connect */
2808 if (unlikely(conn->mxk_status == MXLND_CONN_INIT ||
2809 conn->mxk_status == MXLND_CONN_FAIL)) {
2810 CDEBUG(D_NET, "status=%s\n", mxlnd_connstatus_to_str(conn->mxk_status));
2811 mxlnd_set_conn_status(conn, MXLND_CONN_WAIT);
2812 spin_unlock(&conn->mxk_lock);
2813 mxlnd_iconnect(peer, (u8) MXLND_MSG_ICON_REQ);
2817 while (!cfs_list_empty(&conn->mxk_tx_free_queue) ||
2818 !cfs_list_empty(&conn->mxk_tx_credit_queue)) {
2819 /* We have something to send. If we have a queued tx that does not
2820 * require a credit (free), choose it since its completion will
2821 * return a credit (here or at the peer), complete a DATA or
2822 * CONN_REQ or CONN_ACK. */
2823 cfs_list_t *tmp_tx = NULL;
2824 if (!cfs_list_empty(&conn->mxk_tx_free_queue)) {
2825 tmp_tx = &conn->mxk_tx_free_queue;
2827 tmp_tx = &conn->mxk_tx_credit_queue;
2829 tx = cfs_list_entry(tmp_tx->next, kmx_ctx_t, mxc_list);
2831 msg_type = tx->mxc_msg_type;
2833 /* don't try to send a rx */
2834 LASSERT(tx->mxc_type == MXLND_REQ_TX);
2836 /* ensure that it is a valid msg type */
2837 LASSERT(msg_type == MXLND_MSG_CONN_REQ ||
2838 msg_type == MXLND_MSG_CONN_ACK ||
2839 msg_type == MXLND_MSG_NOOP ||
2840 msg_type == MXLND_MSG_EAGER ||
2841 msg_type == MXLND_MSG_PUT_REQ ||
2842 msg_type == MXLND_MSG_PUT_ACK ||
2843 msg_type == MXLND_MSG_PUT_DATA ||
2844 msg_type == MXLND_MSG_GET_REQ ||
2845 msg_type == MXLND_MSG_GET_DATA);
2846 LASSERT(tx->mxc_peer == peer);
2847 LASSERT(tx->mxc_nid == peer->mxp_nid);
2849 credit = mxlnd_tx_requires_credit(tx);
2852 if (conn->mxk_ntx_posted == *kmxlnd_tunables.kmx_peercredits) {
2853 CDEBUG(D_NET, "%s: posted enough\n",
2854 libcfs_nid2str(peer->mxp_nid));
2858 if (conn->mxk_credits == 0) {
2859 CDEBUG(D_NET, "%s: no credits\n",
2860 libcfs_nid2str(peer->mxp_nid));
2864 if (conn->mxk_credits == 1 && /* last credit reserved for */
2865 conn->mxk_outstanding == 0) { /* giving back credits */
2866 CDEBUG(D_NET, "%s: not using last credit\n",
2867 libcfs_nid2str(peer->mxp_nid));
2872 if (unlikely(conn->mxk_status != MXLND_CONN_READY)) {
2873 if ( ! (msg_type == MXLND_MSG_CONN_REQ ||
2874 msg_type == MXLND_MSG_CONN_ACK)) {
2875 CDEBUG(D_NET, "peer status is %s for tx 0x%llx (%s)\n",
2876 mxlnd_connstatus_to_str(conn->mxk_status),
2878 mxlnd_msgtype_to_str(tx->mxc_msg_type));
2879 if (conn->mxk_status == MXLND_CONN_DISCONNECT ||
2880 cfs_time_aftereq(jiffies, tx->mxc_deadline)) {
2881 cfs_list_del_init(&tx->mxc_list);
2882 tx->mxc_errno = -ECONNABORTED;
2883 spin_unlock(&conn->mxk_lock);
2884 mxlnd_put_idle_tx(tx);
2885 mxlnd_conn_decref(conn);
2892 cfs_list_del_init(&tx->mxc_list);
2894 /* handle credits, etc now while we have the lock to avoid races */
2896 conn->mxk_credits--;
2897 conn->mxk_ntx_posted++;
2899 if (msg_type != MXLND_MSG_PUT_DATA &&
2900 msg_type != MXLND_MSG_GET_DATA) {
2901 if (msg_type != MXLND_MSG_CONN_REQ &&
2902 msg_type != MXLND_MSG_CONN_ACK) {
2903 conn->mxk_ntx_msgs--;
2906 if (tx->mxc_incarnation == 0 &&
2907 conn->mxk_incarnation != 0) {
2908 tx->mxc_incarnation = conn->mxk_incarnation;
2911 /* if this is a NOOP and (1) mxp_conn->mxk_outstanding < CREDIT_HIGHWATER
2912 * or (2) there is a non-DATA msg that can return credits in the
2913 * queue, then drop this duplicate NOOP */
2914 if (unlikely(msg_type == MXLND_MSG_NOOP)) {
2915 if ((conn->mxk_outstanding < MXLND_CREDIT_HIGHWATER()) ||
2916 (conn->mxk_ntx_msgs >= 1)) {
2917 conn->mxk_credits++;
2918 conn->mxk_ntx_posted--;
2919 spin_unlock(&conn->mxk_lock);
2920 /* redundant NOOP */
2921 mxlnd_put_idle_tx(tx);
2922 mxlnd_conn_decref(conn);
2923 CDEBUG(D_NET, "%s: redundant noop\n",
2924 libcfs_nid2str(peer->mxp_nid));
2931 if (likely((msg_type != MXLND_MSG_PUT_DATA) &&
2932 (msg_type != MXLND_MSG_GET_DATA))) {
2933 mxlnd_pack_msg_locked(tx);
2938 status = conn->mxk_status;
2939 spin_unlock(&conn->mxk_lock);
2941 if (likely((status == MXLND_CONN_READY) ||
2942 (msg_type == MXLND_MSG_CONN_REQ) ||
2943 (msg_type == MXLND_MSG_CONN_ACK))) {
2945 if (msg_type != MXLND_MSG_CONN_REQ &&
2946 msg_type != MXLND_MSG_CONN_ACK) {
2947 /* add to the pending list */
2948 ret = mxlnd_q_pending_ctx(tx);
2951 tx->mxc_state = MXLND_CTX_PENDING;
2955 if (likely(msg_type != MXLND_MSG_PUT_DATA &&
2956 msg_type != MXLND_MSG_GET_DATA)) {
2957 /* send a msg style tx */
2958 LASSERT(tx->mxc_nseg == 1);
2959 LASSERT(tx->mxc_pin_type == MX_PIN_PHYSICAL);
2960 CDEBUG(D_NET, "sending %s 0x%llx\n",
2961 mxlnd_msgtype_to_str(msg_type),
2963 mxret = mx_kisend(kmxlnd_data.kmx_endpt,
2972 /* send a DATA tx */
2973 spin_lock(&conn->mxk_lock);
2974 conn->mxk_ntx_data--;
2975 conn->mxk_data_posted++;
2976 spin_unlock(&conn->mxk_lock);
2977 CDEBUG(D_NET, "sending %s 0x%llx\n",
2978 mxlnd_msgtype_to_str(msg_type),
2980 mxret = mx_kisend(kmxlnd_data.kmx_endpt,
2991 mxret = MX_CONNECTION_FAILED;
2993 if (likely(mxret == MX_SUCCESS)) {
2996 CNETERR("mx_kisend() failed with %s (%d) "
2997 "sending to %s\n", mx_strerror(mxret), (int) mxret,
2998 libcfs_nid2str(peer->mxp_nid));
2999 /* NOTE mx_kisend() only fails if there are not enough
3000 * resources. Do not change the connection status. */
3001 if (mxret == MX_NO_RESOURCES) {
3002 tx->mxc_errno = -ENOMEM;
3004 tx->mxc_errno = -ECONNABORTED;
3007 spin_lock(&conn->mxk_lock);
3008 conn->mxk_ntx_posted--;
3009 conn->mxk_credits++;
3010 spin_unlock(&conn->mxk_lock);
3011 } else if (msg_type == MXLND_MSG_PUT_DATA ||
3012 msg_type == MXLND_MSG_GET_DATA) {
3013 spin_lock(&conn->mxk_lock);
3014 conn->mxk_data_posted--;
3015 spin_unlock(&conn->mxk_lock);
3017 if (msg_type != MXLND_MSG_PUT_DATA &&
3018 msg_type != MXLND_MSG_GET_DATA &&
3019 msg_type != MXLND_MSG_CONN_REQ &&
3020 msg_type != MXLND_MSG_CONN_ACK) {
3021 spin_lock(&conn->mxk_lock);
3022 conn->mxk_outstanding +=
3023 tx->mxc_msg->mxm_credits;
3024 spin_unlock(&conn->mxk_lock);
3026 if (msg_type != MXLND_MSG_CONN_REQ &&
3027 msg_type != MXLND_MSG_CONN_ACK) {
3028 /* remove from the pending list */
3029 mxlnd_deq_pending_ctx(tx);
3031 mxlnd_put_idle_tx(tx);
3032 mxlnd_conn_decref(conn);
3035 spin_lock(&conn->mxk_lock);
3038 spin_unlock(&conn->mxk_lock);
3040 mxlnd_conn_decref(conn); /* drop ref taken at start of function */
3046 * mxlnd_handle_tx_completion - a tx completed, progress or complete the msg
3047 * @ctx - the tx descriptor
3049 * Determine which type of send request it was and start the next step, if needed,
3050 * or, if done, signal completion to LNET. After we are done, put back on the
3054 mxlnd_handle_tx_completion(kmx_ctx_t *tx)
3056 int code = tx->mxc_status.code;
3057 int failed = (code != MX_STATUS_SUCCESS || tx->mxc_errno != 0);
3058 kmx_msg_t *msg = tx->mxc_msg;
3059 kmx_peer_t *peer = tx->mxc_peer;
3060 kmx_conn_t *conn = tx->mxc_conn;
3061 u8 type = tx->mxc_msg_type;
3062 int credit = mxlnd_tx_requires_credit(tx);
3063 u64 cookie = tx->mxc_cookie;
3065 CDEBUG(D_NET, "entering %s (0x%llx):\n",
3066 mxlnd_msgtype_to_str(tx->mxc_msg_type), cookie);
3068 LASSERT (peer != NULL);
3069 LASSERT (conn != NULL);
3071 if (type != MXLND_MSG_PUT_DATA && type != MXLND_MSG_GET_DATA) {
3072 LASSERT (type == msg->mxm_type);
3076 if (tx->mxc_errno == 0) tx->mxc_errno = -EIO;
3078 spin_lock(&conn->mxk_lock);
3079 conn->mxk_last_tx = cfs_time_current(); /* jiffies */
3080 spin_unlock(&conn->mxk_lock);
3085 case MXLND_MSG_GET_DATA:
3086 spin_lock(&conn->mxk_lock);
3087 if (conn->mxk_incarnation == tx->mxc_incarnation) {
3088 conn->mxk_outstanding++;
3089 conn->mxk_data_posted--;
3091 spin_unlock(&conn->mxk_lock);
3094 case MXLND_MSG_PUT_DATA:
3095 spin_lock(&conn->mxk_lock);
3096 if (conn->mxk_incarnation == tx->mxc_incarnation) {
3097 conn->mxk_data_posted--;
3099 spin_unlock(&conn->mxk_lock);
3102 case MXLND_MSG_NOOP:
3103 case MXLND_MSG_PUT_REQ:
3104 case MXLND_MSG_PUT_ACK:
3105 case MXLND_MSG_GET_REQ:
3106 case MXLND_MSG_EAGER:
3109 case MXLND_MSG_CONN_ACK:
3110 if (peer->mxp_incompatible) {
3111 /* we sent our params, now close this conn */
3112 mxlnd_conn_disconnect(conn, 0, 1);
3114 case MXLND_MSG_CONN_REQ:
3116 CNETERR("%s failed with %s (%d) (errno = %d) to %s\n",
3117 type == MXLND_MSG_CONN_REQ ? "CONN_REQ" : "CONN_ACK",
3118 mx_strstatus(code), code, tx->mxc_errno,
3119 libcfs_nid2str(tx->mxc_nid));
3120 if (!peer->mxp_incompatible) {
3121 spin_lock(&conn->mxk_lock);
3122 if (code == MX_STATUS_BAD_SESSION)
3123 mxlnd_set_conn_status(conn,
3126 mxlnd_set_conn_status(conn,
3128 spin_unlock(&conn->mxk_lock);
3134 CNETERR("Unknown msg type of %d\n", type);
3139 spin_lock(&conn->mxk_lock);
3140 if (conn->mxk_incarnation == tx->mxc_incarnation) {
3141 conn->mxk_ntx_posted--;
3143 spin_unlock(&conn->mxk_lock);
3146 mxlnd_put_idle_tx(tx);
3147 mxlnd_conn_decref(conn);
3149 mxlnd_check_sends(peer);
3151 CDEBUG(D_NET, "leaving\n");
3155 /* Handle completion of MSG or DATA rx.
3156 * CONN_REQ and CONN_ACK are handled elsewhere. */
3158 mxlnd_handle_rx_completion(kmx_ctx_t *rx)
3163 u32 nob = rx->mxc_status.xfer_length;
3164 u64 bits = rx->mxc_status.match_info;
3165 kmx_msg_t *msg = rx->mxc_msg;
3166 kmx_peer_t *peer = rx->mxc_peer;
3167 kmx_conn_t *conn = rx->mxc_conn;
3168 u8 type = rx->mxc_msg_type;
3170 lnet_msg_t *lntmsg[2];
3175 /* NOTE We may only know the peer's nid if it is a PUT_REQ, GET_REQ,
3176 * failed GET reply */
3178 /* NOTE peer may still be NULL if it is a new peer and
3179 * conn may be NULL if this is a re-connect */
3180 if (likely(peer != NULL && conn != NULL)) {
3181 /* we have a reference on the conn */
3183 } else if (peer != NULL && conn == NULL) {
3184 /* we have a reference on the peer */
3186 } else if (peer == NULL && conn != NULL) {
3188 CERROR("rx 0x%llx from %s has conn but no peer\n",
3189 bits, libcfs_nid2str(rx->mxc_nid));
3191 } /* else peer and conn == NULL */
3193 if (conn == NULL && peer != NULL) {
3194 write_lock(&kmxlnd_data.kmx_global_lock);
3195 conn = peer->mxp_conn;
3197 mxlnd_conn_addref(conn); /* conn takes ref... */
3198 mxlnd_peer_decref(peer); /* from peer */
3202 write_unlock(&kmxlnd_data.kmx_global_lock);
3203 rx->mxc_conn = conn;
3207 CDEBUG(D_NET, "receiving msg bits=0x%llx nob=%d peer=0x%p\n", bits, nob, peer);
3213 if (rx->mxc_status.code != MX_STATUS_SUCCESS &&
3214 rx->mxc_status.code != MX_STATUS_TRUNCATED) {
3215 CNETERR("rx from %s failed with %s (%d)\n",
3216 libcfs_nid2str(rx->mxc_nid),
3217 mx_strstatus(rx->mxc_status.code),
3218 rx->mxc_status.code);
3224 /* this may be a failed GET reply */
3225 if (type == MXLND_MSG_GET_DATA) {
3226 /* get the error (52-59) bits from the match bits */
3227 ret = (u32) MXLND_ERROR_VAL(rx->mxc_status.match_info);
3228 lntmsg[0] = rx->mxc_lntmsg[0];
3232 /* we had a rx complete with 0 bytes (no hdr, nothing) */
3233 CNETERR("rx from %s returned with 0 bytes\n",
3234 libcfs_nid2str(rx->mxc_nid));
3239 /* NOTE PUT_DATA and GET_DATA do not have mxc_msg, do not call unpack() */
3240 if (type == MXLND_MSG_PUT_DATA) {
3242 lntmsg[0] = rx->mxc_lntmsg[0];
3244 } else if (type == MXLND_MSG_GET_DATA) {
3246 lntmsg[0] = rx->mxc_lntmsg[0];
3247 lntmsg[1] = rx->mxc_lntmsg[1];
3251 ret = mxlnd_unpack_msg(msg, nob);
3253 CNETERR("Error %d unpacking rx from %s\n",
3254 ret, libcfs_nid2str(rx->mxc_nid));
3258 type = msg->mxm_type;
3260 if (rx->mxc_nid != msg->mxm_srcnid ||
3261 kmxlnd_data.kmx_ni->ni_nid != msg->mxm_dstnid) {
3262 CNETERR("rx with mismatched NID (type %s) (my nid is "
3263 "0x%llx and rx msg dst is 0x%llx)\n",
3264 mxlnd_msgtype_to_str(type), kmxlnd_data.kmx_ni->ni_nid,
3269 if ((conn != NULL && msg->mxm_srcstamp != conn->mxk_incarnation) ||
3270 msg->mxm_dststamp != kmxlnd_data.kmx_incarnation) {
3271 CNETERR("Stale rx from %s with type %s "
3272 "(mxm_srcstamp (%lld) != mxk_incarnation (%lld) "
3273 "|| mxm_dststamp (%lld) != kmx_incarnation (%lld))\n",
3274 libcfs_nid2str(rx->mxc_nid), mxlnd_msgtype_to_str(type),
3275 msg->mxm_srcstamp, conn->mxk_incarnation,
3276 msg->mxm_dststamp, kmxlnd_data.kmx_incarnation);
3281 CDEBUG(D_NET, "Received %s with %d credits\n",
3282 mxlnd_msgtype_to_str(type), msg->mxm_credits);
3284 LASSERT(peer != NULL && conn != NULL);
3285 if (msg->mxm_credits != 0) {
3286 spin_lock(&conn->mxk_lock);
3287 if (msg->mxm_srcstamp == conn->mxk_incarnation) {
3288 if ((conn->mxk_credits + msg->mxm_credits) >
3289 *kmxlnd_tunables.kmx_peercredits) {
3290 CNETERR("mxk_credits %d mxm_credits %d\n",
3291 conn->mxk_credits, msg->mxm_credits);
3293 conn->mxk_credits += msg->mxm_credits;
3294 LASSERT(conn->mxk_credits >= 0);
3295 LASSERT(conn->mxk_credits <= *kmxlnd_tunables.kmx_peercredits);
3297 spin_unlock(&conn->mxk_lock);
3300 CDEBUG(D_NET, "switch %s for rx (0x%llx)\n", mxlnd_msgtype_to_str(type), seq);
3302 case MXLND_MSG_NOOP:
3305 case MXLND_MSG_EAGER:
3306 ret = lnet_parse(kmxlnd_data.kmx_ni, &msg->mxm_u.eager.mxem_hdr,
3307 msg->mxm_srcnid, rx, 0);
3311 case MXLND_MSG_PUT_REQ:
3312 ret = lnet_parse(kmxlnd_data.kmx_ni, &msg->mxm_u.put_req.mxprm_hdr,
3313 msg->mxm_srcnid, rx, 1);
3317 case MXLND_MSG_PUT_ACK: {
3318 u64 cookie = (u64) msg->mxm_u.put_ack.mxpam_dst_cookie;
3319 if (cookie > MXLND_MAX_COOKIE) {
3320 CNETERR("NAK for msg_type %d from %s\n", rx->mxc_msg_type,
3321 libcfs_nid2str(rx->mxc_nid));
3322 result = -((u32) MXLND_ERROR_VAL(cookie));
3323 lntmsg[0] = rx->mxc_lntmsg[0];
3325 mxlnd_send_data(kmxlnd_data.kmx_ni, rx->mxc_lntmsg[0],
3326 rx->mxc_peer, MXLND_MSG_PUT_DATA,
3327 rx->mxc_msg->mxm_u.put_ack.mxpam_dst_cookie);
3332 case MXLND_MSG_GET_REQ:
3333 ret = lnet_parse(kmxlnd_data.kmx_ni, &msg->mxm_u.get_req.mxgrm_hdr,
3334 msg->mxm_srcnid, rx, 1);
3339 CNETERR("Bad MXLND message type %x from %s\n", msg->mxm_type,
3340 libcfs_nid2str(rx->mxc_nid));
3346 CDEBUG(D_NET, "setting PEER_CONN_FAILED\n");
3347 spin_lock(&conn->mxk_lock);
3348 mxlnd_set_conn_status(conn, MXLND_CONN_FAIL);
3349 spin_unlock(&conn->mxk_lock);
3354 spin_lock(&conn->mxk_lock);
3355 conn->mxk_last_rx = cfs_time_current(); /* jiffies */
3356 spin_unlock(&conn->mxk_lock);
3360 /* lnet_parse() failed, etc., repost now */
3361 mxlnd_put_idle_rx(rx);
3362 if (conn != NULL && credit == 1) {
3363 if (type == MXLND_MSG_PUT_DATA ||
3364 type == MXLND_MSG_EAGER ||
3365 type == MXLND_MSG_PUT_REQ ||
3366 type == MXLND_MSG_NOOP) {
3367 spin_lock(&conn->mxk_lock);
3368 conn->mxk_outstanding++;
3369 spin_unlock(&conn->mxk_lock);
3372 if (conn_ref) mxlnd_conn_decref(conn);
3373 LASSERT(peer_ref == 0);
3376 if (type == MXLND_MSG_PUT_DATA || type == MXLND_MSG_GET_DATA) {
3377 CDEBUG(D_NET, "leaving for rx (0x%llx)\n", bits);
3379 CDEBUG(D_NET, "leaving for rx (0x%llx)\n", seq);
3382 if (lntmsg[0] != NULL) lnet_finalize(kmxlnd_data.kmx_ni, lntmsg[0], result);
3383 if (lntmsg[1] != NULL) lnet_finalize(kmxlnd_data.kmx_ni, lntmsg[1], result);
3385 if (conn != NULL && credit == 1) mxlnd_check_sends(peer);
3391 mxlnd_handle_connect_msg(kmx_peer_t *peer, u8 msg_type, mx_status_t status)
3393 kmx_ctx_t *tx = NULL;
3394 kmx_msg_t *txmsg = NULL;
3395 kmx_conn_t *conn = peer->mxp_conn;
3399 u8 type = (msg_type == MXLND_MSG_ICON_REQ ?
3400 MXLND_MSG_CONN_REQ : MXLND_MSG_CONN_ACK);
3402 /* a conn ref was taken when calling mx_iconnect(),
3403 * hold it until CONN_REQ or CONN_ACK completes */
3405 CDEBUG(D_NET, "entering\n");
3406 if (status.code != MX_STATUS_SUCCESS) {
3407 int send_bye = (msg_type == MXLND_MSG_ICON_REQ ? 0 : 1);
3409 CNETERR("mx_iconnect() failed for %s with %s (%d) "
3410 "to %s mxp_nid = 0x%llx mxp_nic_id = 0x%0llx mxp_ep_id = %d\n",
3411 mxlnd_msgtype_to_str(msg_type),
3412 mx_strstatus(status.code), status.code,
3413 libcfs_nid2str(peer->mxp_nid),
3417 spin_lock(&conn->mxk_lock);
3418 mxlnd_set_conn_status(conn, MXLND_CONN_FAIL);
3419 spin_unlock(&conn->mxk_lock);
3421 if (cfs_time_after(jiffies, peer->mxp_reconnect_time +
3422 MXLND_CONNECT_TIMEOUT)) {
3423 CNETERR("timeout, calling conn_disconnect()\n");
3424 mxlnd_conn_disconnect(conn, 0, send_bye);
3427 mxlnd_conn_decref(conn);
3430 mx_decompose_endpoint_addr2(status.source, &nic_id, &ep_id, &sid);
3431 write_lock(&kmxlnd_data.kmx_global_lock);
3432 spin_lock(&conn->mxk_lock);
3433 conn->mxk_epa = status.source;
3434 mx_set_endpoint_addr_context(conn->mxk_epa, (void *) conn);
3435 if (msg_type == MXLND_MSG_ICON_ACK && likely(!peer->mxp_incompatible)) {
3436 mxlnd_set_conn_status(conn, MXLND_CONN_READY);
3438 spin_unlock(&conn->mxk_lock);
3439 write_unlock(&kmxlnd_data.kmx_global_lock);
3441 /* mx_iconnect() succeeded, reset delay to 0 */
3442 write_lock(&kmxlnd_data.kmx_global_lock);
3443 peer->mxp_reconnect_time = 0;
3444 peer->mxp_conn->mxk_sid = sid;
3445 write_unlock(&kmxlnd_data.kmx_global_lock);
3447 /* marshal CONN_REQ or CONN_ACK msg */
3448 /* we are still using the conn ref from iconnect() - do not take another */
3449 tx = mxlnd_get_idle_tx();
3451 CNETERR("Can't obtain %s tx for %s\n",
3452 mxlnd_msgtype_to_str(type),
3453 libcfs_nid2str(peer->mxp_nid));
3454 spin_lock(&conn->mxk_lock);
3455 mxlnd_set_conn_status(conn, MXLND_CONN_FAIL);
3456 spin_unlock(&conn->mxk_lock);
3457 mxlnd_conn_decref(conn);
3461 tx->mxc_peer = peer;
3462 tx->mxc_conn = conn;
3463 tx->mxc_deadline = jiffies + MXLND_CONNECT_TIMEOUT;
3464 CDEBUG(D_NET, "sending %s\n", mxlnd_msgtype_to_str(type));
3465 mxlnd_init_tx_msg (tx, type, sizeof(kmx_connreq_msg_t), peer->mxp_nid);
3466 txmsg = tx->mxc_msg;
3467 txmsg->mxm_u.conn_req.mxcrm_queue_depth = *kmxlnd_tunables.kmx_peercredits;
3468 txmsg->mxm_u.conn_req.mxcrm_eager_size = MXLND_MSG_SIZE;
3469 tx->mxc_match = mxlnd_create_match(tx, 0);
3476 * mxlnd_request_waitd - the MX request completion thread(s)
3477 * @arg - thread id (as a void *)
3479 * This thread waits for a MX completion and then completes the request.
3480 * We will create one thread per CPU.
3483 mxlnd_request_waitd(void *arg)
3485 long id = (long) arg;
3487 mx_return_t mxret = MX_SUCCESS;
3489 kmx_ctx_t *ctx = NULL;
3490 enum kmx_req_state req_type = MXLND_REQ_TX;
3491 kmx_peer_t *peer = NULL;
3492 kmx_conn_t *conn = NULL;
3497 memset(&status, 0, sizeof(status));
3499 CDEBUG(D_NET, "%s starting\n", name);
3501 while (!(atomic_read(&kmxlnd_data.kmx_shutdown))) {
3507 if (id == 0 && count++ < *kmxlnd_tunables.kmx_polling) {
3508 mxret = mx_test_any(kmxlnd_data.kmx_endpt, 0ULL, 0ULL,
3512 mxret = mx_wait_any(kmxlnd_data.kmx_endpt, MXLND_WAIT_TIMEOUT,
3513 0ULL, 0ULL, &status, &result);
3516 mxret = mx_wait_any(kmxlnd_data.kmx_endpt, MXLND_WAIT_TIMEOUT,
3517 0ULL, 0ULL, &status, &result);
3519 if (unlikely(atomic_read(&kmxlnd_data.kmx_shutdown)))
3523 /* nothing completed... */
3527 CDEBUG(D_NET, "wait_any() returned with %s (%d) with "
3528 "match_info 0x%llx and length %d\n",
3529 mx_strstatus(status.code), status.code,
3530 (u64) status.match_info, status.msg_length);
3532 if (status.code != MX_STATUS_SUCCESS) {
3533 CNETERR("wait_any() failed with %s (%d) with "
3534 "match_info 0x%llx and length %d\n",
3535 mx_strstatus(status.code), status.code,
3536 (u64) status.match_info, status.msg_length);
3539 msg_type = MXLND_MSG_TYPE(status.match_info);
3541 /* This may be a mx_iconnect() request completing,
3542 * check the bit mask for CONN_REQ and CONN_ACK */
3543 if (msg_type == MXLND_MSG_ICON_REQ ||
3544 msg_type == MXLND_MSG_ICON_ACK) {
3545 peer = (kmx_peer_t*) status.context;
3546 mxlnd_handle_connect_msg(peer, msg_type, status);
3550 /* This must be a tx or rx */
3552 /* NOTE: if this is a RX from the unexpected callback, it may
3553 * have very little info. If we dropped it in unexpected_recv(),
3554 * it will not have a context. If so, ignore it. */
3555 ctx = (kmx_ctx_t *) status.context;
3558 req_type = ctx->mxc_type;
3559 conn = ctx->mxc_conn; /* this may be NULL */
3560 mxlnd_deq_pending_ctx(ctx);
3562 /* copy status to ctx->mxc_status */
3563 ctx->mxc_status = status;
3567 mxlnd_handle_tx_completion(ctx);
3570 mxlnd_handle_rx_completion(ctx);
3573 CNETERR("Unknown ctx type %d\n", req_type);
3578 /* conn is always set except for the first CONN_REQ rx
3579 * from a new peer */
3580 if (status.code != MX_STATUS_SUCCESS && conn != NULL) {
3581 mxlnd_conn_disconnect(conn, 1, 1);
3584 CDEBUG(D_NET, "waitd() completed task\n");
3586 CDEBUG(D_NET, "%s stopping\n", name);
3587 mxlnd_thread_stop(id);
3593 mxlnd_check_timeouts(unsigned long now)
3597 unsigned long next = 0; /* jiffies */
3598 kmx_peer_t *peer = NULL;
3599 kmx_conn_t *conn = NULL;
3600 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
3603 for (i = 0; i < MXLND_HASH_SIZE; i++) {
3604 cfs_list_for_each_entry(peer, &kmxlnd_data.kmx_peers[i],
3607 if (unlikely(atomic_read(&kmxlnd_data.kmx_shutdown))) {
3608 read_unlock(g_lock);
3612 conn = peer->mxp_conn;
3614 mxlnd_conn_addref(conn);
3619 spin_lock(&conn->mxk_lock);
3621 /* if nothing pending (timeout == 0) or
3622 * if conn is already disconnected,
3624 if (conn->mxk_timeout == 0 ||
3625 conn->mxk_status == MXLND_CONN_DISCONNECT) {
3626 spin_unlock(&conn->mxk_lock);
3627 mxlnd_conn_decref(conn);
3631 /* we want to find the timeout that will occur first.
3632 * if it is in the future, we will sleep until then.
3633 * if it is in the past, then we will sleep one
3634 * second and repeat the process. */
3636 (cfs_time_before(conn->mxk_timeout, next))) {
3637 next = conn->mxk_timeout;
3642 if (cfs_time_aftereq(now, conn->mxk_timeout))
3644 spin_unlock(&conn->mxk_lock);
3647 mxlnd_conn_disconnect(conn, 1, 1);
3648 mxlnd_conn_decref(conn);
3651 read_unlock(g_lock);
3653 next = now + MXLND_COMM_TIMEOUT;
3659 mxlnd_passive_connect(kmx_connparams_t *cp)
3662 int incompatible = 0;
3667 kmx_msg_t *msg = &cp->mxr_msg;
3668 kmx_peer_t *peer = cp->mxr_peer;
3669 kmx_conn_t *conn = NULL;
3670 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
3672 mx_decompose_endpoint_addr2(cp->mxr_epa, &nic_id, &ep_id, &sid);
3674 ret = mxlnd_unpack_msg(msg, cp->mxr_nob);
3677 CNETERR("Error %d unpacking CONN_REQ from %s\n",
3678 ret, libcfs_nid2str(peer->mxp_nid));
3680 CNETERR("Error %d unpacking CONN_REQ from "
3681 "unknown host with nic_id 0x%llx\n", ret, nic_id);
3685 if (kmxlnd_data.kmx_ni->ni_nid != msg->mxm_dstnid) {
3686 CNETERR("Can't accept %s: bad dst nid %s\n",
3687 libcfs_nid2str(msg->mxm_srcnid),
3688 libcfs_nid2str(msg->mxm_dstnid));
3691 if (msg->mxm_u.conn_req.mxcrm_queue_depth != *kmxlnd_tunables.kmx_peercredits) {
3692 CNETERR("Can't accept %s: incompatible queue depth "
3694 libcfs_nid2str(msg->mxm_srcnid),
3695 msg->mxm_u.conn_req.mxcrm_queue_depth,
3696 *kmxlnd_tunables.kmx_peercredits);
3699 if (msg->mxm_u.conn_req.mxcrm_eager_size != MXLND_MSG_SIZE) {
3700 CNETERR("Can't accept %s: incompatible EAGER size "
3702 libcfs_nid2str(msg->mxm_srcnid),
3703 msg->mxm_u.conn_req.mxcrm_eager_size,
3704 (int) MXLND_MSG_SIZE);
3709 peer = mxlnd_find_peer_by_nid(msg->mxm_srcnid, 0); /* adds peer ref */
3713 kmx_peer_t *existing_peer = NULL;
3715 hash = mxlnd_nid_to_hash(msg->mxm_srcnid);
3717 mx_nic_id_to_board_number(nic_id, &board);
3719 /* adds conn ref for peer and one for this function */
3720 ret = mxlnd_peer_alloc(&peer, msg->mxm_srcnid,
3721 board, ep_id, 0ULL);
3725 peer->mxp_conn->mxk_sid = sid;
3726 LASSERT(peer->mxp_ep_id == ep_id);
3728 existing_peer = mxlnd_find_peer_by_nid_locked(msg->mxm_srcnid);
3729 if (existing_peer) {
3730 mxlnd_conn_decref(peer->mxp_conn);
3731 mxlnd_peer_decref(peer);
3732 peer = existing_peer;
3733 mxlnd_conn_addref(peer->mxp_conn);
3734 conn = peer->mxp_conn;
3736 cfs_list_add_tail(&peer->mxp_list,
3737 &kmxlnd_data.kmx_peers[hash]);
3738 atomic_inc(&kmxlnd_data.kmx_npeers);
3740 write_unlock(g_lock);
3742 ret = mxlnd_conn_alloc(&conn, peer); /* adds 2nd ref */
3744 mxlnd_peer_decref(peer); /* drop ref taken above */
3745 write_unlock(g_lock);
3747 CNETERR("Cannot allocate mxp_conn\n");
3751 conn_ref = 1; /* peer/conn_alloc() added ref for this function */
3752 conn = peer->mxp_conn;
3753 } else { /* unexpected handler found peer */
3754 kmx_conn_t *old_conn = peer->mxp_conn;
3756 if (sid != peer->mxp_conn->mxk_sid) {
3757 /* do not call mx_disconnect() or send a BYE */
3758 mxlnd_conn_disconnect(old_conn, 0, 0);
3760 /* This allocs a conn, points peer->mxp_conn to this one.
3761 * The old conn is still on the peer->mxp_conns list.
3762 * As the pending requests complete, they will call
3763 * conn_decref() which will eventually free it. */
3764 ret = mxlnd_conn_alloc(&conn, peer);
3766 CNETERR("Cannot allocate peer->mxp_conn\n");
3769 /* conn_alloc() adds one ref for the peer and one
3770 * for this function */
3773 peer->mxp_conn->mxk_sid = sid;
3776 conn = peer->mxp_conn;
3780 peer->mxp_incompatible = incompatible;
3781 write_unlock(g_lock);
3782 spin_lock(&conn->mxk_lock);
3783 conn->mxk_incarnation = msg->mxm_srcstamp;
3784 mxlnd_set_conn_status(conn, MXLND_CONN_WAIT);
3785 spin_unlock(&conn->mxk_lock);
3787 /* handle_conn_ack() will create the CONN_ACK msg */
3788 mxlnd_iconnect(peer, (u8) MXLND_MSG_ICON_ACK);
3791 if (conn_ref) mxlnd_conn_decref(conn);
3793 mxlnd_connparams_free(cp);
3798 mxlnd_check_conn_ack(kmx_connparams_t *cp)
3801 int incompatible = 0;
3805 kmx_msg_t *msg = &cp->mxr_msg;
3806 kmx_peer_t *peer = cp->mxr_peer;
3807 kmx_conn_t *conn = cp->mxr_conn;
3809 mx_decompose_endpoint_addr2(cp->mxr_epa, &nic_id, &ep_id, &sid);
3811 ret = mxlnd_unpack_msg(msg, cp->mxr_nob);
3814 CNETERR("Error %d unpacking CONN_ACK from %s\n",
3815 ret, libcfs_nid2str(peer->mxp_nid));
3817 CNETERR("Error %d unpacking CONN_ACK from "
3818 "unknown host with nic_id 0x%llx\n", ret, nic_id);
3824 if (kmxlnd_data.kmx_ni->ni_nid != msg->mxm_dstnid) {
3825 CNETERR("Can't accept CONN_ACK from %s: "
3826 "bad dst nid %s\n", libcfs_nid2str(msg->mxm_srcnid),
3827 libcfs_nid2str(msg->mxm_dstnid));
3831 if (msg->mxm_u.conn_req.mxcrm_queue_depth != *kmxlnd_tunables.kmx_peercredits) {
3832 CNETERR("Can't accept CONN_ACK from %s: "
3833 "incompatible queue depth %d (%d wanted)\n",
3834 libcfs_nid2str(msg->mxm_srcnid),
3835 msg->mxm_u.conn_req.mxcrm_queue_depth,
3836 *kmxlnd_tunables.kmx_peercredits);
3841 if (msg->mxm_u.conn_req.mxcrm_eager_size != MXLND_MSG_SIZE) {
3842 CNETERR("Can't accept CONN_ACK from %s: "
3843 "incompatible EAGER size %d (%d wanted)\n",
3844 libcfs_nid2str(msg->mxm_srcnid),
3845 msg->mxm_u.conn_req.mxcrm_eager_size,
3846 (int) MXLND_MSG_SIZE);
3851 write_lock(&kmxlnd_data.kmx_global_lock);
3852 peer->mxp_incompatible = incompatible;
3853 write_unlock(&kmxlnd_data.kmx_global_lock);
3854 spin_lock(&conn->mxk_lock);
3855 conn->mxk_credits = *kmxlnd_tunables.kmx_peercredits;
3856 conn->mxk_outstanding = 0;
3857 conn->mxk_incarnation = msg->mxm_srcstamp;
3858 conn->mxk_timeout = 0;
3859 if (!incompatible) {
3860 CDEBUG(D_NET, "setting peer %s CONN_READY\n",
3861 libcfs_nid2str(msg->mxm_srcnid));
3862 mxlnd_set_conn_status(conn, MXLND_CONN_READY);
3864 spin_unlock(&conn->mxk_lock);
3867 mxlnd_check_sends(peer);
3871 spin_lock(&conn->mxk_lock);
3872 mxlnd_set_conn_status(conn, MXLND_CONN_FAIL);
3873 spin_unlock(&conn->mxk_lock);
3876 if (incompatible) mxlnd_conn_disconnect(conn, 0, 0);
3878 mxlnd_connparams_free(cp);
3883 mxlnd_abort_msgs(void)
3886 cfs_list_t *orphans = &kmxlnd_data.kmx_orphan_msgs;
3887 spinlock_t *g_conn_lock = &kmxlnd_data.kmx_conn_lock;
3890 spin_lock(g_conn_lock);
3891 while (!cfs_list_empty(orphans)) {
3892 kmx_ctx_t *ctx = NULL;
3893 kmx_conn_t *conn = NULL;
3895 ctx = cfs_list_entry(orphans->next, kmx_ctx_t, mxc_list);
3896 cfs_list_del_init(&ctx->mxc_list);
3897 spin_unlock(g_conn_lock);
3899 ctx->mxc_errno = -ECONNABORTED;
3900 conn = ctx->mxc_conn;
3901 CDEBUG(D_NET, "aborting %s %s %s\n",
3902 mxlnd_msgtype_to_str(ctx->mxc_msg_type),
3903 ctx->mxc_type == MXLND_REQ_TX ? "(TX) to" : "(RX) from",
3904 libcfs_nid2str(ctx->mxc_nid));
3905 if (ctx->mxc_type == MXLND_REQ_TX) {
3906 mxlnd_put_idle_tx(ctx); /* do not hold any locks */
3907 if (conn) mxlnd_conn_decref(conn); /* for this tx */
3909 ctx->mxc_state = MXLND_CTX_CANCELED;
3910 mxlnd_handle_rx_completion(ctx);
3914 spin_lock(g_conn_lock);
3916 spin_unlock(g_conn_lock);
3922 mxlnd_free_conn_zombies(void)
3925 cfs_list_t *zombies = &kmxlnd_data.kmx_conn_zombies;
3926 spinlock_t *g_conn_lock = &kmxlnd_data.kmx_conn_lock;
3927 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
3929 /* cleanup any zombies */
3930 spin_lock(g_conn_lock);
3931 while (!cfs_list_empty(zombies)) {
3932 kmx_conn_t *conn = NULL;
3934 conn = cfs_list_entry(zombies->next, kmx_conn_t, mxk_zombie);
3935 cfs_list_del_init(&conn->mxk_zombie);
3936 spin_unlock(g_conn_lock);
3939 mxlnd_conn_free_locked(conn);
3940 write_unlock(g_lock);
3943 spin_lock(g_conn_lock);
3945 spin_unlock(g_conn_lock);
3946 CDEBUG(D_NET, "%s: freed %d zombies\n", __func__, count);
3951 * mxlnd_connd - handles incoming connection requests
3952 * @arg - thread id (as a void *)
3954 * This thread handles incoming connection requests
3957 mxlnd_connd(void *arg)
3959 long id = (long) arg;
3961 CDEBUG(D_NET, "connd starting\n");
3963 while (!(atomic_read(&kmxlnd_data.kmx_shutdown))) {
3965 kmx_connparams_t *cp = NULL;
3966 spinlock_t *g_conn_lock = &kmxlnd_data.kmx_conn_lock;
3967 cfs_list_t *conn_reqs = &kmxlnd_data.kmx_conn_reqs;
3969 ret = down_interruptible(&kmxlnd_data.kmx_conn_sem);
3971 if (atomic_read(&kmxlnd_data.kmx_shutdown))
3977 ret = mxlnd_abort_msgs();
3978 ret += mxlnd_free_conn_zombies();
3980 spin_lock(g_conn_lock);
3981 if (cfs_list_empty(conn_reqs)) {
3983 CNETERR("connd woke up but did not find a "
3984 "kmx_connparams_t or zombie conn\n");
3985 spin_unlock(g_conn_lock);
3988 cp = cfs_list_entry(conn_reqs->next, kmx_connparams_t,
3990 cfs_list_del_init(&cp->mxr_list);
3991 spin_unlock(g_conn_lock);
3993 switch (MXLND_MSG_TYPE(cp->mxr_match)) {
3994 case MXLND_MSG_CONN_REQ:
3995 /* We have a connection request. Handle it. */
3996 mxlnd_passive_connect(cp);
3998 case MXLND_MSG_CONN_ACK:
3999 /* The peer is ready for messages */
4000 mxlnd_check_conn_ack(cp);
4005 mxlnd_free_conn_zombies();
4007 CDEBUG(D_NET, "connd stopping\n");
4008 mxlnd_thread_stop(id);
4013 * mxlnd_timeoutd - enforces timeouts on messages
4014 * @arg - thread id (as a void *)
4016 * This thread queries each peer for its earliest timeout. If a peer has timed out,
4017 * it calls mxlnd_conn_disconnect().
4019 * After checking for timeouts, try progressing sends (call check_sends()).
4022 mxlnd_timeoutd(void *arg)
4025 long id = (long) arg;
4026 unsigned long now = 0;
4027 unsigned long next = 0;
4028 unsigned long delay = HZ;
4029 kmx_peer_t *peer = NULL;
4030 kmx_peer_t *temp = NULL;
4031 kmx_conn_t *conn = NULL;
4032 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
4034 CDEBUG(D_NET, "timeoutd starting\n");
4036 while (!(atomic_read(&kmxlnd_data.kmx_shutdown))) {
4039 /* if the next timeout has not arrived, go back to sleep */
4040 if (cfs_time_after(now, next)) {
4041 next = mxlnd_check_timeouts(now);
4044 /* try to progress peers' txs */
4046 for (i = 0; i < MXLND_HASH_SIZE; i++) {
4047 cfs_list_t *peers = &kmxlnd_data.kmx_peers[i];
4049 /* NOTE we are safe against the removal of peer, but
4050 * not against the removal of temp */
4051 cfs_list_for_each_entry_safe(peer, temp, peers,
4053 if (atomic_read(&kmxlnd_data.kmx_shutdown))
4055 mxlnd_peer_addref(peer); /* add ref... */
4056 conn = peer->mxp_conn;
4057 if (conn && conn->mxk_status != MXLND_CONN_DISCONNECT) {
4058 mxlnd_conn_addref(conn); /* take ref... */
4060 CDEBUG(D_NET, "ignoring %s\n",
4061 libcfs_nid2str(peer->mxp_nid));
4062 mxlnd_peer_decref(peer); /* ...to here */
4066 if ((conn->mxk_status == MXLND_CONN_READY ||
4067 conn->mxk_status == MXLND_CONN_FAIL) &&
4071 write_unlock(g_lock);
4072 mxlnd_check_sends(peer);
4075 mxlnd_conn_decref(conn); /* until here */
4076 mxlnd_peer_decref(peer); /* ...to here */
4079 write_unlock(g_lock);
4083 CDEBUG(D_NET, "timeoutd stopping\n");
4084 mxlnd_thread_stop(id);