4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
27 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Use is subject to license terms.
30 * Copyright (c) 2012, Intel Corporation.
32 * Copyright (C) 2006 Myricom, Inc.
35 * This file is part of Lustre, http://www.lustre.org/
36 * Lustre is a trademark of Sun Microsystems, Inc.
38 * lnet/klnds/mxlnd/mxlnd.c
40 * Author: Eric Barton <eric@bartonsoftware.com>
41 * Author: Scott Atchley <atchley at myri.com>
46 mx_endpoint_addr_t MX_EPA_NULL; /* use to determine if an endpoint is NULL */
49 mxlnd_endpoint_addr_null(mx_endpoint_addr_t epa)
51 /* if memcmp() == 0, it is NULL */
52 return !(memcmp(&epa, &MX_EPA_NULL, sizeof(epa)));
56 mxlnd_ctxstate_to_str(int mxc_state)
60 return "MXLND_CTX_INIT";
62 return "MXLND_CTX_IDLE";
64 return "MXLND_CTX_PREP";
65 case MXLND_CTX_PENDING:
66 return "MXLND_CTX_PENDING";
67 case MXLND_CTX_COMPLETED:
68 return "MXLND_CTX_COMPLETED";
69 case MXLND_CTX_CANCELED:
70 return "MXLND_CTX_CANCELED";
77 mxlnd_connstatus_to_str(int mxk_status)
80 case MXLND_CONN_READY:
81 return "MXLND_CONN_READY";
83 return "MXLND_CONN_INIT";
85 return "MXLND_CONN_WAIT";
86 case MXLND_CONN_DISCONNECT:
87 return "MXLND_CONN_DISCONNECT";
89 return "MXLND_CONN_FAIL";
96 mxlnd_msgtype_to_str(int type) {
99 return "MXLND_MSG_EAGER";
100 case MXLND_MSG_CONN_REQ:
101 return "MXLND_MSG_CONN_REQ";
102 case MXLND_MSG_CONN_ACK:
103 return "MXLND_MSG_CONN_ACK";
105 return "MXLND_MSG_BYE";
107 return "MXLND_MSG_NOOP";
108 case MXLND_MSG_PUT_REQ:
109 return "MXLND_MSG_PUT_REQ";
110 case MXLND_MSG_PUT_ACK:
111 return "MXLND_MSG_PUT_ACK";
112 case MXLND_MSG_PUT_DATA:
113 return "MXLND_MSG_PUT_DATA";
114 case MXLND_MSG_GET_REQ:
115 return "MXLND_MSG_GET_REQ";
116 case MXLND_MSG_GET_DATA:
117 return "MXLND_MSG_GET_DATA";
124 mxlnd_lnetmsg_to_str(int type)
128 return "LNET_MSG_ACK";
130 return "LNET_MSG_PUT";
132 return "LNET_MSG_GET";
134 return "LNET_MSG_REPLY";
136 return "LNET_MSG_HELLO";
144 mxlnd_create_match(kmx_ctx_t *ctx, u8 error)
146 u64 type = (u64) ctx->mxc_msg_type;
147 u64 err = (u64) error;
150 mxlnd_valid_msg_type(ctx->mxc_msg_type);
151 LASSERT(ctx->mxc_cookie >> MXLND_ERROR_OFFSET == 0);
152 match = (type << MXLND_MSG_OFFSET) | (err << MXLND_ERROR_OFFSET) | ctx->mxc_cookie;
157 mxlnd_parse_match(u64 match, u8 *msg_type, u8 *error, u64 *cookie)
159 *msg_type = (u8) MXLND_MSG_TYPE(match);
160 *error = (u8) MXLND_ERROR_VAL(match);
161 *cookie = match & MXLND_MAX_COOKIE;
162 mxlnd_valid_msg_type(*msg_type);
167 mxlnd_get_idle_rx(kmx_conn_t *conn)
169 cfs_list_t *rxs = NULL;
170 kmx_ctx_t *rx = NULL;
172 LASSERT(conn != NULL);
174 rxs = &conn->mxk_rx_idle;
176 spin_lock(&conn->mxk_lock);
178 if (cfs_list_empty (rxs)) {
179 spin_unlock(&conn->mxk_lock);
183 rx = cfs_list_entry (rxs->next, kmx_ctx_t, mxc_list);
184 cfs_list_del_init(&rx->mxc_list);
185 spin_unlock(&conn->mxk_lock);
188 if (rx->mxc_get != rx->mxc_put) {
189 CNETERR("*** RX get (%llu) != put (%llu) ***\n", rx->mxc_get, rx->mxc_put);
190 CNETERR("*** incarnation= %lld ***\n", rx->mxc_incarnation);
191 CNETERR("*** deadline= %ld ***\n", rx->mxc_deadline);
192 CNETERR("*** state= %s ***\n", mxlnd_ctxstate_to_str(rx->mxc_state));
193 CNETERR("*** listed?= %d ***\n", !cfs_list_empty(&rx->mxc_list));
194 CNETERR("*** nid= 0x%llx ***\n", rx->mxc_nid);
195 CNETERR("*** peer= 0x%p ***\n", rx->mxc_peer);
196 CNETERR("*** msg_type= %s ***\n", mxlnd_msgtype_to_str(rx->mxc_msg_type));
197 CNETERR("*** cookie= 0x%llx ***\n", rx->mxc_cookie);
198 CNETERR("*** nob= %d ***\n", rx->mxc_nob);
201 LASSERT (rx->mxc_get == rx->mxc_put);
205 LASSERT (rx->mxc_state == MXLND_CTX_IDLE);
206 rx->mxc_state = MXLND_CTX_PREP;
207 rx->mxc_deadline = jiffies + MXLND_COMM_TIMEOUT;
213 mxlnd_put_idle_rx(kmx_ctx_t *rx)
215 kmx_conn_t *conn = rx->mxc_conn;
216 cfs_list_t *rxs = &conn->mxk_rx_idle;
218 LASSERT(rx->mxc_type == MXLND_REQ_RX);
223 LASSERT(rx->mxc_get == rx->mxc_put);
225 spin_lock(&conn->mxk_lock);
226 cfs_list_add(&rx->mxc_list, rxs);
227 spin_unlock(&conn->mxk_lock);
232 mxlnd_get_idle_tx(void)
234 cfs_list_t *tmp = &kmxlnd_data.kmx_tx_idle;
235 kmx_ctx_t *tx = NULL;
237 spin_lock(&kmxlnd_data.kmx_tx_idle_lock);
239 if (cfs_list_empty (&kmxlnd_data.kmx_tx_idle)) {
240 CNETERR("%d txs in use\n", kmxlnd_data.kmx_tx_used);
241 spin_unlock(&kmxlnd_data.kmx_tx_idle_lock);
245 tmp = &kmxlnd_data.kmx_tx_idle;
246 tx = cfs_list_entry (tmp->next, kmx_ctx_t, mxc_list);
247 cfs_list_del_init(&tx->mxc_list);
249 /* Allocate a new completion cookie. It might not be needed,
250 * but we've got a lock right now and we're unlikely to
252 tx->mxc_cookie = kmxlnd_data.kmx_tx_next_cookie++;
253 if (kmxlnd_data.kmx_tx_next_cookie > MXLND_MAX_COOKIE) {
254 kmxlnd_data.kmx_tx_next_cookie = 1;
256 kmxlnd_data.kmx_tx_used++;
257 spin_unlock(&kmxlnd_data.kmx_tx_idle_lock);
259 LASSERT (tx->mxc_get == tx->mxc_put);
263 LASSERT (tx->mxc_state == MXLND_CTX_IDLE);
264 LASSERT (tx->mxc_lntmsg[0] == NULL);
265 LASSERT (tx->mxc_lntmsg[1] == NULL);
267 tx->mxc_state = MXLND_CTX_PREP;
268 tx->mxc_deadline = jiffies + MXLND_COMM_TIMEOUT;
274 mxlnd_conn_disconnect(kmx_conn_t *conn, int mx_dis, int send_bye);
277 mxlnd_put_idle_tx(kmx_ctx_t *tx)
280 lnet_msg_t *lntmsg[2];
282 LASSERT(tx->mxc_type == MXLND_REQ_TX);
284 if (tx->mxc_status.code != MX_STATUS_SUCCESS || tx->mxc_errno != 0) {
285 kmx_conn_t *conn = tx->mxc_conn;
288 if (tx->mxc_errno != 0) result = tx->mxc_errno;
289 /* FIXME should we set mx_dis? */
290 mxlnd_conn_disconnect(conn, 0, 1);
293 lntmsg[0] = tx->mxc_lntmsg[0];
294 lntmsg[1] = tx->mxc_lntmsg[1];
299 LASSERT(tx->mxc_get == tx->mxc_put);
301 spin_lock(&kmxlnd_data.kmx_tx_idle_lock);
302 cfs_list_add_tail(&tx->mxc_list, &kmxlnd_data.kmx_tx_idle);
303 kmxlnd_data.kmx_tx_used--;
304 spin_unlock(&kmxlnd_data.kmx_tx_idle_lock);
306 if (lntmsg[0] != NULL)
307 lnet_finalize(kmxlnd_data.kmx_ni, lntmsg[0], result);
308 if (lntmsg[1] != NULL)
309 lnet_finalize(kmxlnd_data.kmx_ni, lntmsg[1], result);
315 mxlnd_connparams_free(kmx_connparams_t *cp)
317 LASSERT(cfs_list_empty(&cp->mxr_list));
318 MXLND_FREE(cp, sizeof(*cp));
323 mxlnd_connparams_alloc(kmx_connparams_t **cp, void *context,
324 mx_endpoint_addr_t epa, u64 match, u32 length,
325 kmx_conn_t *conn, kmx_peer_t *peer, void *data)
327 kmx_connparams_t *c = NULL;
329 MXLND_ALLOC(c, sizeof(*c));
330 if (!c) return -ENOMEM;
332 CFS_INIT_LIST_HEAD(&c->mxr_list);
333 c->mxr_context = context;
335 c->mxr_match = match;
339 c->mxr_msg = *((kmx_msg_t *) data);
346 mxlnd_set_conn_status(kmx_conn_t *conn, int status)
348 conn->mxk_status = status;
353 * mxlnd_conn_free_locked - free the conn
354 * @conn - a kmx_conn pointer
356 * The calling function should remove the conn from the conns list first
357 * then destroy it. Caller should have write-locked kmx_global_lock.
360 mxlnd_conn_free_locked(kmx_conn_t *conn)
362 int valid = !mxlnd_endpoint_addr_null(conn->mxk_epa);
363 kmx_peer_t *peer = conn->mxk_peer;
365 CDEBUG(D_NET, "freeing conn 0x%p *****\n", conn);
366 LASSERT (cfs_list_empty (&conn->mxk_tx_credit_queue) &&
367 cfs_list_empty (&conn->mxk_tx_free_queue) &&
368 cfs_list_empty (&conn->mxk_pending));
369 if (!cfs_list_empty(&conn->mxk_list)) {
370 cfs_list_del_init(&conn->mxk_list);
371 if (peer->mxp_conn == conn) {
372 peer->mxp_conn = NULL;
374 kmx_conn_t *temp = NULL;
376 mx_get_endpoint_addr_context(conn->mxk_epa,
379 mx_set_endpoint_addr_context(conn->mxk_epa,
383 /* unlink from global list and drop its ref */
384 cfs_list_del_init(&peer->mxp_list);
385 mxlnd_peer_decref(peer);
388 mxlnd_peer_decref(peer); /* drop conn's ref to peer */
389 if (conn->mxk_rx_pages) {
390 LASSERT (conn->mxk_rxs != NULL);
391 mxlnd_free_pages(conn->mxk_rx_pages);
395 kmx_ctx_t *rx = NULL;
397 for (i = 0; i < MXLND_RX_MSGS(); i++) {
398 rx = &conn->mxk_rxs[i];
399 if (rx->mxc_seg_list != NULL) {
400 LASSERT(rx->mxc_nseg > 0);
401 MXLND_FREE(rx->mxc_seg_list,
403 sizeof(*rx->mxc_seg_list));
406 MXLND_FREE(conn->mxk_rxs, MXLND_RX_MSGS() * sizeof(kmx_ctx_t));
409 MXLND_FREE(conn, sizeof (*conn));
415 mxlnd_conn_cancel_pending_rxs(kmx_conn_t *conn)
419 kmx_ctx_t *ctx = NULL;
420 kmx_ctx_t *next = NULL;
421 mx_return_t mxret = MX_SUCCESS;
426 spin_lock(&conn->mxk_lock);
427 cfs_list_for_each_entry_safe(ctx, next, &conn->mxk_pending,
429 cfs_list_del_init(&ctx->mxc_list);
430 if (ctx->mxc_type == MXLND_REQ_RX) {
432 mxret = mx_cancel(kmxlnd_data.kmx_endpt,
435 if (mxret != MX_SUCCESS) {
436 CNETERR("mx_cancel() returned %s (%d)\n", mx_strerror(mxret), mxret);
439 ctx->mxc_errno = -ECONNABORTED;
440 ctx->mxc_state = MXLND_CTX_CANCELED;
441 spin_unlock(&conn->mxk_lock);
442 spin_lock(&kmxlnd_data.kmx_conn_lock);
443 /* we may be holding the global lock,
444 * move to orphan list so that it can free it */
445 cfs_list_add_tail(&ctx->mxc_list,
446 &kmxlnd_data.kmx_orphan_msgs);
448 spin_unlock(&kmxlnd_data.kmx_conn_lock);
449 spin_lock(&conn->mxk_lock);
454 spin_unlock(&conn->mxk_lock);
461 mxlnd_cancel_queued_txs(kmx_conn_t *conn)
464 cfs_list_t *tmp = NULL;
466 spin_lock(&conn->mxk_lock);
467 while (!cfs_list_empty(&conn->mxk_tx_free_queue) ||
468 !cfs_list_empty(&conn->mxk_tx_credit_queue)) {
470 kmx_ctx_t *tx = NULL;
472 if (!cfs_list_empty(&conn->mxk_tx_free_queue)) {
473 tmp = &conn->mxk_tx_free_queue;
475 tmp = &conn->mxk_tx_credit_queue;
478 tx = cfs_list_entry(tmp->next, kmx_ctx_t, mxc_list);
479 cfs_list_del_init(&tx->mxc_list);
480 spin_unlock(&conn->mxk_lock);
481 tx->mxc_errno = -ECONNABORTED;
482 tx->mxc_state = MXLND_CTX_CANCELED;
483 /* move to orphan list and then abort */
484 spin_lock(&kmxlnd_data.kmx_conn_lock);
485 cfs_list_add_tail(&tx->mxc_list, &kmxlnd_data.kmx_orphan_msgs);
486 spin_unlock(&kmxlnd_data.kmx_conn_lock);
488 spin_lock(&conn->mxk_lock);
490 spin_unlock(&conn->mxk_lock);
496 mxlnd_send_message(mx_endpoint_addr_t epa, u8 msg_type, int error, u64 cookie)
498 u64 match = (((u64) msg_type) << MXLND_MSG_OFFSET) |
499 (((u64) error) << MXLND_ERROR_OFFSET) | cookie;
501 mx_kisend(kmxlnd_data.kmx_endpt, NULL, 0, MX_PIN_PHYSICAL,
502 epa, match, NULL, NULL);
507 * mxlnd_conn_disconnect - shutdown a connection
508 * @conn - a kmx_conn pointer
509 * @mx_dis - call mx_disconnect()
510 * @send_bye - send peer a BYE msg
512 * This function sets the status to DISCONNECT, completes queued
513 * txs with failure, calls mx_disconnect, which will complete
514 * pending txs and matched rxs with failure.
517 mxlnd_conn_disconnect(kmx_conn_t *conn, int mx_dis, int send_bye)
519 mx_endpoint_addr_t epa = conn->mxk_epa;
520 int valid = !mxlnd_endpoint_addr_null(epa);
523 spin_lock(&conn->mxk_lock);
524 if (conn->mxk_status == MXLND_CONN_DISCONNECT) {
525 spin_unlock(&conn->mxk_lock);
528 mxlnd_set_conn_status(conn, MXLND_CONN_DISCONNECT);
529 conn->mxk_timeout = 0;
530 spin_unlock(&conn->mxk_lock);
532 count = mxlnd_cancel_queued_txs(conn);
533 count += mxlnd_conn_cancel_pending_rxs(conn);
535 if (count) /* let connd call kmxlnd_abort_msgs() */
536 up(&kmxlnd_data.kmx_conn_sem);
538 if (send_bye && valid &&
539 conn->mxk_peer->mxp_nid != kmxlnd_data.kmx_ni->ni_nid) {
540 /* send a BYE to the peer */
541 CDEBUG(D_NET, "%s: sending a BYE msg to %s\n", __func__,
542 libcfs_nid2str(conn->mxk_peer->mxp_nid));
543 mxlnd_send_message(epa, MXLND_MSG_BYE, 0, 0);
544 /* wait to allow the peer to ack our message */
545 mxlnd_sleep(msecs_to_jiffies(20));
548 if (cfs_atomic_read(&kmxlnd_data.kmx_shutdown) != 1) {
549 unsigned long last_msg = 0;
551 /* notify LNET that we are giving up on this peer */
552 if (cfs_time_after(conn->mxk_last_rx, conn->mxk_last_tx))
553 last_msg = conn->mxk_last_rx;
555 last_msg = conn->mxk_last_tx;
557 lnet_notify(kmxlnd_data.kmx_ni, conn->mxk_peer->mxp_nid, 0, last_msg);
559 if (mx_dis && valid &&
560 (memcmp(&epa, &kmxlnd_data.kmx_epa, sizeof(epa) != 0)))
561 mx_disconnect(kmxlnd_data.kmx_endpt, epa);
563 mxlnd_conn_decref(conn); /* drop the owning peer's reference */
569 * mxlnd_conn_alloc - allocate and initialize a new conn struct
570 * @connp - address of a kmx_conn pointer
571 * @peer - owning kmx_peer
573 * Returns 0 on success and -ENOMEM on failure
576 mxlnd_conn_alloc_locked(kmx_conn_t **connp, kmx_peer_t *peer)
583 kmx_conn_t *conn = NULL;
584 kmx_pages_t *pages = NULL;
585 struct page *page = NULL;
586 kmx_ctx_t *rx = NULL;
588 LASSERT(peer != NULL);
590 MXLND_ALLOC(conn, sizeof (*conn));
592 CNETERR("Cannot allocate conn\n");
595 CDEBUG(D_NET, "allocated conn 0x%p for peer 0x%p\n", conn, peer);
597 memset(conn, 0, sizeof(*conn));
599 ret = mxlnd_alloc_pages(&pages, MXLND_RX_MSG_PAGES());
601 CERROR("Can't allocate rx pages\n");
602 MXLND_FREE(conn, sizeof(*conn));
605 conn->mxk_rx_pages = pages;
607 MXLND_ALLOC(conn->mxk_rxs, MXLND_RX_MSGS() * sizeof(kmx_ctx_t));
608 if (conn->mxk_rxs == NULL) {
609 CERROR("Can't allocate %d rx descriptors\n", MXLND_RX_MSGS());
610 mxlnd_free_pages(pages);
611 MXLND_FREE(conn, sizeof(*conn));
615 memset(conn->mxk_rxs, 0, MXLND_RX_MSGS() * sizeof(kmx_ctx_t));
617 conn->mxk_peer = peer;
618 CFS_INIT_LIST_HEAD(&conn->mxk_list);
619 CFS_INIT_LIST_HEAD(&conn->mxk_zombie);
620 cfs_atomic_set(&conn->mxk_refcount, 2); /* ref for owning peer
621 and one for the caller */
622 if (peer->mxp_nid == kmxlnd_data.kmx_ni->ni_nid) {
626 /* this is localhost, set the epa and status as up */
627 mxlnd_set_conn_status(conn, MXLND_CONN_READY);
628 conn->mxk_epa = kmxlnd_data.kmx_epa;
629 mx_set_endpoint_addr_context(conn->mxk_epa, (void *) conn);
630 peer->mxp_reconnect_time = 0;
631 mx_decompose_endpoint_addr(kmxlnd_data.kmx_epa, &nic_id, &ep_id);
632 peer->mxp_nic_id = nic_id;
633 peer->mxp_ep_id = ep_id;
634 conn->mxk_incarnation = kmxlnd_data.kmx_incarnation;
635 conn->mxk_timeout = 0;
637 /* conn->mxk_incarnation = 0 - will be set by peer */
638 /* conn->mxk_sid = 0 - will be set by peer */
639 mxlnd_set_conn_status(conn, MXLND_CONN_INIT);
640 /* mxk_epa - to be set after mx_iconnect() */
642 spin_lock_init(&conn->mxk_lock);
643 /* conn->mxk_timeout = 0 */
644 /* conn->mxk_last_tx = 0 */
645 /* conn->mxk_last_rx = 0 */
646 CFS_INIT_LIST_HEAD(&conn->mxk_rx_idle);
648 conn->mxk_credits = *kmxlnd_tunables.kmx_peercredits;
649 /* mxk_outstanding = 0 */
651 CFS_INIT_LIST_HEAD(&conn->mxk_tx_credit_queue);
652 CFS_INIT_LIST_HEAD(&conn->mxk_tx_free_queue);
653 /* conn->mxk_ntx_msgs = 0 */
654 /* conn->mxk_ntx_data = 0 */
655 /* conn->mxk_ntx_posted = 0 */
656 /* conn->mxk_data_posted = 0 */
657 CFS_INIT_LIST_HEAD(&conn->mxk_pending);
659 for (i = 0; i < MXLND_RX_MSGS(); i++) {
661 rx = &conn->mxk_rxs[i];
662 rx->mxc_type = MXLND_REQ_RX;
663 CFS_INIT_LIST_HEAD(&rx->mxc_list);
665 /* map mxc_msg to page */
666 page = pages->mxg_pages[ipage];
667 addr = page_address(page);
668 LASSERT(addr != NULL);
669 rx->mxc_msg = (kmx_msg_t *)(addr + offset);
670 rx->mxc_seg.segment_ptr = MX_PA_TO_U64(virt_to_phys(rx->mxc_msg));
674 rx->mxc_nid = peer->mxp_nid;
678 offset += MXLND_MSG_SIZE;
679 LASSERT (offset <= PAGE_SIZE);
681 if (offset == PAGE_SIZE) {
684 LASSERT (ipage <= MXLND_TX_MSG_PAGES());
687 cfs_list_add_tail(&rx->mxc_list, &conn->mxk_rx_idle);
692 mxlnd_peer_addref(peer); /* add a ref for this conn */
694 /* add to front of peer's conns list */
695 cfs_list_add(&conn->mxk_list, &peer->mxp_conns);
696 peer->mxp_conn = conn;
701 mxlnd_conn_alloc(kmx_conn_t **connp, kmx_peer_t *peer)
704 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
707 ret = mxlnd_conn_alloc_locked(connp, peer);
708 write_unlock(g_lock);
713 mxlnd_q_pending_ctx(kmx_ctx_t *ctx)
716 kmx_conn_t *conn = ctx->mxc_conn;
718 ctx->mxc_state = MXLND_CTX_PENDING;
720 spin_lock(&conn->mxk_lock);
721 if (conn->mxk_status >= MXLND_CONN_INIT) {
722 cfs_list_add_tail(&ctx->mxc_list, &conn->mxk_pending);
723 if (conn->mxk_timeout == 0 || ctx->mxc_deadline < conn->mxk_timeout) {
724 conn->mxk_timeout = ctx->mxc_deadline;
727 ctx->mxc_state = MXLND_CTX_COMPLETED;
730 spin_unlock(&conn->mxk_lock);
736 mxlnd_deq_pending_ctx(kmx_ctx_t *ctx)
738 LASSERT(ctx->mxc_state == MXLND_CTX_PENDING ||
739 ctx->mxc_state == MXLND_CTX_COMPLETED);
740 if (ctx->mxc_state != MXLND_CTX_PENDING &&
741 ctx->mxc_state != MXLND_CTX_COMPLETED) {
742 CNETERR("deq ctx->mxc_state = %s\n",
743 mxlnd_ctxstate_to_str(ctx->mxc_state));
745 ctx->mxc_state = MXLND_CTX_COMPLETED;
746 if (!cfs_list_empty(&ctx->mxc_list)) {
747 kmx_conn_t *conn = ctx->mxc_conn;
748 kmx_ctx_t *next = NULL;
750 LASSERT(conn != NULL);
751 spin_lock(&conn->mxk_lock);
752 cfs_list_del_init(&ctx->mxc_list);
753 conn->mxk_timeout = 0;
754 if (!cfs_list_empty(&conn->mxk_pending)) {
755 next = cfs_list_entry(conn->mxk_pending.next,
756 kmx_ctx_t, mxc_list);
757 conn->mxk_timeout = next->mxc_deadline;
759 spin_unlock(&conn->mxk_lock);
765 * mxlnd_peer_free - free the peer
766 * @peer - a kmx_peer pointer
768 * The calling function should decrement the rxs, drain the tx queues and
769 * remove the peer from the peers list first then destroy it.
772 mxlnd_peer_free(kmx_peer_t *peer)
774 CDEBUG(D_NET, "freeing peer 0x%p %s\n", peer, libcfs_nid2str(peer->mxp_nid));
776 LASSERT (cfs_atomic_read(&peer->mxp_refcount) == 0);
778 if (!cfs_list_empty(&peer->mxp_list)) {
779 /* assume we are locked */
780 cfs_list_del_init(&peer->mxp_list);
783 MXLND_FREE(peer, sizeof (*peer));
784 cfs_atomic_dec(&kmxlnd_data.kmx_npeers);
789 mxlnd_lookup_mac(u32 ip, u64 *tmp_id)
791 int ret = -EHOSTUNREACH;
792 unsigned char *haddr = NULL;
793 struct net_device *dev = NULL;
794 struct neighbour *n = NULL;
795 __be32 dst_ip = htonl(ip);
797 dev = dev_get_by_name(*kmxlnd_tunables.kmx_default_ipif);
801 haddr = (unsigned char *) tmp_id + 2; /* MAC is only 6 bytes */
803 n = neigh_lookup(&arp_tbl, &dst_ip, dev);
806 if (n->nud_state & NUD_VALID) {
807 memcpy(haddr, n->ha, dev->addr_len);
819 /* We only want the MAC address of the peer's Myricom NIC. We
820 * require that each node has the IPoMX interface (myriN) up.
821 * We will not pass any traffic over IPoMX, but it allows us
822 * to get the MAC address. */
824 mxlnd_ip2nic_id(u32 ip, u64 *nic_id, int tries)
830 cfs_socket_t *sock = NULL;
833 CDEBUG(D_NET, "try %d of %d tries\n", try, tries);
834 ret = mxlnd_lookup_mac(ip, &tmp_id);
838 /* not found, try to connect (force an arp) */
839 ret = libcfs_sock_connect(&sock, &fatal, 0, 0, ip, 987);
840 if (ret == -ECONNREFUSED) {
841 /* peer is there, get the MAC address */
842 mxlnd_lookup_mac(ip, &tmp_id);
846 } else if (ret == -EHOSTUNREACH && try < tries) {
847 /* add a little backoff */
848 CDEBUG(D_NET, "sleeping for %d jiffies\n",
850 mxlnd_sleep(CFS_HZ/4);
853 } while (try++ < tries);
854 CDEBUG(D_NET, "done trying. ret = %d\n", ret);
858 #ifdef __LITTLE_ENDIAN
859 *nic_id = ___arch__swab64(tmp_id);
867 * mxlnd_peer_alloc - allocate and initialize a new peer struct
868 * @peerp - address of a kmx_peer pointer
869 * @nid - LNET node id
871 * Returns 0 on success and -ENOMEM on failure
874 mxlnd_peer_alloc(kmx_peer_t **peerp, lnet_nid_t nid, u32 board, u32 ep_id, u64 nic_id)
877 u32 ip = LNET_NIDADDR(nid);
878 kmx_peer_t *peer = NULL;
880 LASSERT (nid != LNET_NID_ANY && nid != 0LL);
882 MXLND_ALLOC(peer, sizeof (*peer));
884 CNETERR("Cannot allocate peer for NID 0x%llx\n",
888 CDEBUG(D_NET, "allocated peer 0x%p for NID 0x%llx\n", peer, nid);
890 memset(peer, 0, sizeof(*peer));
892 CFS_INIT_LIST_HEAD(&peer->mxp_list);
894 /* peer->mxp_ni unused - may be used for multi-rail */
895 cfs_atomic_set(&peer->mxp_refcount, 1); /* ref for kmx_peers list */
897 peer->mxp_board = board;
898 peer->mxp_ep_id = ep_id;
899 peer->mxp_nic_id = nic_id;
901 CFS_INIT_LIST_HEAD(&peer->mxp_conns);
902 ret = mxlnd_conn_alloc(&peer->mxp_conn, peer); /* adds 2nd conn ref here... */
904 mxlnd_peer_decref(peer);
907 CFS_INIT_LIST_HEAD(&peer->mxp_tx_queue);
909 if (peer->mxp_nic_id != 0ULL)
910 nic_id = peer->mxp_nic_id;
912 if (nic_id == 0ULL) {
913 ret = mxlnd_ip2nic_id(ip, &nic_id, 1);
915 peer->mxp_nic_id = nic_id;
916 mx_nic_id_to_board_number(nic_id, &peer->mxp_board);
920 peer->mxp_nic_id = nic_id; /* may be 0ULL if ip2nic_id() failed */
922 /* peer->mxp_reconnect_time = 0 */
923 /* peer->mxp_incompatible = 0 */
929 static inline kmx_peer_t *
930 mxlnd_find_peer_by_nid_locked(lnet_nid_t nid)
934 kmx_peer_t *peer = NULL;
936 hash = mxlnd_nid_to_hash(nid);
938 cfs_list_for_each_entry(peer, &kmxlnd_data.kmx_peers[hash], mxp_list) {
939 if (peer->mxp_nid == nid) {
941 mxlnd_peer_addref(peer);
945 return (found ? peer : NULL);
949 mxlnd_find_peer_by_nid(lnet_nid_t nid, int create)
953 kmx_peer_t *peer = NULL;
954 kmx_peer_t *old = NULL;
955 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
958 peer = mxlnd_find_peer_by_nid_locked(nid); /* adds peer ref */
960 if ((peer && peer->mxp_conn) || /* found peer with conn or */
961 (!peer && !create)) { /* did not find peer and do not create one */
968 /* if peer but _not_ conn */
969 if (peer && !peer->mxp_conn) {
972 if (!peer->mxp_conn) { /* check again */
973 /* create the conn */
974 ret = mxlnd_conn_alloc_locked(&peer->mxp_conn, peer);
976 /* we tried, return the peer only.
977 * the caller needs to see if the conn exists */
978 CNETERR("%s: %s could not alloc conn\n",
979 __func__, libcfs_nid2str(peer->mxp_nid));
981 /* drop extra conn ref */
982 mxlnd_conn_decref(peer->mxp_conn);
985 write_unlock(g_lock);
990 /* peer not found and we need to create one */
991 hash = mxlnd_nid_to_hash(nid);
993 /* create peer (and conn) */
994 /* adds conn ref for peer and one for this function */
995 ret = mxlnd_peer_alloc(&peer, nid, *kmxlnd_tunables.kmx_board,
996 *kmxlnd_tunables.kmx_ep_id, 0ULL);
997 if (ret != 0) /* no memory, peer is NULL */
1003 old = mxlnd_find_peer_by_nid_locked(nid);
1005 /* someone already created one */
1006 mxlnd_conn_decref(peer->mxp_conn); /* drop ref taken above.. */
1007 mxlnd_conn_decref(peer->mxp_conn); /* drop peer's ref */
1008 mxlnd_peer_decref(peer);
1011 /* no other peer, use this one */
1012 cfs_list_add_tail(&peer->mxp_list,
1013 &kmxlnd_data.kmx_peers[hash]);
1014 cfs_atomic_inc(&kmxlnd_data.kmx_npeers);
1015 mxlnd_peer_addref(peer);
1016 mxlnd_conn_decref(peer->mxp_conn); /* drop ref from peer_alloc */
1019 write_unlock(g_lock);
1025 mxlnd_tx_requires_credit(kmx_ctx_t *tx)
1027 return (tx->mxc_msg_type == MXLND_MSG_EAGER ||
1028 tx->mxc_msg_type == MXLND_MSG_GET_REQ ||
1029 tx->mxc_msg_type == MXLND_MSG_PUT_REQ ||
1030 tx->mxc_msg_type == MXLND_MSG_NOOP);
1034 * mxlnd_init_msg - set type and number of bytes
1035 * @msg - msg pointer
1036 * @type - of message
1037 * @body_nob - bytes in msg body
1040 mxlnd_init_msg(kmx_msg_t *msg, u8 type, int body_nob)
1042 msg->mxm_type = type;
1043 msg->mxm_nob = offsetof(kmx_msg_t, mxm_u) + body_nob;
1047 mxlnd_init_tx_msg (kmx_ctx_t *tx, u8 type, int body_nob, lnet_nid_t nid)
1049 int nob = offsetof (kmx_msg_t, mxm_u) + body_nob;
1050 kmx_msg_t *msg = NULL;
1052 LASSERT (tx != NULL);
1053 LASSERT (nob <= MXLND_MSG_SIZE);
1056 /* tx->mxc_peer should have already been set if we know it */
1057 tx->mxc_msg_type = type;
1059 /* tx->mxc_seg.segment_ptr is already pointing to mxc_page */
1060 tx->mxc_seg.segment_length = nob;
1061 tx->mxc_pin_type = MX_PIN_PHYSICAL;
1064 msg->mxm_type = type;
1071 mxlnd_cksum (void *ptr, int nob)
1077 sum = ((sum << 1) | (sum >> 31)) + *c++;
1079 /* ensure I don't return 0 (== no checksum) */
1080 return (sum == 0) ? 1 : sum;
1084 * mxlnd_pack_msg_locked - complete msg info
1088 mxlnd_pack_msg_locked(kmx_ctx_t *tx)
1090 kmx_msg_t *msg = tx->mxc_msg;
1092 /* type and nob should already be set in init_msg() */
1093 msg->mxm_magic = MXLND_MSG_MAGIC;
1094 msg->mxm_version = MXLND_MSG_VERSION;
1096 /* don't use mxlnd_tx_requires_credit() since we want PUT_ACK to
1097 * return credits as well */
1098 if (tx->mxc_msg_type != MXLND_MSG_CONN_REQ &&
1099 tx->mxc_msg_type != MXLND_MSG_CONN_ACK) {
1100 msg->mxm_credits = tx->mxc_conn->mxk_outstanding;
1101 tx->mxc_conn->mxk_outstanding = 0;
1103 msg->mxm_credits = 0;
1107 msg->mxm_srcnid = kmxlnd_data.kmx_ni->ni_nid;
1108 msg->mxm_srcstamp = kmxlnd_data.kmx_incarnation;
1109 msg->mxm_dstnid = tx->mxc_nid;
1110 /* if it is a new peer, the dststamp will be 0 */
1111 msg->mxm_dststamp = tx->mxc_conn->mxk_incarnation;
1113 if (*kmxlnd_tunables.kmx_cksum) {
1114 msg->mxm_cksum = mxlnd_cksum(msg, msg->mxm_nob);
1119 mxlnd_unpack_msg(kmx_msg_t *msg, int nob)
1121 const int hdr_size = offsetof(kmx_msg_t, mxm_u);
1122 __u32 msg_cksum = 0;
1126 /* 6 bytes are enough to have received magic + version */
1128 CNETERR("not enough bytes for magic + hdr: %d\n", nob);
1132 if (msg->mxm_magic == MXLND_MSG_MAGIC) {
1134 } else if (msg->mxm_magic == __swab32(MXLND_MSG_MAGIC)) {
1137 CNETERR("Bad magic: %08x\n", msg->mxm_magic);
1141 if (msg->mxm_version !=
1142 (flip ? __swab16(MXLND_MSG_VERSION) : MXLND_MSG_VERSION)) {
1143 CNETERR("Bad version: %d\n", msg->mxm_version);
1147 if (nob < hdr_size) {
1148 CNETERR("not enough for a header: %d\n", nob);
1152 msg_nob = flip ? __swab32(msg->mxm_nob) : msg->mxm_nob;
1153 if (msg_nob > nob) {
1154 CNETERR("Short message: got %d, wanted %d\n", nob, msg_nob);
1158 /* checksum must be computed with mxm_cksum zero and BEFORE anything
1160 msg_cksum = flip ? __swab32(msg->mxm_cksum) : msg->mxm_cksum;
1162 if (msg_cksum != 0 && msg_cksum != mxlnd_cksum(msg, msg_nob)) {
1163 CNETERR("Bad checksum\n");
1166 msg->mxm_cksum = msg_cksum;
1169 /* leave magic unflipped as a clue to peer endianness */
1170 __swab16s(&msg->mxm_version);
1171 CLASSERT (sizeof(msg->mxm_type) == 1);
1172 CLASSERT (sizeof(msg->mxm_credits) == 1);
1173 msg->mxm_nob = msg_nob;
1174 __swab64s(&msg->mxm_srcnid);
1175 __swab64s(&msg->mxm_srcstamp);
1176 __swab64s(&msg->mxm_dstnid);
1177 __swab64s(&msg->mxm_dststamp);
1180 if (msg->mxm_srcnid == LNET_NID_ANY) {
1181 CNETERR("Bad src nid: %s\n", libcfs_nid2str(msg->mxm_srcnid));
1185 switch (msg->mxm_type) {
1187 CNETERR("Unknown message type %x\n", msg->mxm_type);
1190 case MXLND_MSG_NOOP:
1193 case MXLND_MSG_EAGER:
1194 if (msg_nob < offsetof(kmx_msg_t, mxm_u.eager.mxem_payload[0])) {
1195 CNETERR("Short EAGER: %d(%d)\n", msg_nob,
1196 (int)offsetof(kmx_msg_t, mxm_u.eager.mxem_payload[0]));
1201 case MXLND_MSG_PUT_REQ:
1202 if (msg_nob < hdr_size + sizeof(msg->mxm_u.put_req)) {
1203 CNETERR("Short PUT_REQ: %d(%d)\n", msg_nob,
1204 (int)(hdr_size + sizeof(msg->mxm_u.put_req)));
1208 __swab64s(&msg->mxm_u.put_req.mxprm_cookie);
1211 case MXLND_MSG_PUT_ACK:
1212 if (msg_nob < hdr_size + sizeof(msg->mxm_u.put_ack)) {
1213 CNETERR("Short PUT_ACK: %d(%d)\n", msg_nob,
1214 (int)(hdr_size + sizeof(msg->mxm_u.put_ack)));
1218 __swab64s(&msg->mxm_u.put_ack.mxpam_src_cookie);
1219 __swab64s(&msg->mxm_u.put_ack.mxpam_dst_cookie);
1223 case MXLND_MSG_GET_REQ:
1224 if (msg_nob < hdr_size + sizeof(msg->mxm_u.get_req)) {
1225 CNETERR("Short GET_REQ: %d(%d)\n", msg_nob,
1226 (int)(hdr_size + sizeof(msg->mxm_u.get_req)));
1230 __swab64s(&msg->mxm_u.get_req.mxgrm_cookie);
1234 case MXLND_MSG_CONN_REQ:
1235 case MXLND_MSG_CONN_ACK:
1236 if (msg_nob < hdr_size + sizeof(msg->mxm_u.conn_req)) {
1237 CNETERR("Short connreq/ack: %d(%d)\n", msg_nob,
1238 (int)(hdr_size + sizeof(msg->mxm_u.conn_req)));
1242 __swab32s(&msg->mxm_u.conn_req.mxcrm_queue_depth);
1243 __swab32s(&msg->mxm_u.conn_req.mxcrm_eager_size);
1253 * @lntmsg - the LNET msg that this is continuing. If EAGER, then NULL.
1257 * @length - length of incoming message
1258 * @pending - add to kmx_pending (0 is NO and 1 is YES)
1260 * The caller gets the rx and sets nid, peer and conn if known.
1262 * Returns 0 on success and -1 on failure
1265 mxlnd_recv_msg(lnet_msg_t *lntmsg, kmx_ctx_t *rx, u8 msg_type, u64 cookie, u32 length)
1268 mx_return_t mxret = MX_SUCCESS;
1269 uint64_t mask = ~(MXLND_ERROR_MASK);
1271 rx->mxc_msg_type = msg_type;
1272 rx->mxc_lntmsg[0] = lntmsg; /* may be NULL if EAGER */
1273 rx->mxc_cookie = cookie;
1274 /* rx->mxc_match may already be set */
1275 /* rx->mxc_seg.segment_ptr is already set */
1276 rx->mxc_seg.segment_length = length;
1277 ret = mxlnd_q_pending_ctx(rx);
1279 /* the caller is responsible for calling conn_decref() if needed */
1282 mxret = mx_kirecv(kmxlnd_data.kmx_endpt, &rx->mxc_seg, 1, MX_PIN_PHYSICAL,
1283 cookie, mask, (void *) rx, &rx->mxc_mxreq);
1284 if (mxret != MX_SUCCESS) {
1285 mxlnd_deq_pending_ctx(rx);
1286 CNETERR("mx_kirecv() failed with %s (%d)\n",
1287 mx_strerror(mxret), (int) mxret);
1295 * mxlnd_unexpected_recv - this is the callback function that will handle
1296 * unexpected receives
1297 * @context - NULL, ignore
1298 * @source - the peer's mx_endpoint_addr_t
1299 * @match_value - the msg's bits, should be MXLND_MSG_EAGER
1300 * @length - length of incoming message
1301 * @data_if_available - used for CONN_[REQ|ACK]
1303 * If it is an eager-sized msg, we will call recv_msg() with the actual
1304 * length. If it is a large message, we will call recv_msg() with a
1305 * length of 0 bytes to drop it because we should never have a large,
1306 * unexpected message.
1308 * NOTE - The MX library blocks until this function completes. Make it as fast as
1309 * possible. DO NOT allocate memory which can block!
1311 * If we cannot get a rx or the conn is closed, drop the message on the floor
1312 * (i.e. recv 0 bytes and ignore).
1314 mx_unexp_handler_action_t
1315 mxlnd_unexpected_recv(void *context, mx_endpoint_addr_t source,
1316 uint64_t match_value, uint32_t length, void *data_if_available)
1319 kmx_ctx_t *rx = NULL;
1324 kmx_conn_t *conn = NULL;
1325 kmx_peer_t *peer = NULL;
1330 /* TODO this will change to the net struct */
1331 if (context != NULL) {
1332 CNETERR("non-NULL context\n");
1336 CDEBUG(D_NET, "bits=0x%llx length=%d\n", match_value, length);
1339 mx_decompose_endpoint_addr2(source, &nic_id, &ep_id, &sid);
1340 mxlnd_parse_match(match_value, &msg_type, &error, &cookie);
1341 read_lock(&kmxlnd_data.kmx_global_lock);
1342 mx_get_endpoint_addr_context(source, (void **) &conn);
1344 mxlnd_conn_addref(conn); /* add ref for this function */
1345 peer = conn->mxk_peer;
1347 read_unlock(&kmxlnd_data.kmx_global_lock);
1349 if (msg_type == MXLND_MSG_BYE) {
1351 CDEBUG(D_NET, "peer %s sent BYE msg\n",
1352 libcfs_nid2str(peer->mxp_nid));
1353 mxlnd_conn_disconnect(conn, 1, 0);
1354 mxlnd_conn_decref(conn); /* drop ref taken above */
1356 return MX_RECV_FINISHED;
1359 if (msg_type == MXLND_MSG_CONN_REQ) {
1360 kmx_connparams_t *cp = NULL;
1361 const int expected = offsetof(kmx_msg_t, mxm_u) +
1362 sizeof(kmx_connreq_msg_t);
1364 if (conn) mxlnd_conn_decref(conn); /* drop ref taken above */
1365 if (unlikely(length != expected || !data_if_available)) {
1366 CNETERR("received invalid CONN_REQ from %llx "
1367 "length=%d (expected %d)\n", nic_id, length, expected);
1368 mxlnd_send_message(source, MXLND_MSG_CONN_ACK, EPROTO, 0);
1369 return MX_RECV_FINISHED;
1372 ret = mxlnd_connparams_alloc(&cp, context, source, match_value, length,
1373 conn, peer, data_if_available);
1374 if (unlikely(ret != 0)) {
1375 CNETERR("unable to alloc CONN_REQ from %llx:%d\n",
1377 mxlnd_send_message(source, MXLND_MSG_CONN_ACK, ENOMEM, 0);
1378 return MX_RECV_FINISHED;
1380 spin_lock(&kmxlnd_data.kmx_conn_lock);
1381 cfs_list_add_tail(&cp->mxr_list, &kmxlnd_data.kmx_conn_reqs);
1382 spin_unlock(&kmxlnd_data.kmx_conn_lock);
1383 up(&kmxlnd_data.kmx_conn_sem);
1384 return MX_RECV_FINISHED;
1386 if (msg_type == MXLND_MSG_CONN_ACK) {
1387 kmx_connparams_t *cp = NULL;
1388 const int expected = offsetof(kmx_msg_t, mxm_u) +
1389 sizeof(kmx_connreq_msg_t);
1392 if (unlikely(error != 0)) {
1393 CNETERR("received CONN_ACK from %s with error -%d\n",
1394 libcfs_nid2str(peer->mxp_nid), (int) error);
1395 mxlnd_conn_disconnect(conn, 1, 0);
1396 } else if (unlikely(length != expected || !data_if_available)) {
1397 CNETERR("received %s CONN_ACK from %s "
1398 "length=%d (expected %d)\n",
1399 data_if_available ? "short" : "missing",
1400 libcfs_nid2str(peer->mxp_nid), length, expected);
1401 mxlnd_conn_disconnect(conn, 1, 1);
1403 /* peer is ready for messages */
1404 ret = mxlnd_connparams_alloc(&cp, context, source, match_value, length,
1405 conn, peer, data_if_available);
1406 if (unlikely(ret != 0)) {
1407 CNETERR("unable to alloc kmx_connparams_t"
1408 " from %llx:%d\n", nic_id, ep_id);
1409 mxlnd_conn_disconnect(conn, 1, 1);
1411 spin_lock(&kmxlnd_data.kmx_conn_lock);
1412 cfs_list_add_tail(&cp->mxr_list,
1413 &kmxlnd_data.kmx_conn_reqs);
1414 spin_unlock(&kmxlnd_data.kmx_conn_lock);
1415 up(&kmxlnd_data.kmx_conn_sem);
1418 mxlnd_conn_decref(conn); /* drop ref taken above */
1420 return MX_RECV_FINISHED;
1423 /* Handle unexpected messages (PUT_REQ and GET_REQ) */
1425 LASSERT(peer != NULL && conn != NULL);
1427 rx = mxlnd_get_idle_rx(conn);
1429 if (length <= MXLND_MSG_SIZE) {
1430 ret = mxlnd_recv_msg(NULL, rx, msg_type, match_value, length);
1432 CNETERR("unexpected large receive with "
1433 "match_value=0x%llx length=%d\n",
1434 match_value, length);
1435 ret = mxlnd_recv_msg(NULL, rx, msg_type, match_value, 0);
1439 /* hold conn ref until rx completes */
1440 rx->mxc_conn = conn;
1441 rx->mxc_peer = peer;
1442 rx->mxc_nid = peer->mxp_nid;
1444 CNETERR("could not post receive\n");
1445 mxlnd_put_idle_rx(rx);
1449 /* Encountered error, drop incoming message on the floor */
1450 /* We could use MX_RECV_FINISHED but posting the receive of 0 bytes
1451 * uses the standard code path and acks the sender normally */
1453 if (rx == NULL || ret != 0) {
1454 mxlnd_conn_decref(conn); /* drop ref taken above */
1456 CNETERR("no idle rxs available - dropping rx"
1457 " 0x%llx from %s\n", match_value,
1458 libcfs_nid2str(peer->mxp_nid));
1461 CNETERR("disconnected peer - dropping rx\n");
1463 seg.segment_ptr = 0ULL;
1464 seg.segment_length = 0;
1465 mx_kirecv(kmxlnd_data.kmx_endpt, &seg, 1, MX_PIN_PHYSICAL,
1466 match_value, ~0ULL, NULL, NULL);
1469 return MX_RECV_CONTINUE;
1474 mxlnd_get_peer_info(int index, lnet_nid_t *nidp, int *count)
1478 kmx_peer_t *peer = NULL;
1480 read_lock(&kmxlnd_data.kmx_global_lock);
1481 for (i = 0; i < MXLND_HASH_SIZE; i++) {
1482 cfs_list_for_each_entry(peer, &kmxlnd_data.kmx_peers[i],
1485 *nidp = peer->mxp_nid;
1486 *count = cfs_atomic_read(&peer->mxp_refcount);
1492 read_unlock(&kmxlnd_data.kmx_global_lock);
1498 mxlnd_del_peer_locked(kmx_peer_t *peer)
1500 if (peer->mxp_conn) {
1501 mxlnd_conn_disconnect(peer->mxp_conn, 1, 1);
1503 cfs_list_del_init(&peer->mxp_list); /* remove from the global list */
1504 mxlnd_peer_decref(peer); /* drop global list ref */
1510 mxlnd_del_peer(lnet_nid_t nid)
1514 kmx_peer_t *peer = NULL;
1515 kmx_peer_t *next = NULL;
1517 if (nid != LNET_NID_ANY) {
1518 peer = mxlnd_find_peer_by_nid(nid, 0); /* adds peer ref */
1520 write_lock(&kmxlnd_data.kmx_global_lock);
1521 if (nid != LNET_NID_ANY) {
1525 mxlnd_peer_decref(peer); /* and drops it */
1526 mxlnd_del_peer_locked(peer);
1528 } else { /* LNET_NID_ANY */
1529 for (i = 0; i < MXLND_HASH_SIZE; i++) {
1530 cfs_list_for_each_entry_safe(peer, next,
1531 &kmxlnd_data.kmx_peers[i],
1533 mxlnd_del_peer_locked(peer);
1537 write_unlock(&kmxlnd_data.kmx_global_lock);
1543 mxlnd_get_conn_by_idx(int index)
1546 kmx_peer_t *peer = NULL;
1547 kmx_conn_t *conn = NULL;
1549 read_lock(&kmxlnd_data.kmx_global_lock);
1550 for (i = 0; i < MXLND_HASH_SIZE; i++) {
1551 cfs_list_for_each_entry(peer, &kmxlnd_data.kmx_peers[i],
1553 cfs_list_for_each_entry(conn, &peer->mxp_conns,
1559 mxlnd_conn_addref(conn); /* add ref here, dec in ctl() */
1560 read_unlock(&kmxlnd_data.kmx_global_lock);
1565 read_unlock(&kmxlnd_data.kmx_global_lock);
1571 mxlnd_close_matching_conns_locked(kmx_peer_t *peer)
1573 kmx_conn_t *conn = NULL;
1574 kmx_conn_t *next = NULL;
1576 cfs_list_for_each_entry_safe(conn, next, &peer->mxp_conns, mxk_list)
1577 mxlnd_conn_disconnect(conn, 0, 1);
1583 mxlnd_close_matching_conns(lnet_nid_t nid)
1587 kmx_peer_t *peer = NULL;
1589 write_lock(&kmxlnd_data.kmx_global_lock);
1590 if (nid != LNET_NID_ANY) {
1591 peer = mxlnd_find_peer_by_nid_locked(nid); /* adds peer ref */
1595 mxlnd_close_matching_conns_locked(peer);
1596 mxlnd_peer_decref(peer); /* and drops it here */
1598 } else { /* LNET_NID_ANY */
1599 for (i = 0; i < MXLND_HASH_SIZE; i++) {
1600 cfs_list_for_each_entry(peer, &kmxlnd_data.kmx_peers[i], mxp_list)
1601 mxlnd_close_matching_conns_locked(peer);
1604 write_unlock(&kmxlnd_data.kmx_global_lock);
1610 * mxlnd_ctl - modify MXLND parameters
1611 * @ni - LNET interface handle
1612 * @cmd - command to change
1613 * @arg - the ioctl data
1616 mxlnd_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg)
1618 struct libcfs_ioctl_data *data = arg;
1621 LASSERT (ni == kmxlnd_data.kmx_ni);
1624 case IOC_LIBCFS_GET_PEER: {
1628 ret = mxlnd_get_peer_info(data->ioc_count, &nid, &count);
1629 data->ioc_nid = nid;
1630 data->ioc_count = count;
1633 case IOC_LIBCFS_DEL_PEER: {
1634 ret = mxlnd_del_peer(data->ioc_nid);
1637 case IOC_LIBCFS_GET_CONN: {
1638 kmx_conn_t *conn = NULL;
1640 conn = mxlnd_get_conn_by_idx(data->ioc_count);
1645 data->ioc_nid = conn->mxk_peer->mxp_nid;
1646 mxlnd_conn_decref(conn); /* dec ref taken in get_conn_by_idx() */
1650 case IOC_LIBCFS_CLOSE_CONNECTION: {
1651 ret = mxlnd_close_matching_conns(data->ioc_nid);
1655 CNETERR("unknown ctl(%d)\n", cmd);
1663 * mxlnd_peer_queue_tx_locked - add the tx to the peer's tx queue
1666 * Add the tx to the peer's msg or data queue. The caller has locked the peer.
1669 mxlnd_peer_queue_tx_locked(kmx_ctx_t *tx)
1671 u8 msg_type = tx->mxc_msg_type;
1672 kmx_conn_t *conn = tx->mxc_conn;
1674 LASSERT (msg_type != 0);
1675 LASSERT (tx->mxc_nid != 0);
1676 LASSERT (tx->mxc_peer != NULL);
1677 LASSERT (tx->mxc_conn != NULL);
1679 tx->mxc_incarnation = conn->mxk_incarnation;
1681 if (msg_type != MXLND_MSG_PUT_DATA &&
1682 msg_type != MXLND_MSG_GET_DATA) {
1684 if (mxlnd_tx_requires_credit(tx)) {
1685 cfs_list_add_tail(&tx->mxc_list,
1686 &conn->mxk_tx_credit_queue);
1687 conn->mxk_ntx_msgs++;
1688 } else if (msg_type == MXLND_MSG_CONN_REQ ||
1689 msg_type == MXLND_MSG_CONN_ACK) {
1690 /* put conn msgs at the front of the queue */
1691 cfs_list_add(&tx->mxc_list, &conn->mxk_tx_free_queue);
1693 /* PUT_ACK, PUT_NAK */
1694 cfs_list_add_tail(&tx->mxc_list,
1695 &conn->mxk_tx_free_queue);
1696 conn->mxk_ntx_msgs++;
1700 cfs_list_add_tail(&tx->mxc_list, &conn->mxk_tx_free_queue);
1701 conn->mxk_ntx_data++;
1708 * mxlnd_peer_queue_tx - add the tx to the global tx queue
1711 * Add the tx to the peer's msg or data queue
1714 mxlnd_peer_queue_tx(kmx_ctx_t *tx)
1716 LASSERT(tx->mxc_peer != NULL);
1717 LASSERT(tx->mxc_conn != NULL);
1718 spin_lock(&tx->mxc_conn->mxk_lock);
1719 mxlnd_peer_queue_tx_locked(tx);
1720 spin_unlock(&tx->mxc_conn->mxk_lock);
1726 * mxlnd_queue_tx - add the tx to the global tx queue
1729 * Add the tx to the global queue and up the tx_queue_sem
1732 mxlnd_queue_tx(kmx_ctx_t *tx)
1734 kmx_peer_t *peer = tx->mxc_peer;
1735 LASSERT (tx->mxc_nid != 0);
1738 if (peer->mxp_incompatible &&
1739 tx->mxc_msg_type != MXLND_MSG_CONN_ACK) {
1740 /* let this fail now */
1741 tx->mxc_errno = -ECONNABORTED;
1742 mxlnd_conn_decref(peer->mxp_conn);
1743 mxlnd_put_idle_tx(tx);
1746 if (tx->mxc_conn == NULL) {
1748 kmx_conn_t *conn = NULL;
1750 ret = mxlnd_conn_alloc(&conn, peer); /* adds 2nd ref for tx... */
1752 tx->mxc_errno = ret;
1753 mxlnd_put_idle_tx(tx);
1756 tx->mxc_conn = conn;
1757 mxlnd_peer_decref(peer); /* and takes it from peer */
1759 LASSERT(tx->mxc_conn != NULL);
1760 mxlnd_peer_queue_tx(tx);
1761 mxlnd_check_sends(peer);
1763 spin_lock(&kmxlnd_data.kmx_tx_queue_lock);
1764 cfs_list_add_tail(&tx->mxc_list, &kmxlnd_data.kmx_tx_queue);
1765 spin_unlock(&kmxlnd_data.kmx_tx_queue_lock);
1766 up(&kmxlnd_data.kmx_tx_queue_sem);
1773 mxlnd_setup_iov(kmx_ctx_t *ctx, u32 niov, struct iovec *iov, u32 offset, u32 nob)
1780 int first_iov_offset = 0;
1781 int first_found = 0;
1783 int last_iov_length = 0;
1784 mx_ksegment_t *seg = NULL;
1786 if (niov == 0) return 0;
1787 LASSERT(iov != NULL);
1789 for (i = 0; i < niov; i++) {
1790 sum = old_sum + (u32) iov[i].iov_len;
1791 if (!first_found && (sum > offset)) {
1793 first_iov_offset = offset - old_sum;
1795 sum = (u32) iov[i].iov_len - first_iov_offset;
1800 last_iov_length = (u32) iov[i].iov_len - (sum - nob);
1801 if (first_iov == last_iov) last_iov_length -= first_iov_offset;
1806 LASSERT(first_iov >= 0 && last_iov >= first_iov);
1807 nseg = last_iov - first_iov + 1;
1810 MXLND_ALLOC(seg, nseg * sizeof(*seg));
1812 CNETERR("MXLND_ALLOC() failed\n");
1815 memset(seg, 0, nseg * sizeof(*seg));
1816 ctx->mxc_nseg = nseg;
1818 for (i = 0; i < nseg; i++) {
1819 seg[i].segment_ptr = MX_PA_TO_U64(virt_to_phys(iov[first_iov + i].iov_base));
1820 seg[i].segment_length = (u32) iov[first_iov + i].iov_len;
1822 seg[i].segment_ptr += (u64) first_iov_offset;
1823 seg[i].segment_length -= (u32) first_iov_offset;
1825 if (i == (nseg - 1)) {
1826 seg[i].segment_length = (u32) last_iov_length;
1828 sum += seg[i].segment_length;
1830 ctx->mxc_seg_list = seg;
1831 ctx->mxc_pin_type = MX_PIN_PHYSICAL;
1832 #ifdef MX_PIN_FULLPAGES
1833 ctx->mxc_pin_type |= MX_PIN_FULLPAGES;
1835 LASSERT(nob == sum);
1840 mxlnd_setup_kiov(kmx_ctx_t *ctx, u32 niov, lnet_kiov_t *kiov, u32 offset, u32 nob)
1846 int first_kiov = -1;
1847 int first_kiov_offset = 0;
1848 int first_found = 0;
1850 int last_kiov_length = 0;
1851 mx_ksegment_t *seg = NULL;
1853 if (niov == 0) return 0;
1854 LASSERT(kiov != NULL);
1856 for (i = 0; i < niov; i++) {
1857 sum = old_sum + kiov[i].kiov_len;
1858 if (i == 0) sum -= kiov[i].kiov_offset;
1859 if (!first_found && (sum > offset)) {
1861 first_kiov_offset = offset - old_sum;
1862 if (i == 0) first_kiov_offset = kiov[i].kiov_offset;
1864 sum = kiov[i].kiov_len - first_kiov_offset;
1869 last_kiov_length = kiov[i].kiov_len - (sum - nob);
1870 if (first_kiov == last_kiov) last_kiov_length -= first_kiov_offset;
1875 LASSERT(first_kiov >= 0 && last_kiov >= first_kiov);
1876 nseg = last_kiov - first_kiov + 1;
1879 MXLND_ALLOC(seg, nseg * sizeof(*seg));
1881 CNETERR("MXLND_ALLOC() failed\n");
1884 memset(seg, 0, niov * sizeof(*seg));
1885 ctx->mxc_nseg = niov;
1887 for (i = 0; i < niov; i++) {
1888 seg[i].segment_ptr = lnet_page2phys(kiov[first_kiov + i].kiov_page);
1889 seg[i].segment_length = kiov[first_kiov + i].kiov_len;
1891 seg[i].segment_ptr += (u64) first_kiov_offset;
1892 /* we have to add back the original kiov_offset */
1893 seg[i].segment_length -= first_kiov_offset +
1894 kiov[first_kiov].kiov_offset;
1896 if (i == (nseg - 1)) {
1897 seg[i].segment_length = last_kiov_length;
1899 sum += seg[i].segment_length;
1901 ctx->mxc_seg_list = seg;
1902 ctx->mxc_pin_type = MX_PIN_PHYSICAL;
1903 #ifdef MX_PIN_FULLPAGES
1904 ctx->mxc_pin_type |= MX_PIN_FULLPAGES;
1906 LASSERT(nob == sum);
1911 mxlnd_send_nak(kmx_ctx_t *tx, lnet_nid_t nid, int type, int status, __u64 cookie)
1913 LASSERT(type == MXLND_MSG_PUT_ACK);
1914 mxlnd_init_tx_msg(tx, type, sizeof(kmx_putack_msg_t), tx->mxc_nid);
1915 tx->mxc_cookie = cookie;
1916 tx->mxc_msg->mxm_u.put_ack.mxpam_src_cookie = cookie;
1917 tx->mxc_msg->mxm_u.put_ack.mxpam_dst_cookie = ((u64) status << MXLND_ERROR_OFFSET); /* error code */
1918 tx->mxc_match = mxlnd_create_match(tx, status);
1925 * mxlnd_send_data - get tx, map [k]iov, queue tx
1932 * This setups the DATA send for PUT or GET.
1934 * On success, it queues the tx, on failure it calls lnet_finalize()
1937 mxlnd_send_data(lnet_ni_t *ni, lnet_msg_t *lntmsg, kmx_peer_t *peer, u8 msg_type, u64 cookie)
1940 lnet_process_id_t target = lntmsg->msg_target;
1941 unsigned int niov = lntmsg->msg_niov;
1942 struct iovec *iov = lntmsg->msg_iov;
1943 lnet_kiov_t *kiov = lntmsg->msg_kiov;
1944 unsigned int offset = lntmsg->msg_offset;
1945 unsigned int nob = lntmsg->msg_len;
1946 kmx_ctx_t *tx = NULL;
1948 LASSERT(lntmsg != NULL);
1949 LASSERT(peer != NULL);
1950 LASSERT(msg_type == MXLND_MSG_PUT_DATA || msg_type == MXLND_MSG_GET_DATA);
1951 LASSERT((cookie>>MXLND_ERROR_OFFSET) == 0);
1953 tx = mxlnd_get_idle_tx();
1955 CNETERR("Can't allocate %s tx for %s\n",
1956 msg_type == MXLND_MSG_PUT_DATA ? "PUT_DATA" : "GET_DATA",
1957 libcfs_nid2str(target.nid));
1960 tx->mxc_nid = target.nid;
1961 /* NOTE called when we have a ref on the conn, get one for this tx */
1962 mxlnd_conn_addref(peer->mxp_conn);
1963 tx->mxc_peer = peer;
1964 tx->mxc_conn = peer->mxp_conn;
1965 tx->mxc_msg_type = msg_type;
1966 tx->mxc_lntmsg[0] = lntmsg;
1967 tx->mxc_cookie = cookie;
1968 tx->mxc_match = mxlnd_create_match(tx, 0);
1970 /* This setups up the mx_ksegment_t to send the DATA payload */
1972 /* do not setup the segments */
1973 CNETERR("nob = 0; why didn't we use an EAGER reply "
1974 "to %s?\n", libcfs_nid2str(target.nid));
1976 } else if (kiov == NULL) {
1977 ret = mxlnd_setup_iov(tx, niov, iov, offset, nob);
1979 ret = mxlnd_setup_kiov(tx, niov, kiov, offset, nob);
1982 CNETERR("Can't setup send DATA for %s\n",
1983 libcfs_nid2str(target.nid));
1984 tx->mxc_errno = -EIO;
1991 mxlnd_conn_decref(peer->mxp_conn);
1992 mxlnd_put_idle_tx(tx);
1996 CNETERR("no tx avail\n");
1997 lnet_finalize(ni, lntmsg, -EIO);
2002 * mxlnd_recv_data - map [k]iov, post rx
2009 * This setups the DATA receive for PUT or GET.
2011 * On success, it returns 0, on failure it returns -1
2014 mxlnd_recv_data(lnet_ni_t *ni, lnet_msg_t *lntmsg, kmx_ctx_t *rx, u8 msg_type, u64 cookie)
2017 lnet_process_id_t target = lntmsg->msg_target;
2018 unsigned int niov = lntmsg->msg_niov;
2019 struct iovec *iov = lntmsg->msg_iov;
2020 lnet_kiov_t *kiov = lntmsg->msg_kiov;
2021 unsigned int offset = lntmsg->msg_offset;
2022 unsigned int nob = lntmsg->msg_len;
2023 mx_return_t mxret = MX_SUCCESS;
2024 u64 mask = ~(MXLND_ERROR_MASK);
2026 /* above assumes MXLND_MSG_PUT_DATA */
2027 if (msg_type == MXLND_MSG_GET_DATA) {
2028 niov = lntmsg->msg_md->md_niov;
2029 iov = lntmsg->msg_md->md_iov.iov;
2030 kiov = lntmsg->msg_md->md_iov.kiov;
2032 nob = lntmsg->msg_md->md_length;
2035 LASSERT(lntmsg != NULL);
2036 LASSERT(rx != NULL);
2037 LASSERT(msg_type == MXLND_MSG_PUT_DATA || msg_type == MXLND_MSG_GET_DATA);
2038 LASSERT((cookie>>MXLND_ERROR_OFFSET) == 0); /* ensure top 12 bits are 0 */
2040 rx->mxc_msg_type = msg_type;
2041 rx->mxc_state = MXLND_CTX_PENDING;
2042 rx->mxc_nid = target.nid;
2043 /* if posting a GET_DATA, we may not yet know the peer */
2044 if (rx->mxc_peer != NULL) {
2045 rx->mxc_conn = rx->mxc_peer->mxp_conn;
2047 rx->mxc_lntmsg[0] = lntmsg;
2048 rx->mxc_cookie = cookie;
2049 rx->mxc_match = mxlnd_create_match(rx, 0);
2050 /* This setups up the mx_ksegment_t to receive the DATA payload */
2052 ret = mxlnd_setup_iov(rx, niov, iov, offset, nob);
2054 ret = mxlnd_setup_kiov(rx, niov, kiov, offset, nob);
2056 if (msg_type == MXLND_MSG_GET_DATA) {
2057 rx->mxc_lntmsg[1] = lnet_create_reply_msg(kmxlnd_data.kmx_ni, lntmsg);
2058 if (rx->mxc_lntmsg[1] == NULL) {
2059 CNETERR("Can't create reply for GET -> %s\n",
2060 libcfs_nid2str(target.nid));
2065 CNETERR("Can't setup %s rx for %s\n",
2066 msg_type == MXLND_MSG_PUT_DATA ? "PUT_DATA" : "GET_DATA",
2067 libcfs_nid2str(target.nid));
2070 ret = mxlnd_q_pending_ctx(rx);
2074 CDEBUG(D_NET, "receiving %s 0x%llx\n", mxlnd_msgtype_to_str(msg_type), rx->mxc_cookie);
2075 mxret = mx_kirecv(kmxlnd_data.kmx_endpt,
2076 rx->mxc_seg_list, rx->mxc_nseg,
2077 rx->mxc_pin_type, rx->mxc_match,
2080 if (mxret != MX_SUCCESS) {
2081 if (rx->mxc_conn != NULL) {
2082 mxlnd_deq_pending_ctx(rx);
2084 CNETERR("mx_kirecv() failed with %d for %s\n",
2085 (int) mxret, libcfs_nid2str(target.nid));
2093 * mxlnd_send - the LND required send function
2098 * This must not block. Since we may not have a peer struct for the receiver,
2099 * it will append send messages on a global tx list. We will then up the
2100 * tx_queued's semaphore to notify it of the new send.
2103 mxlnd_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg)
2106 int type = lntmsg->msg_type;
2107 lnet_hdr_t *hdr = &lntmsg->msg_hdr;
2108 lnet_process_id_t target = lntmsg->msg_target;
2109 lnet_nid_t nid = target.nid;
2110 int target_is_router = lntmsg->msg_target_is_router;
2111 int routing = lntmsg->msg_routing;
2112 unsigned int payload_niov = lntmsg->msg_niov;
2113 struct iovec *payload_iov = lntmsg->msg_iov;
2114 lnet_kiov_t *payload_kiov = lntmsg->msg_kiov;
2115 unsigned int payload_offset = lntmsg->msg_offset;
2116 unsigned int payload_nob = lntmsg->msg_len;
2117 kmx_ctx_t *tx = NULL;
2118 kmx_msg_t *txmsg = NULL;
2119 kmx_ctx_t *rx = (kmx_ctx_t *) private; /* for REPLY */
2120 kmx_ctx_t *rx_data = NULL;
2121 kmx_conn_t *conn = NULL;
2123 uint32_t length = 0;
2124 kmx_peer_t *peer = NULL;
2125 rwlock_t *g_lock =&kmxlnd_data.kmx_global_lock;
2127 CDEBUG(D_NET, "sending %d bytes in %d frags to %s\n",
2128 payload_nob, payload_niov, libcfs_id2str(target));
2130 LASSERT (payload_nob == 0 || payload_niov > 0);
2131 LASSERT (payload_niov <= LNET_MAX_IOV);
2132 /* payload is either all vaddrs or all pages */
2133 LASSERT (!(payload_kiov != NULL && payload_iov != NULL));
2135 /* private is used on LNET_GET_REPLY only, NULL for all other cases */
2137 /* NOTE we may not know the peer if it is the very first PUT_REQ or GET_REQ
2138 * to a new peer, so create one if not found */
2139 peer = mxlnd_find_peer_by_nid(nid, 1); /* adds peer ref */
2140 if (peer == NULL || peer->mxp_conn == NULL) {
2141 /* we could not find it nor could we create one or
2142 * one exists but we cannot create a conn,
2143 * fail this message */
2145 /* found peer without conn, drop ref taken above */
2146 LASSERT(peer->mxp_conn == NULL);
2147 mxlnd_peer_decref(peer);
2152 /* we have a peer with a conn */
2154 if (unlikely(peer->mxp_incompatible)) {
2155 mxlnd_peer_decref(peer); /* drop ref taken above */
2158 conn = peer->mxp_conn;
2159 if (conn && conn->mxk_status != MXLND_CONN_DISCONNECT)
2160 mxlnd_conn_addref(conn);
2163 read_unlock(g_lock);
2164 mxlnd_peer_decref(peer); /* drop peer ref taken above */
2169 LASSERT(peer && conn);
2171 CDEBUG(D_NET, "%s: peer 0x%llx is 0x%p\n", __func__, nid, peer);
2175 LASSERT (payload_nob == 0);
2178 case LNET_MSG_REPLY:
2180 /* Is the payload small enough not to need DATA? */
2181 nob = offsetof(kmx_msg_t, mxm_u.eager.mxem_payload[payload_nob]);
2182 if (nob <= MXLND_MSG_SIZE)
2183 break; /* send EAGER */
2185 tx = mxlnd_get_idle_tx();
2186 if (unlikely(tx == NULL)) {
2187 CNETERR("Can't allocate %s tx for %s\n",
2188 type == LNET_MSG_PUT ? "PUT" : "REPLY",
2189 libcfs_nid2str(nid));
2190 if (conn) mxlnd_conn_decref(conn);
2194 tx->mxc_peer = peer;
2195 tx->mxc_conn = conn;
2196 /* we added a conn ref above */
2197 mxlnd_init_tx_msg (tx, MXLND_MSG_PUT_REQ, sizeof(kmx_putreq_msg_t), nid);
2198 txmsg = tx->mxc_msg;
2199 txmsg->mxm_u.put_req.mxprm_hdr = *hdr;
2200 txmsg->mxm_u.put_req.mxprm_cookie = tx->mxc_cookie;
2201 tx->mxc_match = mxlnd_create_match(tx, 0);
2203 /* we must post a receive _before_ sending the request.
2204 * we need to determine how much to receive, it will be either
2205 * a put_ack or a put_nak. The put_ack is larger, so use it. */
2207 rx = mxlnd_get_idle_rx(conn);
2208 if (unlikely(rx == NULL)) {
2209 CNETERR("Can't allocate rx for PUT_ACK for %s\n",
2210 libcfs_nid2str(nid));
2211 mxlnd_put_idle_tx(tx);
2212 if (conn) mxlnd_conn_decref(conn); /* for the ref taken above */
2216 rx->mxc_peer = peer;
2217 mxlnd_conn_addref(conn); /* for this rx */
2218 rx->mxc_conn = conn;
2219 rx->mxc_msg_type = MXLND_MSG_PUT_ACK;
2220 rx->mxc_cookie = tx->mxc_cookie;
2221 rx->mxc_match = mxlnd_create_match(rx, 0);
2223 length = offsetof(kmx_msg_t, mxm_u) + sizeof(kmx_putack_msg_t);
2224 ret = mxlnd_recv_msg(lntmsg, rx, MXLND_MSG_PUT_ACK, rx->mxc_match, length);
2225 if (unlikely(ret != 0)) {
2226 CNETERR("recv_msg() failed for PUT_ACK for %s\n",
2227 libcfs_nid2str(nid));
2228 rx->mxc_lntmsg[0] = NULL;
2229 mxlnd_put_idle_rx(rx);
2230 mxlnd_put_idle_tx(tx);
2231 mxlnd_conn_decref(conn); /* for the rx... */
2232 mxlnd_conn_decref(conn); /* and for the tx */
2233 return -EHOSTUNREACH;
2240 if (routing || target_is_router)
2241 break; /* send EAGER */
2243 /* is the REPLY message too small for DATA? */
2244 nob = offsetof(kmx_msg_t, mxm_u.eager.mxem_payload[lntmsg->msg_md->md_length]);
2245 if (nob <= MXLND_MSG_SIZE)
2246 break; /* send EAGER */
2248 /* get tx (we need the cookie) , post rx for incoming DATA,
2249 * then post GET_REQ tx */
2250 tx = mxlnd_get_idle_tx();
2251 if (unlikely(tx == NULL)) {
2252 CNETERR("Can't allocate GET tx for %s\n",
2253 libcfs_nid2str(nid));
2254 mxlnd_conn_decref(conn); /* for the ref taken above */
2257 rx_data = mxlnd_get_idle_rx(conn);
2258 if (unlikely(rx_data == NULL)) {
2259 CNETERR("Can't allocate DATA rx for %s\n",
2260 libcfs_nid2str(nid));
2261 mxlnd_put_idle_tx(tx);
2262 mxlnd_conn_decref(conn); /* for the ref taken above */
2265 rx_data->mxc_peer = peer;
2266 /* NOTE no need to lock peer before adding conn ref since we took
2267 * a conn ref for the tx (it cannot be freed between there and here ) */
2268 mxlnd_conn_addref(conn); /* for the rx_data */
2269 rx_data->mxc_conn = conn;
2271 ret = mxlnd_recv_data(ni, lntmsg, rx_data, MXLND_MSG_GET_DATA, tx->mxc_cookie);
2272 if (unlikely(ret != 0)) {
2273 CNETERR("Can't setup GET sink for %s\n",
2274 libcfs_nid2str(nid));
2275 mxlnd_put_idle_rx(rx_data);
2276 mxlnd_put_idle_tx(tx);
2277 mxlnd_conn_decref(conn); /* for the rx_data... */
2278 mxlnd_conn_decref(conn); /* and for the tx */
2282 tx->mxc_peer = peer;
2283 tx->mxc_conn = conn;
2284 /* conn ref taken above */
2285 mxlnd_init_tx_msg(tx, MXLND_MSG_GET_REQ, sizeof(kmx_getreq_msg_t), nid);
2286 txmsg = tx->mxc_msg;
2287 txmsg->mxm_u.get_req.mxgrm_hdr = *hdr;
2288 txmsg->mxm_u.get_req.mxgrm_cookie = tx->mxc_cookie;
2289 tx->mxc_match = mxlnd_create_match(tx, 0);
2296 mxlnd_conn_decref(conn); /* drop ref taken above */
2302 LASSERT (offsetof(kmx_msg_t, mxm_u.eager.mxem_payload[payload_nob])
2305 tx = mxlnd_get_idle_tx();
2306 if (unlikely(tx == NULL)) {
2307 CNETERR("Can't send %s to %s: tx descs exhausted\n",
2308 mxlnd_lnetmsg_to_str(type), libcfs_nid2str(nid));
2309 mxlnd_conn_decref(conn); /* drop ref taken above */
2313 tx->mxc_peer = peer;
2314 tx->mxc_conn = conn;
2315 /* conn ref taken above */
2316 nob = offsetof(kmx_eager_msg_t, mxem_payload[payload_nob]);
2317 mxlnd_init_tx_msg (tx, MXLND_MSG_EAGER, nob, nid);
2318 tx->mxc_match = mxlnd_create_match(tx, 0);
2320 txmsg = tx->mxc_msg;
2321 txmsg->mxm_u.eager.mxem_hdr = *hdr;
2323 if (payload_kiov != NULL)
2324 lnet_copy_kiov2flat(MXLND_MSG_SIZE, txmsg,
2325 offsetof(kmx_msg_t, mxm_u.eager.mxem_payload),
2326 payload_niov, payload_kiov, payload_offset, payload_nob);
2328 lnet_copy_iov2flat(MXLND_MSG_SIZE, txmsg,
2329 offsetof(kmx_msg_t, mxm_u.eager.mxem_payload),
2330 payload_niov, payload_iov, payload_offset, payload_nob);
2332 tx->mxc_lntmsg[0] = lntmsg; /* finalise lntmsg on completion */
2338 * mxlnd_recv - the LND required recv function
2349 * This must not block.
2352 mxlnd_recv (lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, int delayed,
2353 unsigned int niov, struct iovec *iov, lnet_kiov_t *kiov,
2354 unsigned int offset, unsigned int mlen, unsigned int rlen)
2359 kmx_ctx_t *rx = private;
2360 kmx_msg_t *rxmsg = rx->mxc_msg;
2361 lnet_nid_t nid = rx->mxc_nid;
2362 kmx_ctx_t *tx = NULL;
2363 kmx_msg_t *txmsg = NULL;
2364 kmx_peer_t *peer = rx->mxc_peer;
2365 kmx_conn_t *conn = peer->mxp_conn;
2367 int msg_type = rxmsg->mxm_type;
2372 LASSERT (mlen <= rlen);
2373 /* Either all pages or all vaddrs */
2374 LASSERT (!(kiov != NULL && iov != NULL));
2375 LASSERT (peer && conn);
2377 /* conn_addref(conn) already taken for the primary rx */
2380 case MXLND_MSG_EAGER:
2381 nob = offsetof(kmx_msg_t, mxm_u.eager.mxem_payload[rlen]);
2382 len = rx->mxc_status.xfer_length;
2383 if (unlikely(nob > len)) {
2384 CNETERR("Eager message from %s too big: %d(%d)\n",
2385 libcfs_nid2str(nid), nob, len);
2391 lnet_copy_flat2kiov(niov, kiov, offset,
2392 MXLND_MSG_SIZE, rxmsg,
2393 offsetof(kmx_msg_t, mxm_u.eager.mxem_payload),
2396 lnet_copy_flat2iov(niov, iov, offset,
2397 MXLND_MSG_SIZE, rxmsg,
2398 offsetof(kmx_msg_t, mxm_u.eager.mxem_payload),
2404 case MXLND_MSG_PUT_REQ:
2405 /* we are going to reuse the rx, store the needed info */
2406 cookie = rxmsg->mxm_u.put_req.mxprm_cookie;
2408 /* get tx, post rx, send PUT_ACK */
2410 tx = mxlnd_get_idle_tx();
2411 if (unlikely(tx == NULL)) {
2412 CNETERR("Can't allocate tx for %s\n", libcfs_nid2str(nid));
2413 /* Not replying will break the connection */
2417 if (unlikely(mlen == 0)) {
2419 tx->mxc_peer = peer;
2420 tx->mxc_conn = conn;
2421 mxlnd_send_nak(tx, nid, MXLND_MSG_PUT_ACK, 0, cookie);
2426 mxlnd_init_tx_msg(tx, MXLND_MSG_PUT_ACK, sizeof(kmx_putack_msg_t), nid);
2427 tx->mxc_peer = peer;
2428 tx->mxc_conn = conn;
2429 /* no need to lock peer first since we already have a ref */
2430 mxlnd_conn_addref(conn); /* for the tx */
2431 txmsg = tx->mxc_msg;
2432 txmsg->mxm_u.put_ack.mxpam_src_cookie = cookie;
2433 txmsg->mxm_u.put_ack.mxpam_dst_cookie = tx->mxc_cookie;
2434 tx->mxc_cookie = cookie;
2435 tx->mxc_match = mxlnd_create_match(tx, 0);
2437 /* we must post a receive _before_ sending the PUT_ACK */
2439 rx->mxc_state = MXLND_CTX_PREP;
2440 rx->mxc_peer = peer;
2441 rx->mxc_conn = conn;
2442 /* do not take another ref for this rx, it is already taken */
2443 rx->mxc_nid = peer->mxp_nid;
2444 ret = mxlnd_recv_data(ni, lntmsg, rx, MXLND_MSG_PUT_DATA,
2445 txmsg->mxm_u.put_ack.mxpam_dst_cookie);
2447 if (unlikely(ret != 0)) {
2448 /* Notify peer that it's over */
2449 CNETERR("Can't setup PUT_DATA rx for %s: %d\n",
2450 libcfs_nid2str(nid), ret);
2452 tx->mxc_state = MXLND_CTX_PREP;
2453 tx->mxc_peer = peer;
2454 tx->mxc_conn = conn;
2455 /* finalize = 0, let the PUT_ACK tx finalize this */
2456 tx->mxc_lntmsg[0] = rx->mxc_lntmsg[0];
2457 tx->mxc_lntmsg[1] = rx->mxc_lntmsg[1];
2458 /* conn ref already taken above */
2459 mxlnd_send_nak(tx, nid, MXLND_MSG_PUT_ACK, ret, cookie);
2465 /* do not return a credit until after PUT_DATA returns */
2469 case MXLND_MSG_GET_REQ:
2470 cookie = rxmsg->mxm_u.get_req.mxgrm_cookie;
2472 if (likely(lntmsg != NULL)) {
2473 mxlnd_send_data(ni, lntmsg, rx->mxc_peer, MXLND_MSG_GET_DATA,
2476 /* GET didn't match anything */
2477 /* The initiator has a rx mapped to [k]iov. We cannot send a nak.
2478 * We have to embed the error code in the match bits.
2479 * Send the error in bits 52-59 and the cookie in bits 0-51 */
2480 tx = mxlnd_get_idle_tx();
2481 if (unlikely(tx == NULL)) {
2482 CNETERR("Can't get tx for GET NAK for %s\n",
2483 libcfs_nid2str(nid));
2484 /* we can't get a tx, notify the peer that the GET failed */
2485 mxlnd_send_message(conn->mxk_epa, MXLND_MSG_GET_DATA,
2490 tx->mxc_msg_type = MXLND_MSG_GET_DATA;
2491 tx->mxc_state = MXLND_CTX_PENDING;
2493 tx->mxc_peer = peer;
2494 tx->mxc_conn = conn;
2495 /* no need to lock peer first since we already have a ref */
2496 mxlnd_conn_addref(conn); /* for this tx */
2497 tx->mxc_cookie = cookie;
2498 tx->mxc_match = mxlnd_create_match(tx, ENODATA);
2499 tx->mxc_pin_type = MX_PIN_PHYSICAL;
2502 /* finalize lntmsg after tx completes */
2510 /* we received a message, increment peer's outstanding credits */
2512 spin_lock(&conn->mxk_lock);
2513 conn->mxk_outstanding++;
2514 spin_unlock(&conn->mxk_lock);
2516 /* we are done with the rx */
2517 mxlnd_put_idle_rx(rx);
2518 mxlnd_conn_decref(conn);
2521 if (finalize == 1) lnet_finalize(kmxlnd_data.kmx_ni, lntmsg, 0);
2523 /* we received a credit, see if we can use it to send a msg */
2524 if (credit) mxlnd_check_sends(peer);
2530 mxlnd_sleep(unsigned long timeout)
2532 cfs_set_current_state(CFS_TASK_INTERRUPTIBLE);
2533 cfs_schedule_timeout(timeout);
2538 * mxlnd_tx_queued - the generic send queue thread
2539 * @arg - thread id (as a void *)
2541 * This thread moves send messages from the global tx_queue to the owning
2542 * peer's tx_[msg|data]_queue. If the peer does not exist, it creates one and adds
2543 * it to the global peer list.
2546 mxlnd_tx_queued(void *arg)
2548 long id = (long) arg;
2551 kmx_ctx_t *tx = NULL;
2552 kmx_peer_t *peer = NULL;
2553 cfs_list_t *queue = &kmxlnd_data.kmx_tx_queue;
2554 spinlock_t *tx_q_lock = &kmxlnd_data.kmx_tx_queue_lock;
2555 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
2557 cfs_daemonize("mxlnd_tx_queued");
2559 while (!(cfs_atomic_read(&kmxlnd_data.kmx_shutdown))) {
2560 ret = down_interruptible(&kmxlnd_data.kmx_tx_queue_sem);
2561 if (cfs_atomic_read(&kmxlnd_data.kmx_shutdown))
2563 if (ret != 0) /* Should we check for -EINTR? */
2565 spin_lock(tx_q_lock);
2566 if (cfs_list_empty(&kmxlnd_data.kmx_tx_queue)) {
2567 spin_unlock(tx_q_lock);
2570 tx = cfs_list_entry(queue->next, kmx_ctx_t, mxc_list);
2571 cfs_list_del_init(&tx->mxc_list);
2572 spin_unlock(tx_q_lock);
2575 peer = mxlnd_find_peer_by_nid(tx->mxc_nid, 0); /* adds ref*/
2577 tx->mxc_peer = peer;
2579 if (peer->mxp_conn == NULL) {
2580 ret = mxlnd_conn_alloc_locked(&peer->mxp_conn,
2583 /* out of memory: give up, fail tx */
2584 tx->mxc_errno = -ENOMEM;
2585 mxlnd_peer_decref(peer);
2586 write_unlock(g_lock);
2587 mxlnd_put_idle_tx(tx);
2591 tx->mxc_conn = peer->mxp_conn;
2592 mxlnd_conn_addref(tx->mxc_conn); /* for this tx */
2593 mxlnd_peer_decref(peer); /* drop peer ref taken above */
2594 write_unlock(g_lock);
2600 kmx_peer_t *peer = NULL;
2601 kmx_peer_t *old = NULL;
2603 hash = mxlnd_nid_to_hash(tx->mxc_nid);
2605 LASSERT(tx->mxc_msg_type != MXLND_MSG_PUT_DATA &&
2606 tx->mxc_msg_type != MXLND_MSG_GET_DATA);
2608 /* adds conn ref for this function */
2609 ret = mxlnd_peer_alloc(&peer, tx->mxc_nid,
2610 *kmxlnd_tunables.kmx_board,
2611 *kmxlnd_tunables.kmx_ep_id, 0ULL);
2613 /* finalize message */
2614 tx->mxc_errno = ret;
2615 mxlnd_put_idle_tx(tx);
2618 tx->mxc_peer = peer;
2619 tx->mxc_conn = peer->mxp_conn;
2620 /* this tx will keep the conn ref taken in peer_alloc() */
2622 /* add peer to global peer list, but look to see
2623 * if someone already created it after we released
2626 old = mxlnd_find_peer_by_nid_locked(peer->mxp_nid);
2628 /* we have a peer ref on old */
2629 if (old->mxp_conn) {
2633 /* drop our ref taken above... */
2634 mxlnd_peer_decref(old);
2636 mxlnd_del_peer_locked(old);
2641 cfs_list_add_tail(&peer->mxp_list,
2642 &kmxlnd_data.kmx_peers[hash]);
2643 cfs_atomic_inc(&kmxlnd_data.kmx_npeers);
2646 tx->mxc_conn = old->mxp_conn;
2647 LASSERT(old->mxp_conn != NULL);
2648 mxlnd_conn_addref(old->mxp_conn);
2649 mxlnd_conn_decref(peer->mxp_conn); /* drop ref taken above.. */
2650 mxlnd_conn_decref(peer->mxp_conn); /* drop peer's ref */
2651 mxlnd_peer_decref(peer);
2653 write_unlock(g_lock);
2658 mxlnd_thread_stop(id);
2662 /* When calling this, we must not have the peer lock. */
2664 mxlnd_iconnect(kmx_peer_t *peer, u8 msg_type)
2666 mx_return_t mxret = MX_SUCCESS;
2667 mx_request_t request;
2668 kmx_conn_t *conn = peer->mxp_conn;
2669 u64 match = ((u64) msg_type) << MXLND_MSG_OFFSET;
2671 /* NOTE we are holding a conn ref every time we call this function,
2672 * we do not need to lock the peer before taking another ref */
2673 mxlnd_conn_addref(conn); /* hold until CONN_REQ or CONN_ACK completes */
2675 LASSERT(msg_type == MXLND_MSG_ICON_REQ || msg_type == MXLND_MSG_ICON_ACK);
2677 if (peer->mxp_reconnect_time == 0) {
2678 peer->mxp_reconnect_time = jiffies;
2681 if (peer->mxp_nic_id == 0ULL) {
2684 ret = mxlnd_ip2nic_id(LNET_NIDADDR(peer->mxp_nid),
2685 &peer->mxp_nic_id, MXLND_LOOKUP_COUNT);
2687 mx_nic_id_to_board_number(peer->mxp_nic_id, &peer->mxp_board);
2689 if (peer->mxp_nic_id == 0ULL && conn->mxk_status == MXLND_CONN_WAIT) {
2690 /* not mapped yet, return */
2691 spin_lock(&conn->mxk_lock);
2692 mxlnd_set_conn_status(conn, MXLND_CONN_INIT);
2693 spin_unlock(&conn->mxk_lock);
2697 if (cfs_time_after(jiffies,
2698 peer->mxp_reconnect_time + MXLND_CONNECT_TIMEOUT) &&
2699 conn->mxk_status != MXLND_CONN_DISCONNECT) {
2700 /* give up and notify LNET */
2701 CDEBUG(D_NET, "timeout trying to connect to %s\n",
2702 libcfs_nid2str(peer->mxp_nid));
2703 mxlnd_conn_disconnect(conn, 0, 0);
2704 mxlnd_conn_decref(conn);
2708 mxret = mx_iconnect(kmxlnd_data.kmx_endpt, peer->mxp_nic_id,
2709 peer->mxp_ep_id, MXLND_MSG_MAGIC, match,
2710 (void *) peer, &request);
2711 if (unlikely(mxret != MX_SUCCESS)) {
2712 spin_lock(&conn->mxk_lock);
2713 mxlnd_set_conn_status(conn, MXLND_CONN_FAIL);
2714 spin_unlock(&conn->mxk_lock);
2715 CNETERR("mx_iconnect() failed with %s (%d) to %s\n",
2716 mx_strerror(mxret), mxret, libcfs_nid2str(peer->mxp_nid));
2717 mxlnd_conn_decref(conn);
2719 mx_set_request_timeout(kmxlnd_data.kmx_endpt, request,
2720 MXLND_CONNECT_TIMEOUT/CFS_HZ*1000);
2724 #define MXLND_STATS 0
2727 mxlnd_check_sends(kmx_peer_t *peer)
2731 mx_return_t mxret = MX_SUCCESS;
2732 kmx_ctx_t *tx = NULL;
2733 kmx_conn_t *conn = NULL;
2740 static unsigned long last = 0;
2743 if (unlikely(peer == NULL)) {
2744 LASSERT(peer != NULL);
2747 write_lock(&kmxlnd_data.kmx_global_lock);
2748 conn = peer->mxp_conn;
2749 /* NOTE take a ref for the duration of this function since it is
2750 * called when there might not be any queued txs for this peer */
2752 if (conn->mxk_status == MXLND_CONN_DISCONNECT) {
2753 write_unlock(&kmxlnd_data.kmx_global_lock);
2756 mxlnd_conn_addref(conn); /* for duration of this function */
2758 write_unlock(&kmxlnd_data.kmx_global_lock);
2760 /* do not add another ref for this tx */
2763 /* we do not have any conns */
2764 CNETERR("peer %s has no conn\n", libcfs_nid2str(peer->mxp_nid));
2769 if (cfs_time_after(jiffies, last)) {
2770 last = jiffies + CFS_HZ;
2771 CDEBUG(D_NET, "status= %s credits= %d outstanding= %d ntx_msgs= %d "
2772 "ntx_posted= %d ntx_data= %d data_posted= %d\n",
2773 mxlnd_connstatus_to_str(conn->mxk_status), conn->mxk_credits,
2774 conn->mxk_outstanding, conn->mxk_ntx_msgs, conn->mxk_ntx_posted,
2775 conn->mxk_ntx_data, conn->mxk_data_posted);
2779 spin_lock(&conn->mxk_lock);
2780 ntx_posted = conn->mxk_ntx_posted;
2781 credits = conn->mxk_credits;
2783 LASSERT(ntx_posted <= *kmxlnd_tunables.kmx_peercredits);
2784 LASSERT(ntx_posted >= 0);
2786 LASSERT(credits <= *kmxlnd_tunables.kmx_peercredits);
2787 LASSERT(credits >= 0);
2789 /* check number of queued msgs, ignore data */
2790 if (conn->mxk_outstanding >= MXLND_CREDIT_HIGHWATER()) {
2791 /* check if any txs queued that could return credits... */
2792 if (cfs_list_empty(&conn->mxk_tx_credit_queue) ||
2793 conn->mxk_ntx_msgs == 0) {
2794 /* if not, send a NOOP */
2795 tx = mxlnd_get_idle_tx();
2796 if (likely(tx != NULL)) {
2797 tx->mxc_peer = peer;
2798 tx->mxc_conn = peer->mxp_conn;
2799 mxlnd_conn_addref(conn); /* for this tx */
2800 mxlnd_init_tx_msg (tx, MXLND_MSG_NOOP, 0, peer->mxp_nid);
2801 tx->mxc_match = mxlnd_create_match(tx, 0);
2802 mxlnd_peer_queue_tx_locked(tx);
2809 /* if the peer is not ready, try to connect */
2810 if (unlikely(conn->mxk_status == MXLND_CONN_INIT ||
2811 conn->mxk_status == MXLND_CONN_FAIL)) {
2812 CDEBUG(D_NET, "status=%s\n", mxlnd_connstatus_to_str(conn->mxk_status));
2813 mxlnd_set_conn_status(conn, MXLND_CONN_WAIT);
2814 spin_unlock(&conn->mxk_lock);
2815 mxlnd_iconnect(peer, (u8) MXLND_MSG_ICON_REQ);
2819 while (!cfs_list_empty(&conn->mxk_tx_free_queue) ||
2820 !cfs_list_empty(&conn->mxk_tx_credit_queue)) {
2821 /* We have something to send. If we have a queued tx that does not
2822 * require a credit (free), choose it since its completion will
2823 * return a credit (here or at the peer), complete a DATA or
2824 * CONN_REQ or CONN_ACK. */
2825 cfs_list_t *tmp_tx = NULL;
2826 if (!cfs_list_empty(&conn->mxk_tx_free_queue)) {
2827 tmp_tx = &conn->mxk_tx_free_queue;
2829 tmp_tx = &conn->mxk_tx_credit_queue;
2831 tx = cfs_list_entry(tmp_tx->next, kmx_ctx_t, mxc_list);
2833 msg_type = tx->mxc_msg_type;
2835 /* don't try to send a rx */
2836 LASSERT(tx->mxc_type == MXLND_REQ_TX);
2838 /* ensure that it is a valid msg type */
2839 LASSERT(msg_type == MXLND_MSG_CONN_REQ ||
2840 msg_type == MXLND_MSG_CONN_ACK ||
2841 msg_type == MXLND_MSG_NOOP ||
2842 msg_type == MXLND_MSG_EAGER ||
2843 msg_type == MXLND_MSG_PUT_REQ ||
2844 msg_type == MXLND_MSG_PUT_ACK ||
2845 msg_type == MXLND_MSG_PUT_DATA ||
2846 msg_type == MXLND_MSG_GET_REQ ||
2847 msg_type == MXLND_MSG_GET_DATA);
2848 LASSERT(tx->mxc_peer == peer);
2849 LASSERT(tx->mxc_nid == peer->mxp_nid);
2851 credit = mxlnd_tx_requires_credit(tx);
2854 if (conn->mxk_ntx_posted == *kmxlnd_tunables.kmx_peercredits) {
2855 CDEBUG(D_NET, "%s: posted enough\n",
2856 libcfs_nid2str(peer->mxp_nid));
2860 if (conn->mxk_credits == 0) {
2861 CDEBUG(D_NET, "%s: no credits\n",
2862 libcfs_nid2str(peer->mxp_nid));
2866 if (conn->mxk_credits == 1 && /* last credit reserved for */
2867 conn->mxk_outstanding == 0) { /* giving back credits */
2868 CDEBUG(D_NET, "%s: not using last credit\n",
2869 libcfs_nid2str(peer->mxp_nid));
2874 if (unlikely(conn->mxk_status != MXLND_CONN_READY)) {
2875 if ( ! (msg_type == MXLND_MSG_CONN_REQ ||
2876 msg_type == MXLND_MSG_CONN_ACK)) {
2877 CDEBUG(D_NET, "peer status is %s for tx 0x%llx (%s)\n",
2878 mxlnd_connstatus_to_str(conn->mxk_status),
2880 mxlnd_msgtype_to_str(tx->mxc_msg_type));
2881 if (conn->mxk_status == MXLND_CONN_DISCONNECT ||
2882 cfs_time_aftereq(jiffies, tx->mxc_deadline)) {
2883 cfs_list_del_init(&tx->mxc_list);
2884 tx->mxc_errno = -ECONNABORTED;
2885 spin_unlock(&conn->mxk_lock);
2886 mxlnd_put_idle_tx(tx);
2887 mxlnd_conn_decref(conn);
2894 cfs_list_del_init(&tx->mxc_list);
2896 /* handle credits, etc now while we have the lock to avoid races */
2898 conn->mxk_credits--;
2899 conn->mxk_ntx_posted++;
2901 if (msg_type != MXLND_MSG_PUT_DATA &&
2902 msg_type != MXLND_MSG_GET_DATA) {
2903 if (msg_type != MXLND_MSG_CONN_REQ &&
2904 msg_type != MXLND_MSG_CONN_ACK) {
2905 conn->mxk_ntx_msgs--;
2908 if (tx->mxc_incarnation == 0 &&
2909 conn->mxk_incarnation != 0) {
2910 tx->mxc_incarnation = conn->mxk_incarnation;
2913 /* if this is a NOOP and (1) mxp_conn->mxk_outstanding < CREDIT_HIGHWATER
2914 * or (2) there is a non-DATA msg that can return credits in the
2915 * queue, then drop this duplicate NOOP */
2916 if (unlikely(msg_type == MXLND_MSG_NOOP)) {
2917 if ((conn->mxk_outstanding < MXLND_CREDIT_HIGHWATER()) ||
2918 (conn->mxk_ntx_msgs >= 1)) {
2919 conn->mxk_credits++;
2920 conn->mxk_ntx_posted--;
2921 spin_unlock(&conn->mxk_lock);
2922 /* redundant NOOP */
2923 mxlnd_put_idle_tx(tx);
2924 mxlnd_conn_decref(conn);
2925 CDEBUG(D_NET, "%s: redundant noop\n",
2926 libcfs_nid2str(peer->mxp_nid));
2933 if (likely((msg_type != MXLND_MSG_PUT_DATA) &&
2934 (msg_type != MXLND_MSG_GET_DATA))) {
2935 mxlnd_pack_msg_locked(tx);
2940 status = conn->mxk_status;
2941 spin_unlock(&conn->mxk_lock);
2943 if (likely((status == MXLND_CONN_READY) ||
2944 (msg_type == MXLND_MSG_CONN_REQ) ||
2945 (msg_type == MXLND_MSG_CONN_ACK))) {
2947 if (msg_type != MXLND_MSG_CONN_REQ &&
2948 msg_type != MXLND_MSG_CONN_ACK) {
2949 /* add to the pending list */
2950 ret = mxlnd_q_pending_ctx(tx);
2953 tx->mxc_state = MXLND_CTX_PENDING;
2957 if (likely(msg_type != MXLND_MSG_PUT_DATA &&
2958 msg_type != MXLND_MSG_GET_DATA)) {
2959 /* send a msg style tx */
2960 LASSERT(tx->mxc_nseg == 1);
2961 LASSERT(tx->mxc_pin_type == MX_PIN_PHYSICAL);
2962 CDEBUG(D_NET, "sending %s 0x%llx\n",
2963 mxlnd_msgtype_to_str(msg_type),
2965 mxret = mx_kisend(kmxlnd_data.kmx_endpt,
2974 /* send a DATA tx */
2975 spin_lock(&conn->mxk_lock);
2976 conn->mxk_ntx_data--;
2977 conn->mxk_data_posted++;
2978 spin_unlock(&conn->mxk_lock);
2979 CDEBUG(D_NET, "sending %s 0x%llx\n",
2980 mxlnd_msgtype_to_str(msg_type),
2982 mxret = mx_kisend(kmxlnd_data.kmx_endpt,
2993 mxret = MX_CONNECTION_FAILED;
2995 if (likely(mxret == MX_SUCCESS)) {
2998 CNETERR("mx_kisend() failed with %s (%d) "
2999 "sending to %s\n", mx_strerror(mxret), (int) mxret,
3000 libcfs_nid2str(peer->mxp_nid));
3001 /* NOTE mx_kisend() only fails if there are not enough
3002 * resources. Do not change the connection status. */
3003 if (mxret == MX_NO_RESOURCES) {
3004 tx->mxc_errno = -ENOMEM;
3006 tx->mxc_errno = -ECONNABORTED;
3009 spin_lock(&conn->mxk_lock);
3010 conn->mxk_ntx_posted--;
3011 conn->mxk_credits++;
3012 spin_unlock(&conn->mxk_lock);
3013 } else if (msg_type == MXLND_MSG_PUT_DATA ||
3014 msg_type == MXLND_MSG_GET_DATA) {
3015 spin_lock(&conn->mxk_lock);
3016 conn->mxk_data_posted--;
3017 spin_unlock(&conn->mxk_lock);
3019 if (msg_type != MXLND_MSG_PUT_DATA &&
3020 msg_type != MXLND_MSG_GET_DATA &&
3021 msg_type != MXLND_MSG_CONN_REQ &&
3022 msg_type != MXLND_MSG_CONN_ACK) {
3023 spin_lock(&conn->mxk_lock);
3024 conn->mxk_outstanding +=
3025 tx->mxc_msg->mxm_credits;
3026 spin_unlock(&conn->mxk_lock);
3028 if (msg_type != MXLND_MSG_CONN_REQ &&
3029 msg_type != MXLND_MSG_CONN_ACK) {
3030 /* remove from the pending list */
3031 mxlnd_deq_pending_ctx(tx);
3033 mxlnd_put_idle_tx(tx);
3034 mxlnd_conn_decref(conn);
3037 spin_lock(&conn->mxk_lock);
3040 spin_unlock(&conn->mxk_lock);
3042 mxlnd_conn_decref(conn); /* drop ref taken at start of function */
3048 * mxlnd_handle_tx_completion - a tx completed, progress or complete the msg
3049 * @ctx - the tx descriptor
3051 * Determine which type of send request it was and start the next step, if needed,
3052 * or, if done, signal completion to LNET. After we are done, put back on the
3056 mxlnd_handle_tx_completion(kmx_ctx_t *tx)
3058 int code = tx->mxc_status.code;
3059 int failed = (code != MX_STATUS_SUCCESS || tx->mxc_errno != 0);
3060 kmx_msg_t *msg = tx->mxc_msg;
3061 kmx_peer_t *peer = tx->mxc_peer;
3062 kmx_conn_t *conn = tx->mxc_conn;
3063 u8 type = tx->mxc_msg_type;
3064 int credit = mxlnd_tx_requires_credit(tx);
3065 u64 cookie = tx->mxc_cookie;
3067 CDEBUG(D_NET, "entering %s (0x%llx):\n",
3068 mxlnd_msgtype_to_str(tx->mxc_msg_type), cookie);
3070 LASSERT (peer != NULL);
3071 LASSERT (conn != NULL);
3073 if (type != MXLND_MSG_PUT_DATA && type != MXLND_MSG_GET_DATA) {
3074 LASSERT (type == msg->mxm_type);
3078 if (tx->mxc_errno == 0) tx->mxc_errno = -EIO;
3080 spin_lock(&conn->mxk_lock);
3081 conn->mxk_last_tx = cfs_time_current(); /* jiffies */
3082 spin_unlock(&conn->mxk_lock);
3087 case MXLND_MSG_GET_DATA:
3088 spin_lock(&conn->mxk_lock);
3089 if (conn->mxk_incarnation == tx->mxc_incarnation) {
3090 conn->mxk_outstanding++;
3091 conn->mxk_data_posted--;
3093 spin_unlock(&conn->mxk_lock);
3096 case MXLND_MSG_PUT_DATA:
3097 spin_lock(&conn->mxk_lock);
3098 if (conn->mxk_incarnation == tx->mxc_incarnation) {
3099 conn->mxk_data_posted--;
3101 spin_unlock(&conn->mxk_lock);
3104 case MXLND_MSG_NOOP:
3105 case MXLND_MSG_PUT_REQ:
3106 case MXLND_MSG_PUT_ACK:
3107 case MXLND_MSG_GET_REQ:
3108 case MXLND_MSG_EAGER:
3111 case MXLND_MSG_CONN_ACK:
3112 if (peer->mxp_incompatible) {
3113 /* we sent our params, now close this conn */
3114 mxlnd_conn_disconnect(conn, 0, 1);
3116 case MXLND_MSG_CONN_REQ:
3118 CNETERR("%s failed with %s (%d) (errno = %d) to %s\n",
3119 type == MXLND_MSG_CONN_REQ ? "CONN_REQ" : "CONN_ACK",
3120 mx_strstatus(code), code, tx->mxc_errno,
3121 libcfs_nid2str(tx->mxc_nid));
3122 if (!peer->mxp_incompatible) {
3123 spin_lock(&conn->mxk_lock);
3124 if (code == MX_STATUS_BAD_SESSION)
3125 mxlnd_set_conn_status(conn,
3128 mxlnd_set_conn_status(conn,
3130 spin_unlock(&conn->mxk_lock);
3136 CNETERR("Unknown msg type of %d\n", type);
3141 spin_lock(&conn->mxk_lock);
3142 if (conn->mxk_incarnation == tx->mxc_incarnation) {
3143 conn->mxk_ntx_posted--;
3145 spin_unlock(&conn->mxk_lock);
3148 mxlnd_put_idle_tx(tx);
3149 mxlnd_conn_decref(conn);
3151 mxlnd_check_sends(peer);
3153 CDEBUG(D_NET, "leaving\n");
3157 /* Handle completion of MSG or DATA rx.
3158 * CONN_REQ and CONN_ACK are handled elsewhere. */
3160 mxlnd_handle_rx_completion(kmx_ctx_t *rx)
3165 u32 nob = rx->mxc_status.xfer_length;
3166 u64 bits = rx->mxc_status.match_info;
3167 kmx_msg_t *msg = rx->mxc_msg;
3168 kmx_peer_t *peer = rx->mxc_peer;
3169 kmx_conn_t *conn = rx->mxc_conn;
3170 u8 type = rx->mxc_msg_type;
3172 lnet_msg_t *lntmsg[2];
3177 /* NOTE We may only know the peer's nid if it is a PUT_REQ, GET_REQ,
3178 * failed GET reply */
3180 /* NOTE peer may still be NULL if it is a new peer and
3181 * conn may be NULL if this is a re-connect */
3182 if (likely(peer != NULL && conn != NULL)) {
3183 /* we have a reference on the conn */
3185 } else if (peer != NULL && conn == NULL) {
3186 /* we have a reference on the peer */
3188 } else if (peer == NULL && conn != NULL) {
3190 CERROR("rx 0x%llx from %s has conn but no peer\n",
3191 bits, libcfs_nid2str(rx->mxc_nid));
3193 } /* else peer and conn == NULL */
3195 if (conn == NULL && peer != NULL) {
3196 write_lock(&kmxlnd_data.kmx_global_lock);
3197 conn = peer->mxp_conn;
3199 mxlnd_conn_addref(conn); /* conn takes ref... */
3200 mxlnd_peer_decref(peer); /* from peer */
3204 write_unlock(&kmxlnd_data.kmx_global_lock);
3205 rx->mxc_conn = conn;
3209 CDEBUG(D_NET, "receiving msg bits=0x%llx nob=%d peer=0x%p\n", bits, nob, peer);
3215 if (rx->mxc_status.code != MX_STATUS_SUCCESS &&
3216 rx->mxc_status.code != MX_STATUS_TRUNCATED) {
3217 CNETERR("rx from %s failed with %s (%d)\n",
3218 libcfs_nid2str(rx->mxc_nid),
3219 mx_strstatus(rx->mxc_status.code),
3220 rx->mxc_status.code);
3226 /* this may be a failed GET reply */
3227 if (type == MXLND_MSG_GET_DATA) {
3228 /* get the error (52-59) bits from the match bits */
3229 ret = (u32) MXLND_ERROR_VAL(rx->mxc_status.match_info);
3230 lntmsg[0] = rx->mxc_lntmsg[0];
3234 /* we had a rx complete with 0 bytes (no hdr, nothing) */
3235 CNETERR("rx from %s returned with 0 bytes\n",
3236 libcfs_nid2str(rx->mxc_nid));
3241 /* NOTE PUT_DATA and GET_DATA do not have mxc_msg, do not call unpack() */
3242 if (type == MXLND_MSG_PUT_DATA) {
3244 lntmsg[0] = rx->mxc_lntmsg[0];
3246 } else if (type == MXLND_MSG_GET_DATA) {
3248 lntmsg[0] = rx->mxc_lntmsg[0];
3249 lntmsg[1] = rx->mxc_lntmsg[1];
3253 ret = mxlnd_unpack_msg(msg, nob);
3255 CNETERR("Error %d unpacking rx from %s\n",
3256 ret, libcfs_nid2str(rx->mxc_nid));
3260 type = msg->mxm_type;
3262 if (rx->mxc_nid != msg->mxm_srcnid ||
3263 kmxlnd_data.kmx_ni->ni_nid != msg->mxm_dstnid) {
3264 CNETERR("rx with mismatched NID (type %s) (my nid is "
3265 "0x%llx and rx msg dst is 0x%llx)\n",
3266 mxlnd_msgtype_to_str(type), kmxlnd_data.kmx_ni->ni_nid,
3271 if ((conn != NULL && msg->mxm_srcstamp != conn->mxk_incarnation) ||
3272 msg->mxm_dststamp != kmxlnd_data.kmx_incarnation) {
3273 CNETERR("Stale rx from %s with type %s "
3274 "(mxm_srcstamp (%lld) != mxk_incarnation (%lld) "
3275 "|| mxm_dststamp (%lld) != kmx_incarnation (%lld))\n",
3276 libcfs_nid2str(rx->mxc_nid), mxlnd_msgtype_to_str(type),
3277 msg->mxm_srcstamp, conn->mxk_incarnation,
3278 msg->mxm_dststamp, kmxlnd_data.kmx_incarnation);
3283 CDEBUG(D_NET, "Received %s with %d credits\n",
3284 mxlnd_msgtype_to_str(type), msg->mxm_credits);
3286 LASSERT(peer != NULL && conn != NULL);
3287 if (msg->mxm_credits != 0) {
3288 spin_lock(&conn->mxk_lock);
3289 if (msg->mxm_srcstamp == conn->mxk_incarnation) {
3290 if ((conn->mxk_credits + msg->mxm_credits) >
3291 *kmxlnd_tunables.kmx_peercredits) {
3292 CNETERR("mxk_credits %d mxm_credits %d\n",
3293 conn->mxk_credits, msg->mxm_credits);
3295 conn->mxk_credits += msg->mxm_credits;
3296 LASSERT(conn->mxk_credits >= 0);
3297 LASSERT(conn->mxk_credits <= *kmxlnd_tunables.kmx_peercredits);
3299 spin_unlock(&conn->mxk_lock);
3302 CDEBUG(D_NET, "switch %s for rx (0x%llx)\n", mxlnd_msgtype_to_str(type), seq);
3304 case MXLND_MSG_NOOP:
3307 case MXLND_MSG_EAGER:
3308 ret = lnet_parse(kmxlnd_data.kmx_ni, &msg->mxm_u.eager.mxem_hdr,
3309 msg->mxm_srcnid, rx, 0);
3313 case MXLND_MSG_PUT_REQ:
3314 ret = lnet_parse(kmxlnd_data.kmx_ni, &msg->mxm_u.put_req.mxprm_hdr,
3315 msg->mxm_srcnid, rx, 1);
3319 case MXLND_MSG_PUT_ACK: {
3320 u64 cookie = (u64) msg->mxm_u.put_ack.mxpam_dst_cookie;
3321 if (cookie > MXLND_MAX_COOKIE) {
3322 CNETERR("NAK for msg_type %d from %s\n", rx->mxc_msg_type,
3323 libcfs_nid2str(rx->mxc_nid));
3324 result = -((u32) MXLND_ERROR_VAL(cookie));
3325 lntmsg[0] = rx->mxc_lntmsg[0];
3327 mxlnd_send_data(kmxlnd_data.kmx_ni, rx->mxc_lntmsg[0],
3328 rx->mxc_peer, MXLND_MSG_PUT_DATA,
3329 rx->mxc_msg->mxm_u.put_ack.mxpam_dst_cookie);
3334 case MXLND_MSG_GET_REQ:
3335 ret = lnet_parse(kmxlnd_data.kmx_ni, &msg->mxm_u.get_req.mxgrm_hdr,
3336 msg->mxm_srcnid, rx, 1);
3341 CNETERR("Bad MXLND message type %x from %s\n", msg->mxm_type,
3342 libcfs_nid2str(rx->mxc_nid));
3348 CDEBUG(D_NET, "setting PEER_CONN_FAILED\n");
3349 spin_lock(&conn->mxk_lock);
3350 mxlnd_set_conn_status(conn, MXLND_CONN_FAIL);
3351 spin_unlock(&conn->mxk_lock);
3356 spin_lock(&conn->mxk_lock);
3357 conn->mxk_last_rx = cfs_time_current(); /* jiffies */
3358 spin_unlock(&conn->mxk_lock);
3362 /* lnet_parse() failed, etc., repost now */
3363 mxlnd_put_idle_rx(rx);
3364 if (conn != NULL && credit == 1) {
3365 if (type == MXLND_MSG_PUT_DATA ||
3366 type == MXLND_MSG_EAGER ||
3367 type == MXLND_MSG_PUT_REQ ||
3368 type == MXLND_MSG_NOOP) {
3369 spin_lock(&conn->mxk_lock);
3370 conn->mxk_outstanding++;
3371 spin_unlock(&conn->mxk_lock);
3374 if (conn_ref) mxlnd_conn_decref(conn);
3375 LASSERT(peer_ref == 0);
3378 if (type == MXLND_MSG_PUT_DATA || type == MXLND_MSG_GET_DATA) {
3379 CDEBUG(D_NET, "leaving for rx (0x%llx)\n", bits);
3381 CDEBUG(D_NET, "leaving for rx (0x%llx)\n", seq);
3384 if (lntmsg[0] != NULL) lnet_finalize(kmxlnd_data.kmx_ni, lntmsg[0], result);
3385 if (lntmsg[1] != NULL) lnet_finalize(kmxlnd_data.kmx_ni, lntmsg[1], result);
3387 if (conn != NULL && credit == 1) mxlnd_check_sends(peer);
3393 mxlnd_handle_connect_msg(kmx_peer_t *peer, u8 msg_type, mx_status_t status)
3395 kmx_ctx_t *tx = NULL;
3396 kmx_msg_t *txmsg = NULL;
3397 kmx_conn_t *conn = peer->mxp_conn;
3401 u8 type = (msg_type == MXLND_MSG_ICON_REQ ?
3402 MXLND_MSG_CONN_REQ : MXLND_MSG_CONN_ACK);
3404 /* a conn ref was taken when calling mx_iconnect(),
3405 * hold it until CONN_REQ or CONN_ACK completes */
3407 CDEBUG(D_NET, "entering\n");
3408 if (status.code != MX_STATUS_SUCCESS) {
3409 int send_bye = (msg_type == MXLND_MSG_ICON_REQ ? 0 : 1);
3411 CNETERR("mx_iconnect() failed for %s with %s (%d) "
3412 "to %s mxp_nid = 0x%llx mxp_nic_id = 0x%0llx mxp_ep_id = %d\n",
3413 mxlnd_msgtype_to_str(msg_type),
3414 mx_strstatus(status.code), status.code,
3415 libcfs_nid2str(peer->mxp_nid),
3419 spin_lock(&conn->mxk_lock);
3420 mxlnd_set_conn_status(conn, MXLND_CONN_FAIL);
3421 spin_unlock(&conn->mxk_lock);
3423 if (cfs_time_after(jiffies, peer->mxp_reconnect_time +
3424 MXLND_CONNECT_TIMEOUT)) {
3425 CNETERR("timeout, calling conn_disconnect()\n");
3426 mxlnd_conn_disconnect(conn, 0, send_bye);
3429 mxlnd_conn_decref(conn);
3432 mx_decompose_endpoint_addr2(status.source, &nic_id, &ep_id, &sid);
3433 write_lock(&kmxlnd_data.kmx_global_lock);
3434 spin_lock(&conn->mxk_lock);
3435 conn->mxk_epa = status.source;
3436 mx_set_endpoint_addr_context(conn->mxk_epa, (void *) conn);
3437 if (msg_type == MXLND_MSG_ICON_ACK && likely(!peer->mxp_incompatible)) {
3438 mxlnd_set_conn_status(conn, MXLND_CONN_READY);
3440 spin_unlock(&conn->mxk_lock);
3441 write_unlock(&kmxlnd_data.kmx_global_lock);
3443 /* mx_iconnect() succeeded, reset delay to 0 */
3444 write_lock(&kmxlnd_data.kmx_global_lock);
3445 peer->mxp_reconnect_time = 0;
3446 peer->mxp_conn->mxk_sid = sid;
3447 write_unlock(&kmxlnd_data.kmx_global_lock);
3449 /* marshal CONN_REQ or CONN_ACK msg */
3450 /* we are still using the conn ref from iconnect() - do not take another */
3451 tx = mxlnd_get_idle_tx();
3453 CNETERR("Can't obtain %s tx for %s\n",
3454 mxlnd_msgtype_to_str(type),
3455 libcfs_nid2str(peer->mxp_nid));
3456 spin_lock(&conn->mxk_lock);
3457 mxlnd_set_conn_status(conn, MXLND_CONN_FAIL);
3458 spin_unlock(&conn->mxk_lock);
3459 mxlnd_conn_decref(conn);
3463 tx->mxc_peer = peer;
3464 tx->mxc_conn = conn;
3465 tx->mxc_deadline = jiffies + MXLND_CONNECT_TIMEOUT;
3466 CDEBUG(D_NET, "sending %s\n", mxlnd_msgtype_to_str(type));
3467 mxlnd_init_tx_msg (tx, type, sizeof(kmx_connreq_msg_t), peer->mxp_nid);
3468 txmsg = tx->mxc_msg;
3469 txmsg->mxm_u.conn_req.mxcrm_queue_depth = *kmxlnd_tunables.kmx_peercredits;
3470 txmsg->mxm_u.conn_req.mxcrm_eager_size = MXLND_MSG_SIZE;
3471 tx->mxc_match = mxlnd_create_match(tx, 0);
3478 * mxlnd_request_waitd - the MX request completion thread(s)
3479 * @arg - thread id (as a void *)
3481 * This thread waits for a MX completion and then completes the request.
3482 * We will create one thread per CPU.
3485 mxlnd_request_waitd(void *arg)
3487 long id = (long) arg;
3490 mx_return_t mxret = MX_SUCCESS;
3492 kmx_ctx_t *ctx = NULL;
3493 enum kmx_req_state req_type = MXLND_REQ_TX;
3494 kmx_peer_t *peer = NULL;
3495 kmx_conn_t *conn = NULL;
3500 memset(name, 0, sizeof(name));
3501 snprintf(name, sizeof(name), "mxlnd_request_waitd_%02ld", id);
3502 cfs_daemonize(name);
3504 memset(&status, 0, sizeof(status));
3506 CDEBUG(D_NET, "%s starting\n", name);
3508 while (!(cfs_atomic_read(&kmxlnd_data.kmx_shutdown))) {
3514 if (id == 0 && count++ < *kmxlnd_tunables.kmx_polling) {
3515 mxret = mx_test_any(kmxlnd_data.kmx_endpt, 0ULL, 0ULL,
3519 mxret = mx_wait_any(kmxlnd_data.kmx_endpt, MXLND_WAIT_TIMEOUT,
3520 0ULL, 0ULL, &status, &result);
3523 mxret = mx_wait_any(kmxlnd_data.kmx_endpt, MXLND_WAIT_TIMEOUT,
3524 0ULL, 0ULL, &status, &result);
3526 if (unlikely(cfs_atomic_read(&kmxlnd_data.kmx_shutdown)))
3530 /* nothing completed... */
3534 CDEBUG(D_NET, "wait_any() returned with %s (%d) with "
3535 "match_info 0x%llx and length %d\n",
3536 mx_strstatus(status.code), status.code,
3537 (u64) status.match_info, status.msg_length);
3539 if (status.code != MX_STATUS_SUCCESS) {
3540 CNETERR("wait_any() failed with %s (%d) with "
3541 "match_info 0x%llx and length %d\n",
3542 mx_strstatus(status.code), status.code,
3543 (u64) status.match_info, status.msg_length);
3546 msg_type = MXLND_MSG_TYPE(status.match_info);
3548 /* This may be a mx_iconnect() request completing,
3549 * check the bit mask for CONN_REQ and CONN_ACK */
3550 if (msg_type == MXLND_MSG_ICON_REQ ||
3551 msg_type == MXLND_MSG_ICON_ACK) {
3552 peer = (kmx_peer_t*) status.context;
3553 mxlnd_handle_connect_msg(peer, msg_type, status);
3557 /* This must be a tx or rx */
3559 /* NOTE: if this is a RX from the unexpected callback, it may
3560 * have very little info. If we dropped it in unexpected_recv(),
3561 * it will not have a context. If so, ignore it. */
3562 ctx = (kmx_ctx_t *) status.context;
3565 req_type = ctx->mxc_type;
3566 conn = ctx->mxc_conn; /* this may be NULL */
3567 mxlnd_deq_pending_ctx(ctx);
3569 /* copy status to ctx->mxc_status */
3570 ctx->mxc_status = status;
3574 mxlnd_handle_tx_completion(ctx);
3577 mxlnd_handle_rx_completion(ctx);
3580 CNETERR("Unknown ctx type %d\n", req_type);
3585 /* conn is always set except for the first CONN_REQ rx
3586 * from a new peer */
3587 if (status.code != MX_STATUS_SUCCESS && conn != NULL) {
3588 mxlnd_conn_disconnect(conn, 1, 1);
3591 CDEBUG(D_NET, "waitd() completed task\n");
3593 CDEBUG(D_NET, "%s stopping\n", name);
3594 mxlnd_thread_stop(id);
3600 mxlnd_check_timeouts(unsigned long now)
3604 unsigned long next = 0; /* jiffies */
3605 kmx_peer_t *peer = NULL;
3606 kmx_conn_t *conn = NULL;
3607 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
3610 for (i = 0; i < MXLND_HASH_SIZE; i++) {
3611 cfs_list_for_each_entry(peer, &kmxlnd_data.kmx_peers[i],
3614 if (unlikely(cfs_atomic_read(&kmxlnd_data.kmx_shutdown))) {
3615 read_unlock(g_lock);
3619 conn = peer->mxp_conn;
3621 mxlnd_conn_addref(conn);
3626 spin_lock(&conn->mxk_lock);
3628 /* if nothing pending (timeout == 0) or
3629 * if conn is already disconnected,
3631 if (conn->mxk_timeout == 0 ||
3632 conn->mxk_status == MXLND_CONN_DISCONNECT) {
3633 spin_unlock(&conn->mxk_lock);
3634 mxlnd_conn_decref(conn);
3638 /* we want to find the timeout that will occur first.
3639 * if it is in the future, we will sleep until then.
3640 * if it is in the past, then we will sleep one
3641 * second and repeat the process. */
3643 (cfs_time_before(conn->mxk_timeout, next))) {
3644 next = conn->mxk_timeout;
3649 if (cfs_time_aftereq(now, conn->mxk_timeout))
3651 spin_unlock(&conn->mxk_lock);
3654 mxlnd_conn_disconnect(conn, 1, 1);
3655 mxlnd_conn_decref(conn);
3658 read_unlock(g_lock);
3660 next = now + MXLND_COMM_TIMEOUT;
3666 mxlnd_passive_connect(kmx_connparams_t *cp)
3669 int incompatible = 0;
3674 kmx_msg_t *msg = &cp->mxr_msg;
3675 kmx_peer_t *peer = cp->mxr_peer;
3676 kmx_conn_t *conn = NULL;
3677 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
3679 mx_decompose_endpoint_addr2(cp->mxr_epa, &nic_id, &ep_id, &sid);
3681 ret = mxlnd_unpack_msg(msg, cp->mxr_nob);
3684 CNETERR("Error %d unpacking CONN_REQ from %s\n",
3685 ret, libcfs_nid2str(peer->mxp_nid));
3687 CNETERR("Error %d unpacking CONN_REQ from "
3688 "unknown host with nic_id 0x%llx\n", ret, nic_id);
3692 if (kmxlnd_data.kmx_ni->ni_nid != msg->mxm_dstnid) {
3693 CNETERR("Can't accept %s: bad dst nid %s\n",
3694 libcfs_nid2str(msg->mxm_srcnid),
3695 libcfs_nid2str(msg->mxm_dstnid));
3698 if (msg->mxm_u.conn_req.mxcrm_queue_depth != *kmxlnd_tunables.kmx_peercredits) {
3699 CNETERR("Can't accept %s: incompatible queue depth "
3701 libcfs_nid2str(msg->mxm_srcnid),
3702 msg->mxm_u.conn_req.mxcrm_queue_depth,
3703 *kmxlnd_tunables.kmx_peercredits);
3706 if (msg->mxm_u.conn_req.mxcrm_eager_size != MXLND_MSG_SIZE) {
3707 CNETERR("Can't accept %s: incompatible EAGER size "
3709 libcfs_nid2str(msg->mxm_srcnid),
3710 msg->mxm_u.conn_req.mxcrm_eager_size,
3711 (int) MXLND_MSG_SIZE);
3716 peer = mxlnd_find_peer_by_nid(msg->mxm_srcnid, 0); /* adds peer ref */
3720 kmx_peer_t *existing_peer = NULL;
3722 hash = mxlnd_nid_to_hash(msg->mxm_srcnid);
3724 mx_nic_id_to_board_number(nic_id, &board);
3726 /* adds conn ref for peer and one for this function */
3727 ret = mxlnd_peer_alloc(&peer, msg->mxm_srcnid,
3728 board, ep_id, 0ULL);
3732 peer->mxp_conn->mxk_sid = sid;
3733 LASSERT(peer->mxp_ep_id == ep_id);
3735 existing_peer = mxlnd_find_peer_by_nid_locked(msg->mxm_srcnid);
3736 if (existing_peer) {
3737 mxlnd_conn_decref(peer->mxp_conn);
3738 mxlnd_peer_decref(peer);
3739 peer = existing_peer;
3740 mxlnd_conn_addref(peer->mxp_conn);
3741 conn = peer->mxp_conn;
3743 cfs_list_add_tail(&peer->mxp_list,
3744 &kmxlnd_data.kmx_peers[hash]);
3745 cfs_atomic_inc(&kmxlnd_data.kmx_npeers);
3747 write_unlock(g_lock);
3749 ret = mxlnd_conn_alloc(&conn, peer); /* adds 2nd ref */
3751 mxlnd_peer_decref(peer); /* drop ref taken above */
3752 write_unlock(g_lock);
3754 CNETERR("Cannot allocate mxp_conn\n");
3758 conn_ref = 1; /* peer/conn_alloc() added ref for this function */
3759 conn = peer->mxp_conn;
3760 } else { /* unexpected handler found peer */
3761 kmx_conn_t *old_conn = peer->mxp_conn;
3763 if (sid != peer->mxp_conn->mxk_sid) {
3764 /* do not call mx_disconnect() or send a BYE */
3765 mxlnd_conn_disconnect(old_conn, 0, 0);
3767 /* This allocs a conn, points peer->mxp_conn to this one.
3768 * The old conn is still on the peer->mxp_conns list.
3769 * As the pending requests complete, they will call
3770 * conn_decref() which will eventually free it. */
3771 ret = mxlnd_conn_alloc(&conn, peer);
3773 CNETERR("Cannot allocate peer->mxp_conn\n");
3776 /* conn_alloc() adds one ref for the peer and one
3777 * for this function */
3780 peer->mxp_conn->mxk_sid = sid;
3783 conn = peer->mxp_conn;
3787 peer->mxp_incompatible = incompatible;
3788 write_unlock(g_lock);
3789 spin_lock(&conn->mxk_lock);
3790 conn->mxk_incarnation = msg->mxm_srcstamp;
3791 mxlnd_set_conn_status(conn, MXLND_CONN_WAIT);
3792 spin_unlock(&conn->mxk_lock);
3794 /* handle_conn_ack() will create the CONN_ACK msg */
3795 mxlnd_iconnect(peer, (u8) MXLND_MSG_ICON_ACK);
3798 if (conn_ref) mxlnd_conn_decref(conn);
3800 mxlnd_connparams_free(cp);
3805 mxlnd_check_conn_ack(kmx_connparams_t *cp)
3808 int incompatible = 0;
3812 kmx_msg_t *msg = &cp->mxr_msg;
3813 kmx_peer_t *peer = cp->mxr_peer;
3814 kmx_conn_t *conn = cp->mxr_conn;
3816 mx_decompose_endpoint_addr2(cp->mxr_epa, &nic_id, &ep_id, &sid);
3818 ret = mxlnd_unpack_msg(msg, cp->mxr_nob);
3821 CNETERR("Error %d unpacking CONN_ACK from %s\n",
3822 ret, libcfs_nid2str(peer->mxp_nid));
3824 CNETERR("Error %d unpacking CONN_ACK from "
3825 "unknown host with nic_id 0x%llx\n", ret, nic_id);
3831 if (kmxlnd_data.kmx_ni->ni_nid != msg->mxm_dstnid) {
3832 CNETERR("Can't accept CONN_ACK from %s: "
3833 "bad dst nid %s\n", libcfs_nid2str(msg->mxm_srcnid),
3834 libcfs_nid2str(msg->mxm_dstnid));
3838 if (msg->mxm_u.conn_req.mxcrm_queue_depth != *kmxlnd_tunables.kmx_peercredits) {
3839 CNETERR("Can't accept CONN_ACK from %s: "
3840 "incompatible queue depth %d (%d wanted)\n",
3841 libcfs_nid2str(msg->mxm_srcnid),
3842 msg->mxm_u.conn_req.mxcrm_queue_depth,
3843 *kmxlnd_tunables.kmx_peercredits);
3848 if (msg->mxm_u.conn_req.mxcrm_eager_size != MXLND_MSG_SIZE) {
3849 CNETERR("Can't accept CONN_ACK from %s: "
3850 "incompatible EAGER size %d (%d wanted)\n",
3851 libcfs_nid2str(msg->mxm_srcnid),
3852 msg->mxm_u.conn_req.mxcrm_eager_size,
3853 (int) MXLND_MSG_SIZE);
3858 write_lock(&kmxlnd_data.kmx_global_lock);
3859 peer->mxp_incompatible = incompatible;
3860 write_unlock(&kmxlnd_data.kmx_global_lock);
3861 spin_lock(&conn->mxk_lock);
3862 conn->mxk_credits = *kmxlnd_tunables.kmx_peercredits;
3863 conn->mxk_outstanding = 0;
3864 conn->mxk_incarnation = msg->mxm_srcstamp;
3865 conn->mxk_timeout = 0;
3866 if (!incompatible) {
3867 CDEBUG(D_NET, "setting peer %s CONN_READY\n",
3868 libcfs_nid2str(msg->mxm_srcnid));
3869 mxlnd_set_conn_status(conn, MXLND_CONN_READY);
3871 spin_unlock(&conn->mxk_lock);
3874 mxlnd_check_sends(peer);
3878 spin_lock(&conn->mxk_lock);
3879 mxlnd_set_conn_status(conn, MXLND_CONN_FAIL);
3880 spin_unlock(&conn->mxk_lock);
3883 if (incompatible) mxlnd_conn_disconnect(conn, 0, 0);
3885 mxlnd_connparams_free(cp);
3890 mxlnd_abort_msgs(void)
3893 cfs_list_t *orphans = &kmxlnd_data.kmx_orphan_msgs;
3894 spinlock_t *g_conn_lock = &kmxlnd_data.kmx_conn_lock;
3897 spin_lock(g_conn_lock);
3898 while (!cfs_list_empty(orphans)) {
3899 kmx_ctx_t *ctx = NULL;
3900 kmx_conn_t *conn = NULL;
3902 ctx = cfs_list_entry(orphans->next, kmx_ctx_t, mxc_list);
3903 cfs_list_del_init(&ctx->mxc_list);
3904 spin_unlock(g_conn_lock);
3906 ctx->mxc_errno = -ECONNABORTED;
3907 conn = ctx->mxc_conn;
3908 CDEBUG(D_NET, "aborting %s %s %s\n",
3909 mxlnd_msgtype_to_str(ctx->mxc_msg_type),
3910 ctx->mxc_type == MXLND_REQ_TX ? "(TX) to" : "(RX) from",
3911 libcfs_nid2str(ctx->mxc_nid));
3912 if (ctx->mxc_type == MXLND_REQ_TX) {
3913 mxlnd_put_idle_tx(ctx); /* do not hold any locks */
3914 if (conn) mxlnd_conn_decref(conn); /* for this tx */
3916 ctx->mxc_state = MXLND_CTX_CANCELED;
3917 mxlnd_handle_rx_completion(ctx);
3921 spin_lock(g_conn_lock);
3923 spin_unlock(g_conn_lock);
3929 mxlnd_free_conn_zombies(void)
3932 cfs_list_t *zombies = &kmxlnd_data.kmx_conn_zombies;
3933 spinlock_t *g_conn_lock = &kmxlnd_data.kmx_conn_lock;
3934 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
3936 /* cleanup any zombies */
3937 spin_lock(g_conn_lock);
3938 while (!cfs_list_empty(zombies)) {
3939 kmx_conn_t *conn = NULL;
3941 conn = cfs_list_entry(zombies->next, kmx_conn_t, mxk_zombie);
3942 cfs_list_del_init(&conn->mxk_zombie);
3943 spin_unlock(g_conn_lock);
3946 mxlnd_conn_free_locked(conn);
3947 write_unlock(g_lock);
3950 spin_lock(g_conn_lock);
3952 spin_unlock(g_conn_lock);
3953 CDEBUG(D_NET, "%s: freed %d zombies\n", __func__, count);
3958 * mxlnd_connd - handles incoming connection requests
3959 * @arg - thread id (as a void *)
3961 * This thread handles incoming connection requests
3964 mxlnd_connd(void *arg)
3966 long id = (long) arg;
3968 cfs_daemonize("mxlnd_connd");
3970 CDEBUG(D_NET, "connd starting\n");
3972 while (!(cfs_atomic_read(&kmxlnd_data.kmx_shutdown))) {
3974 kmx_connparams_t *cp = NULL;
3975 spinlock_t *g_conn_lock = &kmxlnd_data.kmx_conn_lock;
3976 cfs_list_t *conn_reqs = &kmxlnd_data.kmx_conn_reqs;
3978 ret = down_interruptible(&kmxlnd_data.kmx_conn_sem);
3980 if (cfs_atomic_read(&kmxlnd_data.kmx_shutdown))
3986 ret = mxlnd_abort_msgs();
3987 ret += mxlnd_free_conn_zombies();
3989 spin_lock(g_conn_lock);
3990 if (cfs_list_empty(conn_reqs)) {
3992 CNETERR("connd woke up but did not find a "
3993 "kmx_connparams_t or zombie conn\n");
3994 spin_unlock(g_conn_lock);
3997 cp = cfs_list_entry(conn_reqs->next, kmx_connparams_t,
3999 cfs_list_del_init(&cp->mxr_list);
4000 spin_unlock(g_conn_lock);
4002 switch (MXLND_MSG_TYPE(cp->mxr_match)) {
4003 case MXLND_MSG_CONN_REQ:
4004 /* We have a connection request. Handle it. */
4005 mxlnd_passive_connect(cp);
4007 case MXLND_MSG_CONN_ACK:
4008 /* The peer is ready for messages */
4009 mxlnd_check_conn_ack(cp);
4014 mxlnd_free_conn_zombies();
4016 CDEBUG(D_NET, "connd stopping\n");
4017 mxlnd_thread_stop(id);
4022 * mxlnd_timeoutd - enforces timeouts on messages
4023 * @arg - thread id (as a void *)
4025 * This thread queries each peer for its earliest timeout. If a peer has timed out,
4026 * it calls mxlnd_conn_disconnect().
4028 * After checking for timeouts, try progressing sends (call check_sends()).
4031 mxlnd_timeoutd(void *arg)
4034 long id = (long) arg;
4035 unsigned long now = 0;
4036 unsigned long next = 0;
4037 unsigned long delay = CFS_HZ;
4038 kmx_peer_t *peer = NULL;
4039 kmx_peer_t *temp = NULL;
4040 kmx_conn_t *conn = NULL;
4041 rwlock_t *g_lock = &kmxlnd_data.kmx_global_lock;
4043 cfs_daemonize("mxlnd_timeoutd");
4045 CDEBUG(D_NET, "timeoutd starting\n");
4047 while (!(cfs_atomic_read(&kmxlnd_data.kmx_shutdown))) {
4050 /* if the next timeout has not arrived, go back to sleep */
4051 if (cfs_time_after(now, next)) {
4052 next = mxlnd_check_timeouts(now);
4055 /* try to progress peers' txs */
4057 for (i = 0; i < MXLND_HASH_SIZE; i++) {
4058 cfs_list_t *peers = &kmxlnd_data.kmx_peers[i];
4060 /* NOTE we are safe against the removal of peer, but
4061 * not against the removal of temp */
4062 cfs_list_for_each_entry_safe(peer, temp, peers,
4064 if (cfs_atomic_read(&kmxlnd_data.kmx_shutdown))
4066 mxlnd_peer_addref(peer); /* add ref... */
4067 conn = peer->mxp_conn;
4068 if (conn && conn->mxk_status != MXLND_CONN_DISCONNECT) {
4069 mxlnd_conn_addref(conn); /* take ref... */
4071 CDEBUG(D_NET, "ignoring %s\n",
4072 libcfs_nid2str(peer->mxp_nid));
4073 mxlnd_peer_decref(peer); /* ...to here */
4077 if ((conn->mxk_status == MXLND_CONN_READY ||
4078 conn->mxk_status == MXLND_CONN_FAIL) &&
4082 write_unlock(g_lock);
4083 mxlnd_check_sends(peer);
4086 mxlnd_conn_decref(conn); /* until here */
4087 mxlnd_peer_decref(peer); /* ...to here */
4090 write_unlock(g_lock);
4094 CDEBUG(D_NET, "timeoutd stopping\n");
4095 mxlnd_thread_stop(id);