1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
6 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 only,
10 * as published by the Free Software Foundation.
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License version 2 for more details (a copy is included
16 * in the LICENSE file that accompanied this code).
18 * You should have received a copy of the GNU General Public License
19 * version 2 along with this program; If not, see
20 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
22 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
23 * CA 95054 USA or visit www.sun.com if you need additional information or
29 * Copyright 2008 Sun Microsystems, Inc. All rights reserved
30 * Use is subject to license terms.
32 * Copyright (C) 2006 Myricom, Inc.
35 * This file is part of Lustre, http://www.lustre.org/
36 * Lustre is a trademark of Sun Microsystems, Inc.
38 * lnet/klnds/mxlnd/mxlnd.c
40 * Author: Eric Barton <eric@bartonsoftware.com>
41 * Author: Scott Atchley <atchley at myri.com>
48 .lnd_startup = mxlnd_startup,
49 .lnd_shutdown = mxlnd_shutdown,
51 .lnd_send = mxlnd_send,
52 .lnd_recv = mxlnd_recv,
55 kmx_data_t kmxlnd_data;
58 mxlnd_free_pages(kmx_pages_t *p)
60 int npages = p->mxg_npages;
63 CDEBUG(D_MALLOC, "freeing %d pages\n", npages);
65 for (i = 0; i < npages; i++) {
66 if (p->mxg_pages[i] != NULL) {
67 __free_page(p->mxg_pages[i]);
68 spin_lock(&kmxlnd_data.kmx_mem_lock);
69 kmxlnd_data.kmx_mem_used -= PAGE_SIZE;
70 spin_unlock(&kmxlnd_data.kmx_mem_lock);
74 MXLND_FREE(p, offsetof(kmx_pages_t, mxg_pages[npages]));
78 mxlnd_alloc_pages(kmx_pages_t **pp, int npages)
80 kmx_pages_t *p = NULL;
83 CDEBUG(D_MALLOC, "allocing %d pages\n", npages);
85 MXLND_ALLOC(p, offsetof(kmx_pages_t, mxg_pages[npages]));
87 CERROR("Can't allocate descriptor for %d pages\n", npages);
91 memset(p, 0, offsetof(kmx_pages_t, mxg_pages[npages]));
92 p->mxg_npages = npages;
94 for (i = 0; i < npages; i++) {
95 p->mxg_pages[i] = alloc_page(GFP_KERNEL);
96 if (p->mxg_pages[i] == NULL) {
97 CERROR("Can't allocate page %d of %d\n", i, npages);
101 spin_lock(&kmxlnd_data.kmx_mem_lock);
102 kmxlnd_data.kmx_mem_used += PAGE_SIZE;
103 spin_unlock(&kmxlnd_data.kmx_mem_lock);
111 * mxlnd_ctx_init - reset ctx struct to the default values
112 * @ctx - a kmx_ctx pointer
115 mxlnd_ctx_init(kmx_ctx_t *ctx)
117 if (ctx == NULL) return;
119 /* do not change mxc_type */
120 ctx->mxc_incarnation = 0;
121 ctx->mxc_deadline = 0;
122 ctx->mxc_state = MXLND_CTX_IDLE;
123 if (!list_empty(&ctx->mxc_list))
124 list_del_init(&ctx->mxc_list);
125 /* ignore mxc_rx_list */
126 if (ctx->mxc_type == MXLND_REQ_TX) {
128 ctx->mxc_peer = NULL;
129 ctx->mxc_conn = NULL;
132 ctx->mxc_lntmsg[0] = NULL;
133 ctx->mxc_lntmsg[1] = NULL;
134 ctx->mxc_msg_type = 0;
135 ctx->mxc_cookie = 0LL;
136 ctx->mxc_match = 0LL;
137 /* ctx->mxc_seg.segment_ptr points to backing page */
138 ctx->mxc_seg.segment_length = 0;
139 if (ctx->mxc_seg_list != NULL) {
140 LASSERT(ctx->mxc_nseg > 0);
141 MXLND_FREE(ctx->mxc_seg_list, ctx->mxc_nseg * sizeof(mx_ksegment_t));
143 ctx->mxc_seg_list = NULL;
146 memset(&ctx->mxc_mxreq, 0, sizeof(mx_request_t));
147 memset(&ctx->mxc_status, 0, sizeof(mx_status_t));
152 ctx->mxc_msg->mxm_type = 0;
153 ctx->mxc_msg->mxm_credits = 0;
154 ctx->mxc_msg->mxm_nob = 0;
160 * mxlnd_free_txs - free kmx_txs and associated pages
162 * Called from mxlnd_shutdown()
168 kmx_ctx_t *tx = NULL;
170 if (kmxlnd_data.kmx_tx_pages) {
171 for (i = 0; i < MXLND_TX_MSGS(); i++) {
172 tx = &kmxlnd_data.kmx_txs[i];
173 if (tx->mxc_seg_list != NULL) {
174 LASSERT(tx->mxc_nseg > 0);
175 MXLND_FREE(tx->mxc_seg_list,
177 sizeof(*tx->mxc_seg_list));
180 MXLND_FREE(kmxlnd_data.kmx_txs,
181 MXLND_TX_MSGS() * sizeof(kmx_ctx_t));
182 mxlnd_free_pages(kmxlnd_data.kmx_tx_pages);
189 * mxlnd_init_txs - allocate tx descriptors then stash on txs and idle tx lists
191 * Called from mxlnd_startup()
192 * returns 0 on success, else -ENOMEM
202 kmx_ctx_t *tx = NULL;
203 kmx_pages_t *pages = NULL;
204 struct page *page = NULL;
206 /* pre-mapped messages are not bigger than 1 page */
207 CLASSERT(MXLND_MSG_SIZE <= PAGE_SIZE);
209 /* No fancy arithmetic when we do the buffer calculations */
210 CLASSERT (PAGE_SIZE % MXLND_MSG_SIZE == 0);
212 ret = mxlnd_alloc_pages(&pages, MXLND_TX_MSG_PAGES());
214 CERROR("Can't allocate tx pages\n");
217 kmxlnd_data.kmx_tx_pages = pages;
219 MXLND_ALLOC(kmxlnd_data.kmx_txs, MXLND_TX_MSGS() * sizeof(kmx_ctx_t));
220 if (&kmxlnd_data.kmx_txs == NULL) {
221 CERROR("Can't allocate %d tx descriptors\n", MXLND_TX_MSGS());
222 mxlnd_free_pages(pages);
226 memset(kmxlnd_data.kmx_txs, 0, MXLND_TX_MSGS() * sizeof(kmx_ctx_t));
228 for (i = 0; i < MXLND_TX_MSGS(); i++) {
230 tx = &kmxlnd_data.kmx_txs[i];
231 tx->mxc_type = MXLND_REQ_TX;
233 INIT_LIST_HEAD(&tx->mxc_list);
235 /* map mxc_msg to page */
236 page = pages->mxg_pages[ipage];
237 addr = page_address(page);
238 LASSERT(addr != NULL);
239 tx->mxc_msg = (kmx_msg_t *)(addr + offset);
240 tx->mxc_seg.segment_ptr = MX_PA_TO_U64(virt_to_phys(tx->mxc_msg));
244 offset += MXLND_MSG_SIZE;
245 LASSERT (offset <= PAGE_SIZE);
247 if (offset == PAGE_SIZE) {
250 LASSERT (ipage <= MXLND_TX_MSG_PAGES());
253 /* in startup(), no locks required */
254 list_add_tail(&tx->mxc_list, &kmxlnd_data.kmx_tx_idle);
261 * mxlnd_free_peers - free peers
263 * Called from mxlnd_shutdown()
266 mxlnd_free_peers(void)
270 kmx_peer_t *peer = NULL;
271 kmx_peer_t *next = NULL;
273 for (i = 0; i < MXLND_HASH_SIZE; i++) {
274 list_for_each_entry_safe(peer, next, &kmxlnd_data.kmx_peers[i], mxp_list) {
275 list_del_init(&peer->mxp_list);
276 if (peer->mxp_conn) mxlnd_conn_decref(peer->mxp_conn);
277 mxlnd_peer_decref(peer);
281 CDEBUG(D_NET, "%s: freed %d peers\n", __func__, count);
285 * mxlnd_init_mx - open the endpoint, set our ID, register the EAGER callback
286 * @ni - the network interface
288 * Returns 0 on success, -1 on failure
291 mxlnd_init_mx(lnet_ni_t *ni)
295 u32 board = *kmxlnd_tunables.kmx_board;
296 u32 ep_id = *kmxlnd_tunables.kmx_ep_id;
304 if (mxret != MX_SUCCESS) {
305 CERROR("mx_init() failed with %s (%d)\n", mx_strerror(mxret), mxret);
309 if (ni->ni_interfaces[0] != NULL) {
310 /* Use the IPoMX interface specified in 'networks=' */
312 CLASSERT (LNET_MAX_INTERFACES > 1);
313 if (ni->ni_interfaces[1] != NULL) {
314 CERROR("Multiple interfaces not supported\n");
315 goto failed_with_init;
318 ifname = ni->ni_interfaces[0];
320 ifname = *kmxlnd_tunables.kmx_default_ipif;
323 ret = libcfs_ipif_query(ifname, &if_up, &ip, &netmask);
325 CERROR("Can't query IPoMX interface %s: %d\n",
327 goto failed_with_init;
331 CERROR("Can't query IPoMX interface %s: it's down\n",
333 goto failed_with_init;
336 mxret = mx_open_endpoint(board, ep_id, MXLND_MSG_MAGIC,
337 NULL, 0, &kmxlnd_data.kmx_endpt);
338 if (mxret != MX_SUCCESS) {
339 CERROR("mx_open_endpoint() failed with %d\n", mxret);
340 goto failed_with_init;
343 mx_get_endpoint_addr(kmxlnd_data.kmx_endpt, &kmxlnd_data.kmx_epa);
344 mx_decompose_endpoint_addr(kmxlnd_data.kmx_epa, &nic_id, &ep_id);
345 mxret = mx_connect(kmxlnd_data.kmx_endpt, nic_id, ep_id, MXLND_MSG_MAGIC,
346 MXLND_CONNECT_TIMEOUT/HZ*1000, &kmxlnd_data.kmx_epa);
347 if (mxret != MX_SUCCESS) {
348 CDEBUG(D_NETERROR, "unable to connect to myself (%s)\n", mx_strerror(mxret));
349 goto failed_with_endpoint;
352 ni->ni_nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), ip);
353 CDEBUG(D_NET, "My NID is 0x%llx\n", ni->ni_nid);
355 /* this will catch all unexpected receives. */
356 mxret = mx_register_unexp_handler(kmxlnd_data.kmx_endpt,
357 (mx_unexp_handler_t) mxlnd_unexpected_recv,
359 if (mxret != MX_SUCCESS) {
360 CERROR("mx_register_unexp_callback() failed with %s\n",
362 goto failed_with_endpoint;
364 mxret = mx_set_request_timeout(kmxlnd_data.kmx_endpt, NULL, MXLND_COMM_TIMEOUT/HZ*1000);
365 if (mxret != MX_SUCCESS) {
366 CERROR("mx_set_request_timeout() failed with %s\n",
368 goto failed_with_endpoint;
372 failed_with_endpoint:
373 mx_close_endpoint(kmxlnd_data.kmx_endpt);
381 * mxlnd_thread_start - spawn a kernel thread with this function
382 * @fn - function pointer
383 * @arg - pointer to the parameter data
385 * Returns 0 on success and a negative value on failure
388 mxlnd_thread_start(int (*fn)(void *arg), void *arg)
391 int i = (int) ((long) arg);
393 atomic_inc(&kmxlnd_data.kmx_nthreads);
394 init_completion(&kmxlnd_data.kmx_completions[i]);
396 pid = kernel_thread (fn, arg, 0);
398 CERROR("kernel_thread() failed with %d\n", pid);
399 atomic_dec(&kmxlnd_data.kmx_nthreads);
405 * mxlnd_thread_stop - decrement thread counter
407 * The thread returns 0 when it detects shutdown.
408 * We are simply decrementing the thread counter.
411 mxlnd_thread_stop(long id)
414 atomic_dec (&kmxlnd_data.kmx_nthreads);
415 complete(&kmxlnd_data.kmx_completions[i]);
419 * mxlnd_shutdown - stop IO, clean up state
420 * @ni - LNET interface handle
422 * No calls to the LND should be made after calling this function.
425 mxlnd_shutdown (lnet_ni_t *ni)
428 int nthreads = MXLND_NDAEMONS
429 + *kmxlnd_tunables.kmx_n_waitd;
431 LASSERT (ni == kmxlnd_data.kmx_ni);
432 LASSERT (ni->ni_data == &kmxlnd_data);
433 CDEBUG(D_NET, "in shutdown()\n");
435 CDEBUG(D_MALLOC, "before MXLND cleanup: libcfs_kmemory %d "
436 "kmx_mem_used %ld\n", atomic_read (&libcfs_kmemory),
437 kmxlnd_data.kmx_mem_used);
440 CDEBUG(D_NET, "setting shutdown = 1\n");
441 atomic_set(&kmxlnd_data.kmx_shutdown, 1);
443 switch (kmxlnd_data.kmx_init) {
447 /* calls write_[un]lock(kmx_global_lock) */
448 mxlnd_del_peer(LNET_NID_ANY);
450 /* wakeup request_waitds */
451 mx_wakeup(kmxlnd_data.kmx_endpt);
452 up(&kmxlnd_data.kmx_tx_queue_sem);
453 up(&kmxlnd_data.kmx_conn_sem);
458 case MXLND_INIT_THREADS:
460 CDEBUG(D_NET, "waiting on threads\n");
461 /* wait for threads to complete */
462 for (i = 0; i < nthreads; i++) {
463 wait_for_completion(&kmxlnd_data.kmx_completions[i]);
465 LASSERT(atomic_read(&kmxlnd_data.kmx_nthreads) == 0);
467 CDEBUG(D_NET, "freeing completions\n");
468 MXLND_FREE(kmxlnd_data.kmx_completions,
469 nthreads * sizeof(struct completion));
475 CDEBUG(D_NET, "stopping mx\n");
477 /* no peers left, close the endpoint */
478 mx_close_endpoint(kmxlnd_data.kmx_endpt);
485 CDEBUG(D_NET, "freeing txs\n");
487 /* free all txs and associated pages */
492 case MXLND_INIT_DATA:
494 CDEBUG(D_NET, "freeing peers\n");
496 /* peers should be gone, but check again */
499 /* conn zombies should be gone, but check again */
500 mxlnd_free_conn_zombies();
504 case MXLND_INIT_NOTHING:
507 CDEBUG(D_NET, "shutdown complete\n");
509 CDEBUG(D_MALLOC, "after MXLND cleanup: libcfs_kmemory %d "
510 "kmx_mem_used %ld\n", atomic_read (&libcfs_kmemory),
511 kmxlnd_data.kmx_mem_used);
513 kmxlnd_data.kmx_init = MXLND_INIT_NOTHING;
519 * mxlnd_startup - initialize state, open an endpoint, start IO
520 * @ni - LNET interface handle
522 * Initialize state, open an endpoint, start monitoring threads.
523 * Should only be called once.
526 mxlnd_startup (lnet_ni_t *ni)
530 int nthreads = MXLND_NDAEMONS /* tx_queued, timeoutd, connd */
531 + *kmxlnd_tunables.kmx_n_waitd;
534 LASSERT (ni->ni_lnd == &the_kmxlnd);
536 if (kmxlnd_data.kmx_init != MXLND_INIT_NOTHING) {
537 CERROR("Only 1 instance supported\n");
540 CDEBUG(D_MALLOC, "before MXLND startup: libcfs_kmemory %d "
541 "kmx_mem_used %ld\n", atomic_read (&libcfs_kmemory),
542 kmxlnd_data.kmx_mem_used);
544 ni->ni_maxtxcredits = MXLND_TX_MSGS();
545 ni->ni_peertxcredits = *kmxlnd_tunables.kmx_peercredits;
546 if (ni->ni_maxtxcredits < ni->ni_peertxcredits)
547 ni->ni_maxtxcredits = ni->ni_peertxcredits;
550 memset (&kmxlnd_data, 0, sizeof (kmxlnd_data));
552 kmxlnd_data.kmx_ni = ni;
553 ni->ni_data = &kmxlnd_data;
555 do_gettimeofday(&tv);
556 kmxlnd_data.kmx_incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec;
557 CDEBUG(D_NET, "my incarnation is %llu\n", kmxlnd_data.kmx_incarnation);
559 rwlock_init (&kmxlnd_data.kmx_global_lock);
560 spin_lock_init (&kmxlnd_data.kmx_mem_lock);
562 INIT_LIST_HEAD (&kmxlnd_data.kmx_conn_reqs);
563 INIT_LIST_HEAD (&kmxlnd_data.kmx_conn_zombies);
564 INIT_LIST_HEAD (&kmxlnd_data.kmx_orphan_msgs);
565 spin_lock_init (&kmxlnd_data.kmx_conn_lock);
566 sema_init(&kmxlnd_data.kmx_conn_sem, 0);
568 for (i = 0; i < MXLND_HASH_SIZE; i++) {
569 INIT_LIST_HEAD (&kmxlnd_data.kmx_peers[i]);
572 INIT_LIST_HEAD (&kmxlnd_data.kmx_tx_idle);
573 spin_lock_init (&kmxlnd_data.kmx_tx_idle_lock);
574 kmxlnd_data.kmx_tx_next_cookie = 1;
575 INIT_LIST_HEAD (&kmxlnd_data.kmx_tx_queue);
576 spin_lock_init (&kmxlnd_data.kmx_tx_queue_lock);
577 sema_init(&kmxlnd_data.kmx_tx_queue_sem, 0);
579 kmxlnd_data.kmx_init = MXLND_INIT_DATA;
580 /*****************************************************/
582 ret = mxlnd_init_txs();
584 CERROR("Can't alloc tx descs: %d\n", ret);
587 kmxlnd_data.kmx_init = MXLND_INIT_TXS;
588 /*****************************************************/
590 ret = mxlnd_init_mx(ni);
592 CERROR("Can't init mx\n");
596 kmxlnd_data.kmx_init = MXLND_INIT_MX;
597 /*****************************************************/
601 MXLND_ALLOC(kmxlnd_data.kmx_completions,
602 nthreads * sizeof(struct completion));
603 if (kmxlnd_data.kmx_completions == NULL) {
604 CERROR("failed to alloc kmxlnd_data.kmx_completions\n");
607 memset(kmxlnd_data.kmx_completions, 0,
608 nthreads * sizeof(struct completion));
610 CDEBUG(D_NET, "using %d %s in mx_wait_any()\n",
611 *kmxlnd_tunables.kmx_n_waitd,
612 *kmxlnd_tunables.kmx_n_waitd == 1 ? "thread" : "threads");
614 for (i = 0; i < *kmxlnd_tunables.kmx_n_waitd; i++) {
615 ret = mxlnd_thread_start(mxlnd_request_waitd, (void*)((long)i));
617 CERROR("Starting mxlnd_request_waitd[%d] failed with %d\n", i, ret);
618 atomic_set(&kmxlnd_data.kmx_shutdown, 1);
619 mx_wakeup(kmxlnd_data.kmx_endpt);
620 for (--i; i >= 0; i--) {
621 wait_for_completion(&kmxlnd_data.kmx_completions[i]);
623 LASSERT(atomic_read(&kmxlnd_data.kmx_nthreads) == 0);
624 MXLND_FREE(kmxlnd_data.kmx_completions,
625 nthreads * sizeof(struct completion));
630 ret = mxlnd_thread_start(mxlnd_tx_queued, (void*)((long)i++));
632 CERROR("Starting mxlnd_tx_queued failed with %d\n", ret);
633 atomic_set(&kmxlnd_data.kmx_shutdown, 1);
634 mx_wakeup(kmxlnd_data.kmx_endpt);
635 for (--i; i >= 0; i--) {
636 wait_for_completion(&kmxlnd_data.kmx_completions[i]);
638 LASSERT(atomic_read(&kmxlnd_data.kmx_nthreads) == 0);
639 MXLND_FREE(kmxlnd_data.kmx_completions,
640 nthreads * sizeof(struct completion));
643 ret = mxlnd_thread_start(mxlnd_timeoutd, (void*)((long)i++));
645 CERROR("Starting mxlnd_timeoutd failed with %d\n", ret);
646 atomic_set(&kmxlnd_data.kmx_shutdown, 1);
647 mx_wakeup(kmxlnd_data.kmx_endpt);
648 up(&kmxlnd_data.kmx_tx_queue_sem);
649 for (--i; i >= 0; i--) {
650 wait_for_completion(&kmxlnd_data.kmx_completions[i]);
652 LASSERT(atomic_read(&kmxlnd_data.kmx_nthreads) == 0);
653 MXLND_FREE(kmxlnd_data.kmx_completions,
654 nthreads * sizeof(struct completion));
657 ret = mxlnd_thread_start(mxlnd_connd, (void*)((long)i++));
659 CERROR("Starting mxlnd_connd failed with %d\n", ret);
660 atomic_set(&kmxlnd_data.kmx_shutdown, 1);
661 mx_wakeup(kmxlnd_data.kmx_endpt);
662 up(&kmxlnd_data.kmx_tx_queue_sem);
663 for (--i; i >= 0; i--) {
664 wait_for_completion(&kmxlnd_data.kmx_completions[i]);
666 LASSERT(atomic_read(&kmxlnd_data.kmx_nthreads) == 0);
667 MXLND_FREE(kmxlnd_data.kmx_completions,
668 nthreads * sizeof(struct completion));
672 kmxlnd_data.kmx_init = MXLND_INIT_THREADS;
673 /*****************************************************/
675 kmxlnd_data.kmx_init = MXLND_INIT_ALL;
676 CDEBUG(D_MALLOC, "startup complete (kmx_mem_used %ld)\n", kmxlnd_data.kmx_mem_used);
680 CERROR("mxlnd_startup failed\n");
685 static int mxlnd_init(void)
687 lnet_register_lnd(&the_kmxlnd);
691 static void mxlnd_exit(void)
693 lnet_unregister_lnd(&the_kmxlnd);
697 module_init(mxlnd_init);
698 module_exit(mxlnd_exit);
700 MODULE_LICENSE("GPL");
701 MODULE_AUTHOR("Myricom, Inc. - help@myri.com");
702 MODULE_DESCRIPTION("Kernel MyrinetExpress LND");
703 MODULE_VERSION("0.6.0");