4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
27 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Use is subject to license terms.
30 * Copyright (c) 2012, Intel Corporation.
32 * Copyright (C) 2006 Myricom, Inc.
35 * This file is part of Lustre, http://www.lustre.org/
36 * Lustre is a trademark of Sun Microsystems, Inc.
38 * lnet/klnds/mxlnd/mxlnd.c
40 * Author: Eric Barton <eric@bartonsoftware.com>
41 * Author: Scott Atchley <atchley at myri.com>
48 .lnd_startup = mxlnd_startup,
49 .lnd_shutdown = mxlnd_shutdown,
51 .lnd_send = mxlnd_send,
52 .lnd_recv = mxlnd_recv,
55 kmx_data_t kmxlnd_data;
58 mxlnd_free_pages(kmx_pages_t *p)
60 int npages = p->mxg_npages;
63 CDEBUG(D_MALLOC, "freeing %d pages\n", npages);
65 for (i = 0; i < npages; i++) {
66 if (p->mxg_pages[i] != NULL) {
67 __free_page(p->mxg_pages[i]);
68 spin_lock(&kmxlnd_data.kmx_mem_lock);
69 kmxlnd_data.kmx_mem_used -= PAGE_SIZE;
70 spin_unlock(&kmxlnd_data.kmx_mem_lock);
74 MXLND_FREE(p, offsetof(kmx_pages_t, mxg_pages[npages]));
78 mxlnd_alloc_pages(kmx_pages_t **pp, int npages)
80 kmx_pages_t *p = NULL;
83 CDEBUG(D_MALLOC, "allocing %d pages\n", npages);
85 MXLND_ALLOC(p, offsetof(kmx_pages_t, mxg_pages[npages]));
87 CERROR("Can't allocate descriptor for %d pages\n", npages);
91 memset(p, 0, offsetof(kmx_pages_t, mxg_pages[npages]));
92 p->mxg_npages = npages;
94 for (i = 0; i < npages; i++) {
95 p->mxg_pages[i] = alloc_page(GFP_KERNEL);
96 if (p->mxg_pages[i] == NULL) {
97 CERROR("Can't allocate page %d of %d\n", i, npages);
101 spin_lock(&kmxlnd_data.kmx_mem_lock);
102 kmxlnd_data.kmx_mem_used += PAGE_SIZE;
103 spin_unlock(&kmxlnd_data.kmx_mem_lock);
111 * mxlnd_ctx_init - reset ctx struct to the default values
112 * @ctx - a kmx_ctx pointer
115 mxlnd_ctx_init(kmx_ctx_t *ctx)
117 if (ctx == NULL) return;
119 /* do not change mxc_type */
120 ctx->mxc_incarnation = 0;
121 ctx->mxc_deadline = 0;
122 ctx->mxc_state = MXLND_CTX_IDLE;
123 if (!cfs_list_empty(&ctx->mxc_list))
124 cfs_list_del_init(&ctx->mxc_list);
125 /* ignore mxc_rx_list */
126 if (ctx->mxc_type == MXLND_REQ_TX) {
128 ctx->mxc_peer = NULL;
129 ctx->mxc_conn = NULL;
132 ctx->mxc_lntmsg[0] = NULL;
133 ctx->mxc_lntmsg[1] = NULL;
134 ctx->mxc_msg_type = 0;
135 ctx->mxc_cookie = 0LL;
136 ctx->mxc_match = 0LL;
137 /* ctx->mxc_seg.segment_ptr points to backing page */
138 ctx->mxc_seg.segment_length = 0;
139 if (ctx->mxc_seg_list != NULL) {
140 LASSERT(ctx->mxc_nseg > 0);
141 MXLND_FREE(ctx->mxc_seg_list, ctx->mxc_nseg * sizeof(mx_ksegment_t));
143 ctx->mxc_seg_list = NULL;
146 memset(&ctx->mxc_mxreq, 0, sizeof(mx_request_t));
147 memset(&ctx->mxc_status, 0, sizeof(mx_status_t));
152 ctx->mxc_msg->mxm_type = 0;
153 ctx->mxc_msg->mxm_credits = 0;
154 ctx->mxc_msg->mxm_nob = 0;
160 * mxlnd_free_txs - free kmx_txs and associated pages
162 * Called from mxlnd_shutdown()
168 kmx_ctx_t *tx = NULL;
170 if (kmxlnd_data.kmx_tx_pages) {
171 for (i = 0; i < MXLND_TX_MSGS(); i++) {
172 tx = &kmxlnd_data.kmx_txs[i];
173 if (tx->mxc_seg_list != NULL) {
174 LASSERT(tx->mxc_nseg > 0);
175 MXLND_FREE(tx->mxc_seg_list,
177 sizeof(*tx->mxc_seg_list));
180 MXLND_FREE(kmxlnd_data.kmx_txs,
181 MXLND_TX_MSGS() * sizeof(kmx_ctx_t));
182 mxlnd_free_pages(kmxlnd_data.kmx_tx_pages);
189 * mxlnd_init_txs - allocate tx descriptors then stash on txs and idle tx lists
191 * Called from mxlnd_startup()
192 * returns 0 on success, else -ENOMEM
202 kmx_ctx_t *tx = NULL;
203 kmx_pages_t *pages = NULL;
204 struct page *page = NULL;
206 /* pre-mapped messages are not bigger than 1 page */
207 CLASSERT(MXLND_MSG_SIZE <= PAGE_SIZE);
209 /* No fancy arithmetic when we do the buffer calculations */
210 CLASSERT (PAGE_SIZE % MXLND_MSG_SIZE == 0);
212 ret = mxlnd_alloc_pages(&pages, MXLND_TX_MSG_PAGES());
214 CERROR("Can't allocate tx pages\n");
217 kmxlnd_data.kmx_tx_pages = pages;
219 MXLND_ALLOC(kmxlnd_data.kmx_txs, MXLND_TX_MSGS() * sizeof(kmx_ctx_t));
220 if (&kmxlnd_data.kmx_txs == NULL) {
221 CERROR("Can't allocate %d tx descriptors\n", MXLND_TX_MSGS());
222 mxlnd_free_pages(pages);
226 memset(kmxlnd_data.kmx_txs, 0, MXLND_TX_MSGS() * sizeof(kmx_ctx_t));
228 for (i = 0; i < MXLND_TX_MSGS(); i++) {
230 tx = &kmxlnd_data.kmx_txs[i];
231 tx->mxc_type = MXLND_REQ_TX;
233 CFS_INIT_LIST_HEAD(&tx->mxc_list);
235 /* map mxc_msg to page */
236 page = pages->mxg_pages[ipage];
237 addr = page_address(page);
238 LASSERT(addr != NULL);
239 tx->mxc_msg = (kmx_msg_t *)(addr + offset);
240 tx->mxc_seg.segment_ptr = MX_PA_TO_U64(virt_to_phys(tx->mxc_msg));
244 offset += MXLND_MSG_SIZE;
245 LASSERT (offset <= PAGE_SIZE);
247 if (offset == PAGE_SIZE) {
250 LASSERT (ipage <= MXLND_TX_MSG_PAGES());
253 /* in startup(), no locks required */
254 cfs_list_add_tail(&tx->mxc_list, &kmxlnd_data.kmx_tx_idle);
261 * mxlnd_free_peers - free peers
263 * Called from mxlnd_shutdown()
266 mxlnd_free_peers(void)
270 kmx_peer_t *peer = NULL;
271 kmx_peer_t *next = NULL;
273 for (i = 0; i < MXLND_HASH_SIZE; i++) {
274 cfs_list_for_each_entry_safe(peer, next,
275 &kmxlnd_data.kmx_peers[i],
277 cfs_list_del_init(&peer->mxp_list);
278 if (peer->mxp_conn) mxlnd_conn_decref(peer->mxp_conn);
279 mxlnd_peer_decref(peer);
283 CDEBUG(D_NET, "%s: freed %d peers\n", __func__, count);
287 * mxlnd_init_mx - open the endpoint, set our ID, register the EAGER callback
288 * @ni - the network interface
290 * Returns 0 on success, -1 on failure
293 mxlnd_init_mx(lnet_ni_t *ni)
297 u32 board = *kmxlnd_tunables.kmx_board;
298 u32 ep_id = *kmxlnd_tunables.kmx_ep_id;
306 if (mxret != MX_SUCCESS) {
307 CERROR("mx_init() failed with %s (%d)\n", mx_strerror(mxret), mxret);
311 if (ni->ni_interfaces[0] != NULL) {
312 /* Use the IPoMX interface specified in 'networks=' */
314 CLASSERT (LNET_MAX_INTERFACES > 1);
315 if (ni->ni_interfaces[1] != NULL) {
316 CERROR("Multiple interfaces not supported\n");
317 goto failed_with_init;
320 ifname = ni->ni_interfaces[0];
322 ifname = *kmxlnd_tunables.kmx_default_ipif;
325 ret = libcfs_ipif_query(ifname, &if_up, &ip, &netmask);
327 CERROR("Can't query IPoMX interface %s: %d\n",
329 goto failed_with_init;
333 CERROR("Can't query IPoMX interface %s: it's down\n",
335 goto failed_with_init;
338 mxret = mx_open_endpoint(board, ep_id, MXLND_MSG_MAGIC,
339 NULL, 0, &kmxlnd_data.kmx_endpt);
340 if (mxret != MX_SUCCESS) {
341 CERROR("mx_open_endpoint() failed with %d\n", mxret);
342 goto failed_with_init;
345 mx_get_endpoint_addr(kmxlnd_data.kmx_endpt, &kmxlnd_data.kmx_epa);
346 mx_decompose_endpoint_addr(kmxlnd_data.kmx_epa, &nic_id, &ep_id);
347 mxret = mx_connect(kmxlnd_data.kmx_endpt, nic_id, ep_id,
348 MXLND_MSG_MAGIC, MXLND_CONNECT_TIMEOUT/CFS_HZ*1000,
349 &kmxlnd_data.kmx_epa);
350 if (mxret != MX_SUCCESS) {
351 CNETERR("unable to connect to myself (%s)\n", mx_strerror(mxret));
352 goto failed_with_endpoint;
355 ni->ni_nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), ip);
356 CDEBUG(D_NET, "My NID is 0x%llx\n", ni->ni_nid);
358 /* this will catch all unexpected receives. */
359 mxret = mx_register_unexp_handler(kmxlnd_data.kmx_endpt,
360 (mx_unexp_handler_t) mxlnd_unexpected_recv,
362 if (mxret != MX_SUCCESS) {
363 CERROR("mx_register_unexp_callback() failed with %s\n",
365 goto failed_with_endpoint;
367 mxret = mx_set_request_timeout(kmxlnd_data.kmx_endpt, NULL,
368 MXLND_COMM_TIMEOUT/CFS_HZ*1000);
369 if (mxret != MX_SUCCESS) {
370 CERROR("mx_set_request_timeout() failed with %s\n",
372 goto failed_with_endpoint;
376 failed_with_endpoint:
377 mx_close_endpoint(kmxlnd_data.kmx_endpt);
385 * mxlnd_thread_start - spawn a kernel thread with this function
386 * @fn - function pointer
387 * @arg - pointer to the parameter data
388 * @name - name of new thread
390 * Returns 0 on success and a negative value on failure
393 mxlnd_thread_start(int (*fn)(void *arg), void *arg, char *name)
396 int i = (int) ((long) arg);
398 cfs_atomic_inc(&kmxlnd_data.kmx_nthreads);
399 init_completion(&kmxlnd_data.kmx_completions[i]);
401 task = kthread_run(fn, arg, name);
403 CERROR("cfs_create_thread() failed with %d\n", PTR_ERR(task));
404 cfs_atomic_dec(&kmxlnd_data.kmx_nthreads);
406 return PTR_ERR(task);
410 * mxlnd_thread_stop - decrement thread counter
412 * The thread returns 0 when it detects shutdown.
413 * We are simply decrementing the thread counter.
416 mxlnd_thread_stop(long id)
419 cfs_atomic_dec (&kmxlnd_data.kmx_nthreads);
420 complete(&kmxlnd_data.kmx_completions[i]);
424 * mxlnd_shutdown - stop IO, clean up state
425 * @ni - LNET interface handle
427 * No calls to the LND should be made after calling this function.
430 mxlnd_shutdown (lnet_ni_t *ni)
433 int nthreads = MXLND_NDAEMONS
434 + *kmxlnd_tunables.kmx_n_waitd;
436 LASSERT (ni == kmxlnd_data.kmx_ni);
437 LASSERT (ni->ni_data == &kmxlnd_data);
438 CDEBUG(D_NET, "in shutdown()\n");
440 CDEBUG(D_MALLOC, "before MXLND cleanup: libcfs_kmemory %d "
441 "kmx_mem_used %ld\n", cfs_atomic_read(&libcfs_kmemory),
442 kmxlnd_data.kmx_mem_used);
445 CDEBUG(D_NET, "setting shutdown = 1\n");
446 cfs_atomic_set(&kmxlnd_data.kmx_shutdown, 1);
448 switch (kmxlnd_data.kmx_init) {
452 /* calls write_[un]lock(kmx_global_lock) */
453 mxlnd_del_peer(LNET_NID_ANY);
455 /* wakeup request_waitds */
456 mx_wakeup(kmxlnd_data.kmx_endpt);
457 up(&kmxlnd_data.kmx_tx_queue_sem);
458 up(&kmxlnd_data.kmx_conn_sem);
459 mxlnd_sleep(2 * CFS_HZ);
463 case MXLND_INIT_THREADS:
465 CDEBUG(D_NET, "waiting on threads\n");
466 /* wait for threads to complete */
467 for (i = 0; i < nthreads; i++) {
468 wait_for_completion(&kmxlnd_data.kmx_completions[i]);
470 LASSERT(cfs_atomic_read(&kmxlnd_data.kmx_nthreads) == 0);
472 CDEBUG(D_NET, "freeing completions\n");
473 MXLND_FREE(kmxlnd_data.kmx_completions,
474 nthreads * sizeof(struct completion));
480 CDEBUG(D_NET, "stopping mx\n");
482 /* no peers left, close the endpoint */
483 mx_close_endpoint(kmxlnd_data.kmx_endpt);
490 CDEBUG(D_NET, "freeing txs\n");
492 /* free all txs and associated pages */
497 case MXLND_INIT_DATA:
499 CDEBUG(D_NET, "freeing peers\n");
501 /* peers should be gone, but check again */
504 /* conn zombies should be gone, but check again */
505 mxlnd_free_conn_zombies();
509 case MXLND_INIT_NOTHING:
512 CDEBUG(D_NET, "shutdown complete\n");
514 CDEBUG(D_MALLOC, "after MXLND cleanup: libcfs_kmemory %d "
515 "kmx_mem_used %ld\n", cfs_atomic_read(&libcfs_kmemory),
516 kmxlnd_data.kmx_mem_used);
518 kmxlnd_data.kmx_init = MXLND_INIT_NOTHING;
524 * mxlnd_startup - initialize state, open an endpoint, start IO
525 * @ni - LNET interface handle
527 * Initialize state, open an endpoint, start monitoring threads.
528 * Should only be called once.
531 mxlnd_startup (lnet_ni_t *ni)
535 int nthreads = MXLND_NDAEMONS /* tx_queued, timeoutd, connd */
536 + *kmxlnd_tunables.kmx_n_waitd;
539 LASSERT (ni->ni_lnd == &the_kmxlnd);
541 if (kmxlnd_data.kmx_init != MXLND_INIT_NOTHING) {
542 CERROR("Only 1 instance supported\n");
545 CDEBUG(D_MALLOC, "before MXLND startup: libcfs_kmemory %d "
546 "kmx_mem_used %ld\n", cfs_atomic_read(&libcfs_kmemory),
547 kmxlnd_data.kmx_mem_used);
549 ni->ni_maxtxcredits = MXLND_TX_MSGS();
550 ni->ni_peertxcredits = *kmxlnd_tunables.kmx_peercredits;
551 if (ni->ni_maxtxcredits < ni->ni_peertxcredits)
552 ni->ni_maxtxcredits = ni->ni_peertxcredits;
555 memset (&kmxlnd_data, 0, sizeof (kmxlnd_data));
557 kmxlnd_data.kmx_ni = ni;
558 ni->ni_data = &kmxlnd_data;
560 cfs_gettimeofday(&tv);
561 kmxlnd_data.kmx_incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec;
562 CDEBUG(D_NET, "my incarnation is %llu\n", kmxlnd_data.kmx_incarnation);
564 rwlock_init (&kmxlnd_data.kmx_global_lock);
565 spin_lock_init (&kmxlnd_data.kmx_mem_lock);
567 CFS_INIT_LIST_HEAD (&kmxlnd_data.kmx_conn_reqs);
568 CFS_INIT_LIST_HEAD (&kmxlnd_data.kmx_conn_zombies);
569 CFS_INIT_LIST_HEAD (&kmxlnd_data.kmx_orphan_msgs);
570 spin_lock_init (&kmxlnd_data.kmx_conn_lock);
571 sema_init(&kmxlnd_data.kmx_conn_sem, 0);
573 for (i = 0; i < MXLND_HASH_SIZE; i++) {
574 CFS_INIT_LIST_HEAD (&kmxlnd_data.kmx_peers[i]);
577 CFS_INIT_LIST_HEAD (&kmxlnd_data.kmx_tx_idle);
578 spin_lock_init (&kmxlnd_data.kmx_tx_idle_lock);
579 kmxlnd_data.kmx_tx_next_cookie = 1;
580 CFS_INIT_LIST_HEAD (&kmxlnd_data.kmx_tx_queue);
581 spin_lock_init (&kmxlnd_data.kmx_tx_queue_lock);
582 sema_init(&kmxlnd_data.kmx_tx_queue_sem, 0);
584 kmxlnd_data.kmx_init = MXLND_INIT_DATA;
585 /*****************************************************/
587 ret = mxlnd_init_txs();
589 CERROR("Can't alloc tx descs: %d\n", ret);
592 kmxlnd_data.kmx_init = MXLND_INIT_TXS;
593 /*****************************************************/
595 ret = mxlnd_init_mx(ni);
597 CERROR("Can't init mx\n");
601 kmxlnd_data.kmx_init = MXLND_INIT_MX;
602 /*****************************************************/
606 MXLND_ALLOC(kmxlnd_data.kmx_completions,
607 nthreads * sizeof(struct completion));
608 if (kmxlnd_data.kmx_completions == NULL) {
609 CERROR("failed to alloc kmxlnd_data.kmx_completions\n");
612 memset(kmxlnd_data.kmx_completions, 0,
613 nthreads * sizeof(struct completion));
615 CDEBUG(D_NET, "using %d %s in mx_wait_any()\n",
616 *kmxlnd_tunables.kmx_n_waitd,
617 *kmxlnd_tunables.kmx_n_waitd == 1 ? "thread" : "threads");
619 for (i = 0; i < *kmxlnd_tunables.kmx_n_waitd; i++) {
621 memset(name, 0, sizeof(name));
622 snprintf(name, sizeof(name), "mxlnd_request_waitd_%02ld", i);
623 ret = mxlnd_thread_start(mxlnd_request_waitd, (void*)((long)i));
625 CERROR("Starting mxlnd_request_waitd[%d] "
626 "failed with %d\n", i, ret);
627 cfs_atomic_set(&kmxlnd_data.kmx_shutdown, 1);
628 mx_wakeup(kmxlnd_data.kmx_endpt);
629 for (--i; i >= 0; i--) {
630 wait_for_completion(&kmxlnd_data.kmx_completions[i]);
632 LASSERT(cfs_atomic_read(&kmxlnd_data.kmx_nthreads) == 0);
633 MXLND_FREE(kmxlnd_data.kmx_completions,
634 nthreads * sizeof(struct completion));
639 ret = mxlnd_thread_start(mxlnd_tx_queued, (void *)((long)i++),
642 CERROR("Starting mxlnd_tx_queued failed with %d\n", ret);
643 cfs_atomic_set(&kmxlnd_data.kmx_shutdown, 1);
644 mx_wakeup(kmxlnd_data.kmx_endpt);
645 for (--i; i >= 0; i--) {
646 wait_for_completion(&kmxlnd_data.kmx_completions[i]);
648 LASSERT(cfs_atomic_read(&kmxlnd_data.kmx_nthreads) == 0);
649 MXLND_FREE(kmxlnd_data.kmx_completions,
650 nthreads * sizeof(struct completion));
653 ret = mxlnd_thread_start(mxlnd_timeoutd, (void *)((long)i++),
656 CERROR("Starting mxlnd_timeoutd failed with %d\n", ret);
657 cfs_atomic_set(&kmxlnd_data.kmx_shutdown, 1);
658 mx_wakeup(kmxlnd_data.kmx_endpt);
659 up(&kmxlnd_data.kmx_tx_queue_sem);
660 for (--i; i >= 0; i--) {
661 wait_for_completion(&kmxlnd_data.kmx_completions[i]);
663 LASSERT(cfs_atomic_read(&kmxlnd_data.kmx_nthreads) == 0);
664 MXLND_FREE(kmxlnd_data.kmx_completions,
665 nthreads * sizeof(struct completion));
668 ret = mxlnd_thread_start(mxlnd_connd, (void *)((long)i++),
671 CERROR("Starting mxlnd_connd failed with %d\n", ret);
672 cfs_atomic_set(&kmxlnd_data.kmx_shutdown, 1);
673 mx_wakeup(kmxlnd_data.kmx_endpt);
674 up(&kmxlnd_data.kmx_tx_queue_sem);
675 for (--i; i >= 0; i--) {
676 wait_for_completion(&kmxlnd_data.kmx_completions[i]);
678 LASSERT(cfs_atomic_read(&kmxlnd_data.kmx_nthreads) == 0);
679 MXLND_FREE(kmxlnd_data.kmx_completions,
680 nthreads * sizeof(struct completion));
684 kmxlnd_data.kmx_init = MXLND_INIT_THREADS;
685 /*****************************************************/
687 kmxlnd_data.kmx_init = MXLND_INIT_ALL;
688 CDEBUG(D_MALLOC, "startup complete (kmx_mem_used %ld)\n", kmxlnd_data.kmx_mem_used);
692 CERROR("mxlnd_startup failed\n");
697 static int mxlnd_init(void)
699 lnet_register_lnd(&the_kmxlnd);
703 static void mxlnd_exit(void)
705 lnet_unregister_lnd(&the_kmxlnd);
709 module_init(mxlnd_init);
710 module_exit(mxlnd_exit);
712 MODULE_LICENSE("GPL");
713 MODULE_AUTHOR("Myricom, Inc. - help@myri.com");
714 MODULE_DESCRIPTION("Kernel MyrinetExpress LND");
715 MODULE_VERSION("0.6.0");