1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
6 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 only,
10 * as published by the Free Software Foundation.
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License version 2 for more details (a copy is included
16 * in the LICENSE file that accompanied this code).
18 * You should have received a copy of the GNU General Public License
19 * version 2 along with this program; If not, see
20 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
22 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
23 * CA 95054 USA or visit www.sun.com if you need additional information or
29 * Copyright 2008 Sun Microsystems, Inc. All rights reserved
30 * Use is subject to license terms.
33 * This file is part of Lustre, http://www.lustre.org/
34 * Lustre is a trademark of Sun Microsystems, Inc.
37 #define DEBUG_SUBSYSTEM S_LNET
38 #include <lnet/lib-lnet.h>
41 static char *accept = "secure";
42 CFS_MODULE_PARM(accept, "s", charp, 0444,
43 "Accept connections (secure|all|none)");
45 static int accept_port = 988;
46 CFS_MODULE_PARM(accept_port, "i", int, 0444,
47 "Acceptor's port (same on all nodes)");
49 static int accept_backlog = 127;
50 CFS_MODULE_PARM(accept_backlog, "i", int, 0444,
51 "Acceptor's listen backlog");
53 static int accept_timeout = 5;
54 CFS_MODULE_PARM(accept_timeout, "i", int, 0644,
55 "Acceptor's timeout (seconds)");
59 cfs_socket_t *pta_sock;
60 struct semaphore pta_signal;
61 } lnet_acceptor_state;
64 lnet_acceptor_timeout(void)
66 return accept_timeout;
68 EXPORT_SYMBOL(lnet_acceptor_timeout);
71 lnet_acceptor_port(void)
75 EXPORT_SYMBOL(lnet_acceptor_port);
78 lnet_connect_console_error (int rc, lnet_nid_t peer_nid,
79 __u32 peer_ip, int peer_port)
84 CDEBUG(D_NETERROR, "Connection to %s at host %u.%u.%u.%u "
85 "on port %d was refused: "
86 "check that Lustre is running on that node.\n",
87 libcfs_nid2str(peer_nid),
88 HIPQUAD(peer_ip), peer_port);
92 CDEBUG(D_NETERROR, "Connection to %s at host %u.%u.%u.%u "
93 "was unreachable: the network or that node may "
94 "be down, or Lustre may be misconfigured.\n",
95 libcfs_nid2str(peer_nid), HIPQUAD(peer_ip));
98 CDEBUG(D_NETERROR, "Connection to %s at host %u.%u.%u.%u on "
99 "port %d took too long: that node may be hung "
100 "or experiencing high load.\n",
101 libcfs_nid2str(peer_nid),
102 HIPQUAD(peer_ip), peer_port);
105 LCONSOLE_ERROR_MSG(0x11b, "Connection to %s at host %u.%u.%u.%u"
106 " on port %d was reset: "
107 "is it running a compatible version of "
108 "Lustre and is %s one of its NIDs?\n",
109 libcfs_nid2str(peer_nid),
110 HIPQUAD(peer_ip), peer_port,
111 libcfs_nid2str(peer_nid));
114 LCONSOLE_ERROR_MSG(0x11c, "Protocol error connecting to %s at "
115 "host %u.%u.%u.%u on port %d: is it running "
116 "a compatible version of Lustre?\n",
117 libcfs_nid2str(peer_nid),
118 HIPQUAD(peer_ip), peer_port);
121 LCONSOLE_ERROR_MSG(0x11d, "No privileged ports available to "
122 "connect to %s at host %u.%u.%u.%u on port "
123 "%d\n", libcfs_nid2str(peer_nid),
124 HIPQUAD(peer_ip), peer_port);
127 LCONSOLE_ERROR_MSG(0x11e, "Unexpected error %d connecting to %s"
128 " at host %u.%u.%u.%u on port %d\n", rc,
129 libcfs_nid2str(peer_nid),
130 HIPQUAD(peer_ip), peer_port);
134 EXPORT_SYMBOL(lnet_connect_console_error);
137 lnet_connect(cfs_socket_t **sockp, lnet_nid_t peer_nid,
138 __u32 local_ip, __u32 peer_ip, int peer_port)
140 lnet_acceptor_connreq_t cr;
146 CLASSERT (sizeof(cr) <= 16); /* not too big to be on the stack */
148 for (port = LNET_ACCEPTOR_MAX_RESERVED_PORT;
149 port >= LNET_ACCEPTOR_MIN_RESERVED_PORT;
151 /* Iterate through reserved ports. */
153 rc = libcfs_sock_connect(&sock, &fatal,
162 CLASSERT (LNET_PROTO_ACCEPTOR_VERSION == 1);
164 cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
165 cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
166 cr.acr_nid = peer_nid;
168 if (the_lnet.ln_testprotocompat != 0) {
169 /* single-shot proto check */
171 if ((the_lnet.ln_testprotocompat & 4) != 0) {
173 the_lnet.ln_testprotocompat &= ~4;
175 if ((the_lnet.ln_testprotocompat & 8) != 0) {
176 cr.acr_magic = LNET_PROTO_MAGIC;
177 the_lnet.ln_testprotocompat &= ~8;
182 rc = libcfs_sock_write(sock, &cr, sizeof(cr),
195 libcfs_sock_release(sock);
197 lnet_connect_console_error(rc, peer_nid, peer_ip, peer_port);
200 EXPORT_SYMBOL(lnet_connect);
203 lnet_accept_magic(__u32 magic, __u32 constant)
205 return (magic == constant ||
206 magic == __swab32(constant));
210 lnet_accept(cfs_socket_t *sock, __u32 magic)
212 lnet_acceptor_connreq_t cr;
220 LASSERT (sizeof(cr) <= 16); /* not too big for the stack */
222 rc = libcfs_sock_getaddr(sock, 1, &peer_ip, &peer_port);
223 LASSERT (rc == 0); /* we succeeded before */
225 if (!lnet_accept_magic(magic, LNET_PROTO_ACCEPTOR_MAGIC)) {
227 if (lnet_accept_magic(magic, LNET_PROTO_MAGIC)) {
228 /* future version compatibility!
229 * When LNET unifies protocols over all LNDs, the first
230 * thing sent will be a version query. I send back
231 * LNET_PROTO_ACCEPTOR_MAGIC to tell her I'm "old" */
233 memset (&cr, 0, sizeof(cr));
234 cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
235 cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
236 rc = libcfs_sock_write(sock, &cr, sizeof(cr),
240 CERROR("Error sending magic+version in response"
241 "to LNET magic from %u.%u.%u.%u: %d\n",
242 HIPQUAD(peer_ip), rc);
246 if (magic == le32_to_cpu(LNET_PROTO_TCP_MAGIC))
247 str = "'old' socknal/tcpnal";
248 else if (lnet_accept_magic(magic, LNET_PROTO_RA_MAGIC))
250 else if (lnet_accept_magic(magic, LNET_PROTO_OPENIB_MAGIC))
251 str = "'old' openibnal";
253 str = "unrecognised";
255 LCONSOLE_ERROR_MSG(0x11f, "Refusing connection from %u.%u.%u.%u"
256 " magic %08x: %s acceptor protocol\n",
257 HIPQUAD(peer_ip), magic, str);
261 flip = (magic != LNET_PROTO_ACCEPTOR_MAGIC);
263 rc = libcfs_sock_read(sock, &cr.acr_version,
264 sizeof(cr.acr_version),
267 CERROR("Error %d reading connection request version from "
268 "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
273 __swab32s(&cr.acr_version);
275 if (cr.acr_version != LNET_PROTO_ACCEPTOR_VERSION) {
276 /* future version compatibility!
277 * An acceptor-specific protocol rev will first send a version
278 * query. I send back my current version to tell her I'm
280 int peer_version = cr.acr_version;
282 memset (&cr, 0, sizeof(cr));
283 cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
284 cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
286 rc = libcfs_sock_write(sock, &cr, sizeof(cr),
290 CERROR("Error sending magic+version in response"
291 "to version %d from %u.%u.%u.%u: %d\n",
292 peer_version, HIPQUAD(peer_ip), rc);
296 rc = libcfs_sock_read(sock, &cr.acr_nid,
298 offsetof(lnet_acceptor_connreq_t, acr_nid),
301 CERROR("Error %d reading connection request from "
302 "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
307 __swab64s(&cr.acr_nid);
309 ni = lnet_net2ni(LNET_NIDNET(cr.acr_nid));
310 if (ni == NULL || /* no matching net */
311 ni->ni_nid != cr.acr_nid) { /* right NET, wrong NID! */
314 LCONSOLE_ERROR_MSG(0x120, "Refusing connection from %u.%u.%u.%u"
315 " for %s: No matching NI\n",
316 HIPQUAD(peer_ip), libcfs_nid2str(cr.acr_nid));
320 if (ni->ni_lnd->lnd_accept == NULL) {
321 /* This catches a request for the loopback LND */
323 LCONSOLE_ERROR_MSG(0x121, "Refusing connection from %u.%u.%u.%u"
324 " for %s: NI doesn not accept IP connections\n",
325 HIPQUAD(peer_ip), libcfs_nid2str(cr.acr_nid));
329 CDEBUG(D_NET, "Accept %s from %u.%u.%u.%u\n",
330 libcfs_nid2str(cr.acr_nid), HIPQUAD(peer_ip));
332 rc = ni->ni_lnd->lnd_accept(ni, sock);
338 lnet_acceptor(void *arg)
341 cfs_socket_t *newsock;
346 int secure = (int)((long_ptr_t)arg);
348 LASSERT (lnet_acceptor_state.pta_sock == NULL);
350 snprintf(name, sizeof(name), "acceptor_%03d", accept_port);
354 rc = libcfs_sock_listen(&lnet_acceptor_state.pta_sock,
355 0, accept_port, accept_backlog);
357 if (rc == -EADDRINUSE)
358 LCONSOLE_ERROR_MSG(0x122, "Can't start acceptor on port"
359 " %d: port already in use\n",
362 LCONSOLE_ERROR_MSG(0x123, "Can't start acceptor on port "
363 "%d: unexpected error %d\n",
366 lnet_acceptor_state.pta_sock = NULL;
368 LCONSOLE(0, "Accept %s, port %d\n", accept, accept_port);
371 /* set init status and unblock parent */
372 lnet_acceptor_state.pta_shutdown = rc;
373 mutex_up(&lnet_acceptor_state.pta_signal);
378 while (!lnet_acceptor_state.pta_shutdown) {
380 rc = libcfs_sock_accept(&newsock, lnet_acceptor_state.pta_sock);
383 CWARN("Accept error %d: pausing...\n", rc);
384 cfs_pause(cfs_time_seconds(1));
389 rc = libcfs_sock_getaddr(newsock, 1, &peer_ip, &peer_port);
391 CERROR("Can't determine new connection's address\n");
395 if (secure && peer_port > LNET_ACCEPTOR_MAX_RESERVED_PORT) {
396 CERROR("Refusing connection from %u.%u.%u.%u: "
397 "insecure port %d\n",
398 HIPQUAD(peer_ip), peer_port);
402 rc = libcfs_sock_read(newsock, &magic, sizeof(magic),
405 CERROR("Error %d reading connection request from "
406 "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
410 rc = lnet_accept(newsock, magic);
417 libcfs_sock_release(newsock);
420 libcfs_sock_release(lnet_acceptor_state.pta_sock);
421 lnet_acceptor_state.pta_sock = NULL;
423 LCONSOLE(0, "Acceptor stopping\n");
425 /* unblock lnet_acceptor_stop() */
426 mutex_up(&lnet_acceptor_state.pta_signal);
431 lnet_acceptor_start(void)
436 LASSERT (lnet_acceptor_state.pta_sock == NULL);
437 init_mutex_locked(&lnet_acceptor_state.pta_signal);
439 if (!strcmp(accept, "secure")) {
441 } else if (!strcmp(accept, "all")) {
443 } else if (!strcmp(accept, "none")) {
446 LCONSOLE_ERROR_MSG(0x124, "Can't parse 'accept=\"%s\"'\n",
451 if (lnet_count_acceptor_nis() == 0) /* not required */
454 pid = cfs_kernel_thread(lnet_acceptor, (void *)(ulong_ptr_t)secure, 0);
456 CERROR("Can't start acceptor thread: %ld\n", pid);
460 mutex_down(&lnet_acceptor_state.pta_signal); /* wait for acceptor to startup */
462 if (!lnet_acceptor_state.pta_shutdown) {
464 LASSERT (lnet_acceptor_state.pta_sock != NULL);
468 LASSERT (lnet_acceptor_state.pta_sock == NULL);
473 lnet_acceptor_stop(void)
475 if (lnet_acceptor_state.pta_sock == NULL) /* not running */
478 lnet_acceptor_state.pta_shutdown = 1;
479 libcfs_sock_abort_accept(lnet_acceptor_state.pta_sock);
481 /* block until acceptor signals exit */
482 mutex_down(&lnet_acceptor_state.pta_signal);
485 #else /* __KERNEL__ */
486 #ifdef HAVE_LIBPTHREAD
488 static char *accept_type;
489 static int accept_port = 988;
490 static int accept_backlog;
491 static int accept_timeout;
496 struct cfs_completion pta_completion;
497 } lnet_acceptor_state;
500 lnet_acceptor_port(void)
506 lnet_parse_int_tunable(int *value, char *name, int dflt)
508 char *env = getenv(name);
516 *value = strtoull(env, &end, 0);
520 CERROR("Can't parse tunable %s=%s\n", name, env);
525 lnet_parse_string_tunable(char **value, char *name, char *dflt)
527 char *env = getenv(name);
538 lnet_acceptor_get_tunables()
541 rc = lnet_parse_string_tunable(&accept_type, "LNET_ACCEPT", "secure");
546 rc = lnet_parse_int_tunable(&accept_port, "LNET_ACCEPT_PORT", 988);
551 rc = lnet_parse_int_tunable(&accept_backlog, "LNET_ACCEPT_BACKLOG", 127);
556 rc = lnet_parse_int_tunable(&accept_timeout, "LNET_ACCEPT_TIMEOUT", 5);
561 CDEBUG(D_NET, "accept_type = %s\n", accept_type);
562 CDEBUG(D_NET, "accept_port = %d\n", accept_port);
563 CDEBUG(D_NET, "accept_backlog = %d\n", accept_backlog);
564 CDEBUG(D_NET, "accept_timeout = %d\n", accept_timeout);
569 lnet_accept_magic(__u32 magic, __u32 constant)
571 return (magic == constant ||
572 magic == __swab32(constant));
575 /* user-land lnet_accept() isn't used by any LND's directly. So, we don't
576 * do it visible outside acceptor.c and we can change its prototype
579 lnet_accept(int sock, __u32 magic, __u32 peer_ip, int peer_port)
582 lnet_acceptor_connreq_t cr;
585 if (!lnet_accept_magic(magic, LNET_PROTO_ACCEPTOR_MAGIC)) {
586 LCONSOLE_ERROR("Refusing connection from %u.%u.%u.%u magic %08x: "
587 "unsupported acceptor protocol\n",
588 HIPQUAD(peer_ip), magic);
592 flip = (magic != LNET_PROTO_ACCEPTOR_MAGIC);
594 rc = libcfs_sock_read(sock, &cr.acr_version,
595 sizeof(cr.acr_version),
598 CERROR("Error %d reading connection request version from "
599 "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
604 __swab32s(&cr.acr_version);
606 if (cr.acr_version != LNET_PROTO_ACCEPTOR_VERSION)
609 rc = libcfs_sock_read(sock, &cr.acr_nid,
611 offsetof(lnet_acceptor_connreq_t, acr_nid),
614 CERROR("Error %d reading connection request from "
615 "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
620 __swab64s(&cr.acr_nid);
622 ni = lnet_net2ni(LNET_NIDNET(cr.acr_nid));
624 if (ni == NULL || /* no matching net */
625 ni->ni_nid != cr.acr_nid) { /* right NET, wrong NID! */
628 LCONSOLE_ERROR("Refusing connection from %u.%u.%u.%u for %s: "
630 HIPQUAD(peer_ip), libcfs_nid2str(cr.acr_nid));
634 if (ni->ni_lnd->lnd_accept == NULL) {
636 LCONSOLE_ERROR("Refusing connection from %u.%u.%u.%u for %s: "
637 " NI doesn not accept IP connections\n",
638 HIPQUAD(peer_ip), libcfs_nid2str(cr.acr_nid));
642 CDEBUG(D_NET, "Accept %s from %u.%u.%u.%u\n",
643 libcfs_nid2str(cr.acr_nid), HIPQUAD(peer_ip));
645 rc = ni->ni_lnd->lnd_accept(ni, sock);
652 lnet_acceptor(void *arg)
655 int secure = (int)((unsigned long)arg);
662 snprintf(name, sizeof(name), "acceptor_%03d", accept_port);
666 rc = libcfs_sock_listen(&lnet_acceptor_state.pta_sock,
667 0, accept_port, accept_backlog);
669 if (rc == -EADDRINUSE)
670 LCONSOLE_ERROR("Can't start acceptor on port %d: "
671 "port already in use\n",
674 LCONSOLE_ERROR("Can't start acceptor on port %d: "
675 "unexpected error %d\n",
679 LCONSOLE(0, "Accept %s, port %d\n", accept_type, accept_port);
682 /* set init status and unblock parent */
683 lnet_acceptor_state.pta_shutdown = rc;
684 cfs_complete(&lnet_acceptor_state.pta_completion);
689 while (!lnet_acceptor_state.pta_shutdown) {
691 rc = libcfs_sock_accept(&newsock, lnet_acceptor_state.pta_sock,
692 &peer_ip, &peer_port);
696 /* maybe we're waken up with libcfs_sock_abort_accept() */
697 if (lnet_acceptor_state.pta_shutdown) {
698 libcfs_sock_release(newsock);
702 if (secure && peer_port > LNET_ACCEPTOR_MAX_RESERVED_PORT) {
703 CERROR("Refusing connection from %u.%u.%u.%u: "
704 "insecure port %d\n",
705 HIPQUAD(peer_ip), peer_port);
709 rc = libcfs_sock_read(newsock, &magic, sizeof(magic),
712 CERROR("Error %d reading connection request from "
713 "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
717 rc = lnet_accept(newsock, magic, peer_ip, peer_port);
724 libcfs_sock_release(newsock);
727 libcfs_sock_release(lnet_acceptor_state.pta_sock);
728 LCONSOLE(0,"Acceptor stopping\n");
730 /* unblock lnet_acceptor_stop() */
731 cfs_complete(&lnet_acceptor_state.pta_completion);
736 static int skip_waiting_for_completion;
739 lnet_acceptor_start(void)
744 rc = lnet_acceptor_get_tunables();
748 /* Do nothing if we're liblustre clients */
749 if ((the_lnet.ln_pid & LNET_PID_USERFLAG) != 0)
752 cfs_init_completion(&lnet_acceptor_state.pta_completion);
754 if (!strcmp(accept_type, "secure")) {
756 } else if (!strcmp(accept_type, "all")) {
758 } else if (!strcmp(accept_type, "none")) {
759 skip_waiting_for_completion = 1;
762 LCONSOLE_ERROR ("Can't parse 'accept_type=\"%s\"'\n", accept_type);
763 cfs_fini_completion(&lnet_acceptor_state.pta_completion);
767 if (lnet_count_acceptor_nis() == 0) { /* not required */
768 skip_waiting_for_completion = 1;
772 rc = cfs_create_thread(lnet_acceptor, (void *)secure);
774 CERROR("Can't start acceptor thread: %d\n", rc);
775 cfs_fini_completion(&lnet_acceptor_state.pta_completion);
779 /* wait for acceptor to startup */
780 cfs_wait_for_completion(&lnet_acceptor_state.pta_completion);
782 if (!lnet_acceptor_state.pta_shutdown)
785 cfs_fini_completion(&lnet_acceptor_state.pta_completion);
790 lnet_acceptor_stop(void)
792 /* Do nothing if we're liblustre clients */
793 if ((the_lnet.ln_pid & LNET_PID_USERFLAG) != 0)
796 if (!skip_waiting_for_completion) {
797 lnet_acceptor_state.pta_shutdown = 1;
798 libcfs_sock_abort_accept(accept_port);
800 /* block until acceptor signals exit */
801 cfs_wait_for_completion(&lnet_acceptor_state.pta_completion);
804 cfs_fini_completion(&lnet_acceptor_state.pta_completion);
808 lnet_acceptor_start(void)
814 lnet_acceptor_stop(void)
817 #endif /* !HAVE_LIBPTHREAD */
818 #endif /* !__KERNEL__ */