X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lnet%2Flnet%2Facceptor.c;h=908623730a3f324e5e651c04d94781620d2610b9;hp=cf0a56adde20345d11382d98ba166eb8f9492e33;hb=26adc2e13a877507b483b2d3c3e873fbd13d2596;hpb=d2b8a0efaa4b5faea675bd4bd4bfe1f80dad4011 diff --git a/lnet/lnet/acceptor.c b/lnet/lnet/acceptor.c index cf0a56a..9086237 100644 --- a/lnet/lnet/acceptor.c +++ b/lnet/lnet/acceptor.c @@ -1,6 +1,4 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * +/* * GPL HEADER START * * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -28,6 +26,8 @@ /* * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. + * + * Copyright (c) 2011, 2015, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -35,23 +35,25 @@ */ #define DEBUG_SUBSYSTEM S_LNET -#include -#if defined(__KERNEL__) || defined(HAVE_LIBPTHREAD) +#ifdef HAVE_COMPAT_RDMA +#include +#endif +#include +#include +#include static int accept_port = 988; static int accept_backlog = 127; static int accept_timeout = 5; -struct { - int pta_shutdown; - cfs_socket_t *pta_sock; -#ifdef __KERNEL__ - cfs_completion_t pta_signal; -#else - cfs_mt_completion_t pta_signal; -#endif -} lnet_acceptor_state; +static struct { + int pta_shutdown; + struct socket *pta_sock; + struct completion pta_signal; +} lnet_acceptor_state = { + .pta_shutdown = 1 +}; int lnet_acceptor_port(void) @@ -66,14 +68,6 @@ lnet_accept_magic(__u32 magic, __u32 constant) magic == __swab32(constant)); } -#ifdef __KERNEL__ - -#define cfs_mt_init_completion(c) cfs_init_completion(c) -#define cfs_mt_wait_for_completion(c) cfs_wait_for_completion(c) -#define cfs_mt_complete(c) cfs_complete(c) -#define cfs_mt_fini_completion(c) cfs_fini_completion(c) -#define cfs_create_thread(func, a) cfs_kernel_thread(func, a, 0) - EXPORT_SYMBOL(lnet_acceptor_port); static char *accept = "secure"; @@ -89,7 +83,7 @@ CFS_MODULE_PARM(accept_timeout, "i", int, 0644, static char *accept_type = NULL; -int +static int lnet_acceptor_get_tunables(void) { /* Userland acceptor uses 'accept_type' instead of 'accept', due to @@ -113,64 +107,60 @@ lnet_connect_console_error (int rc, lnet_nid_t peer_nid, switch (rc) { /* "normal" errors */ case -ECONNREFUSED: - CDEBUG(D_NETERROR, "Connection to %s at host %u.%u.%u.%u " - "on port %d was refused: " - "check that Lustre is running on that node.\n", - libcfs_nid2str(peer_nid), - HIPQUAD(peer_ip), peer_port); + CNETERR("Connection to %s at host %pI4h on port %d was " + "refused: check that Lustre is running on that node.\n", + libcfs_nid2str(peer_nid), &peer_ip, peer_port); break; case -EHOSTUNREACH: case -ENETUNREACH: - CDEBUG(D_NETERROR, "Connection to %s at host %u.%u.%u.%u " - "was unreachable: the network or that node may " - "be down, or Lustre may be misconfigured.\n", - libcfs_nid2str(peer_nid), HIPQUAD(peer_ip)); + CNETERR("Connection to %s at host %pI4h " + "was unreachable: the network or that node may " + "be down, or Lustre may be misconfigured.\n", + libcfs_nid2str(peer_nid), &peer_ip); break; case -ETIMEDOUT: - CDEBUG(D_NETERROR, "Connection to %s at host %u.%u.%u.%u on " - "port %d took too long: that node may be hung " - "or experiencing high load.\n", - libcfs_nid2str(peer_nid), - HIPQUAD(peer_ip), peer_port); + CNETERR("Connection to %s at host %pI4h on " + "port %d took too long: that node may be hung " + "or experiencing high load.\n", + libcfs_nid2str(peer_nid), &peer_ip, peer_port); break; case -ECONNRESET: - LCONSOLE_ERROR_MSG(0x11b, "Connection to %s at host %u.%u.%u.%u" + LCONSOLE_ERROR_MSG(0x11b, "Connection to %s at host %pI4h" " on port %d was reset: " "is it running a compatible version of " "Lustre and is %s one of its NIDs?\n", - libcfs_nid2str(peer_nid), - HIPQUAD(peer_ip), peer_port, - libcfs_nid2str(peer_nid)); + libcfs_nid2str(peer_nid), &peer_ip, + peer_port, libcfs_nid2str(peer_nid)); break; case -EPROTO: LCONSOLE_ERROR_MSG(0x11c, "Protocol error connecting to %s at " - "host %u.%u.%u.%u on port %d: is it running " + "host %pI4h on port %d: is it running " "a compatible version of Lustre?\n", - libcfs_nid2str(peer_nid), - HIPQUAD(peer_ip), peer_port); + libcfs_nid2str(peer_nid), &peer_ip, + peer_port); break; case -EADDRINUSE: LCONSOLE_ERROR_MSG(0x11d, "No privileged ports available to " - "connect to %s at host %u.%u.%u.%u on port " + "connect to %s at host %pI4h on port " "%d\n", libcfs_nid2str(peer_nid), - HIPQUAD(peer_ip), peer_port); + &peer_ip, peer_port); break; default: LCONSOLE_ERROR_MSG(0x11e, "Unexpected error %d connecting to %s" - " at host %u.%u.%u.%u on port %d\n", rc, + " at host %pI4h on port %d\n", rc, libcfs_nid2str(peer_nid), - HIPQUAD(peer_ip), peer_port); + &peer_ip, peer_port); break; } } EXPORT_SYMBOL(lnet_connect_console_error); int -lnet_connect(cfs_socket_t **sockp, lnet_nid_t peer_nid, +lnet_connect(struct socket **sockp, lnet_nid_t peer_nid, __u32 local_ip, __u32 peer_ip, int peer_port) { lnet_acceptor_connreq_t cr; - cfs_socket_t *sock; + struct socket *sock; int rc; int port; int fatal; @@ -182,7 +172,7 @@ lnet_connect(cfs_socket_t **sockp, lnet_nid_t peer_nid, --port) { /* Iterate through reserved ports. */ - rc = libcfs_sock_connect(&sock, &fatal, + rc = lnet_sock_connect(&sock, &fatal, local_ip, port, peer_ip, peer_port); if (rc != 0) { @@ -199,7 +189,7 @@ lnet_connect(cfs_socket_t **sockp, lnet_nid_t peer_nid, if (the_lnet.ln_testprotocompat != 0) { /* single-shot proto check */ - LNET_LOCK(); + lnet_net_lock(LNET_LOCK_EX); if ((the_lnet.ln_testprotocompat & 4) != 0) { cr.acr_version++; the_lnet.ln_testprotocompat &= ~4; @@ -208,10 +198,10 @@ lnet_connect(cfs_socket_t **sockp, lnet_nid_t peer_nid, cr.acr_magic = LNET_PROTO_MAGIC; the_lnet.ln_testprotocompat &= ~8; } - LNET_UNLOCK(); + lnet_net_unlock(LNET_LOCK_EX); } - rc = libcfs_sock_write(sock, &cr, sizeof(cr), + rc = lnet_sock_write(sock, &cr, sizeof(cr), accept_timeout); if (rc != 0) goto failed_sock; @@ -223,55 +213,16 @@ lnet_connect(cfs_socket_t **sockp, lnet_nid_t peer_nid, rc = -EADDRINUSE; goto failed; - failed_sock: - libcfs_sock_release(sock); - failed: - lnet_connect_console_error(rc, peer_nid, peer_ip, peer_port); - return rc; +failed_sock: + sock_release(sock); +failed: + lnet_connect_console_error(rc, peer_nid, peer_ip, peer_port); + return rc; } EXPORT_SYMBOL(lnet_connect); -#else /* below is multi-threaded user-space code */ - -static char *accept_type = "secure"; - -int -lnet_acceptor_get_tunables() -{ - int rc; - char *env = getenv("LNET_ACCEPT"); - - if (env != NULL) - accept_type = env; - - rc = lnet_parse_int_tunable(&accept_port, "LNET_ACCEPT_PORT"); - - if (rc != 0) - return rc; - - rc = lnet_parse_int_tunable(&accept_backlog, "LNET_ACCEPT_BACKLOG"); - - if (rc != 0) - return rc; - - rc = lnet_parse_int_tunable(&accept_timeout, "LNET_ACCEPT_TIMEOUT"); - - if (rc != 0) - return rc; - - CDEBUG(D_NET, "accept_type = %s\n", accept_type); - CDEBUG(D_NET, "accept_port = %d\n", accept_port); - CDEBUG(D_NET, "accept_backlog = %d\n", accept_backlog); - CDEBUG(D_NET, "accept_timeout = %d\n", accept_timeout); - return 0; -} - -#endif /* __KERNEL__ */ - -/* Below is the code common for both kernel and MT user-space */ - -int -lnet_accept(cfs_socket_t *sock, __u32 magic) +static int +lnet_accept(struct socket *sock, __u32 magic) { lnet_acceptor_connreq_t cr; __u32 peer_ip; @@ -283,7 +234,7 @@ lnet_accept(cfs_socket_t *sock, __u32 magic) LASSERT (sizeof(cr) <= 16); /* not too big for the stack */ - rc = libcfs_sock_getaddr(sock, 1, &peer_ip, &peer_port); + rc = lnet_sock_getaddr(sock, true, &peer_ip, &peer_port); LASSERT (rc == 0); /* we succeeded before */ if (!lnet_accept_magic(magic, LNET_PROTO_ACCEPTOR_MAGIC)) { @@ -297,37 +248,35 @@ lnet_accept(cfs_socket_t *sock, __u32 magic) memset (&cr, 0, sizeof(cr)); cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC; cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION; - rc = libcfs_sock_write(sock, &cr, sizeof(cr), + rc = lnet_sock_write(sock, &cr, sizeof(cr), accept_timeout); if (rc != 0) CERROR("Error sending magic+version in response" - "to LNET magic from %u.%u.%u.%u: %d\n", - HIPQUAD(peer_ip), rc); + "to LNET magic from %pI4h: %d\n", + &peer_ip, rc); return -EPROTO; } if (magic == le32_to_cpu(LNET_PROTO_TCP_MAGIC)) str = "'old' socknal/tcpnal"; - else if (lnet_accept_magic(magic, LNET_PROTO_RA_MAGIC)) - str = "'old' ranal"; else str = "unrecognised"; - LCONSOLE_ERROR_MSG(0x11f, "Refusing connection from %u.%u.%u.%u" + LCONSOLE_ERROR_MSG(0x11f, "Refusing connection from %pI4h" " magic %08x: %s acceptor protocol\n", - HIPQUAD(peer_ip), magic, str); + &peer_ip, magic, str); return -EPROTO; } flip = (magic != LNET_PROTO_ACCEPTOR_MAGIC); - rc = libcfs_sock_read(sock, &cr.acr_version, + rc = lnet_sock_read(sock, &cr.acr_version, sizeof(cr.acr_version), accept_timeout); if (rc != 0) { CERROR("Error %d reading connection request version from " - "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip)); + "%pI4h\n", rc, &peer_ip); return -EIO; } @@ -345,23 +294,23 @@ lnet_accept(cfs_socket_t *sock, __u32 magic) cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC; cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION; - rc = libcfs_sock_write(sock, &cr, sizeof(cr), + rc = lnet_sock_write(sock, &cr, sizeof(cr), accept_timeout); if (rc != 0) CERROR("Error sending magic+version in response" - "to version %d from %u.%u.%u.%u: %d\n", - peer_version, HIPQUAD(peer_ip), rc); + "to version %d from %pI4h: %d\n", + peer_version, &peer_ip, rc); return -EPROTO; } - rc = libcfs_sock_read(sock, &cr.acr_nid, + rc = lnet_sock_read(sock, &cr.acr_nid, sizeof(cr) - offsetof(lnet_acceptor_connreq_t, acr_nid), accept_timeout); if (rc != 0) { CERROR("Error %d reading connection request from " - "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip)); + "%pI4h\n", rc, &peer_ip); return -EIO; } @@ -373,23 +322,23 @@ lnet_accept(cfs_socket_t *sock, __u32 magic) ni->ni_nid != cr.acr_nid) { /* right NET, wrong NID! */ if (ni != NULL) lnet_ni_decref(ni); - LCONSOLE_ERROR_MSG(0x120, "Refusing connection from %u.%u.%u.%u" - " for %s: No matching NI\n", - HIPQUAD(peer_ip), libcfs_nid2str(cr.acr_nid)); + LCONSOLE_ERROR_MSG(0x120, "Refusing connection from %pI4h " + "for %s: No matching NI\n", + &peer_ip, libcfs_nid2str(cr.acr_nid)); return -EPERM; } if (ni->ni_lnd->lnd_accept == NULL) { /* This catches a request for the loopback LND */ lnet_ni_decref(ni); - LCONSOLE_ERROR_MSG(0x121, "Refusing connection from %u.%u.%u.%u" - " for %s: NI doesn not accept IP connections\n", - HIPQUAD(peer_ip), libcfs_nid2str(cr.acr_nid)); + LCONSOLE_ERROR_MSG(0x121, "Refusing connection from %pI4h " + "for %s: NI doesn not accept IP connections\n", + &peer_ip, libcfs_nid2str(cr.acr_nid)); return -EPERM; } - CDEBUG(D_NET, "Accept %s from %u.%u.%u.%u\n", - libcfs_nid2str(cr.acr_nid), HIPQUAD(peer_ip)); + CDEBUG(D_NET, "Accept %s from %pI4h\n", + libcfs_nid2str(cr.acr_nid), &peer_ip); rc = ni->ni_lnd->lnd_accept(ni, sock); @@ -397,11 +346,10 @@ lnet_accept(cfs_socket_t *sock, __u32 magic) return rc; } -int +static int lnet_acceptor(void *arg) { - char name[16]; - cfs_socket_t *newsock; + struct socket *newsock; int rc; __u32 magic; __u32 peer_ip; @@ -410,11 +358,9 @@ lnet_acceptor(void *arg) LASSERT (lnet_acceptor_state.pta_sock == NULL); - snprintf(name, sizeof(name), "acceptor_%03d", accept_port); - cfs_daemonize(name); cfs_block_allsigs(); - rc = libcfs_sock_listen(&lnet_acceptor_state.pta_sock, + rc = lnet_sock_listen(&lnet_acceptor_state.pta_sock, 0, accept_port, accept_backlog); if (rc != 0) { if (rc == -EADDRINUSE) @@ -431,48 +377,48 @@ lnet_acceptor(void *arg) LCONSOLE(0, "Accept %s, port %d\n", accept_type, accept_port); } - /* set init status and unblock parent */ - lnet_acceptor_state.pta_shutdown = rc; - cfs_mt_complete(&lnet_acceptor_state.pta_signal); + /* set init status and unblock parent */ + lnet_acceptor_state.pta_shutdown = rc; + complete(&lnet_acceptor_state.pta_signal); if (rc != 0) return rc; while (!lnet_acceptor_state.pta_shutdown) { - rc = libcfs_sock_accept(&newsock, lnet_acceptor_state.pta_sock); - if (rc != 0) { - if (rc != -EAGAIN) { - CWARN("Accept error %d: pausing...\n", rc); - cfs_pause(cfs_time_seconds(1)); - } - continue; - } - - /* maybe we're waken up with libcfs_sock_abort_accept() */ - if (lnet_acceptor_state.pta_shutdown) { - libcfs_sock_release(newsock); - break; - } - - rc = libcfs_sock_getaddr(newsock, 1, &peer_ip, &peer_port); + rc = lnet_sock_accept(&newsock, lnet_acceptor_state.pta_sock); + if (rc != 0) { + if (rc != -EAGAIN) { + CWARN("Accept error %d: pausing...\n", rc); + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_timeout(cfs_time_seconds(1)); + } + continue; + } + + /* maybe we're waken up with lnet_sock_abort_accept() */ + if (lnet_acceptor_state.pta_shutdown) { + sock_release(newsock); + break; + } + + rc = lnet_sock_getaddr(newsock, true, &peer_ip, &peer_port); if (rc != 0) { CERROR("Can't determine new connection's address\n"); goto failed; } if (secure && peer_port > LNET_ACCEPTOR_MAX_RESERVED_PORT) { - CERROR("Refusing connection from %u.%u.%u.%u: " - "insecure port %d\n", - HIPQUAD(peer_ip), peer_port); + CERROR("Refusing connection from %pI4h: " + "insecure port %d\n", &peer_ip, peer_port); goto failed; } - rc = libcfs_sock_read(newsock, &magic, sizeof(magic), + rc = lnet_sock_read(newsock, &magic, sizeof(magic), accept_timeout); if (rc != 0) { CERROR("Error %d reading connection request from " - "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip)); + "%pI4h\n", rc, &peer_ip); goto failed; } @@ -482,18 +428,18 @@ lnet_acceptor(void *arg) continue; - failed: - libcfs_sock_release(newsock); - } +failed: + sock_release(newsock); + } - libcfs_sock_release(lnet_acceptor_state.pta_sock); - lnet_acceptor_state.pta_sock = NULL; + sock_release(lnet_acceptor_state.pta_sock); + lnet_acceptor_state.pta_sock = NULL; - LCONSOLE(0, "Acceptor stopping\n"); + CDEBUG(D_NET, "Acceptor stopping\n"); - /* unblock lnet_acceptor_stop() */ - cfs_mt_complete(&lnet_acceptor_state.pta_signal); - return 0; + /* unblock lnet_acceptor_stop() */ + complete(&lnet_acceptor_state.pta_signal); + return 0; } static inline int @@ -517,79 +463,67 @@ accept2secure(const char *acc, long *sec) int lnet_acceptor_start(void) { + struct task_struct *task; int rc; long rc2; long secure; + /* if acceptor is already running return immediately */ + if (!lnet_acceptor_state.pta_shutdown) + return 0; + LASSERT (lnet_acceptor_state.pta_sock == NULL); rc = lnet_acceptor_get_tunables(); if (rc != 0) return rc; -#ifndef __KERNEL__ - /* Do nothing if we're liblustre clients */ - if ((the_lnet.ln_pid & LNET_PID_USERFLAG) != 0) - return 0; -#endif + init_completion(&lnet_acceptor_state.pta_signal); + rc = accept2secure(accept_type, &secure); + if (rc <= 0) + return rc; - cfs_mt_init_completion(&lnet_acceptor_state.pta_signal); - rc = accept2secure(accept_type, &secure); - if (rc <= 0) { - cfs_mt_fini_completion(&lnet_acceptor_state.pta_signal); - return rc; - } - - if (lnet_count_acceptor_nis() == 0) /* not required */ - return 0; + if (lnet_count_acceptor_nis() == 0) /* not required */ + return 0; - rc2 = cfs_create_thread(lnet_acceptor, (void *)(ulong_ptr_t)secure); - if (rc2 < 0) { - CERROR("Can't start acceptor thread: %d\n", rc); - cfs_mt_fini_completion(&lnet_acceptor_state.pta_signal); + task = kthread_run(lnet_acceptor, (void *)(ulong_ptr_t)secure, + "acceptor_%03ld", secure); + if (IS_ERR(task)) { + rc2 = PTR_ERR(task); + CERROR("Can't start acceptor thread: %ld\n", rc2); - return -ESRCH; - } + return -ESRCH; + } - /* wait for acceptor to startup */ - cfs_mt_wait_for_completion(&lnet_acceptor_state.pta_signal); + /* wait for acceptor to startup */ + wait_for_completion(&lnet_acceptor_state.pta_signal); - if (!lnet_acceptor_state.pta_shutdown) { - /* started OK */ - LASSERT (lnet_acceptor_state.pta_sock != NULL); - return 0; - } + if (!lnet_acceptor_state.pta_shutdown) { + /* started OK */ + LASSERT(lnet_acceptor_state.pta_sock != NULL); + return 0; + } - LASSERT (lnet_acceptor_state.pta_sock == NULL); - cfs_mt_fini_completion(&lnet_acceptor_state.pta_signal); + LASSERT(lnet_acceptor_state.pta_sock == NULL); - return -ENETDOWN; + return -ENETDOWN; } void lnet_acceptor_stop(void) { - if (lnet_acceptor_state.pta_sock == NULL) /* not running */ - return; + struct sock *sk; - lnet_acceptor_state.pta_shutdown = 1; - libcfs_sock_abort_accept(lnet_acceptor_state.pta_sock); + if (lnet_acceptor_state.pta_shutdown) /* not running */ + return; - /* block until acceptor signals exit */ - cfs_mt_wait_for_completion(&lnet_acceptor_state.pta_signal); + lnet_acceptor_state.pta_shutdown = 1; - cfs_mt_fini_completion(&lnet_acceptor_state.pta_signal); -} + sk = lnet_acceptor_state.pta_sock->sk; -#else /* single-threaded user-space */ -int -lnet_acceptor_start(void) -{ - return 0; -} + /* awake any sleepers using safe method */ + sk->sk_state_change(sk); -void -lnet_acceptor_stop(void) -{ + /* block until acceptor signals exit */ + wait_for_completion(&lnet_acceptor_state.pta_signal); } -#endif /* defined(__KERNEL__) || defined(HAVE_LIBPTHREAD) */