lnet/lnet/acceptor.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Copyright (C) 2005 Cluster File Systems, Inc.
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org.
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  *
  21  */
  22
  23 #define DEBUG_SUBSYSTEM S_LNET
  24 #include <lnet/lib-lnet.h>
  25
  26 #ifdef __KERNEL__
  27 static char *accept = "secure";
  28 CFS_MODULE_PARM(accept, "s", charp, 0444,
  29                 "Accept connections (secure|all|none)");
  30
  31 static int accept_port = 988;
  32 CFS_MODULE_PARM(accept_port, "i", int, 0444,
  33                 "Acceptor's port (same on all nodes)");
  34
  35 static int accept_backlog = 127;
  36 CFS_MODULE_PARM(accept_backlog, "i", int, 0444,
  37                 "Acceptor's listen backlog");
  38
  39 static int accept_timeout = 5;
  40 CFS_MODULE_PARM(accept_timeout, "i", int, 0644,
  41                 "Acceptor's timeout (seconds)");
  42
  43 struct {
  44         int               pta_shutdown;
  45         cfs_socket_t     *pta_sock;
  46         struct semaphore  pta_signal;
  47 } lnet_acceptor_state;
  48
  49 int
  50 lnet_acceptor_timeout(void)
  51 {
  52         return accept_timeout;
  53 }
  54 EXPORT_SYMBOL(lnet_acceptor_timeout);
  55
  56 int
  57 lnet_acceptor_port(void)
  58 {
  59         return accept_port;
  60 }
  61 EXPORT_SYMBOL(lnet_acceptor_port);
  62
  63 void
  64 lnet_connect_console_error (int rc, lnet_nid_t peer_nid,
  65                            __u32 peer_ip, int peer_port)
  66 {
  67         switch (rc) {
  68         /* "normal" errors */
  69         case -ECONNREFUSED:
  70                 CDEBUG(D_NETERROR, "Connection to %s at host %u.%u.%u.%u "
  71                        "on port %d was refused: "
  72                        "check that Lustre is running on that node.\n",
  73                        libcfs_nid2str(peer_nid),
  74                        HIPQUAD(peer_ip), peer_port);
  75                 break;
  76         case -EHOSTUNREACH:
  77         case -ENETUNREACH:
  78                 CDEBUG(D_NETERROR, "Connection to %s at host %u.%u.%u.%u "
  79                        "was unreachable: the network or that node may "
  80                        "be down, or Lustre may be misconfigured.\n",
  81                        libcfs_nid2str(peer_nid), HIPQUAD(peer_ip));
  82                 break;
  83         case -ETIMEDOUT:
  84                 CDEBUG(D_NETERROR, "Connection to %s at host %u.%u.%u.%u on "
  85                        "port %d took too long: that node may be hung "
  86                        "or experiencing high load.\n",
  87                        libcfs_nid2str(peer_nid),
  88                        HIPQUAD(peer_ip), peer_port);
  89                 break;
  90         case -ECONNRESET:
  91                 LCONSOLE_ERROR_MSG(0x11b, "Connection to %s at host %u.%u.%u.%u"
  92                                    " on port %d was reset: "
  93                                    "is it running a compatible version of "
  94                                    "Lustre and is %s one of its NIDs?\n",
  95                                    libcfs_nid2str(peer_nid),
  96                                    HIPQUAD(peer_ip), peer_port,
  97                                    libcfs_nid2str(peer_nid));
  98                 break;
  99         case -EPROTO:
 100                 LCONSOLE_ERROR_MSG(0x11c, "Protocol error connecting to %s at "
 101                                    "host %u.%u.%u.%u on port %d: is it running "
 102                                    "a compatible version of Lustre?\n",
 103                                    libcfs_nid2str(peer_nid),
 104                                    HIPQUAD(peer_ip), peer_port);
 105                 break;
 106         case -EADDRINUSE:
 107                 LCONSOLE_ERROR_MSG(0x11d, "No privileged ports available to "
 108                                    "connect to %s at host %u.%u.%u.%u on port "
 109                                    "%d\n", libcfs_nid2str(peer_nid),
 110                                    HIPQUAD(peer_ip), peer_port);
 111                 break;
 112         default:
 113                 LCONSOLE_ERROR_MSG(0x11e, "Unexpected error %d connecting to %s"
 114                                    " at host %u.%u.%u.%u on port %d\n", rc,
 115                                    libcfs_nid2str(peer_nid),
 116                                    HIPQUAD(peer_ip), peer_port);
 117                 break;
 118         }
 119 }
 120 EXPORT_SYMBOL(lnet_connect_console_error);
 121
 122 int
 123 lnet_connect(cfs_socket_t **sockp, lnet_nid_t peer_nid,
 124             __u32 local_ip, __u32 peer_ip, int peer_port)
 125 {
 126         lnet_acceptor_connreq_t cr;
 127         cfs_socket_t           *sock;
 128         int                     rc;
 129         int                     port;
 130         int                     fatal;
 131
 132         CLASSERT (sizeof(cr) <= 16);            /* not too big to be on the stack */
 133
 134         for (port = LNET_ACCEPTOR_MAX_RESERVED_PORT;
 135              port >= LNET_ACCEPTOR_MIN_RESERVED_PORT;
 136              --port) {
 137                 /* Iterate through reserved ports. */
 138
 139                 rc = libcfs_sock_connect(&sock, &fatal,
 140                                          local_ip, port,
 141                                          peer_ip, peer_port);
 142                 if (rc != 0) {
 143                         if (fatal)
 144                                 goto failed;
 145                         continue;
 146                 }
 147
 148                 CLASSERT (LNET_PROTO_ACCEPTOR_VERSION == 1);
 149
 150                 if (the_lnet.ln_ptlcompat != 2) {
 151                         /* When portals compatibility is "strong", simply
 152                          * connect (i.e. send no acceptor connection request).
 153                          * Othewise send an acceptor connection request. I can
 154                          * have no portals peers so everyone else should
 155                          * understand my protocol. */
 156                         cr.acr_magic   = LNET_PROTO_ACCEPTOR_MAGIC;
 157                         cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
 158                         cr.acr_nid     = peer_nid;
 159
 160                         if (the_lnet.ln_testprotocompat != 0) {
 161                                 /* single-shot proto check */
 162                                 LNET_LOCK();
 163                                 if ((the_lnet.ln_testprotocompat & 4) != 0) {
 164                                         cr.acr_version++;
 165                                         the_lnet.ln_testprotocompat &= ~4;
 166                                 }
 167                                 if ((the_lnet.ln_testprotocompat & 8) != 0) {
 168                                         cr.acr_magic = LNET_PROTO_MAGIC;
 169                                         the_lnet.ln_testprotocompat &= ~8;
 170                                 }
 171                                 LNET_UNLOCK();
 172                         }
 173
 174                         rc = libcfs_sock_write(sock, &cr, sizeof(cr),
 175                                                accept_timeout);
 176                         if (rc != 0)
 177                                 goto failed_sock;
 178                 }
 179
 180                 *sockp = sock;
 181                 return 0;
 182         }
 183
 184         rc = -EADDRINUSE;
 185         goto failed;
 186
 187  failed_sock:
 188         libcfs_sock_release(sock);
 189  failed:
 190         lnet_connect_console_error(rc, peer_nid, peer_ip, peer_port);
 191         return rc;
 192 }
 193 EXPORT_SYMBOL(lnet_connect);
 194
 195 static inline int
 196 lnet_accept_magic(__u32 magic, __u32 constant)
 197 {
 198         return (magic == constant ||
 199                 magic == __swab32(constant));
 200 }
 201
 202 int
 203 lnet_accept(lnet_ni_t *blind_ni, cfs_socket_t *sock, __u32 magic)
 204 {
 205         lnet_acceptor_connreq_t cr;
 206         __u32                   peer_ip;
 207         int                     peer_port;
 208         int                     rc;
 209         int                     flip;
 210         lnet_ni_t              *ni;
 211         char                   *str;
 212
 213         /* CAVEAT EMPTOR: I may be called by an LND in any thread's context if
 214          * I passed the new socket "blindly" to the single NI that needed an
 215          * acceptor.  If so, blind_ni != NULL... */
 216
 217         LASSERT (sizeof(cr) <= 16);             /* not too big for the stack */
 218
 219         rc = libcfs_sock_getaddr(sock, 1, &peer_ip, &peer_port);
 220         LASSERT (rc == 0);                      /* we succeeded before */
 221
 222         if (!lnet_accept_magic(magic, LNET_PROTO_ACCEPTOR_MAGIC)) {
 223
 224                 if (lnet_accept_magic(magic, LNET_PROTO_MAGIC)) {
 225                         /* future version compatibility!
 226                          * When LNET unifies protocols over all LNDs, the first
 227                          * thing sent will be a version query.  I send back
 228                          * LNET_PROTO_ACCEPTOR_MAGIC to tell her I'm "old" */
 229
 230                         memset (&cr, 0, sizeof(cr));
 231                         cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
 232                         cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
 233                         rc = libcfs_sock_write(sock, &cr, sizeof(cr),
 234                                                accept_timeout);
 235
 236                         if (rc != 0)
 237                                 CERROR("Error sending magic+version in response"
 238                                        "to LNET magic from %u.%u.%u.%u: %d\n",
 239                                        HIPQUAD(peer_ip), rc);
 240                         return -EPROTO;
 241                 }
 242
 243                 if (magic == le32_to_cpu(LNET_PROTO_TCP_MAGIC))
 244                         str = "'old' socknal/tcpnal";
 245                 else if (lnet_accept_magic(magic, LNET_PROTO_RA_MAGIC))
 246                         str = "'old' ranal";
 247                 else if (lnet_accept_magic(magic, LNET_PROTO_OPENIB_MAGIC))
 248                         str = "'old' openibnal";
 249                 else
 250                         str = "unrecognised";
 251
 252                 LCONSOLE_ERROR_MSG(0x11f, "Refusing connection from %u.%u.%u.%u"
 253                                    " magic %08x: %s acceptor protocol\n",
 254                                    HIPQUAD(peer_ip), magic, str);
 255                 return -EPROTO;
 256         }
 257
 258         flip = (magic != LNET_PROTO_ACCEPTOR_MAGIC);
 259
 260         rc = libcfs_sock_read(sock, &cr.acr_version,
 261                               sizeof(cr.acr_version),
 262                               accept_timeout);
 263         if (rc != 0) {
 264                 CERROR("Error %d reading connection request version from "
 265                        "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
 266                 return -EIO;
 267         }
 268
 269         if (flip)
 270                 __swab32s(&cr.acr_version);
 271
 272         if (cr.acr_version != LNET_PROTO_ACCEPTOR_VERSION) {
 273                 /* future version compatibility!
 274                  * An acceptor-specific protocol rev will first send a version
 275                  * query.  I send back my current version to tell her I'm
 276                  * "old". */
 277                 int peer_version = cr.acr_version;
 278
 279                 memset (&cr, 0, sizeof(cr));
 280                 cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
 281                 cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
 282
 283                 rc = libcfs_sock_write(sock, &cr, sizeof(cr),
 284                                        accept_timeout);
 285
 286                 if (rc != 0)
 287                         CERROR("Error sending magic+version in response"
 288                                "to version %d from %u.%u.%u.%u: %d\n",
 289                                peer_version, HIPQUAD(peer_ip), rc);
 290                 return -EPROTO;
 291         }
 292
 293         rc = libcfs_sock_read(sock, &cr.acr_nid,
 294                               sizeof(cr) -
 295                               offsetof(lnet_acceptor_connreq_t, acr_nid),
 296                               accept_timeout);
 297         if (rc != 0) {
 298                 CERROR("Error %d reading connection request from "
 299                        "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
 300                 return -EIO;
 301         }
 302
 303         if (flip)
 304                 __swab64s(&cr.acr_nid);
 305
 306         ni = lnet_net2ni(LNET_NIDNET(cr.acr_nid));
 307         if (ni == NULL ||               /* no matching net */
 308             ni->ni_nid != cr.acr_nid) { /* right NET, wrong NID! */
 309                 if (ni != NULL)
 310                         lnet_ni_decref(ni);
 311                 LCONSOLE_ERROR_MSG(0x120, "Refusing connection from %u.%u.%u.%u"
 312                                    " for %s: No matching NI\n",
 313                                    HIPQUAD(peer_ip), libcfs_nid2str(cr.acr_nid));
 314                 return -EPERM;
 315         }
 316
 317         if (ni->ni_lnd->lnd_accept == NULL) {
 318                 /* This catches a request for the loopback LND */
 319                 lnet_ni_decref(ni);
 320                 LCONSOLE_ERROR_MSG(0x121, "Refusing connection from %u.%u.%u.%u"
 321                                   " for %s: NI doesn not accept IP connections\n",
 322                                   HIPQUAD(peer_ip), libcfs_nid2str(cr.acr_nid));
 323                 return -EPERM;
 324         }
 325
 326         CDEBUG(D_NET, "Accept %s from %u.%u.%u.%u%s\n",
 327                libcfs_nid2str(cr.acr_nid), HIPQUAD(peer_ip),
 328                blind_ni == NULL ? "" : " (blind)");
 329
 330         if (blind_ni == NULL) {
 331                 /* called by the acceptor: call into the requested NI... */
 332                 rc = ni->ni_lnd->lnd_accept(ni, sock);
 333         } else {
 334                 /* portals_compatible set and the (only) NI called me to verify
 335                  * and skip the connection request... */
 336                 LASSERT (the_lnet.ln_ptlcompat != 0);
 337                 LASSERT (ni == blind_ni);
 338                 rc = 0;
 339         }
 340
 341         lnet_ni_decref(ni);
 342         return rc;
 343 }
 344 EXPORT_SYMBOL(lnet_accept);
 345
 346 int
 347 lnet_acceptor(void *arg)
 348 {
 349         char           name[16];
 350         cfs_socket_t  *newsock;
 351         int            rc;
 352         int            n_acceptor_nis;
 353         __u32          magic;
 354         __u32          peer_ip;
 355         int            peer_port;
 356         lnet_ni_t     *blind_ni = NULL;
 357         int            secure = (int)((unsigned long)arg);
 358
 359         LASSERT (lnet_acceptor_state.pta_sock == NULL);
 360
 361         if (the_lnet.ln_ptlcompat != 0) {
 362                 /* When portals_compatibility is enabled, peers may connect
 363                  * without sending an acceptor connection request.  There is no
 364                  * ambiguity about which network the peer wants to connect to
 365                  * since there can only be 1 network, so I pass connections
 366                  * "blindly" to it. */
 367                 n_acceptor_nis = lnet_count_acceptor_nis(&blind_ni);
 368                 LASSERT (n_acceptor_nis == 1);
 369                 LASSERT (blind_ni != NULL);
 370         }
 371
 372         snprintf(name, sizeof(name), "acceptor_%03d", accept_port);
 373         cfs_daemonize(name);
 374         cfs_block_allsigs();
 375
 376         rc = libcfs_sock_listen(&lnet_acceptor_state.pta_sock,
 377                                 0, accept_port, accept_backlog);
 378         if (rc != 0) {
 379                 if (rc == -EADDRINUSE)
 380                         LCONSOLE_ERROR_MSG(0x122, "Can't start acceptor on port"
 381                                            " %d: port already in use\n",
 382                                            accept_port);
 383                 else
 384                         LCONSOLE_ERROR_MSG(0x123, "Can't start acceptor on port "
 385                                            "%d: unexpected error %d\n",
 386                                            accept_port, rc);
 387
 388                 lnet_acceptor_state.pta_sock = NULL;
 389         } else {
 390                 LCONSOLE(0, "Accept %s, port %d%s\n",
 391                          accept, accept_port,
 392                          blind_ni == NULL ? "" : " (proto compatible)");
 393         }
 394
 395         /* set init status and unblock parent */
 396         lnet_acceptor_state.pta_shutdown = rc;
 397         mutex_up(&lnet_acceptor_state.pta_signal);
 398
 399         if (rc != 0)
 400                 return rc;
 401
 402         while (!lnet_acceptor_state.pta_shutdown) {
 403
 404                 rc = libcfs_sock_accept(&newsock, lnet_acceptor_state.pta_sock);
 405                 if (rc != 0) {
 406                         if (rc != -EAGAIN) {
 407                                 CWARN("Accept error %d: pausing...\n", rc);
 408                                 cfs_pause(cfs_time_seconds(1));
 409                         }
 410                         continue;
 411                 }
 412
 413                 rc = libcfs_sock_getaddr(newsock, 1, &peer_ip, &peer_port);
 414                 if (rc != 0) {
 415                         CERROR("Can't determine new connection's address\n");
 416                         goto failed;
 417                 }
 418
 419                 if (secure && peer_port > LNET_ACCEPTOR_MAX_RESERVED_PORT) {
 420                         CERROR("Refusing connection from %u.%u.%u.%u: "
 421                                "insecure port %d\n",
 422                                HIPQUAD(peer_ip), peer_port);
 423                         goto failed;
 424                 }
 425
 426                 if (blind_ni != NULL) {
 427                         rc = blind_ni->ni_lnd->lnd_accept(blind_ni, newsock);
 428                         if (rc != 0) {
 429                                 CERROR("NI %s refused 'blind' connection from "
 430                                        "%u.%u.%u.%u\n",
 431                                        libcfs_nid2str(blind_ni->ni_nid),
 432                                        HIPQUAD(peer_ip));
 433                                 goto failed;
 434                         }
 435                         continue;
 436                 }
 437
 438                 rc = libcfs_sock_read(newsock, &magic, sizeof(magic),
 439                                       accept_timeout);
 440                 if (rc != 0) {
 441                         CERROR("Error %d reading connection request from "
 442                                "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
 443                         goto failed;
 444                 }
 445
 446                 rc = lnet_accept(NULL, newsock, magic);
 447                 if (rc != 0)
 448                         goto failed;
 449
 450                 continue;
 451
 452         failed:
 453                 libcfs_sock_release(newsock);
 454         }
 455
 456         libcfs_sock_release(lnet_acceptor_state.pta_sock);
 457         lnet_acceptor_state.pta_sock = NULL;
 458
 459         if (blind_ni != NULL)
 460                 lnet_ni_decref(blind_ni);
 461
 462         LCONSOLE(0,"Acceptor stopping\n");
 463
 464         /* unblock lnet_acceptor_stop() */
 465         mutex_up(&lnet_acceptor_state.pta_signal);
 466         return 0;
 467 }
 468
 469 int
 470 lnet_acceptor_start(void)
 471 {
 472         long   pid;
 473         long   secure;
 474
 475         LASSERT (lnet_acceptor_state.pta_sock == NULL);
 476         init_mutex_locked(&lnet_acceptor_state.pta_signal);
 477
 478         if (!strcmp(accept, "secure")) {
 479                 secure = 1;
 480         } else if (!strcmp(accept, "all")) {
 481                 secure = 0;
 482         } else if (!strcmp(accept, "none")) {
 483                 return 0;
 484         } else {
 485                 LCONSOLE_ERROR_MSG(0x124, "Can't parse 'accept=\"%s\"'\n",
 486                                    accept);
 487                 return -EINVAL;
 488         }
 489
 490         if (lnet_count_acceptor_nis(NULL) == 0)  /* not required */
 491                 return 0;
 492
 493         pid = cfs_kernel_thread(lnet_acceptor, (void *)secure, 0);
 494         if (pid < 0) {
 495                 CERROR("Can't start acceptor thread: %ld\n", pid);
 496                 return -ESRCH;
 497         }
 498
 499         mutex_down(&lnet_acceptor_state.pta_signal); /* wait for acceptor to startup */
 500
 501         if (!lnet_acceptor_state.pta_shutdown) {
 502                 /* started OK */
 503                 LASSERT (lnet_acceptor_state.pta_sock != NULL);
 504                 return 0;
 505         }
 506
 507         LASSERT (lnet_acceptor_state.pta_sock == NULL);
 508         return -ENETDOWN;
 509 }
 510
 511 void
 512 lnet_acceptor_stop(void)
 513 {
 514         if (lnet_acceptor_state.pta_sock == NULL) /* not running */
 515                 return;
 516
 517         lnet_acceptor_state.pta_shutdown = 1;
 518         libcfs_sock_abort_accept(lnet_acceptor_state.pta_sock);
 519
 520         /* block until acceptor signals exit */
 521         mutex_down(&lnet_acceptor_state.pta_signal);
 522 }
 523
 524 #else /* __KERNEL__ */
 525 #ifdef HAVE_LIBPTHREAD
 526
 527 static char *accept_type;
 528 static int accept_port = 988;
 529 static int accept_backlog;
 530 static int accept_timeout;
 531
 532 struct {
 533         int                   pta_shutdown;
 534         int                   pta_sock;
 535         struct cfs_completion pta_completion;
 536 } lnet_acceptor_state;
 537
 538 int
 539 lnet_acceptor_port(void)
 540 {
 541         return accept_port;
 542 }
 543
 544 int
 545 lnet_parse_int_tunable(int *value, char *name, int dflt)
 546 {
 547         char    *env = getenv(name);
 548         char    *end;
 549
 550         if (env == NULL) {
 551                 *value = dflt;
 552                 return 0;
 553         }
 554
 555         *value = strtoull(env, &end, 0);
 556         if (*end == 0)
 557                 return 0;
 558
 559         CERROR("Can't parse tunable %s=%s\n", name, env);
 560         return -EINVAL;
 561 }
 562
 563 int
 564 lnet_parse_string_tunable(char **value, char *name, char *dflt)
 565 {
 566         char    *env = getenv(name);
 567
 568         if (env == NULL)
 569                 *value = dflt;
 570         else
 571                 *value = env;
 572
 573         return 0;
 574 }
 575
 576 int
 577 lnet_acceptor_get_tunables()
 578 {
 579         int rc;
 580         rc = lnet_parse_string_tunable(&accept_type, "LNET_ACCEPT", "secure");
 581
 582         if (rc != 0)
 583                 return rc;
 584
 585         rc = lnet_parse_int_tunable(&accept_port, "LNET_ACCEPT_PORT", 988);
 586
 587         if (rc != 0)
 588                 return rc;
 589
 590         rc = lnet_parse_int_tunable(&accept_backlog, "LNET_ACCEPT_BACKLOG", 127);
 591
 592         if (rc != 0)
 593                 return rc;
 594
 595         rc = lnet_parse_int_tunable(&accept_timeout, "LNET_ACCEPT_TIMEOUT", 5);
 596
 597         if (rc != 0)
 598                 return rc;
 599
 600         CDEBUG(D_NET, "accept_type     = %s\n", accept_type);
 601         CDEBUG(D_NET, "accept_port     = %d\n", accept_port);
 602         CDEBUG(D_NET, "accept_backlog  = %d\n", accept_backlog);
 603         CDEBUG(D_NET, "accept_timeout  = %d\n", accept_timeout);
 604         return 0;
 605 }
 606
 607 static inline int
 608 lnet_accept_magic(__u32 magic, __u32 constant)
 609 {
 610         return (magic == constant ||
 611                 magic == __swab32(constant));
 612 }
 613
 614 /* user-land lnet_accept() isn't used by any LND's directly. So, we don't
 615  * do it visible outside acceptor.c and we can change its prototype
 616  * freely */
 617 static int
 618 lnet_accept(int sock, __u32 magic, __u32 peer_ip, int peer_port)
 619 {
 620         int rc, flip;
 621         lnet_acceptor_connreq_t cr;
 622         lnet_ni_t *ni;
 623
 624         if (!lnet_accept_magic(magic, LNET_PROTO_ACCEPTOR_MAGIC)) {
 625                 LCONSOLE_ERROR("Refusing connection from %u.%u.%u.%u magic %08x: "
 626                                "unsupported acceptor protocol\n",
 627                                HIPQUAD(peer_ip), magic);
 628                 return -EPROTO;
 629         }
 630
 631         flip = (magic != LNET_PROTO_ACCEPTOR_MAGIC);
 632
 633         rc = libcfs_sock_read(sock, &cr.acr_version,
 634                               sizeof(cr.acr_version),
 635                               accept_timeout);
 636         if (rc != 0) {
 637                 CERROR("Error %d reading connection request version from "
 638                        "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
 639                 return -EIO;
 640         }
 641
 642         if (flip)
 643                 __swab32s(&cr.acr_version);
 644
 645         if (cr.acr_version != LNET_PROTO_ACCEPTOR_VERSION)
 646                 return -EPROTO;
 647
 648         rc = libcfs_sock_read(sock, &cr.acr_nid,
 649                               sizeof(cr) -
 650                               offsetof(lnet_acceptor_connreq_t, acr_nid),
 651                               accept_timeout);
 652         if (rc != 0) {
 653                 CERROR("Error %d reading connection request from "
 654                        "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
 655                 return -EIO;
 656         }
 657
 658         if (flip)
 659                 __swab64s(&cr.acr_nid);
 660
 661         ni = lnet_net2ni(LNET_NIDNET(cr.acr_nid));
 662
 663         if (ni == NULL ||                    /* no matching net */
 664              ni->ni_nid != cr.acr_nid) {     /* right NET, wrong NID! */
 665                 if (ni != NULL)
 666                         lnet_ni_decref(ni);
 667                 LCONSOLE_ERROR("Refusing connection from %u.%u.%u.%u for %s: "
 668                                " No matching NI\n",
 669                                HIPQUAD(peer_ip), libcfs_nid2str(cr.acr_nid));
 670                 return -EPERM;
 671         }
 672
 673         if (ni->ni_lnd->lnd_accept == NULL) {
 674                 lnet_ni_decref(ni);
 675                 LCONSOLE_ERROR("Refusing connection from %u.%u.%u.%u for %s: "
 676                                " NI doesn not accept IP connections\n",
 677                                HIPQUAD(peer_ip), libcfs_nid2str(cr.acr_nid));
 678                 return -EPERM;
 679         }
 680
 681         CDEBUG(D_NET, "Accept %s from %u.%u.%u.%u\n",
 682                libcfs_nid2str(cr.acr_nid), HIPQUAD(peer_ip));
 683
 684         rc = ni->ni_lnd->lnd_accept(ni, sock);
 685
 686         lnet_ni_decref(ni);
 687         return rc;
 688 }
 689
 690 int
 691 lnet_acceptor(void *arg)
 692 {
 693         char           name[16];
 694         int            secure = (int)((unsigned long)arg);
 695         int            rc;
 696         int            newsock;
 697         __u32          peer_ip;
 698         int            peer_port;
 699         __u32          magic;
 700
 701         snprintf(name, sizeof(name), "acceptor_%03d", accept_port);
 702         cfs_daemonize(name);
 703         cfs_block_allsigs();
 704
 705         rc = libcfs_sock_listen(&lnet_acceptor_state.pta_sock,
 706                                 0, accept_port, accept_backlog);
 707         if (rc != 0) {
 708                 if (rc == -EADDRINUSE)
 709                         LCONSOLE_ERROR("Can't start acceptor on port %d: "
 710                                        "port already in use\n",
 711                                        accept_port);
 712                 else
 713                         LCONSOLE_ERROR("Can't start acceptor on port %d: "
 714                                        "unexpected error %d\n",
 715                                        accept_port, rc);
 716
 717         } else {
 718                 LCONSOLE(0, "Accept %s, port %d\n", accept_type, accept_port);
 719         }
 720
 721         /* set init status and unblock parent */
 722         lnet_acceptor_state.pta_shutdown = rc;
 723         cfs_complete(&lnet_acceptor_state.pta_completion);
 724
 725         if (rc != 0)
 726                 return rc;
 727
 728         while (!lnet_acceptor_state.pta_shutdown) {
 729
 730                 rc = libcfs_sock_accept(&newsock, lnet_acceptor_state.pta_sock,
 731                                         &peer_ip, &peer_port);
 732                 if (rc != 0)
 733                         continue;
 734
 735                 /* maybe we're waken up with libcfs_sock_abort_accept() */
 736                 if (lnet_acceptor_state.pta_shutdown) {
 737                         close(newsock);
 738                         break;
 739                 }
 740
 741                 if (secure && peer_port > LNET_ACCEPTOR_MAX_RESERVED_PORT) {
 742                         CERROR("Refusing connection from %u.%u.%u.%u: "
 743                                "insecure port %d\n",
 744                                HIPQUAD(peer_ip), peer_port);
 745                         goto failed;
 746                 }
 747
 748                 rc = libcfs_sock_read(newsock, &magic, sizeof(magic),
 749                                       accept_timeout);
 750                 if (rc != 0) {
 751                         CERROR("Error %d reading connection request from "
 752                                "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
 753                         goto failed;
 754                 }
 755
 756                 rc = lnet_accept(newsock, magic, peer_ip, peer_port);
 757                 if (rc != 0)
 758                         goto failed;
 759
 760                 continue;
 761
 762           failed:
 763                 close(newsock);
 764         }
 765
 766         close(lnet_acceptor_state.pta_sock);
 767         LCONSOLE(0,"Acceptor stopping\n");
 768
 769         /* unblock lnet_acceptor_stop() */
 770         cfs_complete(&lnet_acceptor_state.pta_completion);
 771
 772         return 0;
 773 }
 774
 775 static int skip_waiting_for_completion;
 776
 777 int
 778 lnet_acceptor_start(void)
 779 {
 780         long   secure;
 781         int rc;
 782
 783         rc = lnet_acceptor_get_tunables();
 784         if (rc != 0)
 785                 return rc;
 786
 787         /* Do nothing if we're liblustre clients */
 788         if ((the_lnet.ln_pid & LNET_PID_USERFLAG) != 0)
 789                 return 0;
 790
 791         cfs_init_completion(&lnet_acceptor_state.pta_completion);
 792
 793         if (!strcmp(accept_type, "secure")) {
 794                 secure = 1;
 795         } else if (!strcmp(accept_type, "all")) {
 796                 secure = 0;
 797         } else if (!strcmp(accept_type, "none")) {
 798                 skip_waiting_for_completion = 1;
 799                 return 0;
 800         } else {
 801                 LCONSOLE_ERROR ("Can't parse 'accept_type=\"%s\"'\n", accept_type);
 802                 cfs_fini_completion(&lnet_acceptor_state.pta_completion);
 803                 return -EINVAL;
 804         }
 805
 806         if (lnet_count_acceptor_nis(NULL) == 0) { /* not required */
 807                 skip_waiting_for_completion = 1;
 808                 return 0;
 809         }
 810
 811         rc = cfs_create_thread(lnet_acceptor, (void *)secure);
 812         if (rc != 0) {
 813                 CERROR("Can't start acceptor thread: %d\n", rc);
 814                 cfs_fini_completion(&lnet_acceptor_state.pta_completion);
 815                 return rc;
 816         }
 817
 818         /* wait for acceptor to startup */
 819         cfs_wait_for_completion(&lnet_acceptor_state.pta_completion);
 820
 821         if (!lnet_acceptor_state.pta_shutdown)
 822                 return 0;
 823
 824         cfs_fini_completion(&lnet_acceptor_state.pta_completion);
 825         return -ENETDOWN;
 826 }
 827
 828 void
 829 lnet_acceptor_stop(void)
 830 {
 831         /* Do nothing if we're liblustre clients */
 832         if ((the_lnet.ln_pid & LNET_PID_USERFLAG) != 0)
 833                 return;
 834
 835         if (!skip_waiting_for_completion) {
 836                 lnet_acceptor_state.pta_shutdown = 1;
 837                 libcfs_sock_abort_accept(accept_port);
 838
 839                 /* block until acceptor signals exit */
 840                 cfs_wait_for_completion(&lnet_acceptor_state.pta_completion);
 841         }
 842
 843         cfs_fini_completion(&lnet_acceptor_state.pta_completion);
 844 }
 845 #else
 846 int
 847 lnet_acceptor_start(void)
 848 {
 849         return 0;
 850 }
 851
 852 void
 853 lnet_acceptor_stop(void)
 854 {
 855 }
 856 #endif /* !HAVE_LIBPTHREAD */
 857 #endif /* !__KERNEL__ */