lnet/klnds/gmlnd/gmlnd_api.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  Copyright (c) 2003 Los Alamos National Laboratory (LANL)
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org/
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  */
  21
  22 /*
  23  *      Implements the API NAL functions
  24  */
  25
  26 #include "gmnal.h"
  27
  28 int
  29 gmnal_cmd(struct portals_cfg *pcfg, void *private)
  30 {
  31         gmnal_ni_t      *gmnalni = private;
  32         char            *name;
  33         int              nid;
  34         int              gmid;
  35         gm_status_t      gm_status;
  36
  37         CDEBUG(D_TRACE, "gmnal_cmd [%d] private [%p]\n",
  38                pcfg->pcfg_command, private);
  39         gmnalni = (gmnal_ni_t*)private;
  40
  41         switch(pcfg->pcfg_command) {
  42         case GMNAL_IOC_GET_GNID:
  43
  44                 PORTAL_ALLOC(name, pcfg->pcfg_plen1);
  45                 copy_from_user(name, PCFG_PBUF(pcfg, 1), pcfg->pcfg_plen1);
  46
  47                 gm_status = gm_host_name_to_node_id_ex(gmnalni->gmni_port, 0,
  48                                                        name, &nid);
  49                 if (gm_status != GM_SUCCESS) {
  50                         CDEBUG(D_NET, "gm_host_name_to_node_id_ex(...host %s) "
  51                                "failed[%d]\n", name, gm_status);
  52                         return -ENOENT;
  53                 }
  54
  55                 CDEBUG(D_NET, "Local node %s id is [%d]\n", name, nid);
  56                 gm_status = gm_node_id_to_global_id(gmnalni->gmni_port,
  57                                                     nid, &gmid);
  58                 if (gm_status != GM_SUCCESS) {
  59                         CDEBUG(D_NET, "gm_node_id_to_global_id failed[%d]\n",
  60                                gm_status);
  61                         return -ENOENT;
  62                 }
  63
  64                 CDEBUG(D_NET, "Global node is is [%u][%x]\n", gmid, gmid);
  65                 copy_to_user(PCFG_PBUF(pcfg, 2), &gmid, pcfg->pcfg_plen2);
  66                 return 0;
  67
  68         case NAL_CMD_REGISTER_MYNID:
  69                 /* Same NID OK */
  70                 if (pcfg->pcfg_nid == gmnalni->gmni_libnal->libnal_ni.ni_pid.nid)
  71                         return 0;
  72
  73                 CERROR("Can't change NID from "LPD64" to "LPD64"\n",
  74                        gmnalni->gmni_libnal->libnal_ni.ni_pid.nid,
  75                        pcfg->pcfg_nid);
  76                 return -EINVAL;
  77
  78         default:
  79                 CERROR ("gmnal_cmd UNKNOWN[%d]\n", pcfg->pcfg_command);
  80                 return -EINVAL;
  81         }
  82         /* not reached */
  83 }
  84
  85 ptl_nid_t
  86 gmnal_get_local_nid (gmnal_ni_t *gmnalni)
  87 {
  88         unsigned int     local_gmid;
  89         unsigned int     global_gmid;
  90         ptl_nid_t        nid;
  91         gm_status_t      gm_status;
  92
  93         /* Called before anything initialised: no need to lock */
  94         gm_status = gm_get_node_id(gmnalni->gmni_port, &local_gmid);
  95         if (gm_status != GM_SUCCESS)
  96                 return PTL_NID_ANY;
  97
  98         CDEBUG(D_NET, "Local node id is [%u]\n", local_gmid);
  99
 100         gm_status = gm_node_id_to_global_id(gmnalni->gmni_port,
 101                                             local_gmid,
 102                                             &global_gmid);
 103         if (gm_status != GM_SUCCESS)
 104                 return PTL_NID_ANY;
 105
 106         CDEBUG(D_NET, "Global node id is [%u]\n", global_gmid);
 107
 108         nid = (__u64)global_gmid;
 109         LASSERT (nid != PTL_NID_ANY);
 110
 111         return global_gmid;
 112 }
 113
 114
 115 void
 116 gmnal_api_shutdown(nal_t *nal)
 117 {
 118         lib_nal_t       *libnal = nal->nal_data;
 119         gmnal_ni_t      *gmnalni = libnal->libnal_data;
 120
 121         if (nal->nal_refct != 0) {
 122                 /* This module got the first ref */
 123                 PORTAL_MODULE_UNUSE;
 124                 return;
 125         }
 126
 127         CDEBUG(D_TRACE, "gmnal_api_shutdown: gmnalni [%p]\n", gmnalni);
 128
 129         /* Stop portals calling our ioctl handler */
 130         libcfs_nal_cmd_unregister(GMNAL);
 131
 132         /* stop processing messages */
 133         gmnal_stop_ctthread(gmnalni);
 134         gmnal_stop_rxthread(gmnalni);
 135
 136         gm_close(gmnalni->gmni_port);
 137         gm_finalize();
 138
 139         lib_fini(libnal);
 140
 141         gmnal_free_txs(gmnalni);
 142         gmnal_free_rxs(gmnalni);
 143
 144         PORTAL_FREE(gmnalni, sizeof(*gmnalni));
 145         PORTAL_FREE(libnal, sizeof(*libnal));
 146 }
 147
 148 int
 149 gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid,
 150                   ptl_ni_limits_t *requested_limits,
 151                   ptl_ni_limits_t *actual_limits)
 152 {
 153
 154         lib_nal_t       *libnal = NULL;
 155         gmnal_ni_t      *gmnalni = NULL;
 156         gmnal_rx_t      *rx = NULL;
 157         gm_status_t      gm_status;
 158         ptl_process_id_t process_id;
 159         int              rc;
 160
 161         if (nal->nal_refct != 0) {
 162                 if (actual_limits != NULL) {
 163                         libnal = (lib_nal_t *)nal->nal_data;
 164                         *actual_limits = libnal->libnal_ni.ni_actual_limits;
 165                 }
 166                 PORTAL_MODULE_USE;
 167                 return PTL_OK;
 168         }
 169
 170         /* Called on first PtlNIInit() */
 171         CDEBUG(D_TRACE, "startup\n");
 172
 173         PORTAL_ALLOC(gmnalni, sizeof(*gmnalni));
 174         if (gmnalni == NULL) {
 175                 CERROR("can't allocate gmnalni\n");
 176                 return PTL_FAIL;
 177         }
 178
 179         PORTAL_ALLOC(libnal, sizeof(*libnal));
 180         if (libnal == NULL) {
 181                 CERROR("can't allocate lib_nal\n");
 182                 goto failed_0;
 183         }
 184
 185         memset(gmnalni, 0, sizeof(*gmnalni));
 186         gmnalni->gmni_libnal = libnal;
 187         spin_lock_init(&gmnalni->gmni_gm_lock);
 188
 189         *libnal = (lib_nal_t) {
 190                 .libnal_send       = gmnal_cb_send,
 191                 .libnal_send_pages = gmnal_cb_send_pages,
 192                 .libnal_recv       = gmnal_cb_recv,
 193                 .libnal_recv_pages = gmnal_cb_recv_pages,
 194                 .libnal_dist       = gmnal_cb_dist,
 195                 .libnal_data       = gmnalni,
 196         };
 197
 198         /*
 199          *      initialise the interface,
 200          */
 201         CDEBUG(D_NET, "Calling gm_init\n");
 202         if (gm_init() != GM_SUCCESS) {
 203                 CERROR("call to gm_init failed\n");
 204                 goto failed_1;
 205         }
 206
 207         CDEBUG(D_NET, "Calling gm_open with port [%d], "
 208                "name [%s], version [%d]\n", gm_port_id,
 209                "gmnal", GM_API_VERSION);
 210
 211         gm_status = gm_open(&gmnalni->gmni_port, 0, gm_port_id, "gmnal",
 212                             GM_API_VERSION);
 213
 214         if (gm_status != GM_SUCCESS) {
 215                 CERROR("Can't open GM port %d: %d (%s)\n",
 216                        gm_port_id, gm_status, gmnal_gmstatus2str(gm_status));
 217                 goto failed_2;
 218         }
 219
 220         CDEBUG(D_NET,"gm_open succeeded port[%p]\n",gmnalni->gmni_port);
 221
 222         gmnalni->gmni_msg_size = offsetof(gmnal_msg_t,
 223                                           gmm_u.immediate.gmim_payload[PTL_MTU]);
 224         CWARN("Msg size %08x\n", gmnalni->gmni_msg_size);
 225
 226         if (gmnal_alloc_rxs(gmnalni) != 0) {
 227                 CERROR("Failed to allocate rx descriptors\n");
 228                 goto failed_3;
 229         }
 230
 231         if (gmnal_alloc_txs(gmnalni) != 0) {
 232                 CERROR("Failed to allocate tx descriptors\n");
 233                 goto failed_3;
 234         }
 235
 236         process_id.pid = requested_pid;
 237         process_id.nid = gmnal_get_local_nid(gmnalni);
 238         if (process_id.nid == PTL_NID_ANY)
 239                 goto failed_3;
 240
 241         CDEBUG(D_NET, "portals_pid is [%u]\n", process_id.pid);
 242         CDEBUG(D_NET, "portals_nid is ["LPU64"]\n", process_id.nid);
 243
 244         /*      Hang out a bunch of small receive buffers
 245          *      In fact hang them all out */
 246         for (rx = gmnalni->gmni_rx; rx != NULL; rx = rx->rx_next)
 247                 gmnal_post_rx(gmnalni, rx);
 248
 249         if (lib_init(libnal, nal, process_id,
 250                      requested_limits, actual_limits) != PTL_OK) {
 251                 CERROR("lib_init failed\n");
 252                 goto failed_3;
 253         }
 254
 255         /* Now that we have initialised the portals library, start receive
 256          * threads, we do this to avoid processing messages before we can parse
 257          * them */
 258         rc = gmnal_start_kernel_threads(gmnalni);
 259         if (rc != 0) {
 260                 CERROR("Can't start threads: %d\n", rc);
 261                 goto failed_3;
 262         }
 263
 264         rc = libcfs_nal_cmd_register(GMNAL, &gmnal_cmd, libnal->libnal_data);
 265         if (rc != 0) {
 266                 CDEBUG(D_NET, "libcfs_nal_cmd_register failed: %d\n", rc);
 267                 goto failed_4;
 268         }
 269
 270         CDEBUG(D_NET, "gmnal_init finished\n");
 271         return PTL_OK;
 272
 273  failed_4:
 274         gmnal_stop_rxthread(gmnalni);
 275         gmnal_stop_ctthread(gmnalni);
 276
 277  failed_3:
 278         gm_close(gmnalni->gmni_port);
 279
 280  failed_2:
 281         gm_finalize();
 282
 283         /* safe to free buffers after network has been shut down */
 284         gmnal_free_txs(gmnalni);
 285         gmnal_free_rxs(gmnalni);
 286
 287  failed_1:
 288         PORTAL_FREE(libnal, sizeof(*libnal));
 289
 290  failed_0:
 291         PORTAL_FREE(gmnalni, sizeof(*gmnalni));
 292
 293         return PTL_FAIL;
 294 }
 295
 296 ptl_handle_ni_t kgmnal_ni;
 297 nal_t           the_gm_nal;
 298
 299 /*
 300  *        Called when module loaded
 301  */
 302 int gmnal_init(void)
 303 {
 304         int    rc;
 305
 306         CDEBUG(D_NET, "reset nal[%p]\n", &the_gm_nal);
 307
 308         the_gm_nal = (nal_t) {
 309                 .nal_ni_init = gmnal_api_startup,
 310                 .nal_ni_fini = gmnal_api_shutdown,
 311                 .nal_data = NULL,
 312         };
 313
 314         rc = ptl_register_nal(GMNAL, &the_gm_nal);
 315         if (rc != PTL_OK)
 316                 CERROR("Can't register GMNAL: %d\n", rc);
 317         rc = PtlNIInit(GMNAL, LUSTRE_SRV_PTL_PID, NULL, NULL, &kgmnal_ni);
 318         if (rc != PTL_OK && rc != PTL_IFACE_DUP) {
 319                 ptl_unregister_nal(GMNAL);
 320                 return (-ENODEV);
 321         }
 322
 323         return (rc);
 324 }
 325
 326
 327 /*
 328  *      Called when module removed
 329  */
 330 void gmnal_fini()
 331 {
 332         CDEBUG(D_TRACE, "gmnal_fini\n");
 333
 334         PtlNIFini(kgmnal_ni);
 335
 336         ptl_unregister_nal(GMNAL);
 337 }