lnet/klnds/gmlnd/gmlnd_api.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  Copyright (c) 2003 Los Alamos National Laboratory (LANL)
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org/
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  */
  21
  22 /*
  23  *      Implements the API NAL functions
  24  */
  25
  26 #include "gmnal.h"
  27
  28 int
  29 gmnal_cmd(struct portals_cfg *pcfg, void *private)
  30 {
  31         gmnal_ni_t      *gmnalni = private;
  32         char            *name;
  33         int              nid;
  34         int              gmid;
  35         gm_status_t      gm_status;
  36
  37         CDEBUG(D_TRACE, "gmnal_cmd [%d] private [%p]\n",
  38                pcfg->pcfg_command, private);
  39         gmnalni = (gmnal_ni_t*)private;
  40
  41         switch(pcfg->pcfg_command) {
  42         case GMNAL_IOC_GET_GNID:
  43
  44                 PORTAL_ALLOC(name, pcfg->pcfg_plen1);
  45                 copy_from_user(name, PCFG_PBUF(pcfg, 1), pcfg->pcfg_plen1);
  46
  47                 gm_status = gm_host_name_to_node_id_ex(gmnalni->gmni_port, 0,
  48                                                        name, &nid);
  49                 if (gm_status != GM_SUCCESS) {
  50                         CDEBUG(D_NET, "gm_host_name_to_node_id_ex(...host %s) "
  51                                "failed[%d]\n", name, gm_status);
  52                         return -ENOENT;
  53                 }
  54
  55                 CDEBUG(D_NET, "Local node %s id is [%d]\n", name, nid);
  56                 gm_status = gm_node_id_to_global_id(gmnalni->gmni_port,
  57                                                     nid, &gmid);
  58                 if (gm_status != GM_SUCCESS) {
  59                         CDEBUG(D_NET, "gm_node_id_to_global_id failed[%d]\n",
  60                                gm_status);
  61                         return -ENOENT;
  62                 }
  63
  64                 CDEBUG(D_NET, "Global node is is [%u][%x]\n", gmid, gmid);
  65                 copy_to_user(PCFG_PBUF(pcfg, 2), &gmid, pcfg->pcfg_plen2);
  66                 return 0;
  67
  68         case NAL_CMD_REGISTER_MYNID:
  69                 /* Same NID OK */
  70                 if (pcfg->pcfg_nid == gmnalni->gmni_libnal->libnal_ni.ni_pid.nid)
  71                         return 0;
  72
  73                 CERROR("Can't change NID from "LPD64" to "LPD64"\n",
  74                        gmnalni->gmni_libnal->libnal_ni.ni_pid.nid,
  75                        pcfg->pcfg_nid);
  76                 return -EINVAL;
  77
  78         default:
  79                 CERROR ("gmnal_cmd UNKNOWN[%d]\n", pcfg->pcfg_command);
  80                 return -EINVAL;
  81         }
  82         /* not reached */
  83 }
  84
  85 ptl_nid_t
  86 gmnal_get_local_nid (gmnal_ni_t *gmnalni)
  87 {
  88         unsigned int     local_gmid;
  89         unsigned int     global_gmid;
  90         ptl_nid_t        nid;
  91         gm_status_t      gm_status;
  92
  93         /* Called before anything initialised: no need to lock */
  94         gm_status = gm_get_node_id(gmnalni->gmni_port, &local_gmid);
  95         if (gm_status != GM_SUCCESS)
  96                 return PTL_NID_ANY;
  97
  98         CDEBUG(D_NET, "Local node id is [%u]\n", local_gmid);
  99
 100         gm_status = gm_node_id_to_global_id(gmnalni->gmni_port,
 101                                             local_gmid,
 102                                             &global_gmid);
 103         if (gm_status != GM_SUCCESS)
 104                 return PTL_NID_ANY;
 105
 106         CDEBUG(D_NET, "Global node id is [%u]\n", global_gmid);
 107
 108         nid = (__u64)global_gmid;
 109         LASSERT (nid != PTL_NID_ANY);
 110
 111         return global_gmid;
 112 }
 113
 114
 115 void
 116 gmnal_api_shutdown(nal_t *nal)
 117 {
 118         lib_nal_t       *libnal = nal->nal_data;
 119         gmnal_ni_t      *gmnalni = libnal->libnal_data;
 120
 121         if (nal->nal_refct != 0) {
 122                 /* This module got the first ref */
 123                 PORTAL_MODULE_UNUSE;
 124                 return;
 125         }
 126
 127         CDEBUG(D_TRACE, "gmnal_api_shutdown: gmnalni [%p]\n", gmnalni);
 128
 129         /* Stop portals calling our ioctl handler */
 130         libcfs_nal_cmd_unregister(GMNAL);
 131
 132         /* stop processing messages */
 133         gmnal_stop_threads(gmnalni);
 134
 135         gm_close(gmnalni->gmni_port);
 136         gm_finalize();
 137
 138         lib_fini(libnal);
 139
 140         gmnal_free_txs(gmnalni);
 141         gmnal_free_rxs(gmnalni);
 142
 143         PORTAL_FREE(gmnalni, sizeof(*gmnalni));
 144         PORTAL_FREE(libnal, sizeof(*libnal));
 145 }
 146
 147 int
 148 gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid,
 149                   ptl_ni_limits_t *requested_limits,
 150                   ptl_ni_limits_t *actual_limits)
 151 {
 152
 153         lib_nal_t       *libnal = NULL;
 154         gmnal_ni_t      *gmnalni = NULL;
 155         gmnal_rx_t      *rx = NULL;
 156         gm_status_t      gm_status;
 157         ptl_process_id_t process_id;
 158         int              rc;
 159
 160         if (nal->nal_refct != 0) {
 161                 if (actual_limits != NULL) {
 162                         libnal = (lib_nal_t *)nal->nal_data;
 163                         *actual_limits = libnal->libnal_ni.ni_actual_limits;
 164                 }
 165                 PORTAL_MODULE_USE;
 166                 return PTL_OK;
 167         }
 168
 169         /* Called on first PtlNIInit() */
 170         CDEBUG(D_TRACE, "startup\n");
 171
 172         PORTAL_ALLOC(gmnalni, sizeof(*gmnalni));
 173         if (gmnalni == NULL) {
 174                 CERROR("can't allocate gmnalni\n");
 175                 return PTL_FAIL;
 176         }
 177
 178         PORTAL_ALLOC(libnal, sizeof(*libnal));
 179         if (libnal == NULL) {
 180                 CERROR("can't allocate lib_nal\n");
 181                 goto failed_0;
 182         }
 183
 184         memset(gmnalni, 0, sizeof(*gmnalni));
 185         gmnalni->gmni_libnal = libnal;
 186         spin_lock_init(&gmnalni->gmni_gm_lock);
 187
 188         *libnal = (lib_nal_t) {
 189                 .libnal_send       = gmnal_cb_send,
 190                 .libnal_send_pages = gmnal_cb_send_pages,
 191                 .libnal_recv       = gmnal_cb_recv,
 192                 .libnal_recv_pages = gmnal_cb_recv_pages,
 193                 .libnal_dist       = gmnal_cb_dist,
 194                 .libnal_data       = gmnalni,
 195         };
 196
 197         /*
 198          *      initialise the interface,
 199          */
 200         CDEBUG(D_NET, "Calling gm_init\n");
 201         if (gm_init() != GM_SUCCESS) {
 202                 CERROR("call to gm_init failed\n");
 203                 goto failed_1;
 204         }
 205
 206         CDEBUG(D_NET, "Calling gm_open with port [%d], "
 207                "name [%s], version [%d]\n", gm_port_id,
 208                "gmnal", GM_API_VERSION);
 209
 210         gm_status = gm_open(&gmnalni->gmni_port, 0, gm_port_id, "gmnal",
 211                             GM_API_VERSION);
 212
 213         if (gm_status != GM_SUCCESS) {
 214                 CERROR("Can't open GM port %d: %d (%s)\n",
 215                        gm_port_id, gm_status, gmnal_gmstatus2str(gm_status));
 216                 goto failed_2;
 217         }
 218
 219         CDEBUG(D_NET,"gm_open succeeded port[%p]\n",gmnalni->gmni_port);
 220
 221         gmnalni->gmni_msg_size = offsetof(gmnal_msg_t,
 222                                           gmm_u.immediate.gmim_payload[PTL_MTU]);
 223         CWARN("Msg size %08x\n", gmnalni->gmni_msg_size);
 224
 225         if (gmnal_alloc_rxs(gmnalni) != 0) {
 226                 CERROR("Failed to allocate rx descriptors\n");
 227                 goto failed_3;
 228         }
 229
 230         if (gmnal_alloc_txs(gmnalni) != 0) {
 231                 CERROR("Failed to allocate tx descriptors\n");
 232                 goto failed_3;
 233         }
 234
 235         process_id.pid = requested_pid;
 236         process_id.nid = gmnal_get_local_nid(gmnalni);
 237         if (process_id.nid == PTL_NID_ANY)
 238                 goto failed_3;
 239
 240         CDEBUG(D_NET, "portals_pid is [%u]\n", process_id.pid);
 241         CDEBUG(D_NET, "portals_nid is ["LPU64"]\n", process_id.nid);
 242
 243         /*      Hang out a bunch of small receive buffers
 244          *      In fact hang them all out */
 245         for (rx = gmnalni->gmni_rx; rx != NULL; rx = rx->rx_next)
 246                 gmnal_post_rx(gmnalni, rx);
 247
 248         if (lib_init(libnal, nal, process_id,
 249                      requested_limits, actual_limits) != PTL_OK) {
 250                 CERROR("lib_init failed\n");
 251                 goto failed_3;
 252         }
 253
 254         /* Now that we have initialised the portals library, start receive
 255          * threads, we do this to avoid processing messages before we can parse
 256          * them */
 257         rc = gmnal_start_threads(gmnalni);
 258         if (rc != 0) {
 259                 CERROR("Can't start threads: %d\n", rc);
 260                 goto failed_3;
 261         }
 262
 263         rc = libcfs_nal_cmd_register(GMNAL, &gmnal_cmd, libnal->libnal_data);
 264         if (rc != 0) {
 265                 CDEBUG(D_NET, "libcfs_nal_cmd_register failed: %d\n", rc);
 266                 goto failed_4;
 267         }
 268
 269         CDEBUG(D_NET, "gmnal_init finished\n");
 270         return PTL_OK;
 271
 272  failed_4:
 273         gmnal_stop_threads(gmnalni);
 274
 275  failed_3:
 276         gm_close(gmnalni->gmni_port);
 277
 278  failed_2:
 279         gm_finalize();
 280
 281         /* safe to free buffers after network has been shut down */
 282         gmnal_free_txs(gmnalni);
 283         gmnal_free_rxs(gmnalni);
 284
 285  failed_1:
 286         PORTAL_FREE(libnal, sizeof(*libnal));
 287
 288  failed_0:
 289         PORTAL_FREE(gmnalni, sizeof(*gmnalni));
 290
 291         return PTL_FAIL;
 292 }
 293
 294 ptl_handle_ni_t kgmnal_ni;
 295 nal_t           the_gm_nal;
 296
 297 /*
 298  *        Called when module loaded
 299  */
 300 int gmnal_init(void)
 301 {
 302         int    rc;
 303
 304         CDEBUG(D_NET, "reset nal[%p]\n", &the_gm_nal);
 305
 306         the_gm_nal = (nal_t) {
 307                 .nal_ni_init = gmnal_api_startup,
 308                 .nal_ni_fini = gmnal_api_shutdown,
 309                 .nal_data = NULL,
 310         };
 311
 312         rc = ptl_register_nal(GMNAL, &the_gm_nal);
 313         if (rc != PTL_OK)
 314                 CERROR("Can't register GMNAL: %d\n", rc);
 315         rc = PtlNIInit(GMNAL, LUSTRE_SRV_PTL_PID, NULL, NULL, &kgmnal_ni);
 316         if (rc != PTL_OK && rc != PTL_IFACE_DUP) {
 317                 ptl_unregister_nal(GMNAL);
 318                 return (-ENODEV);
 319         }
 320
 321         return (rc);
 322 }
 323
 324
 325 /*
 326  *      Called when module removed
 327  */
 328 void gmnal_fini()
 329 {
 330         CDEBUG(D_TRACE, "gmnal_fini\n");
 331
 332         PtlNIFini(kgmnal_ni);
 333
 334         ptl_unregister_nal(GMNAL);
 335 }