lustre/portals/knals/gmnal/gmnal_api.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  Copyright (c) 2003 Los Alamos National Laboratory (LANL)
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org/
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  */
  21
  22 /*
  23  *      Implements the API NAL functions
  24  */
  25
  26 #include "gmnal.h"
  27
  28
  29
  30 gmnal_data_t    *global_nal_data = NULL;
  31 #define         GLOBAL_NID_STR_LEN      16
  32 char            global_nid_str[GLOBAL_NID_STR_LEN] = {0};
  33
  34 /*
  35  *      Write the global nid /proc/sys/gmnal/globalnid
  36  */
  37 #define GMNAL_SYSCTL    201
  38 #define GMNAL_SYSCTL_GLOBALNID  1
  39
  40 static ctl_table gmnal_sysctl_table[] = {
  41         {GMNAL_SYSCTL_GLOBALNID, "globalnid",
  42          global_nid_str, GLOBAL_NID_STR_LEN,
  43          0444, NULL, &proc_dostring},
  44         { 0 }
  45 };
  46
  47
  48 static ctl_table gmnalnal_top_sysctl_table[] = {
  49         {GMNAL_SYSCTL, "gmnal", NULL, 0, 0555, gmnal_sysctl_table},
  50         { 0 }
  51 };
  52
  53 /*
  54  *      gmnal_api_shutdown
  55  *      nal_refct == 0 => called on last matching PtlNIFini()
  56  *      Close down this interface and free any resources associated with it
  57  *      nal_t   nal     our nal to shutdown
  58  */
  59 void
  60 gmnal_api_shutdown(nal_t *nal, int interface)
  61 {
  62         gmnal_data_t    *nal_data;
  63         lib_nal_t       *libnal;
  64
  65         if (nal->nal_refct != 0)
  66                 return;
  67
  68         CDEBUG(D_TRACE, "gmnal_api_shutdown: nal_data [%p]\n", nal_data);
  69
  70         LASSERT(nal == global_nal_data->nal);
  71         libnal = (lib_nal_t *)nal->nal_data;
  72         nal_data = (gmnal_data_t *)libnal->libnal_data;
  73         LASSERT(nal_data == global_nal_data);
  74
  75         /* Stop portals calling our ioctl handler */
  76         libcfs_nal_cmd_unregister(GMNAL);
  77
  78         /* XXX for shutdown "under fire" we probably need to set a shutdown
  79          * flag so when lib calls us we fail immediately and dont queue any
  80          * more work but our threads can still call into lib OK.  THEN
  81          * shutdown our threads, THEN lib_fini() */
  82         lib_fini(libnal);
  83
  84         gmnal_stop_rxthread(nal_data);
  85         gmnal_stop_ctthread(nal_data);
  86         gmnal_free_txd(nal_data);
  87         gmnal_free_srxd(nal_data);
  88         GMNAL_GM_LOCK(nal_data);
  89         gm_close(nal_data->gm_port);
  90         gm_finalize();
  91         GMNAL_GM_UNLOCK(nal_data);
  92         if (nal_data->sysctl)
  93                 unregister_sysctl_table (nal_data->sysctl);
  94         /* Don't free 'nal'; it's a static struct */
  95         PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
  96         PORTAL_FREE(libnal, sizeof(lib_nal_t));
  97
  98         global_nal_data = NULL;
  99         PORTAL_MODULE_UNUSE;
 100 }
 101
 102
 103 int
 104 gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid,
 105                   ptl_ni_limits_t *requested_limits,
 106                   ptl_ni_limits_t *actual_limits)
 107 {
 108
 109         lib_nal_t       *libnal = NULL;
 110         gmnal_data_t    *nal_data = NULL;
 111         gmnal_srxd_t    *srxd = NULL;
 112         gm_status_t     gm_status;
 113         unsigned int    local_nid = 0, global_nid = 0;
 114         ptl_process_id_t process_id;
 115
 116         if (nal->nal_refct != 0) {
 117                 if (actual_limits != NULL) {
 118                         libnal = (lib_nal_t *)nal->nal_data;
 119                         *actual_limits = nal->libnal_ni.ni_actual_limits;
 120                 return (PTL_OK);
 121         }
 122
 123         /* Called on first PtlNIInit() */
 124
 125         CDEBUG(D_TRACE, "startup\n");
 126
 127         LASSERT(global_nal_data == NULL);
 128
 129         PORTAL_ALLOC(nal_data, sizeof(gmnal_data_t));
 130         if (!nal_data) {
 131                 CDEBUG(D_ERROR, "can't get memory\n");
 132                 return(PTL_NO_SPACE);
 133         }
 134         memset(nal_data, 0, sizeof(gmnal_data_t));
 135         /*
 136          *      set the small message buffer size
 137          */
 138
 139         CDEBUG(D_INFO, "Allocd and reset nal_data[%p]\n", nal_data);
 140         CDEBUG(D_INFO, "small_msg_size is [%d]\n", nal_data->small_msg_size);
 141
 142         PORTAL_ALLOC(libnal, sizeof(lib_nal_t));
 143         if (!libnal) {
 144                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 145                 return(PTL_NO_SPACE);
 146         }
 147         memset(libnal, 0, sizeof(lib_nal_t));
 148         CDEBUG(D_INFO, "Allocd and reset libnal[%p]\n", libnal);
 149
 150         GMNAL_INIT_NAL_CB(libnal);
 151         /*
 152          *      String them all together
 153          */
 154         libnal->libnal_data = (void*)nal_data;
 155         nal_data->nal = nal;
 156         nal_data->libnal = libnal;
 157
 158         GMNAL_GM_LOCK_INIT(nal_data);
 159
 160
 161         /*
 162          *      initialise the interface,
 163          */
 164         CDEBUG(D_INFO, "Calling gm_init\n");
 165         if (gm_init() != GM_SUCCESS) {
 166                 CDEBUG(D_ERROR, "call to gm_init failed\n");
 167                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 168                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 169                 return(PTL_FAIL);
 170         }
 171
 172
 173         CDEBUG(D_NET, "Calling gm_open with interface [%d], port [%d], "
 174                "name [%s], version [%d]\n", interface, GMNAL_GM_PORT,
 175                "gmnal", GM_API_VERSION);
 176
 177         GMNAL_GM_LOCK(nal_data);
 178         gm_status = gm_open(&nal_data->gm_port, 0, GMNAL_GM_PORT, "gmnal",
 179                             GM_API_VERSION);
 180         GMNAL_GM_UNLOCK(nal_data);
 181
 182         CDEBUG(D_INFO, "gm_open returned [%d]\n", gm_status);
 183         if (gm_status == GM_SUCCESS) {
 184                 CDEBUG(D_INFO, "gm_open succeeded port[%p]\n",
 185                        nal_data->gm_port);
 186         } else {
 187                 switch(gm_status) {
 188                 case(GM_INVALID_PARAMETER):
 189                         CDEBUG(D_ERROR, "gm_open Failure. Invalid Parameter\n");
 190                         break;
 191                 case(GM_BUSY):
 192                         CDEBUG(D_ERROR, "gm_open Failure. GM Busy\n");
 193                         break;
 194                 case(GM_NO_SUCH_DEVICE):
 195                         CDEBUG(D_ERROR, "gm_open Failure. No such device\n");
 196                         break;
 197                 case(GM_INCOMPATIBLE_LIB_AND_DRIVER):
 198                         CDEBUG(D_ERROR, "gm_open Failure. Incompatile lib "
 199                                "and driver\n");
 200                         break;
 201                 case(GM_OUT_OF_MEMORY):
 202                         CDEBUG(D_ERROR, "gm_open Failure. Out of Memory\n");
 203                         break;
 204                 default:
 205                         CDEBUG(D_ERROR, "gm_open Failure. Unknow error "
 206                                "code [%d]\n", gm_status);
 207                         break;
 208                 }
 209                 GMNAL_GM_LOCK(nal_data);
 210                 gm_finalize();
 211                 GMNAL_GM_UNLOCK(nal_data);
 212                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 213                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 214                 return(PTL_FAIL);
 215         }
 216
 217
 218         nal_data->small_msg_size = gmnal_small_msg_size;
 219         nal_data->small_msg_gmsize =
 220                         gm_min_size_for_length(gmnal_small_msg_size);
 221
 222         if (gmnal_alloc_srxd(nal_data) != GMNAL_STATUS_OK) {
 223                 CDEBUG(D_ERROR, "Failed to allocate small rx descriptors\n");
 224                 gmnal_free_txd(nal_data);
 225                 GMNAL_GM_LOCK(nal_data);
 226                 gm_close(nal_data->gm_port);
 227                 gm_finalize();
 228                 GMNAL_GM_UNLOCK(nal_data);
 229                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 230                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 231                 return(PTL_FAIL);
 232         }
 233
 234
 235         /*
 236          *      Hang out a bunch of small receive buffers
 237          *      In fact hang them all out
 238          */
 239         while((srxd = gmnal_get_srxd(nal_data, 0))) {
 240                 CDEBUG(D_NET, "giving [%p] to gm_provide_recvive_buffer\n",
 241                        srxd->buffer);
 242                 GMNAL_GM_LOCK(nal_data);
 243                 gm_provide_receive_buffer_with_tag(nal_data->gm_port,
 244                                                    srxd->buffer, srxd->gmsize,
 245                                                    GM_LOW_PRIORITY, 0);
 246                 GMNAL_GM_UNLOCK(nal_data);
 247         }
 248
 249         /*
 250          *      Allocate pools of small tx buffers and descriptors
 251          */
 252         if (gmnal_alloc_txd(nal_data) != GMNAL_STATUS_OK) {
 253                 CDEBUG(D_ERROR, "Failed to allocate small tx descriptors\n");
 254                 GMNAL_GM_LOCK(nal_data);
 255                 gm_close(nal_data->gm_port);
 256                 gm_finalize();
 257                 GMNAL_GM_UNLOCK(nal_data);
 258                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 259                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 260                 return(PTL_FAIL);
 261         }
 262
 263         gmnal_start_kernel_threads(nal_data);
 264
 265         while (nal_data->rxthread_flag != GMNAL_RXTHREADS_STARTED) {
 266                 gmnal_yield(1);
 267                 CDEBUG(D_INFO, "Waiting for receive thread signs of life\n");
 268         }
 269
 270         CDEBUG(D_INFO, "receive thread seems to have started\n");
 271
 272
 273         /*
 274          *      Initialise the portals library
 275          */
 276         CDEBUG(D_NET, "Getting node id\n");
 277         GMNAL_GM_LOCK(nal_data);
 278         gm_status = gm_get_node_id(nal_data->gm_port, &local_nid);
 279         GMNAL_GM_UNLOCK(nal_data);
 280         if (gm_status != GM_SUCCESS) {
 281                 gmnal_stop_rxthread(nal_data);
 282                 gmnal_stop_ctthread(nal_data);
 283                 CDEBUG(D_ERROR, "can't determine node id\n");
 284                 gmnal_free_txd(nal_data);
 285                 gmnal_free_srxd(nal_data);
 286                 GMNAL_GM_LOCK(nal_data);
 287                 gm_close(nal_data->gm_port);
 288                 gm_finalize();
 289                 GMNAL_GM_UNLOCK(nal_data);
 290                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 291                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 292                 return(PTL_FAIL);
 293         }
 294         nal_data->gm_local_nid = local_nid;
 295         CDEBUG(D_INFO, "Local node id is [%u]\n", local_nid);
 296         GMNAL_GM_LOCK(nal_data);
 297         gm_status = gm_node_id_to_global_id(nal_data->gm_port, local_nid,
 298                                             &global_nid);
 299         GMNAL_GM_UNLOCK(nal_data);
 300         if (gm_status != GM_SUCCESS) {
 301                 CDEBUG(D_ERROR, "failed to obtain global id\n");
 302                 gmnal_stop_rxthread(nal_data);
 303                 gmnal_stop_ctthread(nal_data);
 304                 gmnal_free_txd(nal_data);
 305                 gmnal_free_srxd(nal_data);
 306                 GMNAL_GM_LOCK(nal_data);
 307                 gm_close(nal_data->gm_port);
 308                 gm_finalize();
 309                 GMNAL_GM_UNLOCK(nal_data);
 310                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 311                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 312                 return(PTL_FAIL);
 313         }
 314         CDEBUG(D_INFO, "Global node id is [%u]\n", global_nid);
 315         nal_data->gm_global_nid = global_nid;
 316         snprintf(global_nid_str, GLOBAL_NID_STR_LEN, "%u", global_nid);
 317
 318 /*
 319         pid = gm_getpid();
 320 */
 321         process_id.pid = requested_pid;
 322         process_id.nid = global_nid;
 323
 324         CDEBUG(D_INFO, "portals_pid is [%u]\n", process_id.pid);
 325         CDEBUG(D_INFO, "portals_nid is ["LPU64"]\n", process_id.nid);
 326
 327         CDEBUG(D_PORTALS, "calling lib_init\n");
 328         if (lib_init(libnal, nal, process_id,
 329                      requested_limits, actual_limits) != PTL_OK) {
 330                 CDEBUG(D_ERROR, "lib_init failed\n");
 331                 gmnal_stop_rxthread(nal_data);
 332                 gmnal_stop_ctthread(nal_data);
 333                 gmnal_free_txd(nal_data);
 334                 gmnal_free_srxd(nal_data);
 335                 GMNAL_GM_LOCK(nal_data);
 336                 gm_close(nal_data->gm_port);
 337                 gm_finalize();
 338                 GMNAL_GM_UNLOCK(nal_data);
 339                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 340                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 341                 return(PTL_FAIL);
 342
 343         }
 344
 345         if (libcfs_nal_cmd_register(GMNAL, &gmnal_cmd, nal->nal_data) != 0) {
 346                 CDEBUG(D_INFO, "libcfs_nal_cmd_register failed\n");
 347
 348                 /* XXX these cleanup cases should be restructured to
 349                  * minimise duplication... */
 350                 lib_fini(libnal);
 351
 352                 gmnal_stop_rxthread(nal_data);
 353                 gmnal_stop_ctthread(nal_data);
 354                 gmnal_free_txd(nal_data);
 355                 gmnal_free_srxd(nal_data);
 356                 GMNAL_GM_LOCK(nal_data);
 357                 gm_close(nal_data->gm_port);
 358                 gm_finalize();
 359                 GMNAL_GM_UNLOCK(nal_data);
 360                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 361                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 362                 return(PTL_FAIL);
 363         }
 364
 365         /* might be better to initialise this at module load rather than in
 366          * NAL startup */
 367         nal_data->sysctl = NULL;
 368         nal_data->sysctl = register_sysctl_table (gmnalnal_top_sysctl_table, 0);
 369
 370
 371         CDEBUG(D_INFO, "gmnal_init finished\n");
 372         global_nal_data = nal->nal_data;
 373
 374         /* no unload now until shutdown */
 375         PORTAL_MODULE_USE;
 376
 377         return(PTL_OK);
 378 }
 379
 380 nal_t the_gm_nal;
 381
 382 /*
 383  *        Called when module loaded
 384  */
 385 int gmnal_init(void)
 386 {
 387         int    rc;
 388
 389         memset(&the_gm_nal, 0, sizeof(nal_t));
 390         CDEBUG(D_INFO, "reset nal[%p]\n", &the_gm_nal);
 391         GMNAL_INIT_NAL(&the_gm_nal);
 392
 393         rc = ptl_register_nal(GMNAL, &the_gm_nal);
 394         if (rc != PTL_OK)
 395                 CERROR("Can't register GMNAL: %d\n", rc);
 396
 397         return (rc);
 398 }
 399
 400
 401
 402 /*
 403  *      Called when module removed
 404  */
 405 void gmnal_fini()
 406 {
 407         CDEBUG(D_TRACE, "gmnal_fini\n");
 408
 409         LASSERT(global_nal_data == NULL);
 410
 411         ptl_unregister_nal(GMNAL);
 412 }