lustre/portals/knals/gmnal/gmnal_api.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  Copyright (c) 2003 Los Alamos National Laboratory (LANL)
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org/
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  */
  21
  22 /*
  23  *      Implements the API NAL functions
  24  */
  25
  26 #include "gmnal.h"
  27
  28
  29
  30 gmnal_data_t    *global_nal_data = NULL;
  31 #define         GLOBAL_NID_STR_LEN      16
  32 char            global_nid_str[GLOBAL_NID_STR_LEN] = {0};
  33 ptl_handle_ni_t kgmnal_ni;
  34
  35 extern int gmnal_cmd(struct portals_cfg *pcfg, void *private);
  36
  37 /*
  38  *      Write the global nid /proc/sys/gmnal/globalnid
  39  */
  40 #define GMNAL_SYSCTL    201
  41 #define GMNAL_SYSCTL_GLOBALNID  1
  42
  43 static ctl_table gmnal_sysctl_table[] = {
  44         {GMNAL_SYSCTL_GLOBALNID, "globalnid",
  45          global_nid_str, GLOBAL_NID_STR_LEN,
  46          0444, NULL, &proc_dostring},
  47         { 0 }
  48 };
  49
  50
  51 static ctl_table gmnalnal_top_sysctl_table[] = {
  52         {GMNAL_SYSCTL, "gmnal", NULL, 0, 0555, gmnal_sysctl_table},
  53         { 0 }
  54 };
  55
  56 /*
  57  *      gmnal_api_shutdown
  58  *      nal_refct == 0 => called on last matching PtlNIFini()
  59  *      Close down this interface and free any resources associated with it
  60  *      nal_t   nal     our nal to shutdown
  61  */
  62 void
  63 gmnal_api_shutdown(nal_t *nal)
  64 {
  65         gmnal_data_t    *nal_data;
  66         lib_nal_t       *libnal;
  67
  68         if (nal->nal_refct != 0)
  69                 return;
  70
  71
  72         LASSERT(nal == global_nal_data->nal);
  73         libnal = (lib_nal_t *)nal->nal_data;
  74         nal_data = (gmnal_data_t *)libnal->libnal_data;
  75         LASSERT(nal_data == global_nal_data);
  76         CDEBUG(D_TRACE, "gmnal_api_shutdown: nal_data [%p]\n", nal_data);
  77
  78         /* Stop portals calling our ioctl handler */
  79         libcfs_nal_cmd_unregister(GMNAL);
  80
  81         /* XXX for shutdown "under fire" we probably need to set a shutdown
  82          * flag so when lib calls us we fail immediately and dont queue any
  83          * more work but our threads can still call into lib OK.  THEN
  84          * shutdown our threads, THEN lib_fini() */
  85         lib_fini(libnal);
  86
  87         gmnal_stop_rxthread(nal_data);
  88         gmnal_stop_ctthread(nal_data);
  89         gmnal_free_txd(nal_data);
  90         gmnal_free_srxd(nal_data);
  91         GMNAL_GM_LOCK(nal_data);
  92         gm_close(nal_data->gm_port);
  93         gm_finalize();
  94         GMNAL_GM_UNLOCK(nal_data);
  95         if (nal_data->sysctl)
  96                 unregister_sysctl_table (nal_data->sysctl);
  97         /* Don't free 'nal'; it's a static struct */
  98         PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
  99         PORTAL_FREE(libnal, sizeof(lib_nal_t));
 100
 101         global_nal_data = NULL;
 102         PORTAL_MODULE_UNUSE;
 103 }
 104
 105
 106 int
 107 gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid,
 108                   ptl_ni_limits_t *requested_limits,
 109                   ptl_ni_limits_t *actual_limits)
 110 {
 111
 112         lib_nal_t       *libnal = NULL;
 113         gmnal_data_t    *nal_data = NULL;
 114         gmnal_srxd_t    *srxd = NULL;
 115         gm_status_t     gm_status;
 116         unsigned int    local_nid = 0, global_nid = 0;
 117         ptl_process_id_t process_id;
 118
 119         if (nal->nal_refct != 0) {
 120                 if (actual_limits != NULL) {
 121                         libnal = (lib_nal_t *)nal->nal_data;
 122                         *actual_limits = libnal->libnal_ni.ni_actual_limits;
 123                 }
 124                 return (PTL_OK);
 125         }
 126
 127         /* Called on first PtlNIInit() */
 128
 129         CDEBUG(D_TRACE, "startup\n");
 130
 131         LASSERT(global_nal_data == NULL);
 132
 133         PORTAL_ALLOC(nal_data, sizeof(gmnal_data_t));
 134         if (!nal_data) {
 135                 CDEBUG(D_ERROR, "can't get memory\n");
 136                 return(PTL_NO_SPACE);
 137         }
 138         memset(nal_data, 0, sizeof(gmnal_data_t));
 139         /*
 140          *      set the small message buffer size
 141          */
 142
 143         CDEBUG(D_INFO, "Allocd and reset nal_data[%p]\n", nal_data);
 144         CDEBUG(D_INFO, "small_msg_size is [%d]\n", nal_data->small_msg_size);
 145
 146         PORTAL_ALLOC(libnal, sizeof(lib_nal_t));
 147         if (!libnal) {
 148                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 149                 return(PTL_NO_SPACE);
 150         }
 151         memset(libnal, 0, sizeof(lib_nal_t));
 152         CDEBUG(D_INFO, "Allocd and reset libnal[%p]\n", libnal);
 153
 154         GMNAL_INIT_NAL_CB(libnal);
 155         /*
 156          *      String them all together
 157          */
 158         libnal->libnal_data = (void*)nal_data;
 159         nal_data->nal = nal;
 160         nal_data->libnal = libnal;
 161
 162         GMNAL_GM_LOCK_INIT(nal_data);
 163
 164
 165         /*
 166          *      initialise the interface,
 167          */
 168         CDEBUG(D_INFO, "Calling gm_init\n");
 169         if (gm_init() != GM_SUCCESS) {
 170                 CDEBUG(D_ERROR, "call to gm_init failed\n");
 171                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 172                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 173                 return(PTL_FAIL);
 174         }
 175
 176
 177         CDEBUG(D_NET, "Calling gm_open with port [%d], "
 178                "name [%s], version [%d]\n", GMNAL_GM_PORT,
 179                "gmnal", GM_API_VERSION);
 180
 181         GMNAL_GM_LOCK(nal_data);
 182         gm_status = gm_open(&nal_data->gm_port, 0, GMNAL_GM_PORT, "gmnal",
 183                             GM_API_VERSION);
 184         GMNAL_GM_UNLOCK(nal_data);
 185
 186         CDEBUG(D_INFO, "gm_open returned [%d]\n", gm_status);
 187         if (gm_status == GM_SUCCESS) {
 188                 CDEBUG(D_INFO, "gm_open succeeded port[%p]\n",
 189                        nal_data->gm_port);
 190         } else {
 191                 switch(gm_status) {
 192                 case(GM_INVALID_PARAMETER):
 193                         CDEBUG(D_ERROR, "gm_open Failure. Invalid Parameter\n");
 194                         break;
 195                 case(GM_BUSY):
 196                         CDEBUG(D_ERROR, "gm_open Failure. GM Busy\n");
 197                         break;
 198                 case(GM_NO_SUCH_DEVICE):
 199                         CDEBUG(D_ERROR, "gm_open Failure. No such device\n");
 200                         break;
 201                 case(GM_INCOMPATIBLE_LIB_AND_DRIVER):
 202                         CDEBUG(D_ERROR, "gm_open Failure. Incompatile lib "
 203                                "and driver\n");
 204                         break;
 205                 case(GM_OUT_OF_MEMORY):
 206                         CDEBUG(D_ERROR, "gm_open Failure. Out of Memory\n");
 207                         break;
 208                 default:
 209                         CDEBUG(D_ERROR, "gm_open Failure. Unknow error "
 210                                "code [%d]\n", gm_status);
 211                         break;
 212                 }
 213                 GMNAL_GM_LOCK(nal_data);
 214                 gm_finalize();
 215                 GMNAL_GM_UNLOCK(nal_data);
 216                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 217                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 218                 return(PTL_FAIL);
 219         }
 220
 221
 222         nal_data->small_msg_size = gmnal_small_msg_size;
 223         nal_data->small_msg_gmsize =
 224                         gm_min_size_for_length(gmnal_small_msg_size);
 225
 226         if (gmnal_alloc_srxd(nal_data) != GMNAL_STATUS_OK) {
 227                 CDEBUG(D_ERROR, "Failed to allocate small rx descriptors\n");
 228                 gmnal_free_txd(nal_data);
 229                 GMNAL_GM_LOCK(nal_data);
 230                 gm_close(nal_data->gm_port);
 231                 gm_finalize();
 232                 GMNAL_GM_UNLOCK(nal_data);
 233                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 234                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 235                 return(PTL_FAIL);
 236         }
 237
 238
 239         /*
 240          *      Hang out a bunch of small receive buffers
 241          *      In fact hang them all out
 242          */
 243         while((srxd = gmnal_get_srxd(nal_data, 0))) {
 244                 CDEBUG(D_NET, "giving [%p] to gm_provide_recvive_buffer\n",
 245                        srxd->buffer);
 246                 GMNAL_GM_LOCK(nal_data);
 247                 gm_provide_receive_buffer_with_tag(nal_data->gm_port,
 248                                                    srxd->buffer, srxd->gmsize,
 249                                                    GM_LOW_PRIORITY, 0);
 250                 GMNAL_GM_UNLOCK(nal_data);
 251         }
 252
 253         /*
 254          *      Allocate pools of small tx buffers and descriptors
 255          */
 256         if (gmnal_alloc_txd(nal_data) != GMNAL_STATUS_OK) {
 257                 CDEBUG(D_ERROR, "Failed to allocate small tx descriptors\n");
 258                 GMNAL_GM_LOCK(nal_data);
 259                 gm_close(nal_data->gm_port);
 260                 gm_finalize();
 261                 GMNAL_GM_UNLOCK(nal_data);
 262                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 263                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 264                 return(PTL_FAIL);
 265         }
 266
 267         gmnal_start_kernel_threads(nal_data);
 268
 269         while (nal_data->rxthread_flag != GMNAL_RXTHREADS_STARTED) {
 270                 gmnal_yield(1);
 271                 CDEBUG(D_INFO, "Waiting for receive thread signs of life\n");
 272         }
 273
 274         CDEBUG(D_INFO, "receive thread seems to have started\n");
 275
 276
 277         /*
 278          *      Initialise the portals library
 279          */
 280         CDEBUG(D_NET, "Getting node id\n");
 281         GMNAL_GM_LOCK(nal_data);
 282         gm_status = gm_get_node_id(nal_data->gm_port, &local_nid);
 283         GMNAL_GM_UNLOCK(nal_data);
 284         if (gm_status != GM_SUCCESS) {
 285                 gmnal_stop_rxthread(nal_data);
 286                 gmnal_stop_ctthread(nal_data);
 287                 CDEBUG(D_ERROR, "can't determine node id\n");
 288                 gmnal_free_txd(nal_data);
 289                 gmnal_free_srxd(nal_data);
 290                 GMNAL_GM_LOCK(nal_data);
 291                 gm_close(nal_data->gm_port);
 292                 gm_finalize();
 293                 GMNAL_GM_UNLOCK(nal_data);
 294                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 295                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 296                 return(PTL_FAIL);
 297         }
 298
 299         nal_data->gm_local_nid = local_nid;
 300         CDEBUG(D_INFO, "Local node id is [%u]\n", local_nid);
 301
 302         GMNAL_GM_LOCK(nal_data);
 303         gm_status = gm_node_id_to_global_id(nal_data->gm_port, local_nid,
 304                                             &global_nid);
 305         GMNAL_GM_UNLOCK(nal_data);
 306         if (gm_status != GM_SUCCESS) {
 307                 CDEBUG(D_ERROR, "failed to obtain global id\n");
 308                 gmnal_stop_rxthread(nal_data);
 309                 gmnal_stop_ctthread(nal_data);
 310                 gmnal_free_txd(nal_data);
 311                 gmnal_free_srxd(nal_data);
 312                 GMNAL_GM_LOCK(nal_data);
 313                 gm_close(nal_data->gm_port);
 314                 gm_finalize();
 315                 GMNAL_GM_UNLOCK(nal_data);
 316                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 317                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 318                 return(PTL_FAIL);
 319         }
 320         CDEBUG(D_INFO, "Global node id is [%u]\n", global_nid);
 321         nal_data->gm_global_nid = global_nid;
 322         snprintf(global_nid_str, GLOBAL_NID_STR_LEN, "%u", global_nid);
 323
 324 /*
 325         pid = gm_getpid();
 326 */
 327         process_id.pid = requested_pid;
 328         process_id.nid = global_nid;
 329
 330         CDEBUG(D_INFO, "portals_pid is [%u]\n", process_id.pid);
 331         CDEBUG(D_INFO, "portals_nid is ["LPU64"]\n", process_id.nid);
 332
 333         CDEBUG(D_PORTALS, "calling lib_init\n");
 334         if (lib_init(libnal, nal, process_id,
 335                      requested_limits, actual_limits) != PTL_OK) {
 336                 CDEBUG(D_ERROR, "lib_init failed\n");
 337                 gmnal_stop_rxthread(nal_data);
 338                 gmnal_stop_ctthread(nal_data);
 339                 gmnal_free_txd(nal_data);
 340                 gmnal_free_srxd(nal_data);
 341                 GMNAL_GM_LOCK(nal_data);
 342                 gm_close(nal_data->gm_port);
 343                 gm_finalize();
 344                 GMNAL_GM_UNLOCK(nal_data);
 345                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 346                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 347                 return(PTL_FAIL);
 348
 349         }
 350
 351         if (libcfs_nal_cmd_register(GMNAL, &gmnal_cmd, libnal->libnal_data) != 0) {
 352                 CDEBUG(D_INFO, "libcfs_nal_cmd_register failed\n");
 353
 354                 /* XXX these cleanup cases should be restructured to
 355                  * minimise duplication... */
 356                 lib_fini(libnal);
 357
 358                 gmnal_stop_rxthread(nal_data);
 359                 gmnal_stop_ctthread(nal_data);
 360                 gmnal_free_txd(nal_data);
 361                 gmnal_free_srxd(nal_data);
 362                 GMNAL_GM_LOCK(nal_data);
 363                 gm_close(nal_data->gm_port);
 364                 gm_finalize();
 365                 GMNAL_GM_UNLOCK(nal_data);
 366                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 367                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
 368                 return(PTL_FAIL);
 369         }
 370
 371         /* might be better to initialise this at module load rather than in
 372          * NAL startup */
 373         nal_data->sysctl = NULL;
 374         nal_data->sysctl = register_sysctl_table (gmnalnal_top_sysctl_table, 0);
 375
 376
 377         CDEBUG(D_INFO, "gmnal_init finished\n");
 378         global_nal_data = nal->nal_data;
 379
 380         /* no unload now until shutdown */
 381         PORTAL_MODULE_USE;
 382
 383         return(PTL_OK);
 384 }
 385
 386 nal_t the_gm_nal;
 387
 388 /*
 389  *        Called when module loaded
 390  */
 391 int gmnal_init(void)
 392 {
 393         int    rc;
 394
 395         memset(&the_gm_nal, 0, sizeof(nal_t));
 396         CDEBUG(D_INFO, "reset nal[%p]\n", &the_gm_nal);
 397         GMNAL_INIT_NAL(&the_gm_nal);
 398
 399         rc = ptl_register_nal(GMNAL, &the_gm_nal);
 400         if (rc != PTL_OK)
 401                 CERROR("Can't register GMNAL: %d\n", rc);
 402         rc = PtlNIInit(GMNAL, LUSTRE_SRV_PTL_PID, NULL, NULL, &kgmnal_ni);
 403         if (rc != PTL_OK && rc != PTL_IFACE_DUP) {
 404                 ptl_unregister_nal(GMNAL);
 405                 return (-ENODEV);
 406         }
 407
 408         return (rc);
 409 }
 410
 411
 412
 413 /*
 414  *      Called when module removed
 415  */
 416 void gmnal_fini()
 417 {
 418         CDEBUG(D_TRACE, "gmnal_fini\n");
 419
 420         LASSERT(global_nal_data == NULL);
 421         PtlNIFini(kgmnal_ni);
 422
 423         ptl_unregister_nal(GMNAL);
 424 }