lnet/klnds/gmlnd/gmlnd_api.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  Copyright (c) 2003 Los Alamos National Laboratory (LANL)
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org/
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  */
  21
  22 /*
  23  *      Implements the API NAL functions
  24  */
  25
  26 #include "gmnal.h"
  27
  28 gmnal_data_t    *global_nal_data = NULL;
  29 /*
  30  *      gmnal_api_forward
  31  *      This function takes a pack block of arguments from the NAL API
  32  *      module and passes them to the NAL CB module. The CB module unpacks
  33  *      the args and calls the appropriate function indicated by index.
  34  *      Typically this function is used to pass args between kernel and use
  35  *      space.
  36  *      As lgmanl exists entirely in kernel, just pass the arg block directly
  37  *      to the NAL CB, buy passing the args to lib_dispatch
  38  *      Arguments are
  39  *      nal_t   nal     Our nal
  40  *      int     index   the api function that initiated this call
  41  *      void    *args   packed block of function args
  42  *      size_t  arg_len length of args block
  43  *      void    *ret    A return value for the API NAL
  44  *      size_t  ret_len Size of the return value
  45  *
  46  */
  47
  48 int
  49 gmnal_api_forward(nal_t *nal, int index, void *args, size_t arg_len,
  50                 void *ret, size_t ret_len)
  51 {
  52
  53         nal_cb_t        *nal_cb = NULL;
  54         gmnal_data_t    *nal_data = NULL;
  55
  56
  57
  58
  59
  60         if (!nal || !args || (index < 0) || (arg_len < 0)) {
  61                         CDEBUG(D_ERROR, "Bad args to gmnal_api_forward\n");
  62                 return (PTL_FAIL);
  63         }
  64
  65         if (ret && (ret_len <= 0)) {
  66                 CDEBUG(D_ERROR, "Bad args to gmnal_api_forward\n");
  67                 return (PTL_FAIL);
  68         }
  69
  70
  71         if (!nal->nal_data) {
  72                 CDEBUG(D_ERROR, "bad nal, no nal data\n");
  73                 return (PTL_FAIL);
  74         }
  75
  76         nal_data = nal->nal_data;
  77         CDEBUG(D_INFO, "nal_data is [%p]\n", nal_data);
  78
  79         if (!nal_data->nal_cb) {
  80                 CDEBUG(D_ERROR, "bad nal_data, no nal_cb\n");
  81                 return (PTL_FAIL);
  82         }
  83
  84         nal_cb = nal_data->nal_cb;
  85         CDEBUG(D_INFO, "nal_cb is [%p]\n", nal_cb);
  86
  87         CDEBUG(D_PORTALS, "gmnal_api_forward calling lib_dispatch\n");
  88         lib_dispatch(nal_cb, NULL, index, args, ret);
  89         CDEBUG(D_PORTALS, "gmnal_api_forward returns from lib_dispatch\n");
  90
  91         return(PTL_OK);
  92 }
  93
  94
  95 /*
  96  *      gmnal_api_shutdown
  97  *      Close down this interface and free any resources associated with it
  98  *      nal_t   nal     our nal to shutdown
  99  */
 100 int
 101 gmnal_api_shutdown(nal_t *nal, int interface)
 102 {
 103
 104         gmnal_data_t    *nal_data = nal->nal_data;
 105
 106         CDEBUG(D_TRACE, "gmnal_api_shutdown: nal_data [%p]\n", nal_data);
 107
 108         return(PTL_OK);
 109 }
 110
 111
 112 /*
 113  *      gmnal_api_validate
 114  *      validate a user address for use in communications
 115  *      There's nothing to be done here
 116  */
 117 int
 118 gmnal_api_validate(nal_t *nal, void *base, size_t extent)
 119 {
 120
 121         return(PTL_OK);
 122 }
 123
 124
 125
 126 /*
 127  *      gmnal_api_yield
 128  *      Give up the processor
 129  */
 130 void
 131 gmnal_api_yield(nal_t *nal)
 132 {
 133         CDEBUG(D_TRACE, "gmnal_api_yield : nal [%p]\n", nal);
 134
 135         set_current_state(TASK_INTERRUPTIBLE);
 136         schedule();
 137
 138         return;
 139 }
 140
 141
 142
 143 /*
 144  *      gmnal_api_lock
 145  *      Take a threadsafe lock
 146  */
 147 void
 148 gmnal_api_lock(nal_t *nal, unsigned long *flags)
 149 {
 150
 151         gmnal_data_t    *nal_data;
 152         nal_cb_t        *nal_cb;
 153
 154         nal_data = nal->nal_data;
 155         nal_cb = nal_data->nal_cb;
 156
 157         nal_cb->cb_cli(nal_cb, flags);
 158
 159         return;
 160 }
 161
 162 /*
 163  *      gmnal_api_unlock
 164  *      Release a threadsafe lock
 165  */
 166 void
 167 gmnal_api_unlock(nal_t *nal, unsigned long *flags)
 168 {
 169         gmnal_data_t    *nal_data;
 170         nal_cb_t        *nal_cb;
 171
 172         nal_data = nal->nal_data;
 173         nal_cb = nal_data->nal_cb;
 174
 175         nal_cb->cb_sti(nal_cb, flags);
 176
 177         return;
 178 }
 179
 180
 181 nal_t *
 182 gmnal_init(int interface, ptl_pt_index_t ptl_size, ptl_ac_index_t ac_size,
 183             ptl_pid_t rpid)
 184 {
 185
 186         nal_t           *nal = NULL;
 187         nal_cb_t        *nal_cb = NULL;
 188         gmnal_data_t    *nal_data = NULL;
 189         gmnal_srxd_t    *srxd = NULL;
 190         gm_status_t     gm_status;
 191         unsigned int    local_nid = 0, global_nid = 0;
 192         ptl_nid_t       portals_nid;
 193         ptl_pid_t       portals_pid = 0;
 194
 195
 196         CDEBUG(D_TRACE, "gmnal_init : interface [%d], ptl_size [%d],
 197                ac_size[%d]\n", interface, ptl_size, ac_size);
 198
 199
 200         PORTAL_ALLOC(nal_data, sizeof(gmnal_data_t));
 201         if (!nal_data) {
 202                 CDEBUG(D_ERROR, "can't get memory\n");
 203                 return(NULL);
 204         }
 205         memset(nal_data, 0, sizeof(gmnal_data_t));
 206         /*
 207          *      set the small message buffer size
 208          */
 209         nal_data->refcnt = 1;
 210
 211         CDEBUG(D_INFO, "Allocd and reset nal_data[%p]\n", nal_data);
 212         CDEBUG(D_INFO, "small_msg_size is [%d]\n", nal_data->small_msg_size);
 213
 214         PORTAL_ALLOC(nal, sizeof(nal_t));
 215         if (!nal) {
 216                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 217                 return(NULL);
 218         }
 219         memset(nal, 0, sizeof(nal_t));
 220         CDEBUG(D_INFO, "Allocd and reset nal[%p]\n", nal);
 221
 222         PORTAL_ALLOC(nal_cb, sizeof(nal_cb_t));
 223         if (!nal_cb) {
 224                 PORTAL_FREE(nal, sizeof(nal_t));
 225                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 226                 return(NULL);
 227         }
 228         memset(nal_cb, 0, sizeof(nal_cb_t));
 229         CDEBUG(D_INFO, "Allocd and reset nal_cb[%p]\n", nal_cb);
 230
 231         GMNAL_INIT_NAL(nal);
 232         GMNAL_INIT_NAL_CB(nal_cb);
 233         /*
 234          *      String them all together
 235          */
 236         nal->nal_data = (void*)nal_data;
 237         nal_cb->nal_data = (void*)nal_data;
 238         nal_data->nal = nal;
 239         nal_data->nal_cb = nal_cb;
 240
 241         GMNAL_CB_LOCK_INIT(nal_data);
 242         GMNAL_GM_LOCK_INIT(nal_data);
 243
 244
 245         /*
 246          *      initialise the interface,
 247          */
 248         CDEBUG(D_INFO, "Calling gm_init\n");
 249         if (gm_init() != GM_SUCCESS) {
 250                 CDEBUG(D_ERROR, "call to gm_init failed\n");
 251                 PORTAL_FREE(nal, sizeof(nal_t));
 252                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 253                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 254                 return(NULL);
 255         }
 256
 257
 258         CDEBUG(D_NET, "Calling gm_open with interface [%d], port [%d],
 259                name [%s], version [%d]\n", interface, GMNAL_GM_PORT,
 260                "gmnal", GM_API_VERSION);
 261
 262         GMNAL_GM_LOCK(nal_data);
 263         gm_status = gm_open(&nal_data->gm_port, 0, GMNAL_GM_PORT, "gmnal",
 264                             GM_API_VERSION);
 265         GMNAL_GM_UNLOCK(nal_data);
 266
 267         CDEBUG(D_INFO, "gm_open returned [%d]\n", gm_status);
 268         if (gm_status == GM_SUCCESS) {
 269                 CDEBUG(D_INFO, "gm_open succeeded port[%p]\n",
 270                        nal_data->gm_port);
 271         } else {
 272                 switch(gm_status) {
 273                 case(GM_INVALID_PARAMETER):
 274                         CDEBUG(D_ERROR, "gm_open Failure. Invalid Parameter\n");
 275                         break;
 276                 case(GM_BUSY):
 277                         CDEBUG(D_ERROR, "gm_open Failure. GM Busy\n");
 278                         break;
 279                 case(GM_NO_SUCH_DEVICE):
 280                         CDEBUG(D_ERROR, "gm_open Failure. No such device\n");
 281                         break;
 282                 case(GM_INCOMPATIBLE_LIB_AND_DRIVER):
 283                         CDEBUG(D_ERROR, "gm_open Failure. Incompatile lib
 284                                and driver\n");
 285                         break;
 286                 case(GM_OUT_OF_MEMORY):
 287                         CDEBUG(D_ERROR, "gm_open Failure. Out of Memory\n");
 288                         break;
 289                 default:
 290                         CDEBUG(D_ERROR, "gm_open Failure. Unknow error
 291                                code [%d]\n", gm_status);
 292                         break;
 293                 }
 294                 GMNAL_GM_LOCK(nal_data);
 295                 gm_finalize();
 296                 GMNAL_GM_UNLOCK(nal_data);
 297                 PORTAL_FREE(nal, sizeof(nal_t));
 298                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 299                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 300                 return(NULL);
 301         }
 302
 303
 304         nal_data->small_msg_size = gmnal_small_msg_size;
 305         nal_data->small_msg_gmsize =
 306                         gm_min_size_for_length(gmnal_small_msg_size);
 307
 308         if (gmnal_alloc_srxd(nal_data) != GMNAL_STATUS_OK) {
 309                 CDEBUG(D_ERROR, "Failed to allocate small rx descriptors\n");
 310                 gmnal_free_txd(nal_data);
 311                 GMNAL_GM_LOCK(nal_data);
 312                 gm_close(nal_data->gm_port);
 313                 gm_finalize();
 314                 GMNAL_GM_UNLOCK(nal_data);
 315                 PORTAL_FREE(nal, sizeof(nal_t));
 316                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 317                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 318                 return(NULL);
 319         }
 320
 321
 322         /*
 323          *      Hang out a bunch of small receive buffers
 324          *      In fact hang them all out
 325          */
 326         while((srxd = gmnal_get_srxd(nal_data, 0))) {
 327                 CDEBUG(D_NET, "giving [%p] to gm_provide_recvive_buffer\n",
 328                        srxd->buffer);
 329                 GMNAL_GM_LOCK(nal_data);
 330                 gm_provide_receive_buffer_with_tag(nal_data->gm_port,
 331                                                    srxd->buffer, srxd->gmsize,
 332                                                    GM_LOW_PRIORITY, 0);
 333                 GMNAL_GM_UNLOCK(nal_data);
 334         }
 335
 336         /*
 337          *      Allocate pools of small tx buffers and descriptors
 338          */
 339         if (gmnal_alloc_txd(nal_data) != GMNAL_STATUS_OK) {
 340                 CDEBUG(D_ERROR, "Failed to allocate small tx descriptors\n");
 341                 GMNAL_GM_LOCK(nal_data);
 342                 gm_close(nal_data->gm_port);
 343                 gm_finalize();
 344                 GMNAL_GM_UNLOCK(nal_data);
 345                 PORTAL_FREE(nal, sizeof(nal_t));
 346                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 347                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 348                 return(NULL);
 349         }
 350
 351         gmnal_start_kernel_threads(nal_data);
 352
 353         while (nal_data->rxthread_flag != GMNAL_RXTHREADS_STARTED) {
 354                 gmnal_yield(1);
 355                 CDEBUG(D_INFO, "Waiting for receive thread signs of life\n");
 356         }
 357
 358         CDEBUG(D_INFO, "receive thread seems to have started\n");
 359
 360
 361         /*
 362          *      Initialise the portals library
 363          */
 364         CDEBUG(D_NET, "Getting node id\n");
 365         GMNAL_GM_LOCK(nal_data);
 366         gm_status = gm_get_node_id(nal_data->gm_port, &local_nid);
 367         GMNAL_GM_UNLOCK(nal_data);
 368         if (gm_status != GM_SUCCESS) {
 369                 gmnal_stop_rxthread(nal_data);
 370                 gmnal_stop_ctthread(nal_data);
 371                 CDEBUG(D_ERROR, "can't determine node id\n");
 372                 gmnal_free_txd(nal_data);
 373                 gmnal_free_srxd(nal_data);
 374                 GMNAL_GM_LOCK(nal_data);
 375                 gm_close(nal_data->gm_port);
 376                 gm_finalize();
 377                 GMNAL_GM_UNLOCK(nal_data);
 378                 PORTAL_FREE(nal, sizeof(nal_t));
 379                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 380                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 381                 return(NULL);
 382         }
 383         nal_data->gm_local_nid = local_nid;
 384         CDEBUG(D_INFO, "Local node id is [%u]\n", local_nid);
 385         GMNAL_GM_LOCK(nal_data);
 386         gm_status = gm_node_id_to_global_id(nal_data->gm_port, local_nid,
 387                                             &global_nid);
 388         GMNAL_GM_UNLOCK(nal_data);
 389         if (gm_status != GM_SUCCESS) {
 390                 CDEBUG(D_ERROR, "failed to obtain global id\n");
 391                 gmnal_stop_rxthread(nal_data);
 392                 gmnal_stop_ctthread(nal_data);
 393                 gmnal_free_txd(nal_data);
 394                 gmnal_free_srxd(nal_data);
 395                 GMNAL_GM_LOCK(nal_data);
 396                 gm_close(nal_data->gm_port);
 397                 gm_finalize();
 398                 GMNAL_GM_UNLOCK(nal_data);
 399                 PORTAL_FREE(nal, sizeof(nal_t));
 400                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 401                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 402                 return(NULL);
 403         }
 404         CDEBUG(D_INFO, "Global node id is [%u]\n", global_nid);
 405         nal_data->gm_global_nid = global_nid;
 406
 407 /*
 408         pid = gm_getpid();
 409 */
 410         CDEBUG(D_INFO, "portals_pid is [%u]\n", portals_pid);
 411         portals_nid = (unsigned long)global_nid;
 412         CDEBUG(D_INFO, "portals_nid is ["LPU64"]\n", portals_nid);
 413
 414         CDEBUG(D_PORTALS, "calling lib_init\n");
 415         if (lib_init(nal_cb, portals_nid, portals_pid, 1024, ptl_size,
 416                      ac_size) != PTL_OK) {
 417                 CDEBUG(D_ERROR, "lib_init failed\n");
 418                 gmnal_stop_rxthread(nal_data);
 419                 gmnal_stop_ctthread(nal_data);
 420                 gmnal_free_txd(nal_data);
 421                 gmnal_free_srxd(nal_data);
 422                 GMNAL_GM_LOCK(nal_data);
 423                 gm_close(nal_data->gm_port);
 424                 gm_finalize();
 425                 GMNAL_GM_UNLOCK(nal_data);
 426                 PORTAL_FREE(nal, sizeof(nal_t));
 427                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 428                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 429                 return(NULL);
 430
 431         }
 432
 433         CDEBUG(D_INFO, "gmnal_init finished\n");
 434         global_nal_data = nal->nal_data;
 435         return(nal);
 436 }
 437
 438
 439
 440 /*
 441  *      Called when module removed
 442  */
 443 void gmnal_fini()
 444 {
 445         gmnal_data_t    *nal_data = global_nal_data;
 446         nal_t           *nal = nal_data->nal;
 447         nal_cb_t        *nal_cb = nal_data->nal_cb;
 448
 449         CDEBUG(D_TRACE, "gmnal_fini\n");
 450
 451         PtlNIFini(kgmnal_ni);
 452         lib_fini(nal_cb);
 453
 454         gmnal_stop_rxthread(nal_data);
 455         gmnal_stop_ctthread(nal_data);
 456         gmnal_free_txd(nal_data);
 457         gmnal_free_srxd(nal_data);
 458         GMNAL_GM_LOCK(nal_data);
 459         gm_close(nal_data->gm_port);
 460         gm_finalize();
 461         GMNAL_GM_UNLOCK(nal_data);
 462         PORTAL_FREE(nal, sizeof(nal_t));
 463         PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 464         PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 465 }