lnet/klnds/gmlnd/gmlnd_api.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  Copyright (c) 2003 Los Alamos National Laboratory (LANL)
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org/
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  */
  21
  22 /*
  23  *      Implements the API NAL functions
  24  */
  25
  26 #include "gmnal.h"
  27
  28
  29
  30 gmnal_data_t    *global_nal_data = NULL;
  31 #define         GLOBAL_NID_STR_LEN      16
  32 char            global_nid_str[GLOBAL_NID_STR_LEN] = {0};
  33
  34 /*
  35  *      Write the global nid /proc/sys/gmnal/globalnid
  36  */
  37 #define GMNAL_SYSCTL    201
  38 #define GMNAL_SYSCTL_GLOBALNID  1
  39
  40 static ctl_table gmnal_sysctl_table[] = {
  41         {GMNAL_SYSCTL_GLOBALNID, "globalnid",
  42          global_nid_str, GLOBAL_NID_STR_LEN,
  43          0444, NULL, &proc_dostring},
  44         { 0 }
  45 };
  46
  47
  48 static ctl_table gmnalnal_top_sysctl_table[] = {
  49         {GMNAL_SYSCTL, "gmnal", NULL, 0, 0555, gmnal_sysctl_table},
  50         { 0 }
  51 };
  52
  53
  54
  55
  56
  57
  58 /*
  59  *      gmnal_api_forward
  60  *      This function takes a pack block of arguments from the NAL API
  61  *      module and passes them to the NAL CB module. The CB module unpacks
  62  *      the args and calls the appropriate function indicated by index.
  63  *      Typically this function is used to pass args between kernel and use
  64  *      space.
  65  *      As lgmanl exists entirely in kernel, just pass the arg block directly
  66  *      to the NAL CB, buy passing the args to lib_dispatch
  67  *      Arguments are
  68  *      nal_t   nal     Our nal
  69  *      int     index   the api function that initiated this call
  70  *      void    *args   packed block of function args
  71  *      size_t  arg_len length of args block
  72  *      void    *ret    A return value for the API NAL
  73  *      size_t  ret_len Size of the return value
  74  *
  75  */
  76
  77 int
  78 gmnal_api_forward(nal_t *nal, int index, void *args, size_t arg_len,
  79                 void *ret, size_t ret_len)
  80 {
  81
  82         nal_cb_t        *nal_cb = NULL;
  83         gmnal_data_t    *nal_data = NULL;
  84
  85
  86
  87
  88
  89         if (!nal || !args || (index < 0) || (arg_len < 0)) {
  90                         CDEBUG(D_ERROR, "Bad args to gmnal_api_forward\n");
  91                 return (PTL_FAIL);
  92         }
  93
  94         if (ret && (ret_len <= 0)) {
  95                 CDEBUG(D_ERROR, "Bad args to gmnal_api_forward\n");
  96                 return (PTL_FAIL);
  97         }
  98
  99
 100         if (!nal->nal_data) {
 101                 CDEBUG(D_ERROR, "bad nal, no nal data\n");
 102                 return (PTL_FAIL);
 103         }
 104
 105         nal_data = nal->nal_data;
 106         CDEBUG(D_INFO, "nal_data is [%p]\n", nal_data);
 107
 108         if (!nal_data->nal_cb) {
 109                 CDEBUG(D_ERROR, "bad nal_data, no nal_cb\n");
 110                 return (PTL_FAIL);
 111         }
 112
 113         nal_cb = nal_data->nal_cb;
 114         CDEBUG(D_INFO, "nal_cb is [%p]\n", nal_cb);
 115
 116         CDEBUG(D_PORTALS, "gmnal_api_forward calling lib_dispatch\n");
 117         lib_dispatch(nal_cb, NULL, index, args, ret);
 118         CDEBUG(D_PORTALS, "gmnal_api_forward returns from lib_dispatch\n");
 119
 120         return(PTL_OK);
 121 }
 122
 123
 124 /*
 125  *      gmnal_api_shutdown
 126  *      Close down this interface and free any resources associated with it
 127  *      nal_t   nal     our nal to shutdown
 128  */
 129 int
 130 gmnal_api_shutdown(nal_t *nal, int interface)
 131 {
 132
 133         gmnal_data_t    *nal_data = nal->nal_data;
 134
 135         CDEBUG(D_TRACE, "gmnal_api_shutdown: nal_data [%p]\n", nal_data);
 136
 137         return(PTL_OK);
 138 }
 139
 140
 141 /*
 142  *      gmnal_api_validate
 143  *      validate a user address for use in communications
 144  *      There's nothing to be done here
 145  */
 146 int
 147 gmnal_api_validate(nal_t *nal, void *base, size_t extent)
 148 {
 149
 150         return(PTL_OK);
 151 }
 152
 153
 154
 155 /*
 156  *      gmnal_api_yield
 157  *      Give up the processor
 158  */
 159 void
 160 gmnal_api_yield(nal_t *nal, unsigned long *flags, int milliseconds)
 161 {
 162         CDEBUG(D_TRACE, "gmnal_api_yield : nal [%p]\n", nal);
 163
 164         if (milliseconds != 0) {
 165                 CERROR("Blocking yield not implemented yet\n");
 166                 LBUG();
 167         }
 168
 169         our_cond_resched();
 170         return;
 171 }
 172
 173
 174
 175 /*
 176  *      gmnal_api_lock
 177  *      Take a threadsafe lock
 178  */
 179 void
 180 gmnal_api_lock(nal_t *nal, unsigned long *flags)
 181 {
 182
 183         gmnal_data_t    *nal_data;
 184         nal_cb_t        *nal_cb;
 185
 186         nal_data = nal->nal_data;
 187         nal_cb = nal_data->nal_cb;
 188
 189         nal_cb->cb_cli(nal_cb, flags);
 190
 191         return;
 192 }
 193
 194 /*
 195  *      gmnal_api_unlock
 196  *      Release a threadsafe lock
 197  */
 198 void
 199 gmnal_api_unlock(nal_t *nal, unsigned long *flags)
 200 {
 201         gmnal_data_t    *nal_data;
 202         nal_cb_t        *nal_cb;
 203
 204         nal_data = nal->nal_data;
 205         nal_cb = nal_data->nal_cb;
 206
 207         nal_cb->cb_sti(nal_cb, flags);
 208
 209         return;
 210 }
 211
 212
 213 nal_t *
 214 gmnal_init(int interface, ptl_pt_index_t ptl_size, ptl_ac_index_t ac_size,
 215             ptl_pid_t rpid)
 216 {
 217
 218         nal_t           *nal = NULL;
 219         nal_cb_t        *nal_cb = NULL;
 220         gmnal_data_t    *nal_data = NULL;
 221         gmnal_srxd_t    *srxd = NULL;
 222         gm_status_t     gm_status;
 223         unsigned int    local_nid = 0, global_nid = 0;
 224         ptl_nid_t       portals_nid;
 225         ptl_pid_t       portals_pid = 0;
 226
 227
 228         CDEBUG(D_TRACE, "gmnal_init : interface [%d], ptl_size [%d], "
 229                "ac_size[%d]\n", interface, ptl_size, ac_size);
 230
 231
 232         PORTAL_ALLOC(nal_data, sizeof(gmnal_data_t));
 233         if (!nal_data) {
 234                 CDEBUG(D_ERROR, "can't get memory\n");
 235                 return(NULL);
 236         }
 237         memset(nal_data, 0, sizeof(gmnal_data_t));
 238         /*
 239          *      set the small message buffer size
 240          */
 241         nal_data->refcnt = 1;
 242
 243         CDEBUG(D_INFO, "Allocd and reset nal_data[%p]\n", nal_data);
 244         CDEBUG(D_INFO, "small_msg_size is [%d]\n", nal_data->small_msg_size);
 245
 246         PORTAL_ALLOC(nal, sizeof(nal_t));
 247         if (!nal) {
 248                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 249                 return(NULL);
 250         }
 251         memset(nal, 0, sizeof(nal_t));
 252         CDEBUG(D_INFO, "Allocd and reset nal[%p]\n", nal);
 253
 254         PORTAL_ALLOC(nal_cb, sizeof(nal_cb_t));
 255         if (!nal_cb) {
 256                 PORTAL_FREE(nal, sizeof(nal_t));
 257                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 258                 return(NULL);
 259         }
 260         memset(nal_cb, 0, sizeof(nal_cb_t));
 261         CDEBUG(D_INFO, "Allocd and reset nal_cb[%p]\n", nal_cb);
 262
 263         GMNAL_INIT_NAL(nal);
 264         GMNAL_INIT_NAL_CB(nal_cb);
 265         /*
 266          *      String them all together
 267          */
 268         nal->nal_data = (void*)nal_data;
 269         nal_cb->nal_data = (void*)nal_data;
 270         nal_data->nal = nal;
 271         nal_data->nal_cb = nal_cb;
 272
 273         GMNAL_CB_LOCK_INIT(nal_data);
 274         GMNAL_GM_LOCK_INIT(nal_data);
 275
 276
 277         /*
 278          *      initialise the interface,
 279          */
 280         CDEBUG(D_INFO, "Calling gm_init\n");
 281         if (gm_init() != GM_SUCCESS) {
 282                 CDEBUG(D_ERROR, "call to gm_init failed\n");
 283                 PORTAL_FREE(nal, sizeof(nal_t));
 284                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 285                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 286                 return(NULL);
 287         }
 288
 289
 290         CDEBUG(D_NET, "Calling gm_open with interface [%d], port [%d], "
 291                "name [%s], version [%d]\n", interface, GMNAL_GM_PORT,
 292                "gmnal", GM_API_VERSION);
 293
 294         GMNAL_GM_LOCK(nal_data);
 295         gm_status = gm_open(&nal_data->gm_port, 0, GMNAL_GM_PORT, "gmnal",
 296                             GM_API_VERSION);
 297         GMNAL_GM_UNLOCK(nal_data);
 298
 299         CDEBUG(D_INFO, "gm_open returned [%d]\n", gm_status);
 300         if (gm_status == GM_SUCCESS) {
 301                 CDEBUG(D_INFO, "gm_open succeeded port[%p]\n",
 302                        nal_data->gm_port);
 303         } else {
 304                 switch(gm_status) {
 305                 case(GM_INVALID_PARAMETER):
 306                         CDEBUG(D_ERROR, "gm_open Failure. Invalid Parameter\n");
 307                         break;
 308                 case(GM_BUSY):
 309                         CDEBUG(D_ERROR, "gm_open Failure. GM Busy\n");
 310                         break;
 311                 case(GM_NO_SUCH_DEVICE):
 312                         CDEBUG(D_ERROR, "gm_open Failure. No such device\n");
 313                         break;
 314                 case(GM_INCOMPATIBLE_LIB_AND_DRIVER):
 315                         CDEBUG(D_ERROR, "gm_open Failure. Incompatile lib "
 316                                "and driver\n");
 317                         break;
 318                 case(GM_OUT_OF_MEMORY):
 319                         CDEBUG(D_ERROR, "gm_open Failure. Out of Memory\n");
 320                         break;
 321                 default:
 322                         CDEBUG(D_ERROR, "gm_open Failure. Unknow error "
 323                                "code [%d]\n", gm_status);
 324                         break;
 325                 }
 326                 GMNAL_GM_LOCK(nal_data);
 327                 gm_finalize();
 328                 GMNAL_GM_UNLOCK(nal_data);
 329                 PORTAL_FREE(nal, sizeof(nal_t));
 330                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 331                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 332                 return(NULL);
 333         }
 334
 335
 336         nal_data->small_msg_size = gmnal_small_msg_size;
 337         nal_data->small_msg_gmsize =
 338                         gm_min_size_for_length(gmnal_small_msg_size);
 339
 340         if (gmnal_alloc_srxd(nal_data) != GMNAL_STATUS_OK) {
 341                 CDEBUG(D_ERROR, "Failed to allocate small rx descriptors\n");
 342                 gmnal_free_txd(nal_data);
 343                 GMNAL_GM_LOCK(nal_data);
 344                 gm_close(nal_data->gm_port);
 345                 gm_finalize();
 346                 GMNAL_GM_UNLOCK(nal_data);
 347                 PORTAL_FREE(nal, sizeof(nal_t));
 348                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 349                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 350                 return(NULL);
 351         }
 352
 353
 354         /*
 355          *      Hang out a bunch of small receive buffers
 356          *      In fact hang them all out
 357          */
 358         while((srxd = gmnal_get_srxd(nal_data, 0))) {
 359                 CDEBUG(D_NET, "giving [%p] to gm_provide_recvive_buffer\n",
 360                        srxd->buffer);
 361                 GMNAL_GM_LOCK(nal_data);
 362                 gm_provide_receive_buffer_with_tag(nal_data->gm_port,
 363                                                    srxd->buffer, srxd->gmsize,
 364                                                    GM_LOW_PRIORITY, 0);
 365                 GMNAL_GM_UNLOCK(nal_data);
 366         }
 367
 368         /*
 369          *      Allocate pools of small tx buffers and descriptors
 370          */
 371         if (gmnal_alloc_txd(nal_data) != GMNAL_STATUS_OK) {
 372                 CDEBUG(D_ERROR, "Failed to allocate small tx descriptors\n");
 373                 GMNAL_GM_LOCK(nal_data);
 374                 gm_close(nal_data->gm_port);
 375                 gm_finalize();
 376                 GMNAL_GM_UNLOCK(nal_data);
 377                 PORTAL_FREE(nal, sizeof(nal_t));
 378                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 379                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 380                 return(NULL);
 381         }
 382
 383         gmnal_start_kernel_threads(nal_data);
 384
 385         while (nal_data->rxthread_flag != GMNAL_RXTHREADS_STARTED) {
 386                 gmnal_yield(1);
 387                 CDEBUG(D_INFO, "Waiting for receive thread signs of life\n");
 388         }
 389
 390         CDEBUG(D_INFO, "receive thread seems to have started\n");
 391
 392
 393         /*
 394          *      Initialise the portals library
 395          */
 396         CDEBUG(D_NET, "Getting node id\n");
 397         GMNAL_GM_LOCK(nal_data);
 398         gm_status = gm_get_node_id(nal_data->gm_port, &local_nid);
 399         GMNAL_GM_UNLOCK(nal_data);
 400         if (gm_status != GM_SUCCESS) {
 401                 gmnal_stop_rxthread(nal_data);
 402                 gmnal_stop_ctthread(nal_data);
 403                 CDEBUG(D_ERROR, "can't determine node id\n");
 404                 gmnal_free_txd(nal_data);
 405                 gmnal_free_srxd(nal_data);
 406                 GMNAL_GM_LOCK(nal_data);
 407                 gm_close(nal_data->gm_port);
 408                 gm_finalize();
 409                 GMNAL_GM_UNLOCK(nal_data);
 410                 PORTAL_FREE(nal, sizeof(nal_t));
 411                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 412                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 413                 return(NULL);
 414         }
 415         nal_data->gm_local_nid = local_nid;
 416         CDEBUG(D_INFO, "Local node id is [%u]\n", local_nid);
 417         GMNAL_GM_LOCK(nal_data);
 418         gm_status = gm_node_id_to_global_id(nal_data->gm_port, local_nid,
 419                                             &global_nid);
 420         GMNAL_GM_UNLOCK(nal_data);
 421         if (gm_status != GM_SUCCESS) {
 422                 CDEBUG(D_ERROR, "failed to obtain global id\n");
 423                 gmnal_stop_rxthread(nal_data);
 424                 gmnal_stop_ctthread(nal_data);
 425                 gmnal_free_txd(nal_data);
 426                 gmnal_free_srxd(nal_data);
 427                 GMNAL_GM_LOCK(nal_data);
 428                 gm_close(nal_data->gm_port);
 429                 gm_finalize();
 430                 GMNAL_GM_UNLOCK(nal_data);
 431                 PORTAL_FREE(nal, sizeof(nal_t));
 432                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 433                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 434                 return(NULL);
 435         }
 436         CDEBUG(D_INFO, "Global node id is [%u]\n", global_nid);
 437         nal_data->gm_global_nid = global_nid;
 438         snprintf(global_nid_str, GLOBAL_NID_STR_LEN, "%u", global_nid);
 439
 440 /*
 441         pid = gm_getpid();
 442 */
 443         CDEBUG(D_INFO, "portals_pid is [%u]\n", portals_pid);
 444         portals_nid = (unsigned long)global_nid;
 445         CDEBUG(D_INFO, "portals_nid is ["LPU64"]\n", portals_nid);
 446
 447         CDEBUG(D_PORTALS, "calling lib_init\n");
 448         if (lib_init(nal_cb, portals_nid, portals_pid, 1024, ptl_size,
 449                      ac_size) != PTL_OK) {
 450                 CDEBUG(D_ERROR, "lib_init failed\n");
 451                 gmnal_stop_rxthread(nal_data);
 452                 gmnal_stop_ctthread(nal_data);
 453                 gmnal_free_txd(nal_data);
 454                 gmnal_free_srxd(nal_data);
 455                 GMNAL_GM_LOCK(nal_data);
 456                 gm_close(nal_data->gm_port);
 457                 gm_finalize();
 458                 GMNAL_GM_UNLOCK(nal_data);
 459                 PORTAL_FREE(nal, sizeof(nal_t));
 460                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 461                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 462                 return(NULL);
 463
 464         }
 465         nal_data->sysctl = NULL;
 466         nal_data->sysctl = register_sysctl_table (gmnalnal_top_sysctl_table, 0);
 467
 468
 469         CDEBUG(D_INFO, "gmnal_init finished\n");
 470         global_nal_data = nal->nal_data;
 471         return(nal);
 472 }
 473
 474
 475
 476 /*
 477  *      Called when module removed
 478  */
 479 void gmnal_fini()
 480 {
 481         gmnal_data_t    *nal_data = global_nal_data;
 482         nal_t           *nal = nal_data->nal;
 483         nal_cb_t        *nal_cb = nal_data->nal_cb;
 484
 485         CDEBUG(D_TRACE, "gmnal_fini\n");
 486
 487         PtlNIFini(kgmnal_ni);
 488         lib_fini(nal_cb);
 489
 490         gmnal_stop_rxthread(nal_data);
 491         gmnal_stop_ctthread(nal_data);
 492         gmnal_free_txd(nal_data);
 493         gmnal_free_srxd(nal_data);
 494         GMNAL_GM_LOCK(nal_data);
 495         gm_close(nal_data->gm_port);
 496         gm_finalize();
 497         GMNAL_GM_UNLOCK(nal_data);
 498         if (nal_data->sysctl)
 499                 unregister_sysctl_table (nal_data->sysctl);
 500         PORTAL_FREE(nal, sizeof(nal_t));
 501         PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 502         PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 503 }