lnet/klnds/gmlnd/gmlnd_api.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  Copyright (c) 2003 Los Alamos National Laboratory (LANL)
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org/
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  */
  21
  22 /*
  23  *      Implements the API NAL functions
  24  */
  25
  26 #include "gmnal.h"
  27
  28
  29
  30 gmnal_data_t    *global_nal_data = NULL;
  31 #define         GLOBAL_NID_STR_LEN      16
  32 char            global_nid_str[GLOBAL_NID_STR_LEN] = {0};
  33
  34 /*
  35  *      Write the global nid /proc/sys/gmnal/globalnid
  36  */
  37 #define GMNAL_SYSCTL    201
  38 #define GMNAL_SYSCTL_GLOBALNID  1
  39
  40 static ctl_table gmnal_sysctl_table[] = {
  41         {GMNAL_SYSCTL_GLOBALNID, "globalnid",
  42          global_nid_str, GLOBAL_NID_STR_LEN,
  43          0444, NULL, &proc_dostring},
  44         { 0 }
  45 };
  46
  47
  48 static ctl_table gmnalnal_top_sysctl_table[] = {
  49         {GMNAL_SYSCTL, "gmnal", NULL, 0, 0555, gmnal_sysctl_table},
  50         { 0 }
  51 };
  52
  53
  54
  55
  56
  57
  58 /*
  59  *      gmnal_api_forward
  60  *      This function takes a pack block of arguments from the NAL API
  61  *      module and passes them to the NAL CB module. The CB module unpacks
  62  *      the args and calls the appropriate function indicated by index.
  63  *      Typically this function is used to pass args between kernel and use
  64  *      space.
  65  *      As lgmanl exists entirely in kernel, just pass the arg block directly
  66  *      to the NAL CB, buy passing the args to lib_dispatch
  67  *      Arguments are
  68  *      nal_t   nal     Our nal
  69  *      int     index   the api function that initiated this call
  70  *      void    *args   packed block of function args
  71  *      size_t  arg_len length of args block
  72  *      void    *ret    A return value for the API NAL
  73  *      size_t  ret_len Size of the return value
  74  *
  75  */
  76
  77 int
  78 gmnal_api_forward(nal_t *nal, int index, void *args, size_t arg_len,
  79                 void *ret, size_t ret_len)
  80 {
  81
  82         nal_cb_t        *nal_cb = NULL;
  83         gmnal_data_t    *nal_data = NULL;
  84
  85
  86
  87
  88
  89         if (!nal || !args || (index < 0) || (arg_len < 0)) {
  90                         CDEBUG(D_ERROR, "Bad args to gmnal_api_forward\n");
  91                 return (PTL_FAIL);
  92         }
  93
  94         if (ret && (ret_len <= 0)) {
  95                 CDEBUG(D_ERROR, "Bad args to gmnal_api_forward\n");
  96                 return (PTL_FAIL);
  97         }
  98
  99
 100         if (!nal->nal_data) {
 101                 CDEBUG(D_ERROR, "bad nal, no nal data\n");
 102                 return (PTL_FAIL);
 103         }
 104
 105         nal_data = nal->nal_data;
 106         CDEBUG(D_INFO, "nal_data is [%p]\n", nal_data);
 107
 108         if (!nal_data->nal_cb) {
 109                 CDEBUG(D_ERROR, "bad nal_data, no nal_cb\n");
 110                 return (PTL_FAIL);
 111         }
 112
 113         nal_cb = nal_data->nal_cb;
 114         CDEBUG(D_INFO, "nal_cb is [%p]\n", nal_cb);
 115
 116         CDEBUG(D_PORTALS, "gmnal_api_forward calling lib_dispatch\n");
 117         lib_dispatch(nal_cb, NULL, index, args, ret);
 118         CDEBUG(D_PORTALS, "gmnal_api_forward returns from lib_dispatch\n");
 119
 120         return(PTL_OK);
 121 }
 122
 123
 124 /*
 125  *      gmnal_api_shutdown
 126  *      Close down this interface and free any resources associated with it
 127  *      nal_t   nal     our nal to shutdown
 128  */
 129 int
 130 gmnal_api_shutdown(nal_t *nal, int interface)
 131 {
 132
 133         gmnal_data_t    *nal_data = nal->nal_data;
 134
 135         CDEBUG(D_TRACE, "gmnal_api_shutdown: nal_data [%p]\n", nal_data);
 136
 137         return(PTL_OK);
 138 }
 139
 140
 141 /*
 142  *      gmnal_api_validate
 143  *      validate a user address for use in communications
 144  *      There's nothing to be done here
 145  */
 146 int
 147 gmnal_api_validate(nal_t *nal, void *base, size_t extent)
 148 {
 149
 150         return(PTL_OK);
 151 }
 152
 153
 154
 155 /*
 156  *      gmnal_api_yield
 157  *      Give up the processor
 158  */
 159 void
 160 gmnal_api_yield(nal_t *nal)
 161 {
 162         CDEBUG(D_TRACE, "gmnal_api_yield : nal [%p]\n", nal);
 163
 164         set_current_state(TASK_INTERRUPTIBLE);
 165         schedule();
 166
 167         return;
 168 }
 169
 170
 171
 172 /*
 173  *      gmnal_api_lock
 174  *      Take a threadsafe lock
 175  */
 176 void
 177 gmnal_api_lock(nal_t *nal, unsigned long *flags)
 178 {
 179
 180         gmnal_data_t    *nal_data;
 181         nal_cb_t        *nal_cb;
 182
 183         nal_data = nal->nal_data;
 184         nal_cb = nal_data->nal_cb;
 185
 186         nal_cb->cb_cli(nal_cb, flags);
 187
 188         return;
 189 }
 190
 191 /*
 192  *      gmnal_api_unlock
 193  *      Release a threadsafe lock
 194  */
 195 void
 196 gmnal_api_unlock(nal_t *nal, unsigned long *flags)
 197 {
 198         gmnal_data_t    *nal_data;
 199         nal_cb_t        *nal_cb;
 200
 201         nal_data = nal->nal_data;
 202         nal_cb = nal_data->nal_cb;
 203
 204         nal_cb->cb_sti(nal_cb, flags);
 205
 206         return;
 207 }
 208
 209
 210 nal_t *
 211 gmnal_init(int interface, ptl_pt_index_t ptl_size, ptl_ac_index_t ac_size,
 212             ptl_pid_t rpid)
 213 {
 214
 215         nal_t           *nal = NULL;
 216         nal_cb_t        *nal_cb = NULL;
 217         gmnal_data_t    *nal_data = NULL;
 218         gmnal_srxd_t    *srxd = NULL;
 219         gm_status_t     gm_status;
 220         unsigned int    local_nid = 0, global_nid = 0;
 221         ptl_nid_t       portals_nid;
 222         ptl_pid_t       portals_pid = 0;
 223
 224
 225         CDEBUG(D_TRACE, "gmnal_init : interface [%d], ptl_size [%d], "
 226                "ac_size[%d]\n", interface, ptl_size, ac_size);
 227
 228
 229         PORTAL_ALLOC(nal_data, sizeof(gmnal_data_t));
 230         if (!nal_data) {
 231                 CDEBUG(D_ERROR, "can't get memory\n");
 232                 return(NULL);
 233         }
 234         memset(nal_data, 0, sizeof(gmnal_data_t));
 235         /*
 236          *      set the small message buffer size
 237          */
 238         nal_data->refcnt = 1;
 239
 240         CDEBUG(D_INFO, "Allocd and reset nal_data[%p]\n", nal_data);
 241         CDEBUG(D_INFO, "small_msg_size is [%d]\n", nal_data->small_msg_size);
 242
 243         PORTAL_ALLOC(nal, sizeof(nal_t));
 244         if (!nal) {
 245                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 246                 return(NULL);
 247         }
 248         memset(nal, 0, sizeof(nal_t));
 249         CDEBUG(D_INFO, "Allocd and reset nal[%p]\n", nal);
 250
 251         PORTAL_ALLOC(nal_cb, sizeof(nal_cb_t));
 252         if (!nal_cb) {
 253                 PORTAL_FREE(nal, sizeof(nal_t));
 254                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 255                 return(NULL);
 256         }
 257         memset(nal_cb, 0, sizeof(nal_cb_t));
 258         CDEBUG(D_INFO, "Allocd and reset nal_cb[%p]\n", nal_cb);
 259
 260         GMNAL_INIT_NAL(nal);
 261         GMNAL_INIT_NAL_CB(nal_cb);
 262         /*
 263          *      String them all together
 264          */
 265         nal->nal_data = (void*)nal_data;
 266         nal_cb->nal_data = (void*)nal_data;
 267         nal_data->nal = nal;
 268         nal_data->nal_cb = nal_cb;
 269
 270         GMNAL_CB_LOCK_INIT(nal_data);
 271         GMNAL_GM_LOCK_INIT(nal_data);
 272
 273
 274         /*
 275          *      initialise the interface,
 276          */
 277         CDEBUG(D_INFO, "Calling gm_init\n");
 278         if (gm_init() != GM_SUCCESS) {
 279                 CDEBUG(D_ERROR, "call to gm_init failed\n");
 280                 PORTAL_FREE(nal, sizeof(nal_t));
 281                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 282                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 283                 return(NULL);
 284         }
 285
 286
 287         CDEBUG(D_NET, "Calling gm_open with interface [%d], port [%d], "
 288                "name [%s], version [%d]\n", interface, GMNAL_GM_PORT,
 289                "gmnal", GM_API_VERSION);
 290
 291         GMNAL_GM_LOCK(nal_data);
 292         gm_status = gm_open(&nal_data->gm_port, 0, GMNAL_GM_PORT, "gmnal",
 293                             GM_API_VERSION);
 294         GMNAL_GM_UNLOCK(nal_data);
 295
 296         CDEBUG(D_INFO, "gm_open returned [%d]\n", gm_status);
 297         if (gm_status == GM_SUCCESS) {
 298                 CDEBUG(D_INFO, "gm_open succeeded port[%p]\n",
 299                        nal_data->gm_port);
 300         } else {
 301                 switch(gm_status) {
 302                 case(GM_INVALID_PARAMETER):
 303                         CDEBUG(D_ERROR, "gm_open Failure. Invalid Parameter\n");
 304                         break;
 305                 case(GM_BUSY):
 306                         CDEBUG(D_ERROR, "gm_open Failure. GM Busy\n");
 307                         break;
 308                 case(GM_NO_SUCH_DEVICE):
 309                         CDEBUG(D_ERROR, "gm_open Failure. No such device\n");
 310                         break;
 311                 case(GM_INCOMPATIBLE_LIB_AND_DRIVER):
 312                         CDEBUG(D_ERROR, "gm_open Failure. Incompatile lib "
 313                                "and driver\n");
 314                         break;
 315                 case(GM_OUT_OF_MEMORY):
 316                         CDEBUG(D_ERROR, "gm_open Failure. Out of Memory\n");
 317                         break;
 318                 default:
 319                         CDEBUG(D_ERROR, "gm_open Failure. Unknow error "
 320                                "code [%d]\n", gm_status);
 321                         break;
 322                 }
 323                 GMNAL_GM_LOCK(nal_data);
 324                 gm_finalize();
 325                 GMNAL_GM_UNLOCK(nal_data);
 326                 PORTAL_FREE(nal, sizeof(nal_t));
 327                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 328                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 329                 return(NULL);
 330         }
 331
 332
 333         nal_data->small_msg_size = gmnal_small_msg_size;
 334         nal_data->small_msg_gmsize =
 335                         gm_min_size_for_length(gmnal_small_msg_size);
 336
 337         if (gmnal_alloc_srxd(nal_data) != GMNAL_STATUS_OK) {
 338                 CDEBUG(D_ERROR, "Failed to allocate small rx descriptors\n");
 339                 gmnal_free_txd(nal_data);
 340                 GMNAL_GM_LOCK(nal_data);
 341                 gm_close(nal_data->gm_port);
 342                 gm_finalize();
 343                 GMNAL_GM_UNLOCK(nal_data);
 344                 PORTAL_FREE(nal, sizeof(nal_t));
 345                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 346                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 347                 return(NULL);
 348         }
 349
 350
 351         /*
 352          *      Hang out a bunch of small receive buffers
 353          *      In fact hang them all out
 354          */
 355         while((srxd = gmnal_get_srxd(nal_data, 0))) {
 356                 CDEBUG(D_NET, "giving [%p] to gm_provide_recvive_buffer\n",
 357                        srxd->buffer);
 358                 GMNAL_GM_LOCK(nal_data);
 359                 gm_provide_receive_buffer_with_tag(nal_data->gm_port,
 360                                                    srxd->buffer, srxd->gmsize,
 361                                                    GM_LOW_PRIORITY, 0);
 362                 GMNAL_GM_UNLOCK(nal_data);
 363         }
 364
 365         /*
 366          *      Allocate pools of small tx buffers and descriptors
 367          */
 368         if (gmnal_alloc_txd(nal_data) != GMNAL_STATUS_OK) {
 369                 CDEBUG(D_ERROR, "Failed to allocate small tx descriptors\n");
 370                 GMNAL_GM_LOCK(nal_data);
 371                 gm_close(nal_data->gm_port);
 372                 gm_finalize();
 373                 GMNAL_GM_UNLOCK(nal_data);
 374                 PORTAL_FREE(nal, sizeof(nal_t));
 375                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 376                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 377                 return(NULL);
 378         }
 379
 380         gmnal_start_kernel_threads(nal_data);
 381
 382         while (nal_data->rxthread_flag != GMNAL_RXTHREADS_STARTED) {
 383                 gmnal_yield(1);
 384                 CDEBUG(D_INFO, "Waiting for receive thread signs of life\n");
 385         }
 386
 387         CDEBUG(D_INFO, "receive thread seems to have started\n");
 388
 389
 390         /*
 391          *      Initialise the portals library
 392          */
 393         CDEBUG(D_NET, "Getting node id\n");
 394         GMNAL_GM_LOCK(nal_data);
 395         gm_status = gm_get_node_id(nal_data->gm_port, &local_nid);
 396         GMNAL_GM_UNLOCK(nal_data);
 397         if (gm_status != GM_SUCCESS) {
 398                 gmnal_stop_rxthread(nal_data);
 399                 gmnal_stop_ctthread(nal_data);
 400                 CDEBUG(D_ERROR, "can't determine node id\n");
 401                 gmnal_free_txd(nal_data);
 402                 gmnal_free_srxd(nal_data);
 403                 GMNAL_GM_LOCK(nal_data);
 404                 gm_close(nal_data->gm_port);
 405                 gm_finalize();
 406                 GMNAL_GM_UNLOCK(nal_data);
 407                 PORTAL_FREE(nal, sizeof(nal_t));
 408                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 409                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 410                 return(NULL);
 411         }
 412         nal_data->gm_local_nid = local_nid;
 413         CDEBUG(D_INFO, "Local node id is [%u]\n", local_nid);
 414         GMNAL_GM_LOCK(nal_data);
 415         gm_status = gm_node_id_to_global_id(nal_data->gm_port, local_nid,
 416                                             &global_nid);
 417         GMNAL_GM_UNLOCK(nal_data);
 418         if (gm_status != GM_SUCCESS) {
 419                 CDEBUG(D_ERROR, "failed to obtain global id\n");
 420                 gmnal_stop_rxthread(nal_data);
 421                 gmnal_stop_ctthread(nal_data);
 422                 gmnal_free_txd(nal_data);
 423                 gmnal_free_srxd(nal_data);
 424                 GMNAL_GM_LOCK(nal_data);
 425                 gm_close(nal_data->gm_port);
 426                 gm_finalize();
 427                 GMNAL_GM_UNLOCK(nal_data);
 428                 PORTAL_FREE(nal, sizeof(nal_t));
 429                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 430                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 431                 return(NULL);
 432         }
 433         CDEBUG(D_INFO, "Global node id is [%u]\n", global_nid);
 434         nal_data->gm_global_nid = global_nid;
 435         snprintf(global_nid_str, GLOBAL_NID_STR_LEN, "%u", global_nid);
 436
 437 /*
 438         pid = gm_getpid();
 439 */
 440         CDEBUG(D_INFO, "portals_pid is [%u]\n", portals_pid);
 441         portals_nid = (unsigned long)global_nid;
 442         CDEBUG(D_INFO, "portals_nid is ["LPU64"]\n", portals_nid);
 443
 444         CDEBUG(D_PORTALS, "calling lib_init\n");
 445         if (lib_init(nal_cb, portals_nid, portals_pid, 1024, ptl_size,
 446                      ac_size) != PTL_OK) {
 447                 CDEBUG(D_ERROR, "lib_init failed\n");
 448                 gmnal_stop_rxthread(nal_data);
 449                 gmnal_stop_ctthread(nal_data);
 450                 gmnal_free_txd(nal_data);
 451                 gmnal_free_srxd(nal_data);
 452                 GMNAL_GM_LOCK(nal_data);
 453                 gm_close(nal_data->gm_port);
 454                 gm_finalize();
 455                 GMNAL_GM_UNLOCK(nal_data);
 456                 PORTAL_FREE(nal, sizeof(nal_t));
 457                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 458                 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 459                 return(NULL);
 460
 461         }
 462         nal_data->sysctl = NULL;
 463         nal_data->sysctl = register_sysctl_table (gmnalnal_top_sysctl_table, 0);
 464
 465
 466         CDEBUG(D_INFO, "gmnal_init finished\n");
 467         global_nal_data = nal->nal_data;
 468         return(nal);
 469 }
 470
 471
 472
 473 /*
 474  *      Called when module removed
 475  */
 476 void gmnal_fini()
 477 {
 478         gmnal_data_t    *nal_data = global_nal_data;
 479         nal_t           *nal = nal_data->nal;
 480         nal_cb_t        *nal_cb = nal_data->nal_cb;
 481
 482         CDEBUG(D_TRACE, "gmnal_fini\n");
 483
 484         PtlNIFini(kgmnal_ni);
 485         lib_fini(nal_cb);
 486
 487         gmnal_stop_rxthread(nal_data);
 488         gmnal_stop_ctthread(nal_data);
 489         gmnal_free_txd(nal_data);
 490         gmnal_free_srxd(nal_data);
 491         GMNAL_GM_LOCK(nal_data);
 492         gm_close(nal_data->gm_port);
 493         gm_finalize();
 494         GMNAL_GM_UNLOCK(nal_data);
 495         if (nal_data->sysctl)
 496                 unregister_sysctl_table (nal_data->sysctl);
 497         PORTAL_FREE(nal, sizeof(nal_t));
 498         PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
 499         PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
 500 }