1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (c) 2003 Los Alamos National Laboratory (LANL)
6 * This file is part of Lustre, http://www.lustre.org/
8 * Lustre is free software; you can redistribute it and/or
9 * modify it under the terms of version 2 of the GNU General Public
10 * License as published by the Free Software Foundation.
12 * Lustre is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with Lustre; if not, write to the Free Software
19 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23 * Implements the API NAL functions
30 gmnal_data_t *global_nal_data = NULL;
31 #define GLOBAL_NID_STR_LEN 16
32 char global_nid_str[GLOBAL_NID_STR_LEN] = {0};
35 * Write the global nid /proc/sys/gmnal/globalnid
37 #define GMNAL_SYSCTL 201
38 #define GMNAL_SYSCTL_GLOBALNID 1
40 static ctl_table gmnal_sysctl_table[] = {
41 {GMNAL_SYSCTL_GLOBALNID, "globalnid",
42 global_nid_str, GLOBAL_NID_STR_LEN,
43 0444, NULL, &proc_dostring},
48 static ctl_table gmnalnal_top_sysctl_table[] = {
49 {GMNAL_SYSCTL, "gmnal", NULL, 0, 0555, gmnal_sysctl_table},
60 * This function takes a pack block of arguments from the NAL API
61 * module and passes them to the NAL CB module. The CB module unpacks
62 * the args and calls the appropriate function indicated by index.
63 * Typically this function is used to pass args between kernel and use
65 * As lgmanl exists entirely in kernel, just pass the arg block directly
66 * to the NAL CB, buy passing the args to lib_dispatch
69 * int index the api function that initiated this call
70 * void *args packed block of function args
71 * size_t arg_len length of args block
72 * void *ret A return value for the API NAL
73 * size_t ret_len Size of the return value
78 gmnal_api_forward(nal_t *nal, int index, void *args, size_t arg_len,
79 void *ret, size_t ret_len)
82 nal_cb_t *nal_cb = NULL;
83 gmnal_data_t *nal_data = NULL;
89 if (!nal || !args || (index < 0) || (arg_len < 0)) {
90 CDEBUG(D_ERROR, "Bad args to gmnal_api_forward\n");
94 if (ret && (ret_len <= 0)) {
95 CDEBUG(D_ERROR, "Bad args to gmnal_api_forward\n");
100 if (!nal->nal_data) {
101 CDEBUG(D_ERROR, "bad nal, no nal data\n");
105 nal_data = nal->nal_data;
106 CDEBUG(D_INFO, "nal_data is [%p]\n", nal_data);
108 if (!nal_data->nal_cb) {
109 CDEBUG(D_ERROR, "bad nal_data, no nal_cb\n");
113 nal_cb = nal_data->nal_cb;
114 CDEBUG(D_INFO, "nal_cb is [%p]\n", nal_cb);
116 CDEBUG(D_PORTALS, "gmnal_api_forward calling lib_dispatch\n");
117 lib_dispatch(nal_cb, NULL, index, args, ret);
118 CDEBUG(D_PORTALS, "gmnal_api_forward returns from lib_dispatch\n");
126 * Close down this interface and free any resources associated with it
127 * nal_t nal our nal to shutdown
130 gmnal_api_shutdown(nal_t *nal, int interface)
133 gmnal_data_t *nal_data = nal->nal_data;
135 CDEBUG(D_TRACE, "gmnal_api_shutdown: nal_data [%p]\n", nal_data);
143 * validate a user address for use in communications
144 * There's nothing to be done here
147 gmnal_api_validate(nal_t *nal, void *base, size_t extent)
157 * Give up the processor
160 gmnal_api_yield(nal_t *nal, unsigned long *flags, int milliseconds)
162 CDEBUG(D_TRACE, "gmnal_api_yield : nal [%p]\n", nal);
164 if (milliseconds != 0) {
165 CERROR("Blocking yield not implemented yet\n");
177 * Take a threadsafe lock
180 gmnal_api_lock(nal_t *nal, unsigned long *flags)
183 gmnal_data_t *nal_data;
186 nal_data = nal->nal_data;
187 nal_cb = nal_data->nal_cb;
189 nal_cb->cb_cli(nal_cb, flags);
196 * Release a threadsafe lock
199 gmnal_api_unlock(nal_t *nal, unsigned long *flags)
201 gmnal_data_t *nal_data;
204 nal_data = nal->nal_data;
205 nal_cb = nal_data->nal_cb;
207 nal_cb->cb_sti(nal_cb, flags);
214 gmnal_init(int interface, ptl_pt_index_t ptl_size, ptl_ac_index_t ac_size,
219 nal_cb_t *nal_cb = NULL;
220 gmnal_data_t *nal_data = NULL;
221 gmnal_srxd_t *srxd = NULL;
222 gm_status_t gm_status;
223 unsigned int local_nid = 0, global_nid = 0;
224 ptl_nid_t portals_nid;
225 ptl_pid_t portals_pid = 0;
228 CDEBUG(D_TRACE, "gmnal_init : interface [%d], ptl_size [%d], "
229 "ac_size[%d]\n", interface, ptl_size, ac_size);
232 PORTAL_ALLOC(nal_data, sizeof(gmnal_data_t));
234 CDEBUG(D_ERROR, "can't get memory\n");
237 memset(nal_data, 0, sizeof(gmnal_data_t));
239 * set the small message buffer size
241 nal_data->refcnt = 1;
243 CDEBUG(D_INFO, "Allocd and reset nal_data[%p]\n", nal_data);
244 CDEBUG(D_INFO, "small_msg_size is [%d]\n", nal_data->small_msg_size);
246 PORTAL_ALLOC(nal, sizeof(nal_t));
248 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
251 memset(nal, 0, sizeof(nal_t));
252 CDEBUG(D_INFO, "Allocd and reset nal[%p]\n", nal);
254 PORTAL_ALLOC(nal_cb, sizeof(nal_cb_t));
256 PORTAL_FREE(nal, sizeof(nal_t));
257 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
260 memset(nal_cb, 0, sizeof(nal_cb_t));
261 CDEBUG(D_INFO, "Allocd and reset nal_cb[%p]\n", nal_cb);
264 GMNAL_INIT_NAL_CB(nal_cb);
266 * String them all together
268 nal->nal_data = (void*)nal_data;
269 nal_cb->nal_data = (void*)nal_data;
271 nal_data->nal_cb = nal_cb;
273 GMNAL_CB_LOCK_INIT(nal_data);
274 GMNAL_GM_LOCK_INIT(nal_data);
278 * initialise the interface,
280 CDEBUG(D_INFO, "Calling gm_init\n");
281 if (gm_init() != GM_SUCCESS) {
282 CDEBUG(D_ERROR, "call to gm_init failed\n");
283 PORTAL_FREE(nal, sizeof(nal_t));
284 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
285 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
290 CDEBUG(D_NET, "Calling gm_open with interface [%d], port [%d], "
291 "name [%s], version [%d]\n", interface, GMNAL_GM_PORT,
292 "gmnal", GM_API_VERSION);
294 GMNAL_GM_LOCK(nal_data);
295 gm_status = gm_open(&nal_data->gm_port, 0, GMNAL_GM_PORT, "gmnal",
297 GMNAL_GM_UNLOCK(nal_data);
299 CDEBUG(D_INFO, "gm_open returned [%d]\n", gm_status);
300 if (gm_status == GM_SUCCESS) {
301 CDEBUG(D_INFO, "gm_open succeeded port[%p]\n",
305 case(GM_INVALID_PARAMETER):
306 CDEBUG(D_ERROR, "gm_open Failure. Invalid Parameter\n");
309 CDEBUG(D_ERROR, "gm_open Failure. GM Busy\n");
311 case(GM_NO_SUCH_DEVICE):
312 CDEBUG(D_ERROR, "gm_open Failure. No such device\n");
314 case(GM_INCOMPATIBLE_LIB_AND_DRIVER):
315 CDEBUG(D_ERROR, "gm_open Failure. Incompatile lib "
318 case(GM_OUT_OF_MEMORY):
319 CDEBUG(D_ERROR, "gm_open Failure. Out of Memory\n");
322 CDEBUG(D_ERROR, "gm_open Failure. Unknow error "
323 "code [%d]\n", gm_status);
326 GMNAL_GM_LOCK(nal_data);
328 GMNAL_GM_UNLOCK(nal_data);
329 PORTAL_FREE(nal, sizeof(nal_t));
330 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
331 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
336 nal_data->small_msg_size = gmnal_small_msg_size;
337 nal_data->small_msg_gmsize =
338 gm_min_size_for_length(gmnal_small_msg_size);
340 if (gmnal_alloc_srxd(nal_data) != GMNAL_STATUS_OK) {
341 CDEBUG(D_ERROR, "Failed to allocate small rx descriptors\n");
342 gmnal_free_txd(nal_data);
343 GMNAL_GM_LOCK(nal_data);
344 gm_close(nal_data->gm_port);
346 GMNAL_GM_UNLOCK(nal_data);
347 PORTAL_FREE(nal, sizeof(nal_t));
348 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
349 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
355 * Hang out a bunch of small receive buffers
356 * In fact hang them all out
358 while((srxd = gmnal_get_srxd(nal_data, 0))) {
359 CDEBUG(D_NET, "giving [%p] to gm_provide_recvive_buffer\n",
361 GMNAL_GM_LOCK(nal_data);
362 gm_provide_receive_buffer_with_tag(nal_data->gm_port,
363 srxd->buffer, srxd->gmsize,
365 GMNAL_GM_UNLOCK(nal_data);
369 * Allocate pools of small tx buffers and descriptors
371 if (gmnal_alloc_txd(nal_data) != GMNAL_STATUS_OK) {
372 CDEBUG(D_ERROR, "Failed to allocate small tx descriptors\n");
373 GMNAL_GM_LOCK(nal_data);
374 gm_close(nal_data->gm_port);
376 GMNAL_GM_UNLOCK(nal_data);
377 PORTAL_FREE(nal, sizeof(nal_t));
378 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
379 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
383 gmnal_start_kernel_threads(nal_data);
385 while (nal_data->rxthread_flag != GMNAL_RXTHREADS_STARTED) {
387 CDEBUG(D_INFO, "Waiting for receive thread signs of life\n");
390 CDEBUG(D_INFO, "receive thread seems to have started\n");
394 * Initialise the portals library
396 CDEBUG(D_NET, "Getting node id\n");
397 GMNAL_GM_LOCK(nal_data);
398 gm_status = gm_get_node_id(nal_data->gm_port, &local_nid);
399 GMNAL_GM_UNLOCK(nal_data);
400 if (gm_status != GM_SUCCESS) {
401 gmnal_stop_rxthread(nal_data);
402 gmnal_stop_ctthread(nal_data);
403 CDEBUG(D_ERROR, "can't determine node id\n");
404 gmnal_free_txd(nal_data);
405 gmnal_free_srxd(nal_data);
406 GMNAL_GM_LOCK(nal_data);
407 gm_close(nal_data->gm_port);
409 GMNAL_GM_UNLOCK(nal_data);
410 PORTAL_FREE(nal, sizeof(nal_t));
411 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
412 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
415 nal_data->gm_local_nid = local_nid;
416 CDEBUG(D_INFO, "Local node id is [%u]\n", local_nid);
417 GMNAL_GM_LOCK(nal_data);
418 gm_status = gm_node_id_to_global_id(nal_data->gm_port, local_nid,
420 GMNAL_GM_UNLOCK(nal_data);
421 if (gm_status != GM_SUCCESS) {
422 CDEBUG(D_ERROR, "failed to obtain global id\n");
423 gmnal_stop_rxthread(nal_data);
424 gmnal_stop_ctthread(nal_data);
425 gmnal_free_txd(nal_data);
426 gmnal_free_srxd(nal_data);
427 GMNAL_GM_LOCK(nal_data);
428 gm_close(nal_data->gm_port);
430 GMNAL_GM_UNLOCK(nal_data);
431 PORTAL_FREE(nal, sizeof(nal_t));
432 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
433 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
436 CDEBUG(D_INFO, "Global node id is [%u]\n", global_nid);
437 nal_data->gm_global_nid = global_nid;
438 snprintf(global_nid_str, GLOBAL_NID_STR_LEN, "%u", global_nid);
443 CDEBUG(D_INFO, "portals_pid is [%u]\n", portals_pid);
444 portals_nid = (unsigned long)global_nid;
445 CDEBUG(D_INFO, "portals_nid is ["LPU64"]\n", portals_nid);
447 CDEBUG(D_PORTALS, "calling lib_init\n");
448 if (lib_init(nal_cb, portals_nid, portals_pid, 1024, ptl_size,
449 ac_size) != PTL_OK) {
450 CDEBUG(D_ERROR, "lib_init failed\n");
451 gmnal_stop_rxthread(nal_data);
452 gmnal_stop_ctthread(nal_data);
453 gmnal_free_txd(nal_data);
454 gmnal_free_srxd(nal_data);
455 GMNAL_GM_LOCK(nal_data);
456 gm_close(nal_data->gm_port);
458 GMNAL_GM_UNLOCK(nal_data);
459 PORTAL_FREE(nal, sizeof(nal_t));
460 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
461 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
465 nal_data->sysctl = NULL;
466 nal_data->sysctl = register_sysctl_table (gmnalnal_top_sysctl_table, 0);
469 CDEBUG(D_INFO, "gmnal_init finished\n");
470 global_nal_data = nal->nal_data;
477 * Called when module removed
481 gmnal_data_t *nal_data = global_nal_data;
482 nal_t *nal = nal_data->nal;
483 nal_cb_t *nal_cb = nal_data->nal_cb;
485 CDEBUG(D_TRACE, "gmnal_fini\n");
487 PtlNIFini(kgmnal_ni);
490 gmnal_stop_rxthread(nal_data);
491 gmnal_stop_ctthread(nal_data);
492 gmnal_free_txd(nal_data);
493 gmnal_free_srxd(nal_data);
494 GMNAL_GM_LOCK(nal_data);
495 gm_close(nal_data->gm_port);
497 GMNAL_GM_UNLOCK(nal_data);
498 if (nal_data->sysctl)
499 unregister_sysctl_table (nal_data->sysctl);
500 PORTAL_FREE(nal, sizeof(nal_t));
501 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
502 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));