1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (c) 2003 Los Alamos National Laboratory (LANL)
6 * This file is part of Lustre, http://www.lustre.org/
8 * Lustre is free software; you can redistribute it and/or
9 * modify it under the terms of version 2 of the GNU General Public
10 * License as published by the Free Software Foundation.
12 * Lustre is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with Lustre; if not, write to the Free Software
19 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23 * Implements the API NAL functions
28 lgmnal_data_t *global_nal_data = NULL;
31 * This function takes a pack block of arguments from the NAL API
32 * module and passes them to the NAL CB module. The CB module unpacks
33 * the args and calls the appropriate function indicated by index.
34 * Typically this function is used to pass args between kernel and use
36 * As lgmanl exists entirely in kernel, just pass the arg block directly to
37 * the NAL CB, buy passing the args to lib_dispatch
40 * int index the api function that initiated this call
41 * void *args packed block of function args
42 * size_t arg_len length of args block
43 * void *ret A return value for the API NAL
44 * size_t ret_len Size of the return value
49 lgmnal_api_forward(nal_t *nal, int index, void *args, size_t arg_len,
50 void *ret, size_t ret_len)
53 nal_cb_t *nal_cb = NULL;
54 lgmnal_data_t *nal_data = NULL;
59 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_api_forward: nal [%p], index [%d], args [%p], arglen [%d], ret [%p], retlen [%d]\n", nal, index, args, arg_len, ret, ret_len));
61 if (!nal || !args || (index < 0) || (arg_len < 0)) {
62 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("Bad args to lgmnal_api_forward\n"));
65 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("No nal specified\n"));
67 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("No args specified\n"));
69 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("Index is negative[%d]\n", index));
71 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("arg_len is negative [%d]\n", arg_len));
76 if (ret && (ret_len <= 0)) {
77 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("Bad args to lgmnal_api_forward\n"));
79 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("ret_len is [%d]\n", ret_len));
86 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("bad nal, no nal data\n"));
90 nal_data = nal->nal_data;
91 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("nal_data is [%p]\n", nal_data));
93 if (!nal_data->nal_cb) {
94 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("bad nal_data, no nal_cb\n"));
98 nal_cb = nal_data->nal_cb;
99 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("nal_cb is [%p]\n", nal_cb));
101 LGMNAL_PRINT(LGMNAL_DEBUG_V, ("lgmnal_api_forward calling lib_dispatch\n"));
102 lib_dispatch(nal_cb, NULL, index, args, ret);
103 LGMNAL_PRINT(LGMNAL_DEBUG_V, ("lgmnal_api_forward returns from lib_dispatch\n"));
110 * lgmnal_api_shutdown
111 * Close down this interface and free any resources associated with it
112 * nal_t nal our nal to shutdown
115 lgmnal_api_shutdown(nal_t *nal, int interface)
118 lgmnal_data_t *nal_data = nal->nal_data;
120 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_api_shutdown: nal_data [%p]\n", nal_data));
123 * TO DO lgmnal_api_shutdown what is to be done?
131 * lgmnal_api_validate
132 * validate a user address for use in communications
133 * There's nothing to be done here
136 lgmnal_api_validate(nal_t *nal, void *base, size_t extent)
139 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_api_validate : nal [%p], base [%p], extent [%d]\n", nal, base, extent));
148 * Give up the processor
151 lgmnal_api_yield(nal_t *nal)
153 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_api_yield : nal [%p]\n", nal));
155 set_current_state(TASK_INTERRUPTIBLE);
165 * Take a threadsafe lock
168 lgmnal_api_lock(nal_t *nal, unsigned long *flags)
171 lgmnal_data_t *nal_data;
173 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_api_lock : nal [%p], flagsa [%p] flags[%ul]\n", nal, flags, *flags));
175 nal_data = nal->nal_data;
176 nal_cb = nal_data->nal_cb;
178 nal_cb->cb_cli(nal_cb, flags);
180 LGMNAL_API_LOCK(nal_data);
188 * Release a threadsafe lock
191 lgmnal_api_unlock(nal_t *nal, unsigned long *flags)
193 lgmnal_data_t *nal_data;
195 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_api_lock : nal [%p], flags [%p]\n", nal, flags));
197 nal_data = nal->nal_data;
199 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("lgmnal_api_unlock bad nal, no nal_data\n"));
201 nal_cb = nal_data->nal_cb;
203 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("lgmnal_api_unlock bad nal_data, no nal_cb\n"));
206 nal_cb->cb_sti(nal_cb, flags);
208 LGMNAL_API_UNLOCK(nal_data);
216 lgmnal_init(int interface, ptl_pt_index_t ptl_size, ptl_ac_index_t ac_size, ptl_pid_t rpid)
220 nal_cb_t *nal_cb = NULL;
221 lgmnal_data_t *nal_data = NULL;
222 lgmnal_srxd_t *srxd = NULL;
223 gm_status_t gm_status;
224 unsigned int local_nid = 0, global_nid = 0;
225 ptl_nid_t portals_nid;
226 ptl_pid_t portals_pid = 0;
229 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_init : interface [%d], ptl_size [%d], ac_size[%d]\n",
230 interface, ptl_size, ac_size));
232 if ((interface < 0) || (interface > LGMNAL_NUM_IF) || (ptl_size <= 0) || (ac_size <= 0) ) {
233 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("bad args\n"));
236 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("parameters check out ok\n"));
239 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Acquired global lock\n"));
242 PORTAL_ALLOC(nal_data, sizeof(lgmnal_data_t));
244 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("can't get memory\n"));
247 memset(nal_data, 0, sizeof(lgmnal_data_t));
249 * set the small message buffer size
251 nal_data->refcnt = 1;
253 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Allocd and reset nal_data[%p]\n", nal_data));
254 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("small_msg_size is [%d]\n", nal_data->small_msg_size));
256 PORTAL_ALLOC(nal, sizeof(nal_t));
258 PORTAL_FREE(nal_data, sizeof(lgmnal_data_t));
261 memset(nal, 0, sizeof(nal_t));
262 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Allocd and reset nal[%p]\n", nal));
264 PORTAL_ALLOC(nal_cb, sizeof(nal_cb_t));
266 PORTAL_FREE(nal, sizeof(nal_t));
267 PORTAL_FREE(nal_data, sizeof(lgmnal_data_t));
270 memset(nal_cb, 0, sizeof(nal_cb_t));
271 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Allocd and reset nal_cb[%p]\n", nal_cb));
273 LGMNAL_INIT_NAL(nal);
274 LGMNAL_INIT_NAL_CB(nal_cb);
276 * String them all together
278 nal->nal_data = (void*)nal_data;
279 nal_cb->nal_data = (void*)nal_data;
281 nal_data->nal_cb = nal_cb;
283 LGMNAL_API_LOCK_INIT(nal_data);
284 LGMNAL_CB_LOCK_INIT(nal_data);
285 LGMNAL_GM_LOCK_INIT(nal_data);
289 * initialise the interface,
291 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Calling gm_init\n"));
292 if (gm_init() != GM_SUCCESS) {
293 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("call to gm_init failed\n"));
294 PORTAL_FREE(nal, sizeof(nal_t));
295 PORTAL_FREE(nal_data, sizeof(lgmnal_data_t));
296 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
301 LGMNAL_PRINT(LGMNAL_DEBUG_V, ("Calling gm_open with interface [%d], port [%d], name [%s], version [%d]\n", interface, LGMNAL_GM_PORT, "lgmnal", GM_API_VERSION));
303 LGMNAL_GM_LOCK(nal_data);
304 gm_status = gm_open(&nal_data->gm_port, 0, LGMNAL_GM_PORT, "lgmnal", GM_API_VERSION);
305 LGMNAL_GM_UNLOCK(nal_data);
307 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("gm_open returned [%d]\n", gm_status));
308 if (gm_status == GM_SUCCESS) {
309 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("gm_open succeeded port[%p]\n", nal_data->gm_port));
312 case(GM_INVALID_PARAMETER):
313 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("gm_open Failure. Invalid Parameter\n"));
316 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("gm_open Failure. GM Busy\n"));
318 case(GM_NO_SUCH_DEVICE):
319 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("gm_open Failure. No such device\n"));
321 case(GM_INCOMPATIBLE_LIB_AND_DRIVER):
322 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("gm_open Failure. Incompatile lib and driver\n"));
324 case(GM_OUT_OF_MEMORY):
325 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("gm_open Failure. Out of Memory\n"));
328 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("gm_open Failure. Unknow error code [%d]\n", gm_status));
331 LGMNAL_GM_LOCK(nal_data);
333 LGMNAL_GM_UNLOCK(nal_data);
334 PORTAL_FREE(nal, sizeof(nal_t));
335 PORTAL_FREE(nal_data, sizeof(lgmnal_data_t));
336 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
341 nal_data->small_msg_size = lgmnal_small_msg_size;
342 nal_data->small_msg_gmsize = gm_min_size_for_length(lgmnal_small_msg_size);
344 if (lgmnal_alloc_srxd(nal_data) != LGMNAL_STATUS_OK) {
345 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("Failed to allocate small rx descriptors\n"));
346 lgmnal_free_stxd(nal_data);
347 LGMNAL_GM_LOCK(nal_data);
348 gm_close(nal_data->gm_port);
350 LGMNAL_GM_UNLOCK(nal_data);
351 PORTAL_FREE(nal, sizeof(nal_t));
352 PORTAL_FREE(nal_data, sizeof(lgmnal_data_t));
353 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
359 * Hang out a bunch of small receive buffers
360 * In fact hang them all out
362 while((srxd = lgmnal_get_srxd(nal_data, 0))) {
363 LGMNAL_PRINT(LGMNAL_DEBUG_V, ("giving [%p] to gm_provide_recvive_buffer\n", srxd->buffer));
364 LGMNAL_GM_LOCK(nal_data);
365 gm_provide_receive_buffer_with_tag(nal_data->gm_port, srxd->buffer,
366 srxd->gmsize, GM_LOW_PRIORITY, 0);
367 LGMNAL_GM_UNLOCK(nal_data);
371 * Allocate pools of small tx buffers and descriptors
373 if (lgmnal_alloc_stxd(nal_data) != LGMNAL_STATUS_OK) {
374 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("Failed to allocate small tx descriptors\n"));
375 LGMNAL_GM_LOCK(nal_data);
376 gm_close(nal_data->gm_port);
378 LGMNAL_GM_UNLOCK(nal_data);
379 PORTAL_FREE(nal, sizeof(nal_t));
380 PORTAL_FREE(nal_data, sizeof(lgmnal_data_t));
381 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
386 * Start the recieve thread
387 * Initialise the gm_alarm we will use to wake the thread is
388 * it needs to be stopped
390 LGMNAL_PRINT(LGMNAL_DEBUG_V, ("Initializing receive thread alarm and flag\n"));
391 gm_initialize_alarm(&nal_data->rxthread_alarm);
392 nal_data->rxthread_flag = LGMNAL_THREAD_START;
395 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Starting receive thread\n"));
396 nal_data->rxthread_pid = kernel_thread(lgmnal_receive_thread, (void*)nal_data, 0);
397 if (nal_data->rxthread_pid <= 0) {
398 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("Receive thread failed to start\n"));
399 lgmnal_free_stxd(nal_data);
400 lgmnal_free_srxd(nal_data);
401 LGMNAL_GM_LOCK(nal_data);
402 gm_close(nal_data->gm_port);
404 LGMNAL_GM_UNLOCK(nal_data);
405 PORTAL_FREE(nal, sizeof(nal_t));
406 PORTAL_FREE(nal_data, sizeof(lgmnal_data_t));
407 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
410 while (nal_data->rxthread_flag != LGMNAL_THREAD_STARTED) {
411 set_current_state(TASK_INTERRUPTIBLE);
412 schedule_timeout(1024);
413 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Waiting for receive thread signs of life\n"));
415 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("receive thread seems to have started\n"));
416 nal_data->rxthread_flag = LGMNAL_THREAD_CONTINUE;
421 * Initialise the portals library
423 LGMNAL_PRINT(LGMNAL_DEBUG_V, ("Getting node id\n"));
424 LGMNAL_GM_LOCK(nal_data);
425 gm_status = gm_get_node_id(nal_data->gm_port, &local_nid);
426 LGMNAL_GM_UNLOCK(nal_data);
427 if (gm_status != GM_SUCCESS) {
428 lgmnal_stop_rxthread(nal_data);
429 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("can't determine node id\n"));
430 lgmnal_free_stxd(nal_data);
431 lgmnal_free_srxd(nal_data);
432 LGMNAL_GM_LOCK(nal_data);
433 gm_close(nal_data->gm_port);
435 LGMNAL_GM_UNLOCK(nal_data);
436 PORTAL_FREE(nal, sizeof(nal_t));
437 PORTAL_FREE(nal_data, sizeof(lgmnal_data_t));
438 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
441 nal_data->gm_local_nid = local_nid;
442 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Local node id is [%u]\n", local_nid));
443 LGMNAL_GM_LOCK(nal_data);
444 gm_status = gm_node_id_to_global_id(nal_data->gm_port, local_nid, &global_nid);
445 LGMNAL_GM_UNLOCK(nal_data);
446 if (gm_status != GM_SUCCESS) {
447 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("failed to obtain global id\n"));
448 lgmnal_stop_rxthread(nal_data);
449 lgmnal_free_stxd(nal_data);
450 lgmnal_free_srxd(nal_data);
451 LGMNAL_GM_LOCK(nal_data);
452 gm_close(nal_data->gm_port);
454 LGMNAL_GM_UNLOCK(nal_data);
455 PORTAL_FREE(nal, sizeof(nal_t));
456 PORTAL_FREE(nal_data, sizeof(lgmnal_data_t));
457 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
460 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("Global node id is [%u][%x]\n", global_nid));
461 nal_data->gm_global_nid = global_nid;
466 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("portals_pid is [%u]\n", portals_pid));
467 portals_nid = (unsigned long)global_nid;
468 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("portals_nid is [%lu]\n", portals_nid));
470 LGMNAL_PRINT(LGMNAL_DEBUG_V, ("calling lib_init\n"));
471 if (lib_init(nal_cb, portals_nid, portals_pid, 1024, ptl_size, ac_size) != PTL_OK) {
472 LGMNAL_PRINT(LGMNAL_DEBUG_ERR, ("lib_init failed\n"));
473 lgmnal_stop_rxthread(nal_data);
474 lgmnal_free_stxd(nal_data);
475 lgmnal_free_srxd(nal_data);
476 LGMNAL_GM_LOCK(nal_data);
477 gm_close(nal_data->gm_port);
479 LGMNAL_GM_UNLOCK(nal_data);
480 PORTAL_FREE(nal, sizeof(nal_t));
481 PORTAL_FREE(nal_data, sizeof(lgmnal_data_t));
482 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
487 LGMNAL_PRINT(LGMNAL_DEBUG_VV, ("lgmnal_init finished\n"));
488 global_nal_data = nal->nal_data;
495 * Called when module removed
499 lgmnal_data_t *nal_data = global_nal_data;
500 nal_t *nal = nal_data->nal;
501 nal_cb_t *nal_cb = nal_data->nal_cb;
503 LGMNAL_PRINT(LGMNAL_DEBUG_TRACE, ("lgmnal_fini\n"));
505 PtlNIFini(lgmnal_ni);
508 lgmnal_stop_rxthread(nal_data);
509 lgmnal_free_stxd(nal_data);
510 lgmnal_free_srxd(nal_data);
511 LGMNAL_GM_LOCK(nal_data);
512 gm_close(nal_data->gm_port);
514 LGMNAL_GM_UNLOCK(nal_data);
515 PORTAL_FREE(nal, sizeof(nal_t));
516 PORTAL_FREE(nal_data, sizeof(lgmnal_data_t));
517 PORTAL_FREE(nal_cb, sizeof(nal_cb_t));
520 EXPORT_SYMBOL(lgmnal_init);
521 EXPORT_SYMBOL(lgmnal_fini);
522 EXPORT_SYMBOL(lgmnal_api_forward);
523 EXPORT_SYMBOL(lgmnal_api_validate);
524 EXPORT_SYMBOL(lgmnal_api_yield);
525 EXPORT_SYMBOL(lgmnal_api_lock);
526 EXPORT_SYMBOL(lgmnal_api_unlock);
527 EXPORT_SYMBOL(lgmnal_api_shutdown);