Whamcloud - gitweb
bd09d786dec9a67de527d2cf882028a2faa1f801
[fs/lustre-release.git] / lnet / klnds / gmlnd / gmlnd_api.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  *  Copyright (c) 2003 Los Alamos National Laboratory (LANL)
5  *
6  *   This file is part of Lustre, http://www.lustre.org/
7  *
8  *   Lustre is free software; you can redistribute it and/or
9  *   modify it under the terms of version 2 of the GNU General Public
10  *   License as published by the Free Software Foundation.
11  *
12  *   Lustre is distributed in the hope that it will be useful,
13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  *   GNU General Public License for more details.
16  *
17  *   You should have received a copy of the GNU General Public License
18  *   along with Lustre; if not, write to the Free Software
19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20  */
21
22 /*
23  *      Implements the API NAL functions
24  */
25
26 #include "gmnal.h"
27
28
29
30 gmnal_data_t    *global_nal_data = NULL;
31 #define         GLOBAL_NID_STR_LEN      16
32 char            global_nid_str[GLOBAL_NID_STR_LEN] = {0};
33 ptl_handle_ni_t kgmnal_ni;
34
35 extern int gmnal_cmd(struct portals_cfg *pcfg, void *private);
36
37 /*
38  *      Write the global nid /proc/sys/gmnal/globalnid
39  */
40 #define GMNAL_SYSCTL    201
41 #define GMNAL_SYSCTL_GLOBALNID  1
42
43 static ctl_table gmnal_sysctl_table[] = {
44         {GMNAL_SYSCTL_GLOBALNID, "globalnid",
45          global_nid_str, GLOBAL_NID_STR_LEN,
46          0444, NULL, &proc_dostring},
47         { 0 }
48 };
49
50
51 static ctl_table gmnalnal_top_sysctl_table[] = {
52         {GMNAL_SYSCTL, "gmnal", NULL, 0, 0555, gmnal_sysctl_table},
53         { 0 }
54 };
55
56 /*
57  *      gmnal_api_shutdown
58  *      nal_refct == 0 => called on last matching PtlNIFini()
59  *      Close down this interface and free any resources associated with it
60  *      nal_t   nal     our nal to shutdown
61  */
62 void
63 gmnal_api_shutdown(nal_t *nal)
64 {
65         gmnal_data_t    *nal_data;
66         lib_nal_t       *libnal;
67
68         if (nal->nal_refct != 0)
69                 return;
70         
71
72         LASSERT(nal == global_nal_data->nal);
73         libnal = (lib_nal_t *)nal->nal_data;
74         nal_data = (gmnal_data_t *)libnal->libnal_data;
75         LASSERT(nal_data == global_nal_data);
76         CDEBUG(D_TRACE, "gmnal_api_shutdown: nal_data [%p]\n", nal_data);
77
78         /* Stop portals calling our ioctl handler */
79         libcfs_nal_cmd_unregister(GMNAL);
80
81         /* XXX for shutdown "under fire" we probably need to set a shutdown
82          * flag so when lib calls us we fail immediately and dont queue any
83          * more work but our threads can still call into lib OK.  THEN
84          * shutdown our threads, THEN lib_fini() */
85         lib_fini(libnal);
86
87         gmnal_stop_rxthread(nal_data);
88         gmnal_stop_ctthread(nal_data);
89         gmnal_free_txd(nal_data);
90         gmnal_free_srxd(nal_data);
91         GMNAL_GM_LOCK(nal_data);
92         gm_close(nal_data->gm_port);
93         gm_finalize();
94         GMNAL_GM_UNLOCK(nal_data);
95         if (nal_data->sysctl)
96                 unregister_sysctl_table (nal_data->sysctl);
97         /* Don't free 'nal'; it's a static struct */
98         PORTAL_FREE(nal_data, sizeof(gmnal_data_t));    
99         PORTAL_FREE(libnal, sizeof(lib_nal_t));
100
101         global_nal_data = NULL;
102         PORTAL_MODULE_UNUSE;
103 }
104
105
106 int
107 gmnal_api_startup(nal_t *nal, ptl_pid_t requested_pid,
108                   ptl_ni_limits_t *requested_limits,
109                   ptl_ni_limits_t *actual_limits)
110 {
111
112         lib_nal_t       *libnal = NULL;
113         gmnal_data_t    *nal_data = NULL;
114         gmnal_srxd_t    *srxd = NULL;
115         gm_status_t     gm_status;
116         unsigned int    local_nid = 0, global_nid = 0;
117         ptl_process_id_t process_id;
118
119         if (nal->nal_refct != 0) {
120                 if (actual_limits != NULL) {
121                         libnal = (lib_nal_t *)nal->nal_data;
122                         *actual_limits = libnal->libnal_ni.ni_actual_limits;
123                 }
124                 return (PTL_OK);
125         }
126
127         /* Called on first PtlNIInit() */
128
129         CDEBUG(D_TRACE, "startup\n");
130
131         LASSERT(global_nal_data == NULL);
132
133         PORTAL_ALLOC(nal_data, sizeof(gmnal_data_t));
134         if (!nal_data) {
135                 CDEBUG(D_ERROR, "can't get memory\n");
136                 return(PTL_NO_SPACE);
137         }       
138         memset(nal_data, 0, sizeof(gmnal_data_t));
139         /*
140          *      set the small message buffer size 
141          */
142
143         CDEBUG(D_INFO, "Allocd and reset nal_data[%p]\n", nal_data);
144         CDEBUG(D_INFO, "small_msg_size is [%d]\n", nal_data->small_msg_size);
145
146         PORTAL_ALLOC(libnal, sizeof(lib_nal_t));
147         if (!libnal) {
148                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));
149                 return(PTL_NO_SPACE);
150         }
151         memset(libnal, 0, sizeof(lib_nal_t));
152         CDEBUG(D_INFO, "Allocd and reset libnal[%p]\n", libnal);
153
154         GMNAL_INIT_NAL_CB(libnal);
155         /*
156          *      String them all together
157          */
158         libnal->libnal_data = (void*)nal_data;
159         nal_data->nal = nal;
160         nal_data->libnal = libnal;
161
162         GMNAL_GM_LOCK_INIT(nal_data);
163
164
165         /*
166          *      initialise the interface, 
167          */
168         CDEBUG(D_INFO, "Calling gm_init\n");
169         if (gm_init() != GM_SUCCESS) {
170                 CDEBUG(D_ERROR, "call to gm_init failed\n");
171                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));    
172                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
173                 return(PTL_FAIL);
174         }
175
176
177         CDEBUG(D_NET, "Calling gm_open with port [%d], "
178                "name [%s], version [%d]\n", GMNAL_GM_PORT_ID, 
179                "gmnal", GM_API_VERSION);
180
181         GMNAL_GM_LOCK(nal_data);
182         gm_status = gm_open(&nal_data->gm_port, 0, GMNAL_GM_PORT_ID, "gmnal", 
183                             GM_API_VERSION);
184         GMNAL_GM_UNLOCK(nal_data);
185
186         CDEBUG(D_INFO, "gm_open returned [%d]\n", gm_status);
187         if (gm_status == GM_SUCCESS) {
188                 CDEBUG(D_INFO, "gm_open succeeded port[%p]\n", 
189                        nal_data->gm_port);
190         } else {
191                 switch(gm_status) {
192                 case(GM_INVALID_PARAMETER):
193                         CDEBUG(D_ERROR, "gm_open Failure. Invalid Parameter\n");
194                         break;
195                 case(GM_BUSY):
196                         CDEBUG(D_ERROR, "gm_open Failure. GM Busy\n");
197                         break;
198                 case(GM_NO_SUCH_DEVICE):
199                         CDEBUG(D_ERROR, "gm_open Failure. No such device\n");
200                         break;
201                 case(GM_INCOMPATIBLE_LIB_AND_DRIVER):
202                         CDEBUG(D_ERROR, "gm_open Failure. Incompatile lib "
203                                "and driver\n");
204                         break;
205                 case(GM_OUT_OF_MEMORY):
206                         CDEBUG(D_ERROR, "gm_open Failure. Out of Memory\n");
207                         break;
208                 default:
209                         CDEBUG(D_ERROR, "gm_open Failure. Unknow error "
210                                "code [%d]\n", gm_status);
211                         break;
212                 }       
213                 GMNAL_GM_LOCK(nal_data);
214                 gm_finalize();
215                 GMNAL_GM_UNLOCK(nal_data);
216                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));    
217                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
218                 return(PTL_FAIL);
219         }
220
221         
222         nal_data->small_msg_size = gmnal_small_msg_size;
223         nal_data->small_msg_gmsize = 
224                         gm_min_size_for_length(gmnal_small_msg_size);
225
226         if (gmnal_alloc_srxd(nal_data) != GMNAL_STATUS_OK) {
227                 CDEBUG(D_ERROR, "Failed to allocate small rx descriptors\n");
228                 gmnal_free_txd(nal_data);
229                 GMNAL_GM_LOCK(nal_data);
230                 gm_close(nal_data->gm_port);
231                 gm_finalize();
232                 GMNAL_GM_UNLOCK(nal_data);
233                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));    
234                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
235                 return(PTL_FAIL);
236         }
237
238
239         /*
240          *      Hang out a bunch of small receive buffers
241          *      In fact hang them all out
242          */
243         while((srxd = gmnal_get_srxd(nal_data, 0))) {
244                 CDEBUG(D_NET, "giving [%p] to gm_provide_recvive_buffer\n", 
245                        srxd->buffer);
246                 GMNAL_GM_LOCK(nal_data);
247                 gm_provide_receive_buffer_with_tag(nal_data->gm_port, 
248                                                    srxd->buffer, srxd->gmsize, 
249                                                    GM_LOW_PRIORITY, 0);
250                 GMNAL_GM_UNLOCK(nal_data);
251         }
252         
253         /*
254          *      Allocate pools of small tx buffers and descriptors
255          */
256         if (gmnal_alloc_txd(nal_data) != GMNAL_STATUS_OK) {
257                 CDEBUG(D_ERROR, "Failed to allocate small tx descriptors\n");
258                 GMNAL_GM_LOCK(nal_data);
259                 gm_close(nal_data->gm_port);
260                 gm_finalize();
261                 GMNAL_GM_UNLOCK(nal_data);
262                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));    
263                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
264                 return(PTL_FAIL);
265         }
266
267         gmnal_start_kernel_threads(nal_data);
268
269         while (nal_data->rxthread_flag != GMNAL_RXTHREADS_STARTED) {
270                 gmnal_yield(1);
271                 CDEBUG(D_INFO, "Waiting for receive thread signs of life\n");
272         }
273
274         CDEBUG(D_INFO, "receive thread seems to have started\n");
275
276
277         /*
278          *      Initialise the portals library
279          */
280         CDEBUG(D_NET, "Getting node id\n");
281         GMNAL_GM_LOCK(nal_data);
282         gm_status = gm_get_node_id(nal_data->gm_port, &local_nid);
283         GMNAL_GM_UNLOCK(nal_data);
284         if (gm_status != GM_SUCCESS) {
285                 gmnal_stop_rxthread(nal_data);
286                 gmnal_stop_ctthread(nal_data);
287                 CDEBUG(D_ERROR, "can't determine node id\n");
288                 gmnal_free_txd(nal_data);
289                 gmnal_free_srxd(nal_data);
290                 GMNAL_GM_LOCK(nal_data);
291                 gm_close(nal_data->gm_port);
292                 gm_finalize();
293                 GMNAL_GM_UNLOCK(nal_data);
294                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));    
295                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
296                 return(PTL_FAIL);
297         }
298
299         nal_data->gm_local_nid = local_nid;
300         CDEBUG(D_INFO, "Local node id is [%u]\n", local_nid);
301
302         GMNAL_GM_LOCK(nal_data);
303         gm_status = gm_node_id_to_global_id(nal_data->gm_port, local_nid, 
304                                             &global_nid);
305         GMNAL_GM_UNLOCK(nal_data);
306         if (gm_status != GM_SUCCESS) {
307                 CDEBUG(D_ERROR, "failed to obtain global id\n");
308                 gmnal_stop_rxthread(nal_data);
309                 gmnal_stop_ctthread(nal_data);
310                 gmnal_free_txd(nal_data);
311                 gmnal_free_srxd(nal_data);
312                 GMNAL_GM_LOCK(nal_data);
313                 gm_close(nal_data->gm_port);
314                 gm_finalize();
315                 GMNAL_GM_UNLOCK(nal_data);
316                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));    
317                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
318                 return(PTL_FAIL);
319         }
320         CDEBUG(D_INFO, "Global node id is [%u]\n", global_nid);
321         nal_data->gm_global_nid = global_nid;
322         snprintf(global_nid_str, GLOBAL_NID_STR_LEN, "%u", global_nid);
323
324 /*
325         pid = gm_getpid();
326 */
327         process_id.pid = requested_pid;
328         process_id.nid = global_nid;
329         
330         CDEBUG(D_INFO, "portals_pid is [%u]\n", process_id.pid);
331         CDEBUG(D_INFO, "portals_nid is ["LPU64"]\n", process_id.nid);
332         
333         CDEBUG(D_PORTALS, "calling lib_init\n");
334         if (lib_init(libnal, nal, process_id, 
335                      requested_limits, actual_limits) != PTL_OK) {
336                 CDEBUG(D_ERROR, "lib_init failed\n");
337                 gmnal_stop_rxthread(nal_data);
338                 gmnal_stop_ctthread(nal_data);
339                 gmnal_free_txd(nal_data);
340                 gmnal_free_srxd(nal_data);
341                 GMNAL_GM_LOCK(nal_data);
342                 gm_close(nal_data->gm_port);
343                 gm_finalize();
344                 GMNAL_GM_UNLOCK(nal_data);
345                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));    
346                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
347                 return(PTL_FAIL);
348                 
349         }
350
351         if (libcfs_nal_cmd_register(GMNAL, &gmnal_cmd, libnal->libnal_data) != 0) {
352                 CDEBUG(D_INFO, "libcfs_nal_cmd_register failed\n");
353
354                 /* XXX these cleanup cases should be restructured to
355                  * minimise duplication... */
356                 lib_fini(libnal);
357                 
358                 gmnal_stop_rxthread(nal_data);
359                 gmnal_stop_ctthread(nal_data);
360                 gmnal_free_txd(nal_data);
361                 gmnal_free_srxd(nal_data);
362                 GMNAL_GM_LOCK(nal_data);
363                 gm_close(nal_data->gm_port);
364                 gm_finalize();
365                 GMNAL_GM_UNLOCK(nal_data);
366                 PORTAL_FREE(nal_data, sizeof(gmnal_data_t));    
367                 PORTAL_FREE(libnal, sizeof(lib_nal_t));
368                 return(PTL_FAIL);
369         }
370
371         /* might be better to initialise this at module load rather than in
372          * NAL startup */
373         nal_data->sysctl = NULL;
374         nal_data->sysctl = register_sysctl_table (gmnalnal_top_sysctl_table, 0);
375
376         
377         CDEBUG(D_INFO, "gmnal_init finished\n");
378         global_nal_data = nal->nal_data;
379
380         /* no unload now until shutdown */
381         PORTAL_MODULE_USE;
382         
383         return(PTL_OK);
384 }
385
386 nal_t the_gm_nal;
387
388 /* 
389  *        Called when module loaded
390  */
391 int gmnal_init(void)
392 {
393         int    rc;
394
395         memset(&the_gm_nal, 0, sizeof(nal_t));
396         CDEBUG(D_INFO, "reset nal[%p]\n", &the_gm_nal);
397         GMNAL_INIT_NAL(&the_gm_nal);
398
399         rc = ptl_register_nal(GMNAL, &the_gm_nal);
400         if (rc != PTL_OK)
401                 CERROR("Can't register GMNAL: %d\n", rc);
402         rc = PtlNIInit(GMNAL, LUSTRE_SRV_PTL_PID, NULL, NULL, &kgmnal_ni);
403         if (rc != PTL_OK && rc != PTL_IFACE_DUP) {
404                 ptl_unregister_nal(GMNAL);
405                 return (-ENODEV);
406         }
407
408         return (rc);
409 }
410
411                 
412
413 /*
414  *      Called when module removed
415  */
416 void gmnal_fini()
417 {
418         CDEBUG(D_TRACE, "gmnal_fini\n");
419
420         LASSERT(global_nal_data == NULL);
421         PtlNIFini(kgmnal_ni);
422
423         ptl_unregister_nal(GMNAL);
424 }