1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
5 * Start up the internal library and clear all structures
6 * Called by the NAL when it initializes. Safe to call multiple times.
8 * Copyright (c) 2001-2003 Cluster File Systems, Inc.
10 * This file is part of Lustre, http://www.lustre.org
12 * Lustre is free software; you can redistribute it and/or
13 * modify it under the terms of version 2 of the GNU General Public
14 * License as published by the Free Software Foundation.
16 * Lustre is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with Lustre; if not, write to the Free Software
23 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26 # define DEBUG_SUBSYSTEM S_PORTALS
27 #include <portals/lib-p30.h>
30 # include <libcfs/kp30.h>
33 # include <sys/time.h>
36 #ifndef PTL_USE_LIB_FREELIST
39 kportal_descriptor_setup (lib_nal_t *nal,
40 ptl_ni_limits_t *requested_limits,
41 ptl_ni_limits_t *actual_limits)
43 /* Ignore requested limits! */
44 actual_limits->max_mes = INT_MAX;
45 actual_limits->max_mds = INT_MAX;
46 actual_limits->max_eqs = INT_MAX;
52 kportal_descriptor_cleanup (lib_nal_t *nal)
58 lib_freelist_init (lib_nal_t *nal, lib_freelist_t *fl, int n, int size)
64 size += offsetof (lib_freeobj_t, fo_contents);
66 PORTAL_ALLOC(space, n * size);
68 return (PTL_NO_SPACE);
70 CFS_INIT_LIST_HEAD (&fl->fl_list);
73 fl->fl_objsize = size;
77 memset (space, 0, size);
78 list_add ((struct list_head *)space, &fl->fl_list);
86 lib_freelist_fini (lib_nal_t *nal, lib_freelist_t *fl)
91 if (fl->fl_nobjs == 0)
95 for (el = fl->fl_list.next; el != &fl->fl_list; el = el->next)
98 LASSERT (count == fl->fl_nobjs);
100 PORTAL_FREE(fl->fl_objs, fl->fl_nobjs * fl->fl_objsize);
101 memset (fl, 0, sizeof (fl));
105 kportal_descriptor_setup (lib_nal_t *nal,
106 ptl_ni_limits_t *requested_limits,
107 ptl_ni_limits_t *actual_limits)
109 /* NB on failure caller must still call kportal_descriptor_cleanup */
111 lib_ni_t *ni = &nal->libnal_ni;
114 memset (&ni->ni_free_mes, 0, sizeof (ni->ni_free_mes));
115 memset (&ni->ni_free_msgs, 0, sizeof (ni->ni_free_msgs));
116 memset (&ni->ni_free_mds, 0, sizeof (ni->ni_free_mds));
117 memset (&ni->ni_free_eqs, 0, sizeof (ni->ni_free_eqs));
119 /* Ignore requested limits! */
120 actual_limits->max_mes = MAX_MES;
121 actual_limits->max_mds = MAX_MDS;
122 actual_limits->max_eqs = MAX_EQS;
123 /* Hahahah what a load of bollocks. There's nowhere to
124 * specify the max # messages in-flight */
126 rc = lib_freelist_init (nal, &ni->ni_free_mes,
127 MAX_MES, sizeof (lib_me_t));
131 rc = lib_freelist_init (nal, &ni->ni_free_msgs,
132 MAX_MSGS, sizeof (lib_msg_t));
136 rc = lib_freelist_init (nal, &ni->ni_free_mds,
137 MAX_MDS, sizeof (lib_md_t));
141 rc = lib_freelist_init (nal, &ni->ni_free_eqs,
142 MAX_EQS, sizeof (lib_eq_t));
147 kportal_descriptor_cleanup (lib_nal_t *nal)
149 lib_ni_t *ni = &nal->libnal_ni;
151 lib_freelist_fini (nal, &ni->ni_free_mes);
152 lib_freelist_fini (nal, &ni->ni_free_msgs);
153 lib_freelist_fini (nal, &ni->ni_free_mds);
154 lib_freelist_fini (nal, &ni->ni_free_eqs);
160 lib_create_interface_cookie (lib_nal_t *nal)
162 /* NB the interface cookie in wire handles guards against delayed
163 * replies and ACKs appearing valid in a new instance of the same
164 * interface. Initialisation time, even if it's only implemented
165 * to millisecond resolution is probably easily good enough. */
169 int rc = gettimeofday (&tv, NULL);
172 do_gettimeofday(&tv);
176 cookie += tv.tv_usec;
181 lib_setup_handle_hash (lib_nal_t *nal)
183 lib_ni_t *ni = &nal->libnal_ni;
186 /* Arbitrary choice of hash table size */
188 ni->ni_lh_hash_size = PAGE_SIZE / sizeof (struct list_head);
190 ni->ni_lh_hash_size = (MAX_MES + MAX_MDS + MAX_EQS)/4;
192 PORTAL_ALLOC(ni->ni_lh_hash_table,
193 ni->ni_lh_hash_size * sizeof (struct list_head));
194 if (ni->ni_lh_hash_table == NULL)
195 return (PTL_NO_SPACE);
197 for (i = 0; i < ni->ni_lh_hash_size; i++)
198 CFS_INIT_LIST_HEAD (&ni->ni_lh_hash_table[i]);
200 ni->ni_next_object_cookie = PTL_COOKIE_TYPES;
206 lib_cleanup_handle_hash (lib_nal_t *nal)
208 lib_ni_t *ni = &nal->libnal_ni;
210 if (ni->ni_lh_hash_table == NULL)
213 PORTAL_FREE(ni->ni_lh_hash_table,
214 ni->ni_lh_hash_size * sizeof (struct list_head));
218 lib_lookup_cookie (lib_nal_t *nal, __u64 cookie, int type)
220 /* ALWAYS called with statelock held */
221 lib_ni_t *ni = &nal->libnal_ni;
222 struct list_head *list;
223 struct list_head *el;
226 if ((cookie & (PTL_COOKIE_TYPES - 1)) != type)
229 hash = ((unsigned int)cookie) % ni->ni_lh_hash_size;
230 list = &ni->ni_lh_hash_table[hash];
232 list_for_each (el, list) {
233 lib_handle_t *lh = list_entry (el, lib_handle_t, lh_hash_chain);
235 if (lh->lh_cookie == cookie)
243 lib_initialise_handle (lib_nal_t *nal, lib_handle_t *lh, int type)
245 /* ALWAYS called with statelock held */
246 lib_ni_t *ni = &nal->libnal_ni;
249 LASSERT (type >= 0 && type < PTL_COOKIE_TYPES);
250 lh->lh_cookie = ni->ni_next_object_cookie | type;
251 ni->ni_next_object_cookie += PTL_COOKIE_TYPES;
253 hash = ((unsigned int)lh->lh_cookie) % ni->ni_lh_hash_size;
254 list_add (&lh->lh_hash_chain, &ni->ni_lh_hash_table[hash]);
258 lib_invalidate_handle (lib_nal_t *nal, lib_handle_t *lh)
260 list_del (&lh->lh_hash_chain);
264 lib_init(lib_nal_t *libnal, nal_t *apinal,
265 ptl_process_id_t process_id,
266 ptl_ni_limits_t *requested_limits,
267 ptl_ni_limits_t *actual_limits)
270 lib_ni_t *ni = &libnal->libnal_ni;
275 /* NB serialised in PtlNIInit() */
277 lib_assert_wire_constants ();
279 /* Setup the API nal with the lib API handling functions */
280 apinal->nal_get_id = lib_api_get_id;
281 apinal->nal_ni_status = lib_api_ni_status;
282 apinal->nal_ni_dist = lib_api_ni_dist;
283 apinal->nal_fail_nid = lib_api_fail_nid;
284 apinal->nal_loopback = lib_api_loopback;
285 apinal->nal_me_attach = lib_api_me_attach;
286 apinal->nal_me_insert = lib_api_me_insert;
287 apinal->nal_me_unlink = lib_api_me_unlink;
288 apinal->nal_md_attach = lib_api_md_attach;
289 apinal->nal_md_bind = lib_api_md_bind;
290 apinal->nal_md_unlink = lib_api_md_unlink;
291 apinal->nal_md_update = lib_api_md_update;
292 apinal->nal_eq_alloc = lib_api_eq_alloc;
293 apinal->nal_eq_free = lib_api_eq_free;
294 apinal->nal_eq_poll = lib_api_eq_poll;
295 apinal->nal_put = lib_api_put;
296 apinal->nal_get = lib_api_get;
298 apinal->nal_data = libnal;
301 rc = kportal_descriptor_setup (libnal, requested_limits,
302 &ni->ni_actual_limits);
306 memset(&ni->ni_counters, 0, sizeof(lib_counters_t));
308 CFS_INIT_LIST_HEAD (&ni->ni_active_msgs);
309 CFS_INIT_LIST_HEAD (&ni->ni_active_mds);
310 CFS_INIT_LIST_HEAD (&ni->ni_active_eqs);
311 CFS_INIT_LIST_HEAD (&ni->ni_test_peers);
314 spin_lock_init (&ni->ni_lock);
315 cfs_waitq_init (&ni->ni_waitq);
317 pthread_mutex_init(&ni->ni_mutex, NULL);
318 pthread_cond_init(&ni->ni_cond, NULL);
321 ni->ni_interface_cookie = lib_create_interface_cookie (libnal);
322 ni->ni_next_object_cookie = 0;
323 rc = lib_setup_handle_hash (libnal);
327 ni->ni_pid = process_id;
329 if (requested_limits != NULL)
330 ptl_size = requested_limits->max_pt_index + 1;
334 ni->ni_portals.size = ptl_size;
335 PORTAL_ALLOC(ni->ni_portals.tbl,
336 ptl_size * sizeof(struct list_head));
337 if (ni->ni_portals.tbl == NULL) {
342 for (i = 0; i < ptl_size; i++)
343 CFS_INIT_LIST_HEAD(&(ni->ni_portals.tbl[i]));
345 /* max_{mes,mds,eqs} set in kportal_descriptor_setup */
347 /* We don't have an access control table! */
348 ni->ni_actual_limits.max_ac_index = -1;
350 ni->ni_actual_limits.max_pt_index = ptl_size - 1;
351 ni->ni_actual_limits.max_md_iovecs = PTL_MD_MAX_IOV;
352 ni->ni_actual_limits.max_me_list = INT_MAX;
354 /* We don't support PtlGetPut! */
355 ni->ni_actual_limits.max_getput_md = 0;
357 if (actual_limits != NULL)
358 *actual_limits = ni->ni_actual_limits;
360 /* disable loopback optimisation by default */
365 lib_cleanup_handle_hash (libnal);
366 kportal_descriptor_cleanup (libnal);
373 lib_fini(lib_nal_t *nal)
375 lib_ni_t *ni = &nal->libnal_ni;
378 /* NB no state_lock() since this is the last reference. The NAL
379 * should have shut down already, so it should be safe to unlink
380 * and free all descriptors, even those that appear committed to a
381 * network op (eg MD with non-zero pending count)
384 for (idx = 0; idx < ni->ni_portals.size; idx++)
385 while (!list_empty (&ni->ni_portals.tbl[idx])) {
386 lib_me_t *me = list_entry (ni->ni_portals.tbl[idx].next,
389 CERROR ("Active me %p on exit\n", me);
390 list_del (&me->me_list);
391 lib_me_free (nal, me);
394 while (!list_empty (&ni->ni_active_mds)) {
395 lib_md_t *md = list_entry (ni->ni_active_mds.next,
398 CERROR ("Active md %p on exit\n", md);
399 list_del (&md->md_list);
400 lib_md_free (nal, md);
403 while (!list_empty (&ni->ni_active_eqs)) {
404 lib_eq_t *eq = list_entry (ni->ni_active_eqs.next,
407 CERROR ("Active eq %p on exit\n", eq);
408 list_del (&eq->eq_list);
409 lib_eq_free (nal, eq);
412 while (!list_empty (&ni->ni_active_msgs)) {
413 lib_msg_t *msg = list_entry (ni->ni_active_msgs.next,
414 lib_msg_t, msg_list);
416 CERROR ("Active msg %p on exit\n", msg);
417 list_del (&msg->msg_list);
418 lib_msg_free (nal, msg);
421 PORTAL_FREE(ni->ni_portals.tbl,
422 ni->ni_portals.size * sizeof(struct list_head));
424 lib_cleanup_handle_hash (nal);
425 kportal_descriptor_cleanup (nal);
428 pthread_mutex_destroy(&ni->ni_mutex);
429 pthread_cond_destroy(&ni->ni_cond);