1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
5 * Start up the internal library and clear all structures
6 * Called by the NAL when it initializes. Safe to call multiple times.
8 * Copyright (c) 2001-2003 Cluster File Systems, Inc.
9 * Copyright (c) 2001-2002 Sandia National Laboratories
11 * This file is part of Lustre, http://www.sf.net/projects/lustre/
13 * Lustre is free software; you can redistribute it and/or
14 * modify it under the terms of version 2 of the GNU General Public
15 * License as published by the Free Software Foundation.
17 * Lustre is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
22 * You should have received a copy of the GNU General Public License
23 * along with Lustre; if not, write to the Free Software
24 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
27 # define DEBUG_SUBSYSTEM S_PORTALS
28 #include <portals/lib-p30.h>
31 # include <linux/string.h> /* for memset() */
32 # include <linux/kp30.h>
33 # ifdef KERNEL_ADDR_CACHE
34 # include <compute/OS/addrCache/cache.h>
38 # include <sys/time.h>
41 #ifndef PTL_USE_LIB_FREELIST
44 kportal_descriptor_setup (lib_nal_t *nal,
45 ptl_ni_limits_t *requested_limits,
46 ptl_ni_limits_t *actual_limits)
48 /* Ignore requested limits! */
49 actual_limits->max_mes = INT_MAX;
50 actual_limits->max_mds = INT_MAX;
51 actual_limits->max_eqs = INT_MAX;
57 kportal_descriptor_cleanup (lib_nal_t *nal)
63 lib_freelist_init (lib_nal_t *nal, lib_freelist_t *fl, int n, int size)
69 size += offsetof (lib_freeobj_t, fo_contents);
71 PORTAL_ALLOC(space, n * size);
73 return (PTL_NO_SPACE);
75 INIT_LIST_HEAD (&fl->fl_list);
78 fl->fl_objsize = size;
82 memset (space, 0, size);
83 list_add ((struct list_head *)space, &fl->fl_list);
91 lib_freelist_fini (lib_nal_t *nal, lib_freelist_t *fl)
96 if (fl->fl_nobjs == 0)
100 for (el = fl->fl_list.next; el != &fl->fl_list; el = el->next)
103 LASSERT (count == fl->fl_nobjs);
105 PORTAL_FREE(fl->fl_objs, fl->fl_nobjs * fl->fl_objsize);
106 memset (fl, 0, sizeof (fl));
110 kportal_descriptor_setup (lib_nal_t *nal,
111 ptl_ni_limits_t *requested_limits,
112 ptl_ni_limits_t *actual_limits)
114 /* NB on failure caller must still call kportal_descriptor_cleanup */
116 lib_ni_t *ni = &nal->libnal_ni;
119 memset (&ni->ni_free_mes, 0, sizeof (ni->ni_free_mes));
120 memset (&ni->ni_free_msgs, 0, sizeof (ni->ni_free_msgs));
121 memset (&ni->ni_free_mds, 0, sizeof (ni->ni_free_mds));
122 memset (&ni->ni_free_eqs, 0, sizeof (ni->ni_free_eqs));
124 /* Ignore requested limits! */
125 actual_limits->max_mes = MAX_MES;
126 actual_limits->max_mds = MAX_MDS;
127 actual_limits->max_eqs = MAX_EQS;
128 /* Hahahah what a load of bollocks. There's nowhere to
129 * specify the max # messages in-flight */
131 rc = lib_freelist_init (nal, &ni->ni_free_mes,
132 MAX_MES, sizeof (lib_me_t));
136 rc = lib_freelist_init (nal, &ni->ni_free_msgs,
137 MAX_MSGS, sizeof (lib_msg_t));
141 rc = lib_freelist_init (nal, &ni->ni_free_mds,
142 MAX_MDS, sizeof (lib_md_t));
146 rc = lib_freelist_init (nal, &ni->ni_free_eqs,
147 MAX_EQS, sizeof (lib_eq_t));
152 kportal_descriptor_cleanup (lib_nal_t *nal)
154 lib_ni_t *ni = &nal->libnal_ni;
156 lib_freelist_fini (nal, &ni->ni_free_mes);
157 lib_freelist_fini (nal, &ni->ni_free_msgs);
158 lib_freelist_fini (nal, &ni->ni_free_mds);
159 lib_freelist_fini (nal, &ni->ni_free_eqs);
165 lib_create_interface_cookie (lib_nal_t *nal)
167 /* NB the interface cookie in wire handles guards against delayed
168 * replies and ACKs appearing valid in a new instance of the same
169 * interface. Initialisation time, even if it's only implemented
170 * to millisecond resolution is probably easily good enough. */
174 int rc = gettimeofday (&tv, NULL);
177 do_gettimeofday(&tv);
181 cookie += tv.tv_usec;
186 lib_setup_handle_hash (lib_nal_t *nal)
188 lib_ni_t *ni = &nal->libnal_ni;
191 /* Arbitrary choice of hash table size */
193 ni->ni_lh_hash_size = PAGE_SIZE / sizeof (struct list_head);
195 ni->ni_lh_hash_size = (MAX_MES + MAX_MDS + MAX_EQS)/4;
197 PORTAL_ALLOC(ni->ni_lh_hash_table,
198 ni->ni_lh_hash_size * sizeof (struct list_head));
199 if (ni->ni_lh_hash_table == NULL)
200 return (PTL_NO_SPACE);
202 for (i = 0; i < ni->ni_lh_hash_size; i++)
203 INIT_LIST_HEAD (&ni->ni_lh_hash_table[i]);
205 ni->ni_next_object_cookie = PTL_COOKIE_TYPES;
211 lib_cleanup_handle_hash (lib_nal_t *nal)
213 lib_ni_t *ni = &nal->libnal_ni;
215 if (ni->ni_lh_hash_table == NULL)
218 PORTAL_FREE(ni->ni_lh_hash_table,
219 ni->ni_lh_hash_size * sizeof (struct list_head));
223 lib_lookup_cookie (lib_nal_t *nal, __u64 cookie, int type)
225 /* ALWAYS called with statelock held */
226 lib_ni_t *ni = &nal->libnal_ni;
227 struct list_head *list;
228 struct list_head *el;
231 if ((cookie & (PTL_COOKIE_TYPES - 1)) != type)
234 hash = ((unsigned int)cookie) % ni->ni_lh_hash_size;
235 list = &ni->ni_lh_hash_table[hash];
237 list_for_each (el, list) {
238 lib_handle_t *lh = list_entry (el, lib_handle_t, lh_hash_chain);
240 if (lh->lh_cookie == cookie)
248 lib_initialise_handle (lib_nal_t *nal, lib_handle_t *lh, int type)
250 /* ALWAYS called with statelock held */
251 lib_ni_t *ni = &nal->libnal_ni;
254 LASSERT (type >= 0 && type < PTL_COOKIE_TYPES);
255 lh->lh_cookie = ni->ni_next_object_cookie | type;
256 ni->ni_next_object_cookie += PTL_COOKIE_TYPES;
258 hash = ((unsigned int)lh->lh_cookie) % ni->ni_lh_hash_size;
259 list_add (&lh->lh_hash_chain, &ni->ni_lh_hash_table[hash]);
263 lib_invalidate_handle (lib_nal_t *nal, lib_handle_t *lh)
265 list_del (&lh->lh_hash_chain);
269 lib_init(lib_nal_t *libnal, nal_t *apinal,
270 ptl_process_id_t process_id,
271 ptl_ni_limits_t *requested_limits,
272 ptl_ni_limits_t *actual_limits)
275 lib_ni_t *ni = &libnal->libnal_ni;
280 /* NB serialised in PtlNIInit() */
282 lib_assert_wire_constants ();
284 /* Setup the API nal with the lib API handling functions */
285 apinal->nal_get_id = lib_api_get_id;
286 apinal->nal_ni_status = lib_api_ni_status;
287 apinal->nal_ni_dist = lib_api_ni_dist;
288 apinal->nal_fail_nid = lib_api_fail_nid;
289 apinal->nal_me_attach = lib_api_me_attach;
290 apinal->nal_me_insert = lib_api_me_insert;
291 apinal->nal_me_unlink = lib_api_me_unlink;
292 apinal->nal_md_attach = lib_api_md_attach;
293 apinal->nal_md_bind = lib_api_md_bind;
294 apinal->nal_md_unlink = lib_api_md_unlink;
295 apinal->nal_md_update = lib_api_md_update;
296 apinal->nal_eq_alloc = lib_api_eq_alloc;
297 apinal->nal_eq_free = lib_api_eq_free;
298 apinal->nal_eq_poll = lib_api_eq_poll;
299 apinal->nal_put = lib_api_put;
300 apinal->nal_get = lib_api_get;
302 apinal->nal_data = libnal;
305 rc = kportal_descriptor_setup (libnal, requested_limits,
306 &ni->ni_actual_limits);
310 memset(&ni->ni_counters, 0, sizeof(lib_counters_t));
312 INIT_LIST_HEAD (&ni->ni_active_msgs);
313 INIT_LIST_HEAD (&ni->ni_active_mds);
314 INIT_LIST_HEAD (&ni->ni_active_eqs);
315 INIT_LIST_HEAD (&ni->ni_test_peers);
318 spin_lock_init (&ni->ni_lock);
319 init_waitqueue_head (&ni->ni_waitq);
321 pthread_mutex_init(&ni->ni_mutex, NULL);
322 pthread_cond_init(&ni->ni_cond, NULL);
325 ni->ni_interface_cookie = lib_create_interface_cookie (libnal);
326 ni->ni_next_object_cookie = 0;
327 rc = lib_setup_handle_hash (libnal);
331 ni->ni_pid = process_id;
333 if (requested_limits != NULL)
334 ptl_size = requested_limits->max_pt_index + 1;
338 ni->ni_portals.size = ptl_size;
339 PORTAL_ALLOC(ni->ni_portals.tbl,
340 ptl_size * sizeof(struct list_head));
341 if (ni->ni_portals.tbl == NULL) {
346 for (i = 0; i < ptl_size; i++)
347 INIT_LIST_HEAD(&(ni->ni_portals.tbl[i]));
349 /* max_{mes,mds,eqs} set in kportal_descriptor_setup */
351 /* We don't have an access control table! */
352 ni->ni_actual_limits.max_ac_index = -1;
354 ni->ni_actual_limits.max_pt_index = ptl_size - 1;
355 ni->ni_actual_limits.max_md_iovecs = PTL_MD_MAX_IOV;
356 ni->ni_actual_limits.max_me_list = INT_MAX;
358 /* We don't support PtlGetPut! */
359 ni->ni_actual_limits.max_getput_md = 0;
361 if (actual_limits != NULL)
362 *actual_limits = ni->ni_actual_limits;
366 lib_cleanup_handle_hash (libnal);
367 kportal_descriptor_cleanup (libnal);
374 lib_fini(lib_nal_t *nal)
376 lib_ni_t *ni = &nal->libnal_ni;
379 /* NB no state_lock() since this is the last reference. The NAL
380 * should have shut down already, so it should be safe to unlink
381 * and free all descriptors, even those that appear committed to a
382 * network op (eg MD with non-zero pending count)
385 for (idx = 0; idx < ni->ni_portals.size; idx++)
386 while (!list_empty (&ni->ni_portals.tbl[idx])) {
387 lib_me_t *me = list_entry (ni->ni_portals.tbl[idx].next,
390 CERROR ("Active me %p on exit\n", me);
391 list_del (&me->me_list);
392 lib_me_free (nal, me);
395 while (!list_empty (&ni->ni_active_mds)) {
396 lib_md_t *md = list_entry (ni->ni_active_mds.next,
399 CERROR ("Active md %p on exit\n", md);
400 list_del (&md->md_list);
401 lib_md_free (nal, md);
404 while (!list_empty (&ni->ni_active_eqs)) {
405 lib_eq_t *eq = list_entry (ni->ni_active_eqs.next,
408 CERROR ("Active eq %p on exit\n", eq);
409 list_del (&eq->eq_list);
410 lib_eq_free (nal, eq);
413 while (!list_empty (&ni->ni_active_msgs)) {
414 lib_msg_t *msg = list_entry (ni->ni_active_msgs.next,
415 lib_msg_t, msg_list);
417 CERROR ("Active msg %p on exit\n", msg);
418 list_del (&msg->msg_list);
419 lib_msg_free (nal, msg);
422 PORTAL_FREE(ni->ni_portals.tbl,
423 ni->ni_portals.size * sizeof(struct list_head));
425 lib_cleanup_handle_hash (nal);
426 kportal_descriptor_cleanup (nal);
429 pthread_mutex_destroy(&ni->ni_mutex);
430 pthread_cond_destroy(&ni->ni_cond);