1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
5 * Start up the internal library and clear all structures
6 * Called by the NAL when it initializes. Safe to call multiple times.
8 * Copyright (c) 2001-2003 Cluster File Systems, Inc.
9 * Copyright (c) 2001-2002 Sandia National Laboratories
11 * This file is part of Lustre, http://www.sf.net/projects/lustre/
13 * Lustre is free software; you can redistribute it and/or
14 * modify it under the terms of version 2 of the GNU General Public
15 * License as published by the Free Software Foundation.
17 * Lustre is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
22 * You should have received a copy of the GNU General Public License
23 * along with Lustre; if not, write to the Free Software
24 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
27 # define DEBUG_SUBSYSTEM S_PORTALS
28 #include <portals/lib-p30.h>
31 # include <linux/string.h> /* for memset() */
32 # include <linux/kp30.h>
33 # ifdef KERNEL_ADDR_CACHE
34 # include <compute/OS/addrCache/cache.h>
38 # include <sys/time.h>
41 #ifndef PTL_USE_LIB_FREELIST
44 kportal_descriptor_setup (nal_cb_t *nal,
45 ptl_ni_limits_t *requested_limits,
46 ptl_ni_limits_t *actual_limits)
48 /* Ignore requested limits! */
49 actual_limits->max_mes = INT_MAX;
50 actual_limits->max_mds = INT_MAX;
51 actual_limits->max_eqs = INT_MAX;
57 kportal_descriptor_cleanup (nal_cb_t *nal)
63 lib_freelist_init (nal_cb_t *nal, lib_freelist_t *fl, int n, int size)
69 size += offsetof (lib_freeobj_t, fo_contents);
71 space = nal->cb_malloc (nal, n * size);
73 return (PTL_NO_SPACE);
75 INIT_LIST_HEAD (&fl->fl_list);
78 fl->fl_objsize = size;
82 memset (space, 0, size);
83 list_add ((struct list_head *)space, &fl->fl_list);
91 lib_freelist_fini (nal_cb_t *nal, lib_freelist_t *fl)
96 if (fl->fl_nobjs == 0)
100 for (el = fl->fl_list.next; el != &fl->fl_list; el = el->next)
103 LASSERT (count == fl->fl_nobjs);
105 nal->cb_free (nal, fl->fl_objs, fl->fl_nobjs * fl->fl_objsize);
106 memset (fl, 0, sizeof (fl));
110 kportal_descriptor_setup (nal_cb_t *nal,
111 ptl_ni_limits_t *requested_limits,
112 ptl_ni_limits_t *actual_limits)
114 /* NB on failure caller must still call kportal_descriptor_cleanup */
118 memset (&nal->ni.ni_free_mes, 0, sizeof (nal->ni.ni_free_mes));
119 memset (&nal->ni.ni_free_msgs, 0, sizeof (nal->ni.ni_free_msgs));
120 memset (&nal->ni.ni_free_mds, 0, sizeof (nal->ni.ni_free_mds));
121 memset (&nal->ni.ni_free_eqs, 0, sizeof (nal->ni.ni_free_eqs));
123 /* Ignore requested limits! */
124 actual_limits->max_mes = MAX_MES;
125 actual_limits->max_mds = MAX_MDS;
126 actual_limits->max_eqs = MAX_EQS;
127 /* Hahahah what a load of bollocks. There's nowhere to
128 * specify the max # messages in-flight */
130 rc = lib_freelist_init (nal, &nal->ni.ni_free_mes,
131 MAX_MES, sizeof (lib_me_t));
135 rc = lib_freelist_init (nal, &nal->ni.ni_free_msgs,
136 MAX_MSGS, sizeof (lib_msg_t));
140 rc = lib_freelist_init (nal, &nal->ni.ni_free_mds,
141 MAX_MDS, sizeof (lib_md_t));
145 rc = lib_freelist_init (nal, &nal->ni.ni_free_eqs,
146 MAX_EQS, sizeof (lib_eq_t));
151 kportal_descriptor_cleanup (nal_cb_t *nal)
153 lib_freelist_fini (nal, &nal->ni.ni_free_mes);
154 lib_freelist_fini (nal, &nal->ni.ni_free_msgs);
155 lib_freelist_fini (nal, &nal->ni.ni_free_mds);
156 lib_freelist_fini (nal, &nal->ni.ni_free_eqs);
162 lib_create_interface_cookie (nal_cb_t *nal)
164 /* NB the interface cookie in wire handles guards against delayed
165 * replies and ACKs appearing valid in a new instance of the same
166 * interface. Initialisation time, even if it's only implemented
167 * to millisecond resolution is probably easily good enough. */
171 int rc = gettimeofday (&tv, NULL);
174 do_gettimeofday(&tv);
178 cookie += tv.tv_usec;
183 lib_setup_handle_hash (nal_cb_t *nal)
185 lib_ni_t *ni = &nal->ni;
188 /* Arbitrary choice of hash table size */
190 ni->ni_lh_hash_size = PAGE_SIZE / sizeof (struct list_head);
192 ni->ni_lh_hash_size = (MAX_MES + MAX_MDS + MAX_EQS)/4;
194 ni->ni_lh_hash_table =
195 (struct list_head *)nal->cb_malloc (nal, ni->ni_lh_hash_size
196 * sizeof (struct list_head));
197 if (ni->ni_lh_hash_table == NULL)
198 return (PTL_NO_SPACE);
200 for (i = 0; i < ni->ni_lh_hash_size; i++)
201 INIT_LIST_HEAD (&ni->ni_lh_hash_table[i]);
203 ni->ni_next_object_cookie = PTL_COOKIE_TYPES;
209 lib_cleanup_handle_hash (nal_cb_t *nal)
211 lib_ni_t *ni = &nal->ni;
213 if (ni->ni_lh_hash_table == NULL)
216 nal->cb_free (nal, ni->ni_lh_hash_table,
217 ni->ni_lh_hash_size * sizeof (struct list_head));
221 lib_lookup_cookie (nal_cb_t *nal, __u64 cookie, int type)
223 /* ALWAYS called with statelock held */
224 lib_ni_t *ni = &nal->ni;
225 struct list_head *list;
226 struct list_head *el;
229 if ((cookie & (PTL_COOKIE_TYPES - 1)) != type)
232 hash = ((unsigned int)cookie) % ni->ni_lh_hash_size;
233 list = &ni->ni_lh_hash_table[hash];
235 list_for_each (el, list) {
236 lib_handle_t *lh = list_entry (el, lib_handle_t, lh_hash_chain);
238 if (lh->lh_cookie == cookie)
246 lib_initialise_handle (nal_cb_t *nal, lib_handle_t *lh, int type)
248 /* ALWAYS called with statelock held */
249 lib_ni_t *ni = &nal->ni;
252 LASSERT (type >= 0 && type < PTL_COOKIE_TYPES);
253 lh->lh_cookie = ni->ni_next_object_cookie | type;
254 ni->ni_next_object_cookie += PTL_COOKIE_TYPES;
256 hash = ((unsigned int)lh->lh_cookie) % ni->ni_lh_hash_size;
257 list_add (&lh->lh_hash_chain, &ni->ni_lh_hash_table[hash]);
261 lib_invalidate_handle (nal_cb_t *nal, lib_handle_t *lh)
263 list_del (&lh->lh_hash_chain);
267 lib_init(nal_cb_t *nal, ptl_process_id_t process_id,
268 ptl_ni_limits_t *requested_limits,
269 ptl_ni_limits_t *actual_limits)
272 lib_ni_t *ni = &nal->ni;
277 /* NB serialised in PtlNIInit() */
279 lib_assert_wire_constants ();
282 * Allocate the portal table for this interface
283 * and all per-interface objects.
285 memset(&ni->counters, 0, sizeof(lib_counters_t));
287 rc = kportal_descriptor_setup (nal, requested_limits,
292 INIT_LIST_HEAD (&ni->ni_active_msgs);
293 INIT_LIST_HEAD (&ni->ni_active_mds);
294 INIT_LIST_HEAD (&ni->ni_active_eqs);
296 INIT_LIST_HEAD (&ni->ni_test_peers);
298 ni->ni_interface_cookie = lib_create_interface_cookie (nal);
299 ni->ni_next_object_cookie = 0;
300 rc = lib_setup_handle_hash (nal);
304 ni->nid = process_id.nid;
305 ni->pid = process_id.pid;
307 if (requested_limits != NULL)
308 ptl_size = requested_limits->max_pt_index + 1;
312 ni->tbl.size = ptl_size;
313 ni->tbl.tbl = nal->cb_malloc(nal, sizeof(struct list_head) * ptl_size);
314 if (ni->tbl.tbl == NULL) {
319 for (i = 0; i < ptl_size; i++)
320 INIT_LIST_HEAD(&(ni->tbl.tbl[i]));
322 /* max_{mes,mds,eqs} set in kportal_descriptor_setup */
324 /* We don't have an access control table! */
325 ni->actual_limits.max_ac_index = -1;
327 ni->actual_limits.max_pt_index = ptl_size - 1;
328 ni->actual_limits.max_md_iovecs = PTL_MD_MAX_IOV;
329 ni->actual_limits.max_me_list = INT_MAX;
331 /* We don't support PtlGetPut! */
332 ni->actual_limits.max_getput_md = 0;
334 if (actual_limits != NULL)
335 *actual_limits = ni->actual_limits;
339 lib_cleanup_handle_hash (nal);
340 kportal_descriptor_cleanup (nal);
347 lib_fini(nal_cb_t * nal)
349 lib_ni_t *ni = &nal->ni;
352 /* NB no state_lock() since this is the last reference. The NAL
353 * should have shut down already, so it should be safe to unlink
354 * and free all descriptors, even those that appear committed to a
355 * network op (eg MD with non-zero pending count)
358 for (idx = 0; idx < ni->tbl.size; idx++)
359 while (!list_empty (&ni->tbl.tbl[idx])) {
360 lib_me_t *me = list_entry (ni->tbl.tbl[idx].next,
363 CERROR ("Active me %p on exit\n", me);
364 list_del (&me->me_list);
365 lib_me_free (nal, me);
368 while (!list_empty (&ni->ni_active_mds)) {
369 lib_md_t *md = list_entry (ni->ni_active_mds.next,
372 CERROR ("Active md %p on exit\n", md);
373 list_del (&md->md_list);
374 lib_md_free (nal, md);
377 while (!list_empty (&ni->ni_active_eqs)) {
378 lib_eq_t *eq = list_entry (ni->ni_active_eqs.next,
381 CERROR ("Active eq %p on exit\n", eq);
382 list_del (&eq->eq_list);
383 lib_eq_free (nal, eq);
386 while (!list_empty (&ni->ni_active_msgs)) {
387 lib_msg_t *msg = list_entry (ni->ni_active_msgs.next,
388 lib_msg_t, msg_list);
390 CERROR ("Active msg %p on exit\n", msg);
391 list_del (&msg->msg_list);
392 lib_msg_free (nal, msg);
395 nal->cb_free(nal, ni->tbl.tbl, sizeof(struct list_head) * ni->tbl.size);
397 lib_cleanup_handle_hash (nal);
398 kportal_descriptor_cleanup (nal);