From: Liang Zhen Date: Sun, 27 May 2012 06:22:34 +0000 (+0800) Subject: LU-56 lnet: abstract container for EQ/ME/MD X-Git-Tag: 2.2.55~28 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=c3a57ec36441c75df03cfbec8f718e053aaad12a LU-56 lnet: abstract container for EQ/ME/MD This patch implemented a simple resource container for LNet, it also created several instances of it to manage different LNet objects (EQ/ME/MD), instead of putting resources in global table/list. It's a work step of LNet SMP improvements, we will create instances for each CPT in following patches. Signed-off-by: Liang Zhen Change-Id: I5e03d618a5c932f032b45b4acdbd4aec546cf9a9 Reviewed-on: http://review.whamcloud.com/2921 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Bobi Jam Reviewed-by: Lai Siyao Reviewed-by: Doug Oucharek Reviewed-by: Andreas Dilger --- diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 4b6c25f..28b69e6 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -126,12 +126,17 @@ do { \ #define MAX_PORTALS 64 +/* these are only used by code with LNET_USE_LIB_FREELIST, but we still + * exported them to !LNET_USE_LIB_FREELIST for easy implemetation */ +#define LNET_FL_MAX_MES 2048 +#define LNET_FL_MAX_MDS 2048 +#define LNET_FL_MAX_EQS 512 +#define LNET_FL_MAX_MSGS 2048 /* Outstanding messages */ + #ifdef LNET_USE_LIB_FREELIST -#define MAX_MES 2048 -#define MAX_MDS 2048 -#define MAX_MSGS 2048 /* Outstanding messages */ -#define MAX_EQS 512 +int lnet_freelist_init(lnet_freelist_t *fl, int n, int size); +void lnet_freelist_fini(lnet_freelist_t *fl); static inline void * lnet_freelist_alloc (lnet_freelist_t *fl) @@ -160,21 +165,24 @@ lnet_freelist_free (lnet_freelist_t *fl, void *obj) static inline lnet_eq_t * lnet_eq_alloc (void) { - /* NEVER called with liblock held */ - lnet_eq_t *eq; + /* NEVER called with resource lock held */ + struct lnet_res_container *rec = &the_lnet.ln_eq_container; + lnet_eq_t *eq; - LNET_LOCK(); - eq = (lnet_eq_t *)lnet_freelist_alloc(&the_lnet.ln_free_eqs); - LNET_UNLOCK(); + LNET_LOCK(); + eq = (lnet_eq_t *)lnet_freelist_alloc(&rec->rec_freelist); + LNET_UNLOCK(); - return (eq); + return eq; } static inline void lnet_eq_free_locked(lnet_eq_t *eq) { /* ALWAYS called with resource lock held */ - lnet_freelist_free(&the_lnet.ln_free_eqs, eq); + struct lnet_res_container *rec = &the_lnet.ln_eq_container; + + lnet_freelist_free(&rec->rec_freelist, eq); } static inline void @@ -188,24 +196,27 @@ lnet_eq_free(lnet_eq_t *eq) static inline lnet_libmd_t * lnet_md_alloc (lnet_md_t *umd) { - /* NEVER called with liblock held */ - lnet_libmd_t *md; + /* NEVER called with resource lock held */ + struct lnet_res_container *rec = &the_lnet.ln_md_container; + lnet_libmd_t *md; - LNET_LOCK(); - md = (lnet_libmd_t *)lnet_freelist_alloc(&the_lnet.ln_free_mds); - LNET_UNLOCK(); + LNET_LOCK(); + md = (lnet_libmd_t *)lnet_freelist_alloc(&rec->rec_freelist); + LNET_UNLOCK(); - if (md != NULL) - CFS_INIT_LIST_HEAD(&md->md_list); + if (md != NULL) + CFS_INIT_LIST_HEAD(&md->md_list); - return (md); + return md; } static inline void lnet_md_free_locked(lnet_libmd_t *md) { /* ALWAYS called with resource lock held */ - lnet_freelist_free(&the_lnet.ln_free_mds, md); + struct lnet_res_container *rec = &the_lnet.ln_md_container; + + lnet_freelist_free(&rec->rec_freelist, md); } static inline void @@ -217,23 +228,26 @@ lnet_md_free(lnet_libmd_t *md) } static inline lnet_me_t * -lnet_me_alloc (void) +lnet_me_alloc(void) { - /* NEVER called with liblock held */ - lnet_me_t *me; + /* NEVER called with resource lock held */ + struct lnet_res_container *rec = &the_lnet.ln_me_container; + lnet_me_t *me; - LNET_LOCK(); - me = (lnet_me_t *)lnet_freelist_alloc(&the_lnet.ln_free_mes); - LNET_UNLOCK(); + LNET_LOCK(); + me = (lnet_me_t *)lnet_freelist_alloc(&rec->rec_freelist); + LNET_UNLOCK(); - return (me); + return me; } static inline void lnet_me_free_locked(lnet_me_t *me) { /* ALWAYS called with resource lock held */ - lnet_freelist_free(&the_lnet.ln_free_mes, me); + struct lnet_res_container *rec = &the_lnet.ln_me_container; + + lnet_freelist_free(&rec->rec_freelist, me); } static inline void @@ -392,9 +406,16 @@ lnet_msg_free(lnet_msg_t *msg) #endif /* LNET_USE_LIB_FREELIST */ -extern lnet_libhandle_t *lnet_lookup_cookie (__u64 cookie, int type); -extern void lnet_initialise_handle (lnet_libhandle_t *lh, int type); -extern void lnet_invalidate_handle (lnet_libhandle_t *lh); +lnet_libhandle_t *lnet_res_lh_lookup(struct lnet_res_container *rec, + __u64 cookie); +void lnet_res_lh_initialize(struct lnet_res_container *rec, + lnet_libhandle_t *lh); +static inline void +lnet_res_lh_invalidate(lnet_libhandle_t *lh) +{ + /* ALWAYS called with resource lock held */ + cfs_list_del(&lh->lh_hash_chain); +} static inline void lnet_eq2handle (lnet_handle_eq_t *handle, lnet_eq_t *eq) @@ -408,15 +429,16 @@ lnet_eq2handle (lnet_handle_eq_t *handle, lnet_eq_t *eq) } static inline lnet_eq_t * -lnet_handle2eq (lnet_handle_eq_t *handle) +lnet_handle2eq(lnet_handle_eq_t *handle) { - /* ALWAYS called with liblock held */ - lnet_libhandle_t *lh = lnet_lookup_cookie(handle->cookie, - LNET_COOKIE_TYPE_EQ); - if (lh == NULL) - return (NULL); + /* ALWAYS called with resource lock held */ + lnet_libhandle_t *lh; - return (lh_entry (lh, lnet_eq_t, eq_lh)); + lh = lnet_res_lh_lookup(&the_lnet.ln_eq_container, handle->cookie); + if (lh == NULL) + return NULL; + + return lh_entry(lh, lnet_eq_t, eq_lh); } static inline void @@ -426,32 +448,33 @@ lnet_md2handle (lnet_handle_md_t *handle, lnet_libmd_t *md) } static inline lnet_libmd_t * -lnet_handle2md (lnet_handle_md_t *handle) +lnet_handle2md(lnet_handle_md_t *handle) { - /* ALWAYS called with liblock held */ - lnet_libhandle_t *lh = lnet_lookup_cookie(handle->cookie, - LNET_COOKIE_TYPE_MD); - if (lh == NULL) - return (NULL); + /* ALWAYS called with resource lock held */ + lnet_libhandle_t *lh; + + lh = lnet_res_lh_lookup(&the_lnet.ln_md_container, handle->cookie); + if (lh == NULL) + return NULL; - return (lh_entry (lh, lnet_libmd_t, md_lh)); + return lh_entry(lh, lnet_libmd_t, md_lh); } static inline lnet_libmd_t * -lnet_wire_handle2md (lnet_handle_wire_t *wh) +lnet_wire_handle2md(lnet_handle_wire_t *wh) { - /* ALWAYS called with liblock held */ - lnet_libhandle_t *lh; + /* ALWAYS called with resource lock held */ + lnet_libhandle_t *lh; - if (wh->wh_interface_cookie != the_lnet.ln_interface_cookie) - return (NULL); + if (wh->wh_interface_cookie != the_lnet.ln_interface_cookie) + return NULL; - lh = lnet_lookup_cookie(wh->wh_object_cookie, - LNET_COOKIE_TYPE_MD); - if (lh == NULL) - return (NULL); + lh = lnet_res_lh_lookup(&the_lnet.ln_md_container, + wh->wh_object_cookie); + if (lh == NULL) + return NULL; - return (lh_entry (lh, lnet_libmd_t, md_lh)); + return lh_entry(lh, lnet_libmd_t, md_lh); } static inline void @@ -461,15 +484,16 @@ lnet_me2handle (lnet_handle_me_t *handle, lnet_me_t *me) } static inline lnet_me_t * -lnet_handle2me (lnet_handle_me_t *handle) +lnet_handle2me(lnet_handle_me_t *handle) { - /* ALWAYS called with liblock held */ - lnet_libhandle_t *lh = lnet_lookup_cookie(handle->cookie, - LNET_COOKIE_TYPE_ME); - if (lh == NULL) - return (NULL); + /* ALWAYS called with resource lock held */ + lnet_libhandle_t *lh; + + lh = lnet_res_lh_lookup(&the_lnet.ln_me_container, handle->cookie); + if (lh == NULL) + return NULL; - return (lh_entry (lh, lnet_me_t, me_lh)); + return lh_entry(lh, lnet_me_t, me_lh); } static inline int diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index af33c9e..0dd289e 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -521,6 +521,21 @@ typedef struct { unsigned int ptl_options; } lnet_portal_t; +#define LNET_LH_HASH_BITS 12 +#define LNET_LH_HASH_SIZE (1ULL << LNET_LH_HASH_BITS) +#define LNET_LH_HASH_MASK (LNET_LH_HASH_SIZE - 1) + +/* resource container (ME, MD, EQ) */ +struct lnet_res_container { + unsigned int rec_type; /* container type */ + __u64 rec_lh_cookie; /* cookie generator */ + cfs_list_t rec_active; /* active resource list */ + cfs_list_t *rec_lh_hash; /* handle hash */ +#ifdef LNET_USE_LIB_FREELIST + lnet_freelist_t rec_freelist; /* freelist for resources */ +#endif +}; + /* Router Checker states */ #define LNET_RC_STATE_SHUTDOWN 0 /* not started */ #define LNET_RC_STATE_RUNNING 1 /* started up OK */ @@ -554,6 +569,12 @@ typedef struct pthread_mutex_t ln_lnd_mutex; # endif #endif + /* ME container */ + struct lnet_res_container ln_me_container; + /* MD container */ + struct lnet_res_container ln_md_container; + /* Event Queue container */ + struct lnet_res_container ln_eq_container; /* Stuff initialised at LNetNIInit() */ @@ -582,9 +603,6 @@ typedef struct int ln_routing; /* am I a router? */ lnet_rtrbufpool_t ln_rtrpools[LNET_NRBPOOLS]; /* router buffer pools */ - int ln_lh_hash_size; /* size of lib handle hash table */ - cfs_list_t *ln_lh_hash_table; /* all extant lib handles, this interface */ - __u64 ln_next_object_cookie; /* cookie generator */ __u64 ln_interface_cookie; /* uniquely identifies this ni in this epoch */ char *ln_network_tokens; /* space for network names */ @@ -612,16 +630,10 @@ typedef struct lnet_handle_eq_t ln_rc_eqh; /* router checker's event queue */ lnet_handle_md_t ln_rc_mdh; cfs_list_t ln_zombie_rcd; - #ifdef LNET_USE_LIB_FREELIST - lnet_freelist_t ln_free_mes; lnet_freelist_t ln_free_msgs; - lnet_freelist_t ln_free_mds; - lnet_freelist_t ln_free_eqs; #endif cfs_list_t ln_active_msgs; - cfs_list_t ln_active_mds; - cfs_list_t ln_active_eqs; lnet_counters_t ln_counters; diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 07b9f68..ee09400 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -417,41 +417,21 @@ lnet_descriptor_setup (void) /* ****** */ int rc; - memset (&the_lnet.ln_free_mes, 0, sizeof (the_lnet.ln_free_mes)); memset (&the_lnet.ln_free_msgs, 0, sizeof (the_lnet.ln_free_msgs)); - memset (&the_lnet.ln_free_mds, 0, sizeof (the_lnet.ln_free_mds)); - memset (&the_lnet.ln_free_eqs, 0, sizeof (the_lnet.ln_free_eqs)); - - rc = lnet_freelist_init(&the_lnet.ln_free_mes, - MAX_MES, sizeof (lnet_me_t)); - if (rc != 0) - return (rc); - rc = lnet_freelist_init(&the_lnet.ln_free_msgs, - MAX_MSGS, sizeof (lnet_msg_t)); - if (rc != 0) - return (rc); - - rc = lnet_freelist_init(&the_lnet.ln_free_mds, - MAX_MDS, sizeof (lnet_libmd_t)); + LNET_FL_MAX_MSGS, sizeof(lnet_msg_t)); if (rc != 0) return (rc); - - rc = lnet_freelist_init(&the_lnet.ln_free_eqs, - MAX_EQS, sizeof (lnet_eq_t)); return (rc); } void lnet_descriptor_cleanup (void) { - lnet_freelist_fini (&the_lnet.ln_free_mes); lnet_freelist_fini (&the_lnet.ln_free_msgs); - lnet_freelist_fini (&the_lnet.ln_free_mds); - lnet_freelist_fini (&the_lnet.ln_free_eqs); } -#endif +#endif /* LNET_USE_LIB_FREELIST */ __u64 lnet_create_interface_cookie (void) @@ -474,85 +454,140 @@ lnet_create_interface_cookie (void) return cookie; } -int -lnet_setup_handle_hash (void) +static char * +lnet_res_type2str(int type) { - int i; + switch (type) { + default: + LBUG(); + case LNET_COOKIE_TYPE_MD: + return "MD"; + case LNET_COOKIE_TYPE_ME: + return "ME"; + case LNET_COOKIE_TYPE_EQ: + return "EQ"; + } +} - /* Arbitrary choice of hash table size */ -#ifdef __KERNEL__ - the_lnet.ln_lh_hash_size = - (2 * CFS_PAGE_SIZE) / sizeof (cfs_list_t); -#else - the_lnet.ln_lh_hash_size = (MAX_MES + MAX_MDS + MAX_EQS)/4; -#endif - LIBCFS_ALLOC(the_lnet.ln_lh_hash_table, - the_lnet.ln_lh_hash_size * sizeof (cfs_list_t)); - if (the_lnet.ln_lh_hash_table == NULL) - return (-ENOMEM); +void +lnet_res_container_cleanup(struct lnet_res_container *rec) +{ + int count = 0; - for (i = 0; i < the_lnet.ln_lh_hash_size; i++) - CFS_INIT_LIST_HEAD (&the_lnet.ln_lh_hash_table[i]); + if (rec->rec_type == 0) /* not set yet, it's a uninitialized */ + return; - the_lnet.ln_next_object_cookie = LNET_COOKIE_TYPES; + while (!cfs_list_empty(&rec->rec_active)) { + cfs_list_t *e = rec->rec_active.next; - return (0); + cfs_list_del_init(e); + if (rec->rec_type == LNET_COOKIE_TYPE_EQ) { + lnet_eq_free(cfs_list_entry(e, lnet_eq_t, eq_list)); + + } else if (rec->rec_type == LNET_COOKIE_TYPE_MD) { + lnet_md_free(cfs_list_entry(e, lnet_libmd_t, md_list)); + + } else { /* NB: Active MEs should be attached on portals */ + LBUG(); + } + count++; + } + + if (count > 0) { + /* Found alive MD/ME/EQ, user really should unlink/free + * all of them before finalize LNet, but if someone didn't, + * we have to recycle garbage for him */ + CERROR("%d active elements on exit of %s container\n", + count, lnet_res_type2str(rec->rec_type)); + } + +#ifdef LNET_USE_LIB_FREELIST + lnet_freelist_fini(&rec->rec_freelist); +#endif + if (rec->rec_lh_hash != NULL) { + LIBCFS_FREE(rec->rec_lh_hash, + LNET_LH_HASH_SIZE * sizeof(rec->rec_lh_hash[0])); + rec->rec_lh_hash = NULL; + } + + rec->rec_type = 0; /* mark it as finalized */ } -void -lnet_cleanup_handle_hash (void) +int +lnet_res_container_setup(struct lnet_res_container *rec, + int type, int objnum, int objsz) { - if (the_lnet.ln_lh_hash_table == NULL) - return; + int rc = 0; + int i; + + LASSERT(rec->rec_type == 0); - LIBCFS_FREE(the_lnet.ln_lh_hash_table, - the_lnet.ln_lh_hash_size * sizeof (cfs_list_t)); + rec->rec_type = type; + CFS_INIT_LIST_HEAD(&rec->rec_active); + +#ifdef LNET_USE_LIB_FREELIST + memset(&rec->rec_freelist, 0, sizeof(rec->rec_freelist)); + rc = lnet_freelist_init(&rec->rec_freelist, objnum, objsz); + if (rc != 0) + goto out; +#endif + rec->rec_lh_cookie = type; + + /* Arbitrary choice of hash table size */ + LIBCFS_ALLOC(rec->rec_lh_hash, + LNET_LH_HASH_SIZE * sizeof(rec->rec_lh_hash[0])); + if (rec->rec_lh_hash == NULL) { + rc = -ENOMEM; + goto out; + } + + for (i = 0; i < LNET_LH_HASH_SIZE; i++) + CFS_INIT_LIST_HEAD(&rec->rec_lh_hash[i]); + + return 0; + +out: + CERROR("Failed to setup %s resource container\n", + lnet_res_type2str(type)); + lnet_res_container_cleanup(rec); + return rc; } lnet_libhandle_t * -lnet_lookup_cookie (__u64 cookie, int type) +lnet_res_lh_lookup(struct lnet_res_container *rec, __u64 cookie) { - /* ALWAYS called with LNET_LOCK held */ - cfs_list_t *list; - cfs_list_t *el; - unsigned int hash; + /* ALWAYS called with lnet_res_lock held */ + cfs_list_t *head; + lnet_libhandle_t *lh; + unsigned int hash; - if ((cookie & (LNET_COOKIE_TYPES - 1)) != type) - return (NULL); + if ((cookie & (LNET_COOKIE_TYPES - 1)) != rec->rec_type) + return NULL; - hash = ((unsigned int)(cookie >> LNET_COOKIE_TYPE_BITS)) % the_lnet.ln_lh_hash_size; - list = &the_lnet.ln_lh_hash_table[hash]; + hash = cookie >> LNET_COOKIE_TYPE_BITS; + head = &rec->rec_lh_hash[hash & LNET_LH_HASH_MASK]; - cfs_list_for_each (el, list) { - lnet_libhandle_t *lh = cfs_list_entry (el, lnet_libhandle_t, - lh_hash_chain); + cfs_list_for_each_entry(lh, head, lh_hash_chain) { + if (lh->lh_cookie == cookie) + return lh; + } - if (lh->lh_cookie == cookie) - return (lh); - } - - return (NULL); + return NULL; } void -lnet_initialise_handle (lnet_libhandle_t *lh, int type) +lnet_res_lh_initialize(struct lnet_res_container *rec, lnet_libhandle_t *lh) { - /* ALWAYS called with LNET_LOCK held */ - unsigned int hash; + /* ALWAYS called with lnet_res_lock held */ + unsigned int ibits = LNET_COOKIE_TYPE_BITS; + unsigned int hash; - LASSERT (type >= 0 && type < LNET_COOKIE_TYPES); - lh->lh_cookie = the_lnet.ln_next_object_cookie | type; - the_lnet.ln_next_object_cookie += LNET_COOKIE_TYPES; + lh->lh_cookie = rec->rec_lh_cookie; + rec->rec_lh_cookie += 1 << ibits; - hash = ((unsigned int)(lh->lh_cookie >> LNET_COOKIE_TYPE_BITS)) % the_lnet.ln_lh_hash_size; - cfs_list_add (&lh->lh_hash_chain, &the_lnet.ln_lh_hash_table[hash]); -} + hash = (lh->lh_cookie >> ibits) & LNET_LH_HASH_MASK; -void -lnet_invalidate_handle (lnet_libhandle_t *lh) -{ - /* ALWAYS called with LNET_LOCK held */ - cfs_list_del (&lh->lh_hash_chain); + cfs_list_add(&lh->lh_hash_chain, &rec->rec_lh_hash[hash]); } cfs_list_t * @@ -676,14 +711,12 @@ lnet_prepare(lnet_pid_t requested_pid) rc = lnet_descriptor_setup(); if (rc != 0) - goto failed0; + return -ENOMEM; memset(&the_lnet.ln_counters, 0, sizeof(the_lnet.ln_counters)); CFS_INIT_LIST_HEAD (&the_lnet.ln_active_msgs); - CFS_INIT_LIST_HEAD (&the_lnet.ln_active_mds); - CFS_INIT_LIST_HEAD (&the_lnet.ln_active_eqs); CFS_INIT_LIST_HEAD (&the_lnet.ln_test_peers); CFS_INIT_LIST_HEAD (&the_lnet.ln_nis); CFS_INIT_LIST_HEAD (&the_lnet.ln_zombie_nis); @@ -694,17 +727,39 @@ lnet_prepare(lnet_pid_t requested_pid) lnet_init_rtrpools(); - rc = lnet_setup_handle_hash (); - if (rc != 0) - goto failed0; - rc = lnet_create_peer_table(); if (rc != 0) - goto failed1; - - rc = lnet_init_finalizers(); - if (rc != 0) - goto failed2; + goto failed0; + + rc = lnet_init_finalizers(); + if (rc != 0) + goto failed1; + + rc = lnet_res_container_setup(&the_lnet.ln_eq_container, + LNET_COOKIE_TYPE_EQ, LNET_FL_MAX_EQS, + sizeof(lnet_eq_t)); + if (rc != 0) { + CERROR("Failed to create EQ container for LNet: %d\n", rc); + goto failed2; + } + + /* NB: we will have instance of ME container per CPT soon */ + rc = lnet_res_container_setup(&the_lnet.ln_me_container, + LNET_COOKIE_TYPE_ME, LNET_FL_MAX_MES, + sizeof(lnet_me_t)); + if (rc != 0) { + CERROR("Failed to create ME container for LNet: %d\n", rc); + goto failed3; + } + + /* NB: we will have instance of MD container per CPT soon */ + rc = lnet_res_container_setup(&the_lnet.ln_md_container, + LNET_COOKIE_TYPE_MD, LNET_FL_MAX_MDS, + sizeof(lnet_libmd_t)); + if (rc != 0) { + CERROR("Failed to create MD container for LNet: %d\n", rc); + goto failed3; + } the_lnet.ln_nportals = MAX_PORTALS; LIBCFS_ALLOC(the_lnet.ln_portals, @@ -724,14 +779,18 @@ lnet_prepare(lnet_pid_t requested_pid) return 0; failed3: - lnet_fini_finalizers(); + /* NB: lnet_res_container_cleanup is safe to call for + * uninitialized container */ + lnet_res_container_cleanup(&the_lnet.ln_md_container); + lnet_res_container_cleanup(&the_lnet.ln_me_container); + lnet_res_container_cleanup(&the_lnet.ln_eq_container); failed2: - lnet_destroy_peer_table(); + lnet_fini_finalizers(); failed1: - lnet_cleanup_handle_hash(); + lnet_destroy_peer_table(); failed0: - lnet_descriptor_cleanup(); - return rc; + lnet_descriptor_cleanup(); + return rc; } int @@ -771,23 +830,9 @@ lnet_unprepare (void) } } - while (!cfs_list_empty (&the_lnet.ln_active_mds)) { - lnet_libmd_t *md = cfs_list_entry (the_lnet.ln_active_mds.next, - lnet_libmd_t, md_list); - - CERROR ("Active MD %p on exit\n", md); - cfs_list_del_init (&md->md_list); - lnet_md_free (md); - } - - while (!cfs_list_empty (&the_lnet.ln_active_eqs)) { - lnet_eq_t *eq = cfs_list_entry (the_lnet.ln_active_eqs.next, - lnet_eq_t, eq_list); - - CERROR ("Active EQ %p on exit\n", eq); - cfs_list_del (&eq->eq_list); - lnet_eq_free (eq); - } + lnet_res_container_cleanup(&the_lnet.ln_md_container); + lnet_res_container_cleanup(&the_lnet.ln_me_container); + lnet_res_container_cleanup(&the_lnet.ln_eq_container); while (!cfs_list_empty (&the_lnet.ln_active_msgs)) { lnet_msg_t *msg = cfs_list_entry (the_lnet.ln_active_msgs.next, @@ -806,7 +851,6 @@ lnet_unprepare (void) lnet_free_rtrpools(); lnet_fini_finalizers(); lnet_destroy_peer_table(); - lnet_cleanup_handle_hash(); lnet_descriptor_cleanup(); return (0); diff --git a/lnet/lnet/lib-eq.c b/lnet/lnet/lib-eq.c index 23f94be..4f2e3f8 100644 --- a/lnet/lnet/lib-eq.c +++ b/lnet/lnet/lib-eq.c @@ -111,8 +111,8 @@ LNetEQAlloc(unsigned int count, lnet_eq_handler_t callback, LNET_LOCK(); - lnet_initialise_handle (&eq->eq_lh, LNET_COOKIE_TYPE_EQ); - cfs_list_add (&eq->eq_list, &the_lnet.ln_active_eqs); + lnet_res_lh_initialize(&the_lnet.ln_eq_container, &eq->eq_lh); + cfs_list_add(&eq->eq_list, &the_lnet.ln_eq_container.rec_active); LNET_UNLOCK(); @@ -159,7 +159,7 @@ LNetEQFree(lnet_handle_eq_t eqh) events = eq->eq_events; size = eq->eq_size; - lnet_invalidate_handle (&eq->eq_lh); + lnet_res_lh_invalidate(&eq->eq_lh); cfs_list_del (&eq->eq_list); lnet_eq_free_locked(eq); diff --git a/lnet/lnet/lib-md.c b/lnet/lnet/lib-md.c index 464f9aa..be5b9e1 100644 --- a/lnet/lnet/lib-md.c +++ b/lnet/lnet/lib-md.c @@ -60,7 +60,7 @@ lnet_md_unlink(lnet_libmd_t *md) } /* ensure all future handle lookups fail */ - lnet_invalidate_handle(&md->md_lh); + lnet_res_lh_invalidate(&md->md_lh); } if (md->md_refcount != 0) { @@ -184,11 +184,11 @@ lib_md_build(lnet_libmd_t *lmd, lnet_md_t *umd, int unlink) eq->eq_refcount++; /* It's good; let handle2md succeed and add to active mds */ - lnet_initialise_handle (&lmd->md_lh, LNET_COOKIE_TYPE_MD); - LASSERT (cfs_list_empty(&lmd->md_list)); - cfs_list_add (&lmd->md_list, &the_lnet.ln_active_mds); + lnet_res_lh_initialize(&the_lnet.ln_md_container, &lmd->md_lh); + LASSERT(cfs_list_empty(&lmd->md_list)); + cfs_list_add(&lmd->md_list, &the_lnet.ln_md_container.rec_active); - return 0; + return 0; } /* must be called with LNET_LOCK held */ diff --git a/lnet/lnet/lib-me.c b/lnet/lnet/lib-me.c index c10828d..2ea5309 100644 --- a/lnet/lnet/lib-me.c +++ b/lnet/lnet/lib-me.c @@ -160,7 +160,7 @@ LNetMEAttach(unsigned int portal, me->me_unlink = unlink; me->me_md = NULL; - lnet_initialise_handle (&me->me_lh, LNET_COOKIE_TYPE_ME); + lnet_res_lh_initialize(&the_lnet.ln_me_container, &me->me_lh); head = lnet_portal_me_head(portal, match_id, match_bits); LASSERT (head != NULL); @@ -238,7 +238,7 @@ LNetMEInsert(lnet_handle_me_t current_meh, new_me->me_unlink = unlink; new_me->me_md = NULL; - lnet_initialise_handle (&new_me->me_lh, LNET_COOKIE_TYPE_ME); + lnet_res_lh_initialize(&the_lnet.ln_me_container, &new_me->me_lh); if (pos == LNET_INS_AFTER) cfs_list_add(&new_me->me_list, ¤t_me->me_list); @@ -309,7 +309,7 @@ lnet_me_unlink(lnet_me_t *me) lnet_md_unlink(me->me_md); } - lnet_invalidate_handle (&me->me_lh); + lnet_res_lh_invalidate(&me->me_lh); lnet_me_free_locked(me); }