From e8ad5c163bfe4085a3132872374bc50b38cb9564 Mon Sep 17 00:00:00 2001 From: Liang Zhen Date: Tue, 17 Aug 2010 11:06:13 +0400 Subject: [PATCH] b21619 hash ME on RDMA portal i=isaac i=maxim --- lnet/include/lnet/lib-lnet.h | 63 ++++++++++++++++++++++++++++++++ lnet/include/lnet/lib-types.h | 18 +++++++--- lnet/lnet/api-ni.c | 55 +++++++++++++++++++++++----- lnet/lnet/lib-me.c | 83 ++++++++++++++++++++++++++++++++++++++++--- lnet/lnet/lib-move.c | 27 ++++++++------ 5 files changed, 217 insertions(+), 29 deletions(-) diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index f36791a..1e6b72f 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -88,6 +88,13 @@ static inline int lnet_md_unlinkable (lnet_libmd_t *md) lnet_md_exhausted(md)); } +static inline unsigned int +lnet_match_to_hash(lnet_process_id_t id, __u64 mbits) +{ + mbits += id.nid + id.pid; + return cfs_hash_long((unsigned long)mbits, LNET_PORTAL_HASH_BITS); +} + #ifdef __KERNEL__ #define LNET_LOCK() cfs_spin_lock(&the_lnet.ln_lock) #define LNET_UNLOCK() cfs_spin_unlock(&the_lnet.ln_lock) @@ -429,6 +436,62 @@ lnet_handle2me (lnet_handle_me_t *handle) return (lh_entry (lh, lnet_me_t, me_lh)); } +static inline int +lnet_portal_is_lazy(lnet_portal_t *ptl) +{ + return !!(ptl->ptl_options & LNET_PTL_LAZY); +} + +static inline int +lnet_portal_is_unique(lnet_portal_t *ptl) +{ + return !!(ptl->ptl_options & LNET_PTL_MATCH_UNIQUE); +} + +static inline int +lnet_portal_is_wildcard(lnet_portal_t *ptl) +{ + return !!(ptl->ptl_options & LNET_PTL_MATCH_WILDCARD); +} + +static inline void +lnet_portal_setopt(lnet_portal_t *ptl, int opt) +{ + ptl->ptl_options |= opt; +} + +static inline void +lnet_portal_unsetopt(lnet_portal_t *ptl, int opt) +{ + ptl->ptl_options &= ~opt; +} + +static inline int +lnet_match_is_unique(lnet_process_id_t match_id, + __u64 match_bits, __u64 ignore_bits) +{ + return ignore_bits == 0 && + match_id.nid != LNET_NID_ANY && + match_id.pid != LNET_PID_ANY; +} + +static inline cfs_list_t * +lnet_portal_me_head(int index, lnet_process_id_t id, __u64 mbits) +{ + lnet_portal_t *ptl = &the_lnet.ln_portals[index]; + + if (lnet_portal_is_wildcard(ptl)) { + return &ptl->ptl_mlist; + } else if (lnet_portal_is_unique(ptl)) { + LASSERT (ptl->ptl_mhash != NULL); + return &ptl->ptl_mhash[lnet_match_to_hash(id, mbits)]; + } + return NULL; +} + +cfs_list_t *lnet_portal_mhash_alloc(void); +void lnet_portal_mhash_free(cfs_list_t *mhash); + static inline void lnet_peer_addref_locked(lnet_peer_t *lp) { diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index b6a2bce..8f69c18 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -508,12 +508,20 @@ typedef struct { /* Options for lnet_portal_t::ptl_options */ #define LNET_PTL_LAZY (1 << 0) +#define LNET_PTL_MATCH_UNIQUE (1 << 1) /* unique match, for RDMA */ +#define LNET_PTL_MATCH_WILDCARD (1 << 2) /* wildcard match, request portal */ + +/* ME hash of RDMA portal */ +#define LNET_PORTAL_HASH_BITS 8 +#define LNET_PORTAL_HASH_SIZE (1 << LNET_PORTAL_HASH_BITS) + typedef struct { - cfs_list_t ptl_ml; /* match list */ - cfs_list_t ptl_msgq; /* messages blocking for MD */ - __u64 ptl_ml_version; /* validity stamp, only changed for new attached MD */ - __u64 ptl_msgq_version; /* validity stamp */ - unsigned int ptl_options; + cfs_list_t *ptl_mhash; /* match hash */ + cfs_list_t ptl_mlist; /* match list */ + cfs_list_t ptl_msgq; /* messages blocking for MD */ + __u64 ptl_ml_version; /* validity stamp, only changed for new attached MD */ + __u64 ptl_msgq_version; /* validity stamp */ + unsigned int ptl_options; } lnet_portal_t; /* Router Checker states */ diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 2f681d7..0a3fabb 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -555,6 +555,39 @@ lnet_invalidate_handle (lnet_libhandle_t *lh) cfs_list_del (&lh->lh_hash_chain); } +cfs_list_t * +lnet_portal_mhash_alloc(void) +{ + cfs_list_t *mhash; + int i; + + LIBCFS_ALLOC(mhash, sizeof(cfs_list_t) * LNET_PORTAL_HASH_SIZE); + if (mhash == NULL) + return NULL; + + for (i = 0; i < LNET_PORTAL_HASH_SIZE; i++) + CFS_INIT_LIST_HEAD(&mhash[i]); + + return mhash; +} + +void +lnet_portal_mhash_free(cfs_list_t *mhash) +{ + int i; + + for (i = 0; i < LNET_PORTAL_HASH_SIZE; i++) { + while (!cfs_list_empty(&mhash[i])) { + lnet_me_t *me = cfs_list_entry(mhash[i].next, + lnet_me_t, me_list); + CERROR ("Active ME %p on exit portal mhash\n", me); + cfs_list_del(&me->me_list); + lnet_me_free(me); + } + } + LIBCFS_FREE(mhash, sizeof(cfs_list_t) * LNET_PORTAL_HASH_SIZE); +} + int lnet_init_finalizers(void) { @@ -681,7 +714,7 @@ lnet_prepare(lnet_pid_t requested_pid) } for (i = 0; i < the_lnet.ln_nportals; i++) { - CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_ml)); + CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_mlist)); CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_msgq)); the_lnet.ln_portals[i].ptl_options = 0; } @@ -718,23 +751,29 @@ lnet_unprepare (void) LASSERT (the_lnet.ln_nzombie_nis == 0); for (idx = 0; idx < the_lnet.ln_nportals; idx++) { - LASSERT (cfs_list_empty(&the_lnet.ln_portals[idx].ptl_msgq)); + lnet_portal_t *ptl = &the_lnet.ln_portals[idx]; - while (!cfs_list_empty (&the_lnet.ln_portals[idx].ptl_ml)) { - lnet_me_t *me = cfs_list_entry (the_lnet.ln_portals[idx].ptl_ml.next, - lnet_me_t, me_list); + LASSERT (cfs_list_empty(&ptl->ptl_msgq)); - CERROR ("Active me %p on exit\n", me); + while (!cfs_list_empty(&ptl->ptl_mlist)) { + lnet_me_t *me = cfs_list_entry(ptl->ptl_mlist.next, + lnet_me_t, me_list); + CERROR ("Active ME %p on exit\n", me); cfs_list_del (&me->me_list); lnet_me_free (me); } + + if (ptl->ptl_mhash != NULL) { + LASSERT (lnet_portal_is_unique(ptl)); + lnet_portal_mhash_free(ptl->ptl_mhash); + } } while (!cfs_list_empty (&the_lnet.ln_active_mds)) { lnet_libmd_t *md = cfs_list_entry (the_lnet.ln_active_mds.next, lnet_libmd_t, md_list); - CERROR ("Active md %p on exit\n", md); + CERROR ("Active MD %p on exit\n", md); cfs_list_del_init (&md->md_list); lnet_md_free (md); } @@ -743,7 +782,7 @@ lnet_unprepare (void) lnet_eq_t *eq = cfs_list_entry (the_lnet.ln_active_eqs.next, lnet_eq_t, eq_list); - CERROR ("Active eq %p on exit\n", eq); + CERROR ("Active EQ %p on exit\n", eq); cfs_list_del (&eq->eq_list); lnet_eq_free (eq); } diff --git a/lnet/lnet/lib-me.c b/lnet/lnet/lib-me.c index 65b28dd..381e512 100644 --- a/lnet/lnet/lib-me.c +++ b/lnet/lnet/lib-me.c @@ -42,6 +42,60 @@ #include +static int +lnet_me_match_portal(lnet_portal_t *ptl, lnet_process_id_t id, + __u64 match_bits, __u64 ignore_bits) +{ + cfs_list_t *mhash = NULL; + int unique; + + LASSERT (!(lnet_portal_is_unique(ptl) && + lnet_portal_is_wildcard(ptl))); + + /* prefer to check w/o any lock */ + unique = lnet_match_is_unique(id, match_bits, ignore_bits); + if (likely(lnet_portal_is_unique(ptl) || + lnet_portal_is_wildcard(ptl))) + goto match; + + /* unset, new portal */ + if (unique) { + mhash = lnet_portal_mhash_alloc(); + if (mhash == NULL) + return -ENOMEM; + } + + LNET_LOCK(); + if (lnet_portal_is_unique(ptl) || + lnet_portal_is_wildcard(ptl)) { + /* someone set it before me */ + if (mhash != NULL) + lnet_portal_mhash_free(mhash); + LNET_UNLOCK(); + goto match; + } + + /* still not set */ + LASSERT (ptl->ptl_mhash == NULL); + if (unique) { + ptl->ptl_mhash = mhash; + lnet_portal_setopt(ptl, LNET_PTL_MATCH_UNIQUE); + } else { + lnet_portal_setopt(ptl, LNET_PTL_MATCH_WILDCARD); + } + LNET_UNLOCK(); + return 0; + + match: + if (lnet_portal_is_unique(ptl) && !unique) + return -EPERM; + + if (lnet_portal_is_wildcard(ptl) && unique) + return -EPERM; + + return 0; +} + int LNetMEAttach(unsigned int portal, lnet_process_id_t match_id, @@ -49,7 +103,10 @@ LNetMEAttach(unsigned int portal, lnet_unlink_t unlink, lnet_ins_pos_t pos, lnet_handle_me_t *handle) { - lnet_me_t *me; + lnet_me_t *me; + lnet_portal_t *ptl; + cfs_list_t *head; + int rc; LASSERT (the_lnet.ln_init); LASSERT (the_lnet.ln_refcount > 0); @@ -57,6 +114,11 @@ LNetMEAttach(unsigned int portal, if ((int)portal >= the_lnet.ln_nportals) return -EINVAL; + ptl = &the_lnet.ln_portals[portal]; + rc = lnet_me_match_portal(ptl, match_id, match_bits, ignore_bits); + if (rc != 0) + return rc; + me = lnet_me_alloc(); if (me == NULL) return -ENOMEM; @@ -71,13 +133,13 @@ LNetMEAttach(unsigned int portal, me->me_md = NULL; lnet_initialise_handle (&me->me_lh, LNET_COOKIE_TYPE_ME); + head = lnet_portal_me_head(portal, match_id, match_bits); + LASSERT (head != NULL); if (pos == LNET_INS_AFTER) - cfs_list_add_tail(&me->me_list, - &(the_lnet.ln_portals[portal].ptl_ml)); + cfs_list_add_tail(&me->me_list, head); else - cfs_list_add(&me->me_list, - &(the_lnet.ln_portals[portal].ptl_ml)); + cfs_list_add(&me->me_list, head); lnet_me2handle(handle, me); @@ -95,6 +157,7 @@ LNetMEInsert(lnet_handle_me_t current_meh, { lnet_me_t *current_me; lnet_me_t *new_me; + lnet_portal_t *ptl; LASSERT (the_lnet.ln_init); LASSERT (the_lnet.ln_refcount > 0); @@ -113,6 +176,16 @@ LNetMEInsert(lnet_handle_me_t current_meh, return -ENOENT; } + LASSERT (current_me->me_portal < the_lnet.ln_nportals); + + ptl = &the_lnet.ln_portals[current_me->me_portal]; + if (lnet_portal_is_unique(ptl)) { + /* nosense to insertion on unique portal */ + lnet_me_free (new_me); + LNET_UNLOCK(); + return -EPERM; + } + new_me->me_portal = current_me->me_portal; new_me->me_match_id = match_id; new_me->me_match_bits = match_bits; diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index e46256b5..957f958 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -155,6 +155,7 @@ lnet_match_md(int index, int op_mask, lnet_process_id_t src, lnet_libmd_t **md_out) { lnet_portal_t *ptl = &the_lnet.ln_portals[index]; + cfs_list_t *head; lnet_me_t *me; lnet_me_t *tmp; lnet_libmd_t *md; @@ -169,7 +170,11 @@ lnet_match_md(int index, int op_mask, lnet_process_id_t src, return LNET_MATCHMD_DROP; } - cfs_list_for_each_entry_safe_typed (me, tmp, &ptl->ptl_ml, + head = lnet_portal_me_head(index, src, match_bits); + if (head == NULL) /* nobody posted anything on this portal */ + goto out; + + cfs_list_for_each_entry_safe_typed (me, tmp, head, lnet_me_t, me_list) { md = me->me_md; @@ -199,8 +204,9 @@ lnet_match_md(int index, int op_mask, lnet_process_id_t src, /* not reached */ } + out: if (op_mask == LNET_MD_OP_GET || - (ptl->ptl_options & LNET_PTL_LAZY) == 0) + !lnet_portal_is_lazy(ptl)) return LNET_MATCHMD_DROP; return LNET_MATCHMD_NONE; @@ -1572,9 +1578,7 @@ LNetSetLazyPortal(int portal) CDEBUG(D_NET, "Setting portal %d lazy\n", portal); LNET_LOCK(); - - ptl->ptl_options |= LNET_PTL_LAZY; - + lnet_portal_setopt(ptl, LNET_PTL_LAZY); LNET_UNLOCK(); return 0; @@ -1592,7 +1596,7 @@ LNetClearLazyPortal(int portal) LNET_LOCK(); - if ((ptl->ptl_options & LNET_PTL_LAZY) == 0) { + if (!lnet_portal_is_lazy(ptl)) { LNET_UNLOCK(); return 0; } @@ -1607,7 +1611,7 @@ LNetClearLazyPortal(int portal) cfs_list_del_init(&ptl->ptl_msgq); ptl->ptl_msgq_version++; - ptl->ptl_options &= ~LNET_PTL_LAZY; + lnet_portal_unsetopt(ptl, LNET_PTL_LAZY); LNET_UNLOCK(); @@ -1662,12 +1666,13 @@ lnet_match_blocked_msg(lnet_libmd_t *md) cfs_list_t *tmp; cfs_list_t *entry; lnet_msg_t *msg; + lnet_portal_t *ptl; lnet_me_t *me = md->md_me; - lnet_portal_t *ptl = &the_lnet.ln_portals[me->me_portal]; LASSERT (me->me_portal < (unsigned int)the_lnet.ln_nportals); - if ((ptl->ptl_options & LNET_PTL_LAZY) == 0) { + ptl = &the_lnet.ln_portals[me->me_portal]; + if (!lnet_portal_is_lazy(ptl)) { LASSERT (cfs_list_empty(&ptl->ptl_msgq)); return; } @@ -1773,7 +1778,6 @@ lnet_parse_put(lnet_ni_t *ni, lnet_msg_t *msg) hdr->msg.put.offset = le32_to_cpu(hdr->msg.put.offset); index = hdr->msg.put.ptl_index; - ptl = &the_lnet.ln_portals[index]; LNET_LOCK(); @@ -1792,6 +1796,7 @@ lnet_parse_put(lnet_ni_t *ni, lnet_msg_t *msg) return 0; case LNET_MATCHMD_NONE: + ptl = &the_lnet.ln_portals[index]; version = ptl->ptl_ml_version; rc = 0; @@ -1800,7 +1805,7 @@ lnet_parse_put(lnet_ni_t *ni, lnet_msg_t *msg) if (rc == 0 && !the_lnet.ln_shutdown && - ((ptl->ptl_options & LNET_PTL_LAZY) != 0)) { + lnet_portal_is_lazy(ptl)) { if (version != ptl->ptl_ml_version) goto again; -- 1.8.3.1