b21619 hash ME on RDMA portal i=isaac i=maxim

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index f36791a..1e6b72f 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -88,6 +88,13 @@ static inline int lnet_md_unlinkable (lnet_libmd_t *md)
                 lnet_md_exhausted(md));
 }
 
                 lnet_md_exhausted(md));
 }
 

+static inline unsigned int
+lnet_match_to_hash(lnet_process_id_t id, __u64 mbits)
+{
+        mbits += id.nid + id.pid;
+        return cfs_hash_long((unsigned long)mbits, LNET_PORTAL_HASH_BITS);
+}
+

 #ifdef __KERNEL__
 #define LNET_LOCK()        cfs_spin_lock(&the_lnet.ln_lock)
 #define LNET_UNLOCK()      cfs_spin_unlock(&the_lnet.ln_lock)
 #ifdef __KERNEL__
 #define LNET_LOCK()        cfs_spin_lock(&the_lnet.ln_lock)
 #define LNET_UNLOCK()      cfs_spin_unlock(&the_lnet.ln_lock)


@@ -429,6 +436,62 @@ lnet_handle2me (lnet_handle_me_t *handle)
         return (lh_entry (lh, lnet_me_t, me_lh));
 }
 
         return (lh_entry (lh, lnet_me_t, me_lh));
 }
 

+static inline int
+lnet_portal_is_lazy(lnet_portal_t *ptl)
+{
+        return !!(ptl->ptl_options & LNET_PTL_LAZY);
+}
+
+static inline int
+lnet_portal_is_unique(lnet_portal_t *ptl)
+{
+        return !!(ptl->ptl_options & LNET_PTL_MATCH_UNIQUE); 
+}
+
+static inline int
+lnet_portal_is_wildcard(lnet_portal_t *ptl)
+{
+        return !!(ptl->ptl_options & LNET_PTL_MATCH_WILDCARD);
+}
+
+static inline void
+lnet_portal_setopt(lnet_portal_t *ptl, int opt)
+{
+        ptl->ptl_options |= opt;
+}
+
+static inline void
+lnet_portal_unsetopt(lnet_portal_t *ptl, int opt)
+{
+        ptl->ptl_options &= ~opt;
+}
+
+static inline int
+lnet_match_is_unique(lnet_process_id_t match_id,
+                     __u64 match_bits, __u64 ignore_bits)
+{
+        return ignore_bits == 0 &&
+               match_id.nid != LNET_NID_ANY &&
+               match_id.pid != LNET_PID_ANY;
+}
+
+static inline cfs_list_t *
+lnet_portal_me_head(int index, lnet_process_id_t id, __u64 mbits)
+{
+        lnet_portal_t *ptl = &the_lnet.ln_portals[index];
+
+        if (lnet_portal_is_wildcard(ptl)) {
+                return &ptl->ptl_mlist;
+        } else if (lnet_portal_is_unique(ptl)) {
+                LASSERT (ptl->ptl_mhash != NULL);
+                return &ptl->ptl_mhash[lnet_match_to_hash(id, mbits)];
+        }
+        return NULL;
+}
+
+cfs_list_t *lnet_portal_mhash_alloc(void);
+void lnet_portal_mhash_free(cfs_list_t *mhash);
+

 static inline void
 lnet_peer_addref_locked(lnet_peer_t *lp)
 {
 static inline void
 lnet_peer_addref_locked(lnet_peer_t *lp)
 {

diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h
index b6a2bce..8f69c18 100644 (file)
--- a/lnet/include/lnet/lib-types.h
+++ b/lnet/include/lnet/lib-types.h
@@ -508,12 +508,20 @@ typedef struct {
 
 /* Options for lnet_portal_t::ptl_options */
 #define LNET_PTL_LAZY               (1 << 0)
 
 /* Options for lnet_portal_t::ptl_options */
 #define LNET_PTL_LAZY               (1 << 0)

+#define LNET_PTL_MATCH_UNIQUE       (1 << 1)    /* unique match, for RDMA */
+#define LNET_PTL_MATCH_WILDCARD     (1 << 2)    /* wildcard match, request portal */
+
+/* ME hash of RDMA portal */
+#define LNET_PORTAL_HASH_BITS        8
+#define LNET_PORTAL_HASH_SIZE       (1 << LNET_PORTAL_HASH_BITS)
+

 typedef struct {
 typedef struct {

-        cfs_list_t           ptl_ml;   /* match list */
-        cfs_list_t           ptl_msgq; /* messages blocking for MD */
-        __u64                ptl_ml_version;    /* validity stamp, only changed for new attached MD */
-        __u64                ptl_msgq_version;  /* validity stamp */
-        unsigned int         ptl_options;
+        cfs_list_t       *ptl_mhash;            /* match hash */
+        cfs_list_t        ptl_mlist;            /* match list */
+        cfs_list_t        ptl_msgq;             /* messages blocking for MD */
+        __u64             ptl_ml_version;       /* validity stamp, only changed for new attached MD */
+        __u64             ptl_msgq_version;     /* validity stamp */
+        unsigned int      ptl_options;

 } lnet_portal_t;
 
 /* Router Checker states */
 } lnet_portal_t;
 
 /* Router Checker states */

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 2f681d7..0a3fabb 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -555,6 +555,39 @@ lnet_invalidate_handle (lnet_libhandle_t *lh)
         cfs_list_del (&lh->lh_hash_chain);
 }
 
         cfs_list_del (&lh->lh_hash_chain);
 }
 

+cfs_list_t *
+lnet_portal_mhash_alloc(void)
+{
+        cfs_list_t       *mhash;
+        int               i;
+
+        LIBCFS_ALLOC(mhash, sizeof(cfs_list_t) * LNET_PORTAL_HASH_SIZE);
+        if (mhash == NULL)
+                return NULL;
+
+        for (i = 0; i < LNET_PORTAL_HASH_SIZE; i++)
+                CFS_INIT_LIST_HEAD(&mhash[i]);
+
+        return mhash;
+}
+
+void
+lnet_portal_mhash_free(cfs_list_t *mhash)
+{
+        int     i;
+
+        for (i = 0; i < LNET_PORTAL_HASH_SIZE; i++) {
+                while (!cfs_list_empty(&mhash[i])) {
+                        lnet_me_t *me = cfs_list_entry(mhash[i].next,
+                                                       lnet_me_t, me_list);
+                        CERROR ("Active ME %p on exit portal mhash\n", me);
+                        cfs_list_del(&me->me_list);
+                        lnet_me_free(me);
+                }
+        }
+        LIBCFS_FREE(mhash, sizeof(cfs_list_t) * LNET_PORTAL_HASH_SIZE);
+}
+

 int
 lnet_init_finalizers(void)
 {
 int
 lnet_init_finalizers(void)
 {


@@ -681,7 +714,7 @@ lnet_prepare(lnet_pid_t requested_pid)
         }
 
         for (i = 0; i < the_lnet.ln_nportals; i++) {
         }
 
         for (i = 0; i < the_lnet.ln_nportals; i++) {

-                CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_ml));
+                CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_mlist));

                 CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_msgq));
                 the_lnet.ln_portals[i].ptl_options = 0;
         }
                 CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_msgq));
                 the_lnet.ln_portals[i].ptl_options = 0;
         }


@@ -718,23 +751,29 @@ lnet_unprepare (void)
         LASSERT (the_lnet.ln_nzombie_nis == 0);
 
         for (idx = 0; idx < the_lnet.ln_nportals; idx++) {
         LASSERT (the_lnet.ln_nzombie_nis == 0);
 
         for (idx = 0; idx < the_lnet.ln_nportals; idx++) {

-                LASSERT (cfs_list_empty(&the_lnet.ln_portals[idx].ptl_msgq));
+                lnet_portal_t *ptl = &the_lnet.ln_portals[idx];

 
 

-                while (!cfs_list_empty (&the_lnet.ln_portals[idx].ptl_ml)) {
-                        lnet_me_t *me = cfs_list_entry (the_lnet.ln_portals[idx].ptl_ml.next,
-                                                        lnet_me_t, me_list);
+                LASSERT (cfs_list_empty(&ptl->ptl_msgq));

 
 

-                        CERROR ("Active me %p on exit\n", me);
+                while (!cfs_list_empty(&ptl->ptl_mlist)) {
+                        lnet_me_t *me = cfs_list_entry(ptl->ptl_mlist.next,
+                                                       lnet_me_t, me_list);
+                        CERROR ("Active ME %p on exit\n", me);

                         cfs_list_del (&me->me_list);
                         lnet_me_free (me);
                 }
                         cfs_list_del (&me->me_list);
                         lnet_me_free (me);
                 }

+
+                if (ptl->ptl_mhash != NULL) {
+                        LASSERT (lnet_portal_is_unique(ptl));
+                        lnet_portal_mhash_free(ptl->ptl_mhash);
+                }

         }
 
         while (!cfs_list_empty (&the_lnet.ln_active_mds)) {
                 lnet_libmd_t *md = cfs_list_entry (the_lnet.ln_active_mds.next,
                                                    lnet_libmd_t, md_list);
 
         }
 
         while (!cfs_list_empty (&the_lnet.ln_active_mds)) {
                 lnet_libmd_t *md = cfs_list_entry (the_lnet.ln_active_mds.next,
                                                    lnet_libmd_t, md_list);
 

-                CERROR ("Active md %p on exit\n", md);
+                CERROR ("Active MD %p on exit\n", md);

                 cfs_list_del_init (&md->md_list);
                 lnet_md_free (md);
         }
                 cfs_list_del_init (&md->md_list);
                 lnet_md_free (md);
         }


@@ -743,7 +782,7 @@ lnet_unprepare (void)
                 lnet_eq_t *eq = cfs_list_entry (the_lnet.ln_active_eqs.next,
                                                 lnet_eq_t, eq_list);
 
                 lnet_eq_t *eq = cfs_list_entry (the_lnet.ln_active_eqs.next,
                                                 lnet_eq_t, eq_list);
 

-                CERROR ("Active eq %p on exit\n", eq);
+                CERROR ("Active EQ %p on exit\n", eq);

                 cfs_list_del (&eq->eq_list);
                 lnet_eq_free (eq);
         }
                 cfs_list_del (&eq->eq_list);
                 lnet_eq_free (eq);
         }

diff --git a/lnet/lnet/lib-me.c b/lnet/lnet/lib-me.c
index 65b28dd..381e512 100644 (file)
--- a/lnet/lnet/lib-me.c
+++ b/lnet/lnet/lib-me.c
@@ -42,6 +42,60 @@
 
 #include <lnet/lib-lnet.h>
 
 
 #include <lnet/lib-lnet.h>
 

+static int
+lnet_me_match_portal(lnet_portal_t *ptl, lnet_process_id_t id,
+                     __u64 match_bits, __u64 ignore_bits)
+{
+        cfs_list_t       *mhash = NULL;
+        int               unique;
+
+        LASSERT (!(lnet_portal_is_unique(ptl) &&
+                   lnet_portal_is_wildcard(ptl)));
+
+        /* prefer to check w/o any lock */
+        unique = lnet_match_is_unique(id, match_bits, ignore_bits);
+        if (likely(lnet_portal_is_unique(ptl) ||
+                   lnet_portal_is_wildcard(ptl)))
+                goto match;
+
+        /* unset, new portal */
+        if (unique) {
+                mhash = lnet_portal_mhash_alloc();
+                if (mhash == NULL)
+                        return -ENOMEM;
+        }
+
+        LNET_LOCK();
+        if (lnet_portal_is_unique(ptl) ||
+            lnet_portal_is_wildcard(ptl)) {
+                /* someone set it before me */
+                if (mhash != NULL)
+                        lnet_portal_mhash_free(mhash);
+                LNET_UNLOCK();
+                goto match;
+        }
+
+        /* still not set */
+        LASSERT (ptl->ptl_mhash == NULL);
+        if (unique) {
+                ptl->ptl_mhash = mhash;
+                lnet_portal_setopt(ptl, LNET_PTL_MATCH_UNIQUE);
+        } else {
+                lnet_portal_setopt(ptl, LNET_PTL_MATCH_WILDCARD);
+        }
+        LNET_UNLOCK();
+        return 0;
+
+ match:
+        if (lnet_portal_is_unique(ptl) && !unique)
+                return -EPERM;
+
+        if (lnet_portal_is_wildcard(ptl) && unique)
+                return -EPERM;
+
+        return 0;
+}
+

 int
 LNetMEAttach(unsigned int portal,
              lnet_process_id_t match_id,
 int
 LNetMEAttach(unsigned int portal,
              lnet_process_id_t match_id,


@@ -49,7 +103,10 @@ LNetMEAttach(unsigned int portal,
              lnet_unlink_t unlink, lnet_ins_pos_t pos,
              lnet_handle_me_t *handle)
 {
              lnet_unlink_t unlink, lnet_ins_pos_t pos,
              lnet_handle_me_t *handle)
 {

-        lnet_me_t     *me;
+        lnet_me_t        *me;
+        lnet_portal_t    *ptl;
+        cfs_list_t       *head;
+        int               rc;

 
         LASSERT (the_lnet.ln_init);
         LASSERT (the_lnet.ln_refcount > 0);
 
         LASSERT (the_lnet.ln_init);
         LASSERT (the_lnet.ln_refcount > 0);


@@ -57,6 +114,11 @@ LNetMEAttach(unsigned int portal,
         if ((int)portal >= the_lnet.ln_nportals)
                 return -EINVAL;
 
         if ((int)portal >= the_lnet.ln_nportals)
                 return -EINVAL;
 

+        ptl = &the_lnet.ln_portals[portal];
+        rc = lnet_me_match_portal(ptl, match_id, match_bits, ignore_bits);
+        if (rc != 0)
+                return rc;
+

         me = lnet_me_alloc();
         if (me == NULL)
                 return -ENOMEM;
         me = lnet_me_alloc();
         if (me == NULL)
                 return -ENOMEM;


@@ -71,13 +133,13 @@ LNetMEAttach(unsigned int portal,
         me->me_md = NULL;
 
         lnet_initialise_handle (&me->me_lh, LNET_COOKIE_TYPE_ME);
         me->me_md = NULL;
 
         lnet_initialise_handle (&me->me_lh, LNET_COOKIE_TYPE_ME);

+        head = lnet_portal_me_head(portal, match_id, match_bits);
+        LASSERT (head != NULL);

 
         if (pos == LNET_INS_AFTER)
 
         if (pos == LNET_INS_AFTER)

-                cfs_list_add_tail(&me->me_list,
-                                  &(the_lnet.ln_portals[portal].ptl_ml));
+                cfs_list_add_tail(&me->me_list, head);

         else
         else

-                cfs_list_add(&me->me_list,
-                             &(the_lnet.ln_portals[portal].ptl_ml));
+                cfs_list_add(&me->me_list, head);

 
         lnet_me2handle(handle, me);
 
 
         lnet_me2handle(handle, me);
 


@@ -95,6 +157,7 @@ LNetMEInsert(lnet_handle_me_t current_meh,
 {
         lnet_me_t     *current_me;
         lnet_me_t     *new_me;
 {
         lnet_me_t     *current_me;
         lnet_me_t     *new_me;

+        lnet_portal_t *ptl;

 
         LASSERT (the_lnet.ln_init);
         LASSERT (the_lnet.ln_refcount > 0);
 
         LASSERT (the_lnet.ln_init);
         LASSERT (the_lnet.ln_refcount > 0);


@@ -113,6 +176,16 @@ LNetMEInsert(lnet_handle_me_t current_meh,
                 return -ENOENT;
         }
 
                 return -ENOENT;
         }
 

+        LASSERT (current_me->me_portal < the_lnet.ln_nportals);
+
+        ptl = &the_lnet.ln_portals[current_me->me_portal];
+        if (lnet_portal_is_unique(ptl)) {
+                /* nosense to insertion on unique portal */
+                lnet_me_free (new_me);
+                LNET_UNLOCK();
+                return -EPERM;
+        }
+

         new_me->me_portal = current_me->me_portal;
         new_me->me_match_id = match_id;
         new_me->me_match_bits = match_bits;
         new_me->me_portal = current_me->me_portal;
         new_me->me_match_id = match_id;
         new_me->me_match_bits = match_bits;

diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c
index e46256b..957f958 100644 (file)
--- a/lnet/lnet/lib-move.c
+++ b/lnet/lnet/lib-move.c
@@ -155,6 +155,7 @@ lnet_match_md(int index, int op_mask, lnet_process_id_t src,
               lnet_libmd_t **md_out)
 {
         lnet_portal_t    *ptl = &the_lnet.ln_portals[index];
               lnet_libmd_t **md_out)
 {
         lnet_portal_t    *ptl = &the_lnet.ln_portals[index];

+        cfs_list_t       *head;

         lnet_me_t        *me;
         lnet_me_t        *tmp;
         lnet_libmd_t     *md;
         lnet_me_t        *me;
         lnet_me_t        *tmp;
         lnet_libmd_t     *md;


@@ -169,7 +170,11 @@ lnet_match_md(int index, int op_mask, lnet_process_id_t src,
                 return LNET_MATCHMD_DROP;
         }
 
                 return LNET_MATCHMD_DROP;
         }
 

-        cfs_list_for_each_entry_safe_typed (me, tmp, &ptl->ptl_ml,
+        head = lnet_portal_me_head(index, src, match_bits);
+        if (head == NULL) /* nobody posted anything on this portal */
+                goto out;
+
+        cfs_list_for_each_entry_safe_typed (me, tmp, head,

                                             lnet_me_t, me_list) {
                 md = me->me_md;
 
                                             lnet_me_t, me_list) {
                 md = me->me_md;
 


@@ -199,8 +204,9 @@ lnet_match_md(int index, int op_mask, lnet_process_id_t src,
                 /* not reached */
         }
 
                 /* not reached */
         }
 

+ out:

         if (op_mask == LNET_MD_OP_GET ||
         if (op_mask == LNET_MD_OP_GET ||

-            (ptl->ptl_options & LNET_PTL_LAZY) == 0)
+            !lnet_portal_is_lazy(ptl))

                 return LNET_MATCHMD_DROP;
 
         return LNET_MATCHMD_NONE;
                 return LNET_MATCHMD_DROP;
 
         return LNET_MATCHMD_NONE;


@@ -1572,9 +1578,7 @@ LNetSetLazyPortal(int portal)
         CDEBUG(D_NET, "Setting portal %d lazy\n", portal);
 
         LNET_LOCK();
         CDEBUG(D_NET, "Setting portal %d lazy\n", portal);
 
         LNET_LOCK();

-
-        ptl->ptl_options |= LNET_PTL_LAZY;
-
+        lnet_portal_setopt(ptl, LNET_PTL_LAZY);

         LNET_UNLOCK();
 
         return 0;
         LNET_UNLOCK();
 
         return 0;


@@ -1592,7 +1596,7 @@ LNetClearLazyPortal(int portal)
 
         LNET_LOCK();
 
 
         LNET_LOCK();
 

-        if ((ptl->ptl_options & LNET_PTL_LAZY) == 0) {
+        if (!lnet_portal_is_lazy(ptl)) {

                 LNET_UNLOCK();
                 return 0;
         }
                 LNET_UNLOCK();
                 return 0;
         }


@@ -1607,7 +1611,7 @@ LNetClearLazyPortal(int portal)
         cfs_list_del_init(&ptl->ptl_msgq);
 
         ptl->ptl_msgq_version++;
         cfs_list_del_init(&ptl->ptl_msgq);
 
         ptl->ptl_msgq_version++;

-        ptl->ptl_options &= ~LNET_PTL_LAZY;
+        lnet_portal_unsetopt(ptl, LNET_PTL_LAZY);

 
         LNET_UNLOCK();
 
 
         LNET_UNLOCK();
 


@@ -1662,12 +1666,13 @@ lnet_match_blocked_msg(lnet_libmd_t *md)
         cfs_list_t       *tmp;
         cfs_list_t       *entry;
         lnet_msg_t       *msg;
         cfs_list_t       *tmp;
         cfs_list_t       *entry;
         lnet_msg_t       *msg;

+        lnet_portal_t    *ptl;

         lnet_me_t        *me  = md->md_me;
         lnet_me_t        *me  = md->md_me;

-        lnet_portal_t    *ptl = &the_lnet.ln_portals[me->me_portal];

 
         LASSERT (me->me_portal < (unsigned int)the_lnet.ln_nportals);
 
 
         LASSERT (me->me_portal < (unsigned int)the_lnet.ln_nportals);
 

-        if ((ptl->ptl_options & LNET_PTL_LAZY) == 0) {
+        ptl = &the_lnet.ln_portals[me->me_portal];
+        if (!lnet_portal_is_lazy(ptl)) {

                 LASSERT (cfs_list_empty(&ptl->ptl_msgq));
                 return;
         }
                 LASSERT (cfs_list_empty(&ptl->ptl_msgq));
                 return;
         }


@@ -1773,7 +1778,6 @@ lnet_parse_put(lnet_ni_t *ni, lnet_msg_t *msg)
         hdr->msg.put.offset = le32_to_cpu(hdr->msg.put.offset);
 
         index = hdr->msg.put.ptl_index;
         hdr->msg.put.offset = le32_to_cpu(hdr->msg.put.offset);
 
         index = hdr->msg.put.ptl_index;

-        ptl = &the_lnet.ln_portals[index];

 
         LNET_LOCK();
 
 
         LNET_LOCK();
 


@@ -1792,6 +1796,7 @@ lnet_parse_put(lnet_ni_t *ni, lnet_msg_t *msg)
                 return 0;
 
         case LNET_MATCHMD_NONE:
                 return 0;
 
         case LNET_MATCHMD_NONE:

+                ptl = &the_lnet.ln_portals[index];

                 version = ptl->ptl_ml_version;
 
                 rc = 0;
                 version = ptl->ptl_ml_version;
 
                 rc = 0;


@@ -1800,7 +1805,7 @@ lnet_parse_put(lnet_ni_t *ni, lnet_msg_t *msg)
 
                 if (rc == 0 &&
                     !the_lnet.ln_shutdown &&
 
                 if (rc == 0 &&
                     !the_lnet.ln_shutdown &&

-                    ((ptl->ptl_options & LNET_PTL_LAZY) != 0)) {
+                    lnet_portal_is_lazy(ptl)) {

                         if (version != ptl->ptl_ml_version)
                                 goto again;
 
                         if (version != ptl->ptl_ml_version)
                                 goto again;