lnet_md_exhausted(md));
}
+static inline unsigned int
+lnet_match_to_hash(lnet_process_id_t id, __u64 mbits)
+{
+ mbits += id.nid + id.pid;
+ return cfs_hash_long((unsigned long)mbits, LNET_PORTAL_HASH_BITS);
+}
+
#ifdef __KERNEL__
#define LNET_LOCK() cfs_spin_lock(&the_lnet.ln_lock)
#define LNET_UNLOCK() cfs_spin_unlock(&the_lnet.ln_lock)
return (lh_entry (lh, lnet_me_t, me_lh));
}
+static inline int
+lnet_portal_is_lazy(lnet_portal_t *ptl)
+{
+ return !!(ptl->ptl_options & LNET_PTL_LAZY);
+}
+
+static inline int
+lnet_portal_is_unique(lnet_portal_t *ptl)
+{
+ return !!(ptl->ptl_options & LNET_PTL_MATCH_UNIQUE);
+}
+
+static inline int
+lnet_portal_is_wildcard(lnet_portal_t *ptl)
+{
+ return !!(ptl->ptl_options & LNET_PTL_MATCH_WILDCARD);
+}
+
+static inline void
+lnet_portal_setopt(lnet_portal_t *ptl, int opt)
+{
+ ptl->ptl_options |= opt;
+}
+
+static inline void
+lnet_portal_unsetopt(lnet_portal_t *ptl, int opt)
+{
+ ptl->ptl_options &= ~opt;
+}
+
+static inline int
+lnet_match_is_unique(lnet_process_id_t match_id,
+ __u64 match_bits, __u64 ignore_bits)
+{
+ return ignore_bits == 0 &&
+ match_id.nid != LNET_NID_ANY &&
+ match_id.pid != LNET_PID_ANY;
+}
+
+static inline cfs_list_t *
+lnet_portal_me_head(int index, lnet_process_id_t id, __u64 mbits)
+{
+ lnet_portal_t *ptl = &the_lnet.ln_portals[index];
+
+ if (lnet_portal_is_wildcard(ptl)) {
+ return &ptl->ptl_mlist;
+ } else if (lnet_portal_is_unique(ptl)) {
+ LASSERT (ptl->ptl_mhash != NULL);
+ return &ptl->ptl_mhash[lnet_match_to_hash(id, mbits)];
+ }
+ return NULL;
+}
+
+cfs_list_t *lnet_portal_mhash_alloc(void);
+void lnet_portal_mhash_free(cfs_list_t *mhash);
+
static inline void
lnet_peer_addref_locked(lnet_peer_t *lp)
{
/* Options for lnet_portal_t::ptl_options */
#define LNET_PTL_LAZY (1 << 0)
+#define LNET_PTL_MATCH_UNIQUE (1 << 1) /* unique match, for RDMA */
+#define LNET_PTL_MATCH_WILDCARD (1 << 2) /* wildcard match, request portal */
+
+/* ME hash of RDMA portal */
+#define LNET_PORTAL_HASH_BITS 8
+#define LNET_PORTAL_HASH_SIZE (1 << LNET_PORTAL_HASH_BITS)
+
typedef struct {
- cfs_list_t ptl_ml; /* match list */
- cfs_list_t ptl_msgq; /* messages blocking for MD */
- __u64 ptl_ml_version; /* validity stamp, only changed for new attached MD */
- __u64 ptl_msgq_version; /* validity stamp */
- unsigned int ptl_options;
+ cfs_list_t *ptl_mhash; /* match hash */
+ cfs_list_t ptl_mlist; /* match list */
+ cfs_list_t ptl_msgq; /* messages blocking for MD */
+ __u64 ptl_ml_version; /* validity stamp, only changed for new attached MD */
+ __u64 ptl_msgq_version; /* validity stamp */
+ unsigned int ptl_options;
} lnet_portal_t;
/* Router Checker states */
cfs_list_del (&lh->lh_hash_chain);
}
+cfs_list_t *
+lnet_portal_mhash_alloc(void)
+{
+ cfs_list_t *mhash;
+ int i;
+
+ LIBCFS_ALLOC(mhash, sizeof(cfs_list_t) * LNET_PORTAL_HASH_SIZE);
+ if (mhash == NULL)
+ return NULL;
+
+ for (i = 0; i < LNET_PORTAL_HASH_SIZE; i++)
+ CFS_INIT_LIST_HEAD(&mhash[i]);
+
+ return mhash;
+}
+
+void
+lnet_portal_mhash_free(cfs_list_t *mhash)
+{
+ int i;
+
+ for (i = 0; i < LNET_PORTAL_HASH_SIZE; i++) {
+ while (!cfs_list_empty(&mhash[i])) {
+ lnet_me_t *me = cfs_list_entry(mhash[i].next,
+ lnet_me_t, me_list);
+ CERROR ("Active ME %p on exit portal mhash\n", me);
+ cfs_list_del(&me->me_list);
+ lnet_me_free(me);
+ }
+ }
+ LIBCFS_FREE(mhash, sizeof(cfs_list_t) * LNET_PORTAL_HASH_SIZE);
+}
+
int
lnet_init_finalizers(void)
{
}
for (i = 0; i < the_lnet.ln_nportals; i++) {
- CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_ml));
+ CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_mlist));
CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_msgq));
the_lnet.ln_portals[i].ptl_options = 0;
}
LASSERT (the_lnet.ln_nzombie_nis == 0);
for (idx = 0; idx < the_lnet.ln_nportals; idx++) {
- LASSERT (cfs_list_empty(&the_lnet.ln_portals[idx].ptl_msgq));
+ lnet_portal_t *ptl = &the_lnet.ln_portals[idx];
- while (!cfs_list_empty (&the_lnet.ln_portals[idx].ptl_ml)) {
- lnet_me_t *me = cfs_list_entry (the_lnet.ln_portals[idx].ptl_ml.next,
- lnet_me_t, me_list);
+ LASSERT (cfs_list_empty(&ptl->ptl_msgq));
- CERROR ("Active me %p on exit\n", me);
+ while (!cfs_list_empty(&ptl->ptl_mlist)) {
+ lnet_me_t *me = cfs_list_entry(ptl->ptl_mlist.next,
+ lnet_me_t, me_list);
+ CERROR ("Active ME %p on exit\n", me);
cfs_list_del (&me->me_list);
lnet_me_free (me);
}
+
+ if (ptl->ptl_mhash != NULL) {
+ LASSERT (lnet_portal_is_unique(ptl));
+ lnet_portal_mhash_free(ptl->ptl_mhash);
+ }
}
while (!cfs_list_empty (&the_lnet.ln_active_mds)) {
lnet_libmd_t *md = cfs_list_entry (the_lnet.ln_active_mds.next,
lnet_libmd_t, md_list);
- CERROR ("Active md %p on exit\n", md);
+ CERROR ("Active MD %p on exit\n", md);
cfs_list_del_init (&md->md_list);
lnet_md_free (md);
}
lnet_eq_t *eq = cfs_list_entry (the_lnet.ln_active_eqs.next,
lnet_eq_t, eq_list);
- CERROR ("Active eq %p on exit\n", eq);
+ CERROR ("Active EQ %p on exit\n", eq);
cfs_list_del (&eq->eq_list);
lnet_eq_free (eq);
}
#include <lnet/lib-lnet.h>
+static int
+lnet_me_match_portal(lnet_portal_t *ptl, lnet_process_id_t id,
+ __u64 match_bits, __u64 ignore_bits)
+{
+ cfs_list_t *mhash = NULL;
+ int unique;
+
+ LASSERT (!(lnet_portal_is_unique(ptl) &&
+ lnet_portal_is_wildcard(ptl)));
+
+ /* prefer to check w/o any lock */
+ unique = lnet_match_is_unique(id, match_bits, ignore_bits);
+ if (likely(lnet_portal_is_unique(ptl) ||
+ lnet_portal_is_wildcard(ptl)))
+ goto match;
+
+ /* unset, new portal */
+ if (unique) {
+ mhash = lnet_portal_mhash_alloc();
+ if (mhash == NULL)
+ return -ENOMEM;
+ }
+
+ LNET_LOCK();
+ if (lnet_portal_is_unique(ptl) ||
+ lnet_portal_is_wildcard(ptl)) {
+ /* someone set it before me */
+ if (mhash != NULL)
+ lnet_portal_mhash_free(mhash);
+ LNET_UNLOCK();
+ goto match;
+ }
+
+ /* still not set */
+ LASSERT (ptl->ptl_mhash == NULL);
+ if (unique) {
+ ptl->ptl_mhash = mhash;
+ lnet_portal_setopt(ptl, LNET_PTL_MATCH_UNIQUE);
+ } else {
+ lnet_portal_setopt(ptl, LNET_PTL_MATCH_WILDCARD);
+ }
+ LNET_UNLOCK();
+ return 0;
+
+ match:
+ if (lnet_portal_is_unique(ptl) && !unique)
+ return -EPERM;
+
+ if (lnet_portal_is_wildcard(ptl) && unique)
+ return -EPERM;
+
+ return 0;
+}
+
int
LNetMEAttach(unsigned int portal,
lnet_process_id_t match_id,
lnet_unlink_t unlink, lnet_ins_pos_t pos,
lnet_handle_me_t *handle)
{
- lnet_me_t *me;
+ lnet_me_t *me;
+ lnet_portal_t *ptl;
+ cfs_list_t *head;
+ int rc;
LASSERT (the_lnet.ln_init);
LASSERT (the_lnet.ln_refcount > 0);
if ((int)portal >= the_lnet.ln_nportals)
return -EINVAL;
+ ptl = &the_lnet.ln_portals[portal];
+ rc = lnet_me_match_portal(ptl, match_id, match_bits, ignore_bits);
+ if (rc != 0)
+ return rc;
+
me = lnet_me_alloc();
if (me == NULL)
return -ENOMEM;
me->me_md = NULL;
lnet_initialise_handle (&me->me_lh, LNET_COOKIE_TYPE_ME);
+ head = lnet_portal_me_head(portal, match_id, match_bits);
+ LASSERT (head != NULL);
if (pos == LNET_INS_AFTER)
- cfs_list_add_tail(&me->me_list,
- &(the_lnet.ln_portals[portal].ptl_ml));
+ cfs_list_add_tail(&me->me_list, head);
else
- cfs_list_add(&me->me_list,
- &(the_lnet.ln_portals[portal].ptl_ml));
+ cfs_list_add(&me->me_list, head);
lnet_me2handle(handle, me);
{
lnet_me_t *current_me;
lnet_me_t *new_me;
+ lnet_portal_t *ptl;
LASSERT (the_lnet.ln_init);
LASSERT (the_lnet.ln_refcount > 0);
return -ENOENT;
}
+ LASSERT (current_me->me_portal < the_lnet.ln_nportals);
+
+ ptl = &the_lnet.ln_portals[current_me->me_portal];
+ if (lnet_portal_is_unique(ptl)) {
+ /* nosense to insertion on unique portal */
+ lnet_me_free (new_me);
+ LNET_UNLOCK();
+ return -EPERM;
+ }
+
new_me->me_portal = current_me->me_portal;
new_me->me_match_id = match_id;
new_me->me_match_bits = match_bits;
lnet_libmd_t **md_out)
{
lnet_portal_t *ptl = &the_lnet.ln_portals[index];
+ cfs_list_t *head;
lnet_me_t *me;
lnet_me_t *tmp;
lnet_libmd_t *md;
return LNET_MATCHMD_DROP;
}
- cfs_list_for_each_entry_safe_typed (me, tmp, &ptl->ptl_ml,
+ head = lnet_portal_me_head(index, src, match_bits);
+ if (head == NULL) /* nobody posted anything on this portal */
+ goto out;
+
+ cfs_list_for_each_entry_safe_typed (me, tmp, head,
lnet_me_t, me_list) {
md = me->me_md;
/* not reached */
}
+ out:
if (op_mask == LNET_MD_OP_GET ||
- (ptl->ptl_options & LNET_PTL_LAZY) == 0)
+ !lnet_portal_is_lazy(ptl))
return LNET_MATCHMD_DROP;
return LNET_MATCHMD_NONE;
CDEBUG(D_NET, "Setting portal %d lazy\n", portal);
LNET_LOCK();
-
- ptl->ptl_options |= LNET_PTL_LAZY;
-
+ lnet_portal_setopt(ptl, LNET_PTL_LAZY);
LNET_UNLOCK();
return 0;
LNET_LOCK();
- if ((ptl->ptl_options & LNET_PTL_LAZY) == 0) {
+ if (!lnet_portal_is_lazy(ptl)) {
LNET_UNLOCK();
return 0;
}
cfs_list_del_init(&ptl->ptl_msgq);
ptl->ptl_msgq_version++;
- ptl->ptl_options &= ~LNET_PTL_LAZY;
+ lnet_portal_unsetopt(ptl, LNET_PTL_LAZY);
LNET_UNLOCK();
cfs_list_t *tmp;
cfs_list_t *entry;
lnet_msg_t *msg;
+ lnet_portal_t *ptl;
lnet_me_t *me = md->md_me;
- lnet_portal_t *ptl = &the_lnet.ln_portals[me->me_portal];
LASSERT (me->me_portal < (unsigned int)the_lnet.ln_nportals);
- if ((ptl->ptl_options & LNET_PTL_LAZY) == 0) {
+ ptl = &the_lnet.ln_portals[me->me_portal];
+ if (!lnet_portal_is_lazy(ptl)) {
LASSERT (cfs_list_empty(&ptl->ptl_msgq));
return;
}
hdr->msg.put.offset = le32_to_cpu(hdr->msg.put.offset);
index = hdr->msg.put.ptl_index;
- ptl = &the_lnet.ln_portals[index];
LNET_LOCK();
return 0;
case LNET_MATCHMD_NONE:
+ ptl = &the_lnet.ln_portals[index];
version = ptl->ptl_ml_version;
rc = 0;
if (rc == 0 &&
!the_lnet.ln_shutdown &&
- ((ptl->ptl_options & LNET_PTL_LAZY) != 0)) {
+ lnet_portal_is_lazy(ptl)) {
if (version != ptl->ptl_ml_version)
goto again;