lnet_md_exhausted(md));
}
+static inline unsigned int
+lnet_match_hash_value(lnet_process_id_t id, __u64 mbits)
+{
+ unsigned int val;
+
+ val = (unsigned int)(mbits + (mbits >> 32));
+ val += (unsigned int)(id.nid + (id.nid >> 32));
+ val += (unsigned int)(id.pid);
+
+ return val;
+}
+
+static inline unsigned int
+lnet_match_to_hash(lnet_process_id_t id, __u64 mbits)
+{
+ return lnet_match_hash_value(id, mbits) % LNET_PORTAL_HASH_SIZE;
+}
+
#ifdef __KERNEL__
-#define LNET_LOCK() spin_lock(&the_lnet.ln_lock)
-#define LNET_UNLOCK() spin_unlock(&the_lnet.ln_lock)
+#define LNET_LOCK() spin_lock(&the_lnet.ln_lock)
+#define LNET_UNLOCK() spin_unlock(&the_lnet.ln_lock)
#define LNET_MUTEX_DOWN(m) mutex_down(m)
#define LNET_MUTEX_UP(m) mutex_up(m)
#else
return (lh_entry (lh, lnet_me_t, me_lh));
}
+static inline int
+lnet_portal_is_lazy(lnet_portal_t *ptl)
+{
+ return !!(ptl->ptl_options & LNET_PTL_LAZY);
+}
+
+static inline int
+lnet_portal_is_unique(lnet_portal_t *ptl)
+{
+ return !!(ptl->ptl_options & LNET_PTL_MATCH_UNIQUE);
+}
+
+static inline int
+lnet_portal_is_wildcard(lnet_portal_t *ptl)
+{
+ return !!(ptl->ptl_options & LNET_PTL_MATCH_WILDCARD);
+}
+
+static inline void
+lnet_portal_setopt(lnet_portal_t *ptl, int opt)
+{
+ ptl->ptl_options |= opt;
+}
+
+static inline void
+lnet_portal_unsetopt(lnet_portal_t *ptl, int opt)
+{
+ ptl->ptl_options &= ~opt;
+}
+
+static inline int
+lnet_match_is_unique(lnet_process_id_t match_id,
+ __u64 match_bits, __u64 ignore_bits)
+{
+ return ignore_bits == 0 &&
+ match_id.nid != LNET_NID_ANY &&
+ match_id.pid != LNET_PID_ANY;
+}
+
+static inline struct list_head *
+lnet_portal_me_head(int index, lnet_process_id_t id, __u64 mbits)
+{
+ lnet_portal_t *ptl = &the_lnet.ln_portals[index];
+
+ if (lnet_portal_is_wildcard(ptl)) {
+ return &ptl->ptl_mlist;
+ } else if (lnet_portal_is_unique(ptl)) {
+ LASSERT (ptl->ptl_mhash != NULL);
+ return &ptl->ptl_mhash[lnet_match_to_hash(id, mbits)];
+ }
+ return NULL;
+}
+
+struct list_head *lnet_portal_mhash_alloc(void);
+void lnet_portal_mhash_free(struct list_head *mhash);
+
static inline void
lnet_peer_addref_locked(lnet_peer_t *lp)
{
/* Options for lnet_portal_t::ptl_options */
#define LNET_PTL_LAZY (1 << 0)
+#define LNET_PTL_MATCH_UNIQUE (1 << 1) /* unique match, for RDMA */
+#define LNET_PTL_MATCH_WILDCARD (1 << 2) /* wildcard match, request portal */
+
+#define LNET_PORTAL_HASH_SIZE 113 /* ME hash size of RDMA portal (prime) */
+
typedef struct {
- struct list_head ptl_ml; /* match list */
- struct list_head ptl_msgq; /* messages blocking for MD */
- __u64 ptl_ml_version; /* validity stamp, only changed for new attached MD */
- __u64 ptl_msgq_version; /* validity stamp */
- unsigned int ptl_options;
+ struct list_head *ptl_mhash; /* match hash */
+ struct list_head ptl_mlist; /* match list */
+ struct list_head ptl_msgq; /* messages blocking for MD */
+ __u64 ptl_ml_version; /* validity stamp, only changed for new attached MD */
+ __u64 ptl_msgq_version; /* validity stamp */
+ unsigned int ptl_options;
} lnet_portal_t;
/* Router Checker states */
list_del (&lh->lh_hash_chain);
}
+struct list_head *
+lnet_portal_mhash_alloc(void)
+{
+ struct list_head *mhash;
+ int i;
+
+ LIBCFS_ALLOC(mhash, sizeof(struct list_head) * LNET_PORTAL_HASH_SIZE);
+ if (mhash == NULL)
+ return NULL;
+
+ for (i = 0; i < LNET_PORTAL_HASH_SIZE; i++)
+ CFS_INIT_LIST_HEAD(&mhash[i]);
+
+ return mhash;
+}
+
+void
+lnet_portal_mhash_free(struct list_head *mhash)
+{
+ int i;
+
+ for (i = 0; i < LNET_PORTAL_HASH_SIZE; i++) {
+ while (!list_empty(&mhash[i])) {
+ lnet_me_t *me = list_entry (mhash[i].next,
+ lnet_me_t, me_list);
+ CERROR ("Active ME %p on exit portal mhash\n", me);
+ list_del (&me->me_list);
+ lnet_me_free (me);
+ }
+ }
+ LIBCFS_FREE(mhash, sizeof(struct list_head) * LNET_PORTAL_HASH_SIZE);
+}
+
int
lnet_init_finalizers(void)
{
}
for (i = 0; i < the_lnet.ln_nportals; i++) {
- CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_ml));
+ CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_mlist));
CFS_INIT_LIST_HEAD(&(the_lnet.ln_portals[i].ptl_msgq));
the_lnet.ln_portals[i].ptl_options = 0;
}
for (idx = 0; idx < the_lnet.ln_nportals; idx++) {
LASSERT (list_empty(&the_lnet.ln_portals[idx].ptl_msgq));
- while (!list_empty (&the_lnet.ln_portals[idx].ptl_ml)) {
- lnet_me_t *me = list_entry (the_lnet.ln_portals[idx].ptl_ml.next,
+ while (!list_empty (&the_lnet.ln_portals[idx].ptl_mlist)) {
+ lnet_me_t *me = list_entry (the_lnet.ln_portals[idx].ptl_mlist.next,
lnet_me_t, me_list);
- CERROR ("Active me %p on exit\n", me);
+ CERROR ("Active ME %p on exit\n", me);
list_del (&me->me_list);
lnet_me_free (me);
}
+
+ if (the_lnet.ln_portals[idx].ptl_mhash != NULL) {
+ LASSERT (lnet_portal_is_unique(&the_lnet.ln_portals[idx]));
+ lnet_portal_mhash_free(the_lnet.ln_portals[idx].ptl_mhash);
+ }
}
while (!list_empty (&the_lnet.ln_active_mds)) {
lnet_libmd_t *md = list_entry (the_lnet.ln_active_mds.next,
lnet_libmd_t, md_list);
- CERROR ("Active md %p on exit\n", md);
+ CERROR ("Active MD %p on exit\n", md);
list_del_init (&md->md_list);
lnet_md_free (md);
}
lnet_eq_t *eq = list_entry (the_lnet.ln_active_eqs.next,
lnet_eq_t, eq_list);
- CERROR ("Active eq %p on exit\n", eq);
+ CERROR ("Active EQ %p on exit\n", eq);
list_del (&eq->eq_list);
lnet_eq_free (eq);
}
#include <lnet/lib-lnet.h>
+static int
+lnet_me_match_portal(lnet_portal_t *ptl, lnet_process_id_t id,
+ __u64 match_bits, __u64 ignore_bits)
+{
+ struct list_head *mhash = NULL;
+ int unique;
+
+ LASSERT (!(lnet_portal_is_unique(ptl) &&
+ lnet_portal_is_wildcard(ptl)));
+
+ /* prefer to check w/o any lock */
+ unique = lnet_match_is_unique(id, match_bits, ignore_bits);
+ if (likely(lnet_portal_is_unique(ptl) ||
+ lnet_portal_is_wildcard(ptl)))
+ goto match;
+
+ /* unset, new portal */
+ if (unique) {
+ mhash = lnet_portal_mhash_alloc();
+ if (mhash == NULL)
+ return -ENOMEM;
+ }
+
+ LNET_LOCK();
+ if (lnet_portal_is_unique(ptl) ||
+ lnet_portal_is_wildcard(ptl)) {
+ /* someone set it before me */
+ if (mhash != NULL)
+ lnet_portal_mhash_free(mhash);
+ LNET_UNLOCK();
+ goto match;
+ }
+
+ /* still not set */
+ LASSERT (ptl->ptl_mhash == NULL);
+ if (unique) {
+ ptl->ptl_mhash = mhash;
+ lnet_portal_setopt(ptl, LNET_PTL_MATCH_UNIQUE);
+ } else {
+ lnet_portal_setopt(ptl, LNET_PTL_MATCH_WILDCARD);
+ }
+ LNET_UNLOCK();
+ return 0;
+
+ match:
+ if (lnet_portal_is_unique(ptl) && !unique)
+ return -EPERM;
+
+ if (lnet_portal_is_wildcard(ptl) && unique)
+ return -EPERM;
+
+ return 0;
+}
+
int
LNetMEAttach(unsigned int portal,
lnet_process_id_t match_id,
lnet_unlink_t unlink, lnet_ins_pos_t pos,
lnet_handle_me_t *handle)
{
- lnet_me_t *me;
+ lnet_me_t *me;
+ lnet_portal_t *ptl;
+ struct list_head *head;
+ int rc;
LASSERT (the_lnet.ln_init);
LASSERT (the_lnet.ln_refcount > 0);
if (portal >= the_lnet.ln_nportals)
return -EINVAL;
+ ptl = &the_lnet.ln_portals[portal];
+ rc = lnet_me_match_portal(ptl, match_id, match_bits, ignore_bits);
+ if (rc != 0)
+ return rc;
+
me = lnet_me_alloc();
if (me == NULL)
return -ENOMEM;
lnet_initialise_handle (&me->me_lh, LNET_COOKIE_TYPE_ME);
+ head = lnet_portal_me_head(portal, match_id, match_bits);
+ LASSERT (head != NULL);
+
if (pos == LNET_INS_AFTER)
- list_add_tail(&me->me_list, &(the_lnet.ln_portals[portal].ptl_ml));
+ list_add_tail(&me->me_list, head);
else
- list_add(&me->me_list, &(the_lnet.ln_portals[portal].ptl_ml));
+ list_add(&me->me_list, head);
lnet_me2handle(handle, me);
{
lnet_me_t *current_me;
lnet_me_t *new_me;
+ lnet_portal_t *ptl;
LASSERT (the_lnet.ln_init);
LASSERT (the_lnet.ln_refcount > 0);
return -ENOENT;
}
+ LASSERT (current_me->me_portal < the_lnet.ln_nportals);
+
+ ptl = &the_lnet.ln_portals[current_me->me_portal];
+ if (lnet_portal_is_unique(ptl)) {
+ /* nosense to insertion on unique portal */
+ lnet_me_free (new_me);
+ LNET_UNLOCK();
+ return -EPERM;
+ }
+
new_me->me_portal = current_me->me_portal;
new_me->me_match_id = match_id;
new_me->me_match_bits = match_bits;
lnet_libmd_t **md_out)
{
lnet_portal_t *ptl = &the_lnet.ln_portals[index];
+ struct list_head *head;
lnet_me_t *me;
lnet_me_t *tmp;
lnet_libmd_t *md;
return LNET_MATCHMD_DROP;
}
- list_for_each_entry_safe (me, tmp, &ptl->ptl_ml, me_list) {
+ head = lnet_portal_me_head(index, src, match_bits);
+ if (head == NULL) /* nobody posted anything on this portal */
+ goto out;
+
+ list_for_each_entry_safe (me, tmp, head, me_list) {
md = me->me_md;
/* ME attached but MD not attached yet */
/* not reached */
}
+ out:
if (op_mask == LNET_MD_OP_GET ||
- (ptl->ptl_options & LNET_PTL_LAZY) == 0)
+ !lnet_portal_is_lazy(ptl))
return LNET_MATCHMD_DROP;
return LNET_MATCHMD_NONE;
CDEBUG(D_NET, "Setting portal %d lazy\n", portal);
LNET_LOCK();
-
- ptl->ptl_options |= LNET_PTL_LAZY;
-
+ lnet_portal_setopt(ptl, LNET_PTL_LAZY);
LNET_UNLOCK();
return 0;
LNET_LOCK();
- if ((ptl->ptl_options & LNET_PTL_LAZY) == 0) {
+ if (!lnet_portal_is_lazy(ptl)) {
LNET_UNLOCK();
return 0;
}
list_del_init(&ptl->ptl_msgq);
ptl->ptl_msgq_version++;
- ptl->ptl_options &= ~LNET_PTL_LAZY;
+ lnet_portal_unsetopt(ptl, LNET_PTL_LAZY);
LNET_UNLOCK();
struct list_head *tmp;
struct list_head *entry;
lnet_msg_t *msg;
+ lnet_portal_t *ptl;
lnet_me_t *me = md->md_me;
- lnet_portal_t *ptl = &the_lnet.ln_portals[me->me_portal];
LASSERT (me->me_portal < the_lnet.ln_nportals);
- if ((ptl->ptl_options & LNET_PTL_LAZY) == 0) {
+ ptl = &the_lnet.ln_portals[me->me_portal];
+ if (!lnet_portal_is_lazy(ptl)) {
LASSERT (list_empty(&ptl->ptl_msgq));
return;
}
hdr->msg.put.offset = le32_to_cpu(hdr->msg.put.offset);
index = hdr->msg.put.ptl_index;
- ptl = &the_lnet.ln_portals[index];
LNET_LOCK();
return 0;
case LNET_MATCHMD_NONE:
+ ptl = &the_lnet.ln_portals[index];
version = ptl->ptl_ml_version;
rc = 0;
if (rc == 0 &&
!the_lnet.ln_shutdown &&
- ((ptl->ptl_options & LNET_PTL_LAZY) != 0)) {
+ lnet_portal_is_lazy(ptl)) {
if (version != ptl->ptl_ml_version)
goto again;
git://git.whamcloud.com / fs / lustre-release.git / commitdiff