X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lnet%2Fklnds%2Fo2iblnd%2Fo2iblnd.c;h=c07d4d18d983249348ab8979675df72e8d8905f0;hp=67644216772afe3344d51c8e850eb102d99f210f;hb=fa7276592a0cc12d651a5b2bd39dd7ca45bc25bf;hpb=31d6445718b896290198f5d127f86c174d499c6c

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 6764421..c07d4d18 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -15,11 +15,7 @@
  *
  * You should have received a copy of the GNU General Public License
  * version 2 along with this program; If not, see
- * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
- *
- * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
- * CA 95054 USA or visit www.sun.com if you need additional information or
- * have any questions.
+ * http://www.gnu.org/licenses/gpl-2.0.html
  *
  * GPL HEADER END
  */
@@ -27,7 +23,7 @@
  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  * Use is subject to license terms.
  *
- * Copyright (c) 2011, 2015, Intel Corporation.
+ * Copyright (c) 2011, 2016, Intel Corporation.
  */
 /*
  * This file is part of Lustre, http://www.lustre.org/
@@ -321,7 +317,7 @@ kiblnd_create_peer(lnet_ni_t *ni, kib_peer_t **peerp, lnet_nid_t nid)
 {
 	kib_peer_t	*peer;
 	kib_net_t	*net = ni->ni_data;
-	int		cpt = lnet_cpt_of_nid(nid);
+	int		cpt = lnet_cpt_of_nid(nid, ni);
 	unsigned long   flags;
 
 	LASSERT(net != NULL);
@@ -338,7 +334,7 @@ kiblnd_create_peer(lnet_ni_t *ni, kib_peer_t **peerp, lnet_nid_t nid)
 	peer->ibp_error = 0;
 	peer->ibp_last_alive = 0;
 	peer->ibp_max_frags = kiblnd_cfg_rdma_frags(peer->ibp_ni);
-	peer->ibp_queue_depth = ni->ni_peertxcredits;
+	peer->ibp_queue_depth = ni->ni_net->net_tunables.lct_peer_tx_credits;
 	atomic_set(&peer->ibp_refcount, 1);	/* 1 ref for caller */
 
 	INIT_LIST_HEAD(&peer->ibp_list);	/* not in the peer table yet */
@@ -380,7 +376,7 @@ kiblnd_destroy_peer (kib_peer_t *peer)
 }
 
 kib_peer_t *
-kiblnd_find_peer_locked (lnet_nid_t nid)
+kiblnd_find_peer_locked(struct lnet_ni *ni, lnet_nid_t nid)
 {
 	/* the caller is responsible for accounting the additional reference
 	 * that this creates */
@@ -393,7 +389,14 @@ kiblnd_find_peer_locked (lnet_nid_t nid)
 		peer = list_entry(tmp, kib_peer_t, ibp_list);
 		LASSERT(!kiblnd_peer_idle(peer));
 
-		if (peer->ibp_nid != nid)
+		/*
+		 * Match a peer if its NID and the NID of the local NI it
+		 * communicates over are the same. Otherwise don't match
+		 * the peer, which will result in a new lnd peer being
+		 * created.
+		 */
+		if (peer->ibp_nid != nid ||
+		    peer->ibp_ni->ni_nid != ni->ni_nid)
 			continue;
 
 		CDEBUG(D_NET, "got peer [%p] -> %s (%d) version: %x\n",
@@ -726,7 +729,7 @@ kiblnd_create_conn(kib_peer_t *peer, struct rdma_cm_id *cmid,
 
 	dev = net->ibn_dev;
 
-	cpt = lnet_cpt_of_nid(peer->ibp_nid);
+	cpt = lnet_cpt_of_nid(peer->ibp_nid, peer->ibp_ni);
 	sched = kiblnd_data.kib_scheds[cpt];
 
 	LASSERT(sched->ibs_nthreads > 0);
@@ -1153,7 +1156,7 @@ kiblnd_query(lnet_ni_t *ni, lnet_nid_t nid, cfs_time_t *when)
 
 	read_lock_irqsave(glock, flags);
 
-	peer = kiblnd_find_peer_locked(nid);
+	peer = kiblnd_find_peer_locked(ni, nid);
 	if (peer != NULL)
 		last_alive = peer->ibp_last_alive;
 
@@ -1395,7 +1398,7 @@ kiblnd_find_rd_dma_mr(struct lnet_ni *ni, kib_rdma_desc_t *rd,
 	int	mod;
 	__u16	nfrags;
 
-	tunables = &ni->ni_lnd_tunables->lt_tun_u.lt_o2ib;
+	tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib;
 	mod = tunables->lnd_map_on_demand;
 	nfrags = (negotiated_nfrags != -1) ? negotiated_nfrags : mod;
 
@@ -1540,7 +1543,10 @@ static int kiblnd_alloc_freg_pool(kib_fmr_poolset_t *fps, kib_fmr_pool_t *fpo)
 			goto out_middle;
 		}
 
-		frd->frd_valid = true;
+		/* There appears to be a bug in MLX5 code where you must
+		 * invalidate the rkey of a new FastReg pool before first
+		 * using it. Thus, I am marking the FRD invalid here. */
+		frd->frd_valid = false;
 
 		list_add_tail(&frd->frd_list, &fpo->fast_reg.fpo_pool_list);
 		fpo->fast_reg.fpo_pool_size++;
@@ -1876,8 +1882,13 @@ again:
 				}
 
 #ifdef HAVE_IB_MAP_MR_SG
+#ifdef HAVE_IB_MAP_MR_SG_5ARGS
+				n = ib_map_mr_sg(mr, tx->tx_frags,
+						 tx->tx_nfrags, NULL, PAGE_SIZE);
+#else
 				n = ib_map_mr_sg(mr, tx->tx_frags,
 						 tx->tx_nfrags, PAGE_SIZE);
+#endif
 				if (unlikely(n != tx->tx_nfrags)) {
 					CERROR("Failed to map mr %d/%d "
 					       "elements\n", n, tx->tx_nfrags);
@@ -2391,7 +2402,7 @@ kiblnd_net_init_pools(kib_net_t *net, lnet_ni_t *ni, __u32 *cpts, int ncpts)
 	int		rc;
 	int		i;
 
-	tunables = &ni->ni_lnd_tunables->lt_tun_u.lt_o2ib;
+	tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib;
 
 	read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
 	if (tunables->lnd_map_on_demand == 0) {
@@ -3172,8 +3183,9 @@ kiblnd_startup (lnet_ni_t *ni)
         unsigned long             flags;
         int                       rc;
 	int			  newdev;
+	int			  node_id;
 
-        LASSERT (ni->ni_lnd == &the_o2iblnd);
+        LASSERT (ni->ni_net->net_lnd == &the_o2iblnd);
 
         if (kiblnd_data.kib_init == IBLND_INIT_NOTHING) {
                 rc = kiblnd_base_startup();
@@ -3215,13 +3227,16 @@ kiblnd_startup (lnet_ni_t *ni)
 	newdev = ibdev == NULL;
 	/* hmm...create kib_dev even for alias */
 	if (ibdev == NULL || strcmp(&ibdev->ibd_ifname[0], ifname) != 0)
-                ibdev = kiblnd_create_dev(ifname);
+		ibdev = kiblnd_create_dev(ifname);
 
-        if (ibdev == NULL)
-                goto failed;
+	if (ibdev == NULL)
+		goto failed;
+
+	node_id = dev_to_node(ibdev->ibd_hdev->ibh_ibdev->dma_device);
+	ni->dev_cpt = cfs_cpt_of_node(lnet_cpt_table(), node_id);
 
-        net->ibn_dev = ibdev;
-        ni->ni_nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), ibdev->ibd_ifip);
+	net->ibn_dev = ibdev;
+	ni->ni_nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), ibdev->ibd_ifip);
 
 	rc = kiblnd_dev_start_threads(ibdev, newdev,
 				      ni->ni_cpts, ni->ni_ncpts);