From: Liang Zhen <liang@whamcloud.com>
Date: Mon, 5 Nov 2012 12:29:18 +0000 (+0800)
Subject: LU-2268 o2iblnd: initialize tx-pool after fmr-pool
X-Git-Tag: 2.3.57~39
X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=e97ce7c6bfe0bd5663840f8087e573a66e8d068f;p=fs%2Flustre-release.git

LU-2268 o2iblnd: initialize tx-pool after fmr-pool

TX pool should be created after FMR pool, otherwise
kiblnd_create_tx_pool() will not allocate tx_pages for
kib_tx_t even if FMR is enabled.

Signed-off-by: Liang Zhen <liang@whamcloud.com>
Change-Id: I30d3c53b4e29ac249500cf1f780619dcfaa59b10
Reviewed-on: http://review.whamcloud.com/4670
Tested-by: Hudson
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Isaac Huang <he.huang@intel.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
---

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 46f7007..9da9fa9 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -2238,33 +2238,12 @@ kiblnd_net_init_pools(kib_net_t *net, __u32 *cpts, int ncpts)
 	int		rc;
 	int		i;
 
-	net->ibn_tx_ps = cfs_percpt_alloc(lnet_cpt_table(),
-					  sizeof(kib_tx_poolset_t));
-	if (net->ibn_tx_ps == NULL) {
-		CERROR("Failed to allocate tx pool array\n");
-		return -ENOMEM;
-	}
-
-	for (i = 0; i < ncpts; i++) {
-		cpt = (cpts == NULL) ? i : cpts[i];
-		rc = kiblnd_init_poolset(&net->ibn_tx_ps[cpt]->tps_poolset,
-					 cpt, net, "TX",
-					 kiblnd_tx_pool_size(ncpts),
-					 kiblnd_create_tx_pool,
-					 kiblnd_destroy_tx_pool,
-					 kiblnd_tx_init, NULL);
-		if (rc != 0) {
-			CERROR("Failed to initialize TX pool\n");
-			goto failed;
-		}
-	}
-
 	cfs_read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
 	if (*kiblnd_tunables.kib_map_on_demand == 0 &&
 	    net->ibn_dev->ibd_hdev->ibh_nmrs == 1) {
 		cfs_read_unlock_irqrestore(&kiblnd_data.kib_global_lock,
 					   flags);
-		return 0;
+		goto create_tx_pool;
 	}
 
 	cfs_read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
@@ -2274,9 +2253,14 @@ kiblnd_net_init_pools(kib_net_t *net, __u32 *cpts, int ncpts)
 		CERROR("Can't set fmr pool size (%d) < ntx / 4(%d)\n",
 		       *kiblnd_tunables.kib_fmr_pool_size,
 		       *kiblnd_tunables.kib_ntx / 4);
+		rc = -EINVAL;
 		goto failed;
 	}
 
+	/* TX pool must be created later than FMR/PMR, see LU-2268
+	 * for details */
+	LASSERT(net->ibn_tx_ps == NULL);
+
 	/* premapping can fail if ibd_nmr > 1, so we always create
 	 * FMR/PMR pool and map-on-demand if premapping failed */
 
@@ -2284,6 +2268,7 @@ kiblnd_net_init_pools(kib_net_t *net, __u32 *cpts, int ncpts)
 					   sizeof(kib_fmr_poolset_t));
 	if (net->ibn_fmr_ps == NULL) {
 		CERROR("Failed to allocate FMR pool array\n");
+		rc = -ENOMEM;
 		goto failed;
 	}
 
@@ -2294,18 +2279,30 @@ kiblnd_net_init_pools(kib_net_t *net, __u32 *cpts, int ncpts)
 					     kiblnd_fmr_flush_trigger(ncpts));
 		if (rc == -ENOSYS && i == 0) /* no FMR */
 			break; /* create PMR pool */
-		if (rc != 0)
-			goto failed; /* a real error */
+
+		if (rc != 0) { /* a real error */
+			CERROR("Can't initialize FMR pool for CPT %d: %d\n",
+			       cpt, rc);
+			goto failed;
+		}
+	}
+
+	if (i > 0) {
+		LASSERT(i == ncpts);
+		goto create_tx_pool;
 	}
 
 	cfs_percpt_free(net->ibn_fmr_ps);
 	net->ibn_fmr_ps = NULL;
 
+	CWARN("Device does not support FMR, failing back to PMR\n");
+
 	if (*kiblnd_tunables.kib_pmr_pool_size <
 	    *kiblnd_tunables.kib_ntx / 4) {
 		CERROR("Can't set pmr pool size (%d) < ntx / 4(%d)\n",
 		       *kiblnd_tunables.kib_pmr_pool_size,
 		       *kiblnd_tunables.kib_ntx / 4);
+		rc = -EINVAL;
 		goto failed;
 	}
 
@@ -2313,6 +2310,7 @@ kiblnd_net_init_pools(kib_net_t *net, __u32 *cpts, int ncpts)
 					   sizeof(kib_pmr_poolset_t));
 	if (net->ibn_pmr_ps == NULL) {
 		CERROR("Failed to allocate PMR pool array\n");
+		rc = -ENOMEM;
 		goto failed;
 	}
 
@@ -2323,14 +2321,41 @@ kiblnd_net_init_pools(kib_net_t *net, __u32 *cpts, int ncpts)
 					 kiblnd_pmr_pool_size(ncpts),
 					 kiblnd_create_pmr_pool,
 					 kiblnd_destroy_pmr_pool, NULL, NULL);
-		if (rc != 0)
+		if (rc != 0) {
+			CERROR("Can't initialize PMR pool for CPT %d: %d\n",
+			       cpt, rc);
 			goto failed;
+		}
 	}
 
-	return 0;
+ create_tx_pool:
+	net->ibn_tx_ps = cfs_percpt_alloc(lnet_cpt_table(),
+					  sizeof(kib_tx_poolset_t));
+	if (net->ibn_tx_ps == NULL) {
+		CERROR("Failed to allocate tx pool array\n");
+		rc = -ENOMEM;
+		goto failed;
+	}
 
+	for (i = 0; i < ncpts; i++) {
+		cpt = (cpts == NULL) ? i : cpts[i];
+		rc = kiblnd_init_poolset(&net->ibn_tx_ps[cpt]->tps_poolset,
+					 cpt, net, "TX",
+					 kiblnd_tx_pool_size(ncpts),
+					 kiblnd_create_tx_pool,
+					 kiblnd_destroy_tx_pool,
+					 kiblnd_tx_init, NULL);
+		if (rc != 0) {
+			CERROR("Can't initialize TX pool for CPT %d: %d\n",
+			       cpt, rc);
+			goto failed;
+		}
+	}
+
+	return 0;
  failed:
 	kiblnd_net_fini_pools(net);
+	LASSERT(rc != 0);
 	return rc;
 }