From: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Date: Wed, 7 Jul 2021 19:16:01 +0000 (-0400)
Subject: LU-14733 o2iblnd: Avoid double posting invalidate
X-Git-Tag: 2.14.53~53
X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=5930576791e864529e6ef9b46f3e09cc4b635fc2

LU-14733 o2iblnd: Avoid double posting invalidate

When the kib_tx is provisioned during kiblnd_fmr_pool_map(), spare
WRs in the kib_fast_reg_descriptor are setup and the mapping of
pages is given to the mr.

kiblnd_post_tx_locked() then posts the spare WRs from the
kib_fast_reg_descriptor.

	if (rc == 0)
		return 0;

The code returns and the kib_fast_reg_descriptor is still contains
the spare WRs.   The next time the kib_tx is used, the
now obsolete WRs will be inadvertently posted.   For rdmavt, the
obsolete invalidate will cause an -EINVAL to be returned from
the post send.

Fix by adding a state variable frd_posted to the kib_fast_reg_descriptor.
The variable is set to false in kiblnd_fmr_pool_unmap().
kiblnd_post_tx_locked() is adjusted to avoid prepending the
kib_fast_reg_descriptor WRs when frd_posted is true.   After
the post succeeds, the frd_posted is set to true.

Signed-off-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Change-Id: I426dd05e635392e75d1aa48808782a229e83ce5f
Reviewed-on: https://review.whamcloud.com/44190
Reviewed-by: Amir Shehata <ashehata@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
---

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index f3028b4..0c44a61 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1751,6 +1751,7 @@ kiblnd_fmr_pool_unmap(struct kib_fmr *fmr, int status)
 
 		if (frd) {
 			frd->frd_valid = false;
+			frd->frd_posted = false;
 			fmr->fmr_frd = NULL;
 			spin_lock(&fps->fps_lock);
 			list_add_tail(&frd->frd_list, &fpo->fast_reg.fpo_pool_list);
@@ -1921,6 +1922,7 @@ again:
 				fmr->fmr_key  = is_rx ? mr->rkey : mr->lkey;
 				fmr->fmr_frd  = frd;
 				fmr->fmr_pool = fpo;
+				frd->frd_posted = false;
 				return 0;
 			}
 			spin_unlock(&fps->fps_lock);
diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index e2d05be..652c798 100644
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -348,6 +348,7 @@ struct kib_fast_reg_descriptor { /* For fast registration */
 #endif
 	struct ib_mr			*frd_mr;
 	bool				 frd_valid;
+	bool				 frd_posted;
 };
 
 struct kib_fmr_pool {
diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index b96e0e8..7189676 100644
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -811,6 +811,7 @@ __must_hold(&conn->ibc_lock)
 	struct kib_msg *msg = tx->tx_msg;
 	struct kib_peer_ni *peer_ni = conn->ibc_peer;
 	struct lnet_ni *ni = peer_ni->ibp_ni;
+	struct kib_fast_reg_descriptor *frd = tx->tx_fmr.fmr_frd;
 	int ver = conn->ibc_version;
 	int rc;
 	int done;
@@ -895,11 +896,10 @@ __must_hold(&conn->ibc_lock)
                 /* close_conn will launch failover */
                 rc = -ENETDOWN;
         } else {
-		struct kib_fast_reg_descriptor *frd = tx->tx_fmr.fmr_frd;
 		struct ib_send_wr *bad = &tx->tx_wrq[tx->tx_nwrq - 1].wr;
 		struct ib_send_wr *wr  = &tx->tx_wrq[0].wr;
 
-		if (frd != NULL) {
+		if (frd != NULL && !frd->frd_posted) {
 			if (!frd->frd_valid) {
 				wr = &frd->frd_inv_wr.wr;
 				wr->next = &frd->frd_fastreg_wr.wr;
@@ -928,8 +928,11 @@ __must_hold(&conn->ibc_lock)
 
 	conn->ibc_last_send = ktime_get();
 
-        if (rc == 0)
-                return 0;
+	if (rc == 0) {
+		if (frd != NULL)
+			frd->frd_posted = true;
+		return 0;
+	}
 
         /* NB credits are transferred in the actual
          * message, which can only be the last work item */