LU-14733 o2iblnd: Avoid double posting invalidate

When the kib_tx is provisioned during kiblnd_fmr_pool_map(), spare
WRs in the kib_fast_reg_descriptor are setup and the mapping of
pages is given to the mr.

kiblnd_post_tx_locked() then posts the spare WRs from the
kib_fast_reg_descriptor.

	if (rc == 0)
		return 0;

The code returns and the kib_fast_reg_descriptor is still contains
the spare WRs.   The next time the kib_tx is used, the
now obsolete WRs will be inadvertently posted.   For rdmavt, the
obsolete invalidate will cause an -EINVAL to be returned from
the post send.

Fix by adding a state variable frd_posted to the kib_fast_reg_descriptor.
The variable is set to false in kiblnd_fmr_pool_unmap().
kiblnd_post_tx_locked() is adjusted to avoid prepending the
kib_fast_reg_descriptor WRs when frd_posted is true.   After
the post succeeds, the frd_posted is set to true.

Signed-off-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Change-Id: I426dd05e635392e75d1aa48808782a229e83ce5f
Reviewed-on: https://review.whamcloud.com/44190
Reviewed-by: Amir Shehata <ashehata@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index f3028b4..0c44a61 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1751,6 +1751,7 @@ kiblnd_fmr_pool_unmap(struct kib_fmr *fmr, int status)
 
                if (frd) {
                        frd->frd_valid = false;
+                       frd->frd_posted = false;
                        fmr->fmr_frd = NULL;
                        spin_lock(&fps->fps_lock);
                        list_add_tail(&frd->frd_list, &fpo->fast_reg.fpo_pool_list);
@@ -1921,6 +1922,7 @@ again:
                                fmr->fmr_key  = is_rx ? mr->rkey : mr->lkey;
                                fmr->fmr_frd  = frd;
                                fmr->fmr_pool = fpo;
+                               frd->frd_posted = false;
                                return 0;
                        }
                        spin_unlock(&fps->fps_lock);

diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index e2d05be..652c798 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -348,6 +348,7 @@ struct kib_fast_reg_descriptor { /* For fast registration */
 #endif
        struct ib_mr                    *frd_mr;
        bool                             frd_valid;
+       bool                             frd_posted;
 };
 
 struct kib_fmr_pool {

diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c
index b96e0e8..7189676 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -811,6 +811,7 @@ __must_hold(&conn->ibc_lock)
        struct kib_msg *msg = tx->tx_msg;
        struct kib_peer_ni *peer_ni = conn->ibc_peer;
        struct lnet_ni *ni = peer_ni->ibp_ni;
+       struct kib_fast_reg_descriptor *frd = tx->tx_fmr.fmr_frd;
        int ver = conn->ibc_version;
        int rc;
        int done;
@@ -895,11 +896,10 @@ __must_hold(&conn->ibc_lock)
                 /* close_conn will launch failover */
                 rc = -ENETDOWN;
         } else {
-               struct kib_fast_reg_descriptor *frd = tx->tx_fmr.fmr_frd;
                struct ib_send_wr *bad = &tx->tx_wrq[tx->tx_nwrq - 1].wr;
                struct ib_send_wr *wr  = &tx->tx_wrq[0].wr;
 
-               if (frd != NULL) {
+               if (frd != NULL && !frd->frd_posted) {
                        if (!frd->frd_valid) {
                                wr = &frd->frd_inv_wr.wr;
                                wr->next = &frd->frd_fastreg_wr.wr;
@@ -928,8 +928,11 @@ __must_hold(&conn->ibc_lock)
 
        conn->ibc_last_send = ktime_get();
 
-        if (rc == 0)
-                return 0;
+       if (rc == 0) {
+               if (frd != NULL)
+                       frd->frd_posted = true;
+               return 0;
+       }
 
         /* NB credits are transferred in the actual
          * message, which can only be the last work item */