LU-16995 kfilnd: Handle TAG_RX_OK in TN_STATE_FAIL

It is possible for the fabric to delay packets such that the retry
handler cancels the message but it is still delivered to the target.
If the timing is right then the initiator may receive a TAG_RX_OK
event after the transaction has transitioned to TN_STATE_FAIL. This
currently trips an LBUG, but instead we can allow the transaction to
complete normally.

Test-Parameters: trivial
HPE-bug-id: LUS-11572
Signed-off-by: Chris Horn <chris.horn@hpe.com>
Change-Id: I381d64713a7942fed09d41b30f64be602193057f
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51787
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Ron Gredvig <ron.gredvig@hpe.com>
Reviewed-by: Ian Ziemba <ian.ziemba@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/klnds/kfilnd/kfilnd.h b/lnet/klnds/kfilnd/kfilnd.h
index edd99c0..35e63ed 100644 (file)
--- a/lnet/klnds/kfilnd/kfilnd.h
+++ b/lnet/klnds/kfilnd/kfilnd.h
@@ -103,6 +103,7 @@
 #define CFS_KFI_FAIL_MSG_TYPE 0xF114
 #define CFS_KFI_FAIL_WAIT_SEND_COMP1 0xF115
 #define CFS_KFI_FAIL_WAIT_SEND_COMP2 0xF116
+#define CFS_KFI_FAIL_WAIT_SEND_COMP3 0xF117
 
 /* Maximum number of transaction keys supported. */
 #define KFILND_EP_KEY_BITS 16U

diff --git a/lnet/klnds/kfilnd/kfilnd_tn.c b/lnet/klnds/kfilnd/kfilnd_tn.c
index a895579..7b5cf15 100644 (file)
--- a/lnet/klnds/kfilnd/kfilnd_tn.c
+++ b/lnet/klnds/kfilnd/kfilnd_tn.c
@@ -1090,6 +1090,15 @@ static int kfilnd_tn_state_wait_comp(struct kfilnd_transaction *tn,
                                         CFS_KFI_FAIL_WAIT_SEND_COMP2 |
                                         CFS_FAIL_ONCE))
                        break;
+               if (unlikely(tn->msg_type == KFILND_MSG_BULK_PUT_REQ ||
+                            tn->msg_type == KFILND_MSG_BULK_GET_REQ) &&
+                   CFS_FAIL_CHECK(CFS_KFI_FAIL_WAIT_SEND_COMP3)) {
+                       hstatus = LNET_MSG_STATUS_REMOTE_ERROR;
+                       kfilnd_tn_status_update(tn, -EIO, hstatus);
+                       kfilnd_peer_tn_failed(tn->tn_kp, -EIO);
+                       kfilnd_tn_state_change(tn, TN_STATE_FAIL);
+                       break;
+               }
                kfilnd_peer_alive(tn->tn_kp);
                kfilnd_tn_timeout_enable(tn);
                kfilnd_tn_state_change(tn, TN_STATE_WAIT_TAG_COMP);
@@ -1322,6 +1331,20 @@ static int kfilnd_tn_state_fail(struct kfilnd_transaction *tn,
                kfilnd_peer_alive(tn->tn_kp);
                break;
 
+       case TN_EVENT_TAG_RX_OK:
+               kfilnd_peer_alive(tn->tn_kp);
+               if (tn->tn_status != status) {
+                       KFILND_TN_DEBUG(tn, "%d -> %d status change",
+                                       tn->tn_status, status);
+                       tn->tn_status = status;
+               }
+               if (tn->hstatus != LNET_MSG_STATUS_OK) {
+                       KFILND_TN_DEBUG(tn, "%d -> %d health status change",
+                                       tn->hstatus, LNET_MSG_STATUS_OK);
+                       tn->hstatus = LNET_MSG_STATUS_OK;
+               }
+               break;
+
        case TN_EVENT_TAG_RX_FAIL:
        case TN_EVENT_TAG_RX_CANCEL:
                break;

diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh
index b634154..3074ed0 100755 (executable)
--- a/lustre/tests/sanity-lnet.sh
+++ b/lustre/tests/sanity-lnet.sh
@@ -3635,15 +3635,47 @@ test_311() {
 
        local list=$(comma_list $(osts_nodes))
 
-#define CFS_KFI_FAIL_WAIT_SEND_COMP 0xF115
+#define CFS_KFI_FAIL_WAIT_SEND_COMP1 0xF115
        do_nodes $list $LCTL set_param fail_loc=0x8000F115
        dd if=$DIR/$tdir/$tfile of=/dev/null bs=1M count=1 ||
                error "dd read failed"
 
+       rm -f $DIR/$tdir/$tfile
+       rmdir $DIR/$tdir
+
        cleanupall || error "Failed cleanup"
 }
 run_test 311 "Fail bulk put in send wait completion"
 
+test_312() {
+       [[ $NETTYPE == kfi* ]] ||
+               skip "Need kfi network type"
+
+       setupall || error "setupall failed"
+
+       mkdir -p $DIR/$tdir || error "mkdir failed"
+
+       local list=$(comma_list $(osts_nodes))
+
+#define CFS_KFI_FAIL_WAIT_SEND_COMP3 0xF117
+       do_nodes $list $LCTL set_param fail_loc=0x8000F117
+       dd if=/dev/zero of=$DIR/$tdir/$tfile bs=1M count=1 oflag=direct ||
+               error "dd write failed"
+
+       local tfile2="$DIR/$tdir/testfile2"
+
+       do_nodes $list $LCTL set_param fail_loc=0x8000F117
+       dd if=$DIR/$tdir/$tfile of=$tfile2 bs=1M count=1 oflag=direct ||
+               error "dd read failed"
+
+       rm -f $DIR/$tdir/$tfile
+       rm -f $tfile2
+       rmdir $DIR/$tdir
+
+       cleanupall || error "Failed cleanup"
+}
+run_test 312 "TAG_RX_OK is possible after TX_FAIL"
+
 check_udsp_prio() {
        local target_net="${1}"
        local target_nid="${2}"