LU-5651: ptlrpc: fix import state during replay

Client doesn't restore import state correctly
on reconnect during replay. It resends lock replay
when final ping was queued by server.
Server fails with "target_queue_recovery_request())
ASSERTION( req->rq_export->exp_lock_replay_needed ) failed"

Add imp_replay_state to store last replay state.
imp_state is restored from imp_replay_state
during reconnect.

Xyratex-bug-id: MRP-2022
Signed-off-by: Andriy Skulysh <Andriy_Skulysh@xyratex.com>
Change-Id: Iaa14fe968cc31f266b605785df4fa676083fbca4
Reviewed-on: http://review.whamcloud.com/12015
Reviewed-by: James Simmons <uja.ornl@gmail.com>
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Niu Yawei <yawei.niu@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/include/lustre_import.h b/lustre/include/lustre_import.h
index b5649cf..252157f 100644 (file)
--- a/lustre/include/lustre_import.h
+++ b/lustre/include/lustre_import.h
@@ -223,6 +223,8 @@ struct obd_import {
        atomic_t                  imp_timeouts;
        /** Current import state */
         enum lustre_imp_state     imp_state;
+       /** Last replay state */
+       enum lustre_imp_state     imp_replay_state;
         /** History of import states */
         struct import_state_hist  imp_state_hist[IMP_STATE_HIST_LEN];
         int                       imp_state_hist_idx;

diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h
index 2f70856..d832503 100644 (file)
--- a/lustre/include/obd_support.h
+++ b/lustre/include/obd_support.h
@@ -426,6 +426,7 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type,
 #define OBD_FAIL_TGT_RCVG_FLAG           0x712
 #define OBD_FAIL_TGT_DELAY_CONDITIONAL  0x713
 #define OBD_FAIL_TGT_REPLAY_DELAY2       0x714
+#define OBD_FAIL_TGT_REPLAY_RECONNECT   0x715
 
 #define OBD_FAIL_MDC_REVALIDATE_PAUSE    0x800
 #define OBD_FAIL_MDC_ENQUEUE_PAUSE       0x801

diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c
index 0358433..a759378 100644 (file)
--- a/lustre/ldlm/ldlm_lib.c
+++ b/lustre/ldlm/ldlm_lib.c
@@ -2033,6 +2033,7 @@ static int target_recovery_thread(void *arg)
          * The third stage: reply on final pings, at this moment all clients
          * must have request in final queue
          */
+       CFS_FAIL_TIMEOUT(OBD_FAIL_TGT_REPLAY_RECONNECT, cfs_fail_val);
         CDEBUG(D_INFO, "3: final stage - process recovery completion pings\n");
         /** Update server last boot epoch */
         tgt_boot_epoch_update(lut);

diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c
index 80ad967..af93a29 100644 (file)
--- a/lustre/ptlrpc/import.c
+++ b/lustre/ptlrpc/import.c
@@ -63,6 +63,18 @@ struct ptlrpc_connect_async_args {
 static void __import_set_state(struct obd_import *imp,
                                enum lustre_imp_state state)
 {
+       switch (state) {
+       case LUSTRE_IMP_CLOSED:
+       case LUSTRE_IMP_NEW:
+       case LUSTRE_IMP_DISCON:
+       case LUSTRE_IMP_CONNECTING:
+               break;
+       case LUSTRE_IMP_REPLAY_WAIT:
+               imp->imp_replay_state = LUSTRE_IMP_REPLAY_LOCKS;
+               break;
+       default:
+               imp->imp_replay_state = LUSTRE_IMP_REPLAY;
+       }
         imp->imp_state = state;
         imp->imp_state_hist[imp->imp_state_hist_idx].ish_state = state;
         imp->imp_state_hist[imp->imp_state_hist_idx].ish_time =
@@ -984,7 +996,7 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
                        imp->imp_resend_replay = 1;
                        spin_unlock(&imp->imp_lock);
 
-                        IMPORT_SET_STATE(imp, LUSTRE_IMP_REPLAY);
+                       IMPORT_SET_STATE(imp, imp->imp_replay_state);
                 } else {
                         IMPORT_SET_STATE(imp, LUSTRE_IMP_RECOVER);
                 }

diff --git a/lustre/ptlrpc/service.c b/lustre/ptlrpc/service.c
index 706b042..8747321 100644 (file)
--- a/lustre/ptlrpc/service.c
+++ b/lustre/ptlrpc/service.c
@@ -1267,6 +1267,11 @@ static int ptlrpc_at_send_early_reply(struct ptlrpc_request *req)
         int rc;
         ENTRY;
 
+       if (CFS_FAIL_CHECK(OBD_FAIL_TGT_REPLAY_RECONNECT)) {
+               /* don't send early reply */
+               RETURN(1);
+       }
+
         /* deadline is when the client expects us to reply, margin is the
            difference between clients' and servers' expectations */
         DEBUG_REQ(D_ADAPTTO, req,

diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index e0315fe..b1c2ab3 100755 (executable)
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -2818,6 +2818,22 @@ test_90() { # bug 19494
 }
 run_test 90 "lfs find identifies the missing striped file segments"
 
+test_93() {
+    cancel_lru_locks osc
+
+    $SETSTRIPE -i 0 -c 1 $DIR/$tfile
+    dd if=/dev/zero of=$DIR/$tfile bs=1024 count=1
+#define OBD_FAIL_TGT_REPLAY_RECONNECT     0x715
+    # We need to emulate a state that OST is waiting for other clients
+    # not completing the recovery. Final ping is queued, but reply will be sent
+    # on the recovery completion. It is done by sleep before processing final
+    # pings
+    do_facet ost1 "$LCTL set_param fail_val=40"
+    do_facet ost1 "$LCTL set_param fail_loc=0x715"
+    fail ost1
+}
+run_test 93 "replay + reconnect"
+
 striped_dir_check_100() {
        local striped_dir=$DIR/$tdir/striped_dir
        local stripe_count=$($LFS getdirstripe -c $striped_dir)