From d730750a6311cae8a4427824867410faccc6698f Mon Sep 17 00:00:00 2001 From: Andriy Skulysh Date: Tue, 21 Oct 2014 16:53:11 -0400 Subject: [PATCH] LU-5651: ptlrpc: fix import state during replay Client doesn't restore import state correctly on reconnect during replay. It resends lock replay when final ping was queued by server. Server fails with "target_queue_recovery_request()) ASSERTION( req->rq_export->exp_lock_replay_needed ) failed" Add imp_replay_state to store last replay state. imp_state is restored from imp_replay_state during reconnect. Xyratex-bug-id: MRP-2022 Signed-off-by: Andriy Skulysh Change-Id: Iaa14fe968cc31f266b605785df4fa676083fbca4 Reviewed-on: http://review.whamcloud.com/12015 Reviewed-by: James Simmons Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Niu Yawei Reviewed-by: Oleg Drokin --- lustre/include/lustre_import.h | 2 ++ lustre/include/obd_support.h | 1 + lustre/ldlm/ldlm_lib.c | 1 + lustre/ptlrpc/import.c | 14 +++++++++++++- lustre/ptlrpc/service.c | 5 +++++ lustre/tests/replay-single.sh | 16 ++++++++++++++++ 6 files changed, 38 insertions(+), 1 deletion(-) diff --git a/lustre/include/lustre_import.h b/lustre/include/lustre_import.h index b5649cf..252157f 100644 --- a/lustre/include/lustre_import.h +++ b/lustre/include/lustre_import.h @@ -223,6 +223,8 @@ struct obd_import { atomic_t imp_timeouts; /** Current import state */ enum lustre_imp_state imp_state; + /** Last replay state */ + enum lustre_imp_state imp_replay_state; /** History of import states */ struct import_state_hist imp_state_hist[IMP_STATE_HIST_LEN]; int imp_state_hist_idx; diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 2f70856..d832503 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -426,6 +426,7 @@ int obd_alloc_fail(const void *ptr, const char *name, const char *type, #define OBD_FAIL_TGT_RCVG_FLAG 0x712 #define OBD_FAIL_TGT_DELAY_CONDITIONAL 0x713 #define OBD_FAIL_TGT_REPLAY_DELAY2 0x714 +#define OBD_FAIL_TGT_REPLAY_RECONNECT 0x715 #define OBD_FAIL_MDC_REVALIDATE_PAUSE 0x800 #define OBD_FAIL_MDC_ENQUEUE_PAUSE 0x801 diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index 0358433..a759378 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -2033,6 +2033,7 @@ static int target_recovery_thread(void *arg) * The third stage: reply on final pings, at this moment all clients * must have request in final queue */ + CFS_FAIL_TIMEOUT(OBD_FAIL_TGT_REPLAY_RECONNECT, cfs_fail_val); CDEBUG(D_INFO, "3: final stage - process recovery completion pings\n"); /** Update server last boot epoch */ tgt_boot_epoch_update(lut); diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c index 80ad967..af93a29 100644 --- a/lustre/ptlrpc/import.c +++ b/lustre/ptlrpc/import.c @@ -63,6 +63,18 @@ struct ptlrpc_connect_async_args { static void __import_set_state(struct obd_import *imp, enum lustre_imp_state state) { + switch (state) { + case LUSTRE_IMP_CLOSED: + case LUSTRE_IMP_NEW: + case LUSTRE_IMP_DISCON: + case LUSTRE_IMP_CONNECTING: + break; + case LUSTRE_IMP_REPLAY_WAIT: + imp->imp_replay_state = LUSTRE_IMP_REPLAY_LOCKS; + break; + default: + imp->imp_replay_state = LUSTRE_IMP_REPLAY; + } imp->imp_state = state; imp->imp_state_hist[imp->imp_state_hist_idx].ish_state = state; imp->imp_state_hist[imp->imp_state_hist_idx].ish_time = @@ -984,7 +996,7 @@ static int ptlrpc_connect_interpret(const struct lu_env *env, imp->imp_resend_replay = 1; spin_unlock(&imp->imp_lock); - IMPORT_SET_STATE(imp, LUSTRE_IMP_REPLAY); + IMPORT_SET_STATE(imp, imp->imp_replay_state); } else { IMPORT_SET_STATE(imp, LUSTRE_IMP_RECOVER); } diff --git a/lustre/ptlrpc/service.c b/lustre/ptlrpc/service.c index 706b042..8747321 100644 --- a/lustre/ptlrpc/service.c +++ b/lustre/ptlrpc/service.c @@ -1267,6 +1267,11 @@ static int ptlrpc_at_send_early_reply(struct ptlrpc_request *req) int rc; ENTRY; + if (CFS_FAIL_CHECK(OBD_FAIL_TGT_REPLAY_RECONNECT)) { + /* don't send early reply */ + RETURN(1); + } + /* deadline is when the client expects us to reply, margin is the difference between clients' and servers' expectations */ DEBUG_REQ(D_ADAPTTO, req, diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh index e0315fe..b1c2ab3 100755 --- a/lustre/tests/replay-single.sh +++ b/lustre/tests/replay-single.sh @@ -2818,6 +2818,22 @@ test_90() { # bug 19494 } run_test 90 "lfs find identifies the missing striped file segments" +test_93() { + cancel_lru_locks osc + + $SETSTRIPE -i 0 -c 1 $DIR/$tfile + dd if=/dev/zero of=$DIR/$tfile bs=1024 count=1 +#define OBD_FAIL_TGT_REPLAY_RECONNECT 0x715 + # We need to emulate a state that OST is waiting for other clients + # not completing the recovery. Final ping is queued, but reply will be sent + # on the recovery completion. It is done by sleep before processing final + # pings + do_facet ost1 "$LCTL set_param fail_val=40" + do_facet ost1 "$LCTL set_param fail_loc=0x715" + fail ost1 +} +run_test 93 "replay + reconnect" + striped_dir_check_100() { local striped_dir=$DIR/$tdir/striped_dir local stripe_count=$($LFS getdirstripe -c $striped_dir) -- 1.8.3.1