From 7ca495ec67f474e10352077fc40123e4818b8e69 Mon Sep 17 00:00:00 2001 From: Oleg Drokin Date: Tue, 13 Oct 2020 23:55:02 -0400 Subject: [PATCH] LU-14027 ldlm: Do not hang if recovery restarted during lock replay LU-13600 introduced lock ratelimiting logic, but it did not take into account that if there's a disconnection in the REPLAY_LOCKS phase then yet unsent locks get stuck in the sending queue so the replay locks thread hangs with imp_replay_inflight elevated above zero. The direct consequence from that is recovery state machine never advances from REPLAY to REPLAY_LOCKS status when imp_replay_inflight is non zero. Adjust __ldlm_replay_locks() to check if the import state changed before attempting to send any more requests. Add a testcase. Change-Id: Idbaf5461f33d1884088269d67d01071c7e1bf8a5 Signed-off-by: Oleg Drokin Fixes: 3b613a442b ("LU-13600 ptlrpc: limit rate of lock replays") Reviewed-on: https://review.whamcloud.com/40238 Reviewed-by: Mike Pershin Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- lustre/include/obd_support.h | 1 + lustre/ldlm/ldlm_lib.c | 5 +++++ lustre/ldlm/ldlm_request.c | 6 ++++-- lustre/tests/replay-single.sh | 45 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 2 deletions(-) diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 3edec56..462ead5 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -388,6 +388,7 @@ extern char obd_jobid_var[]; #define OBD_FAIL_LDLM_GRANT_CHECK 0x32a #define OBD_FAIL_LDLM_PROLONG_PAUSE 0x32b #define OBD_FAIL_LDLM_LOCAL_CANCEL_PAUSE 0x32c +#define OBD_FAIL_LDLM_LOCK_REPLAY 0x32d /* LOCKLESS IO */ #define OBD_FAIL_LDLM_SET_CONTENTION 0x385 diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index baa75f5..a3386bc 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -2750,6 +2750,11 @@ static int target_recovery_thread(void *arg) LASSERT(trd->trd_processing_task == current->pid); DEBUG_REQ(D_HA, req, "processing lock from %s:", libcfs_nid2str(req->rq_peer.nid)); + if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_LOCK_REPLAY)) { + req->rq_status = -ENODEV; + target_request_copy_put(req); + continue; + } handle_recovery_req(thread, req, trd->trd_recovery_handler); target_request_copy_put(req); diff --git a/lustre/ldlm/ldlm_request.c b/lustre/ldlm/ldlm_request.c index 46b3751..baccc46 100644 --- a/lustre/ldlm/ldlm_request.c +++ b/lustre/ldlm/ldlm_request.c @@ -2580,9 +2580,11 @@ int __ldlm_replay_locks(struct obd_import *imp, bool rate_limit) list_for_each_entry_safe(lock, next, &list, l_pending_chain) { list_del_init(&lock->l_pending_chain); - if (rc) { + /* If we disconnected in the middle - cleanup and let + * reconnection to happen again. LU-14027 */ + if (rc || (imp->imp_state != LUSTRE_IMP_REPLAY_LOCKS)) { LDLM_LOCK_RELEASE(lock); - continue; /* or try to do the rest? */ + continue; } rc = replay_one_lock(imp, lock); LDLM_LOCK_RELEASE(lock); diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh index 5eac2d7..9c12920 100755 --- a/lustre/tests/replay-single.sh +++ b/lustre/tests/replay-single.sh @@ -4939,6 +4939,51 @@ test_134() { } run_test 134 "replay creation of a file created in a pool" +# LU-14027 +test_135() { + mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed" + + # All files to ost1 + $LFS setstripe -S $((128 * 1024)) -i 0 $DIR/$tdir + + replay_barrier ost1 + + # Create 20 files so we have 20 ost locks + for i in $(seq 20) ; do + echo blah > $DIR/$tdir/file.${i} + done + + shutdown_facet ost1 + reboot_facet ost1 + change_active ost1 + wait_for_facet ost1 + + #define OBD_FAIL_TGT_REPLAY_RECONNECT 0x32d + # Make sure lock replay server side never completes and errors out. + do_facet ost1 "$LCTL set_param fail_val=20" + do_facet ost1 "$LCTL set_param fail_loc=0x32d" + + mount_facet ost1 + + # Now make sure we notice + (sync;sync;sync) & + local PID=$? + sleep 20 # should we do something proactive to make reconnects go? + kill -0 $PID || error "Unexpected sync success" + + shutdown_facet ost1 + reboot_facet ost1 + change_active ost1 + wait_for_facet ost1 + + do_facet ost1 "$LCTL set_param fail_loc=0" + mount_facet ost1 + echo blah > $DIR/$tdir/file.test2 + + rm -rf $DIR/$tdir +} +run_test 135 "Server failure in lock replay phase" + complete $SECONDS check_and_cleanup_lustre exit_status -- 1.8.3.1