From: Vladimir Saveliev Date: Fri, 17 Nov 2023 15:30:06 +0000 (+0300) Subject: LU-17297 grant: move tgt_grant_sanity_check() calls X-Git-Url: https://git.whamcloud.com/gitweb?a=commitdiff_plain;h=04f67c68540f044f2a1ccfcf6285de9538221747;p=fs%2Flustre-release.git LU-17297 grant: move tgt_grant_sanity_check() calls Call tgt_grant_sanity_check() in ofd_obd_disconnect() and in mdt_obd_disconnect() after call to tgt_grant_discard(). Otherwise, sum of grants does not match to total grant counter which is reported as LustreError: ofd_obd_disconnect: tot_granted 0 != fo_tot_granted 8388608 This is because on stale export eviction class_disconnect_stale_exports() moves stale exports to separate list but does not update obd's grant counters. Test to illustrate the issue is included. Lustre-change: https://review.whamcloud.com/53171 Lustre-commit: 9df01eee755bbac5bed560f365fab85c1b1164ae Test-Parameters: trivial testlist=recovery-small env=ONLY=156 serverversion=EXA5 Test-Parameters: trivial testlist=recovery-small env=ONLY=156 serverversion=2.15.4 HPE-bug-id: LUS-11469 Signed-off-by: Vladimir Saveliev Change-Id: I0b4568b88a2fe7b50f4eac50b4b064d7afbc7a75 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/54672 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h index 5548b01..7cadebc 100644 --- a/lustre/include/obd_support.h +++ b/lustre/include/obd_support.h @@ -476,6 +476,7 @@ extern char obd_jobid_var[]; #define OBD_FAIL_PTLRPC_IDLE_RACE 0x533 #define OBD_FAIL_PTLRPC_ENQ_RESEND 0x534 #define OBD_FAIL_PTLRPC_DELAY_SEND_FAIL 0x535 +#define OBD_FAIL_PTLRPC_REPLAY_PAUSE 0x536 #define OBD_FAIL_OBD_PING_NET 0x600 /* OBD_FAIL_OBD_LOG_CANCEL_NET 0x601 obsolete since 1.5 */ diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index c4a9145..c46efc0 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -6877,9 +6877,6 @@ static int mdt_obd_disconnect(struct obd_export *exp) LASSERT(exp); class_export_get(exp); - if (!(exp->exp_flags & OBD_OPT_FORCE)) - tgt_grant_sanity_check(exp->exp_obd, __func__); - if ((exp_connect_flags(exp) & OBD_CONNECT_MDS_MDS) && !(exp_connect_flags(exp) & OBD_CONNECT_LIGHTWEIGHT)) { struct mdt_device *mdt = mdt_dev(exp->exp_obd->obd_lu_dev); @@ -6894,6 +6891,9 @@ static int mdt_obd_disconnect(struct obd_export *exp) tgt_grant_discard(exp); + if (!(exp->exp_flags & OBD_OPT_FORCE)) + tgt_grant_sanity_check(exp->exp_obd, __func__); + rc = mdt_export_cleanup(exp); nodemap_del_member(exp); class_export_put(exp); diff --git a/lustre/ofd/ofd_obd.c b/lustre/ofd/ofd_obd.c index e2dffb0..4946267 100644 --- a/lustre/ofd/ofd_obd.c +++ b/lustre/ofd/ofd_obd.c @@ -428,13 +428,13 @@ int ofd_obd_disconnect(struct obd_export *exp) LASSERT(exp); class_export_get(exp); - if (!(exp->exp_flags & OBD_OPT_FORCE)) - tgt_grant_sanity_check(ofd_obd(ofd), __func__); - rc = server_disconnect_export(exp); tgt_grant_discard(exp); + if (!(exp->exp_flags & OBD_OPT_FORCE)) + tgt_grant_sanity_check(ofd_obd(ofd), __func__); + /* Do not erase record for recoverable client. */ if (exp->exp_obd->obd_replayable && (!exp->exp_obd->obd_fail || exp->exp_failed)) { diff --git a/lustre/ptlrpc/client.c b/lustre/ptlrpc/client.c index 8ee745c..46dfcf8 100644 --- a/lustre/ptlrpc/client.c +++ b/lustre/ptlrpc/client.c @@ -3366,6 +3366,8 @@ int ptlrpc_replay_req(struct ptlrpc_request *req) LASSERT(req->rq_import->imp_state == LUSTRE_IMP_REPLAY); + CFS_FAIL_TIMEOUT(OBD_FAIL_PTLRPC_REPLAY_PAUSE, cfs_fail_val); + aa = ptlrpc_req_async_args(aa, req); memset(aa, 0, sizeof(*aa)); diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh index 3b99bca..e3df682 100755 --- a/lustre/tests/recovery-small.sh +++ b/lustre/tests/recovery-small.sh @@ -569,7 +569,7 @@ test_18a() { local osc2dev=`lctl get_param -n devices | grep ${ost2_svc}-osc- | egrep -v 'MDT' | awk '{print $1}'` $LCTL --device $osc2dev deactivate || return 3 # my understanding is that there should be nothing in the page - # cache after the client reconnects? + # cache after the client reconnects? rc=0 pgcache_empty || rc=2 $LCTL --device $osc2dev activate @@ -603,7 +603,7 @@ test_18b() { # allow recovery to complete sleep $((TIMEOUT + 2)) # my understanding is that there should be nothing in the page - # cache after the client reconnects? + # cache after the client reconnects? rc=0 pgcache_empty || rc=2 rm -f $f $TMP/$tfile @@ -753,7 +753,7 @@ test_20a() { # bug 2983 - ldlm_handle_enqueue cleanup rc=$? [ $rc -eq 0 ] && error "multiop didn't fail enqueue: rc $rc" || true } -run_test 20a "ldlm_handle_enqueue error (should return error)" +run_test 20a "ldlm_handle_enqueue error (should return error)" test_20b() { # bug 2986 - ldlm_handle_enqueue error during open remote_ost_nodsh && skip "remote OST with nodsh" && return 0 @@ -965,7 +965,7 @@ run_test 21h "drop open request and close reply while close and open are both in test_22() { f1=$DIR/${tfile}-1 f2=$DIR/${tfile}-2 - + do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000115" $MULTIOP $f2 Oc & close_pid=$! @@ -1117,8 +1117,8 @@ test_26b() { # bug 10140 - evict dead exports by pinger # PING_INTERVAL max(obd_timeout / 4, 1U) # PING_EVICT_TIMEOUT (PING_INTERVAL * 6) - # evictor takes PING_EVICT_TIMEOUT + 3 * PING_INTERVAL to evict. - # But if there's a race to start the evictor from various obds, + # evictor takes PING_EVICT_TIMEOUT + 3 * PING_INTERVAL to evict. + # But if there's a race to start the evictor from various obds, # the loser might have to wait for the next ping. # = 9 * PING_INTERVAL + PING_INTERVAL # = 10 PING_INTERVAL = 10 obd_timeout / 4 = 2.5 obd_timeout @@ -1146,7 +1146,7 @@ test_27() { facet_failover $SINGLEMDS #no crashes allowed! kill -USR1 $CLIENT_PID - wait $CLIENT_PID + wait $CLIENT_PID true FAILURE_MODE=$save_FAILURE_MODE } @@ -1203,7 +1203,7 @@ test_50() { # client process should see no problems even though MDS went down sleep $TIMEOUT kill -USR1 $CLIENT_PID - wait $CLIENT_PID + wait $CLIENT_PID rc=$? echo writemany returned $rc #these may fail because of eviction due to slow AST response. @@ -1384,7 +1384,7 @@ test_56() { # b=11277 run_test 56 "do not fail on getattr resend" test_57_helper() { - # no oscs means no client or mdt + # no oscs means no client or mdt while lctl get_param osc.*.* > /dev/null 2>&1; do : # loop until proc file is removed done @@ -1500,7 +1500,7 @@ test_61() $LFS setstripe -c 1 -i 0 $DIR/$tdir replay_barrier $SINGLEMDS - createmany -o $DIR/$tdir/$tfile-%d 10 + createmany -o $DIR/$tdir/$tfile-%d 10 local oid=$(do_facet ost1 "lctl get_param -n \ obdfilter.${ost1_svc}.last_id" | sed -e 's/.*://') @@ -3441,6 +3441,57 @@ test_154b() { } run_test 154b "restore update llog after failed recovery" +test_156() +{ + (( OST1_VERSION >= $(version_code v2_15_60-90-g9df01eee75) )) || + skip "Need OST version >= 2.15.60.90 for tgt_granted miscount fix" + (( OST1_VERSION < $(version_code 2.15.0) && + OST1_VERSION >= $(version_code 2.14.0-ddn141) )) || + skip "Need OST version < 2.15.0 && version >= 2.14.0-ddn141 \ + for tgt_granted miscount fix" + + # on failover recovery time hard will be 9 * 5 + local saved_timeout=$(do_facet ost1 $LCTL get_param -n timeout) + + do_facet mgs $LCTL set_param -P timeout=5 || + error "failed to set obd_timeout" + stack_trap "do_facet mgs $LCTL set_param -P timeout=$saved_timeout" \ + EXIT + + $LFS setstripe -c 1 -i 0 $DIR/$tfile || error "setstripe failed" + + # this is to sync last_rcvd, so that the client will have to + # send replay on recovery + $LFS df $MOUNT + do_facet ost1 sync + + replay_barrier ost1 + + $MULTIOP $DIR/$tfile oO_RDWR:O_SYNC:w1048576c || error "multiop failed" + + # delay write replay for 45 sec (OBD_RECOVERY_TIME_HARD) to + # get the client evicted as not sending replays + +#define OBD_FAIL_PTLRPC_REPLAY_PAUSE 0x536 + $LCTL set_param fail_loc=0x80000536 fail_val=45 + + fail ost1 + + # check that ost1 evicted the client in recovery + local clients + clients=($(do_facet ost1 \ + $LCTL get_param -n obdfilter.$FSNAME-OST0000.recovery_status | + awk '/completed_clients/ { print $2 }' | tr '/' '\n')) + [[ $((${clients[0]} + 1)) == ${clients[1]} ]] || + error "client not evicted by ost1" + + local testid=$(echo $TESTNAME | tr '_' ' ') + do_facet ost1 dmesg | tac | sed "/$testid/,$ d" | + grep "ofd_obd_disconnect: tot_granted" && + error "grant miscount" || true +} +run_test 156 "tot_granted miscount after client eviction" + complete_test $SECONDS check_and_cleanup_lustre exit_status