LU-17297 grant: move tgt_grant_sanity_check() calls

Call tgt_grant_sanity_check() in ofd_obd_disconnect() and in
mdt_obd_disconnect() after call to tgt_grant_discard().

Otherwise, sum of grants does not match to total grant counter which
is reported as LustreError:
    ofd_obd_disconnect: tot_granted 0 != fo_tot_granted 8388608

This is because on stale export eviction
class_disconnect_stale_exports() moves stale exports to separate list
but does not update obd's grant counters.

Test to illustrate the issue is included.

Lustre-change: https://review.whamcloud.com/53171
Lustre-commit: 9df01eee755bbac5bed560f365fab85c1b1164ae

Test-Parameters: trivial testlist=recovery-small env=ONLY=156 serverversion=EXA5
Test-Parameters: trivial testlist=recovery-small env=ONLY=156 serverversion=2.15.4
HPE-bug-id: LUS-11469
Signed-off-by: Vladimir Saveliev <vladimir.saveliev@hpe.com>
Change-Id: I0b4568b88a2fe7b50f4eac50b4b064d7afbc7a75
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/54672
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h
index 5548b01..7cadebc 100644 (file)
--- a/lustre/include/obd_support.h
+++ b/lustre/include/obd_support.h
@@ -476,6 +476,7 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_PTLRPC_IDLE_RACE       0x533
 #define OBD_FAIL_PTLRPC_ENQ_RESEND      0x534
 #define OBD_FAIL_PTLRPC_DELAY_SEND_FAIL         0x535
+#define OBD_FAIL_PTLRPC_REPLAY_PAUSE    0x536
 
 #define OBD_FAIL_OBD_PING_NET            0x600
 /*     OBD_FAIL_OBD_LOG_CANCEL_NET      0x601 obsolete since 1.5 */

diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index c4a9145..c46efc0 100644 (file)
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -6877,9 +6877,6 @@ static int mdt_obd_disconnect(struct obd_export *exp)
        LASSERT(exp);
        class_export_get(exp);
 
-       if (!(exp->exp_flags & OBD_OPT_FORCE))
-               tgt_grant_sanity_check(exp->exp_obd, __func__);
-
        if ((exp_connect_flags(exp) & OBD_CONNECT_MDS_MDS) &&
            !(exp_connect_flags(exp) & OBD_CONNECT_LIGHTWEIGHT)) {
                struct mdt_device *mdt = mdt_dev(exp->exp_obd->obd_lu_dev);
@@ -6894,6 +6891,9 @@ static int mdt_obd_disconnect(struct obd_export *exp)
 
        tgt_grant_discard(exp);
 
+       if (!(exp->exp_flags & OBD_OPT_FORCE))
+               tgt_grant_sanity_check(exp->exp_obd, __func__);
+
        rc = mdt_export_cleanup(exp);
        nodemap_del_member(exp);
        class_export_put(exp);

diff --git a/lustre/ofd/ofd_obd.c b/lustre/ofd/ofd_obd.c
index e2dffb0..4946267 100644 (file)
--- a/lustre/ofd/ofd_obd.c
+++ b/lustre/ofd/ofd_obd.c
@@ -428,13 +428,13 @@ int ofd_obd_disconnect(struct obd_export *exp)
        LASSERT(exp);
        class_export_get(exp);
 
-       if (!(exp->exp_flags & OBD_OPT_FORCE))
-               tgt_grant_sanity_check(ofd_obd(ofd), __func__);
-
        rc = server_disconnect_export(exp);
 
        tgt_grant_discard(exp);
 
+       if (!(exp->exp_flags & OBD_OPT_FORCE))
+               tgt_grant_sanity_check(ofd_obd(ofd), __func__);
+
        /* Do not erase record for recoverable client. */
        if (exp->exp_obd->obd_replayable &&
            (!exp->exp_obd->obd_fail || exp->exp_failed)) {

diff --git a/lustre/ptlrpc/client.c b/lustre/ptlrpc/client.c
index 8ee745c..46dfcf8 100644 (file)
--- a/lustre/ptlrpc/client.c
+++ b/lustre/ptlrpc/client.c
@@ -3366,6 +3366,8 @@ int ptlrpc_replay_req(struct ptlrpc_request *req)
 
        LASSERT(req->rq_import->imp_state == LUSTRE_IMP_REPLAY);
 
+       CFS_FAIL_TIMEOUT(OBD_FAIL_PTLRPC_REPLAY_PAUSE, cfs_fail_val);
+
        aa = ptlrpc_req_async_args(aa, req);
        memset(aa, 0, sizeof(*aa));
 

diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 3b99bca..e3df682 100755 (executable)
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -569,7 +569,7 @@ test_18a() {
     local osc2dev=`lctl get_param -n devices | grep ${ost2_svc}-osc- | egrep -v 'MDT' | awk '{print $1}'`
     $LCTL --device $osc2dev deactivate || return 3
     # my understanding is that there should be nothing in the page
-    # cache after the client reconnects?     
+    # cache after the client reconnects?
     rc=0
     pgcache_empty || rc=2
     $LCTL --device $osc2dev activate
@@ -603,7 +603,7 @@ test_18b() {
     # allow recovery to complete
     sleep $((TIMEOUT + 2))
     # my understanding is that there should be nothing in the page
-    # cache after the client reconnects?     
+    # cache after the client reconnects?
     rc=0
     pgcache_empty || rc=2
     rm -f $f $TMP/$tfile
@@ -753,7 +753,7 @@ test_20a() {        # bug 2983 - ldlm_handle_enqueue cleanup
        rc=$?
        [ $rc -eq 0 ] && error "multiop didn't fail enqueue: rc $rc" || true
 }
-run_test 20a "ldlm_handle_enqueue error (should return error)" 
+run_test 20a "ldlm_handle_enqueue error (should return error)"
 
 test_20b() {   # bug 2986 - ldlm_handle_enqueue error during open
        remote_ost_nodsh && skip "remote OST with nodsh" && return 0
@@ -965,7 +965,7 @@ run_test 21h "drop open request and close reply while close and open are both in
 test_22() {
     f1=$DIR/${tfile}-1
     f2=$DIR/${tfile}-2
-    
+
     do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000115"
     $MULTIOP $f2 Oc &
     close_pid=$!
@@ -1117,8 +1117,8 @@ test_26b() {      # bug 10140 - evict dead exports by pinger
        # PING_INTERVAL max(obd_timeout / 4, 1U)
        # PING_EVICT_TIMEOUT (PING_INTERVAL * 6)
 
-       # evictor takes PING_EVICT_TIMEOUT + 3 * PING_INTERVAL to evict.  
-       # But if there's a race to start the evictor from various obds, 
+       # evictor takes PING_EVICT_TIMEOUT + 3 * PING_INTERVAL to evict.
+       # But if there's a race to start the evictor from various obds,
        # the loser might have to wait for the next ping.
        # = 9 * PING_INTERVAL + PING_INTERVAL
        # = 10 PING_INTERVAL = 10 obd_timeout / 4 = 2.5 obd_timeout
@@ -1146,7 +1146,7 @@ test_27() {
        facet_failover $SINGLEMDS
        #no crashes allowed!
         kill -USR1 $CLIENT_PID
-       wait $CLIENT_PID 
+       wait $CLIENT_PID
        true
        FAILURE_MODE=$save_FAILURE_MODE
 }
@@ -1203,7 +1203,7 @@ test_50() {
        # client process should see no problems even though MDS went down
        sleep $TIMEOUT
         kill -USR1 $CLIENT_PID
-       wait $CLIENT_PID 
+       wait $CLIENT_PID
        rc=$?
        echo writemany returned $rc
        #these may fail because of eviction due to slow AST response.
@@ -1384,7 +1384,7 @@ test_56() { # b=11277
 run_test 56 "do not fail on getattr resend"
 
 test_57_helper() {
-        # no oscs means no client or mdt 
+        # no oscs means no client or mdt
         while lctl get_param osc.*.* > /dev/null 2>&1; do
                 : # loop until proc file is removed
         done
@@ -1500,7 +1500,7 @@ test_61()
        $LFS setstripe -c 1 -i 0 $DIR/$tdir
 
        replay_barrier $SINGLEMDS
-       createmany -o $DIR/$tdir/$tfile-%d 10 
+       createmany -o $DIR/$tdir/$tfile-%d 10
        local oid=$(do_facet ost1 "lctl get_param -n \
                obdfilter.${ost1_svc}.last_id" | sed -e 's/.*://')
 
@@ -3441,6 +3441,57 @@ test_154b() {
 }
 run_test 154b "restore update llog after failed recovery"
 
+test_156()
+{
+       (( OST1_VERSION >= $(version_code v2_15_60-90-g9df01eee75) )) ||
+               skip "Need OST version >= 2.15.60.90 for tgt_granted miscount fix"
+       (( OST1_VERSION < $(version_code 2.15.0) &&
+          OST1_VERSION >= $(version_code 2.14.0-ddn141) )) ||
+               skip "Need OST version < 2.15.0 && version >= 2.14.0-ddn141 \
+                     for tgt_granted miscount fix"
+
+       # on failover recovery time hard will be 9 * 5
+       local saved_timeout=$(do_facet ost1 $LCTL get_param -n timeout)
+
+       do_facet mgs $LCTL set_param -P timeout=5 ||
+               error "failed to set obd_timeout"
+       stack_trap "do_facet mgs $LCTL set_param -P timeout=$saved_timeout" \
+           EXIT
+
+       $LFS setstripe -c 1 -i 0 $DIR/$tfile || error "setstripe failed"
+
+       # this is to sync last_rcvd, so that the client will have to
+       # send replay on recovery
+       $LFS df $MOUNT
+       do_facet ost1 sync
+
+       replay_barrier ost1
+
+       $MULTIOP $DIR/$tfile oO_RDWR:O_SYNC:w1048576c || error "multiop failed"
+
+       # delay write replay for 45 sec (OBD_RECOVERY_TIME_HARD) to
+       # get the client evicted as not sending replays
+
+#define OBD_FAIL_PTLRPC_REPLAY_PAUSE    0x536
+       $LCTL set_param fail_loc=0x80000536 fail_val=45
+
+       fail ost1
+
+       # check that ost1 evicted the client in recovery
+       local clients
+       clients=($(do_facet ost1 \
+               $LCTL get_param -n obdfilter.$FSNAME-OST0000.recovery_status |
+               awk '/completed_clients/ { print $2 }' | tr '/' '\n'))
+       [[ $((${clients[0]} + 1)) == ${clients[1]} ]] ||
+               error "client not evicted by ost1"
+
+       local testid=$(echo $TESTNAME | tr '_' ' ')
+       do_facet ost1 dmesg | tac | sed "/$testid/,$ d" |
+               grep "ofd_obd_disconnect: tot_granted" &&
+               error "grant miscount" || true
+}
+run_test 156 "tot_granted miscount after client eviction"
+
 complete_test $SECONDS
 check_and_cleanup_lustre
 exit_status