LU-9704 grant: ignore grant info on read resend

The following scenario makes a message like "claims 28672 GRANT, real
grant 0" to appear:

 1. client owns X grants and run rpcs to shrink part of those
 2. server fails over so that the shrink rpc is to be resent.
 3. on the clinet reconnect server and client sync on initial amount
 of grants for the client.
 4. shrink rpc is resend, if server disk space is enough, shrink does
 not happen and the client adds amount of grants it was going to
 shrink to its newly initial amount of grants. Now, client thinks that
 it owns more grants than it does from server points of view.
 5. the client consumes grants and sends rpcs to server. Server avoids
 allocating new grants for the client if the current amount of grant
 is big enough:
static long tgt_grant_alloc(struct obd_export *exp, u64 curgrant,
...
        if (curgrant >= want || curgrant >= ted->ted_grant + chunk)
                RETURN(0);
 6. client continues grants consuming which eventually leads to
 complains like "claims 28672 GRANT, real grant 0".

In case of resent of read and set_info:shrink RPCs grant info should
be ignored as it was reset on reconnect.

Tests to illustrate the issue is added.

HPE-bug-id: LUS-7666
Change-Id: I8af1db287dc61c713e5439f4cf6bd652ce02c12c
Signed-off-by: Vladimir Saveliev <vlaidimir.saveliev@hpe.com>
Reviewed-on: https://review.whamcloud.com/45371
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Mike Pershin <mpershin@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c
index 1261963..628e7c9 100644 (file)
--- a/lustre/ofd/ofd_dev.c
+++ b/lustre/ofd/ofd_dev.c
@@ -910,6 +910,19 @@ static int ofd_set_info_hdl(struct tgt_session_info *tsi)
        if (is_grant_shrink) {
                body = req_capsule_client_get(tsi->tsi_pill, &RMF_OST_BODY);
 
+               /*
+                * Because we already sync grant info with client when
+                * reconnect, grant info will be cleared for resent
+                * req, otherwise, outdated grant count in the rpc
+                * would de-sync grant counters
+                */
+               if (lustre_msg_get_flags(req->rq_reqmsg) &
+                   (MSG_RESENT | MSG_REPLAY)) {
+                       DEBUG_REQ(D_CACHE, req,
+                                 "clear resent/replay req grant info");
+                       body->oa.o_valid &= ~OBD_MD_FLGRANT;
+               }
+
                repbody = req_capsule_server_get(tsi->tsi_pill, &RMF_OST_BODY);
                *repbody = *body;
 

diff --git a/lustre/target/tgt_handler.c b/lustre/target/tgt_handler.c
index 1a903db..3f3a3b5 100644 (file)
--- a/lustre/target/tgt_handler.c
+++ b/lustre/target/tgt_handler.c
@@ -2340,6 +2340,17 @@ int tgt_brw_read(struct tgt_session_info *tsi)
                GOTO(out_lock, rc = -ETIMEDOUT);
        }
 
+       /*
+        * Because we already sync grant info with client when
+        * reconnect, grant info will be cleared for resent req,
+        * otherwise, outdated grant count in the rpc would de-sync
+        * grant counters in case of shrink
+        */
+       if (lustre_msg_get_flags(req->rq_reqmsg) & (MSG_RESENT | MSG_REPLAY)) {
+               DEBUG_REQ(D_CACHE, req, "clear resent/replay req grant info");
+               body->oa.o_valid &= ~OBD_MD_FLGRANT;
+       }
+
        repbody = req_capsule_server_get(&req->rq_pill, &RMF_OST_BODY);
        repbody->oa = body->oa;
 

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 73e68e8..0a81f05 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -8927,6 +8927,40 @@ test_64h() {
 }
 run_test 64h "grant shrink on read"
 
+test_64i() {
+       (( $OST1_VERSION >= $(version_code 2.14.55) )) ||
+               skip "need OST at least 2.14.55 to avoid grant shrink on replay"
+
+       [ $PARALLEL == "yes" ] && skip "skip parallel run"
+       remote_ost_nodsh && skip "remote OSTs with nodsh"
+
+       $LFS setstripe -c 1 -i 0 $DIR/$tfile
+
+       dd if=/dev/zero of=$DIR/$tfile bs=1M count=64
+
+       # lustre-ffff9fc75e850800 /mnt/lustre -> ffff9fc75e850800
+       local instance=$($LFS getname -i $DIR)
+
+       local osc_tgt="$FSNAME-OST0000-osc-$instance"
+       local cgb=$($LCTL get_param -n osc.$osc_tgt.cur_grant_bytes)
+
+       # shrink grants and simulate rpc loss
+       #define OBD_FAIL_PTLRPC_DROP_REQ_OPC     0x513
+       do_facet ost1 "$LCTL set_param fail_loc=0x80000513 fail_val=17"
+       $LCTL set_param osc.$osc_tgt.cur_grant_bytes=$((cgb/2))B
+
+       fail ost1
+
+       dd if=/dev/zero of=$DIR/$tfile oflag=append bs=1M count=8 conv=notrunc
+
+       local testid=$(echo $TESTNAME | tr '_' ' ')
+
+       do_facet ost1 dmesg | tac | sed "/$testid/,$ d" |
+               grep "GRANT, real grant" &&
+               error "client has more grants then it owns" || true
+}
+run_test 64i "shrink on reconnect"
+
 # bug 1414 - set/get directories' stripe info
 test_65a() {
        [ $PARALLEL == "yes" ] && skip "skip parallel run"