From 38c78ac2e390b30106f3e185d8c4d92b8cb19c2b Mon Sep 17 00:00:00 2001 From: Vladimir Saveliev Date: Wed, 3 Nov 2021 13:52:14 +0300 Subject: [PATCH] LU-9704 grant: ignore grant info on read resend The following scenario makes a message like "claims 28672 GRANT, real grant 0" to appear: 1. client owns X grants and run rpcs to shrink part of those 2. server fails over so that the shrink rpc is to be resent. 3. on the clinet reconnect server and client sync on initial amount of grants for the client. 4. shrink rpc is resend, if server disk space is enough, shrink does not happen and the client adds amount of grants it was going to shrink to its newly initial amount of grants. Now, client thinks that it owns more grants than it does from server points of view. 5. the client consumes grants and sends rpcs to server. Server avoids allocating new grants for the client if the current amount of grant is big enough: static long tgt_grant_alloc(struct obd_export *exp, u64 curgrant, ... if (curgrant >= want || curgrant >= ted->ted_grant + chunk) RETURN(0); 6. client continues grants consuming which eventually leads to complains like "claims 28672 GRANT, real grant 0". In case of resent of read and set_info:shrink RPCs grant info should be ignored as it was reset on reconnect. Tests to illustrate the issue is added. HPE-bug-id: LUS-7666 Change-Id: I8af1db287dc61c713e5439f4cf6bd652ce02c12c Signed-off-by: Vladimir Saveliev Reviewed-on: https://review.whamcloud.com/45371 Reviewed-by: Andreas Dilger Tested-by: jenkins Tested-by: Maloo Reviewed-by: Mike Pershin Reviewed-by: Oleg Drokin --- lustre/ofd/ofd_dev.c | 13 +++++++++++++ lustre/target/tgt_handler.c | 11 +++++++++++ lustre/tests/sanity.sh | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c index 1261963..628e7c9 100644 --- a/lustre/ofd/ofd_dev.c +++ b/lustre/ofd/ofd_dev.c @@ -910,6 +910,19 @@ static int ofd_set_info_hdl(struct tgt_session_info *tsi) if (is_grant_shrink) { body = req_capsule_client_get(tsi->tsi_pill, &RMF_OST_BODY); + /* + * Because we already sync grant info with client when + * reconnect, grant info will be cleared for resent + * req, otherwise, outdated grant count in the rpc + * would de-sync grant counters + */ + if (lustre_msg_get_flags(req->rq_reqmsg) & + (MSG_RESENT | MSG_REPLAY)) { + DEBUG_REQ(D_CACHE, req, + "clear resent/replay req grant info"); + body->oa.o_valid &= ~OBD_MD_FLGRANT; + } + repbody = req_capsule_server_get(tsi->tsi_pill, &RMF_OST_BODY); *repbody = *body; diff --git a/lustre/target/tgt_handler.c b/lustre/target/tgt_handler.c index 1a903db..3f3a3b5 100644 --- a/lustre/target/tgt_handler.c +++ b/lustre/target/tgt_handler.c @@ -2340,6 +2340,17 @@ int tgt_brw_read(struct tgt_session_info *tsi) GOTO(out_lock, rc = -ETIMEDOUT); } + /* + * Because we already sync grant info with client when + * reconnect, grant info will be cleared for resent req, + * otherwise, outdated grant count in the rpc would de-sync + * grant counters in case of shrink + */ + if (lustre_msg_get_flags(req->rq_reqmsg) & (MSG_RESENT | MSG_REPLAY)) { + DEBUG_REQ(D_CACHE, req, "clear resent/replay req grant info"); + body->oa.o_valid &= ~OBD_MD_FLGRANT; + } + repbody = req_capsule_server_get(&req->rq_pill, &RMF_OST_BODY); repbody->oa = body->oa; diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 73e68e8..0a81f05 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -8927,6 +8927,40 @@ test_64h() { } run_test 64h "grant shrink on read" +test_64i() { + (( $OST1_VERSION >= $(version_code 2.14.55) )) || + skip "need OST at least 2.14.55 to avoid grant shrink on replay" + + [ $PARALLEL == "yes" ] && skip "skip parallel run" + remote_ost_nodsh && skip "remote OSTs with nodsh" + + $LFS setstripe -c 1 -i 0 $DIR/$tfile + + dd if=/dev/zero of=$DIR/$tfile bs=1M count=64 + + # lustre-ffff9fc75e850800 /mnt/lustre -> ffff9fc75e850800 + local instance=$($LFS getname -i $DIR) + + local osc_tgt="$FSNAME-OST0000-osc-$instance" + local cgb=$($LCTL get_param -n osc.$osc_tgt.cur_grant_bytes) + + # shrink grants and simulate rpc loss + #define OBD_FAIL_PTLRPC_DROP_REQ_OPC 0x513 + do_facet ost1 "$LCTL set_param fail_loc=0x80000513 fail_val=17" + $LCTL set_param osc.$osc_tgt.cur_grant_bytes=$((cgb/2))B + + fail ost1 + + dd if=/dev/zero of=$DIR/$tfile oflag=append bs=1M count=8 conv=notrunc + + local testid=$(echo $TESTNAME | tr '_' ' ') + + do_facet ost1 dmesg | tac | sed "/$testid/,$ d" | + grep "GRANT, real grant" && + error "client has more grants then it owns" || true +} +run_test 64i "shrink on reconnect" + # bug 1414 - set/get directories' stripe info test_65a() { [ $PARALLEL == "yes" ] && skip "skip parallel run" -- 1.8.3.1