From 655d5e708e01a77368459a1dd9c34fab5b6838b1 Mon Sep 17 00:00:00 2001 From: Vitaly Fertman Date: Wed, 15 Dec 2010 02:07:51 +0300 Subject: [PATCH] b=22755 Don't consume grant twice on recoverable resend o=johann i=green i=tappro --- lustre/include/lustre/lustre_idl.h | 1 + lustre/obdfilter/filter_io.c | 15 ++++++++- lustre/osc/lproc_osc.c | 14 +++++++++ lustre/osc/osc_request.c | 17 ++++++++--- lustre/ptlrpc/wiretest.c | 1 + lustre/tests/test-framework.sh | 62 ++++++++++++++++---------------------- lustre/utils/wirecheck.c | 1 + lustre/utils/wiretest.c | 1 + 8 files changed, 71 insertions(+), 41 deletions(-) diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h index afdfb25..0ea1fdb 100644 --- a/lustre/include/lustre/lustre_idl.h +++ b/lustre/include/lustre/lustre_idl.h @@ -1194,6 +1194,7 @@ enum obdo_flags { OBD_FL_CKSUM_RSVD3 = 0x00010000, /* for future cksum types */ OBD_FL_SHRINK_GRANT = 0x00020000, /* object shrink the grant */ OBD_FL_MMAP = 0x00040000, /* object is mmapped on the client */ + OBD_FL_RECOV_RESEND = 0x00080000, /* recoverable resent */ OBD_FL_CKSUM_ALL = OBD_FL_CKSUM_CRC32 | OBD_FL_CKSUM_ADLER, diff --git a/lustre/obdfilter/filter_io.c b/lustre/obdfilter/filter_io.c index 268dec1..859e290 100644 --- a/lustre/obdfilter/filter_io.c +++ b/lustre/obdfilter/filter_io.c @@ -530,6 +530,14 @@ static int filter_grant_check(struct obd_export *exp, struct obdo *oa, int blocksize = exp->exp_obd->u.obt.obt_sb->s_blocksize; unsigned long used = 0, ungranted = 0, using; int i, rc = -ENOSPC, obj, n = 0; + int resend = 0; + + if ((oa->o_valid & OBD_MD_FLFLAGS) && + (oa->o_flags & OBD_FL_RECOV_RESEND)) { + resend = 1; + CDEBUG(D_CACHE, "Recoverable resend arrived, skipping " + "accounting\n"); + } LASSERT_SPIN_LOCKED(&exp->exp_obd->obd_osfs_lock); @@ -546,7 +554,12 @@ static int filter_grant_check(struct obd_export *exp, struct obdo *oa, if ((lnb[n].flags & OBD_BRW_FROM_GRANT) && (oa->o_valid & OBD_MD_FLGRANT)) { - if (fed->fed_grant < used + bytes) { + if (resend) { + /* this is a recoverable resent */ + lnb[n].flags |= OBD_BRW_GRANTED; + rc = 0; + continue; + } else if (fed->fed_grant < used + bytes) { CDEBUG(D_CACHE, "%s: cli %s/%p claims %ld+%d " "GRANT, real grant %lu idx %d\n", diff --git a/lustre/osc/lproc_osc.c b/lustre/osc/lproc_osc.c index 086c94b..237200b 100644 --- a/lustre/osc/lproc_osc.c +++ b/lustre/osc/lproc_osc.c @@ -256,6 +256,19 @@ static int osc_wr_cur_grant_bytes(struct file *file, const char *buffer, return count; } +static int osc_rd_cur_lost_grant_bytes(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct obd_device *dev = data; + struct client_obd *cli = &dev->u.cli; + int rc; + + client_obd_list_lock(&cli->cl_loi_list_lock); + rc = snprintf(page, count, "%lu\n", cli->cl_lost_grant); + client_obd_list_unlock(&cli->cl_loi_list_lock); + return rc; +} + static int osc_rd_grant_shrink_interval(char *page, char **start, off_t off, int count, int *eof, void *data) { @@ -614,6 +627,7 @@ static struct lprocfs_vars lprocfs_osc_obd_vars[] = { { "cur_dirty_bytes", osc_rd_cur_dirty_bytes, 0, 0 }, { "cur_grant_bytes", osc_rd_cur_grant_bytes, osc_wr_cur_grant_bytes, 0 }, + { "cur_lost_grant_bytes", osc_rd_cur_lost_grant_bytes, 0, 0}, { "grant_shrink_interval", osc_rd_grant_shrink_interval, osc_wr_grant_shrink_interval, 0 }, { "create_count", osc_rd_create_count, osc_wr_create_count, 0 }, diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index 2b5f8a8..a01df94 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -1259,7 +1259,8 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,struct obdo *oa, struct lov_stripe_md *lsm, obd_count page_count, struct brw_page **pga, struct ptlrpc_request **reqp, - struct obd_capa *ocapa, int reserve) + struct obd_capa *ocapa, int reserve, + int resend) { struct ptlrpc_request *req; struct ptlrpc_bulk_desc *desc; @@ -1375,6 +1376,14 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,struct obdo *oa, &RMF_NIOBUF_REMOTE), (void *)(niobuf - niocount)); osc_announce_cached(cli, &body->oa, opc == OST_WRITE ? requested_nob:0); + if (resend) { + if ((body->oa.o_valid & OBD_MD_FLFLAGS) == 0) { + body->oa.o_valid |= OBD_MD_FLFLAGS; + body->oa.o_flags = 0; + } + body->oa.o_flags |= OBD_FL_RECOV_RESEND; + } + if (osc_should_shrink_grant(cli)) osc_shrink_grant_local(cli, &body->oa); @@ -1677,7 +1686,7 @@ static int osc_brw_internal(int cmd, struct obd_export *exp, struct obdo *oa, restart_bulk: rc = osc_brw_prep_request(cmd, &exp->exp_obd->u.cli, oa, lsm, - page_count, pga, &req, ocapa, 0); + page_count, pga, &req, ocapa, 0, resends); if (rc != 0) return (rc); @@ -1730,7 +1739,7 @@ int osc_brw_redo_request(struct ptlrpc_request *request, aa->aa_cli, aa->aa_oa, NULL /* lsm unused by osc currently */, aa->aa_page_count, aa->aa_ppga, - &new_req, aa->aa_ocapa, 0); + &new_req, aa->aa_ocapa, 0, 1); if (rc) RETURN(rc); @@ -2387,7 +2396,7 @@ static struct ptlrpc_request *osc_build_req(const struct lu_env *env, sort_brw_pages(pga, page_count); rc = osc_brw_prep_request(cmd, cli, oa, NULL, page_count, - pga, &req, crattr.cra_capa, 1); + pga, &req, crattr.cra_capa, 1, 0); if (rc != 0) { CERROR("prep_req failed: %d\n", rc); GOTO(out, req = ERR_PTR(rc)); diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c index dc414a4..81fe0ff 100644 --- a/lustre/ptlrpc/wiretest.c +++ b/lustre/ptlrpc/wiretest.c @@ -662,6 +662,7 @@ void lustre_assert_wire_constants(void) CLASSERT(OBD_FL_CKSUM_ADLER == 8192); CLASSERT(OBD_FL_SHRINK_GRANT == 131072); CLASSERT(OBD_FL_MMAP == (0x00040000)); + CLASSERT(OBD_FL_RECOV_RESEND == (0x00080000)); CLASSERT(OBD_CKSUM_CRC32 == 1); CLASSERT(OBD_CKSUM_ADLER == 2); diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 425d524..e76c6a0 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -3135,11 +3135,6 @@ canonical_path() { (cd `dirname $1`; echo $PWD/`basename $1`) } -sync_clients() { - [ -d $DIR1 ] && cd $DIR1 && sync; sleep 1; sync - [ -d $DIR2 ] && cd $DIR2 && sync; sleep 1; sync - cd $SAVE_PWD -} check_grant() { export base=`basetest $1` @@ -3149,39 +3144,34 @@ check_grant() { [ ${!testname}x == x ] && return 0 echo -n "checking grant......" - cd $SAVE_PWD - # write some data to sync client lost_grant - rm -f $DIR1/${tfile}_check_grant_* 2>&1 - for i in `seq $OSTCOUNT`; do - $LFS setstripe $DIR1/${tfile}_check_grant_$i -i $(($i -1)) -c 1 - dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \ - count=1 > /dev/null 2>&1 - done - # sync all the data and make sure no pending data on server - sync_clients - - #get client grant and server grant - client_grant=0 - for d in `lctl get_param -n osc.*.cur_grant_bytes`; do - client_grant=$((client_grant + $d)) - done - server_grant=0 - for d in `lctl get_param -n obdfilter.*.tot_granted`; do - server_grant=$((server_grant + $d)) - done - # cleanup the check_grant file - for i in `seq $OSTCOUNT`; do - rm $DIR1/${tfile}_check_grant_$i - done + local clients=$CLIENTS + [ -z $clients ] && clients=$(hostname) + + # sync all the data and make sure no pending data on server + do_nodes $clients sync + + # get client grant + client_grant=`do_nodes $clients \ + "$LCTL get_param -n osc.${FSNAME}-*.cur_*grant_bytes" | \ + awk '{total += $1} END{print total}'` + + # get server grant + server_grant=`do_nodes $(comma_list $(osts_nodes)) \ + "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" | \ + awk '{total += $1} END{print total}'` + + # check whether client grant == server grant + if [ $client_grant -ne $server_grant ]; then + echo "failed: client:${client_grant} server: ${server_grant}." + do_nodes $(comma_list $(osts_nodes)) \ + "$LCTL get_param obdfilter.${FSNAME}-OST*.tot*" + do_nodes $clients "$LCTL get_param osc.${FSNAME}-*.cur_*_bytes" + return 1 + else + echo "pass: client:${client_grant} server: ${server_grant}" + fi - #check whether client grant == server grant - if [ $client_grant != $server_grant ]; then - echo "failed: client:${client_grant} server: ${server_grant}" - return 1 - else - echo "pass" - fi } ######################## diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c index 88c961c..ab5b5a0 100644 --- a/lustre/utils/wirecheck.c +++ b/lustre/utils/wirecheck.c @@ -309,6 +309,7 @@ check_obdo(void) CHECK_CVALUE(OBD_FL_CKSUM_ADLER); CHECK_CVALUE(OBD_FL_SHRINK_GRANT); CHECK_CVALUE(OBD_FL_MMAP); + CHECK_CDEFINE(OBD_FL_RECOV_RESEND); CHECK_CVALUE(OBD_CKSUM_CRC32); CHECK_CVALUE(OBD_CKSUM_ADLER); } diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c index b330ab0..cab0cc7 100644 --- a/lustre/utils/wiretest.c +++ b/lustre/utils/wiretest.c @@ -659,6 +659,7 @@ void lustre_assert_wire_constants(void) CLASSERT(OBD_FL_CKSUM_ADLER == 8192); CLASSERT(OBD_FL_SHRINK_GRANT == 131072); CLASSERT(OBD_FL_MMAP == (0x00040000)); + CLASSERT(OBD_FL_RECOV_RESEND == (0x00080000)); CLASSERT(OBD_CKSUM_CRC32 == 1); CLASSERT(OBD_CKSUM_ADLER == 2); -- 1.8.3.1