b=22755 Don't consume grant twice on recoverable resend

o=johann
i=green
i=tappro

diff --git a/lustre/include/lustre/lustre_idl.h b/lustre/include/lustre/lustre_idl.h
index afdfb25..0ea1fdb 100644 (file)
--- a/lustre/include/lustre/lustre_idl.h
+++ b/lustre/include/lustre/lustre_idl.h
@@ -1194,6 +1194,7 @@ enum obdo_flags {
         OBD_FL_CKSUM_RSVD3  = 0x00010000, /* for future cksum types */
         OBD_FL_SHRINK_GRANT = 0x00020000, /* object shrink the grant */
         OBD_FL_MMAP         = 0x00040000, /* object is mmapped on the client */
+        OBD_FL_RECOV_RESEND = 0x00080000, /* recoverable resent */
 
         OBD_FL_CKSUM_ALL    = OBD_FL_CKSUM_CRC32 | OBD_FL_CKSUM_ADLER,
 

diff --git a/lustre/obdfilter/filter_io.c b/lustre/obdfilter/filter_io.c
index 268dec1..859e290 100644 (file)
--- a/lustre/obdfilter/filter_io.c
+++ b/lustre/obdfilter/filter_io.c
@@ -530,6 +530,14 @@ static int filter_grant_check(struct obd_export *exp, struct obdo *oa,
         int blocksize = exp->exp_obd->u.obt.obt_sb->s_blocksize;
         unsigned long used = 0, ungranted = 0, using;
         int i, rc = -ENOSPC, obj, n = 0;
+        int resend = 0;
+
+        if ((oa->o_valid & OBD_MD_FLFLAGS) &&
+            (oa->o_flags & OBD_FL_RECOV_RESEND)) {
+                resend = 1;
+                CDEBUG(D_CACHE, "Recoverable resend arrived, skipping "
+                       "accounting\n");
+        }
 
         LASSERT_SPIN_LOCKED(&exp->exp_obd->obd_osfs_lock);
 
@@ -546,7 +554,12 @@ static int filter_grant_check(struct obd_export *exp, struct obdo *oa,
 
                         if ((lnb[n].flags & OBD_BRW_FROM_GRANT) &&
                             (oa->o_valid & OBD_MD_FLGRANT)) {
-                                if (fed->fed_grant < used + bytes) {
+                                if (resend) {
+                                        /* this is a recoverable resent */
+                                        lnb[n].flags |= OBD_BRW_GRANTED;
+                                        rc = 0;
+                                        continue;
+                                } else if (fed->fed_grant < used + bytes) {
                                         CDEBUG(D_CACHE,
                                                "%s: cli %s/%p claims %ld+%d "
                                                "GRANT, real grant %lu idx %d\n",

diff --git a/lustre/osc/lproc_osc.c b/lustre/osc/lproc_osc.c
index 086c94b..237200b 100644 (file)
--- a/lustre/osc/lproc_osc.c
+++ b/lustre/osc/lproc_osc.c
@@ -256,6 +256,19 @@ static int osc_wr_cur_grant_bytes(struct file *file, const char *buffer,
         return count;
 }
 
+static int osc_rd_cur_lost_grant_bytes(char *page, char **start, off_t off,
+                                       int count, int *eof, void *data)
+{
+        struct obd_device *dev = data;
+        struct client_obd *cli = &dev->u.cli;
+        int rc;
+
+        client_obd_list_lock(&cli->cl_loi_list_lock);
+        rc = snprintf(page, count, "%lu\n", cli->cl_lost_grant);
+        client_obd_list_unlock(&cli->cl_loi_list_lock);
+        return rc;
+}
+
 static int osc_rd_grant_shrink_interval(char *page, char **start, off_t off,
                                         int count, int *eof, void *data)
 {
@@ -614,6 +627,7 @@ static struct lprocfs_vars lprocfs_osc_obd_vars[] = {
         { "cur_dirty_bytes", osc_rd_cur_dirty_bytes, 0, 0 },
         { "cur_grant_bytes", osc_rd_cur_grant_bytes,
                              osc_wr_cur_grant_bytes, 0 },
+        { "cur_lost_grant_bytes", osc_rd_cur_lost_grant_bytes, 0, 0},
         { "grant_shrink_interval", osc_rd_grant_shrink_interval,
                                    osc_wr_grant_shrink_interval, 0 },
         { "create_count",    osc_rd_create_count, osc_wr_create_count, 0 },

diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index 2b5f8a8..a01df94 100644 (file)
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -1259,7 +1259,8 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,struct obdo *oa,
                                 struct lov_stripe_md *lsm, obd_count page_count,
                                 struct brw_page **pga,
                                 struct ptlrpc_request **reqp,
-                                struct obd_capa *ocapa, int reserve)
+                                struct obd_capa *ocapa, int reserve,
+                                int resend)
 {
         struct ptlrpc_request   *req;
         struct ptlrpc_bulk_desc *desc;
@@ -1375,6 +1376,14 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,struct obdo *oa,
                 &RMF_NIOBUF_REMOTE), (void *)(niobuf - niocount));
 
         osc_announce_cached(cli, &body->oa, opc == OST_WRITE ? requested_nob:0);
+        if (resend) {
+                if ((body->oa.o_valid & OBD_MD_FLFLAGS) == 0) {
+                        body->oa.o_valid |= OBD_MD_FLFLAGS;
+                        body->oa.o_flags = 0;
+                }
+                body->oa.o_flags |= OBD_FL_RECOV_RESEND;
+        }
+
         if (osc_should_shrink_grant(cli))
                 osc_shrink_grant_local(cli, &body->oa);
 
@@ -1677,7 +1686,7 @@ static int osc_brw_internal(int cmd, struct obd_export *exp, struct obdo *oa,
 
 restart_bulk:
         rc = osc_brw_prep_request(cmd, &exp->exp_obd->u.cli, oa, lsm,
-                                  page_count, pga, &req, ocapa, 0);
+                                  page_count, pga, &req, ocapa, 0, resends);
         if (rc != 0)
                 return (rc);
 
@@ -1730,7 +1739,7 @@ int osc_brw_redo_request(struct ptlrpc_request *request,
                                   aa->aa_cli, aa->aa_oa,
                                   NULL /* lsm unused by osc currently */,
                                   aa->aa_page_count, aa->aa_ppga,
-                                  &new_req, aa->aa_ocapa, 0);
+                                  &new_req, aa->aa_ocapa, 0, 1);
         if (rc)
                 RETURN(rc);
 
@@ -2387,7 +2396,7 @@ static struct ptlrpc_request *osc_build_req(const struct lu_env *env,
 
         sort_brw_pages(pga, page_count);
         rc = osc_brw_prep_request(cmd, cli, oa, NULL, page_count,
-                                  pga, &req, crattr.cra_capa, 1);
+                                  pga, &req, crattr.cra_capa, 1, 0);
         if (rc != 0) {
                 CERROR("prep_req failed: %d\n", rc);
                 GOTO(out, req = ERR_PTR(rc));

diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c
index dc414a4..81fe0ff 100644 (file)
--- a/lustre/ptlrpc/wiretest.c
+++ b/lustre/ptlrpc/wiretest.c
@@ -662,6 +662,7 @@ void lustre_assert_wire_constants(void)
         CLASSERT(OBD_FL_CKSUM_ADLER == 8192);
         CLASSERT(OBD_FL_SHRINK_GRANT == 131072);
         CLASSERT(OBD_FL_MMAP == (0x00040000));
+        CLASSERT(OBD_FL_RECOV_RESEND == (0x00080000));
         CLASSERT(OBD_CKSUM_CRC32 == 1);
         CLASSERT(OBD_CKSUM_ADLER == 2);
 

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 425d524..e76c6a0 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -3135,11 +3135,6 @@ canonical_path() {
     (cd `dirname $1`; echo $PWD/`basename $1`)
 }
 
-sync_clients() {
-    [ -d $DIR1 ] && cd $DIR1 && sync; sleep 1; sync
-    [ -d $DIR2 ] && cd $DIR2 && sync; sleep 1; sync
-        cd $SAVE_PWD
-}
 
 check_grant() {
     export base=`basetest $1`
@@ -3149,39 +3144,34 @@ check_grant() {
         [ ${!testname}x == x ] && return 0
 
     echo -n "checking grant......"
-        cd $SAVE_PWD
-        # write some data to sync client lost_grant
-        rm -f $DIR1/${tfile}_check_grant_* 2>&1
-        for i in `seq $OSTCOUNT`; do
-                $LFS setstripe $DIR1/${tfile}_check_grant_$i -i $(($i -1)) -c 1
-                dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \
-                                              count=1 > /dev/null 2>&1
-        done
-        # sync all the data and make sure no pending data on server
-        sync_clients
-        
-        #get client grant and server grant
-        client_grant=0
-    for d in `lctl get_param -n osc.*.cur_grant_bytes`; do
-                client_grant=$((client_grant + $d))
-        done
-        server_grant=0
-        for d in `lctl get_param -n obdfilter.*.tot_granted`; do
-                server_grant=$((server_grant + $d))
-        done
 
-        # cleanup the check_grant file
-        for i in `seq $OSTCOUNT`; do
-                rm $DIR1/${tfile}_check_grant_$i
-        done
+        local clients=$CLIENTS
+        [ -z $clients ] && clients=$(hostname)
+
+    # sync all the data and make sure no pending data on server
+    do_nodes $clients sync
+
+    # get client grant
+    client_grant=`do_nodes $clients \
+                    "$LCTL get_param -n osc.${FSNAME}-*.cur_*grant_bytes" | \
+                    awk '{total += $1} END{print total}'`
+
+    # get server grant
+    server_grant=`do_nodes $(comma_list $(osts_nodes)) \
+                    "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" | \
+                    awk '{total += $1} END{print total}'`
+
+    # check whether client grant == server grant
+    if [ $client_grant -ne $server_grant ]; then
+        echo "failed: client:${client_grant} server: ${server_grant}."
+        do_nodes $(comma_list $(osts_nodes)) \
+                   "$LCTL get_param obdfilter.${FSNAME}-OST*.tot*"
+        do_nodes $clients "$LCTL get_param osc.${FSNAME}-*.cur_*_bytes"
+        return 1
+    else
+        echo "pass: client:${client_grant} server: ${server_grant}"
+    fi
 
-        #check whether client grant == server grant
-        if [ $client_grant != $server_grant ]; then
-                echo "failed: client:${client_grant} server: ${server_grant}"
-                return 1
-        else
-                echo "pass"
-        fi
 }
 
 ########################

diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c
index 88c961c..ab5b5a0 100644 (file)
--- a/lustre/utils/wirecheck.c
+++ b/lustre/utils/wirecheck.c
@@ -309,6 +309,7 @@ check_obdo(void)
         CHECK_CVALUE(OBD_FL_CKSUM_ADLER);
         CHECK_CVALUE(OBD_FL_SHRINK_GRANT);
         CHECK_CVALUE(OBD_FL_MMAP);
+        CHECK_CDEFINE(OBD_FL_RECOV_RESEND);
         CHECK_CVALUE(OBD_CKSUM_CRC32);
         CHECK_CVALUE(OBD_CKSUM_ADLER);
 }

diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c
index b330ab0..cab0cc7 100644 (file)
--- a/lustre/utils/wiretest.c
+++ b/lustre/utils/wiretest.c
@@ -659,6 +659,7 @@ void lustre_assert_wire_constants(void)
         CLASSERT(OBD_FL_CKSUM_ADLER == 8192);
         CLASSERT(OBD_FL_SHRINK_GRANT == 131072);
         CLASSERT(OBD_FL_MMAP == (0x00040000));
+        CLASSERT(OBD_FL_RECOV_RESEND == (0x00080000));
         CLASSERT(OBD_CKSUM_CRC32 == 1);
         CLASSERT(OBD_CKSUM_ADLER == 2);