b=20278 ASSERTION(cli->cl_avail_grant >= 0) failed

i=tom.wang
i=robert.read

This patch tries to address several issues:
- osc_init_grant(): calculate avail_grant according to recovery status.
- osc_reconnect(): request grant should include cl_dirty.
- filter_grant(): beside server reboot, we should also grant the requested
  amount in case of normal reconnect.
- round-up grant amount instead of round-down, otherwise client would still
  have situation that dirty > granted.

diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c
index 1cf4cfd..6d8558b 100644 (file)
--- a/lustre/obdfilter/filter.c
+++ b/lustre/obdfilter/filter.c
@@ -2275,7 +2275,8 @@ static int filter_cleanup(struct obd_device *obd)
 }
 
 static int filter_connect_internal(struct obd_export *exp,
-                                   struct obd_connect_data *data)
+                                   struct obd_connect_data *data,
+                                   int reconnect)
 {
         if (!data)
                 RETURN(0);
@@ -2303,7 +2304,7 @@ static int filter_connect_internal(struct obd_export *exp,
                 spin_lock(&exp->exp_obd->obd_osfs_lock);
                 left = filter_grant_space_left(exp);
                 want = data->ocd_grant;
-                filter_grant(exp, fed->fed_grant, want, left);
+                filter_grant(exp, fed->fed_grant, want, left, (reconnect == 0));
                 data->ocd_grant = fed->fed_grant;
                 spin_unlock(&exp->exp_obd->obd_osfs_lock);
 
@@ -2390,7 +2391,7 @@ static int filter_reconnect(struct obd_export *exp, struct obd_device *obd,
         if (exp == NULL || obd == NULL || cluuid == NULL)
                 RETURN(-EINVAL);
 
-        rc = filter_connect_internal(exp, data);
+        rc = filter_connect_internal(exp, data, 1);
         if (rc == 0)
                 filter_export_stats_init(obd, exp, 1, localdata);
 
@@ -2423,7 +2424,7 @@ static int filter_connect(struct lustre_handle *conn, struct obd_device *obd,
 
         fed = &exp->exp_filter_data;
 
-        rc = filter_connect_internal(exp, data);
+        rc = filter_connect_internal(exp, data,0);
         if (rc)
                 GOTO(cleanup, rc);
 

diff --git a/lustre/obdfilter/filter_internal.h b/lustre/obdfilter/filter_internal.h
index 1990c7a..23fe364 100644 (file)
--- a/lustre/obdfilter/filter_internal.h
+++ b/lustre/obdfilter/filter_internal.h
@@ -174,7 +174,7 @@ int filter_commitrw_write(struct obd_export *exp, struct obdo *oa, int objcount,
                           int rc);
 obd_size filter_grant_space_left(struct obd_export *exp);
 long filter_grant(struct obd_export *exp, obd_size current_grant,
-                  obd_size want, obd_size fs_space_left);
+                  obd_size want, obd_size fs_space_left, int conservative);
 void filter_grant_commit(struct obd_export *exp, int niocount,
                          struct niobuf_local *res);
 void filter_grant_incoming(struct obd_export *exp, struct obdo *oa);

diff --git a/lustre/obdfilter/filter_io.c b/lustre/obdfilter/filter_io.c
index 3853bc5..cedc414 100644 (file)
--- a/lustre/obdfilter/filter_io.c
+++ b/lustre/obdfilter/filter_io.c
@@ -197,9 +197,13 @@ restat:
 /* Calculate how much grant space to allocate to this client, based on how
  * much space is currently free and how much of that is already granted.
  *
+ * if @conservative != 0, we limit the maximum grant to FILTER_GRANT_CHUNK;
+ * otherwise we'll satisfy the requested amount as possible as we can, this
+ * usually due to client reconnect.
+ *
  * Caller must hold obd_osfs_lock. */
 long filter_grant(struct obd_export *exp, obd_size current_grant,
-                  obd_size want, obd_size fs_space_left)
+                  obd_size want, obd_size fs_space_left, int conservative)
 {
         struct obd_device *obd = exp->exp_obd;
         struct filter_export_data *fed = &exp->exp_filter_data;
@@ -222,16 +226,11 @@ long filter_grant(struct obd_export *exp, obd_size current_grant,
                        obd->obd_name, exp->exp_client_uuid.uuid, exp, want);
         } else if (current_grant < want &&
                    current_grant < fed->fed_grant + FILTER_GRANT_CHUNK) {
-                grant = min((want >> blockbits),
-                            (fs_space_left >> blockbits) / 8);
-                grant <<= blockbits;
+                grant = min(want + (1 << blockbits) - 1, fs_space_left / 8);
+                grant &= ~((1ULL << blockbits) - 1);
 
                 if (grant) {
-                        /* Allow >FILTER_GRANT_CHUNK size when clients
-                         * reconnect due to a server reboot.
-                         */
-                        if ((grant > FILTER_GRANT_CHUNK) &&
-                            (!obd->obd_recovering))
+                        if (grant > FILTER_GRANT_CHUNK && conservative)
                                 grant = FILTER_GRANT_CHUNK;
 
                         obd->u.filter.fo_tot_granted += grant;
@@ -747,7 +746,8 @@ static int filter_preprw_write(int cmd, struct obd_export *exp, struct obdo *oa,
         /* do not zero out oa->o_valid as it is used in filter_commitrw_write()
          * for setting UID/GID and fid EA in first write time. */
         if (oa->o_valid & OBD_MD_FLGRANT)
-                oa->o_grant = filter_grant(exp,oa->o_grant,oa->o_undirty,left);
+                oa->o_grant = filter_grant(exp, oa->o_grant, oa->o_undirty,
+                                           left, 1);
 
         spin_unlock(&obd->obd_osfs_lock);
         filter_fmd_put(exp, fmd);

diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index ea5737e..4fc146f 100644 (file)
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -1008,17 +1008,28 @@ static int osc_del_shrink_grant(struct client_obd *client)
 
 static void osc_init_grant(struct client_obd *cli, struct obd_connect_data *ocd)
 {
+        /*
+         * ocd_grant is the total grant amount we're expect to hold: if we'v
+         * been evicted, it's the new avail_grant amount, cl_dirty will drop
+         * to 0 as inflight rpcs fail out; otherwise, it's avail_grant + dirty.
+         *
+         * race is tolerable here: if we're evicted, but imp_state already
+         * left EVICTED state, then cl_diry must be 0 already.
+         */
         client_obd_list_lock(&cli->cl_loi_list_lock);
-        cli->cl_avail_grant = ocd->ocd_grant;
+        if (cli->cl_import->imp_state == LUSTRE_IMP_EVICTED)
+                cli->cl_avail_grant = ocd->ocd_grant;
+        else
+                cli->cl_avail_grant = ocd->ocd_grant - cli->cl_dirty;
         client_obd_list_unlock(&cli->cl_loi_list_lock);
 
-        if (ocd->ocd_connect_flags & OBD_CONNECT_GRANT_SHRINK &&
-            list_empty(&cli->cl_grant_shrink_list))
-                osc_add_shrink_grant(cli);
-
         CDEBUG(D_CACHE, "setting cl_avail_grant: %ld cl_lost_grant: %ld \n",
                cli->cl_avail_grant, cli->cl_lost_grant);
         LASSERT(cli->cl_avail_grant >= 0);
+
+        if (ocd->ocd_connect_flags & OBD_CONNECT_GRANT_SHRINK &&
+            list_empty(&cli->cl_grant_shrink_list))
+                osc_add_shrink_grant(cli);
 }
 
 /* We assume that the reason this OSC got a short read is because it read
@@ -4206,15 +4217,15 @@ static int osc_reconnect(struct obd_export *exp, struct obd_device *obd,
                 long lost_grant;
 
                 client_obd_list_lock(&cli->cl_loi_list_lock);
-                data->ocd_grant = cli->cl_avail_grant ?:
+                data->ocd_grant = cli->cl_avail_grant + cli->cl_dirty ?:
                                 2 * cli->cl_max_pages_per_rpc << CFS_PAGE_SHIFT;
                 lost_grant = cli->cl_lost_grant;
                 cli->cl_lost_grant = 0;
                 client_obd_list_unlock(&cli->cl_loi_list_lock);
 
                 CDEBUG(D_CACHE, "request ocd_grant: %d cl_avail_grant: %ld "
-                       "cl_lost_grant: %ld\n", data->ocd_grant,
-                       cli->cl_avail_grant, lost_grant);
+                       "cl_dirty: %ld cl_lost_grant: %ld\n", data->ocd_grant,
+                       cli->cl_dirty, cli->cl_avail_grant, lost_grant);
                 CDEBUG(D_RPCTRACE, "ocd_connect_flags: "LPX64" ocd_version: %d"
                        " ocd_grant: %d\n", data->ocd_connect_flags,
                        data->ocd_version, data->ocd_grant);