From c35cc39503dcf8dd105d10f923df9e3a3369e15b Mon Sep 17 00:00:00 2001 From: Cliff White Date: Thu, 25 Mar 2010 22:42:51 -0700 Subject: [PATCH] b=20278 ASSERTION(cli->cl_avail_grant >= 0) failed i=tom.wang i=robert.read This patch tries to address several issues: - osc_init_grant(): calculate avail_grant according to recovery status. - osc_reconnect(): request grant should include cl_dirty. - filter_grant(): beside server reboot, we should also grant the requested amount in case of normal reconnect. - round-up grant amount instead of round-down, otherwise client would still have situation that dirty > granted. --- lustre/obdfilter/filter.c | 9 +++++---- lustre/obdfilter/filter_internal.h | 2 +- lustre/obdfilter/filter_io.c | 20 ++++++++++---------- lustre/osc/osc_request.c | 27 +++++++++++++++++++-------- 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c index 1cf4cfd..6d8558b 100644 --- a/lustre/obdfilter/filter.c +++ b/lustre/obdfilter/filter.c @@ -2275,7 +2275,8 @@ static int filter_cleanup(struct obd_device *obd) } static int filter_connect_internal(struct obd_export *exp, - struct obd_connect_data *data) + struct obd_connect_data *data, + int reconnect) { if (!data) RETURN(0); @@ -2303,7 +2304,7 @@ static int filter_connect_internal(struct obd_export *exp, spin_lock(&exp->exp_obd->obd_osfs_lock); left = filter_grant_space_left(exp); want = data->ocd_grant; - filter_grant(exp, fed->fed_grant, want, left); + filter_grant(exp, fed->fed_grant, want, left, (reconnect == 0)); data->ocd_grant = fed->fed_grant; spin_unlock(&exp->exp_obd->obd_osfs_lock); @@ -2390,7 +2391,7 @@ static int filter_reconnect(struct obd_export *exp, struct obd_device *obd, if (exp == NULL || obd == NULL || cluuid == NULL) RETURN(-EINVAL); - rc = filter_connect_internal(exp, data); + rc = filter_connect_internal(exp, data, 1); if (rc == 0) filter_export_stats_init(obd, exp, 1, localdata); @@ -2423,7 +2424,7 @@ static int filter_connect(struct lustre_handle *conn, struct obd_device *obd, fed = &exp->exp_filter_data; - rc = filter_connect_internal(exp, data); + rc = filter_connect_internal(exp, data,0); if (rc) GOTO(cleanup, rc); diff --git a/lustre/obdfilter/filter_internal.h b/lustre/obdfilter/filter_internal.h index 1990c7a..23fe364 100644 --- a/lustre/obdfilter/filter_internal.h +++ b/lustre/obdfilter/filter_internal.h @@ -174,7 +174,7 @@ int filter_commitrw_write(struct obd_export *exp, struct obdo *oa, int objcount, int rc); obd_size filter_grant_space_left(struct obd_export *exp); long filter_grant(struct obd_export *exp, obd_size current_grant, - obd_size want, obd_size fs_space_left); + obd_size want, obd_size fs_space_left, int conservative); void filter_grant_commit(struct obd_export *exp, int niocount, struct niobuf_local *res); void filter_grant_incoming(struct obd_export *exp, struct obdo *oa); diff --git a/lustre/obdfilter/filter_io.c b/lustre/obdfilter/filter_io.c index 3853bc5..cedc414 100644 --- a/lustre/obdfilter/filter_io.c +++ b/lustre/obdfilter/filter_io.c @@ -197,9 +197,13 @@ restat: /* Calculate how much grant space to allocate to this client, based on how * much space is currently free and how much of that is already granted. * + * if @conservative != 0, we limit the maximum grant to FILTER_GRANT_CHUNK; + * otherwise we'll satisfy the requested amount as possible as we can, this + * usually due to client reconnect. + * * Caller must hold obd_osfs_lock. */ long filter_grant(struct obd_export *exp, obd_size current_grant, - obd_size want, obd_size fs_space_left) + obd_size want, obd_size fs_space_left, int conservative) { struct obd_device *obd = exp->exp_obd; struct filter_export_data *fed = &exp->exp_filter_data; @@ -222,16 +226,11 @@ long filter_grant(struct obd_export *exp, obd_size current_grant, obd->obd_name, exp->exp_client_uuid.uuid, exp, want); } else if (current_grant < want && current_grant < fed->fed_grant + FILTER_GRANT_CHUNK) { - grant = min((want >> blockbits), - (fs_space_left >> blockbits) / 8); - grant <<= blockbits; + grant = min(want + (1 << blockbits) - 1, fs_space_left / 8); + grant &= ~((1ULL << blockbits) - 1); if (grant) { - /* Allow >FILTER_GRANT_CHUNK size when clients - * reconnect due to a server reboot. - */ - if ((grant > FILTER_GRANT_CHUNK) && - (!obd->obd_recovering)) + if (grant > FILTER_GRANT_CHUNK && conservative) grant = FILTER_GRANT_CHUNK; obd->u.filter.fo_tot_granted += grant; @@ -747,7 +746,8 @@ static int filter_preprw_write(int cmd, struct obd_export *exp, struct obdo *oa, /* do not zero out oa->o_valid as it is used in filter_commitrw_write() * for setting UID/GID and fid EA in first write time. */ if (oa->o_valid & OBD_MD_FLGRANT) - oa->o_grant = filter_grant(exp,oa->o_grant,oa->o_undirty,left); + oa->o_grant = filter_grant(exp, oa->o_grant, oa->o_undirty, + left, 1); spin_unlock(&obd->obd_osfs_lock); filter_fmd_put(exp, fmd); diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index ea5737e..4fc146f 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -1008,17 +1008,28 @@ static int osc_del_shrink_grant(struct client_obd *client) static void osc_init_grant(struct client_obd *cli, struct obd_connect_data *ocd) { + /* + * ocd_grant is the total grant amount we're expect to hold: if we'v + * been evicted, it's the new avail_grant amount, cl_dirty will drop + * to 0 as inflight rpcs fail out; otherwise, it's avail_grant + dirty. + * + * race is tolerable here: if we're evicted, but imp_state already + * left EVICTED state, then cl_diry must be 0 already. + */ client_obd_list_lock(&cli->cl_loi_list_lock); - cli->cl_avail_grant = ocd->ocd_grant; + if (cli->cl_import->imp_state == LUSTRE_IMP_EVICTED) + cli->cl_avail_grant = ocd->ocd_grant; + else + cli->cl_avail_grant = ocd->ocd_grant - cli->cl_dirty; client_obd_list_unlock(&cli->cl_loi_list_lock); - if (ocd->ocd_connect_flags & OBD_CONNECT_GRANT_SHRINK && - list_empty(&cli->cl_grant_shrink_list)) - osc_add_shrink_grant(cli); - CDEBUG(D_CACHE, "setting cl_avail_grant: %ld cl_lost_grant: %ld \n", cli->cl_avail_grant, cli->cl_lost_grant); LASSERT(cli->cl_avail_grant >= 0); + + if (ocd->ocd_connect_flags & OBD_CONNECT_GRANT_SHRINK && + list_empty(&cli->cl_grant_shrink_list)) + osc_add_shrink_grant(cli); } /* We assume that the reason this OSC got a short read is because it read @@ -4206,15 +4217,15 @@ static int osc_reconnect(struct obd_export *exp, struct obd_device *obd, long lost_grant; client_obd_list_lock(&cli->cl_loi_list_lock); - data->ocd_grant = cli->cl_avail_grant ?: + data->ocd_grant = cli->cl_avail_grant + cli->cl_dirty ?: 2 * cli->cl_max_pages_per_rpc << CFS_PAGE_SHIFT; lost_grant = cli->cl_lost_grant; cli->cl_lost_grant = 0; client_obd_list_unlock(&cli->cl_loi_list_lock); CDEBUG(D_CACHE, "request ocd_grant: %d cl_avail_grant: %ld " - "cl_lost_grant: %ld\n", data->ocd_grant, - cli->cl_avail_grant, lost_grant); + "cl_dirty: %ld cl_lost_grant: %ld\n", data->ocd_grant, + cli->cl_dirty, cli->cl_avail_grant, lost_grant); CDEBUG(D_RPCTRACE, "ocd_connect_flags: "LPX64" ocd_version: %d" " ocd_grant: %d\n", data->ocd_connect_flags, data->ocd_version, data->ocd_grant); -- 1.8.3.1