From 1c8e3ce72ca8935c3b4cac85f768538f25cf2a41 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 22 Jul 2009 19:51:12 +0000 Subject: [PATCH] - use mutex to protect grant calculation because zfs's df isn't atomic (slowdown is expected, to be fixed at some point) --- lustre/ofd/ofd_dev.c | 1 + lustre/ofd/ofd_grant.c | 28 ++++++++++++++-------------- lustre/ofd/ofd_internal.h | 1 + lustre/ofd/ofd_io.c | 8 ++++---- lustre/ofd/ofd_obd.c | 8 ++++---- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c index b31f0df..29e9fbc 100644 --- a/lustre/ofd/ofd_dev.c +++ b/lustre/ofd/ofd_dev.c @@ -622,6 +622,7 @@ static int filter_init0(const struct lu_env *env, struct filter_device *m, /* grant data */ spin_lock_init(&m->ofd_grant_lock); + sema_init(&m->ofd_grant_sem, 1); m->ofd_tot_dirty = 0; m->ofd_tot_granted = 0; m->ofd_tot_pending = 0; diff --git a/lustre/ofd/ofd_grant.c b/lustre/ofd/ofd_grant.c index 0b2f54f..dc0ce7c 100644 --- a/lustre/ofd/ofd_grant.c +++ b/lustre/ofd/ofd_grant.c @@ -67,7 +67,7 @@ void filter_grant_sanity_check(struct obd_device *obd, const char *func) if (obd->obd_num_exports > 100) return; - spin_lock(&obd->obd_osfs_lock); + mutex_down(&ofd->ofd_grant_sem); spin_lock(&obd->obd_dev_lock); list_for_each_entry(exp, &obd->obd_exports, exp_obd_chain) { int error = 0; @@ -101,7 +101,7 @@ void filter_grant_sanity_check(struct obd_device *obd, const char *func) fo_tot_pending = ofd->ofd_tot_pending; fo_tot_dirty = ofd->ofd_tot_dirty; spin_unlock(&obd->obd_dev_lock); - spin_unlock(&obd->obd_osfs_lock); + mutex_up(&ofd->ofd_grant_sem); /* Do these assertions outside the spinlocks so we don't kill system */ if (tot_granted != fo_tot_granted) @@ -135,7 +135,7 @@ void filter_grant_discard(struct obd_export *exp) struct filter_device *ofd = filter_exp(exp); struct filter_export_data *fed = &exp->exp_filter_data; - spin_lock(&obd->obd_osfs_lock); + mutex_down(&ofd->ofd_grant_sem); spin_lock(&obd->obd_dev_lock); list_del_init(&exp->exp_obd_chain); spin_unlock(&obd->obd_dev_lock); @@ -157,7 +157,7 @@ void filter_grant_discard(struct obd_export *exp) ofd->ofd_tot_dirty -= fed->fed_dirty; fed->fed_dirty = 0; fed->fed_grant = 0; - spin_unlock(&obd->obd_osfs_lock); + mutex_up(&ofd->ofd_grant_sem); } /* @@ -173,7 +173,7 @@ void filter_grant_incoming(const struct lu_env *env, struct obd_export *exp, struct obd_device *obd = exp->exp_obd; ENTRY; - LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock); + //LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock); if ((oa->o_valid & (OBD_MD_FLBLOCKS|OBD_MD_FLGRANT)) != (OBD_MD_FLBLOCKS|OBD_MD_FLGRANT)) { @@ -239,7 +239,7 @@ void filter_grant_incoming(const struct lu_env *env, struct obd_export *exp, CERROR("%s: cli %s/%p dirty %ld pend %ld grant %ld\n", obd->obd_name, exp->exp_client_uuid.uuid, exp, fed->fed_dirty, fed->fed_pending, fed->fed_grant); - spin_unlock(&obd->obd_osfs_lock); + mutex_up(&ofd->ofd_grant_sem); LBUG(); } EXIT; @@ -260,7 +260,7 @@ obd_size filter_grant_space_left(const struct lu_env *env, int statfs_done = 0; long frsize; - LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock); + //LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock); if (cfs_time_before_64(obd->obd_osfs_age, cfs_time_current_64() - HZ)) { @@ -327,7 +327,7 @@ int filter_grant_client_calc(struct obd_export *exp, obd_size *left, unsigned long using = 0; int rc = 0; - LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock); + //LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock); *left -= *ungranted; LASSERT(fed->fed_grant >= *used); @@ -363,7 +363,7 @@ int filter_grant_client_calc(struct obd_export *exp, obd_size *left, CERROR("%s: cli %s/%p dirty %ld pend %ld grant %ld\n", obd->obd_name, exp->exp_client_uuid.uuid, exp, fed->fed_dirty, fed->fed_pending, fed->fed_grant); - spin_unlock(&obd->obd_osfs_lock); + mutex_up(&ofd->ofd_grant_sem); LBUG(); } return rc; @@ -385,7 +385,7 @@ int filter_grant_check(const struct lu_env *env, struct obd_export *exp, struct filter_export_data *fed = &exp->exp_filter_data; int i, rc = -ENOSPC, obj, n = 0; - LASSERT_SPIN_LOCKED(&exp->exp_obd->obd_osfs_lock); + //LASSERT_SPIN_LOCKED(&exp->exp_obd->obd_osfs_lock); for (obj = 0; obj < objcount; obj++) { for (i = 0; i < objs[obj].ioo_bufcnt; i++, n++) { @@ -451,7 +451,7 @@ long _filter_grant(const struct lu_env *env, struct obd_export *exp, long frsize = obd->obd_osfs.os_bsize; __u64 grant = 0; - LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock); + //LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock); LASSERT(frsize); /* Grant some fraction of the client's requested grant space so that @@ -487,7 +487,7 @@ long _filter_grant(const struct lu_env *env, struct obd_export *exp, "current"LPU64"\n", obd->obd_name, exp->exp_client_uuid.uuid, exp, fed->fed_grant, want, curgrant); - spin_unlock(&obd->obd_osfs_lock); + mutex_up(&ofd->ofd_grant_sem); LBUG(); } } @@ -530,7 +530,7 @@ void filter_grant_commit(struct obd_export *exp, int niocount, unsigned long pending = 0; int i; - spin_lock(&exp->exp_obd->obd_osfs_lock); + mutex_down(&ofd->ofd_grant_sem); for (i = 0, lnb = res; i < niocount; i++, lnb++) pending += lnb->lnb_grant_used; @@ -550,6 +550,6 @@ void filter_grant_commit(struct obd_export *exp, int niocount, ofd->ofd_tot_pending, pending); ofd->ofd_tot_pending -= pending; - spin_unlock(&exp->exp_obd->obd_osfs_lock); + mutex_up(&ofd->ofd_grant_sem); } diff --git a/lustre/ofd/ofd_internal.h b/lustre/ofd/ofd_internal.h index 6b093cb..bb75178 100644 --- a/lustre/ofd/ofd_internal.h +++ b/lustre/ofd/ofd_internal.h @@ -161,6 +161,7 @@ struct filter_device { obd_size ofd_tot_granted; obd_size ofd_tot_pending; int ofd_tot_granted_clients; + struct semaphore ofd_grant_sem; /* filter mod data: filter_device wide values */ int ofd_fmd_max_num; /* per ofd filter_mod_data */ diff --git a/lustre/ofd/ofd_io.c b/lustre/ofd/ofd_io.c index 5d9acf6..5d07e7a 100644 --- a/lustre/ofd/ofd_io.c +++ b/lustre/ofd/ofd_io.c @@ -121,7 +121,7 @@ static int filter_preprw_write(const struct lu_env *env, struct obd_export *exp, *nr_local = j; LASSERT(*nr_local > 0 && *nr_local <= PTLRPC_MAX_BRW_PAGES); - spin_lock(&exp->exp_obd->obd_osfs_lock); + mutex_down(&ofd->ofd_grant_sem); filter_grant_incoming(env, exp, oa); left = filter_grant_space_left(env, exp); @@ -138,7 +138,7 @@ static int filter_preprw_write(const struct lu_env *env, struct obd_export *exp, if (oa->o_valid & OBD_MD_FLGRANT) oa->o_grant = filter_grant(env, exp, oa->o_grant, oa->o_undirty, left); - spin_unlock(&exp->exp_obd->obd_osfs_lock); + mutex_up(&ofd->ofd_grant_sem); rc = dt_write_prep(env, filter_object_child(fo), res, *nr_local, &used); @@ -187,11 +187,11 @@ int filter_preprw(int cmd, struct obd_export *exp, struct obdo *oa, int objcount if (rc == 0) { if (oa && oa->o_valid & OBD_MD_FLGRANT) { struct obd_device *obd = filter_obd(ofd); - spin_lock(&obd->obd_osfs_lock); + mutex_down(&ofd->ofd_grant_sem); filter_grant_incoming(&env, exp, oa); if (!(oa->o_flags & OBD_FL_SHRINK_GRANT)) oa->o_grant = 0; - spin_unlock(&obd->obd_osfs_lock); + mutex_up(&ofd->ofd_grant_sem); } rc = filter_preprw_read(&env, ofd, &info->fti_fid, &info->fti_attr, obj->ioo_bufcnt, diff --git a/lustre/ofd/ofd_obd.c b/lustre/ofd/ofd_obd.c index e64a739..a6ee7b8 100644 --- a/lustre/ofd/ofd_obd.c +++ b/lustre/ofd/ofd_obd.c @@ -112,12 +112,12 @@ static int filter_parse_connect_data(const struct lu_env *env, if (exp->exp_connect_flags & OBD_CONNECT_GRANT) { obd_size left, want; - spin_lock(&exp->exp_obd->obd_osfs_lock); + mutex_down(&ofd->ofd_grant_sem); left = filter_grant_space_left(env, exp); want = data->ocd_grant; filter_grant(env, exp, fed->fed_grant, want, left); data->ocd_grant = fed->fed_grant; - spin_unlock(&exp->exp_obd->obd_osfs_lock); + mutex_up(&ofd->ofd_grant_sem); CDEBUG(D_CACHE, "%s: cli %s/%p ocd_grant: %d want: " LPU64" left: "LPU64"\n", exp->exp_obd->obd_name, @@ -474,9 +474,9 @@ static int filter_set_info_async(struct obd_export *exp, __u32 keylen, if (KEY_IS(KEY_GRANT_SHRINK)) { struct ost_body *body = (struct ost_body *)val; /* handle shrink grant */ - spin_lock(&exp->exp_obd->obd_osfs_lock); + mutex_down(&ofd->ofd_grant_sem); filter_grant_incoming(&env, exp, &body->oa); - spin_unlock(&exp->exp_obd->obd_osfs_lock); + mutex_up(&ofd->ofd_grant_sem); GOTO(out, rc = 0); } -- 1.8.3.1