- use mutex to protect grant calculation because zfs's df isn't atomic

   (slowdown is expected, to be fixed at some point)

diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c
index b31f0df..29e9fbc 100644 (file)
--- a/lustre/ofd/ofd_dev.c
+++ b/lustre/ofd/ofd_dev.c
@@ -622,6 +622,7 @@ static int filter_init0(const struct lu_env *env, struct filter_device *m,
 
         /* grant data */
         spin_lock_init(&m->ofd_grant_lock);
+        sema_init(&m->ofd_grant_sem, 1);
         m->ofd_tot_dirty = 0;
         m->ofd_tot_granted = 0;
         m->ofd_tot_pending = 0;

diff --git a/lustre/ofd/ofd_grant.c b/lustre/ofd/ofd_grant.c
index 0b2f54f..dc0ce7c 100644 (file)
--- a/lustre/ofd/ofd_grant.c
+++ b/lustre/ofd/ofd_grant.c
@@ -67,7 +67,7 @@ void filter_grant_sanity_check(struct obd_device *obd, const char *func)
         if (obd->obd_num_exports > 100)
                 return;
 
-        spin_lock(&obd->obd_osfs_lock);
+        mutex_down(&ofd->ofd_grant_sem);
         spin_lock(&obd->obd_dev_lock);
         list_for_each_entry(exp, &obd->obd_exports, exp_obd_chain) {
                 int error = 0;
@@ -101,7 +101,7 @@ void filter_grant_sanity_check(struct obd_device *obd, const char *func)
         fo_tot_pending = ofd->ofd_tot_pending;
         fo_tot_dirty = ofd->ofd_tot_dirty;
         spin_unlock(&obd->obd_dev_lock);
-        spin_unlock(&obd->obd_osfs_lock);
+        mutex_up(&ofd->ofd_grant_sem);
 
         /* Do these assertions outside the spinlocks so we don't kill system */
         if (tot_granted != fo_tot_granted)
@@ -135,7 +135,7 @@ void filter_grant_discard(struct obd_export *exp)
         struct filter_device *ofd = filter_exp(exp);
         struct filter_export_data *fed = &exp->exp_filter_data;
 
-        spin_lock(&obd->obd_osfs_lock);
+        mutex_down(&ofd->ofd_grant_sem);
         spin_lock(&obd->obd_dev_lock);
         list_del_init(&exp->exp_obd_chain);
         spin_unlock(&obd->obd_dev_lock);
@@ -157,7 +157,7 @@ void filter_grant_discard(struct obd_export *exp)
         ofd->ofd_tot_dirty -= fed->fed_dirty;
         fed->fed_dirty = 0;
         fed->fed_grant = 0;
-        spin_unlock(&obd->obd_osfs_lock);
+        mutex_up(&ofd->ofd_grant_sem);
 }
 
 /* 
@@ -173,7 +173,7 @@ void filter_grant_incoming(const struct lu_env *env, struct obd_export *exp,
         struct obd_device *obd = exp->exp_obd;
         ENTRY;
 
-        LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock);
+        //LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock);
 
         if ((oa->o_valid & (OBD_MD_FLBLOCKS|OBD_MD_FLGRANT)) !=
                                         (OBD_MD_FLBLOCKS|OBD_MD_FLGRANT)) {
@@ -239,7 +239,7 @@ void filter_grant_incoming(const struct lu_env *env, struct obd_export *exp,
                 CERROR("%s: cli %s/%p dirty %ld pend %ld grant %ld\n",
                        obd->obd_name, exp->exp_client_uuid.uuid, exp,
                        fed->fed_dirty, fed->fed_pending, fed->fed_grant);
-                spin_unlock(&obd->obd_osfs_lock);
+                mutex_up(&ofd->ofd_grant_sem);
                 LBUG();
         }
         EXIT;
@@ -260,7 +260,7 @@ obd_size filter_grant_space_left(const struct lu_env *env,
         int statfs_done = 0;
         long frsize;
 
-        LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock);
+        //LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock);
 
         if (cfs_time_before_64(obd->obd_osfs_age,
                                cfs_time_current_64() - HZ)) {
@@ -327,7 +327,7 @@ int filter_grant_client_calc(struct obd_export *exp, obd_size *left,
         unsigned long using = 0;
         int rc = 0;
 
-        LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock);
+        //LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock);
 
         *left -= *ungranted;
         LASSERT(fed->fed_grant >= *used);
@@ -363,7 +363,7 @@ int filter_grant_client_calc(struct obd_export *exp, obd_size *left,
                 CERROR("%s: cli %s/%p dirty %ld pend %ld grant %ld\n",
                        obd->obd_name, exp->exp_client_uuid.uuid, exp,
                        fed->fed_dirty, fed->fed_pending, fed->fed_grant);
-                spin_unlock(&obd->obd_osfs_lock);
+                mutex_up(&ofd->ofd_grant_sem);
                 LBUG();
         }
         return rc;
@@ -385,7 +385,7 @@ int filter_grant_check(const struct lu_env *env, struct obd_export *exp,
         struct filter_export_data *fed = &exp->exp_filter_data;
         int i, rc = -ENOSPC, obj, n = 0;
 
-        LASSERT_SPIN_LOCKED(&exp->exp_obd->obd_osfs_lock);
+        //LASSERT_SPIN_LOCKED(&exp->exp_obd->obd_osfs_lock);
 
         for (obj = 0; obj < objcount; obj++) {
                 for (i = 0; i < objs[obj].ioo_bufcnt; i++, n++) {
@@ -451,7 +451,7 @@ long _filter_grant(const struct lu_env *env, struct obd_export *exp,
         long                        frsize = obd->obd_osfs.os_bsize;
         __u64                       grant = 0;
 
-        LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock);
+        //LASSERT_SPIN_LOCKED(&obd->obd_osfs_lock);
         LASSERT(frsize);
 
         /* Grant some fraction of the client's requested grant space so that
@@ -487,7 +487,7 @@ long _filter_grant(const struct lu_env *env, struct obd_export *exp,
                                "current"LPU64"\n", obd->obd_name,
                                 exp->exp_client_uuid.uuid, exp,
                                 fed->fed_grant, want, curgrant);
-                        spin_unlock(&obd->obd_osfs_lock);
+                        mutex_up(&ofd->ofd_grant_sem);
                         LBUG();
                 }
         }
@@ -530,7 +530,7 @@ void filter_grant_commit(struct obd_export *exp, int niocount,
         unsigned long pending = 0;
         int i;
 
-        spin_lock(&exp->exp_obd->obd_osfs_lock);
+        mutex_down(&ofd->ofd_grant_sem);
         for (i = 0, lnb = res; i < niocount; i++, lnb++)
                 pending += lnb->lnb_grant_used;
 
@@ -550,6 +550,6 @@ void filter_grant_commit(struct obd_export *exp, int niocount,
                  ofd->ofd_tot_pending, pending);
         ofd->ofd_tot_pending -= pending;
 
-        spin_unlock(&exp->exp_obd->obd_osfs_lock);
+        mutex_up(&ofd->ofd_grant_sem);
 }
 

diff --git a/lustre/ofd/ofd_internal.h b/lustre/ofd/ofd_internal.h
index 6b093cb..bb75178 100644 (file)
--- a/lustre/ofd/ofd_internal.h
+++ b/lustre/ofd/ofd_internal.h
@@ -161,6 +161,7 @@ struct filter_device {
         obd_size                 ofd_tot_granted;
         obd_size                 ofd_tot_pending;
         int                      ofd_tot_granted_clients;
+        struct semaphore         ofd_grant_sem;
 
         /* filter mod data: filter_device wide values */
         int                      ofd_fmd_max_num; /* per ofd filter_mod_data */

diff --git a/lustre/ofd/ofd_io.c b/lustre/ofd/ofd_io.c
index 5d9acf6..5d07e7a 100644 (file)
--- a/lustre/ofd/ofd_io.c
+++ b/lustre/ofd/ofd_io.c
@@ -121,7 +121,7 @@ static int filter_preprw_write(const struct lu_env *env, struct obd_export *exp,
         *nr_local = j;
         LASSERT(*nr_local > 0 && *nr_local <= PTLRPC_MAX_BRW_PAGES);
 
-        spin_lock(&exp->exp_obd->obd_osfs_lock);
+        mutex_down(&ofd->ofd_grant_sem);
         filter_grant_incoming(env, exp, oa);
         left = filter_grant_space_left(env, exp);
 
@@ -138,7 +138,7 @@ static int filter_preprw_write(const struct lu_env *env, struct obd_export *exp,
         if (oa->o_valid & OBD_MD_FLGRANT)
                 oa->o_grant = filter_grant(env, exp, oa->o_grant,
                                            oa->o_undirty, left);
-        spin_unlock(&exp->exp_obd->obd_osfs_lock);
+        mutex_up(&ofd->ofd_grant_sem);
 
         rc = dt_write_prep(env, filter_object_child(fo), res, *nr_local, &used);
 
@@ -187,11 +187,11 @@ int filter_preprw(int cmd, struct obd_export *exp, struct obdo *oa, int objcount
                 if (rc == 0) {
                         if (oa && oa->o_valid & OBD_MD_FLGRANT) {
                                 struct obd_device *obd = filter_obd(ofd);
-                                spin_lock(&obd->obd_osfs_lock);
+                                mutex_down(&ofd->ofd_grant_sem);
                                 filter_grant_incoming(&env, exp, oa);
                                 if (!(oa->o_flags & OBD_FL_SHRINK_GRANT))
                                         oa->o_grant = 0;
-                                spin_unlock(&obd->obd_osfs_lock);
+                                mutex_up(&ofd->ofd_grant_sem);
                         }
                         rc = filter_preprw_read(&env, ofd, &info->fti_fid,
                                                 &info->fti_attr, obj->ioo_bufcnt,

diff --git a/lustre/ofd/ofd_obd.c b/lustre/ofd/ofd_obd.c
index e64a739..a6ee7b8 100644 (file)
--- a/lustre/ofd/ofd_obd.c
+++ b/lustre/ofd/ofd_obd.c
@@ -112,12 +112,12 @@ static int filter_parse_connect_data(const struct lu_env *env,
         if (exp->exp_connect_flags & OBD_CONNECT_GRANT) {
                 obd_size left, want;
 
-                spin_lock(&exp->exp_obd->obd_osfs_lock);
+                mutex_down(&ofd->ofd_grant_sem);
                 left = filter_grant_space_left(env, exp);
                 want = data->ocd_grant;
                 filter_grant(env, exp, fed->fed_grant, want, left);
                 data->ocd_grant = fed->fed_grant;
-                spin_unlock(&exp->exp_obd->obd_osfs_lock);
+                mutex_up(&ofd->ofd_grant_sem);
 
                 CDEBUG(D_CACHE, "%s: cli %s/%p ocd_grant: %d want: "
                        LPU64" left: "LPU64"\n", exp->exp_obd->obd_name,
@@ -474,9 +474,9 @@ static int filter_set_info_async(struct obd_export *exp, __u32 keylen,
         if (KEY_IS(KEY_GRANT_SHRINK)) {
                 struct ost_body *body = (struct ost_body *)val;
                 /* handle shrink grant */
-                spin_lock(&exp->exp_obd->obd_osfs_lock);
+                mutex_down(&ofd->ofd_grant_sem);
                 filter_grant_incoming(&env, exp, &body->oa);
-                spin_unlock(&exp->exp_obd->obd_osfs_lock);
+                mutex_up(&ofd->ofd_grant_sem);
                 GOTO(out, rc = 0);
         }