Fix reservation vs. grant locking.

b=2059

diff --git a/lustre/ChangeLog b/lustre/ChangeLog
index 8d918da..00cb256 100644 (file)
--- a/lustre/ChangeLog
+++ b/lustre/ChangeLog
@@ -13,6 +13,7 @@ tbd  Cluster File Systems, Inc. <info@clusterfs.com>
        - fix race in target_handle_connect (2898)
        - mds_reint_create() should take same inode create lock (2926)
        - correct journal credits calculated for CANCEL_UNLINK_LOG (2931)
+       - don't close files for self_export to avoid uninitialized obd (2936)
        - allow MDS with the same name as client node (2939)
        - hold dentry reference for closed log files for unlink (2325)
        - reserve space for all logs during transactions (2059)

diff --git a/lustre/include/linux/lustre_fsfilt.h b/lustre/include/linux/lustre_fsfilt.h
index b4f71d3..551f62b 100644 (file)
--- a/lustre/include/linux/lustre_fsfilt.h
+++ b/lustre/include/linux/lustre_fsfilt.h
@@ -98,7 +98,25 @@ struct obd_reservation_handle {
         int orh_reserve;
 };
 
-static inline int fsfilt_reserve(struct obd_device *obd,
+/* very similar to obd_statfs(), but caller already holds obd_osfs_lock */
+static inline int fsfilt_statfs(struct obd_device *obd, struct super_block *sb,
+                                unsigned long max_age)
+{
+        int rc = 0;
+
+        CDEBUG(D_SUPER, "osfs %lu, max_age %lu\n", obd->obd_osfs_age, max_age);
+        if (time_before(obd->obd_osfs_age, max_age)) {
+                rc = obd->obd_fsops->fs_statfs(sb, &obd->obd_osfs);
+                if (rc == 0) /* N.B. statfs can't really fail */
+                        obd->obd_osfs_age = jiffies;
+        } else {
+                CDEBUG(D_SUPER, "using cached obd_statfs data\n");
+        }
+
+        return rc;
+}
+
+static inline int fsfilt_reserve(struct obd_device *obd, struct super_block *sb,
                                  int reserve, struct obd_reservation_handle **h)
 {
         struct obd_reservation_handle *handle;
@@ -109,19 +127,13 @@ static inline int fsfilt_reserve(struct obd_device *obd,
 
         /* Perform space reservation if needed */
         if (reserve) {
-                down(&obd->obd_reserve_guard);
+                spin_lock(&obd->obd_osfs_lock);
                 obd->obd_reserve_freespace_estimated -= reserve;
                 if (obd->obd_reserve_freespace_estimated < 0) {
-                        struct obd_statfs osfs;
-                        /* Can we use jiffies here, or is there a race window
-                           where somebody calls obd_statfs(), caches data, then
-                           uses some space, and then we came and get this same
-                           (now stale) cached data all within same jiffie?
-                           maybe jiffies-1 should be used? */
-                        int rc = obd_statfs(obd, &osfs, jiffies);
+                        int rc = fsfilt_statfs(obd, sb, jiffies - 1);
                         if (rc) {
                                 CERROR("statfs failed during reservation\n");
-                                up(&obd->obd_reserve_guard);
+                                spin_unlock(&obd->obd_osfs_lock);
                                 OBD_FREE(handle, sizeof(*handle));
                                 return rc;
                         }
@@ -129,18 +141,18 @@ static inline int fsfilt_reserve(struct obd_device *obd,
                          * available compared to what is really available
                          * (reiserfs reserves 1996K for itself).
                          */
-                        obd->obd_reserve_freespace_estimated = osfs.os_bavail -
-                                                        obd->obd_reserved_space;
+                        obd->obd_reserve_freespace_estimated =
+                                obd->obd_osfs.os_bfree-obd->obd_reserve_space;
                         if (obd->obd_reserve_freespace_estimated < reserve) {
-                                up(&obd->obd_reserve_guard);
+                                spin_unlock(&obd->obd_osfs_lock);
                                 OBD_FREE(handle, sizeof(*handle));
                                 return -ENOSPC;
                         }
                         obd->obd_reserve_freespace_estimated -= reserve;
                 }
-                obd->obd_reserved_space += reserve;
+                obd->obd_reserve_space += reserve;
                 handle->orh_reserve = reserve;
-                up(&obd->obd_reserve_guard);
+                spin_unlock(&obd->obd_osfs_lock);
         }
         *h = handle;
         return 0;
@@ -159,7 +171,7 @@ static inline void *fsfilt_start_log(struct obd_device *obd,
         if (obd->obd_fsops->fs_get_op_len)
                 reserve = obd->obd_fsops->fs_get_op_len(op, NULL, logs);
 
-        rc = fsfilt_reserve(obd, reserve, &h);
+        rc = fsfilt_reserve(obd, inode->i_sb, reserve, &h);
         if (rc)
                 return ERR_PTR(rc);
 
@@ -190,7 +202,8 @@ static inline void *fsfilt_start(struct obd_device *obd,
         return fsfilt_start_log(obd, inode, op, oti, 0);
 }
 
-static inline void *fsfilt_brw_start_log(struct obd_device *obd, int objcount,
+static inline void *fsfilt_brw_start_log(struct obd_device *obd,
+                                         int objcount,
                                          struct fsfilt_objinfo *fso,
                                          int niocount, struct niobuf_local *nb,
                                          struct obd_trans_info *oti,int numlogs)
@@ -204,7 +217,7 @@ static inline void *fsfilt_brw_start_log(struct obd_device *obd, int objcount,
         if (obd->obd_fsops->fs_get_op_len)
                 reserve = obd->obd_fsops->fs_get_op_len(objcount, fso, numlogs);
 
-        rc = fsfilt_reserve(obd, reserve, &h);
+        rc = fsfilt_reserve(obd, fso->fso_dentry->d_inode->i_sb, reserve, &h);
         if (rc)
                 return ERR_PTR(rc);
 
@@ -252,10 +265,10 @@ static inline int fsfilt_commit(struct obd_device *obd, struct inode *inode,
         if (time_after(jiffies, now + 15 * HZ))
                 CERROR("long journal start time %lus\n", (jiffies - now) / HZ);
 
-        down(&obd->obd_reserve_guard);
-        obd->obd_reserved_space -= h->orh_reserve;
-        LASSERT(obd->obd_reserved_space >= 0);
-        up(&obd->obd_reserve_guard);
+        spin_lock(&obd->obd_osfs_lock);
+        obd->obd_reserve_space -= h->orh_reserve;
+        LASSERT(obd->obd_reserve_space >= 0);
+        spin_unlock(&obd->obd_osfs_lock);
         OBD_FREE(h, sizeof(*h));
 
         return rc;
@@ -277,10 +290,10 @@ static inline int fsfilt_commit_async(struct obd_device *obd,
         if (time_after(jiffies, now + 15 * HZ))
                 CERROR("long journal start time %lus\n", (jiffies - now) / HZ);
 
-        down(&obd->obd_reserve_guard);
-        obd->obd_reserved_space -= h->orh_reserve;
-        LASSERT(obd->obd_reserved_space >= 0);
-        up(&obd->obd_reserve_guard);
+        spin_lock(&obd->obd_osfs_lock);
+        obd->obd_reserve_space -= h->orh_reserve;
+        LASSERT(obd->obd_reserve_space >= 0);
+        spin_unlock(&obd->obd_osfs_lock);
         OBD_FREE(h, sizeof(*h));
 
         return rc;
@@ -346,24 +359,6 @@ static inline int fsfilt_add_journal_cb(struct obd_device *obd, __u64 last_rcvd,
                                                  cb_data);
 }
 
-/* very similar to obd_statfs(), but caller already holds obd_osfs_lock */
-static inline int fsfilt_statfs(struct obd_device *obd, struct super_block *sb,
-                                unsigned long max_age)
-{
-        int rc = 0;
-
-        CDEBUG(D_SUPER, "osfs %lu, max_age %lu\n", obd->obd_osfs_age, max_age);
-        if (time_before(obd->obd_osfs_age, max_age)) {
-                rc = obd->obd_fsops->fs_statfs(sb, &obd->obd_osfs);
-                if (rc == 0) /* N.B. statfs can't really fail */
-                        obd->obd_osfs_age = jiffies;
-        } else {
-                CDEBUG(D_SUPER, "using cached obd_statfs data\n");
-        }
-
-        return rc;
-}
-
 static inline int fsfilt_sync(struct obd_device *obd, struct super_block *sb)
 {
         return obd->obd_fsops->fs_sync(sb);

diff --git a/lustre/include/linux/obd.h b/lustre/include/linux/obd.h
index b989291..561bfe4 100644 (file)
--- a/lustre/include/linux/obd.h
+++ b/lustre/include/linux/obd.h
@@ -476,10 +476,10 @@ struct obd_device {
         __u64                  obd_last_committed;
         struct fsfilt_operations *obd_fsops;
         spinlock_t              obd_osfs_lock;
-        struct llog_ctxt        *obd_llog_ctxt[LLOG_MAX_CTXTS];
         struct obd_statfs       obd_osfs;
         unsigned long           obd_osfs_age;
         struct obd_run_ctxt     obd_ctxt;
+        struct llog_ctxt        *obd_llog_ctxt[LLOG_MAX_CTXTS];
         struct obd_device       *obd_observer;
         struct obd_export       *obd_self_export;
 
@@ -519,14 +519,13 @@ struct obd_device {
         struct proc_dir_entry *obd_svc_procroot;
         struct lprocfs_stats  *obd_svc_stats;
         /* Fields used for fsfilt reservations. */
-        int  obd_reserved_space;
+        int  obd_reserve_space;        /* protected by obd_osfs_lock */
         /* This field contains cached statfs(2) amount of free blocks,
            each time reservation is made, we substract reserved amount from this
            field until zero is reached. Then we call statfs(2) again. This
            allows to minimize statfs(2) calls on filesystems with lots of free
            space. */
         long obd_reserve_freespace_estimated;
-        struct semaphore obd_reserve_guard;
 };
 
 #define OBD_OPT_FORCE           0x0001

diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c
index 7570f83..956f8fb 100644 (file)
--- a/lustre/obdclass/obd_config.c
+++ b/lustre/obdclass/obd_config.c
@@ -125,9 +125,7 @@ int class_attach(struct lustre_cfg *lcfg)
         spin_lock_init(&obd->obd_osfs_lock);
         obd->obd_osfs_age = jiffies - 1000 * HZ;
         init_waitqueue_head(&obd->obd_refcount_waitq);
-        sema_init(&obd->obd_reserve_guard, 1);
-        obd->obd_reserved_space=0;
-        obd->obd_reserve_freespace_estimated=-1;
+        obd->obd_reserve_freespace_estimated = -1;
 
         /* XXX belongs in setup not attach  */
         /* recovery data */
@@ -236,9 +234,9 @@ int class_detach(struct obd_device *obd, struct lustre_cfg *lcfg)
                 CERROR("OBD device %d not attached\n", obd->obd_minor);
                 RETURN(-ENODEV);
         }
-        if (obd->obd_reserved_space != 0)
+        if (obd->obd_reserve_space != 0)
                 CERROR("Reserved space on class_detach is %d\n",
-                       obd->obd_reserved_space);
+                       obd->obd_reserve_space);
         if (OBP(obd, detach))
                 err = OBP(obd,detach)(obd);
 

diff --git a/lustre/obdfilter/filter_io.c b/lustre/obdfilter/filter_io.c
index 84831a2..3dce898 100644 (file)
--- a/lustre/obdfilter/filter_io.c
+++ b/lustre/obdfilter/filter_io.c
@@ -151,7 +151,7 @@ static void filter_grant_incoming(struct obd_export *exp, struct obdo *oa)
         EXIT;
 }
 
-#define GRANT_FOR_LLOG(obd) (obd->obd_reserved_space)
+#define GRANT_FOR_LLOG(obd) (obd->obd_reserve_space)
 
 /* Figure out how much space is available between what we've granted
  * and what remains in the filesystem.  Compensate for ext3 indirect