From d761077b0df4dc4922ea60ac7d2474cf518f9ee8 Mon Sep 17 00:00:00 2001 From: adilger Date: Wed, 17 Mar 2004 04:08:51 +0000 Subject: [PATCH] Fix reservation vs. grant locking. b=2059 --- lustre/ChangeLog | 1 + lustre/include/linux/lustre_fsfilt.h | 83 +++++++++++++++++------------------- lustre/include/linux/obd.h | 5 +-- lustre/obdclass/obd_config.c | 8 ++-- lustre/obdfilter/filter_io.c | 2 +- 5 files changed, 46 insertions(+), 53 deletions(-) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index 8d918da..00cb256 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -13,6 +13,7 @@ tbd Cluster File Systems, Inc. - fix race in target_handle_connect (2898) - mds_reint_create() should take same inode create lock (2926) - correct journal credits calculated for CANCEL_UNLINK_LOG (2931) + - don't close files for self_export to avoid uninitialized obd (2936) - allow MDS with the same name as client node (2939) - hold dentry reference for closed log files for unlink (2325) - reserve space for all logs during transactions (2059) diff --git a/lustre/include/linux/lustre_fsfilt.h b/lustre/include/linux/lustre_fsfilt.h index b4f71d3..551f62b 100644 --- a/lustre/include/linux/lustre_fsfilt.h +++ b/lustre/include/linux/lustre_fsfilt.h @@ -98,7 +98,25 @@ struct obd_reservation_handle { int orh_reserve; }; -static inline int fsfilt_reserve(struct obd_device *obd, +/* very similar to obd_statfs(), but caller already holds obd_osfs_lock */ +static inline int fsfilt_statfs(struct obd_device *obd, struct super_block *sb, + unsigned long max_age) +{ + int rc = 0; + + CDEBUG(D_SUPER, "osfs %lu, max_age %lu\n", obd->obd_osfs_age, max_age); + if (time_before(obd->obd_osfs_age, max_age)) { + rc = obd->obd_fsops->fs_statfs(sb, &obd->obd_osfs); + if (rc == 0) /* N.B. statfs can't really fail */ + obd->obd_osfs_age = jiffies; + } else { + CDEBUG(D_SUPER, "using cached obd_statfs data\n"); + } + + return rc; +} + +static inline int fsfilt_reserve(struct obd_device *obd, struct super_block *sb, int reserve, struct obd_reservation_handle **h) { struct obd_reservation_handle *handle; @@ -109,19 +127,13 @@ static inline int fsfilt_reserve(struct obd_device *obd, /* Perform space reservation if needed */ if (reserve) { - down(&obd->obd_reserve_guard); + spin_lock(&obd->obd_osfs_lock); obd->obd_reserve_freespace_estimated -= reserve; if (obd->obd_reserve_freespace_estimated < 0) { - struct obd_statfs osfs; - /* Can we use jiffies here, or is there a race window - where somebody calls obd_statfs(), caches data, then - uses some space, and then we came and get this same - (now stale) cached data all within same jiffie? - maybe jiffies-1 should be used? */ - int rc = obd_statfs(obd, &osfs, jiffies); + int rc = fsfilt_statfs(obd, sb, jiffies - 1); if (rc) { CERROR("statfs failed during reservation\n"); - up(&obd->obd_reserve_guard); + spin_unlock(&obd->obd_osfs_lock); OBD_FREE(handle, sizeof(*handle)); return rc; } @@ -129,18 +141,18 @@ static inline int fsfilt_reserve(struct obd_device *obd, * available compared to what is really available * (reiserfs reserves 1996K for itself). */ - obd->obd_reserve_freespace_estimated = osfs.os_bavail - - obd->obd_reserved_space; + obd->obd_reserve_freespace_estimated = + obd->obd_osfs.os_bfree-obd->obd_reserve_space; if (obd->obd_reserve_freespace_estimated < reserve) { - up(&obd->obd_reserve_guard); + spin_unlock(&obd->obd_osfs_lock); OBD_FREE(handle, sizeof(*handle)); return -ENOSPC; } obd->obd_reserve_freespace_estimated -= reserve; } - obd->obd_reserved_space += reserve; + obd->obd_reserve_space += reserve; handle->orh_reserve = reserve; - up(&obd->obd_reserve_guard); + spin_unlock(&obd->obd_osfs_lock); } *h = handle; return 0; @@ -159,7 +171,7 @@ static inline void *fsfilt_start_log(struct obd_device *obd, if (obd->obd_fsops->fs_get_op_len) reserve = obd->obd_fsops->fs_get_op_len(op, NULL, logs); - rc = fsfilt_reserve(obd, reserve, &h); + rc = fsfilt_reserve(obd, inode->i_sb, reserve, &h); if (rc) return ERR_PTR(rc); @@ -190,7 +202,8 @@ static inline void *fsfilt_start(struct obd_device *obd, return fsfilt_start_log(obd, inode, op, oti, 0); } -static inline void *fsfilt_brw_start_log(struct obd_device *obd, int objcount, +static inline void *fsfilt_brw_start_log(struct obd_device *obd, + int objcount, struct fsfilt_objinfo *fso, int niocount, struct niobuf_local *nb, struct obd_trans_info *oti,int numlogs) @@ -204,7 +217,7 @@ static inline void *fsfilt_brw_start_log(struct obd_device *obd, int objcount, if (obd->obd_fsops->fs_get_op_len) reserve = obd->obd_fsops->fs_get_op_len(objcount, fso, numlogs); - rc = fsfilt_reserve(obd, reserve, &h); + rc = fsfilt_reserve(obd, fso->fso_dentry->d_inode->i_sb, reserve, &h); if (rc) return ERR_PTR(rc); @@ -252,10 +265,10 @@ static inline int fsfilt_commit(struct obd_device *obd, struct inode *inode, if (time_after(jiffies, now + 15 * HZ)) CERROR("long journal start time %lus\n", (jiffies - now) / HZ); - down(&obd->obd_reserve_guard); - obd->obd_reserved_space -= h->orh_reserve; - LASSERT(obd->obd_reserved_space >= 0); - up(&obd->obd_reserve_guard); + spin_lock(&obd->obd_osfs_lock); + obd->obd_reserve_space -= h->orh_reserve; + LASSERT(obd->obd_reserve_space >= 0); + spin_unlock(&obd->obd_osfs_lock); OBD_FREE(h, sizeof(*h)); return rc; @@ -277,10 +290,10 @@ static inline int fsfilt_commit_async(struct obd_device *obd, if (time_after(jiffies, now + 15 * HZ)) CERROR("long journal start time %lus\n", (jiffies - now) / HZ); - down(&obd->obd_reserve_guard); - obd->obd_reserved_space -= h->orh_reserve; - LASSERT(obd->obd_reserved_space >= 0); - up(&obd->obd_reserve_guard); + spin_lock(&obd->obd_osfs_lock); + obd->obd_reserve_space -= h->orh_reserve; + LASSERT(obd->obd_reserve_space >= 0); + spin_unlock(&obd->obd_osfs_lock); OBD_FREE(h, sizeof(*h)); return rc; @@ -346,24 +359,6 @@ static inline int fsfilt_add_journal_cb(struct obd_device *obd, __u64 last_rcvd, cb_data); } -/* very similar to obd_statfs(), but caller already holds obd_osfs_lock */ -static inline int fsfilt_statfs(struct obd_device *obd, struct super_block *sb, - unsigned long max_age) -{ - int rc = 0; - - CDEBUG(D_SUPER, "osfs %lu, max_age %lu\n", obd->obd_osfs_age, max_age); - if (time_before(obd->obd_osfs_age, max_age)) { - rc = obd->obd_fsops->fs_statfs(sb, &obd->obd_osfs); - if (rc == 0) /* N.B. statfs can't really fail */ - obd->obd_osfs_age = jiffies; - } else { - CDEBUG(D_SUPER, "using cached obd_statfs data\n"); - } - - return rc; -} - static inline int fsfilt_sync(struct obd_device *obd, struct super_block *sb) { return obd->obd_fsops->fs_sync(sb); diff --git a/lustre/include/linux/obd.h b/lustre/include/linux/obd.h index b989291..561bfe4 100644 --- a/lustre/include/linux/obd.h +++ b/lustre/include/linux/obd.h @@ -476,10 +476,10 @@ struct obd_device { __u64 obd_last_committed; struct fsfilt_operations *obd_fsops; spinlock_t obd_osfs_lock; - struct llog_ctxt *obd_llog_ctxt[LLOG_MAX_CTXTS]; struct obd_statfs obd_osfs; unsigned long obd_osfs_age; struct obd_run_ctxt obd_ctxt; + struct llog_ctxt *obd_llog_ctxt[LLOG_MAX_CTXTS]; struct obd_device *obd_observer; struct obd_export *obd_self_export; @@ -519,14 +519,13 @@ struct obd_device { struct proc_dir_entry *obd_svc_procroot; struct lprocfs_stats *obd_svc_stats; /* Fields used for fsfilt reservations. */ - int obd_reserved_space; + int obd_reserve_space; /* protected by obd_osfs_lock */ /* This field contains cached statfs(2) amount of free blocks, each time reservation is made, we substract reserved amount from this field until zero is reached. Then we call statfs(2) again. This allows to minimize statfs(2) calls on filesystems with lots of free space. */ long obd_reserve_freespace_estimated; - struct semaphore obd_reserve_guard; }; #define OBD_OPT_FORCE 0x0001 diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index 7570f83..956f8fb 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -125,9 +125,7 @@ int class_attach(struct lustre_cfg *lcfg) spin_lock_init(&obd->obd_osfs_lock); obd->obd_osfs_age = jiffies - 1000 * HZ; init_waitqueue_head(&obd->obd_refcount_waitq); - sema_init(&obd->obd_reserve_guard, 1); - obd->obd_reserved_space=0; - obd->obd_reserve_freespace_estimated=-1; + obd->obd_reserve_freespace_estimated = -1; /* XXX belongs in setup not attach */ /* recovery data */ @@ -236,9 +234,9 @@ int class_detach(struct obd_device *obd, struct lustre_cfg *lcfg) CERROR("OBD device %d not attached\n", obd->obd_minor); RETURN(-ENODEV); } - if (obd->obd_reserved_space != 0) + if (obd->obd_reserve_space != 0) CERROR("Reserved space on class_detach is %d\n", - obd->obd_reserved_space); + obd->obd_reserve_space); if (OBP(obd, detach)) err = OBP(obd,detach)(obd); diff --git a/lustre/obdfilter/filter_io.c b/lustre/obdfilter/filter_io.c index 84831a2..3dce898 100644 --- a/lustre/obdfilter/filter_io.c +++ b/lustre/obdfilter/filter_io.c @@ -151,7 +151,7 @@ static void filter_grant_incoming(struct obd_export *exp, struct obdo *oa) EXIT; } -#define GRANT_FOR_LLOG(obd) (obd->obd_reserved_space) +#define GRANT_FOR_LLOG(obd) (obd->obd_reserve_space) /* Figure out how much space is available between what we've granted * and what remains in the filesystem. Compensate for ext3 indirect -- 1.8.3.1