From 3a190dfbb6e19f03da6603b12b626ab332197be3 Mon Sep 17 00:00:00 2001 From: yury Date: Fri, 28 Jan 2005 17:20:46 +0000 Subject: [PATCH] - fixed problem with sid of virtual directory "FIDS". By now it is saved to store just like last_rcvd and last_fid and read from there on startup. - a lot of changes about last_fid management. last_fid_changed flag is removed and last_fid_sem is replaced by spin lock. --- lustre/include/linux/obd.h | 5 ++- lustre/mds/handler.c | 79 ++++++++++++++++++----------------- lustre/mds/lproc_mds.c | 4 +- lustre/mds/mds_fs.c | 101 ++++++++++++++++++++++++++++++++++++++++++--- lustre/mds/mds_internal.h | 2 + 5 files changed, 144 insertions(+), 47 deletions(-) diff --git a/lustre/include/linux/obd.h b/lustre/include/linux/obd.h index 52d73c4..7561d41 100644 --- a/lustre/include/linux/obd.h +++ b/lustre/include/linux/obd.h @@ -337,14 +337,15 @@ struct mds_obd { int mds_max_cookiesize; struct file *mds_rcvd_filp; struct file *mds_fid_filp; + struct file *mds_virtid_filp; spinlock_t mds_transno_lock; __u64 mds_last_transno; __u64 mds_mount_count; __u64 mds_io_epoch; __u64 mds_last_fid; - struct semaphore mds_last_fid_sem; - int mds_last_fid_changed; + __u64 mds_virtid_fid; + spinlock_t mds_last_fid_lock; struct semaphore mds_epoch_sem; struct lustre_id mds_rootid; diff --git a/lustre/mds/handler.c b/lustre/mds/handler.c index 60438d4..090f6b8 100644 --- a/lustre/mds/handler.c +++ b/lustre/mds/handler.c @@ -2425,37 +2425,36 @@ int mds_update_last_fid(struct obd_device *obd, void *handle, struct file *filp = mds->mds_fid_filp; struct lvfs_run_ctxt saved; loff_t off = 0; + __u64 last_fid; int rc = 0; ENTRY; - down(&mds->mds_last_fid_sem); - if (mds->mds_last_fid_changed) { - CDEBUG(D_SUPER, "MDS last_fid is #"LPU64"\n", - mds->mds_last_fid); + spin_lock(&mds->mds_last_fid_lock); + last_fid = mds->mds_last_fid; + spin_unlock(&mds->mds_last_fid_lock); - if (handle) { - fsfilt_add_journal_cb(obd, mds->mds_sb, - mds->mds_last_fid, handle, - mds_commit_last_fid_cb, NULL); - } - - push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - rc = fsfilt_write_record(obd, filp, &mds->mds_last_fid, - sizeof(mds->mds_last_fid), - &off, force_sync); - if (rc) { - CERROR("error writing MDS last_fid #"LPU64 - ", err = %d\n", mds->mds_last_fid, rc); - } else { - mds->mds_last_fid_changed = 0; - } + CDEBUG(D_SUPER, "MDS last_fid is #"LPU64"\n", + last_fid); + + if (handle) { + fsfilt_add_journal_cb(obd, mds->mds_sb, last_fid, + handle, mds_commit_last_fid_cb, + NULL); + } - CDEBUG(D_SUPER, "wrote fid #"LPU64" at idx " - "%llu: err = %d\n", mds->mds_last_fid, - off, rc); - pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + rc = fsfilt_write_record(obd, filp, &last_fid, sizeof(last_fid), + &off, force_sync); + pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + + if (rc) { + CERROR("error writing MDS last_fid #"LPU64 + ", err = %d\n", last_fid, rc); + RETURN(rc); } - up(&mds->mds_last_fid_sem); + + CDEBUG(D_SUPER, "wrote fid #"LPU64" at idx " + "%llu: err = %d\n", last_fid, off, rc); RETURN(rc); } @@ -2464,12 +2463,10 @@ void mds_set_last_fid(struct obd_device *obd, __u64 fid) { struct mds_obd *mds = &obd->u.mds; - down(&mds->mds_last_fid_sem); - if (fid > mds->mds_last_fid) { + spin_lock(&mds->mds_last_fid_lock); + if (fid > mds->mds_last_fid) mds->mds_last_fid = fid; - mds->mds_last_fid_changed = 1; - } - up(&mds->mds_last_fid_sem); + spin_unlock(&mds->mds_last_fid_lock); } void mds_commit_last_transno_cb(struct obd_device *obd, @@ -2493,6 +2490,18 @@ void mds_commit_last_fid_cb(struct obd_device *obd, obd->obd_name, fid); } +__u64 mds_alloc_fid(struct obd_device *obd) +{ + struct mds_obd *mds = &obd->u.mds; + __u64 fid; + + spin_lock(&mds->mds_last_fid_lock); + fid = ++mds->mds_last_fid; + spin_unlock(&mds->mds_last_fid_lock); + + return fid; +} + /* * allocates new lustre_id on passed @inode and saves it to inode EA. */ @@ -2514,12 +2523,7 @@ int mds_alloc_inode_sid(struct obd_device *obd, struct inode *inode, } id_group(id) = mds->mds_num; - - down(&mds->mds_last_fid_sem); - mds->mds_last_fid_changed = 1; - id_fid(id) = ++mds->mds_last_fid; - up(&mds->mds_last_fid_sem); - + id_fid(id) = mds_alloc_fid(obd); id_ino(id) = inode->i_ino; id_gen(id) = inode->i_generation; id_type(id) = (S_IFMT & inode->i_mode); @@ -2732,11 +2736,10 @@ static int mds_setup(struct obd_device *obd, obd_count len, void *buf) CDEBUG(D_SUPER, "%s: mnt = %p\n", lcfg->lcfg_inlbuf1, mnt); - mds->mds_last_fid_changed = 0; sema_init(&mds->mds_epoch_sem, 1); - sema_init(&mds->mds_last_fid_sem, 1); atomic_set(&mds->mds_real_clients, 0); spin_lock_init(&mds->mds_transno_lock); + spin_lock_init(&mds->mds_last_fid_lock); sema_init(&mds->mds_orphan_recovery_sem, 1); mds->mds_max_cookiesize = sizeof(struct llog_cookie); diff --git a/lustre/mds/lproc_mds.c b/lustre/mds/lproc_mds.c index dd39312..8eb6819 100644 --- a/lustre/mds/lproc_mds.c +++ b/lustre/mds/lproc_mds.c @@ -103,9 +103,9 @@ static int lprocfs_rd_last_fid(char *page, char **start, off_t off, struct mds_obd *mds = &obd->u.mds; __u64 last_fid; - down(&mds->mds_last_fid_sem); + spin_lock(&mds->mds_last_fid_lock); last_fid = mds->mds_last_fid; - up(&mds->mds_last_fid_sem); + spin_unlock(&mds->mds_last_fid_lock); *eof = 1; return snprintf(page, count, LPD64"\n", last_fid); diff --git a/lustre/mds/mds_fs.c b/lustre/mds/mds_fs.c index 88159ab..8242a7a 100644 --- a/lustre/mds/mds_fs.c +++ b/lustre/mds/mds_fs.c @@ -51,6 +51,7 @@ #define LAST_RCVD "last_rcvd" #define LOV_OBJID "lov_objid" #define LAST_FID "last_fid" +#define VIRT_FID "virt_fid" /* Add client data to the MDS. We use a bitmap to locate a free space * in the last_rcvd file if cl_off is -1 (i.e. a new client). @@ -523,6 +524,58 @@ out_dentry: return rc; } +static int mds_update_virtid_fid(struct obd_device *obd, + void *handle, int force_sync) +{ + struct mds_obd *mds = &obd->u.mds; + struct file *filp = mds->mds_virtid_filp; + struct lvfs_run_ctxt saved; + loff_t off = 0; + int rc = 0; + ENTRY; + + push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + rc = fsfilt_write_record(obd, filp, &mds->mds_virtid_fid, + sizeof(mds->mds_virtid_fid), + &off, force_sync); + if (rc) { + CERROR("error writing MDS virtid_fid #"LPU64 + ", err = %d\n", mds->mds_virtid_fid, rc); + } + + CDEBUG(D_SUPER, "wrote virtid fid #"LPU64" at idx " + "%llu: err = %d\n", mds->mds_virtid_fid, + off, rc); + pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + + RETURN(rc); +} + +static int mds_read_virtid_fid(struct obd_device *obd, + struct file *file) +{ + int rc = 0; + loff_t off = 0; + struct mds_obd *mds = &obd->u.mds; + unsigned long virtid_fid_size = file->f_dentry->d_inode->i_size; + ENTRY; + + if (virtid_fid_size == 0) { + mds->mds_virtid_fid = mds_alloc_fid(obd); + } else { + rc = fsfilt_read_record(obd, file, &mds->mds_virtid_fid, + sizeof(mds->mds_virtid_fid), &off); + if (rc) { + CERROR("error reading MDS %s: rc = %d\n", + file->f_dentry->d_name.name, rc); + RETURN(rc); + } + } + rc = mds_update_virtid_fid(obd, NULL, 1); + + RETURN(rc); +} + /* * initializes lustre_id for virtual id directory, it is needed sometimes, as it * is possible that it will be the parent for object an operations is going to @@ -532,6 +585,7 @@ int mds_fs_setup_virtid(struct obd_device *obd) { int rc = 0; void *handle; + struct lustre_id sid; struct mds_obd *mds = &obd->u.mds; struct inode *inode = mds->mds_id_dir->d_inode; ENTRY; @@ -544,13 +598,20 @@ int mds_fs_setup_virtid(struct obd_device *obd) CERROR("fsfilt_start() failed, rc = %d\n", rc); RETURN(rc); } - + + id_group(&sid) = mds->mds_num; + id_fid(&sid) = mds->mds_virtid_fid; + + id_ino(&sid) = inode->i_ino; + id_gen(&sid) = inode->i_generation; + id_type(&sid) = (S_IFMT & inode->i_mode); + down(&inode->i_sem); - rc = mds_alloc_inode_sid(obd, inode, handle, NULL); + rc = mds_update_inode_sid(obd, inode, handle, &sid); up(&inode->i_sem); - + if (rc) { - CERROR("mds_alloc_inode_sid() failed, rc = %d\n", + CERROR("mds_update_inode_sid() failed, rc = %d\n", rc); RETURN(rc); } @@ -678,7 +739,7 @@ int mds_fs_setup(struct obd_device *obd, struct vfsmount *mnt) GOTO(err_last_rcvd, rc); } - /* open and test the last fid file */ + /* open and test last fid file */ file = filp_open(LAST_FID, O_RDWR | O_CREAT, 0644); if (IS_ERR(file)) { rc = PTR_ERR(file); @@ -699,6 +760,27 @@ int mds_fs_setup(struct obd_device *obd, struct vfsmount *mnt) GOTO(err_last_fid, rc); } + /* open and test virtid fid file */ + file = filp_open(VIRT_FID, O_RDWR | O_CREAT, 0644); + if (IS_ERR(file)) { + rc = PTR_ERR(file); + CERROR("cannot open/create %s file: rc = %d\n", + VIRT_FID, rc); + GOTO(err_last_fid, rc = PTR_ERR(file)); + } + mds->mds_virtid_filp = file; + if (!S_ISREG(file->f_dentry->d_inode->i_mode)) { + CERROR("%s is not a regular file!: mode = %o\n", + VIRT_FID, file->f_dentry->d_inode->i_mode); + GOTO(err_virtid_fid, rc = -ENOENT); + } + + rc = mds_read_virtid_fid(obd, file); + if (rc) { + CERROR("cannot read %s: rc = %d\n", VIRT_FID, rc); + GOTO(err_virtid_fid, rc); + } + /* open and test the lov objid file */ file = filp_open(LOV_OBJID, O_RDWR | O_CREAT, 0644); if (IS_ERR(file)) { @@ -724,6 +806,9 @@ err_pop: err_lov_objid: if (mds->mds_lov_objid_filp && filp_close(mds->mds_lov_objid_filp, 0)) CERROR("can't close %s after error\n", LOV_OBJID); +err_virtid_fid: + if (mds->mds_virtid_filp && filp_close(mds->mds_virtid_filp, 0)) + CERROR("can't close %s after error\n", VIRT_FID); err_last_fid: if (mds->mds_fid_filp && filp_close(mds->mds_fid_filp, 0)) CERROR("can't close %s after error\n", LAST_FID); @@ -769,6 +854,12 @@ int mds_fs_cleanup(struct obd_device *obd, int flags) mds_server_free_data(mds); push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + if (mds->mds_virtid_filp) { + rc = filp_close(mds->mds_virtid_filp, 0); + mds->mds_virtid_filp = NULL; + if (rc) + CERROR("%s file won't close, rc = %d\n", VIRT_FID, rc); + } if (mds->mds_fid_filp) { rc = filp_close(mds->mds_fid_filp, 0); mds->mds_fid_filp = NULL; diff --git a/lustre/mds/mds_internal.h b/lustre/mds/mds_internal.h index 448312f..f09833e 100644 --- a/lustre/mds/mds_internal.h +++ b/lustre/mds/mds_internal.h @@ -174,6 +174,8 @@ extern int mds_lock_mode_for_dir(struct obd_device *, int mds_fs_setup_rootid(struct obd_device *obd); int mds_fs_setup_virtid(struct obd_device *obd); +__u64 mds_alloc_fid(struct obd_device *obd); + int mds_alloc_inode_sid(struct obd_device *, struct inode *, void *, struct lustre_id *); -- 1.8.3.1