From: adilger Date: Thu, 31 Oct 2002 07:39:18 +0000 (+0000) Subject: Hack to force an MDS flush and also get the last_xid/last_committed from X-Git-Tag: 0.5.16~57 X-Git-Url: https://git.whamcloud.com/gitweb?a=commitdiff_plain;h=1c83e36b0591a065c279d8dc3e137fb6b0fc032e;p=fs%2Flustre-release.git Hack to force an MDS flush and also get the last_xid/last_committed from the MDS, so the client can flush its pending RPC requests before umount. Also, make mdc_statfs() into a OBD method, just because we can. --- diff --git a/lustre/include/linux/lustre_mds.h b/lustre/include/linux/lustre_mds.h index dfa3bec..b6b4551 100644 --- a/lustre/include/linux/lustre_mds.h +++ b/lustre/include/linux/lustre_mds.h @@ -157,14 +157,10 @@ int mdc_enqueue(struct lustre_handle *conn, int lock_type, int mdc_cancel_unused(struct lustre_handle *conn, struct inode *, int flags); int mdc_getlovinfo(struct obd_device *obd, struct lustre_handle *mdc_connh, struct ptlrpc_request **request); -int mdc_getstatus(struct lustre_handle *conn, - struct ll_fid *rootfid, __u64 *last_committed, - __u64 *last_xid, struct ptlrpc_request **); +int mdc_getstatus(struct lustre_handle *conn, struct ll_fid *rootfid); int mdc_getattr(struct lustre_handle *conn, obd_id ino, int type, unsigned long valid, size_t ea_size, struct ptlrpc_request **request); -int mdc_statfs(struct lustre_handle *conn, struct obd_statfs *osfs, - struct ptlrpc_request **request); int mdc_setattr(struct lustre_handle *conn, struct inode *, struct iattr *iattr, struct ptlrpc_request **); int mdc_open(struct lustre_handle *conn, obd_id ino, int type, int flags, diff --git a/lustre/llite/super.c b/lustre/llite/super.c index 55c7d71..554c28d 100644 --- a/lustre/llite/super.c +++ b/lustre/llite/super.c @@ -104,8 +104,6 @@ static struct super_block * ll_read_super(struct super_block *sb, int err; struct ll_fid rootfid; struct obd_statfs osfs; - __u64 last_committed; - __u64 last_xid; struct ptlrpc_request *request = NULL; struct ptlrpc_connection *mdc_conn; struct ll_read_inode2_cookie lic; @@ -169,21 +167,16 @@ static struct super_block * ll_read_super(struct super_block *sb, GOTO(out_mdc, sb = NULL); } - /* XXX: need to store the last_* values somewhere */ - err = mdc_getstatus(&sbi->ll_mdc_conn, &rootfid, &last_committed, - &last_xid, &request); - ptlrpc_req_finished(request); + err = mdc_getstatus(&sbi->ll_mdc_conn, &rootfid); if (err) { CERROR("cannot mds_connect: rc = %d\n", err); GOTO(out_mdc, sb = NULL); } - CDEBUG(D_SUPER, "rootfid %Ld\n", (unsigned long long)rootfid.id); + CDEBUG(D_SUPER, "rootfid "LPU64"\n", rootfid.id); sbi->ll_rootino = rootfid.id; memset(&osfs, 0, sizeof(osfs)); - request = NULL; - err = mdc_statfs(&sbi->ll_mdc_conn, &osfs, &request); - ptlrpc_req_finished(request); + err = obd_statfs(&sbi->ll_mdc_conn, &osfs); sb->s_blocksize = osfs.os_bsize; sb->s_blocksize_bits = log2(osfs.os_bsize); sb->s_magic = LL_SUPER_MAGIC; @@ -192,7 +185,6 @@ static struct super_block * ll_read_super(struct super_block *sb, sb->s_op = &ll_super_operations; /* make root inode */ - request = NULL; err = mdc_getattr(&sbi->ll_mdc_conn, sbi->ll_rootino, S_IFDIR, OBD_MD_FLNOTOBD|OBD_MD_FLBLOCKS, 0, &request); if (err) { @@ -224,6 +216,7 @@ static struct super_block * ll_read_super(struct super_block *sb, } ptlrpc_req_finished(request); + request = NULL; out_dev: if (mdc) @@ -250,11 +243,21 @@ out_free: static void ll_put_super(struct super_block *sb) { struct ll_sb_info *sbi = ll_s2sbi(sb); - + struct ll_fid rootfid; ENTRY; + list_del(&sbi->ll_conn_chain); ll_commitcbd_cleanup(sbi); obd_disconnect(&sbi->ll_osc_conn); + + /* NULL request to force sync on the MDS, and get the last_committed + * value to flush remaining RPCs from the sending queue on client. + * + * XXX This should be an mdc_sync() call to sync the whole MDS fs, + * which we can call for other reasons as well. + */ + mdc_getstatus(&sbi->ll_mdc_conn, &rootfid); + obd_disconnect(&sbi->ll_mdc_conn); OBD_FREE(sbi, sizeof(*sbi)); @@ -392,16 +395,14 @@ int ll_setattr(struct dentry *de, struct iattr *attr) static int ll_statfs(struct super_block *sb, struct statfs *sfs) { - struct ptlrpc_request *request = NULL; struct ll_sb_info *sbi = ll_s2sbi(sb); struct obd_statfs osfs; int rc; ENTRY; memset(sfs, 0, sizeof(*sfs)); - rc = mdc_statfs(&sbi->ll_mdc_conn, &osfs, &request); + rc = obd_statfs(&sbi->ll_mdc_conn, &osfs); statfs_unpack(sfs, &osfs); - ptlrpc_req_finished(request); if (rc) CERROR("mdc_statfs fails: rc = %d\n", rc); else diff --git a/lustre/llite/super25.c b/lustre/llite/super25.c index f7521e6..7af645b 100644 --- a/lustre/llite/super25.c +++ b/lustre/llite/super25.c @@ -106,8 +106,6 @@ static int ll_fill_super(struct super_block *sb, void *data, int silent) int err; struct ll_fid rootfid; struct obd_statfs osfs; - __u64 last_committed; - __u64 last_xid; struct ptlrpc_request *request = NULL; struct ptlrpc_connection *mdc_conn; struct ll_read_inode2_cookie lic; @@ -171,21 +169,16 @@ static int ll_fill_super(struct super_block *sb, void *data, int silent) GOTO(out_mdc, sb = NULL); } - /* XXX: need to store the last_* values somewhere */ - err = mdc_getstatus(&sbi->ll_mdc_conn, &rootfid, &last_committed, - &last_xid, &request); - ptlrpc_req_finished(request); + err = mdc_getstatus(&sbi->ll_mdc_conn, &rootfid); if (err) { CERROR("cannot mds_connect: rc = %d\n", err); GOTO(out_mdc, sb = NULL); } - CDEBUG(D_SUPER, "rootfid %Ld\n", (unsigned long long)rootfid.id); + CDEBUG(D_SUPER, "rootfid "LPU64"\n", rootfid.id); sbi->ll_rootino = rootfid.id; memset(&osfs, 0, sizeof(osfs)); - request = NULL; - err = mdc_statfs(&sbi->ll_mdc_conn, &osfs, &request); - ptlrpc_req_finished(request); + err = mdc_statfs(&sbi->ll_mdc_conn, &osfs); sb->s_blocksize = osfs.os_bsize; sb->s_blocksize_bits = log2(osfs.os_bsize); sb->s_magic = LL_SUPER_MAGIC; @@ -194,7 +187,6 @@ static int ll_fill_super(struct super_block *sb, void *data, int silent) sb->s_op = &ll_super_operations; /* make root inode */ - request = NULL; err = mdc_getattr(&sbi->ll_mdc_conn, sbi->ll_rootino, S_IFDIR, OBD_MD_FLNOTOBD|OBD_MD_FLBLOCKS, 0, &request); if (err) { @@ -226,6 +218,7 @@ static int ll_fill_super(struct super_block *sb, void *data, int silent) } ptlrpc_req_finished(request); + request = NULL; out_dev: if (mdc) @@ -252,17 +245,27 @@ out_free: struct super_block * ll_get_sb(struct file_system_type *fs_type, int flags, char *devname, void * data) { - return get_sb_nodev(fs_type, flags, data, ll_fill_super); + return get_sb_nodev(fs_type, flags, data, ll_fill_super); } static void ll_put_super(struct super_block *sb) { struct ll_sb_info *sbi = ll_s2sbi(sb); - + struct ll_fid rootfid; ENTRY; + list_del(&sbi->ll_conn_chain); ll_commitcbd_cleanup(sbi); obd_disconnect(&sbi->ll_osc_conn); + + /* NULL request to force sync on the MDS, and get the last_committed + * value to flush remaining RPCs from the pending queue on client. + * + * XXX This should be an mdc_sync() call to sync the whole MDS fs, + * which we can call for other reasons as well. + */ + mdc_getstatus(&sbi->ll_mdc_conn, &rootfid); + obd_disconnect(&sbi->ll_mdc_conn); OBD_FREE(sbi, sizeof(*sbi)); @@ -391,16 +394,14 @@ int ll_setattr(struct dentry *de, struct iattr *attr) static int ll_statfs(struct super_block *sb, struct statfs *sfs) { - struct ptlrpc_request *request = NULL; struct ll_sb_info *sbi = ll_s2sbi(sb); struct obd_statfs osfs; int rc; ENTRY; memset(sfs, 0, sizeof(*sfs)); - rc = mdc_statfs(&sbi->ll_mdc_conn, &osfs, &request); + rc = obd_statfs(&sbi->ll_mdc_conn, &osfs); statfs_unpack(sfs, &osfs); - ptlrpc_req_finished(request); if (rc) CERROR("mdc_statfs fails: rc = %d\n", rc); else @@ -581,46 +582,46 @@ static kmem_cache_t *ll_inode_cachep; static struct inode *ll_alloc_inode(struct super_block *sb) { - struct ll_inode_info *lli; - lli = kmem_cache_alloc(ll_inode_cachep, SLAB_KERNEL); - if (!lli) - return NULL; + struct ll_inode_info *lli; + lli = kmem_cache_alloc(ll_inode_cachep, SLAB_KERNEL); + if (!lli) + return NULL; - memset(lli, 0, (char *)&lli->lli_vfs_inode - (char *)lli); + memset(lli, 0, (char *)&lli->lli_vfs_inode - (char *)lli); sema_init(&lli->lli_open_sem, 1); - return &lli->lli_vfs_inode; + return &lli->lli_vfs_inode; } static void ll_destroy_inode(struct inode *inode) { - kmem_cache_free(ll_inode_cachep, ll_i2info(inode)); + kmem_cache_free(ll_inode_cachep, ll_i2info(inode)); } static void init_once(void * foo, kmem_cache_t * cachep, unsigned long flags) { - struct ll_inode_info *lli = foo; + struct ll_inode_info *lli = foo; - if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) == - SLAB_CTOR_CONSTRUCTOR) - inode_init_once(&lli->lli_vfs_inode); + if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) == + SLAB_CTOR_CONSTRUCTOR) + inode_init_once(&lli->lli_vfs_inode); } - + int ll_init_inodecache(void) { - ll_inode_cachep = kmem_cache_create("lustre_inode_cache", - sizeof(struct ll_inode_info), - 0, SLAB_HWCACHE_ALIGN, - init_once, NULL); - if (ll_inode_cachep == NULL) - return -ENOMEM; - return 0; + ll_inode_cachep = kmem_cache_create("lustre_inode_cache", + sizeof(struct ll_inode_info), + 0, SLAB_HWCACHE_ALIGN, + init_once, NULL); + if (ll_inode_cachep == NULL) + return -ENOMEM; + return 0; } void ll_destroy_inodecache(void) { - if (kmem_cache_destroy(ll_inode_cachep)) - CERROR("ll_inode_cache: not all structures were freed\n"); + if (kmem_cache_destroy(ll_inode_cachep)) + CERROR("ll_inode_cache: not all structures were freed\n"); } diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index 7c63d23..ff7334d 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -35,9 +35,8 @@ extern int mds_queue_req(struct ptlrpc_request *); -int mdc_getstatus(struct lustre_handle *conn, struct ll_fid *rootfid, - __u64 *last_committed, __u64 *last_xid, - struct ptlrpc_request **request) +/* should become mdc_getinfo() */ +int mdc_getstatus(struct lustre_handle *conn, struct ll_fid *rootfid) { struct ptlrpc_request *req; struct mds_body *body; @@ -61,13 +60,11 @@ int mdc_getstatus(struct lustre_handle *conn, struct ll_fid *rootfid, body = lustre_msg_buf(req->rq_repmsg, 0); mds_unpack_body(body); memcpy(rootfid, &body->fid1, sizeof(*rootfid)); - *last_committed = req->rq_repmsg->last_committed; - *last_xid = req->rq_repmsg->last_xid; - CDEBUG(D_NET,"root ino=%ld, last_committed=%Lu, last_xid=%Ld\n", - (unsigned long)rootfid->id, - (unsigned long long)*last_committed, - (unsigned long long)*last_xid); + CDEBUG(D_NET, "root ino="LPU64", last_committed="LPU64 + ", last_xid="LPU64"\n", + rootfid->id, req->rq_repmsg->last_committed, + req->rq_repmsg->last_xid); } EXIT; @@ -751,8 +748,7 @@ int mdc_readpage(struct lustre_handle *conn, obd_id ino, int type, __u64 offset, return rc; } -int mdc_statfs(struct lustre_handle *conn, struct obd_statfs *osfs, - struct ptlrpc_request **request) +static int mdc_statfs(struct lustre_handle *conn, struct obd_statfs *osfs) { struct ptlrpc_request *req; int rc, size = sizeof(*osfs); @@ -761,7 +757,8 @@ int mdc_statfs(struct lustre_handle *conn, struct obd_statfs *osfs, req = ptlrpc_prep_req(class_conn2cliimp(conn), MDS_STATFS, 0, NULL, NULL); if (!req) - GOTO(out, rc = -ENOMEM); + RETURN(-ENOMEM); + req->rq_replen = lustre_msg_size(1, &size); rc = ptlrpc_queue_wait(req); @@ -774,7 +771,7 @@ int mdc_statfs(struct lustre_handle *conn, struct obd_statfs *osfs, EXIT; out: - *request = req; + ptlrpc_req_finished(req); return rc; } @@ -784,6 +781,7 @@ struct obd_ops mdc_obd_ops = { o_cleanup: client_obd_cleanup, o_connect: client_obd_connect, o_disconnect: client_obd_disconnect, + o_statfs: mdc_statfs, }; static int __init ptlrpc_request_init(void) @@ -805,7 +803,6 @@ EXPORT_SYMBOL(mdc_getlovinfo); EXPORT_SYMBOL(mdc_enqueue); EXPORT_SYMBOL(mdc_cancel_unused); EXPORT_SYMBOL(mdc_getattr); -EXPORT_SYMBOL(mdc_statfs); EXPORT_SYMBOL(mdc_create); EXPORT_SYMBOL(mdc_unlink); EXPORT_SYMBOL(mdc_rename); diff --git a/lustre/mds/handler.c b/lustre/mds/handler.c index 5096bd1..dbd74bf 100644 --- a/lustre/mds/handler.c +++ b/lustre/mds/handler.c @@ -34,6 +34,7 @@ #include #include #include +#include #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,0)) #include #endif @@ -385,7 +386,7 @@ static int mds_disconnect(struct lustre_handle *conn) } } spin_unlock(&med->med_open_lock); - + ldlm_cancel_locks_for_export(export); mds_client_free(export); @@ -396,6 +397,23 @@ static int mds_disconnect(struct lustre_handle *conn) RETURN(rc); } +/* + * XXX This is NOT guaranteed to flush all transactions to disk (even though + * it is equivalent to calling sync()) because it only _starts_ the flush + * and does not wait for completion. It's better than nothing though. + * What we really want is a mild form of fsync_dev_lockfs(), but it is + * non-standard, or enabling do_sync_supers in ext3, just for this call. + */ +static void mds_fsync_super(struct super_block *sb) +{ + lock_kernel(); + lock_super(sb); + if (sb->s_dirt && sb->s_op && sb->s_op->write_super) + sb->s_op->write_super(sb); + unlock_super(sb); + unlock_kernel(); +} + static int mds_getstatus(struct ptlrpc_request *req) { struct mds_obd *mds = mds_req2mds(req); @@ -410,7 +428,13 @@ static int mds_getstatus(struct ptlrpc_request *req) RETURN(0); } - /* Anything we need to do here with the client's trans no or so? */ + /* Flush any outstanding transactions to disk so the client will + * get the latest last_committed value and can drop their local + * requests if they have any. This would be fsync_super() if it + * was exported. + */ + mds_fsync_super(mds->mds_sb); + body = lustre_msg_buf(req->rq_repmsg, 0); memcpy(&body->fid1, &mds->mds_rootfid, sizeof(body->fid1));