From d749b8c3e0c682d945623453061665841efcd631 Mon Sep 17 00:00:00 2001 From: wangdi Date: Tue, 15 Aug 2006 04:19:08 +0000 Subject: [PATCH] Branch: b_new_cmd 1 remove md_lov_info and related info, since we create a tmp MDS obd to access the lov, so just reuse mds lov code, no need share anymore. 2)some other fixes and cleanup. --- lustre/include/obd.h | 49 ++--- lustre/mdd/mdd_handler.c | 2 - lustre/mdd/mdd_internal.h | 2 - lustre/mdd/mdd_lov.c | 123 ------------ lustre/mds/handler.c | 23 ++- lustre/mds/mds_internal.h | 3 +- lustre/mds/mds_lov.c | 441 ++++++++++++++++++-------------------------- lustre/mds/mds_reint.c | 2 +- lustre/obdclass/llog_lvfs.c | 6 - 9 files changed, 219 insertions(+), 432 deletions(-) diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 7d141aa..24b423b 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -416,39 +416,6 @@ struct md_lov_ops { const void *ctxt); }; -struct md_lov_info { - struct obd_device *md_lov_obd; /* XXX lov_obd */ - struct obd_uuid md_lov_uuid; - struct obd_export *md_lov_exp; /* XXX lov_exp */ - struct lov_desc md_lov_desc; - obd_id *md_lov_objids; - int md_lov_objids_size; - __u32 md_lov_objids_in_file; - unsigned int md_lov_objids_dirty:1; - int md_lov_nextid_set; - void *md_lov_objid_obj; - struct lu_fid md_lov_objid_fid; - unsigned long md_lov_objids_valid:1; - int md_lov_max_mdsize; - int md_lov_max_cookiesize; - struct semaphore md_lov_orphan_recovery_sem; - struct md_lov_ops *md_lov_ops; -}; - -#define mds_osc_obd mds_lov_info.md_lov_obd -#define mds_lov_uuid mds_lov_info.md_lov_uuid -#define mds_osc_exp mds_lov_info.md_lov_exp -#define mds_lov_desc mds_lov_info.md_lov_desc -#define mds_lov_objids mds_lov_info.md_lov_objids -#define mds_lov_objids_size mds_lov_info.md_lov_objids_size -#define mds_lov_objids_in_file mds_lov_info.md_lov_objids_in_file -#define mds_lov_objids_dirty mds_lov_info.md_lov_objids_dirty -#define mds_lov_nextid_set mds_lov_info.md_lov_nextid_set -#define mds_lov_objid_filp mds_lov_info.md_lov_objid_obj -#define mds_lov_objids_valid mds_lov_info.md_lov_objids_valid -#define mds_max_mdsize mds_lov_info.md_lov_max_mdsize -#define mds_max_cookiesize mds_lov_info.md_lov_max_cookiesize -#define mds_orphan_recovery_sem mds_lov_info.md_lov_orphan_recovery_sem struct mds_obd { /* NB this field MUST be first */ @@ -458,6 +425,8 @@ struct mds_obd { struct ptlrpc_service *mds_readpage_service; struct vfsmount *mds_vfsmnt; cfs_dentry_t *mds_fid_de; + int mds_max_mdsize; + int mds_max_cookiesize; struct file *mds_rcvd_filp; spinlock_t mds_transno_lock; __u64 mds_last_transno; @@ -472,8 +441,17 @@ struct mds_obd { cfs_dentry_t *mds_objects_dir; struct llog_handle *mds_cfg_llh; // struct llog_handle *mds_catalog; - struct md_lov_info mds_lov_info; + struct obd_device *mds_osc_obd; /* XXX lov_obd */ + struct obd_uuid mds_lov_uuid; char *mds_profile; + struct obd_export *mds_osc_exp; /* XXX lov_exp */ + struct lov_desc mds_lov_desc; + obd_id *mds_lov_objids; + int mds_lov_objids_size; + __u32 mds_lov_objids_in_file; + unsigned int mds_lov_objids_dirty:1; + int mds_lov_nextid_set; + struct file *mds_lov_objid_filp; struct file *mds_health_check_filp; unsigned long *mds_client_bitmap; struct upcall_cache *mds_group_hash; @@ -481,7 +459,8 @@ struct mds_obd { struct lustre_quota_info mds_quota_info; struct semaphore mds_qonoff_sem; struct semaphore mds_health_sem; - unsigned long mds_fl_user_xattr:1, + unsigned long mds_lov_objids_valid:1, + mds_fl_user_xattr:1, mds_fl_acl:1; }; diff --git a/lustre/mdd/mdd_handler.c b/lustre/mdd/mdd_handler.c index 0fb2599..00a9806 100644 --- a/lustre/mdd/mdd_handler.c +++ b/lustre/mdd/mdd_handler.c @@ -453,8 +453,6 @@ static struct lu_device *mdd_device_fini(const struct lu_context *ctx, struct mdd_device *m = lu2mdd_dev(d); struct lu_device *next = &m->mdd_child->dd_lu_dev; - dt_device_fini(&m->mdd_lov_dev); - return next; } diff --git a/lustre/mdd/mdd_internal.h b/lustre/mdd/mdd_internal.h index 17b6b0b..35e8928 100644 --- a/lustre/mdd/mdd_internal.h +++ b/lustre/mdd/mdd_internal.h @@ -35,8 +35,6 @@ struct dt_device; struct mdd_device { struct md_device mdd_md_dev; struct dt_device *mdd_child; - struct md_lov_info mdd_lov_info; - struct dt_device mdd_lov_dev; struct lu_fid mdd_root_fid; struct dt_device_param mdd_dt_conf; }; diff --git a/lustre/mdd/mdd_lov.c b/lustre/mdd/mdd_lov.c index 2592a1d..4374cf0 100644 --- a/lustre/mdd/mdd_lov.c +++ b/lustre/mdd/mdd_lov.c @@ -49,111 +49,6 @@ #include "mdd_internal.h" -static const char mdd_lov_objid_name[] = "lov_objid"; - -static int mdd_lov_read_objids(struct obd_device *obd, struct md_lov_info *mli, - const void *ctxt) -{ - struct dt_object *obj_ids = mli->md_lov_objid_obj; - struct lu_attr *lu_attr = NULL; - obd_id *ids; - int i, rc; - loff_t off = 0; - ENTRY; - - LASSERT(!mli->md_lov_objids_size); - LASSERT(!mli->md_lov_objids_dirty); - - /* Read everything in the file, even if our current lov desc - has fewer targets. Old targets not in the lov descriptor - during mds setup may still have valid objids. */ - - OBD_ALLOC_PTR(lu_attr); - if (!lu_attr) - GOTO(out, rc = -ENOMEM); - rc = obj_ids->do_ops->do_attr_get(ctxt, obj_ids, lu_attr); - if (rc) - GOTO(out, rc); - - if (lu_attr->la_size == 0) - GOTO(out, rc); - - OBD_ALLOC(ids, lu_attr->la_size); - if (ids == NULL) - RETURN(-ENOMEM); - - mli->md_lov_objids = ids; - mli->md_lov_objids_size = lu_attr->la_size; - - rc = obj_ids->do_body_ops->dbo_read(ctxt, obj_ids, (char *)ids, - lu_attr->la_size, &off); - if (rc < 0) { - CERROR("Error reading objids %d\n", rc); - RETURN(rc); - } - - mli->md_lov_objids_in_file = lu_attr->la_size / sizeof(*ids); - - for (i = 0; i < mli->md_lov_objids_in_file; i++) { - CDEBUG(D_INFO, "read last object "LPU64" for idx %d\n", - mli->md_lov_objids[i], i); - } -out: - if (lu_attr) - OBD_FREE_PTR(lu_attr); - RETURN(0); -} - -int mdd_lov_write_objids(struct obd_device *obd, struct md_lov_info *mli, - const void *ctxt) -{ - int i, rc = 0, tgts; - ENTRY; - - if (!mli->md_lov_objids_dirty) - RETURN(0); - - tgts = max(mli->md_lov_desc.ld_tgt_count, - mli->md_lov_objids_in_file); - if (!tgts) - RETURN(0); - - for (i = 0; i < tgts; i++) - CDEBUG(D_INFO, "writing last object "LPU64" for idx %d\n", - mli->md_lov_objids[i], i); -#if 0 - rc = ids_obj->do_body_ops->dbo_write(ctxt, obj_ids, - (char *)mli->mdd_lov_objids, - tgts * sizeof(obd_id), &off, - NULL /* XXX transaction handle */); - if (rc >= 0) { - mli->mdd_lov_objids_dirty = 0; - rc = 0; - } -#endif - RETURN(rc); -} -static int mdd_lov_write_catlist(struct obd_device *obd, void *idarray, int size, - const void *ctxt) -{ - int rc = 0; - RETURN(rc); -} - -static int mdd_lov_read_catlist(struct obd_device *obd, void *idarray, int size, - const void *ctxt) -{ - int rc = 0; - RETURN(rc); -} - -static struct md_lov_ops mdd_lov_ops = { - .ml_read_objids = mdd_lov_read_objids, - .ml_write_objids = mdd_lov_write_objids, - .ml_read_catlist = mdd_lov_read_catlist, - .ml_write_catlist = mdd_lov_write_catlist -}; - static int mdd_lov_update(struct obd_device *host, struct obd_device *watched, enum obd_notify_event ev, void *owner) @@ -181,8 +76,6 @@ int mdd_init_obd(const struct lu_context *ctxt, struct mdd_device *mdd, struct lustre_cfg_bufs bufs; struct lustre_cfg *lcfg; struct obd_device *obd; - struct dt_object *obj_id; - struct md_lov_info *mli; int rc; ENTRY; @@ -206,18 +99,6 @@ int mdd_init_obd(const struct lu_context *ctxt, struct mdd_device *mdd, LBUG(); } - /*init mli, which will be used in following mds setup*/ - mli = &obd->u.mds.mds_lov_info; - mli->md_lov_ops = &mdd_lov_ops; - - obj_id = dt_store_open(ctxt, mdd->mdd_child, mdd_lov_objid_name, - &mli->md_lov_objid_fid); - if (IS_ERR(obj_id)){ - rc = PTR_ERR(obj_id); - RETURN(rc); - } - mli->md_lov_objid_obj = obj_id; - rc = class_setup(obd, lcfg); if (rc) GOTO(class_detach, rc); @@ -238,7 +119,6 @@ lcfg_cleanup: int mdd_cleanup_obd(struct mdd_device *mdd) { struct lustre_cfg_bufs bufs; - struct md_lov_info *mli; struct lustre_cfg *lcfg; struct obd_device *obd; int rc; @@ -247,9 +127,6 @@ int mdd_cleanup_obd(struct mdd_device *mdd) obd = mdd->mdd_md_dev.md_lu_dev.ld_obd; LASSERT(obd); - mli = &obd->u.mds.mds_lov_info; - dt_object_fini(mli->md_lov_objid_obj); - lustre_cfg_bufs_reset(&bufs, MDD_OBD_NAME); lcfg = lustre_cfg_new(LCFG_ATTACH, &bufs); if (!lcfg) diff --git a/lustre/mds/handler.c b/lustre/mds/handler.c index 181d7c4..33a5818 100644 --- a/lustre/mds/handler.c +++ b/lustre/mds/handler.c @@ -1892,7 +1892,7 @@ static int mds_setup(struct obd_device *obd, struct lustre_cfg* lcfg) LASSERT(!lvfs_check_rdonly(lvfs_sbdev(mnt->mnt_sb))); - sema_init(&mds->mds_orphan_recovery_sem, 1); + //sema_init(&mds->mds_orphan_recovery_sem, 1); sema_init(&mds->mds_epoch_sem, 1); spin_lock_init(&mds->mds_transno_lock); mds->mds_max_mdsize = sizeof(struct lov_mds_md); @@ -2685,6 +2685,7 @@ static int mds_cmd_setup(struct obd_device *obd, struct lustre_cfg *lcfg) struct lustre_sb_info *lsi; struct lustre_mount_info *lmi; struct dentry *dentry; + struct file *file; int rc = 0; ENTRY; @@ -2729,6 +2730,20 @@ static int mds_cmd_setup(struct obd_device *obd, struct lustre_cfg *lcfg) CERROR("__iopen__ directory has no inode? rc = %d\n", rc); GOTO(err_fid, rc); } + + /* open and test the lov objd file */ + file = filp_open(LOV_OBJID, O_RDWR | O_CREAT, 0644); + if (IS_ERR(file)) { + rc = PTR_ERR(file); + CERROR("cannot open/create %s file: rc = %d\n", LOV_OBJID, rc); + GOTO(err_fid, rc = PTR_ERR(file)); + } + mds->mds_lov_objid_filp = file; + if (!S_ISREG(file->f_dentry->d_inode->i_mode)) { + CERROR("%s is not a regular file!: mode = %o\n", LOV_OBJID, + file->f_dentry->d_inode->i_mode); + GOTO(err_lov_objid, rc = -ENOENT); + } rc = mds_lov_presetup(mds, lcfg); if (rc < 0) @@ -2742,13 +2757,17 @@ static int mds_cmd_setup(struct obd_device *obd, struct lustre_cfg *lcfg) if (rc) GOTO(err_objects, rc); - sema_init(&mds->mds_orphan_recovery_sem, 1); + //sema_init(&mds->mds_orphan_recovery_sem, 1); mds->mds_max_mdsize = sizeof(struct lov_mds_md); mds->mds_max_cookiesize = sizeof(struct llog_cookie); err_pop: pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); RETURN(rc); +err_lov_objid: + if (mds->mds_lov_objid_filp && + filp_close((struct file *)mds->mds_lov_objid_filp, 0)) + CERROR("can't close %s after error\n", LOV_OBJID); err_fid: dput(mds->mds_fid_de); err_objects: diff --git a/lustre/mds/mds_internal.h b/lustre/mds/mds_internal.h index f6acec3..c947cee 100644 --- a/lustre/mds/mds_internal.h +++ b/lustre/mds/mds_internal.h @@ -174,8 +174,7 @@ int mds_llog_finish(struct obd_device *obd, int count); /* mds/mds_lov.c */ int mds_lov_connect(struct obd_device *obd, char * lov_name); int mds_lov_disconnect(struct obd_device *obd); -int mds_lov_write_objids(struct obd_device *obd, struct md_lov_info *mli, - const void *ctxt); +int mds_lov_write_objids(struct obd_device *obd); int mds_lov_clear_orphans(struct mds_obd *mds, struct obd_uuid *ost_uuid); void mds_lov_update_objids(struct obd_device *obd, obd_id *ids); int mds_lov_set_nextid(struct obd_device *obd); diff --git a/lustre/mds/mds_lov.c b/lustre/mds/mds_lov.c index 2c3268a..1e107d3 100644 --- a/lustre/mds/mds_lov.c +++ b/lustre/mds/mds_lov.c @@ -39,106 +39,95 @@ #include #include #include -#include #include "mds_internal.h" -void md_lov_info_update_objids(struct md_lov_info *mli, obd_id *ids) +void mds_lov_update_objids(struct obd_device *obd, obd_id *ids) { + struct mds_obd *mds = &obd->u.mds; int i; + ENTRY; + lock_kernel(); - for (i = 0; i < mli->md_lov_desc.ld_tgt_count; i++) - if (ids[i] > (mli->md_lov_objids)[i]) { - (mli->md_lov_objids)[i] = ids[i]; - mli->md_lov_objids_dirty = 1; + for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++) + if (ids[i] > (mds->mds_lov_objids)[i]) { + (mds->mds_lov_objids)[i] = ids[i]; + mds->mds_lov_objids_dirty = 1; } unlock_kernel(); + EXIT; } -void mds_lov_update_objids(struct obd_device *obd, obd_id *ids) +static int mds_lov_read_objids(struct obd_device *obd) { struct mds_obd *mds = &obd->u.mds; - - md_lov_info_update_objids(&mds->mds_lov_info, ids); -} - -static int mds_lov_read_objids(struct obd_device *obd, struct md_lov_info *mli, - const void *ctxt) -{ - struct file *filp = (struct file *)mli->md_lov_objid_obj; obd_id *ids; loff_t off = 0; int i, rc, size; ENTRY; - LASSERT(!mli->md_lov_objids_size); - LASSERT(!mli->md_lov_objids_dirty); + LASSERT(!mds->mds_lov_objids_size); + LASSERT(!mds->mds_lov_objids_dirty); - /* Read everything in the file, even if our current lov desc - has fewer targets. Old targets not in the lov descriptor + /* Read everything in the file, even if our current lov desc + has fewer targets. Old targets not in the lov descriptor during mds setup may still have valid objids. */ - size = filp->f_dentry->d_inode->i_size; + size = mds->mds_lov_objid_filp->f_dentry->d_inode->i_size; if (size == 0) RETURN(0); OBD_ALLOC(ids, size); if (ids == NULL) RETURN(-ENOMEM); - mli->md_lov_objids = ids; - mli->md_lov_objids_size = size; + mds->mds_lov_objids = ids; + mds->mds_lov_objids_size = size; - rc = fsfilt_read_record(obd, filp, ids, size, &off); + rc = fsfilt_read_record(obd, mds->mds_lov_objid_filp, ids, size, &off); if (rc < 0) { CERROR("Error reading objids %d\n", rc); RETURN(rc); } - - mli->md_lov_objids_in_file = size / sizeof(*ids); - - for (i = 0; i < mli->md_lov_objids_in_file; i++) { + + mds->mds_lov_objids_in_file = size / sizeof(*ids); + + for (i = 0; i < mds->mds_lov_objids_in_file; i++) { CDEBUG(D_INFO, "read last object "LPU64" for idx %d\n", - mli->md_lov_objids[i], i); + mds->mds_lov_objids[i], i); } RETURN(0); } -int mds_lov_write_objids(struct obd_device *obd, struct md_lov_info *mli, - const void *ctxt) +int mds_lov_write_objids(struct obd_device *obd) { - struct file *filp = (struct file *)mli->md_lov_objid_obj; + struct mds_obd *mds = &obd->u.mds; loff_t off = 0; - int i, rc, tgts; + int i, rc, tgts; ENTRY; - if (!mli->md_lov_objids_dirty) + if (!mds->mds_lov_objids_dirty) RETURN(0); - tgts = max(mli->md_lov_desc.ld_tgt_count, mli->md_lov_objids_in_file); + tgts = max(mds->mds_lov_desc.ld_tgt_count, mds->mds_lov_objids_in_file); if (!tgts) RETURN(0); for (i = 0; i < tgts; i++) CDEBUG(D_INFO, "writing last object "LPU64" for idx %d\n", - mli->md_lov_objids[i], i); + mds->mds_lov_objids[i], i); - rc = fsfilt_write_record(obd, filp, mli->md_lov_objids, - tgts * sizeof(obd_id), + rc = fsfilt_write_record(obd, mds->mds_lov_objid_filp, + mds->mds_lov_objids, tgts * sizeof(obd_id), &off, 0); if (rc >= 0) { - mli->md_lov_objids_dirty = 0; + mds->mds_lov_objids_dirty = 0; rc = 0; } RETURN(rc); } -struct md_lov_ops mli_ops = { - .ml_read_objids = mds_lov_read_objids, - .ml_write_objids = mds_lov_write_objids, -}; - -int md_lov_clear_orphans(struct md_lov_info *mli, struct obd_uuid *ost_uuid) +int mds_lov_clear_orphans(struct mds_obd *mds, struct obd_uuid *ost_uuid) { int rc; struct obdo oa; @@ -146,7 +135,7 @@ int md_lov_clear_orphans(struct md_lov_info *mli, struct obd_uuid *ost_uuid) struct lov_stripe_md *empty_ea = NULL; ENTRY; - LASSERT(mli->md_lov_objids != NULL); + LASSERT(mds->mds_lov_objids != NULL); /* This create will in fact either create or destroy: If the OST is * missing objects below this ID, they will be created. If it finds @@ -158,44 +147,40 @@ int md_lov_clear_orphans(struct md_lov_info *mli, struct obd_uuid *ost_uuid) memcpy(&oa.o_inline, ost_uuid, sizeof(*ost_uuid)); oa.o_valid |= OBD_MD_FLINLINE; } - rc = obd_create(mli->md_lov_exp, &oa, &empty_ea, &oti); + rc = obd_create(mds->mds_osc_exp, &oa, &empty_ea, &oti); RETURN(rc); } -int mds_lov_clear_orphans(struct mds_obd *mds, struct obd_uuid *ost_uuid) -{ - return md_lov_clear_orphans(&mds->mds_lov_info, ost_uuid); -} - /* update the LOV-OSC knowledge of the last used object id's */ -static int md_lov_info_set_nextid(struct obd_device *obd, - struct md_lov_info *mli) +int mds_lov_set_nextid(struct obd_device *obd) { + struct mds_obd *mds = &obd->u.mds; int rc; ENTRY; LASSERT(!obd->obd_recovering); - LASSERT(mli->md_lov_objids != NULL); - rc = obd_set_info_async(mli->md_lov_exp, strlen(KEY_NEXT_ID), + LASSERT(mds->mds_lov_objids != NULL); + + rc = obd_set_info_async(mds->mds_osc_exp, strlen(KEY_NEXT_ID), KEY_NEXT_ID, - mli->md_lov_desc.ld_tgt_count, - mli->md_lov_objids, NULL); + mds->mds_lov_desc.ld_tgt_count, + mds->mds_lov_objids, NULL); + + if (rc) + CERROR ("%s: mds_lov_set_nextid failed (%d)\n", + obd->obd_name, rc); RETURN(rc); } -int mds_lov_set_nextid(struct obd_device *obd) +/* Update the lov desc for a new size lov. */ +static int mds_lov_update_desc(struct obd_device *obd, struct obd_export *lov) { - struct md_lov_info *mli = &obd->u.mds.mds_lov_info; - return md_lov_info_set_nextid(obd, mli); -} - -int md_lov_info_update_desc(struct md_lov_info *mli, struct obd_export *lov) -{ - struct lov_desc *ld; - __u32 size, stripes, valsize = sizeof(mli->md_lov_desc); + struct mds_obd *mds = &obd->u.mds; + struct lov_desc *ld; + __u32 size, stripes, valsize = sizeof(mds->mds_lov_desc); int rc = 0; ENTRY; @@ -210,13 +195,13 @@ int md_lov_info_update_desc(struct md_lov_info *mli, struct obd_export *lov) /* The size of the LOV target table may have increased. */ size = ld->ld_tgt_count * sizeof(obd_id); - if ((mli->md_lov_objids_size == 0) || - (size > mli->md_lov_objids_size)) { + if ((mds->mds_lov_objids_size == 0) || + (size > mds->mds_lov_objids_size)) { obd_id *ids; - + /* add room by powers of 2 */ size = 1; - while (size < ld->ld_tgt_count) + while (size < ld->ld_tgt_count) size = size << 1; size = size * sizeof(obd_id); @@ -224,123 +209,116 @@ int md_lov_info_update_desc(struct md_lov_info *mli, struct obd_export *lov) if (ids == NULL) GOTO(out, rc = -ENOMEM); memset(ids, 0, size); - if (mli->md_lov_objids_size) { - obd_id *old_ids = mli->md_lov_objids; - memcpy(ids, mli->md_lov_objids, - mli->md_lov_objids_size); - mli->md_lov_objids = ids; - OBD_FREE(old_ids, mli->md_lov_objids_size); + if (mds->mds_lov_objids_size) { + obd_id *old_ids = mds->mds_lov_objids; + memcpy(ids, mds->mds_lov_objids, + mds->mds_lov_objids_size); + mds->mds_lov_objids = ids; + OBD_FREE(old_ids, mds->mds_lov_objids_size); } - mli->md_lov_objids = ids; - mli->md_lov_objids_size = size; + mds->mds_lov_objids = ids; + mds->mds_lov_objids_size = size; } /* Don't change the mds_lov_desc until the objids size matches the count (paranoia) */ - mli->md_lov_desc = *ld; + mds->mds_lov_desc = *ld; CDEBUG(D_CONFIG, "updated lov_desc, tgt_count: %d\n", - mli->md_lov_desc.ld_tgt_count); + mds->mds_lov_desc.ld_tgt_count); - stripes = min((__u32)LOV_MAX_STRIPE_COUNT, - max(mli->md_lov_desc.ld_tgt_count, - mli->md_lov_objids_in_file)); - - mli->md_lov_max_mdsize = lov_mds_md_size(stripes); - mli->md_lov_max_cookiesize = stripes * sizeof(struct llog_cookie); + stripes = min((__u32)LOV_MAX_STRIPE_COUNT, + max(mds->mds_lov_desc.ld_tgt_count, + mds->mds_lov_objids_in_file)); + mds->mds_max_mdsize = lov_mds_md_size(stripes); + mds->mds_max_cookiesize = stripes * sizeof(struct llog_cookie); CDEBUG(D_CONFIG, "updated max_mdsize/max_cookiesize: %d/%d\n", - mli->md_lov_max_mdsize, mli->md_lov_max_cookiesize); + mds->mds_max_mdsize, mds->mds_max_cookiesize); out: OBD_FREE(ld, sizeof(*ld)); RETURN(rc); } + #define MDSLOV_NO_INDEX -1 + /* Inform MDS about new/updated target */ -static int mds_lov_update_mds(struct obd_device *obd, - struct md_lov_info *mli, - struct obd_device *watched, - __u32 idx, const void *ctxt) +static int mds_lov_update_mds(struct obd_device *obd, + struct obd_device *watched, + __u32 idx) { + struct mds_obd *mds = &obd->u.mds; int old_count; - int rc; + int rc = 0; ENTRY; - old_count = mli->md_lov_desc.ld_tgt_count; - rc = md_lov_info_update_desc(mli, mli->md_lov_exp); + old_count = mds->mds_lov_desc.ld_tgt_count; + rc = mds_lov_update_desc(obd, mds->mds_osc_exp); if (rc) RETURN(rc); CDEBUG(D_CONFIG, "idx=%d, recov=%d/%d, cnt=%d/%d\n", - idx, obd->obd_recovering, obd->obd_async_recov, old_count, - mli->md_lov_desc.ld_tgt_count); + idx, obd->obd_recovering, obd->obd_async_recov, old_count, + mds->mds_lov_desc.ld_tgt_count); /* idx is set as data from lov_notify. */ if (idx != MDSLOV_NO_INDEX && !obd->obd_recovering) { - if (idx >= mli->md_lov_desc.ld_tgt_count) { - CERROR("index %d > count %d!\n", idx, - mli->md_lov_desc.ld_tgt_count); + if (idx >= mds->mds_lov_desc.ld_tgt_count) { + CERROR("index %d > count %d!\n", idx, + mds->mds_lov_desc.ld_tgt_count); RETURN(-EINVAL); } - - if (idx >= mli->md_lov_objids_in_file) { + + if (idx >= mds->mds_lov_objids_in_file) { /* We never read this lastid; ask the osc */ obd_id lastid; __u32 size = sizeof(lastid); rc = obd_get_info(watched->obd_self_export, - strlen("last_id"), + strlen("last_id"), "last_id", &size, &lastid); if (rc) RETURN(rc); - mli->md_lov_objids[idx] = lastid; - mli->md_lov_objids_dirty = 1; - mli->md_lov_ops->ml_write_objids(obd, mli, ctxt); + mds->mds_lov_objids[idx] = lastid; + mds->mds_lov_objids_dirty = 1; + mds_lov_write_objids(obd); } else { /* We have read this lastid from disk; tell the osc. - Don't call this during recovery. */ - rc = md_lov_info_set_nextid(obd, mli); + Don't call this during recovery. */ + rc = mds_lov_set_nextid(obd); } - + CDEBUG(D_CONFIG, "last object "LPU64" from OST %d\n", - mli->md_lov_objids[idx], idx); + mds->mds_lov_objids[idx], idx); } -#if 0 - /*FIXME: Do not support llog in mdd, so disable this temporarily*/ + /* If we added a target we have to reconnect the llogs */ - /* Only do this at first add (idx), or the first time after recovery */ - if (idx != MDSLOV_NO_INDEX || 1/*FIXME*/) { - CDEBUG(D_CONFIG, "reset llogs idx=%d\n", idx); - /* These two must be atomic */ - down(&mli->md_lov_orphan_recovery_sem); - obd_llog_finish(obd, old_count); - llog_cat_initialize(obd, mli->md_lov_desc.ld_tgt_count); - up(&mli->md_lov_orphan_recovery_sem); - } -#else + /* We only _need_ to do this at first add (idx), or the first time + after recovery. However, it should now be safe to call anytime. */ CDEBUG(D_CONFIG, "reset llogs idx=%d\n", idx); - llog_cat_initialize(obd, mli->md_lov_desc.ld_tgt_count); -#endif + llog_cat_initialize(obd, mds->mds_lov_desc.ld_tgt_count); + RETURN(rc); } -int md_lov_connect(struct obd_device *obd, struct md_lov_info *mli, - char *lov_name, struct obd_uuid *uuid, - struct md_lov_ops *mlo, const void *ctxt) +/* update the LOV-OSC knowledge of the last used object id's */ +int mds_lov_connect(struct obd_device *obd, char * lov_name) { + struct mds_obd *mds = &obd->u.mds; struct lustre_handle conn = {0,}; struct obd_connect_data *data; - int rc; + int rc, i; + ENTRY; - if (IS_ERR(mli->md_lov_obd)) - RETURN(PTR_ERR(mli->md_lov_obd)); + if (IS_ERR(mds->mds_osc_obd)) + RETURN(PTR_ERR(mds->mds_osc_obd)); - if (mli->md_lov_obd) + if (mds->mds_osc_obd) RETURN(0); - mli->md_lov_obd = class_name2obd(lov_name); - if (!mli->md_lov_obd) { + mds->mds_osc_obd = class_name2obd(lov_name); + if (!mds->mds_osc_obd) { CERROR("MDS cannot locate LOV %s\n", lov_name); - mli->md_lov_obd = ERR_PTR(-ENOTCONN); + mds->mds_osc_obd = ERR_PTR(-ENOTCONN); RETURN(-ENOTCONN); } @@ -350,74 +328,30 @@ int md_lov_connect(struct obd_device *obd, struct md_lov_info *mli, data->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_INDEX | OBD_CONNECT_REQPORTAL; data->ocd_version = LUSTRE_VERSION_CODE; - /* NB: lov_connect() needs to fill in .ocd_index for each OST */ - rc = obd_connect(NULL, &conn, mli->md_lov_obd, uuid, data); + rc = obd_connect(NULL, &conn, mds->mds_osc_obd, &obd->obd_uuid, data); OBD_FREE(data, sizeof(*data)); if (rc) { CERROR("MDS cannot connect to LOV %s (%d)\n", lov_name, rc); - GOTO(out, rc); + mds->mds_osc_obd = ERR_PTR(rc); + RETURN(rc); } - mli->md_lov_exp = class_conn2export(&conn); + mds->mds_osc_exp = class_conn2export(&conn); - rc = obd_register_observer(mli->md_lov_obd, obd); + rc = obd_register_observer(mds->mds_osc_obd, obd); if (rc) { CERROR("MDS cannot register as observer of LOV %s (%d)\n", lov_name, rc); - GOTO(out, rc); + GOTO(err_discon, rc); } - CDEBUG(D_INFO, "regist observer %s to lov %s \n", - obd->obd_name, mli->md_lov_obd->obd_name); - rc = mli->md_lov_ops->ml_read_objids(obd, mli, ctxt); + rc = mds_lov_read_objids(obd); if (rc) { CERROR("cannot read %s: rc = %d\n", "lov_objids", rc); - GOTO(out, rc); - } - - rc = md_lov_info_update_desc(mli, mli->md_lov_exp); - if (rc) - GOTO(out, rc); -out: - RETURN(rc); -} -EXPORT_SYMBOL(md_lov_connect); - -int md_lov_update_objids(struct obd_device *obd, struct md_lov_info *mli, - const void *ctxt) -{ - int rc = 0, i; - - /* If we're mounting this code for the first time on an existing FS, - * we need to populate the objids array from the real OST values */ - if (mli->md_lov_desc.ld_tgt_count > mli->md_lov_objids_in_file) { - int size = sizeof(obd_id) * mli->md_lov_desc.ld_tgt_count; - rc = obd_get_info(mli->md_lov_exp, strlen("last_id"), - "last_id", &size, mli->md_lov_objids); - if (!rc) { - for (i = 0; i < mli->md_lov_desc.ld_tgt_count; i++) - CWARN("got last object "LPU64" from OST %d\n", - mli->md_lov_objids[i], i); - mli->md_lov_objids_dirty = 1; - rc = mli->md_lov_ops->ml_write_objids(obd, mli, ctxt); - if (rc) - CERROR("got last objids from OSTs, but error " - "writing objids file: %d\n", rc); - } + GOTO(err_reg, rc); } - return rc; -} - -/* update the LOV-OSC knowledge of the last used object id's */ -int mds_lov_connect(struct obd_device *obd, char * lov_name) -{ - struct mds_obd *mds = &obd->u.mds; - struct md_lov_info *mli = &mds->mds_lov_info; - int rc; - ENTRY; - rc = md_lov_connect(obd, mli, lov_name, &obd->obd_uuid, &mli_ops, - NULL); + rc = mds_lov_update_desc(obd, mds->mds_osc_exp); if (rc) GOTO(err_reg, rc); @@ -430,7 +364,21 @@ int mds_lov_connect(struct obd_device *obd, char * lov_name) /* If we're mounting this code for the first time on an existing FS, * we need to populate the objids array from the real OST values */ - rc = md_lov_update_objids(obd, mli, NULL); + if (mds->mds_lov_desc.ld_tgt_count > mds->mds_lov_objids_in_file) { + int size = sizeof(obd_id) * mds->mds_lov_desc.ld_tgt_count; + rc = obd_get_info(mds->mds_osc_exp, strlen("last_id"), + "last_id", &size, mds->mds_lov_objids); + if (!rc) { + for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++) + CWARN("got last object "LPU64" from OST %d\n", + mds->mds_lov_objids[i], i); + mds->mds_lov_objids_dirty = 1; + rc = mds_lov_write_objids(obd); + if (rc) + CERROR("got last objids from OSTs, but error " + "writing objids file: %d\n", rc); + } + } /* I want to see a callback happen when the OBD moves to a * "For General Use" state, and that's when we'll call @@ -443,11 +391,10 @@ int mds_lov_connect(struct obd_device *obd, char * lov_name) err_reg: obd_register_observer(mds->mds_osc_obd, NULL); - if (mli->md_lov_exp) { - obd_disconnect(mli->md_lov_exp); - mli->md_lov_exp = NULL; - } - mli->md_lov_obd = ERR_PTR(rc); +err_discon: + obd_disconnect(mds->mds_osc_exp); + mds->mds_osc_exp = NULL; + mds->mds_osc_obd = ERR_PTR(rc); RETURN(rc); } @@ -669,27 +616,22 @@ int mds_iocontrol(unsigned int cmd, struct obd_export *exp, int len, } struct mds_lov_sync_info { - struct obd_device *mlsi_obd; /* the lov device to sync */ - struct md_lov_info *mlsi_mli; - struct obd_device *mlsi_watched; /* target osc */ - __u32 mlsi_index; /* index of target */ - const void *mlsi_ctxt; + struct obd_device *mlsi_obd; /* the lov device to sync */ + struct obd_device *mlsi_watched; /* target osc */ + __u32 mlsi_index; /* index of target */ }; - /* We only sync one osc at a time, so that we don't have to hold - any kind of lock on the whole mds_lov_desc, which may change + any kind of lock on the whole mds_lov_desc, which may change (grow) as a result of mds_lov_add_ost. This also avoids any - kind of mismatch between the lov_desc and the mds_lov_desc, + kind of mismatch between the lov_desc and the mds_lov_desc, which are not in lock-step during lov_add_obd */ static int __mds_lov_synchronize(void *data) { struct mds_lov_sync_info *mlsi = data; struct obd_device *obd = mlsi->mlsi_obd; - struct mds_obd *mds = &obd->u.mds; struct obd_device *watched = mlsi->mlsi_watched; - struct md_lov_info *mli = mlsi->mlsi_mli; - const void *ctxt = mlsi->mlsi_ctxt; + struct mds_obd *mds = &obd->u.mds; struct obd_uuid *uuid; __u32 idx = mlsi->mlsi_index; int rc = 0; @@ -702,11 +644,11 @@ static int __mds_lov_synchronize(void *data) uuid = &watched->u.cli.cl_target_uuid; LASSERT(uuid); - rc = mds_lov_update_mds(obd, mli, watched, idx, ctxt); + rc = mds_lov_update_mds(obd, watched, idx); if (rc != 0) GOTO(out, rc); - - rc = obd_set_info_async(mli->md_lov_exp, strlen(KEY_MDS_CONN), + + rc = obd_set_info_async(mds->mds_osc_exp, strlen(KEY_MDS_CONN), KEY_MDS_CONN, 0, uuid, NULL); if (rc != 0) GOTO(out, rc); @@ -714,35 +656,37 @@ static int __mds_lov_synchronize(void *data) rc = llog_connect(llog_get_context(obd, LLOG_MDS_OST_ORIG_CTXT), mds->mds_lov_desc.ld_tgt_count, NULL, NULL, uuid); - + if (rc != 0) { CERROR("%s: failed at llog_origin_connect: %d\n", obd->obd_name, rc); GOTO(out, rc); } + LCONSOLE_INFO("MDS %s: %s now active, resetting orphans\n", obd->obd_name, obd_uuid2str(uuid)); if (obd->obd_stopping) GOTO(out, rc = -ENODEV); - rc = md_lov_clear_orphans(mli, uuid); + rc = mds_lov_clear_orphans(mds, uuid); if (rc != 0) { - CERROR("%s: failed at md_lov_clear_orphans: %d\n", + CERROR("%s: failed at mds_lov_clear_orphans: %d\n", obd->obd_name, rc); GOTO(out, rc); } if (obd->obd_upcall.onu_owner) { /*This is an hack for mds_notify->mdd_notify, - *When the mds obd in mdd is removed, + *When the mds obd in mdd is removed, *This hack should be removed*/ - LASSERT(obd->obd_upcall.onu_upcall != NULL); - obd->obd_upcall.onu_upcall(NULL, NULL, 0, - obd->obd_upcall.onu_owner); + LASSERT(obd->obd_upcall.onu_upcall != NULL); + rc = obd->obd_upcall.onu_upcall(NULL, NULL, 0, + obd->obd_upcall.onu_owner); } + EXIT; out: class_decref(obd); - RETURN(rc); + return rc; } int mds_lov_synchronize(void *data) @@ -750,15 +694,20 @@ int mds_lov_synchronize(void *data) struct mds_lov_sync_info *mlsi = data; char name[20]; - sprintf(name, "ll_mlov_sync_%02u", mlsi->mlsi_index); + if (mlsi->mlsi_index == MDSLOV_NO_INDEX) + /* There is still a watched target, + but we don't know its index */ + sprintf(name, "ll_sync_tgt"); + else + snprintf(name, sizeof(name), "ll_sync_%02u", mlsi->mlsi_index); ptlrpc_daemonize(name); RETURN(__mds_lov_synchronize(data)); } -int md_lov_start_synchronize(struct obd_device *obd, struct md_lov_info *mli, - struct obd_device *watched, - void *data, int nonblock, const void *ctxt) +int mds_lov_start_synchronize(struct obd_device *obd, + struct obd_device *watched, + void *data, int nonblock) { struct mds_lov_sync_info *mlsi; int rc; @@ -773,9 +722,7 @@ int md_lov_start_synchronize(struct obd_device *obd, struct md_lov_info *mli, mlsi->mlsi_obd = obd; mlsi->mlsi_watched = watched; - mlsi->mlsi_mli = mli; - mlsi->mlsi_ctxt = ctxt; - if (data) + if (data) mlsi->mlsi_index = *(__u32 *)data; else mlsi->mlsi_index = MDSLOV_NO_INDEX; @@ -789,6 +736,7 @@ int md_lov_start_synchronize(struct obd_device *obd, struct md_lov_info *mli, disconnect the LOV. This of course means a cleanup won't finish for as long as the sync is blocking. */ class_incref(obd); + if (nonblock) { /* Synchronize in the background */ rc = cfs_kernel_thread(mds_lov_synchronize, mlsi, @@ -806,23 +754,15 @@ int md_lov_start_synchronize(struct obd_device *obd, struct md_lov_info *mli, } else { rc = __mds_lov_synchronize((void *)mlsi); } - RETURN(rc); -} -EXPORT_SYMBOL(md_lov_start_synchronize); -int mds_lov_start_synchronize(struct obd_device *obd, - struct obd_device *watched, - void *data, int nonblock) -{ - return md_lov_start_synchronize(obd, &obd->u.mds.mds_lov_info, - watched, data, nonblock, NULL); + RETURN(rc); } -int md_lov_notity_pre(struct obd_device *obd, struct md_lov_info *mli, - struct obd_device *watched, enum obd_notify_event ev, - void *data) +int mds_notify(struct obd_device *obd, struct obd_device *watched, + enum obd_notify_event ev, void *data) { int rc = 0; + ENTRY; switch (ev) { /* We only handle these: */ @@ -831,7 +771,7 @@ int md_lov_notity_pre(struct obd_device *obd, struct md_lov_info *mli, case OBD_NOTIFY_SYNC_NONBLOCK: break; default: - RETURN(-ENOENT); + RETURN(0); } CDEBUG(D_CONFIG, "notify %s ev=%d\n", watched->obd_name, ev); @@ -844,38 +784,21 @@ int md_lov_notity_pre(struct obd_device *obd, struct md_lov_info *mli, if (obd->obd_recovering) { CWARN("MDS %s: in recovery, not resetting orphans on %s\n", - obd->obd_name, + obd->obd_name, obd_uuid2str(&watched->u.cli.cl_target_uuid)); - /* We still have to fix the lov descriptor for ost's added + /* We still have to fix the lov descriptor for ost's added after the mdt in the config log. They didn't make it into mds_lov_connect. */ - rc = md_lov_info_update_desc(mli, mli->md_lov_exp); - RETURN(-EBUSY); - } - RETURN(rc); -} -EXPORT_SYMBOL(md_lov_notity_pre); - -int mds_notify(struct obd_device *obd, struct obd_device *watched, - enum obd_notify_event ev, void *data) -{ - int rc = 0; - struct md_lov_info *mli = &obd->u.mds.mds_lov_info; - ENTRY; - - rc = md_lov_notity_pre(obd, mli, watched, ev, data); - if (rc) { - if (rc == -ENOENT || rc == -EBUSY) - rc = 0; + rc = mds_lov_update_desc(obd, obd->u.mds.mds_osc_exp); RETURN(rc); } LASSERT(llog_get_context(obd, LLOG_MDS_OST_ORIG_CTXT) != NULL); - rc = mds_lov_start_synchronize(obd, watched, data, + rc = mds_lov_start_synchronize(obd, watched, data, !(ev == OBD_NOTIFY_SYNC)); - + lquota_recovery(quota_interface, obd); - + RETURN(rc); } @@ -899,14 +822,14 @@ int mds_convert_lov_ea(struct obd_device *obd, struct inode *inode, int rc, err; ENTRY; - if (le32_to_cpu(lmm->lmm_magic) == LOV_MAGIC || + if (le32_to_cpu(lmm->lmm_magic) == LOV_MAGIC || le32_to_cpu(lmm->lmm_magic == LOV_MAGIC_JOIN)) RETURN(0); CDEBUG(D_INODE, "converting LOV EA on %lu/%u from %#08x to %#08x\n", inode->i_ino, inode->i_generation, le32_to_cpu(lmm->lmm_magic), LOV_MAGIC); - + rc = obd_unpackmd(obd->u.mds.mds_osc_exp, &lsm, lmm, lmm_size); if (rc < 0) GOTO(conv_end, rc); diff --git a/lustre/mds/mds_reint.c b/lustre/mds/mds_reint.c index 96bb8b4..fbc9c70 100644 --- a/lustre/mds/mds_reint.c +++ b/lustre/mds/mds_reint.c @@ -186,7 +186,7 @@ int mds_finish_transno(struct mds_obd *mds, struct inode *inode, void *handle, "wrote trans #"LPU64" rc %d client %s at idx %u: err = %d", transno, rc, mcd->mcd_uuid, med->med_lr_idx, err); - err = mds_lov_write_objids(obd, &mds->mds_lov_info, NULL); + err = mds_lov_write_objids(obd); if (err) { log_pri = D_ERROR; if (rc == 0) diff --git a/lustre/obdclass/llog_lvfs.c b/lustre/obdclass/llog_lvfs.c index d9cb8a4..faa9328 100644 --- a/lustre/obdclass/llog_lvfs.c +++ b/lustre/obdclass/llog_lvfs.c @@ -758,12 +758,6 @@ int llog_put_cat_list(struct obd_device *obd, struct obd_device *disk_obd, if (!count) return (0); - if (!strcmp(disk_obd->obd_name, MDD_OBD_NAME)) { - struct md_lov_info *mli = &disk_obd->u.mds.mds_lov_info; - rc = mli->md_lov_ops->ml_write_catlist(disk_obd, idarray, - size, NULL); - return (0); - } push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); file = filp_open(name, O_RDWR | O_CREAT | O_LARGEFILE, 0700); if (!file || IS_ERR(file)) { -- 1.8.3.1