From 3d50a71829637e6d798e034532fcd507a137725d Mon Sep 17 00:00:00 2001 From: wangdi Date: Wed, 12 Jul 2006 13:22:26 +0000 Subject: [PATCH] Branch: b_new_cmd 1)add lov_create prototype for testing mdd lov, Note: it will finally removed to share the same api as MDS. 2)reconstruct MDS lov code to export some api, mdd could use. not finished. --- lustre/include/lustre_mds.h | 1 + lustre/include/md_object.h | 1 - lustre/include/obd.h | 40 +++++++++++----- lustre/mdd/mdd_handler.c | 3 ++ lustre/mdd/mdd_internal.h | 2 + lustre/mdd/mdd_lov.c | 82 ++++++++++++++++++++++++++++++-- lustre/mds/mds_lov.c | 112 +++++++++++++++++++++++++++++--------------- 7 files changed, 185 insertions(+), 56 deletions(-) diff --git a/lustre/include/lustre_mds.h b/lustre/include/lustre_mds.h index 40795f1..edbf088 100644 --- a/lustre/include/lustre_mds.h +++ b/lustre/include/lustre_mds.h @@ -93,4 +93,5 @@ int mds_reint_rec(struct mds_update_record *r, int offset, #define IOC_REQUEST_CLOSE _IOWR('f', 35, long) #define IOC_REQUEST_MAX_NR 35 +#define MDS_LOV_MD_NAME "lov" #endif diff --git a/lustre/include/md_object.h b/lustre/include/md_object.h index f1880ef..a7f75ea 100644 --- a/lustre/include/md_object.h +++ b/lustre/include/md_object.h @@ -81,7 +81,6 @@ struct md_dir_operations { int (*mdo_create)(const struct lu_context *, struct md_object *, const char *child_name, struct md_object *, const char *target_name, struct lu_attr *); - int (*mdo_rename)(const struct lu_context *ctxt, struct md_object *spobj, struct md_object *tpobj, const struct lu_fid *lf, const char *sname, diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 916713a..83fa845 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -402,6 +402,32 @@ struct mgs_obd { struct semaphore mgs_sem; }; +struct md_lov_info { + struct obd_device *md_osc_obd; /* XXX lov_obd */ + struct obd_uuid md_lov_uuid; + struct obd_export *md_osc_exp; /* XXX lov_exp */ + struct lov_desc md_lov_desc; + obd_id *md_lov_objids; + int md_lov_objids_size; + __u32 md_lov_objids_in_file; + unsigned int md_lov_objids_dirty:1; + int md_lov_nextid_set; + struct file *md_lov_objid_filp; + unsigned long md_lov_objids_valid:1; +}; + +#define mds_osc_obd mds_lov_info.md_osc_obd +#define mds_lov_uuid mds_lov_info.md_lov_uuid +#define mds_osc_exp mds_lov_info.md_osc_exp +#define mds_lov_desc mds_lov_info.md_lov_desc +#define mds_lov_objids mds_lov_info.md_lov_objids +#define mds_lov_objids_size mds_lov_info.md_lov_objids_size +#define mds_lov_objids_in_file mds_lov_info.md_lov_objids_in_file +#define mds_lov_objids_dirty mds_lov_info.md_lov_objids_dirty +#define mds_lov_nextid_set mds_lov_info.md_lov_nextid_set +#define mds_lov_objid_filp mds_lov_info.md_lov_objid_filp +#define mds_lov_objids_valid mds_lov_info.md_lov_objids_valid + struct mds_obd { /* NB this field MUST be first */ struct obd_device_target mds_obt; @@ -426,17 +452,8 @@ struct mds_obd { cfs_dentry_t *mds_objects_dir; struct llog_handle *mds_cfg_llh; // struct llog_handle *mds_catalog; - struct obd_device *mds_osc_obd; /* XXX lov_obd */ - struct obd_uuid mds_lov_uuid; + struct md_lov_info mds_lov_info; char *mds_profile; - struct obd_export *mds_osc_exp; /* XXX lov_exp */ - struct lov_desc mds_lov_desc; - obd_id *mds_lov_objids; - int mds_lov_objids_size; - __u32 mds_lov_objids_in_file; - unsigned int mds_lov_objids_dirty:1; - int mds_lov_nextid_set; - struct file *mds_lov_objid_filp; struct file *mds_health_check_filp; unsigned long *mds_client_bitmap; struct semaphore mds_orphan_recovery_sem; @@ -445,8 +462,7 @@ struct mds_obd { struct lustre_quota_info mds_quota_info; struct semaphore mds_qonoff_sem; struct semaphore mds_health_sem; - unsigned long mds_lov_objids_valid:1, - mds_fl_user_xattr:1, + unsigned long mds_fl_user_xattr:1, mds_fl_acl:1; }; diff --git a/lustre/mdd/mdd_handler.c b/lustre/mdd/mdd_handler.c index 41bbd34..7746502 100644 --- a/lustre/mdd/mdd_handler.c +++ b/lustre/mdd/mdd_handler.c @@ -790,6 +790,9 @@ static int mdd_create(const struct lu_context *ctxt, struct md_object *pobj, if (rc) GOTO(cleanup, rc); + rc = mdd_lov_create(ctxt, mdd, son); + if (rc) + GOTO(cleanup, rc); created = 1; rc = __mdd_index_insert(ctxt, mdo, lu_object_fid(&child->mo_lu), name, handle); diff --git a/lustre/mdd/mdd_internal.h b/lustre/mdd/mdd_internal.h index f37f2b9..7a9844b 100644 --- a/lustre/mdd/mdd_internal.h +++ b/lustre/mdd/mdd_internal.h @@ -78,6 +78,8 @@ int mdd_xattr_set(const struct lu_context *ctxt, struct md_object *obj, const void *buf, int buf_len, const char *name); int mdd_lov_set_md(const struct lu_context *ctxt, struct md_object *pobj, struct md_object *child); +int mdd_lov_create(const struct lu_context *ctxt, struct mdd_device *mdd, + struct mdd_object *child); struct mdd_thread_info *mdd_ctx_info(const struct lu_context *ctx); extern struct lu_device_operations mdd_lu_ops; static inline int lu_device_is_mdd(struct lu_device *d) diff --git a/lustre/mdd/mdd_lov.c b/lustre/mdd/mdd_lov.c index b315db8..7e96a68 100644 --- a/lustre/mdd/mdd_lov.c +++ b/lustre/mdd/mdd_lov.c @@ -43,6 +43,7 @@ #include #include #include +#include #include "mdd_internal.h" @@ -417,6 +418,33 @@ static int mdd_lov_update_mds(struct lu_context *ctxt, RETURN(rc); } +static int mdd_lov_clear_orphans(struct mdd_lov_info *mli, + struct obd_uuid *ost_uuid) +{ + int rc; + struct obdo oa; + struct obd_trans_info oti = {0}; + struct lov_stripe_md *empty_ea = NULL; + ENTRY; + + LASSERT(mli->mdd_lov_objids != NULL); + + /* This create will in fact either create or destroy: If the OST is + * missing objects below this ID, they will be created. If it finds + * objects above this ID, they will be removed. */ + memset(&oa, 0, sizeof(oa)); + oa.o_valid = OBD_MD_FLFLAGS; + oa.o_flags = OBD_FL_DELORPHAN; + if (ost_uuid != NULL) { + memcpy(&oa.o_inline, ost_uuid, sizeof(*ost_uuid)); + oa.o_valid |= OBD_MD_FLINLINE; + } + rc = obd_create(mli->mdd_lov_obd->obd_self_export, &oa, + &empty_ea, &oti); + + RETURN(rc); +} + /* We only sync one osc at a time, so that we don't have to hold any kind of lock on the whole mds_lov_desc, which may change (grow) as a result of mds_lov_add_ost. This also avoids any @@ -448,11 +476,20 @@ static int __mdd_lov_synchronize(void *data) rc = obd_set_info_async(mdd->mdd_lov_info.mdd_lov_obd->obd_self_export, strlen(KEY_MDS_CONN), KEY_MDS_CONN, 0, uuid, NULL); - if (rc != 0) + if (rc != 0) { + CERROR("failed at obd_set_info_async: %d\n", rc); GOTO(out, rc); + } + + rc = mdd_lov_clear_orphans(&mdd->mdd_lov_info, uuid); + if (rc != 0) { + CERROR("failed at mds_lov_clear_orphans: %d\n", rc); + GOTO(out, rc); + } out: + EXIT; lu_device_put(ld); - RETURN(rc); + return rc; } int mdd_lov_synchronize(void *data) @@ -576,7 +613,7 @@ static int mdd_get_md(const struct lu_context *ctxt, struct md_object *obj, next = mdd_object_child(md2mdd_obj(obj)); rc = next->do_ops->do_xattr_get(ctxt, next, md, *md_size, - "lov"); + MDS_LOV_MD_NAME); if (rc < 0) { CERROR("Error %d reading eadata \n", rc); } else if (rc > 0) { @@ -603,10 +640,47 @@ int mdd_lov_set_md(const struct lu_context *ctxt, struct md_object *pobj, int lmm_size = sizeof(lmm); rc = mdd_get_md(ctxt, pobj, &lmm, &lmm_size, 1); if (rc > 0) { - rc = mdd_xattr_set(ctxt, child, lmm, lmm_size, "lov"); + rc = mdd_xattr_set(ctxt, child, lmm, lmm_size, MDS_LOV_MD_NAME); if (rc) CERROR("error on copy stripe info: rc = %d\n", rc); } } RETURN(rc); } + +int mdd_lov_create(const struct lu_context *ctxt, struct mdd_device *mdd, + struct mdd_object *child) +{ + struct mdd_lov_info *mli = &mdd->mdd_lov_info; + struct obdo *oa; + struct lov_mds_md *lmm = NULL; + struct lov_stripe_md *lsm = NULL; + int rc = 0, lmm_size; + ENTRY; + + oa = obdo_alloc(); + + oa->o_uid = 0; /* must have 0 uid / gid on OST */ + oa->o_gid = 0; + oa->o_mode = S_IFREG | 0600; + oa->o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLFLAGS | + OBD_MD_FLMODE | OBD_MD_FLUID | OBD_MD_FLGID; + oa->o_size = 0; + + rc = obd_create(mli->mdd_lov_obd->obd_self_export, oa, &lsm, NULL); + if (rc) + GOTO(out_oa, rc); + + rc = obd_packmd(mli->mdd_lov_obd->obd_self_export, &lmm, lsm); + if (rc < 0) { + CERROR("cannot pack lsm, err = %d\n", rc); + GOTO(out_oa, rc); + } + lmm_size = rc; + + rc = mdd_xattr_set(ctxt, &child->mod_obj, lmm, lmm_size, + MDS_LOV_MD_NAME); +out_oa: + obdo_free(oa); + RETURN(rc); +} diff --git a/lustre/mds/mds_lov.c b/lustre/mds/mds_lov.c index 0f95347..ed4296b 100644 --- a/lustre/mds/mds_lov.c +++ b/lustre/mds/mds_lov.c @@ -42,20 +42,24 @@ #include "mds_internal.h" -void mds_lov_update_objids(struct obd_device *obd, obd_id *ids) +void md_lov_info_update_objids(struct md_lov_info *mli, obd_id *ids) { - struct mds_obd *mds = &obd->u.mds; int i; - ENTRY; - lock_kernel(); - for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++) - if (ids[i] > (mds->mds_lov_objids)[i]) { - (mds->mds_lov_objids)[i] = ids[i]; - mds->mds_lov_objids_dirty = 1; + for (i = 0; i < mli->md_lov_desc.ld_tgt_count; i++) + if (ids[i] > (mli->md_lov_objids)[i]) { + (mli->md_lov_objids)[i] = ids[i]; + mli->md_lov_objids_dirty = 1; } unlock_kernel(); - EXIT; +} +EXPORT_SYMBOL(md_lov_info_update_objids); + +void mds_lov_update_objids(struct obd_device *obd, obd_id *ids) +{ + struct mds_obd *mds = &obd->u.mds; + + md_lov_info_update_objids(&mds->mds_lov_info, ids); } static int mds_lov_read_objids(struct obd_device *obd) @@ -127,7 +131,8 @@ int mds_lov_write_objids(struct obd_device *obd) RETURN(rc); } -int mds_lov_clear_orphans(struct mds_obd *mds, struct obd_uuid *ost_uuid) +int md_lov_info_clear_orphans(struct md_lov_info *mli, + struct obd_uuid *ost_uuid) { int rc; struct obdo oa; @@ -135,7 +140,7 @@ int mds_lov_clear_orphans(struct mds_obd *mds, struct obd_uuid *ost_uuid) struct lov_stripe_md *empty_ea = NULL; ENTRY; - LASSERT(mds->mds_lov_objids != NULL); + LASSERT(mli->md_lov_objids != NULL); /* This create will in fact either create or destroy: If the OST is * missing objects below this ID, they will be created. If it finds @@ -147,40 +152,54 @@ int mds_lov_clear_orphans(struct mds_obd *mds, struct obd_uuid *ost_uuid) memcpy(&oa.o_inline, ost_uuid, sizeof(*ost_uuid)); oa.o_valid |= OBD_MD_FLINLINE; } - rc = obd_create(mds->mds_osc_exp, &oa, &empty_ea, &oti); + rc = obd_create(mli->md_osc_exp, &oa, &empty_ea, &oti); + + RETURN(rc); +} +EXPORT_SYMBOL(md_lov_info_clear_orphans); + +int mds_lov_clear_orphans(struct mds_obd *mds, struct obd_uuid *ost_uuid) +{ + return md_lov_info_clear_orphans(&mds->mds_lov_info, ost_uuid); +} + +int md_lov_info_set_nextid(struct md_lov_info *mli) +{ + int rc; + ENTRY; + + LASSERT(mli->md_lov_objids != NULL); + + rc = obd_set_info_async(mli->md_osc_exp, strlen(KEY_NEXT_ID), + KEY_NEXT_ID, + mli->md_lov_desc.ld_tgt_count, + mli->md_lov_objids, NULL); + RETURN(rc); + } +EXPORT_SYMBOL(md_lov_info_set_nextid); /* update the LOV-OSC knowledge of the last used object id's */ int mds_lov_set_nextid(struct obd_device *obd) { struct mds_obd *mds = &obd->u.mds; int rc; - ENTRY; LASSERT(!obd->obd_recovering); - - LASSERT(mds->mds_lov_objids != NULL); - - rc = obd_set_info_async(mds->mds_osc_exp, strlen(KEY_NEXT_ID), - KEY_NEXT_ID, - mds->mds_lov_desc.ld_tgt_count, - mds->mds_lov_objids, NULL); + rc = md_lov_info_set_nextid(&mds->mds_lov_info); if (rc) CERROR ("%s: mds_lov_set_nextid failed (%d)\n", obd->obd_name, rc); - RETURN(rc); } -/* Update the lov desc for a new size lov. */ -static int mds_lov_update_desc(struct obd_device *obd, struct obd_export *lov) +int md_lov_info_update_desc(struct md_lov_info *mli, struct obd_export *lov) { - struct mds_obd *mds = &obd->u.mds; struct lov_desc *ld; - __u32 size, stripes, valsize = sizeof(mds->mds_lov_desc); + __u32 size, valsize = sizeof(mli->md_lov_desc); int rc = 0; ENTRY; @@ -195,8 +214,8 @@ static int mds_lov_update_desc(struct obd_device *obd, struct obd_export *lov) /* The size of the LOV target table may have increased. */ size = ld->ld_tgt_count * sizeof(obd_id); - if ((mds->mds_lov_objids_size == 0) || - (size > mds->mds_lov_objids_size)) { + if ((mli->md_lov_objids_size == 0) || + (size > mli->md_lov_objids_size)) { obd_id *ids; /* add room by powers of 2 */ @@ -209,23 +228,39 @@ static int mds_lov_update_desc(struct obd_device *obd, struct obd_export *lov) if (ids == NULL) GOTO(out, rc = -ENOMEM); memset(ids, 0, size); - if (mds->mds_lov_objids_size) { - obd_id *old_ids = mds->mds_lov_objids; - memcpy(ids, mds->mds_lov_objids, - mds->mds_lov_objids_size); - mds->mds_lov_objids = ids; - OBD_FREE(old_ids, mds->mds_lov_objids_size); + if (mli->md_lov_objids_size) { + obd_id *old_ids = mli->md_lov_objids; + memcpy(ids, mli->md_lov_objids, + mli->md_lov_objids_size); + mli->md_lov_objids = ids; + OBD_FREE(old_ids, mli->md_lov_objids_size); } - mds->mds_lov_objids = ids; - mds->mds_lov_objids_size = size; + mli->md_lov_objids = ids; + mli->md_lov_objids_size = size; } /* Don't change the mds_lov_desc until the objids size matches the count (paranoia) */ - mds->mds_lov_desc = *ld; + mli->md_lov_desc = *ld; CDEBUG(D_CONFIG, "updated lov_desc, tgt_count: %d\n", - mds->mds_lov_desc.ld_tgt_count); + mli->md_lov_desc.ld_tgt_count); +out: + OBD_FREE(ld, sizeof(*ld)); + RETURN(rc); +} +/* Update the lov desc for a new size lov. */ +static int mds_lov_update_desc(struct obd_device *obd, struct obd_export *lov) +{ + struct mds_obd *mds = &obd->u.mds; + __u32 stripes; + int rc = 0; + ENTRY; + + rc = md_lov_info_update_desc(&mds->mds_lov_info, lov); + if (rc) + GOTO(out, rc); + stripes = min((__u32)LOV_MAX_STRIPE_COUNT, max(mds->mds_lov_desc.ld_tgt_count, mds->mds_lov_objids_in_file)); @@ -233,9 +268,7 @@ static int mds_lov_update_desc(struct obd_device *obd, struct obd_export *lov) mds->mds_max_cookiesize = stripes * sizeof(struct llog_cookie); CDEBUG(D_CONFIG, "updated max_mdsize/max_cookiesize: %d/%d\n", mds->mds_max_mdsize, mds->mds_max_cookiesize); - out: - OBD_FREE(ld, sizeof(*ld)); RETURN(rc); } @@ -626,6 +659,7 @@ struct mds_lov_sync_info { __u32 mlsi_index; /* index of target */ }; + /* We only sync one osc at a time, so that we don't have to hold any kind of lock on the whole mds_lov_desc, which may change (grow) as a result of mds_lov_add_ost. This also avoids any -- 1.8.3.1