From a1efc1411e2e7b67102d132e656d2a11a99636b3 Mon Sep 17 00:00:00 2001 From: nikita Date: Sat, 18 Oct 2008 17:27:58 +0000 Subject: [PATCH] Move md-specific fields out of struct lu_site into special struct md_site, so that lu_site can be used on a client. b=16450 --- lustre/ChangeLog | 7 +++ lustre/cmm/cmm_device.c | 10 +-- lustre/include/lu_object.h | 23 ------- lustre/include/md_object.h | 41 ++++++++++++- lustre/mdd/mdd_lov.c | 5 +- lustre/mdt/mdt_capa.c | 4 +- lustre/mdt/mdt_handler.c | 148 ++++++++++++++++++++++++--------------------- lustre/mdt/mdt_internal.h | 9 +++ lustre/mdt/mdt_reint.c | 9 +-- 9 files changed, 150 insertions(+), 106 deletions(-) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index 8a2ad27..f9248bf 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -1653,6 +1653,13 @@ Bugzilla : 16450 Description: Constify instances of struct {lu,dt,md}_device_operations. Details : Constify instances of struct {lu,dt,md}_device_operations. +Severity : normal +Bugzilla : 16450 +Description: Introduce struct md_site and move meta-data specific parts of + struct lu_site here. +Details : Move md-specific fields out of struct lu_site into special struct + md_site, so that lu_site can be used on a client. + -------------------------------------------------------------------------------- 2007-08-10 Cluster File Systems, Inc. diff --git a/lustre/cmm/cmm_device.c b/lustre/cmm/cmm_device.c index 571a513..4308533 100644 --- a/lustre/cmm/cmm_device.c +++ b/lustre/cmm/cmm_device.c @@ -294,7 +294,7 @@ static int cmm_process_config(const struct lu_env *env, struct lu_site *ls = cmm2lu_dev(m)->ld_site; struct lu_fld_target target; - target.ft_srv = ls->ls_server_fld; + target.ft_srv = lu_site2md(ls)->ms_server_fld; target.ft_idx = m->cmm_local_num; target.ft_exp = NULL; @@ -446,7 +446,7 @@ static int cmm_device_init(const struct lu_env *env, struct lu_device *d, /* Assign site's fld client ref, needed for asserts in osd. */ ls = cmm2lu_dev(m)->ld_site; - ls->ls_client_fld = m->cmm_fld; + lu_site2md(ls)->ms_client_fld = m->cmm_fld; err = cmm_procfs_init(m, name); RETURN(err); @@ -464,9 +464,11 @@ static struct lu_device *cmm_device_fini(const struct lu_env *env, spin_lock(&cm->cmm_tgt_guard); list_for_each_entry_safe(mc, tmp, &cm->cmm_targets, mc_linkage) { struct lu_device *ld_m = mdc2lu_dev(mc); + struct lu_device *ld_c = cmm2lu_dev(cm); list_del_init(&mc->mc_linkage); - lu_device_put(cmm2lu_dev(cm)); + lu_ref_del(&ld_c->ld_reference, "mdc-child", ld_m); + lu_device_put(ld_c); ld_m->ld_type->ldt_ops->ldto_device_fini(env, ld_m); ld_m->ld_type->ldt_ops->ldto_device_free(env, ld_m); cm->cmm_tgt_count--; @@ -475,7 +477,7 @@ static struct lu_device *cmm_device_fini(const struct lu_env *env, fld_client_fini(cm->cmm_fld); ls = cmm2lu_dev(cm)->ld_site; - ls->ls_client_fld = NULL; + lu_site2md(ls)->ms_client_fld = NULL; cmm_procfs_fini(cm); RETURN (md2lu_dev(cm->cmm_child)); diff --git a/lustre/include/lu_object.h b/lustre/include/lu_object.h index d105001..0453004 100644 --- a/lustre/include/lu_object.h +++ b/lustre/include/lu_object.h @@ -608,29 +608,6 @@ struct lu_site { * Top-level device for this stack. */ struct lu_device *ls_top_dev; - /* - * mds number of this site. - */ - mdsno_t ls_node_id; - /* - * Fid location database - */ - struct lu_server_fld *ls_server_fld; - struct lu_client_fld *ls_client_fld; - - /* - * Server Seq Manager - */ - struct lu_server_seq *ls_server_seq; - - /* - * Controller Seq Manager - */ - struct lu_server_seq *ls_control_seq; - struct obd_export *ls_control_exp; - - /* - * Client Seq Manager /** * Wait-queue signaled when an object in this site is ultimately * destroyed (lu_object_free()). It is used by lu_object_find() to diff --git a/lustre/include/md_object.h b/lustre/include/md_object.h index 774e34b..5b11fb7 100644 --- a/lustre/include/md_object.h +++ b/lustre/include/md_object.h @@ -378,8 +378,40 @@ static inline int md_do_upcall(const struct lu_env *env, struct md_device *m, struct md_object { struct lu_object mo_lu; - struct md_object_operations *mo_ops; - struct md_dir_operations *mo_dir_ops; + const struct md_object_operations *mo_ops; + const struct md_dir_operations *mo_dir_ops; +}; + +/** + * md-server site. + */ +struct md_site { + struct lu_site ms_lu; + /** + * mds number of this site. + */ + mdsno_t ms_node_id; + /** + * Fid location database + */ + struct lu_server_fld *ms_server_fld; + struct lu_client_fld *ms_client_fld; + + /** + * Server Seq Manager + */ + struct lu_server_seq *ms_server_seq; + + /** + * Controller Seq Manager + */ + struct lu_server_seq *ms_control_seq; + struct obd_export *ms_control_exp; + + /** + * Client Seq Manager + */ + struct lu_client_seq *ms_client_seq; }; static inline int lu_device_is_md(const struct lu_device *d) @@ -415,6 +447,11 @@ static inline struct md_device *md_obj2dev(const struct md_object *o) return container_of0(o->mo_lu.lo_dev, struct md_device, md_lu_dev); } +static inline struct md_site *lu_site2md(const struct lu_site *s) +{ + return container_of0(s, struct md_site, ms_lu); +} + static inline int md_device_init(struct md_device *md, struct lu_device_type *t) { return lu_device_init(&md->md_lu_dev, t); diff --git a/lustre/mdd/mdd_lov.c b/lustre/mdd/mdd_lov.c index fcce64e..a67e594 100644 --- a/lustre/mdd/mdd_lov.c +++ b/lustre/mdd/mdd_lov.c @@ -97,7 +97,7 @@ int mdd_init_obd(const struct lu_env *env, struct mdd_device *mdd, struct obd_device *obd; ENTRY; - mds_id = mdd2lu_dev(mdd)->ld_site->ls_node_id; + mds_id = lu_site2md(mdd2lu_dev(mdd)->ld_site)->ms_node_id; name_size = strlen(MDD_OBD_NAME) + 35; uuid_size = strlen(MDD_OBD_UUID) + 35; @@ -438,7 +438,8 @@ int mdd_lov_create(const struct lu_env *env, struct mdd_device *mdd, oa->o_uid = 0; /* must have 0 uid / gid on OST */ oa->o_gid = 0; - oa->o_gr = FILTER_GROUP_MDS0 + mdd2lu_dev(mdd)->ld_site->ls_node_id; + oa->o_gr = FILTER_GROUP_MDS0 + + lu_site2md(mdd2lu_dev(mdd)->ld_site)->ms_node_id; oa->o_mode = S_IFREG | 0600; oa->o_id = mdd_lov_create_id(mdd_object_fid(child)); oa->o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLFLAGS | diff --git a/lustre/mdt/mdt_capa.c b/lustre/mdt/mdt_capa.c index 1c887eb..9abc6df 100644 --- a/lustre/mdt/mdt_capa.c +++ b/lustre/mdt/mdt_capa.c @@ -152,7 +152,7 @@ int mdt_capa_keys_init(const struct lu_env *env, struct mdt_device *mdt) int rc; ENTRY; - mdsnum = mdt->mdt_md_dev.md_lu_dev.ld_site->ls_node_id; + mdsnum = mdt_md_site(mdt)->ms_node_id; mti = lu_context_key_get(&env->le_ctx, &mdt_thread_key); LASSERT(mti != NULL); @@ -233,7 +233,7 @@ static int mdt_ck_thread_main(void *args) LASSERT(info != NULL); tmp = &info->mti_capa_key; - mdsnum = mdt->mdt_md_dev.md_lu_dev.ld_site->ls_node_id; + mdsnum = mdt_md_site(mdt)->ms_node_id; while (1) { l_wait_event(thread->t_ctl_waitq, thread->t_flags & (SVC_STOPPING | SVC_EVENT), diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 37bddb3..ffc8a96 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -3080,15 +3080,15 @@ static int mdt_intent_policy(struct ldlm_namespace *ns, static void mdt_seq_adjust(const struct lu_env *env, struct mdt_device *m, int lost) { - struct lu_site *ls = m->mdt_md_dev.md_lu_dev.ld_site; + struct md_site *ms = mdt_md_site(m); struct lu_range out; ENTRY; - LASSERT(ls && ls->ls_server_seq); + LASSERT(ms && ms->ms_server_seq); LASSERT(lost >= 0); /* get extra seq from seq_server, moving it's range up */ while (lost-- > 0) { - seq_server_alloc_meta(ls->ls_server_seq, NULL, &out, env); + seq_server_alloc_meta(ms->ms_server_seq, NULL, &out, env); } EXIT; } @@ -3096,25 +3096,27 @@ static void mdt_seq_adjust(const struct lu_env *env, static int mdt_seq_fini(const struct lu_env *env, struct mdt_device *m) { - struct lu_site *ls = m->mdt_md_dev.md_lu_dev.ld_site; + struct md_site *ms = mdt_md_site(m); ENTRY; - if (ls && ls->ls_server_seq) { - seq_server_fini(ls->ls_server_seq, env); - OBD_FREE_PTR(ls->ls_server_seq); - ls->ls_server_seq = NULL; + if (ms != NULL) { + if (ms->ms_server_seq) { + seq_server_fini(ms->ms_server_seq, env); + OBD_FREE_PTR(ms->ms_server_seq); + ms->ms_server_seq = NULL; } - if (ls && ls->ls_control_seq) { - seq_server_fini(ls->ls_control_seq, env); - OBD_FREE_PTR(ls->ls_control_seq); - ls->ls_control_seq = NULL; + if (ms->ms_control_seq) { + seq_server_fini(ms->ms_control_seq, env); + OBD_FREE_PTR(ms->ms_control_seq); + ms->ms_control_seq = NULL; } - if (ls && ls->ls_client_seq) { - seq_client_fini(ls->ls_client_seq); - OBD_FREE_PTR(ls->ls_client_seq); - ls->ls_client_seq = NULL; + if (ms->ms_client_seq) { + seq_client_fini(ms->ms_client_seq); + OBD_FREE_PTR(ms->ms_client_seq); + ms->ms_client_seq = NULL; + } } RETURN(0); @@ -3124,25 +3126,25 @@ static int mdt_seq_init(const struct lu_env *env, const char *uuid, struct mdt_device *m) { - struct lu_site *ls; + struct md_site *ms; char *prefix; int rc; ENTRY; - ls = m->mdt_md_dev.md_lu_dev.ld_site; + ms = mdt_md_site(m); /* * This is sequence-controller node. Init seq-controller server on local * MDT. */ - if (ls->ls_node_id == 0) { - LASSERT(ls->ls_control_seq == NULL); + if (ms->ms_node_id == 0) { + LASSERT(ms->ms_control_seq == NULL); - OBD_ALLOC_PTR(ls->ls_control_seq); - if (ls->ls_control_seq == NULL) + OBD_ALLOC_PTR(ms->ms_control_seq); + if (ms->ms_control_seq == NULL) RETURN(-ENOMEM); - rc = seq_server_init(ls->ls_control_seq, + rc = seq_server_init(ms->ms_control_seq, m->mdt_bottom, uuid, LUSTRE_SEQ_CONTROLLER, env); @@ -3150,13 +3152,13 @@ static int mdt_seq_init(const struct lu_env *env, if (rc) GOTO(out_seq_fini, rc); - OBD_ALLOC_PTR(ls->ls_client_seq); - if (ls->ls_client_seq == NULL) + OBD_ALLOC_PTR(ms->ms_client_seq); + if (ms->ms_client_seq == NULL) GOTO(out_seq_fini, rc = -ENOMEM); OBD_ALLOC(prefix, MAX_OBD_NAME + 5); if (prefix == NULL) { - OBD_FREE_PTR(ls->ls_client_seq); + OBD_FREE_PTR(ms->ms_client_seq); GOTO(out_seq_fini, rc = -ENOMEM); } @@ -3165,11 +3167,11 @@ static int mdt_seq_init(const struct lu_env *env, /* * Init seq-controller client after seq-controller server is - * ready. Pass ls->ls_control_seq to it for direct talking. + * ready. Pass ms->ms_control_seq to it for direct talking. */ - rc = seq_client_init(ls->ls_client_seq, NULL, + rc = seq_client_init(ms->ms_client_seq, NULL, LUSTRE_SEQ_METADATA, prefix, - ls->ls_control_seq); + ms->ms_control_seq); OBD_FREE(prefix, MAX_OBD_NAME + 5); if (rc) @@ -3177,13 +3179,13 @@ static int mdt_seq_init(const struct lu_env *env, } /* Init seq-server on local MDT */ - LASSERT(ls->ls_server_seq == NULL); + LASSERT(ms->ms_server_seq == NULL); - OBD_ALLOC_PTR(ls->ls_server_seq); - if (ls->ls_server_seq == NULL) + OBD_ALLOC_PTR(ms->ms_server_seq); + if (ms->ms_server_seq == NULL) GOTO(out_seq_fini, rc = -ENOMEM); - rc = seq_server_init(ls->ls_server_seq, + rc = seq_server_init(ms->ms_server_seq, m->mdt_bottom, uuid, LUSTRE_SEQ_SERVER, env); @@ -3191,11 +3193,11 @@ static int mdt_seq_init(const struct lu_env *env, GOTO(out_seq_fini, rc = -ENOMEM); /* Assign seq-controller client to local seq-server. */ - if (ls->ls_node_id == 0) { - LASSERT(ls->ls_client_seq != NULL); + if (ms->ms_node_id == 0) { + LASSERT(ms->ms_client_seq != NULL); - rc = seq_server_set_cli(ls->ls_server_seq, - ls->ls_client_seq, + rc = seq_server_set_cli(ms->ms_server_seq, + ms->ms_client_seq, env); } @@ -3214,7 +3216,7 @@ static int mdt_seq_init_cli(const struct lu_env *env, struct mdt_device *m, struct lustre_cfg *cfg) { - struct lu_site *ls = m->mdt_md_dev.md_lu_dev.ld_site; + struct md_site *ms = mdt_md_site(m); struct obd_device *mdc; struct obd_uuid *uuidp, *mdcuuidp; char *uuid_str, *mdc_uuid_str; @@ -3238,7 +3240,7 @@ static int mdt_seq_init_cli(const struct lu_env *env, /* check if this is adding the first MDC and controller is not yet * initialized. */ - if (index != 0 || ls->ls_client_seq) + if (index != 0 || ms->ms_client_seq) RETURN(0); uuid_str = lustre_cfg_string(cfg, 1); @@ -3255,9 +3257,9 @@ static int mdt_seq_init_cli(const struct lu_env *env, CERROR("target %s not set up\n", mdc->obd_name); rc = -EINVAL; } else { - LASSERT(ls->ls_control_exp); - OBD_ALLOC_PTR(ls->ls_client_seq); - if (ls->ls_client_seq != NULL) { + LASSERT(ms->ms_control_exp); + OBD_ALLOC_PTR(ms->ms_client_seq); + if (ms->ms_client_seq != NULL) { char *prefix; OBD_ALLOC(prefix, MAX_OBD_NAME + 5); @@ -3267,8 +3269,8 @@ static int mdt_seq_init_cli(const struct lu_env *env, snprintf(prefix, MAX_OBD_NAME + 5, "ctl-%s", mdc->obd_name); - rc = seq_client_init(ls->ls_client_seq, - ls->ls_control_exp, + rc = seq_client_init(ms->ms_client_seq, + ms->ms_control_exp, LUSTRE_SEQ_METADATA, prefix, NULL); OBD_FREE(prefix, MAX_OBD_NAME + 5); @@ -3278,8 +3280,8 @@ static int mdt_seq_init_cli(const struct lu_env *env, if (rc) RETURN(rc); - LASSERT(ls->ls_server_seq != NULL); - rc = seq_server_set_cli(ls->ls_server_seq, ls->ls_client_seq, + LASSERT(ms->ms_server_seq != NULL); + rc = seq_server_set_cli(ms->ms_server_seq, ms->ms_client_seq, env); } @@ -3288,19 +3290,21 @@ static int mdt_seq_init_cli(const struct lu_env *env, static void mdt_seq_fini_cli(struct mdt_device *m) { - struct lu_site *ls; + struct md_site *ms; ENTRY; - ls = m->mdt_md_dev.md_lu_dev.ld_site; + ms = mdt_md_site(m); - if (ls && ls->ls_server_seq) - seq_server_set_cli(ls->ls_server_seq, + if (ms != NULL) { + if (ms->ms_server_seq) + seq_server_set_cli(ms->ms_server_seq, NULL, NULL); - if (ls && ls->ls_control_exp) { - class_export_put(ls->ls_control_exp); - ls->ls_control_exp = NULL; + if (ms->ms_control_exp) { + class_export_put(ms->ms_control_exp); + ms->ms_control_exp = NULL; + } } EXIT; } @@ -3311,13 +3315,13 @@ static void mdt_seq_fini_cli(struct mdt_device *m) static int mdt_fld_fini(const struct lu_env *env, struct mdt_device *m) { - struct lu_site *ls = m->mdt_md_dev.md_lu_dev.ld_site; + struct md_site *ms = mdt_md_site(m); ENTRY; - if (ls && ls->ls_server_fld) { - fld_server_fini(ls->ls_server_fld, env); - OBD_FREE_PTR(ls->ls_server_fld); - ls->ls_server_fld = NULL; + if (ms && ms->ms_server_fld) { + fld_server_fini(ms->ms_server_fld, env); + OBD_FREE_PTR(ms->ms_server_fld); + ms->ms_server_fld = NULL; } RETURN(0); @@ -3327,21 +3331,21 @@ static int mdt_fld_init(const struct lu_env *env, const char *uuid, struct mdt_device *m) { - struct lu_site *ls; + struct md_site *ms; int rc; ENTRY; - ls = m->mdt_md_dev.md_lu_dev.ld_site; + ms = mdt_md_site(m); - OBD_ALLOC_PTR(ls->ls_server_fld); - if (ls->ls_server_fld == NULL) + OBD_ALLOC_PTR(ms->ms_server_fld); + if (ms->ms_server_fld == NULL) RETURN(rc = -ENOMEM); - rc = fld_server_init(ls->ls_server_fld, + rc = fld_server_init(ms->ms_server_fld, m->mdt_bottom, uuid, env); if (rc) { - OBD_FREE_PTR(ls->ls_server_fld); - ls->ls_server_fld = NULL; + OBD_FREE_PTR(ms->ms_server_fld); + ms->ms_server_fld = NULL; RETURN(rc); } @@ -3838,8 +3842,11 @@ static void mdt_fini(const struct lu_env *env, struct mdt_device *m) mdt_stack_fini(env, m, md2lu_dev(m->mdt_child)); if (ls) { + struct md_site *mite; + lu_site_fini(ls); - OBD_FREE_PTR(ls); + mite = lu_site2md(ls); + OBD_FREE_PTR(mite); d->ld_site = NULL; } LASSERT(atomic_read(&d->ld_ref) == 0); @@ -3902,6 +3909,7 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, struct lustre_mount_info *lmi; struct lustre_sb_info *lsi; struct lu_site *s; + struct md_site *mite; const char *identity_upcall = "NONE"; int rc; ENTRY; @@ -3940,11 +3948,13 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, spin_lock_init(&m->mdt_client_bitmap_lock); - OBD_ALLOC_PTR(s); - if (s == NULL) + OBD_ALLOC_PTR(mite); + if (mite == NULL) RETURN(-ENOMEM); md_device_init(&m->mdt_md_dev, ldt); + s = &mite->ms_lu; + m->mdt_md_dev.md_lu_dev.ld_ops = &mdt_lu_ops; m->mdt_md_dev.md_lu_dev.ld_obd = obd; /* set this lu_device to obd, because error handling need it */ @@ -3972,7 +3982,7 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, /* set server index */ LASSERT(num); - s->ls_node_id = simple_strtol(num, NULL, 10); + lu_site2md(s)->ms_node_id = simple_strtol(num, NULL, 10); /* failover is the default * FIXME: we do not failout mds0/mgs, which may cause some problems. diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h index 4ddbf7c..d68fe3c 100644 --- a/lustre/mdt/mdt_internal.h +++ b/lustre/mdt/mdt_internal.h @@ -452,6 +452,15 @@ static inline const struct lu_fid *mdt_object_fid(struct mdt_object *o) return lu_object_fid(&o->mot_obj.mo_lu); } +static inline struct lu_site *mdt_lu_site(const struct mdt_device *mdt) +{ + return mdt->mdt_md_dev.md_lu_dev.ld_site; +} + +static inline struct md_site *mdt_md_site(const struct mdt_device *mdt) +{ + return lu_site2md(mdt_lu_site(mdt)); +} static inline void mdt_export_evict(struct obd_export *exp) { diff --git a/lustre/mdt/mdt_reint.c b/lustre/mdt/mdt_reint.c index 222c9ba..bfdff5f 100644 --- a/lustre/mdt/mdt_reint.c +++ b/lustre/mdt/mdt_reint.c @@ -723,7 +723,7 @@ static int mdt_rename_lock(struct mdt_thread_info *info, struct ldlm_namespace *ns = info->mti_mdt->mdt_namespace; ldlm_policy_data_t *policy = &info->mti_policy; struct ldlm_res_id *res_id = &info->mti_res_id; - struct lu_site *ls; + struct md_site *ms; int rc; ENTRY; @@ -735,13 +735,13 @@ static int mdt_rename_lock(struct mdt_thread_info *info, * such handling in controller mds. XXX */ RETURN(0); - ls = info->mti_mdt->mdt_md_dev.md_lu_dev.ld_site; + ms = mdt_md_site(info->mti_mdt); fid_build_reg_res_name(&LUSTRE_BFL_FID, res_id); memset(policy, 0, sizeof *policy); policy->l_inodebits.bits = MDS_INODELOCK_UPDATE; - if (ls->ls_control_exp == NULL) { + if (ms->ms_control_exp == NULL) { int flags = LDLM_FL_LOCAL_ONLY | LDLM_FL_ATOMIC_CB; /* @@ -761,7 +761,8 @@ static int mdt_rename_lock(struct mdt_thread_info *info, * This is the case mdt0 is remote node, issue DLM lock like * other clients. */ - rc = ldlm_cli_enqueue(ls->ls_control_exp, NULL, &einfo, res_id, + rc = ldlm_cli_enqueue(ms->ms_control_exp, + NULL, &einfo, res_id, policy, &flags, NULL, 0, NULL, lh, 0); } -- 1.8.3.1