From b6cbc96dad17c33200670c0555a81055976f4f42 Mon Sep 17 00:00:00 2001 From: Alex Zhuravlev Date: Wed, 8 Aug 2012 14:34:21 +0400 Subject: [PATCH] LU-1711 mount: lu_site is provided by OSD previously it was created by MDT, which is started after MGS. so we relocate lu_site in OSD and now it's created before any service like MDS/MGS. Signed-off-by: Alex Zhuravlev Change-Id: I43312218c293df509064cc6a854074e652a6e691 Reviewed-on: http://review.whamcloud.com/3651 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Jinshan Xiong --- lustre/fid/fid_handler.c | 2 +- lustre/include/lu_object.h | 4 +++ lustre/include/md_object.h | 4 +-- lustre/mdt/mdt_handler.c | 55 ++++++++++++--------------------------- lustre/mdt/mdt_internal.h | 5 ++-- lustre/obdclass/lu_object.c | 9 ------- lustre/ofd/ofd_dev.c | 27 +++++-------------- lustre/ofd/ofd_internal.h | 2 -- lustre/osd-ldiskfs/osd_handler.c | 18 +++++++++++++ lustre/osd-ldiskfs/osd_internal.h | 1 + lustre/osd-zfs/osd_handler.c | 16 ++++++++++-- 11 files changed, 67 insertions(+), 76 deletions(-) diff --git a/lustre/fid/fid_handler.c b/lustre/fid/fid_handler.c index b831b3a..809e0f5 100644 --- a/lustre/fid/fid_handler.c +++ b/lustre/fid/fid_handler.c @@ -203,7 +203,7 @@ static int range_alloc_set(const struct lu_env *env, if (range_is_exhausted(loset)) { /* reached high water mark. */ - struct lu_device *dev = seq->lss_site->ms_lu.ls_top_dev; + struct lu_device *dev = seq->lss_site->ms_lu->ls_top_dev; int obd_num_clients = dev->ld_obd->obd_num_exports; __u64 set_sz; diff --git a/lustre/include/lu_object.h b/lustre/include/lu_object.h index 2ae02b3..fb7d0b3 100644 --- a/lustre/include/lu_object.h +++ b/lustre/include/lu_object.h @@ -644,6 +644,10 @@ struct lu_site { */ struct lprocfs_stats *ls_stats; struct lprocfs_stats *ls_time_stats; + /** + * XXX: a hack! fld has to find md_site via site, remove when possible + */ + struct md_site *ld_md_site; }; static inline struct lu_site_bkt_data * diff --git a/lustre/include/md_object.h b/lustre/include/md_object.h index 973c050..7cf4c68 100644 --- a/lustre/include/md_object.h +++ b/lustre/include/md_object.h @@ -527,7 +527,7 @@ struct md_object { * md-server site. */ struct md_site { - struct lu_site ms_lu; + struct lu_site *ms_lu; /** * mds number of this site. */ @@ -590,7 +590,7 @@ static inline struct md_device *md_obj2dev(const struct md_object *o) static inline struct md_site *lu_site2md(const struct lu_site *s) { - return container_of0(s, struct md_site, ms_lu); + return s->ld_md_site; } static inline int md_device_init(struct md_device *md, struct lu_device_type *t) diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index c7e1b68..ef39c28 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -4487,7 +4487,13 @@ static int mdt_stack_init(struct lu_env *env, tmp = m->mdt_bottom_exp->exp_obd->obd_lu_dev; LASSERT(tmp); m->mdt_bottom = lu2dt_dev(tmp); - tmp->ld_site = d->ld_site; + + /* initialize site's pointers: md_site, top device */ + d->ld_site = tmp->ld_site; + d->ld_site->ls_top_dev = d; + m->mdt_mite.ms_lu = tmp->ld_site; + tmp->ld_site->ld_md_site = &m->mdt_mite; + LASSERT(d->ld_site); d = tmp; tmp = mdt_layer_setup(env, LUSTRE_MDD_NAME, d, cfg); @@ -4585,7 +4591,6 @@ static void mdt_fini(const struct lu_env *env, struct mdt_device *m) { struct md_device *next = m->mdt_child; struct lu_device *d = &m->mdt_md_dev.md_lu_dev; - struct lu_site *ls = d->ld_site; struct obd_device *obd = mdt2obd_dev(m); ENTRY; @@ -4636,14 +4641,6 @@ static void mdt_fini(const struct lu_env *env, struct mdt_device *m) */ mdt_stack_fini(env, m, md2lu_dev(m->mdt_child)); - if (ls) { - struct md_site *mite; - - lu_site_fini(ls); - mite = lu_site2md(ls); - OBD_FREE_PTR(mite); - d->ld_site = NULL; - } LASSERT(cfs_atomic_read(&d->ld_ref) == 0); server_put_mount(mdt2obd_dev(m)->obd_name, NULL); @@ -4754,25 +4751,23 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, m->mdt_nosquash_strlen = 0; cfs_init_rwsem(&m->mdt_squash_sem); - OBD_ALLOC_PTR(mite); - if (mite == NULL) - GOTO(err_lmi, rc = -ENOMEM); - - s = &mite->ms_lu; - m->mdt_md_dev.md_lu_dev.ld_ops = &mdt_lu_ops; m->mdt_md_dev.md_lu_dev.ld_obd = obd; /* set this lu_device to obd, because error handling need it */ obd->obd_lu_dev = &m->mdt_md_dev.md_lu_dev; - rc = lu_site_init(s, &m->mdt_md_dev.md_lu_dev); - if (rc) { - CERROR("Can't init lu_site, rc %d\n", rc); - GOTO(err_free_site, rc); - } + /* init the stack */ + rc = mdt_stack_init((struct lu_env *)env, m, cfg, lmi); + if (rc) { + CERROR("Can't init device stack, rc %d\n", rc); + RETURN(rc); + } + + s = m->mdt_md_dev.md_lu_dev.ld_site; + mite = &m->mdt_mite; /* set server index */ - lu_site2md(s)->ms_node_id = node_id; + mite->ms_node_id = node_id; /* failover is the default * FIXME: we do not failout mds0/mgs, which may cause some problems. @@ -4790,13 +4785,6 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, } } - /* init the stack */ - rc = mdt_stack_init((struct lu_env *)env, m, cfg, lmi); - if (rc) { - CERROR("Can't init device stack, rc %d\n", rc); - GOTO(err_lu_site, rc); - } - rc = lut_init(env, &m->mdt_lut, obd, m->mdt_bottom); if (rc) GOTO(err_fini_stack, rc); @@ -4891,10 +4879,6 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, ping_evictor_start(); - rc = lu_site_init_finish(s); - if (rc) - GOTO(err_stop_service, rc); - if (obd->obd_recovering == 0) mdt_postrecov(env, m); @@ -4908,7 +4892,6 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, RETURN(0); -err_stop_service: ping_evictor_stop(); mdt_stop_ptlrpc_service(m); err_procfs: @@ -4940,10 +4923,6 @@ err_lut: lut_fini(env, &m->mdt_lut); err_fini_stack: mdt_stack_fini(env, m, md2lu_dev(m->mdt_child)); -err_lu_site: - lu_site_fini(s); -err_free_site: - OBD_FREE_PTR(mite); err_lmi: if (lmi) server_put_mount(dev, lmi->lmi_mnt); diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h index e15524d..4c33a37 100644 --- a/lustre/mdt/mdt_internal.h +++ b/lustre/mdt/mdt_internal.h @@ -98,6 +98,7 @@ struct mdt_file_data { struct mdt_device { /* super-class */ struct md_device mdt_md_dev; + struct md_site mdt_mite; struct ptlrpc_service *mdt_regular_service; struct ptlrpc_service *mdt_readpage_service; struct ptlrpc_service *mdt_xmds_service; @@ -467,9 +468,9 @@ static inline struct lu_site *mdt_lu_site(const struct mdt_device *mdt) return mdt->mdt_md_dev.md_lu_dev.ld_site; } -static inline struct md_site *mdt_md_site(const struct mdt_device *mdt) +static inline struct md_site *mdt_md_site(struct mdt_device *mdt) { - return lu_site2md(mdt_lu_site(mdt)); + return &mdt->mdt_mite; } static inline void mdt_export_evict(struct obd_export *exp) diff --git a/lustre/obdclass/lu_object.c b/lustre/obdclass/lu_object.c index caa2c01..a91b4b6 100644 --- a/lustre/obdclass/lu_object.c +++ b/lustre/obdclass/lu_object.c @@ -1213,15 +1213,6 @@ void lu_stack_fini(const struct lu_env *env, struct lu_device *top) /* purge again. */ lu_site_purge(env, site, ~0); - if (!cfs_hash_is_empty(site->ls_obj_hash)) { - /* - * Uh-oh, objects still exist. - */ - LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL); - - lu_site_print(env, site, &msgdata, lu_cdebug_printer); - } - for (scan = top; scan != NULL; scan = next) { const struct lu_device_type *ldt = scan->ld_type; struct obd_type *type; diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c index b06c04b..7b2710e 100644 --- a/lustre/ofd/ofd_dev.c +++ b/lustre/ofd/ofd_dev.c @@ -90,6 +90,12 @@ static int ofd_connect_to_next(const struct lu_env *env, struct ofd_device *m, GOTO(out, rc); } + m->ofd_dt_dev.dd_lu_dev.ld_site = + m->ofd_osd_exp->exp_obd->obd_lu_dev->ld_site; + LASSERT(m->ofd_dt_dev.dd_lu_dev.ld_site); + m->ofd_osd = lu2dt_dev(m->ofd_osd_exp->exp_obd->obd_lu_dev); + m->ofd_dt_dev.dd_lu_dev.ld_site->ls_top_dev = &m->ofd_dt_dev.dd_lu_dev; + out: if (data) OBD_FREE_PTR(data); @@ -99,7 +105,6 @@ out: static int ofd_stack_init(const struct lu_env *env, struct ofd_device *m, struct lustre_cfg *cfg) { - struct lu_device *ofd_lu = &m->ofd_dt_dev.dd_lu_dev; const char *dev = lustre_cfg_string(cfg, 0); struct lu_device *d; struct ofd_thread_info *info = ofd_info(env); @@ -130,9 +135,6 @@ static int ofd_stack_init(const struct lu_env *env, LASSERT(d); m->ofd_osd = lu2dt_dev(d); - LASSERT(ofd_lu->ld_site); - d->ld_site = ofd_lu->ld_site; - snprintf(info->fti_u.name, sizeof(info->fti_u.name), "%s-osd", lustre_cfg_string(cfg, 0)); @@ -491,15 +493,10 @@ static int ofd_init0(const struct lu_env *env, struct ofd_device *m, if (info == NULL) RETURN(-EFAULT); - rc = lu_site_init(&m->ofd_site, &m->ofd_dt_dev.dd_lu_dev); - if (rc) - GOTO(err_fini_proc, rc); - m->ofd_site.ls_top_dev = &m->ofd_dt_dev.dd_lu_dev; - rc = ofd_stack_init(env, m, cfg); if (rc) { CERROR("Can't init device stack, rc %d\n", rc); - GOTO(err_lu_site, rc); + GOTO(err_fini_proc, rc); } /* populate cached statfs data */ @@ -556,14 +553,7 @@ static int ofd_init0(const struct lu_env *env, struct ofd_device *m, target_recovery_init(&m->ofd_lut, ost_handle); - rc = lu_site_init_finish(&m->ofd_site); - if (rc) - GOTO(err_fs_cleanup, rc); - RETURN(0); -err_fs_cleanup: - target_recovery_fini(obd); - ofd_fs_cleanup(env, m); err_fini_lut: lut_fini(env, &m->ofd_lut); err_free_ns: @@ -571,8 +561,6 @@ err_free_ns: obd->obd_namespace = m->ofd_namespace = NULL; err_fini_stack: ofd_stack_fini(env, m, &m->ofd_osd->dd_lu_dev); -err_lu_site: - lu_site_fini(&m->ofd_site); err_fini_proc: ofd_procfs_fini(m); return rc; @@ -600,7 +588,6 @@ static void ofd_fini(const struct lu_env *env, struct ofd_device *m) } ofd_stack_fini(env, m, &m->ofd_dt_dev.dd_lu_dev); - lu_site_fini(&m->ofd_site); ofd_procfs_fini(m); LASSERT(cfs_atomic_read(&d->ld_ref) == 0); server_put_mount(obd->obd_name, NULL); diff --git a/lustre/ofd/ofd_internal.h b/lustre/ofd/ofd_internal.h index c94a7a2..2aca2d2 100644 --- a/lustre/ofd/ofd_internal.h +++ b/lustre/ofd/ofd_internal.h @@ -143,8 +143,6 @@ struct ofd_device { /* shall we grant space to clients not * supporting OBD_CONNECT_GRANT_PARAM? */ ofd_grant_compat_disable:1; - - struct lu_site ofd_site; }; static inline struct ofd_device *ofd_dev(struct lu_device *d) diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index f1c0506..1c37bd9 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -4497,6 +4497,14 @@ static int osd_device_init0(const struct lu_env *env, GOTO(out_mnt, rc); } + rc = lu_site_init(&o->od_site, l); + if (rc) + GOTO(out_compat, rc); + + rc = lu_site_init_finish(&o->od_site); + if (rc) + GOTO(out_compat, rc); + rc = osd_procfs_init(o, o->od_svname); if (rc != 0) { CERROR("%s: can't initialize procfs: rc = %d\n", @@ -4504,6 +4512,8 @@ static int osd_device_init0(const struct lu_env *env, GOTO(out_compat, rc); } + LASSERT(l->ld_site->ls_linkage.next && l->ld_site->ls_linkage.prev); + RETURN(0); out_compat: osd_compat_fini(o); @@ -4549,6 +4559,14 @@ static struct lu_device *osd_device_free(const struct lu_env *env, ENTRY; cleanup_capa_hash(o->od_capa_hash); + /* XXX: make osd top device in order to release reference */ + d->ld_site->ls_top_dev = d; + lu_site_purge(env, d->ld_site, -1); + if (!cfs_hash_is_empty(d->ld_site->ls_obj_hash)) { + LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL); + lu_site_print(env, d->ld_site, &msgdata, lu_cdebug_printer); + } + lu_site_fini(&o->od_site); dt_device_fini(&o->od_dt_dev); OBD_FREE_PTR(o); RETURN(NULL); diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h index 939d539..add8dd4 100644 --- a/lustre/osd-ldiskfs/osd_internal.h +++ b/lustre/osd-ldiskfs/osd_internal.h @@ -287,6 +287,7 @@ struct osd_device { struct fsfilt_operations *od_fsops; int od_connects; + struct lu_site od_site; /* * mapping for legacy OST objids diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c index fedf1e9..c34ab99 100644 --- a/lustre/osd-zfs/osd_handler.c +++ b/lustre/osd-zfs/osd_handler.c @@ -522,6 +522,14 @@ static int osd_mount(const struct lu_env *env, if (rc) GOTO(err, rc); + rc = lu_site_init(&o->od_site, osd2lu_dev(o)); + if (rc) + GOTO(err, rc); + + rc = lu_site_init_finish(&o->od_site); + if (rc) + GOTO(err, rc); + /* Use our own ZAP for inode accounting by default, this can be changed * via procfs to estimate the inode usage from the block usage */ o->od_quota_iused_est = 0; @@ -619,9 +627,13 @@ static struct lu_device *osd_device_free(const struct lu_env *env, cleanup_capa_hash(o->od_capa_hash); /* XXX: make osd top device in order to release reference */ - /*d->ld_site->ls_top_dev = d; + d->ld_site->ls_top_dev = d; lu_site_purge(env, d->ld_site, -1); - lu_site_fini(&o->od_site);*/ + if (!cfs_hash_is_empty(d->ld_site->ls_obj_hash)) { + LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL); + lu_site_print(env, d->ld_site, &msgdata, lu_cdebug_printer); + } + lu_site_fini(&o->od_site); dt_device_fini(&o->od_dt_dev); OBD_FREE_PTR(o); -- 1.8.3.1