From ef0b61b01f0fdbb500a6f03e84af0a074b613bb3 Mon Sep 17 00:00:00 2001 From: Alex Zhuravlev Date: Mon, 13 Aug 2012 10:31:49 +0400 Subject: [PATCH] LU-1301 mgs: lu device for mgs initial infrastructure to get lu-device for mgs: device itself, support for objects, init/fini old lvfs llog is still used to access on-disk data Signed-off-by: Alex Zhuravlev Change-Id: I46b2ba4bafa0f15970745ca024cd3239b9b63ec5 Reviewed-on: http://review.whamcloud.com/3674 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Mike Pershin --- lustre/include/obd.h | 14 - lustre/mgs/lproc_mgs.c | 15 +- lustre/mgs/mgs_fs.c | 32 ++- lustre/mgs/mgs_handler.c | 542 +++++++++++++++++++++++++-------------- lustre/mgs/mgs_internal.h | 90 ++++++- lustre/mgs/mgs_nids.c | 2 +- lustre/obdclass/obd_mount.c | 3 +- lustre/osd-ldiskfs/osd_handler.c | 5 +- 8 files changed, 486 insertions(+), 217 deletions(-) diff --git a/lustre/include/obd.h b/lustre/include/obd.h index d09369f..e7a66a4 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -552,19 +552,6 @@ struct client_obd { #define CL_NOT_QUOTACHECKED 1 /* client->cl_qchk_stat init value */ -struct mgs_device { - struct obd_device_target mgs_obt; - struct ptlrpc_service *mgs_service; - struct vfsmount *mgs_vfsmnt; - struct super_block *mgs_sb; - struct dentry *mgs_configs_dir; - cfs_list_t mgs_fs_db_list; - cfs_mutex_t mgs_mutex; - cfs_proc_dir_entry_t *mgs_proc_live; - cfs_time_t mgs_start_time; - struct obd_device *mgs_obd; -}; - struct mds_obd { /* NB this field MUST be first */ struct obd_device_target mds_obt; @@ -1157,7 +1144,6 @@ struct obd_device { struct echo_obd echo; struct lov_obd lov; struct lmv_obd lmv; - struct mgs_device mgs; } u; /* Fields used by LProcFS */ unsigned int obd_cntr_base; diff --git a/lustre/mgs/lproc_mgs.c b/lustre/mgs/lproc_mgs.c index 01268ac..fef6baa 100644 --- a/lustre/mgs/lproc_mgs.c +++ b/lustre/mgs/lproc_mgs.c @@ -51,17 +51,22 @@ static int lprocfs_mgs_rd_mntdev(char *page, char **start, off_t off, int count, struct obd_device *obd = (struct obd_device *)data; LASSERT(obd != NULL); +#if 0 + /* will be fixed in the subsequent patch */ LASSERT(mnt_get_devname(obd->u.mgs.mgs_vfsmnt)); *eof = 1; return snprintf(page, count, "%s\n", mnt_get_devname(obd->u.mgs.mgs_vfsmnt)); +#else + return 0; +#endif } static int mgs_fs_seq_show(struct seq_file *seq, void *v) { struct obd_device *obd = seq->private; - struct mgs_device *mgs = &obd->u.mgs; + struct mgs_device *mgs; cfs_list_t dentry_list; struct l_linux_dirent *dirent, *n; struct lu_env env; @@ -69,6 +74,8 @@ static int mgs_fs_seq_show(struct seq_file *seq, void *v) ENTRY; LASSERT(obd != NULL); + LASSERT(obd->obd_lu_dev != NULL); + mgs = lu2mgs_dev(obd->obd_lu_dev); rc = lu_env_init(&env, LCT_MG_THREAD); if (rc) @@ -129,11 +136,15 @@ static void seq_show_srpc_rules(struct seq_file *seq, const char *tgtname, static int mgsself_srpc_seq_show(struct seq_file *seq, void *v) { struct obd_device *obd = seq->private; - struct mgs_device *mgs = &obd->u.mgs; + struct mgs_device *mgs; struct fs_db *fsdb; struct lu_env env; int rc; + LASSERT(obd != NULL); + LASSERT(obd->obd_lu_dev != NULL); + mgs = lu2mgs_dev(obd->obd_lu_dev); + rc = lu_env_init(&env, LCT_MG_THREAD); if (rc) return rc; diff --git a/lustre/mgs/mgs_fs.c b/lustre/mgs/mgs_fs.c index 4c7e1ac..3b57c67 100644 --- a/lustre/mgs/mgs_fs.c +++ b/lustre/mgs/mgs_fs.c @@ -27,7 +27,7 @@ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2011, Whamcloud, Inc. + * Copyright (c) 2011, 2012, Whamcloud, Inc. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -38,6 +38,7 @@ * Lustre Management Server (MGS) filesystem interface code * * Author: Nathan Rutman + * Author: Alex Zhuravlev */ #define DEBUG_SUBSYSTEM S_MGS @@ -123,10 +124,11 @@ static struct dentry *mgs_lvfs_fid2dentry(__u64 id, __u32 gen, { struct fsfilt_fid fid; struct obd_device *obd = (struct obd_device *)data; + struct mgs_device *mgs = lu2mgs_dev(obd->obd_lu_dev); ENTRY; CDEBUG(D_DENTRY, "--> mgs_fid2dentry: ino/gen %lu/%u, sb %p\n", - (unsigned long)id, gen, obd->u.mgs.mgs_sb); + (unsigned long)id, gen, mgs->mgs_sb); if (id == 0) RETURN(ERR_PTR(-ESTALE)); @@ -134,21 +136,30 @@ static struct dentry *mgs_lvfs_fid2dentry(__u64 id, __u32 gen, fid.ino = id; fid.gen = gen; - RETURN(fsfilt_fid2dentry(obd, obd->u.mgs.mgs_vfsmnt, &fid, 0)); + RETURN(fsfilt_fid2dentry(obd, mgs->mgs_vfsmnt, &fid, 0)); } struct lvfs_callback_ops mgs_lvfs_ops = { l_fid2dentry: mgs_lvfs_fid2dentry, }; -int mgs_fs_setup(struct obd_device *obd, struct vfsmount *mnt) +int mgs_fs_setup(const struct lu_env *env, struct mgs_device *mgs) { - struct mgs_device *mgs = &obd->u.mgs; + struct obd_device *obd = mgs->mgs_obd; + struct dt_device_param p; + struct vfsmount *mnt; struct lvfs_run_ctxt saved; struct dentry *dentry; int rc; ENTRY; + dt_conf_get(env, mgs->mgs_bottom, &p); + mnt = p.ddp_mnt; + if (mnt == NULL) { + CERROR("%s: no proper support for OSD yet\n", obd->obd_name); + RETURN(-ENODEV); + } + /* FIXME what's this? Do I need it? */ rc = cfs_cleanup_group_info(); if (rc) @@ -157,6 +168,10 @@ int mgs_fs_setup(struct obd_device *obd, struct vfsmount *mnt) mgs->mgs_vfsmnt = mnt; mgs->mgs_sb = mnt->mnt_root->d_inode->i_sb; + obd->obd_fsops = fsfilt_get_ops(mt_str(p.ddp_mount_type)); + if (IS_ERR(obd->obd_fsops)) + RETURN(PTR_ERR(obd->obd_fsops)); + rc = fsfilt_setup(obd, mgs->mgs_sb); if (rc) RETURN(rc); @@ -196,9 +211,9 @@ err_pop: return rc; } -int mgs_fs_cleanup(struct obd_device *obd) +int mgs_fs_cleanup(const struct lu_env *env, struct mgs_device *mgs) { - struct mgs_device *mgs = &obd->u.mgs; + struct obd_device *obd = mgs->mgs_obd; struct lvfs_run_ctxt saved; int rc = 0; @@ -215,5 +230,8 @@ int mgs_fs_cleanup(struct obd_device *obd) pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + if (obd->obd_fsops) + fsfilt_put_ops(obd->obd_fsops); + return rc; } diff --git a/lustre/mgs/mgs_handler.c b/lustre/mgs/mgs_handler.c index 4840c31..e5bff51 100644 --- a/lustre/mgs/mgs_handler.c +++ b/lustre/mgs/mgs_handler.c @@ -36,6 +36,8 @@ * lustre/mgs/mgs_handler.c * * Author: Nathan Rutman + * Author: Alex Zhuravlev + * Author: Mikhail Pershin */ #define DEBUG_SUBSYSTEM S_MGS @@ -140,7 +142,6 @@ static int mgs_disconnect(struct obd_export *exp) RETURN(rc); } -static int mgs_cleanup(struct obd_device *obd); static int mgs_handle(struct ptlrpc_request *req); static int mgs_llog_init(struct obd_device *obd, struct obd_llog_group *olg, @@ -168,179 +169,6 @@ static int mgs_llog_finish(struct obd_device *obd, int count) RETURN(rc); } -/* Start the MGS obd */ -static int mgs_setup(struct obd_device *obd, struct lustre_cfg *lcfg) -{ - static struct ptlrpc_service_conf conf; - struct lprocfs_static_vars lvars; - struct mgs_device *mgs = &obd->u.mgs; - struct lustre_mount_info *lmi; - struct lustre_sb_info *lsi; - struct vfsmount *mnt; - int rc = 0; - ENTRY; - - CDEBUG(D_CONFIG, "Starting MGS\n"); - - /* Find our disk */ - lmi = server_get_mount(obd->obd_name); - if (!lmi) - RETURN(rc = -EINVAL); - - mnt = lmi->lmi_mnt; - lsi = s2lsi(lmi->lmi_sb); - obd->obd_fsops = fsfilt_get_ops(lsi->lsi_fstype); - if (IS_ERR(obd->obd_fsops)) - GOTO(err_put, rc = PTR_ERR(obd->obd_fsops)); - - if (lvfs_check_rdonly(lvfs_sbdev(mnt->mnt_sb))) { - CERROR("%s: Underlying device is marked as read-only. " - "Setup failed\n", obd->obd_name); - GOTO(err_ops, rc = -EROFS); - } - - obd->u.obt.obt_magic = OBT_MAGIC; - obd->u.obt.obt_instance = 0; - mgs->mgs_obd = obd; - - /* namespace for mgs llog */ - obd->obd_namespace = ldlm_namespace_new(obd ,"MGS", - LDLM_NAMESPACE_SERVER, - LDLM_NAMESPACE_MODEST, - LDLM_NS_TYPE_MGT); - if (obd->obd_namespace == NULL) - GOTO(err_ops, rc = -ENOMEM); - - /* ldlm setup */ - ptlrpc_init_client(LDLM_CB_REQUEST_PORTAL, LDLM_CB_REPLY_PORTAL, - "mgs_ldlm_client", &obd->obd_ldlm_client); - - rc = mgs_fs_setup(obd, mnt); - if (rc) { - CERROR("%s: MGS filesystem method init failed: rc = %d\n", - obd->obd_name, rc); - GOTO(err_ns, rc); - } - - rc = obd_llog_init(obd, &obd->obd_olg, obd, NULL); - if (rc) - GOTO(err_fs, rc); - - /* No recovery for MGC's */ - obd->obd_replayable = 0; - - /* Internal mgs setup */ - mgs_init_fsdb_list(mgs); - cfs_mutex_init(&mgs->mgs_mutex); - mgs->mgs_start_time = cfs_time_current_sec(); - - /* Setup proc */ - lprocfs_mgs_init_vars(&lvars); - if (lprocfs_obd_setup(obd, lvars.obd_vars) == 0) { - lproc_mgs_setup(mgs); - rc = lprocfs_alloc_md_stats(obd, LPROC_MGS_LAST); - if (rc) - GOTO(err_llog, rc); - } - - conf = (typeof(conf)) { - .psc_name = LUSTRE_MGS_NAME, - .psc_watchdog_factor = MGS_SERVICE_WATCHDOG_FACTOR, - .psc_buf = { - .bc_nbufs = MGS_NBUFS, - .bc_buf_size = MGS_BUFSIZE, - .bc_req_max_size = MGS_MAXREQSIZE, - .bc_rep_max_size = MGS_MAXREPSIZE, - .bc_req_portal = MGS_REQUEST_PORTAL, - .bc_rep_portal = MGC_REPLY_PORTAL, - }, - .psc_thr = { - .tc_thr_name = "ll_mgs", - .tc_nthrs_init = MGS_NTHRS_INIT, - .tc_nthrs_max = MGS_NTHRS_MAX, - .tc_ctx_tags = LCT_MG_THREAD, - }, - .psc_ops = { - .so_req_handler = mgs_handle, - .so_req_printer = target_print_req, - }, - }; - /* Start the service threads */ - mgs->mgs_service = ptlrpc_register_service(&conf, obd->obd_proc_entry); - if (IS_ERR(mgs->mgs_service)) { - rc = PTR_ERR(mgs->mgs_service); - CERROR("failed to start service: %d\n", rc); - GOTO(err_llog, rc); - } - - ping_evictor_start(); - - CDEBUG(D_INFO, "MGS %s started\n", obd->obd_name); - - RETURN(0); - -err_llog: - lproc_mgs_cleanup(mgs); - obd_llog_finish(obd, 0); -err_fs: - /* No extra cleanup needed for llog_init_commit_thread() */ - mgs_fs_cleanup(obd); -err_ns: - ldlm_namespace_free(obd->obd_namespace, NULL, 0); - obd->obd_namespace = NULL; -err_ops: - fsfilt_put_ops(obd->obd_fsops); -err_put: - server_put_mount(obd->obd_name, mnt); - mgs->mgs_sb = 0; - return rc; -} - -static int mgs_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) -{ - struct mgs_device *mgs = &obd->u.mgs; - int rc = 0; - ENTRY; - - switch (stage) { - case OBD_CLEANUP_EARLY: - break; - case OBD_CLEANUP_EXPORTS: - ping_evictor_stop(); - ptlrpc_unregister_service(mgs->mgs_service); - mgs_cleanup_fsdb_list(mgs); - rc = obd_llog_finish(obd, 0); - lproc_mgs_cleanup(mgs); - break; - } - RETURN(rc); -} - -/** - * Performs cleanup procedures for passed \a obd given it is mgs obd. - */ -static int mgs_cleanup(struct obd_device *obd) -{ - struct mgs_device *mgs = &obd->u.mgs; - ENTRY; - - if (mgs->mgs_sb == NULL) - RETURN(0); - - mgs_fs_cleanup(obd); - - server_put_mount(obd->obd_name, mgs->mgs_vfsmnt); - mgs->mgs_sb = NULL; - - ldlm_namespace_free(obd->obd_namespace, NULL, 1); - obd->obd_namespace = NULL; - - fsfilt_put_ops(obd->obd_fsops); - - LCONSOLE_INFO("%s has stopped.\n", obd->obd_name); - RETURN(0); -} - static int mgs_completion_ast_config(struct ldlm_lock *lock, int flags, void *cbdata) { @@ -631,7 +459,7 @@ out_nolock: *rep_mti = *mti; /* Flush logs to disk */ - fsfilt_sync(obd, obd->u.mgs.mgs_sb); + dt_sync(req->rq_svc_thread->t_env, mgs->mgs_bottom); RETURN(rc); } @@ -1186,20 +1014,365 @@ out: RETURN(rc); } +static int mgs_connect_to_osd(struct mgs_device *m, const char *nextdev) +{ + struct obd_connect_data *data = NULL; + struct obd_device *obd; + int rc; + ENTRY; + + OBD_ALLOC_PTR(data); + if (data == NULL) + RETURN(-ENOMEM); + + obd = class_name2obd(nextdev); + if (obd == NULL) { + CERROR("can't locate next device: %s\n", nextdev); + GOTO(out, rc = -ENOTCONN); + } + + data->ocd_version = LUSTRE_VERSION_CODE; + + rc = obd_connect(NULL, &m->mgs_bottom_exp, obd, + &obd->obd_uuid, data, NULL); + if (rc) { + CERROR("cannot connect to next dev %s (%d)\n", nextdev, rc); + GOTO(out, rc); + } + + m->mgs_bottom = lu2dt_dev(m->mgs_bottom_exp->exp_obd->obd_lu_dev); + m->mgs_dt_dev.dd_lu_dev.ld_site = m->mgs_bottom->dd_lu_dev.ld_site; + LASSERT(m->mgs_dt_dev.dd_lu_dev.ld_site); +out: + OBD_FREE_PTR(data); + RETURN(rc); +} + + +static int mgs_init0(const struct lu_env *env, struct mgs_device *mgs, + struct lu_device_type *ldt, struct lustre_cfg *lcfg) +{ + static struct ptlrpc_service_conf conf; + struct lprocfs_static_vars lvars = { 0 }; + struct obd_device *obd; + struct lustre_mount_info *lmi; + int rc; + ENTRY; + + lmi = server_get_mount(lustre_cfg_string(lcfg, 0)); + if (lmi == NULL) + RETURN(-ENODEV); + + mgs->mgs_dt_dev.dd_lu_dev.ld_ops = &mgs_lu_ops; + + rc = mgs_connect_to_osd(mgs, lustre_cfg_string(lcfg, 3)); + if (rc) + RETURN(rc); + + obd = class_name2obd(lustre_cfg_string(lcfg, 0)); + LASSERT(obd); + mgs->mgs_obd = obd; + mgs->mgs_obd->obd_lu_dev = &mgs->mgs_dt_dev.dd_lu_dev; + + obd->u.obt.obt_magic = OBT_MAGIC; + obd->u.obt.obt_instance = 0; + + /* namespace for mgs llog */ + obd->obd_namespace = ldlm_namespace_new(obd ,"MGS", + LDLM_NAMESPACE_SERVER, + LDLM_NAMESPACE_MODEST, + LDLM_NS_TYPE_MGT); + if (obd->obd_namespace == NULL) + GOTO(err_ops, rc = -ENOMEM); + + /* ldlm setup */ + ptlrpc_init_client(LDLM_CB_REQUEST_PORTAL, LDLM_CB_REPLY_PORTAL, + "mgs_ldlm_client", &obd->obd_ldlm_client); + + rc = mgs_fs_setup(env, mgs); + if (rc) { + CERROR("%s: MGS filesystem method init failed: rc = %d\n", + obd->obd_name, rc); + GOTO(err_ns, rc); + } + + rc = obd_llog_init(obd, &obd->obd_olg, obd, NULL); + if (rc) + GOTO(err_fs, rc); + + /* No recovery for MGC's */ + obd->obd_replayable = 0; + + /* Internal mgs setup */ + mgs_init_fsdb_list(mgs); + cfs_mutex_init(&mgs->mgs_mutex); + mgs->mgs_start_time = cfs_time_current_sec(); + cfs_spin_lock_init(&mgs->mgs_lock); + + /* Setup proc */ + lprocfs_mgs_init_vars(&lvars); + if (lprocfs_obd_setup(obd, lvars.obd_vars) == 0) { + lproc_mgs_setup(mgs); + rc = lprocfs_alloc_md_stats(obd, LPROC_MGS_LAST); + if (rc) + GOTO(err_llog, rc); + } + + conf = (typeof(conf)) { + .psc_name = LUSTRE_MGS_NAME, + .psc_watchdog_factor = MGS_SERVICE_WATCHDOG_FACTOR, + .psc_buf = { + .bc_nbufs = MGS_NBUFS, + .bc_buf_size = MGS_BUFSIZE, + .bc_req_max_size = MGS_MAXREQSIZE, + .bc_rep_max_size = MGS_MAXREPSIZE, + .bc_req_portal = MGS_REQUEST_PORTAL, + .bc_rep_portal = MGC_REPLY_PORTAL, + }, + .psc_thr = { + .tc_thr_name = "ll_mgs", + .tc_nthrs_init = MGS_NTHRS_INIT, + .tc_nthrs_max = MGS_NTHRS_MAX, + .tc_ctx_tags = LCT_MG_THREAD, + }, + .psc_ops = { + .so_req_handler = mgs_handle, + .so_req_printer = target_print_req, + }, + }; + /* Start the service threads */ + mgs->mgs_service = ptlrpc_register_service(&conf, obd->obd_proc_entry); + if (IS_ERR(mgs->mgs_service)) { + rc = PTR_ERR(mgs->mgs_service); + CERROR("failed to start service: %d\n", rc); + GOTO(err_lproc, rc); + } + + ping_evictor_start(); + + CDEBUG(D_INFO, "MGS %s started\n", obd->obd_name); + + /* device stack is not yet fully setup to keep no objects behind */ + lu_site_purge(env, mgs2lu_dev(mgs)->ld_site, ~0); + RETURN(0); + +err_lproc: + lproc_mgs_cleanup(mgs); +err_llog: +err_fs: + /* No extra cleanup needed for llog_init_commit_thread() */ + mgs_fs_cleanup(env, mgs); +err_ns: + ldlm_namespace_free(obd->obd_namespace, NULL, 0); + obd->obd_namespace = NULL; +err_ops: + lu_site_purge(env, mgs2lu_dev(mgs)->ld_site, ~0); + if (!cfs_hash_is_empty(mgs2lu_dev(mgs)->ld_site->ls_obj_hash)) { + LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL); + lu_site_print(env, mgs2lu_dev(mgs)->ld_site, &msgdata, + lu_cdebug_printer); + } + obd_disconnect(mgs->mgs_bottom_exp); + RETURN(rc); +} + +static struct lu_device *mgs_device_free(const struct lu_env *env, + struct lu_device *lu) +{ + struct mgs_device *mgs = lu2mgs_dev(lu); + ENTRY; + + dt_device_fini(&mgs->mgs_dt_dev); + OBD_FREE_PTR(mgs); + RETURN(NULL); +} + +static int mgs_process_config(const struct lu_env *env, + struct lu_device *dev, + struct lustre_cfg *lcfg) +{ + LBUG(); + return 0; +} + +static int mgs_object_init(const struct lu_env *env, struct lu_object *o, + const struct lu_object_conf *unused) +{ + struct mgs_device *d = lu2mgs_dev(o->lo_dev); + struct lu_device *under; + struct lu_object *below; + int rc = 0; + ENTRY; + + /* do no set .do_ops as mgs calls to bottom osd directly */ + + CDEBUG(D_INFO, "object init, fid = "DFID"\n", + PFID(lu_object_fid(o))); + + under = &d->mgs_bottom->dd_lu_dev; + below = under->ld_ops->ldo_object_alloc(env, o->lo_header, under); + if (below != NULL) + lu_object_add(o, below); + else + rc = -ENOMEM; + + return 0; +} + +static void mgs_object_free(const struct lu_env *env, struct lu_object *o) +{ + struct mgs_object *obj = lu2mgs_obj(o); + struct lu_object_header *h = o->lo_header; + + dt_object_fini(&obj->mgo_obj); + lu_object_header_fini(h); + OBD_FREE_PTR(obj); +} + +static int mgs_object_print(const struct lu_env *env, void *cookie, + lu_printer_t p, const struct lu_object *l) +{ + const struct mgs_object *o = lu2mgs_obj((struct lu_object *) l); + + return (*p)(env, cookie, LUSTRE_MGS_NAME"-object@%p", o); +} + +struct lu_object_operations mgs_lu_obj_ops = { + .loo_object_init = mgs_object_init, + .loo_object_free = mgs_object_free, + .loo_object_print = mgs_object_print, +}; + +struct lu_object *mgs_object_alloc(const struct lu_env *env, + const struct lu_object_header *hdr, + struct lu_device *d) +{ + struct lu_object_header *h; + struct mgs_object *o; + struct lu_object *l; + + LASSERT(hdr == NULL); + + OBD_ALLOC_PTR(o); + if (o != NULL) { + l = &o->mgo_obj.do_lu; + h = &o->mgo_header; + + lu_object_header_init(h); + dt_object_init(&o->mgo_obj, h, d); + lu_object_add_top(h, l); + + l->lo_ops = &mgs_lu_obj_ops; + + return l; + } else { + return NULL; + } +} + +const struct lu_device_operations mgs_lu_ops = { + .ldo_object_alloc = mgs_object_alloc, + .ldo_process_config = mgs_process_config, +}; + +static struct lu_device *mgs_device_alloc(const struct lu_env *env, + struct lu_device_type *type, + struct lustre_cfg *lcfg) +{ + struct mgs_device *mgs; + struct lu_device *ludev; + + OBD_ALLOC_PTR(mgs); + if (mgs == NULL) { + ludev = ERR_PTR(-ENOMEM); + } else { + int rc; + + ludev = mgs2lu_dev(mgs); + dt_device_init(&mgs->mgs_dt_dev, type); + rc = mgs_init0(env, mgs, type, lcfg); + if (rc != 0) { + mgs_device_free(env, ludev); + ludev = ERR_PTR(rc); + } + } + return ludev; +} + +static struct lu_device *mgs_device_fini(const struct lu_env *env, + struct lu_device *d) +{ + struct mgs_device *mgs = lu2mgs_dev(d); + struct obd_device *obd = mgs->mgs_obd; + ENTRY; + + LASSERT(mgs->mgs_bottom); + + ping_evictor_stop(); + + ptlrpc_unregister_service(mgs->mgs_service); + + obd_exports_barrier(obd); + obd_zombie_barrier(); + + mgs_cleanup_fsdb_list(mgs); + lproc_mgs_cleanup(mgs); + + obd_llog_finish(obd, 0); + + mgs_fs_cleanup(env, mgs); + + ldlm_namespace_free(obd->obd_namespace, NULL, 1); + obd->obd_namespace = NULL; + + lu_site_purge(env, d->ld_site, ~0); + if (!cfs_hash_is_empty(d->ld_site->ls_obj_hash)) { + LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL); + lu_site_print(env, d->ld_site, &msgdata, lu_cdebug_printer); + } + + LASSERT(mgs->mgs_bottom_exp); + obd_disconnect(mgs->mgs_bottom_exp); + + server_put_mount(obd->obd_name, NULL); + + RETURN(NULL); +} + /* context key constructor/destructor: mgs_key_init, mgs_key_fini */ LU_KEY_INIT_FINI(mgs, struct mgs_thread_info); +LU_TYPE_INIT_FINI(mgs, &mgs_thread_key); + LU_CONTEXT_KEY_DEFINE(mgs, LCT_MG_THREAD); +static struct lu_device_type_operations mgs_device_type_ops = { + .ldto_init = mgs_type_init, + .ldto_fini = mgs_type_fini, + + .ldto_start = mgs_type_start, + .ldto_stop = mgs_type_stop, + + .ldto_device_alloc = mgs_device_alloc, + .ldto_device_free = mgs_device_free, + + .ldto_device_fini = mgs_device_fini +}; + +static struct lu_device_type mgs_device_type = { + .ldt_tags = LU_DEVICE_DT, + .ldt_name = LUSTRE_MGS_NAME, + .ldt_ops = &mgs_device_type_ops, + .ldt_ctx_tags = LCT_MG_THREAD +}; + + /* use obd ops to offer management infrastructure */ static struct obd_ops mgs_obd_ops = { .o_owner = THIS_MODULE, .o_connect = mgs_connect, .o_reconnect = mgs_reconnect, .o_disconnect = mgs_disconnect, - .o_setup = mgs_setup, - .o_precleanup = mgs_precleanup, - .o_cleanup = mgs_cleanup, .o_init_export = mgs_init_export, .o_destroy_export = mgs_destroy_export, .o_iocontrol = mgs_iocontrol, @@ -1210,24 +1383,17 @@ static struct obd_ops mgs_obd_ops = { static int __init mgs_init(void) { struct lprocfs_static_vars lvars; - int rc; lprocfs_mgs_init_vars(&lvars); - class_register_type(&mgs_obd_ops, NULL, - lvars.module_vars, LUSTRE_MGS_NAME, NULL); - - LU_CONTEXT_KEY_INIT(&mgs_thread_key); - rc = lu_context_key_register(&mgs_thread_key); - if (rc) - class_unregister_type(LUSTRE_MGS_NAME); + class_register_type(&mgs_obd_ops, NULL, lvars.module_vars, + LUSTRE_MGS_NAME, &mgs_device_type); - return rc; + return 0; } static void /*__exit*/ mgs_exit(void) { class_unregister_type(LUSTRE_MGS_NAME); - lu_context_key_degister(&mgs_thread_key); } MODULE_AUTHOR("Sun Microsystems, Inc. "); diff --git a/lustre/mgs/mgs_internal.h b/lustre/mgs/mgs_internal.h index 2585a2e..4eb656f 100644 --- a/lustre/mgs/mgs_internal.h +++ b/lustre/mgs/mgs_internal.h @@ -159,6 +159,32 @@ struct fs_db { int class_dentry_readdir(const struct lu_env *env, struct mgs_device *mgs, cfs_list_t *list); +struct mgs_device { + struct dt_device mgs_dt_dev; + struct ptlrpc_service *mgs_service; + struct dt_device *mgs_bottom; + struct obd_export *mgs_bottom_exp; + struct dentry *mgs_configs_dir; + struct dt_object *mgs_nidtbl_dir; + cfs_list_t mgs_fs_db_list; + cfs_spinlock_t mgs_lock; /* covers mgs_fs_db_list */ + cfs_proc_dir_entry_t *mgs_proc_live; + cfs_time_t mgs_start_time; + struct obd_device *mgs_obd; + struct local_oid_storage *mgs_los; + struct vfsmount *mgs_vfsmnt; + struct super_block *mgs_sb; + cfs_mutex_t mgs_mutex; +}; + +/* this is a top object */ +struct mgs_object { + struct lu_object_header mgo_header; + struct dt_object mgo_obj; + int mgo_no_attrs; + int mgo_reserved; +}; + int mgs_init_fsdb_list(struct mgs_device *mgs); int mgs_cleanup_fsdb_list(struct mgs_device *mgs); int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs, char *name, @@ -213,8 +239,8 @@ int mgs_fsc_attach(const struct lu_env *env, struct obd_export *exp, int mgs_export_stats_init(struct obd_device *obd, struct obd_export *exp, void *localdata); int mgs_client_free(struct obd_export *exp); -int mgs_fs_setup(struct obd_device *obd, struct vfsmount *mnt); -int mgs_fs_cleanup(struct obd_device *obddev); +int mgs_fs_setup(const struct lu_env *env, struct mgs_device *m); +int mgs_fs_cleanup(const struct lu_env *env, struct mgs_device *m); #define strsuf(buf, suffix) (strcmp((buf)+strlen(buf)-strlen(suffix), (suffix))) #ifdef LPROCFS @@ -277,9 +303,67 @@ static inline struct mgs_thread_info *mgs_env_info(const struct lu_env *env) return info; } +extern const struct lu_device_operations mgs_lu_ops; + +static inline int lu_device_is_mgs(struct lu_device *d) +{ + return ergo(d != NULL && d->ld_ops != NULL, d->ld_ops == &mgs_lu_ops); +} + +static inline struct mgs_device* lu2mgs_dev(struct lu_device *d) +{ + LASSERT(lu_device_is_mgs(d)); + return container_of0(d, struct mgs_device, mgs_dt_dev.dd_lu_dev); +} + static inline struct mgs_device *exp2mgs_dev(struct obd_export *exp) { - return &exp->exp_obd->u.mgs; + return lu2mgs_dev(exp->exp_obd->obd_lu_dev); +} + +static inline struct lu_device *mgs2lu_dev(struct mgs_device *d) +{ + return (&d->mgs_dt_dev.dd_lu_dev); +} + +static inline struct mgs_device *dt2mgs_dev(struct dt_device *d) +{ + LASSERT(lu_device_is_mgs(&d->dd_lu_dev)); + return container_of0(d, struct mgs_device, mgs_dt_dev); +} + +static inline struct mgs_object *lu2mgs_obj(struct lu_object *o) +{ + LASSERT(ergo(o != NULL, lu_device_is_mgs(o->lo_dev))); + return container_of0(o, struct mgs_object, mgo_obj.do_lu); +} + +static inline struct lu_object *mgs2lu_obj(struct mgs_object *obj) +{ + return &obj->mgo_obj.do_lu; +} + +static inline struct mgs_object *mgs_obj(const struct lu_object *o) +{ + LASSERT(lu_device_is_mgs(o->lo_dev)); + return container_of0(o, struct mgs_object, mgo_obj.do_lu); +} + +static inline struct mgs_object *dt2mgs_obj(const struct dt_object *d) +{ + return mgs_obj(&d->do_lu); +} + +static inline struct dt_object* mgs_object_child(struct mgs_object *o) +{ + return container_of0(lu_object_next(mgs2lu_obj(o)), + struct dt_object, do_lu); +} + +static inline struct dt_object *dt_object_child(struct dt_object *o) +{ + return container_of0(lu_object_next(&(o)->do_lu), + struct dt_object, do_lu); } #endif /* _MGS_INTERNAL_H */ diff --git a/lustre/mgs/mgs_nids.c b/lustre/mgs/mgs_nids.c index d62f600..b23748f 100644 --- a/lustre/mgs/mgs_nids.c +++ b/lustre/mgs/mgs_nids.c @@ -254,7 +254,7 @@ static int nidtbl_update_version(const struct lu_env *env, if (rc == sizeof(version)) rc = 0; filp_close(file, 0); - fsfilt_sync(obd, obd->u.mgs.mgs_sb); + dt_sync(env, mgs->mgs_bottom); } else { rc = PTR_ERR(file); } diff --git a/lustre/obdclass/obd_mount.c b/lustre/obdclass/obd_mount.c index b193034..55e9666 100644 --- a/lustre/obdclass/obd_mount.c +++ b/lustre/obdclass/obd_mount.c @@ -407,7 +407,8 @@ static int server_start_mgs(struct super_block *sb) if (!rc) { rc = lustre_start_simple(LUSTRE_MGS_OBDNAME, LUSTRE_MGS_NAME, - LUSTRE_MGS_OBDNAME, 0, 0, 0, 0); + LUSTRE_MGS_OBDNAME, 0, 0, + lsi->lsi_osd_obdname, 0); /* Do NOT call server_deregister_mount() here. This leads to * inability cleanup cleanly and free lsi and other stuff when * mgs calls server_put_mount() in error handling case. -umka */ diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c index 17ad09a..b1a2f17 100644 --- a/lustre/osd-ldiskfs/osd_handler.c +++ b/lustre/osd-ldiskfs/osd_handler.c @@ -1008,6 +1008,7 @@ static void osd_conf_get(const struct lu_env *env, param->ddp_max_name_len = LDISKFS_NAME_LEN; param->ddp_max_nlink = LDISKFS_LINK_MAX; param->ddp_block_shift = sb->s_blocksize_bits; + param->ddp_mount_type = LDD_MT_LDISKFS; param->ddp_mntopts = 0; if (test_opt(sb, XATTR_USER)) param->ddp_mntopts |= MNTOPT_USERXATTR; @@ -4508,7 +4509,7 @@ static int osd_mount(const struct lu_env *env, OBD_PAGE_ALLOC(__page, CFS_ALLOC_STD); if (__page == NULL) - RETURN(-ENOMEM); + GOTO(out, rc = -ENOMEM); str = lustre_cfg_string(cfg, 2); s_flags = simple_strtoul(str, NULL, 0); @@ -4572,6 +4573,8 @@ static int osd_mount(const struct lu_env *env, out: if (__page) OBD_PAGE_FREE(__page); + if (rc) + fsfilt_put_ops(o->od_fsops); RETURN(rc); } -- 1.8.3.1