From fd1f64fc062a0a850d8da72acfa22b19ce7b5885 Mon Sep 17 00:00:00 2001 From: tappro Date: Mon, 3 Apr 2006 18:27:35 +0000 Subject: [PATCH] generic cmm device support --- lustre/cmm/cmm_device.c | 331 ++++++++++++++++++++++++++++++++++++++++++++++ lustre/cmm/cmm_internal.h | 101 ++++++++++++++ lustre/cmm/cmm_object.c | 134 +++++++++++++++++++ 3 files changed, 566 insertions(+) create mode 100644 lustre/cmm/cmm_device.c create mode 100644 lustre/cmm/cmm_internal.h create mode 100644 lustre/cmm/cmm_object.c diff --git a/lustre/cmm/cmm_device.c b/lustre/cmm/cmm_device.c new file mode 100644 index 0000000..a99c0b2 --- /dev/null +++ b/lustre/cmm/cmm_device.c @@ -0,0 +1,331 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * lustre/cmm/cmm_device.c + * Lustre Cluster Metadata Manager (cmm) + * + * Copyright (c) 2006 Cluster File Systems, Inc. + * Author: Mike Pershin + * + * This file is part of the Lustre file system, http://www.lustre.org + * Lustre is a trademark of Cluster File Systems, Inc. + * + * You may have signed or agreed to another license before downloading + * this software. If so, you are bound by the terms and conditions + * of that agreement, and the following does not apply to you. See the + * LICENSE file included with this distribution for more information. + * + * If you did not agree to a different license, then this copy of Lustre + * is open source software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In either case, Lustre is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * license text for more details. + */ + +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif +#define DEBUG_SUBSYSTEM S_MDS + +#include + +#include +#include + +#include "cmm_internal.h" + +#include +#include + +static struct obd_ops cmm_obd_device_ops = { + .o_owner = THIS_MODULE +}; + +static struct lu_device_operations cmm_lu_ops; + +static int lu_device_is_cmm(struct lu_device *d) +{ + /* + * XXX for now. Tags in lu_device_type->ldt_something are needed. + */ + return ergo(d->ld_ops != NULL, d->ld_ops == &cmm_lu_ops); +} + +static struct lu_device_operations cmm_lu_ops = { + .ldo_object_alloc = cmm_object_alloc, + .ldo_object_init = cmm_object_init, + .ldo_object_free = cmm_object_free, + .ldo_object_release = cmm_object_release, + .ldo_object_print = cmm_object_print +}; +#if 0 +int mds_md_connect(struct obd_device *obd, char *md_name) +{ + struct mds_obd *mds = &obd->u.mds; + struct lustre_handle conn = {0}; + unsigned long sec_flags = PTLRPC_SEC_FL_MDS; + int rc, value; + __u32 valsize; + ENTRY; + + if (IS_ERR(mds->mds_md_obd)) + RETURN(PTR_ERR(mds->mds_md_obd)); + + if (mds->mds_md_connected) + RETURN(0); + + down(&mds->mds_md_sem); + if (mds->mds_md_connected) { + up(&mds->mds_md_sem); + RETURN(0); + } + + mds->mds_md_obd = class_name2obd(md_name); + if (!mds->mds_md_obd) { + CERROR("MDS cannot locate MD(LMV) %s\n", + md_name); + mds->mds_md_obd = ERR_PTR(-ENOTCONN); + GOTO(err_last, rc = -ENOTCONN); + } + + rc = obd_connect(&conn, mds->mds_md_obd, &obd->obd_uuid, NULL, + OBD_OPT_MDS_CONNECTION); + if (rc) { + CERROR("MDS cannot connect to MD(LMV) %s (%d)\n", + md_name, rc); + mds->mds_md_obd = ERR_PTR(rc); + GOTO(err_last, rc); + } + mds->mds_md_exp = class_conn2export(&conn); + if (mds->mds_md_exp == NULL) + CERROR("can't get export!\n"); + + rc = obd_register_observer(mds->mds_md_obd, obd); + if (rc) { + CERROR("MDS cannot register as observer of MD(LMV) %s, " + "rc = %d\n", md_name, rc); + GOTO(err_discon, rc); + } + + /* retrieve size of EA */ + rc = obd_get_info(mds->mds_md_exp, strlen("mdsize"), + "mdsize", &valsize, &value); + if (rc) + GOTO(err_reg, rc); + + if (value > mds->mds_max_mdsize) + mds->mds_max_mdsize = value; + + /* find our number in LMV cluster */ + rc = obd_get_info(mds->mds_md_exp, strlen("mdsnum"), + "mdsnum", &valsize, &value); + if (rc) + GOTO(err_reg, rc); + + mds->mds_num = value; + + rc = obd_set_info(mds->mds_md_exp, strlen("inter_mds"), + "inter_mds", 0, NULL); + if (rc) + GOTO(err_reg, rc); + + if (mds->mds_mds_sec) { + rc = obd_set_info(mds->mds_md_exp, strlen("sec"), "sec", + strlen(mds->mds_mds_sec), mds->mds_mds_sec); + if (rc) + GOTO(err_reg, rc); + } + + rc = obd_set_info(mds->mds_md_exp, strlen("sec_flags"), "sec_flags", + sizeof(sec_flags), &sec_flags); + if (rc) + GOTO(err_reg, rc); + + mds->mds_md_connected = 1; + up(&mds->mds_md_sem); + RETURN(0); + +err_reg: + obd_register_observer(mds->mds_md_obd, NULL); +err_discon: + obd_disconnect(mds->mds_md_exp, 0); + mds->mds_md_exp = NULL; + mds->mds_md_obd = ERR_PTR(rc); +err_last: + up(&mds->mds_md_sem); + return rc; +} +int mds_md_disconnect(struct obd_device *obd, int flags) +{ + struct mds_obd *mds = &obd->u.mds; + int rc = 0; + ENTRY; + + if (!mds->mds_md_connected) + RETURN(0); + + down(&mds->mds_md_sem); + if (!IS_ERR(mds->mds_md_obd) && mds->mds_md_exp != NULL) { + LASSERT(mds->mds_md_connected); + + obd_register_observer(mds->mds_md_obd, NULL); + + if (flags & OBD_OPT_FORCE) { + struct obd_device *lmv_obd; + struct obd_ioctl_data ioc_data = { 0 }; + + lmv_obd = class_exp2obd(mds->mds_md_exp); + if (lmv_obd == NULL) + GOTO(out, rc = 0); + + /* + * making disconnecting lmv stuff do not send anything + * to all remote MDSs from LMV. This is needed to + * prevent possible hanging with endless recovery, when + * MDS sends disconnect to already disconnected + * target. Probably this is wrong, but client does the + * same in --force mode and I do not see why can't we do + * it here. --umka. + */ + lmv_obd->obd_no_recov = 1; + obd_iocontrol(IOC_OSC_SET_ACTIVE, mds->mds_md_exp, + sizeof(ioc_data), &ioc_data, NULL); + } + + /* + * if obd_disconnect() fails (probably because the export was + * disconnected by class_disconnect_exports()) then we just need + * to drop our ref. + */ + mds->mds_md_connected = 0; + rc = obd_disconnect(mds->mds_md_exp, flags); + if (rc) + class_export_put(mds->mds_md_exp); + + out: + mds->mds_md_exp = NULL; + mds->mds_md_obd = NULL; + } + up(&mds->mds_md_sem); + RETURN(rc); +} +#endif +static int cmm_init(struct cmm_device *m, + struct lu_device_type *t, struct lustre_cfg *cfg) +{ + char ns_name[48]; + struct lu_device *lu_dev = cmm2lu_dev(m); + ENTRY; + + md_device_init(&m->cmm_md_dev, t); + + lu_dev->ld_ops = &cmm_lu_ops; + + snprintf(ns_name, sizeof ns_name, LUSTRE_CMM0_NAME"-%p", m); + + return 0; +} + +struct lu_device *cmm_device_alloc(struct lu_device_type *t, + struct lustre_cfg *cfg) +{ + struct lu_device *l; + struct cmm_device *m; + + OBD_ALLOC_PTR(m); + if (m == NULL) { + l = ERR_PTR(-ENOMEM); + } else { + int err; + + err = cmm_init(m, t, cfg); + if (!err) + l = cmm2lu_dev(m); + else + l = ERR_PTR(err); + } + + return l; +} + +static void cmm_fini(struct lu_device *d) +{ + struct cmm_device *m = lu2cmm_dev(d); + + LASSERT(atomic_read(&d->ld_ref) == 0); + md_device_fini(&m->cmm_md_dev); +} + +void cmm_device_free(struct lu_device *m) +{ + cmm_fini(m); + OBD_FREE_PTR(m); +} + +int cmm_type_init(struct lu_device_type *t) +{ + return 0; +} + +void cmm_type_fini(struct lu_device_type *t) +{ + return; +} + +static struct lu_device_type_operations cmm_device_type_ops = { + .ldto_init = cmm_type_init, + .ldto_fini = cmm_type_fini, + + .ldto_device_alloc = cmm_device_alloc, + .ldto_device_free = cmm_device_free +}; + +static struct lu_device_type cmm_device_type = { + .ldt_name = LUSTRE_CMM0_NAME, + .ldt_ops = &cmm_device_type_ops +}; + +struct lprocfs_vars lprocfs_cmm_obd_vars[] = { + { 0 } +}; + +struct lprocfs_vars lprocfs_cmm_module_vars[] = { + { 0 } +}; + +LPROCFS_INIT_VARS(cmm, lprocfs_cmm_module_vars, lprocfs_cmm_obd_vars); + +static int __init cmm_mod_init(void) +{ + struct lprocfs_static_vars lvars; + struct obd_type *type; + int result; + + lprocfs_init_vars(cmm, &lvars); + result = class_register_type(&cmm_obd_device_ops, + lvars.module_vars, LUSTRE_CMM0_NAME); + if (result == 0) { + type = class_get_type(LUSTRE_CMM0_NAME); + LASSERT(type != NULL); + type->typ_lu = &cmm_device_type; + result = type->typ_lu->ldt_ops->ldto_init(type->typ_lu); + if (result != 0) + class_unregister_type(LUSTRE_CMM0_NAME); + } + return result; +} + +static void __exit cmm_mod_exit(void) +{ + class_unregister_type(LUSTRE_CMM0_NAME); +} + +MODULE_AUTHOR("Cluster File Systems, Inc. "); +MODULE_DESCRIPTION("Lustre Clustered Meta-data Manager Prototype ("LUSTRE_CMM0_NAME")"); +MODULE_LICENSE("GPL"); + +cfs_module(cmm, "0.0.1", cmm_mod_init, cmm_mod_exit); diff --git a/lustre/cmm/cmm_internal.h b/lustre/cmm/cmm_internal.h new file mode 100644 index 0000000..03e452c --- /dev/null +++ b/lustre/cmm/cmm_internal.h @@ -0,0 +1,101 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2006 Cluster File Systems, Inc. + * + * This file is part of Lustre, http://www.lustre.org. + * + * Lustre is free software; you can redistribute it and/or + * modify it under the terms of version 2 of the GNU General Public + * License as published by the Free Software Foundation. + * + * Lustre is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Lustre; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#ifndef _CMM_INTERNAL_H +#define _CMM_INTERNAL_H + +#if defined(__KERNEL__) + +#include +#include + +#define LUSTRE_CMM0_NAME "cmm0" + +struct cmm_device { + struct md_device cmm_md_dev; + /* underlaying device in MDS stack, usually MDD */ + struct md_device *cmm_child; + /* other MD servers in cluster */ + struct md_device *cmm_cluster; +}; + + +#define CMM_DO_CHILD(d) (d->cmm_child->md_ops) + +static inline struct cmm_device *md2cmm_dev(struct md_device *m) +{ + return container_of(m, struct cmm_device, cmm_md_dev); +} + +static struct cmm_device *lu2cmm_dev(struct lu_device *d) +{ + //LASSERT(lu_device_is_cmm(d)); + return container_of(d, struct cmm_device, cmm_md_dev.md_lu_dev); +} + +static inline struct lu_device *cmm2lu_dev(struct cmm_device *d) +{ + return (&d->cmm_md_dev.md_lu_dev); +} + +struct cmm_object { + struct md_object cmo_obj; +}; + +static inline struct cmm_device *cmm_obj2dev(struct cmm_object *c) +{ + return (md2cmm_dev(md_device_get(&c->cmo_obj))); +} + +static struct cmm_object *lu2cmm_obj(struct lu_object *o) +{ + //LASSERT(lu_device_is_cmm(o->lo_dev)); + return container_of(o, struct cmm_object, cmo_obj.mo_lu); +} + +/* get cmm object from md_object */ +static inline struct cmm_object *md2cmm_obj(struct md_object *o) +{ + return container_of(o, struct cmm_object, cmo_obj); +} +/* get lower-layer object */ +static inline struct md_object *cmm2child_obj(struct cmm_object *o) +{ + return lu2md(lu_object_next(&o->cmo_obj.mo_lu)); +} + +/* cmm_object.c */ +int cmm_object_init(struct lu_object*); +struct lu_object *cmm_object_alloc(struct lu_device *); +void cmm_object_free(struct lu_object *o); +void cmm_object_release(struct lu_object *o); +//int cmm_getattr(struct lu_object *o, struct lu_attr *a); +//int cmm_setattr(struct lu_object *o, struct lu_attr *a); +int cmm_object_print(struct seq_file *f, const struct lu_object *o); + +/* cmm md operations */ +int cmm_root_get(struct md_device *m, struct ll_fid *f); +int cmm_mkdir(struct md_object *o, const char *name, + struct md_object *child); + +#endif /* __KERNEL__ */ +#endif /* _CMM_INTERNAL_H */ diff --git a/lustre/cmm/cmm_object.c b/lustre/cmm/cmm_object.c new file mode 100644 index 0000000..9cb3c42 --- /dev/null +++ b/lustre/cmm/cmm_object.c @@ -0,0 +1,134 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * lustre/cmm/cmm_object.c + * Lustre Cluster Metadata Manager (cmm) + * + * Copyright (c) 2006 Cluster File Systems, Inc. + * Author: Mike Pershin + * + * This file is part of the Lustre file system, http://www.lustre.org + * Lustre is a trademark of Cluster File Systems, Inc. + * + * You may have signed or agreed to another license before downloading + * this software. If so, you are bound by the terms and conditions + * of that agreement, and the following does not apply to you. See the + * LICENSE file included with this distribution for more information. + * + * If you did not agree to a different license, then this copy of Lustre + * is open source software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In either case, Lustre is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * license text for more details. + */ + +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif + +#define DEBUG_SUBSYSTEM S_MDS + +#include "cmm_internal.h" + +struct cmm_object *cmm_object_find(struct cmm_device *d, struct ll_fid *f) +{ + struct lu_object *o; + + o = lu_object_find(d->cmm_md_dev.md_lu_dev.ld_site, f); + if (IS_ERR(o)) + return (struct cmm_object *)o; + else + return container_of(o, struct cmm_object, cmo_obj.mo_lu); +} + +void cmm_object_put(struct cmm_object *o) +{ + lu_object_put(&o->cmo_obj.mo_lu); +} + +struct lu_object *cmm_object_alloc(struct lu_device *d) +{ + struct cmm_object *mo; + + OBD_ALLOC_PTR(mo); + if (mo != NULL) { + struct lu_object *o; + o = &mo->cmo_obj.mo_lu; + return o; + } else + return NULL; +} + +int cmm_object_init(struct lu_object *o) +{ + struct cmm_device *d = lu2cmm_dev(o->lo_dev); + struct lu_device *under; + struct lu_object *below; + + under = &d->cmm_child->md_lu_dev; + below = under->ld_ops->ldo_object_alloc(under); + if (below != NULL) { + lu_object_add(o, below); + return 0; + } else + return -ENOMEM; +} + +void cmm_object_free(struct lu_object *o) +{ + lu_object_fini(o); +} + +void cmm_object_release(struct lu_object *o) +{ + return; +} + +int cmm_object_print(struct seq_file *f, const struct lu_object *o) +{ + return seq_printf(f, LUSTRE_CMM0_NAME"-object@%p", o); +} + +/* Locking API */ +#if 0 +static void cmm_lock(struct md_object *obj, __u32 mode) +{ + struct cmm_object *cmm_obj = md2cmm_obj(obj); + struct cmm_device *cmm_dev = cmm_obj2dev(cmm_obj); + + CMM_DO_CHILD(cmm_dev)->ldo_lock_obj(cmm2child_obj(cmm_obj), mode); + return; +} + +static void cmm_unlock(struct md_object *obj, __u32 mode) +{ + struct cmm_object *cmm_obj = md2cmm_obj(obj); + struct cmm_device *cmm_dev = cmm_obj2dev(cmm_obj); + + CMM_DO_CHILD(cmm_dev)->ldo_unlock_obj(cmm2child_obj(cmm_obj), mode); + return; +} +#endif +/* Llog API */ +/* Object API */ +/* Metadata API */ +int cmm_mkdir(struct md_object *md_parent, const char *name, + struct md_object *md_child) +{ + struct cmm_object *cmm_parent = md2cmm_obj(md_parent); + struct cmm_device *cmm_dev = cmm_obj2dev(cmm_parent); + int result; + + result = CMM_DO_CHILD(cmm_dev)->mdo_mkdir(cmm2child_obj(cmm_parent), + name, md_child); + + + + return result; +} + + -- 1.8.3.1