From 044546f88a638b98c5d2016da6e85ea72dd1a391 Mon Sep 17 00:00:00 2001 From: Mikhail Pershin Date: Wed, 12 Sep 2012 18:50:53 +0400 Subject: [PATCH] LU-1303 osp: Basic OSP device implementation Initial OSP code: osp_device, osp_object and structures Signed-off-by: Mikhail Pershin Change-Id: I21ea3d0cdfe07634278d8e7fee9a7c3fef79f81e Reviewed-on: http://review.whamcloud.com/3958 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Johann Lombardi --- lustre/Makefile.in | 3 +- lustre/autoMakefile.am | 3 +- lustre/autoconf/lustre-core.m4 | 2 + lustre/include/lprocfs_status.h | 7 +- lustre/include/lustre_param.h | 2 + lustre/include/obd.h | 2 + lustre/obdclass/lprocfs_status.c | 57 +++- lustre/osp/Makefile.in | 6 + lustre/osp/autoMakefile.am | 44 +++ lustre/osp/lproc_osp.c | 77 +++++ lustre/osp/osp_dev.c | 649 +++++++++++++++++++++++++++++++++++++++ lustre/osp/osp_internal.h | 176 +++++++++++ lustre/osp/osp_object.c | 98 ++++++ 13 files changed, 1120 insertions(+), 6 deletions(-) create mode 100644 lustre/osp/Makefile.in create mode 100644 lustre/osp/autoMakefile.am create mode 100644 lustre/osp/lproc_osp.c create mode 100644 lustre/osp/osp_dev.c create mode 100644 lustre/osp/osp_internal.h create mode 100644 lustre/osp/osp_object.c diff --git a/lustre/Makefile.in b/lustre/Makefile.in index 1eb7682..94714bb 100644 --- a/lustre/Makefile.in +++ b/lustre/Makefile.in @@ -7,7 +7,8 @@ subdir-m += osc subdir-m += obdecho subdir-m += mgc -@SERVER_TRUE@subdir-m += mds obdfilter ost mgs mdt cmm mdd ofd osd-ldiskfs quota +@SERVER_TRUE@subdir-m += mds obdfilter ost mgs mdt cmm mdd ofd osd-ldiskfs +@SERVER_TRUE@subdir-m += quota osp @CLIENT_TRUE@subdir-m += mdc lmv llite fld @ZFS_ENABLED_TRUE@subdir-m += osd-zfs diff --git a/lustre/autoMakefile.am b/lustre/autoMakefile.am index b0a028c..6ed9139 100644 --- a/lustre/autoMakefile.am +++ b/lustre/autoMakefile.am @@ -42,7 +42,8 @@ AUTOMAKE_OPTIONS = foreign ALWAYS_SUBDIRS = include lvfs obdclass ldlm ptlrpc osc lov obdecho \ mgc fid fld doc utils tests scripts autoconf contrib conf -SERVER_SUBDIRS = obdfilter ost mds mgs mdt cmm mdd ofd osd-zfs osd-ldiskfs quota +SERVER_SUBDIRS = obdfilter ost mds mgs mdt cmm mdd ofd osd-zfs osd-ldiskfs \ + quota osp CLIENT_SUBDIRS = mdc lmv llite lclient diff --git a/lustre/autoconf/lustre-core.m4 b/lustre/autoconf/lustre-core.m4 index 7797fd4..e359744 100644 --- a/lustre/autoconf/lustre-core.m4 +++ b/lustre/autoconf/lustre-core.m4 @@ -2512,6 +2512,8 @@ lustre/tests/Makefile lustre/tests/mpi/Makefile lustre/utils/Makefile lustre/utils/gss/Makefile +lustre/osp/Makefile +lustre/osp/autoMakefile lustre/obdclass/darwin/Makefile ]) ]) diff --git a/lustre/include/lprocfs_status.h b/lustre/include/lprocfs_status.h index 8f83201..f5a9181 100644 --- a/lustre/include/lprocfs_status.h +++ b/lustre/include/lprocfs_status.h @@ -552,6 +552,8 @@ extern cfs_proc_dir_entry_t *lprocfs_register(const char *name, extern void lprocfs_remove(cfs_proc_dir_entry_t **root); extern void lprocfs_remove_proc_entry(const char *name, struct proc_dir_entry *parent); +extern void lprocfs_try_remove_proc_entry(const char *name, + struct proc_dir_entry *parent); extern cfs_proc_dir_entry_t *lprocfs_srch(cfs_proc_dir_entry_t *root, const char *name); @@ -823,7 +825,7 @@ extern int lprocfs_quota_wr_qs_factor(struct file *file, #else /* LPROCFS is not defined */ - +#define proc_lustre_root NULL static inline void lprocfs_counter_add(struct lprocfs_stats *stats, int index, long amount) @@ -918,6 +920,9 @@ static inline void lprocfs_remove(cfs_proc_dir_entry_t **root) static inline void lprocfs_remove_proc_entry(const char *name, struct proc_dir_entry *parent) { return; } +static inline void lprocfs_try_remove_proc_entry(const char *name, + struct proc_dir_entry *parent) +{ return; } static inline cfs_proc_dir_entry_t *lprocfs_srch(cfs_proc_dir_entry_t *head, const char *name) { return 0; } diff --git a/lustre/include/lustre_param.h b/lustre/include/lustre_param.h index 26d2935..9c53292 100644 --- a/lustre/include/lustre_param.h +++ b/lustre/include/lustre_param.h @@ -106,6 +106,8 @@ int do_lcfg(char *cfgname, lnet_nid_t nid, int cmd, #define PARAM_MDC "mdc." #define PARAM_LLITE "llite." #define PARAM_LOV "lov." +#define PARAM_LOD "lod." +#define PARAM_OSP "osp." #define PARAM_SYS "sys." /* global */ #define PARAM_SRPC "srpc." #define PARAM_SRPC_FLVR "srpc.flavor." diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 6490a04..afc7a8e 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -842,6 +842,8 @@ struct niobuf_local { #define LUSTRE_LMV_NAME "lmv" #define LUSTRE_CMM_MDC_NAME "cmm-mdc" #define LUSTRE_SLP_NAME "slp" +#define LUSTRE_LOD_NAME "lod" +#define LUSTRE_OSP_NAME "osp" /* obd device type names */ /* FIXME all the references to LUSTRE_MDS_NAME should be swapped with LUSTRE_MDT_NAME */ diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c index e1dc6de..0d072c4 100644 --- a/lustre/obdclass/lprocfs_status.c +++ b/lustre/obdclass/lprocfs_status.c @@ -379,7 +379,7 @@ out: } EXPORT_SYMBOL(lprocfs_add_vars); -void lprocfs_remove(struct proc_dir_entry **rooth) +void lprocfs_remove_nolock(struct proc_dir_entry **rooth) { struct proc_dir_entry *root = *rooth; struct proc_dir_entry *temp = root; @@ -392,7 +392,6 @@ void lprocfs_remove(struct proc_dir_entry **rooth) parent = root->parent; LASSERT(parent != NULL); - LPROCFS_WRITE_ENTRY(); /* search vs remove race */ while (1) { while (temp->subdir != NULL) @@ -428,7 +427,13 @@ void lprocfs_remove(struct proc_dir_entry **rooth) if (temp == parent) break; } - LPROCFS_WRITE_EXIT(); +} + +void lprocfs_remove(struct proc_dir_entry **rooth) +{ + LPROCFS_WRITE_ENTRY(); /* search vs remove race */ + lprocfs_remove_nolock(rooth); + LPROCFS_WRITE_EXIT(); } EXPORT_SYMBOL(lprocfs_remove); @@ -439,6 +444,52 @@ void lprocfs_remove_proc_entry(const char *name, struct proc_dir_entry *parent) } EXPORT_SYMBOL(lprocfs_remove_proc_entry); +void lprocfs_try_remove_proc_entry(const char *name, + struct proc_dir_entry *parent) +{ + struct proc_dir_entry *t = NULL; + struct proc_dir_entry **p; + int len, busy = 0; + + LASSERT(parent != NULL); + len = strlen(name); + + LPROCFS_WRITE_ENTRY(); + + /* lookup target name */ + for (p = &parent->subdir; *p; p = &(*p)->next) { + if ((*p)->namelen != len) + continue; + if (memcmp(name, (*p)->name, len)) + continue; + t = *p; + break; + } + + if (t) { + /* verify it's empty: do not count "num_refs" */ + for (p = &t->subdir; *p; p = &(*p)->next) { + if ((*p)->namelen != strlen("num_refs")) { + busy = 1; + break; + } + if (memcmp("num_refs", (*p)->name, + strlen("num_refs"))) { + busy = 1; + break; + } + } + } + + if (busy == 0) + lprocfs_remove_nolock(&t); + + LPROCFS_WRITE_EXIT(); + + return; +} +EXPORT_SYMBOL(lprocfs_try_remove_proc_entry); + struct proc_dir_entry *lprocfs_register(const char *name, struct proc_dir_entry *parent, struct lprocfs_vars *list, void *data) diff --git a/lustre/osp/Makefile.in b/lustre/osp/Makefile.in new file mode 100644 index 0000000..ff5e554 --- /dev/null +++ b/lustre/osp/Makefile.in @@ -0,0 +1,6 @@ +MODULES := osp +osp-objs := osp_dev.o osp_object.o lproc_osp.o + +EXTRA_DIST = $(osp-objs:.o=.c) osp_internal.h + +@INCLUDE_RULES@ diff --git a/lustre/osp/autoMakefile.am b/lustre/osp/autoMakefile.am new file mode 100644 index 0000000..1e77943 --- /dev/null +++ b/lustre/osp/autoMakefile.am @@ -0,0 +1,44 @@ +# +# GPL HEADER START +# +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 only, +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License version 2 for more details (a copy is included +# in the LICENSE file that accompanied this code). +# +# You should have received a copy of the GNU General Public License +# version 2 along with this program; If not, see +# http://www.sun.com/software/products/lustre/docs/GPLv2.pdf +# +# Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, +# CA 95054 USA or visit www.sun.com if you need additional information or +# have any questions. +# +# GPL HEADER END +# + +# +# Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. +# Use is subject to license terms. +# +# Copyright (c) 2011, 2012, Intel, Inc. +# + +# +# This file is part of Lustre, http://www.lustre.org/ +# Lustre is a trademark of Sun Microsystems, Inc. +# + +if MODULES +modulefs_DATA = osp$(KMODEXT) +endif + +MOSTLYCLEANFILES := @MOSTLYCLEANFILES@ +EXTRA_DIST := $(osp-objs:%.o=%.c) osp_internal.h diff --git a/lustre/osp/lproc_osp.c b/lustre/osp/lproc_osp.c new file mode 100644 index 0000000..e0171d5 --- /dev/null +++ b/lustre/osp/lproc_osp.c @@ -0,0 +1,77 @@ +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + * GPL HEADER END + */ +/* + * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. + * Use is subject to license terms. + * + * Copyright (c) 2011, 2012, Intel, Inc. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + * Lustre is a trademark of Sun Microsystems, Inc. + * + * lustre/osp/lproc_osp.c + * + * Lustre OST Proxy Device, procfs functions + * + * Author: Alex Zhuravlev + */ + +#define DEBUG_SUBSYSTEM S_CLASS + +#include "osp_internal.h" + +#ifdef LPROCFS +static struct lprocfs_vars lprocfs_osp_obd_vars[] = { + { "uuid", lprocfs_rd_uuid, 0, 0 }, + { "ping", 0, lprocfs_wr_ping, 0, 0, 0222 }, + { "connect_flags", lprocfs_rd_connect_flags, 0, 0 }, + { "blocksize", lprocfs_rd_blksize, 0, 0 }, + { "kbytestotal", lprocfs_rd_kbytestotal, 0, 0 }, + { "kbytesfree", lprocfs_rd_kbytesfree, 0, 0 }, + { "kbytesavail", lprocfs_rd_kbytesavail, 0, 0 }, + { "filestotal", lprocfs_rd_filestotal, 0, 0 }, + { "filesfree", lprocfs_rd_filesfree, 0, 0 }, + { "ost_server_uuid", lprocfs_rd_server_uuid, 0, 0 }, + { "ost_conn_uuid", lprocfs_rd_conn_uuid, 0, 0 }, + { "timeouts", lprocfs_rd_timeouts, 0, 0 }, + { "import", lprocfs_rd_import, lprocfs_wr_import, 0 }, + { "state", lprocfs_rd_state, 0, 0 }, + { 0 } +}; + +static struct lprocfs_vars lprocfs_osp_module_vars[] = { + { "num_refs", lprocfs_rd_numrefs, 0, 0 }, + { 0 } +}; + +void lprocfs_osp_init_vars(struct lprocfs_static_vars *lvars) +{ + lvars->module_vars = lprocfs_osp_module_vars; + lvars->obd_vars = lprocfs_osp_obd_vars; +} +#endif /* LPROCFS */ + diff --git a/lustre/osp/osp_dev.c b/lustre/osp/osp_dev.c new file mode 100644 index 0000000..fe8c1e4 --- /dev/null +++ b/lustre/osp/osp_dev.c @@ -0,0 +1,649 @@ +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + * GPL HEADER END + */ +/* + * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. + * Use is subject to license terms. + * + * Copyright (c) 2011, 2012, Intel, Inc. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + * Lustre is a trademark of Sun Microsystems, Inc. + * + * lustre/osp/osp_dev.c + * + * Lustre OST Proxy Device + * + * Author: Alex Zhuravlev + * Author: Mikhail Pershin + */ + +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif +#define DEBUG_SUBSYSTEM S_MDS + +#include +#include + +#include "osp_internal.h" + +/* Slab for OSP object allocation */ +cfs_mem_cache_t *osp_object_kmem; + +static struct lu_kmem_descr osp_caches[] = { + { + .ckd_cache = &osp_object_kmem, + .ckd_name = "osp_obj", + .ckd_size = sizeof(struct osp_object) + }, + { + .ckd_cache = NULL + } +}; + +struct lu_object *osp_object_alloc(const struct lu_env *env, + const struct lu_object_header *hdr, + struct lu_device *d) +{ + struct lu_object_header *h; + struct osp_object *o; + struct lu_object *l; + + LASSERT(hdr == NULL); + + OBD_SLAB_ALLOC_PTR_GFP(o, osp_object_kmem, CFS_ALLOC_IO); + if (o != NULL) { + l = &o->opo_obj.do_lu; + h = &o->opo_header; + + lu_object_header_init(h); + dt_object_init(&o->opo_obj, h, d); + lu_object_add_top(h, l); + + l->lo_ops = &osp_lu_obj_ops; + + return l; + } else { + return NULL; + } +} + +static int osp_shutdown(const struct lu_env *env, struct osp_device *d) +{ + struct obd_import *imp; + int rc = 0; + + ENTRY; + + imp = d->opd_obd->u.cli.cl_import; + + /* Mark import deactivated now, so we don't try to reconnect if any + * of the cleanup RPCs fails (e.g. ldlm cancel, etc). We don't + * fully deactivate the import, or that would drop all requests. */ + cfs_spin_lock(&imp->imp_lock); + imp->imp_deactive = 1; + cfs_spin_unlock(&imp->imp_lock); + + ptlrpc_deactivate_import(imp); + + /* Some non-replayable imports (MDS's OSCs) are pinged, so just + * delete it regardless. (It's safe to delete an import that was + * never added.) */ + (void)ptlrpc_pinger_del_import(imp); + + rc = ptlrpc_disconnect_import(imp, 0); + if (rc) + CERROR("%s: can't disconnect: rc = %d\n", + d->opd_obd->obd_name, rc); + + ptlrpc_invalidate_import(imp); + + RETURN(rc); +} + +static int osp_process_config(const struct lu_env *env, + struct lu_device *dev, struct lustre_cfg *lcfg) +{ + struct osp_device *d = lu2osp_dev(dev); + struct lprocfs_static_vars lvars = { 0 }; + int rc; + + ENTRY; + + switch (lcfg->lcfg_command) { + case LCFG_CLEANUP: + lu_dev_del_linkage(dev->ld_site, dev); + rc = osp_shutdown(env, d); + break; + case LCFG_PARAM: + lprocfs_osp_init_vars(&lvars); + + LASSERT(d->opd_obd); + rc = class_process_proc_param(PARAM_OSP, lvars.obd_vars, + lcfg, d->opd_obd); + if (rc > 0) + rc = 0; + if (rc == -ENOSYS) { + /* class_process_proc_param() haven't found matching + * parameter and returned ENOSYS so that layer(s) + * below could use that. But OSP is the bottom, so + * just ignore it */ + CERROR("%s: unknown param %s\n", + (char *)lustre_cfg_string(lcfg, 0), + (char *)lustre_cfg_string(lcfg, 1)); + rc = 0; + } + break; + default: + CERROR("%s: unknown command %u\n", + (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command); + rc = 0; + break; + } + + RETURN(rc); +} + +static int osp_recovery_complete(const struct lu_env *env, + struct lu_device *dev) +{ + struct osp_device *osp = lu2osp_dev(dev); + int rc = 0; + + ENTRY; + osp->opd_recovery_completed = 1; + RETURN(rc); +} + +const struct lu_device_operations osp_lu_ops = { + .ldo_object_alloc = osp_object_alloc, + .ldo_process_config = osp_process_config, + .ldo_recovery_complete = osp_recovery_complete, +}; + +static int osp_sync(const struct lu_env *env, struct dt_device *dev) +{ + ENTRY; + + /* + * XXX: wake up sync thread, command it to start flushing asap? + */ + + RETURN(0); +} + +static const struct dt_device_operations osp_dt_ops = { + .dt_sync = osp_sync, +}; + +static int osp_connect_to_osd(const struct lu_env *env, struct osp_device *m, + const char *nextdev) +{ + struct obd_connect_data *data = NULL; + struct obd_device *obd; + int rc; + + ENTRY; + + LASSERT(m->opd_storage_exp == NULL); + + OBD_ALLOC_PTR(data); + if (data == NULL) + RETURN(-ENOMEM); + + obd = class_name2obd(nextdev); + if (obd == NULL) { + CERROR("%s: can't locate next device: %s\n", + m->opd_obd->obd_name, nextdev); + GOTO(out, rc = -ENOTCONN); + } + + rc = obd_connect(env, &m->opd_storage_exp, obd, &obd->obd_uuid, data, + NULL); + if (rc) { + CERROR("%s: cannot connect to next dev %s: rc = %d\n", + m->opd_obd->obd_name, nextdev, rc); + GOTO(out, rc); + } + + m->opd_dt_dev.dd_lu_dev.ld_site = + m->opd_storage_exp->exp_obd->obd_lu_dev->ld_site; + LASSERT(m->opd_dt_dev.dd_lu_dev.ld_site); + m->opd_storage = lu2dt_dev(m->opd_storage_exp->exp_obd->obd_lu_dev); + +out: + OBD_FREE_PTR(data); + RETURN(rc); +} + +static int osp_init0(const struct lu_env *env, struct osp_device *m, + struct lu_device_type *ldt, struct lustre_cfg *cfg) +{ + struct lprocfs_static_vars lvars = { 0 }; + struct proc_dir_entry *osc_proc_dir; + struct obd_import *imp; + class_uuid_t uuid; + char *src, *ost, *mdt, *osdname = NULL; + int rc, idx; + + ENTRY; + + m->opd_obd = class_name2obd(lustre_cfg_string(cfg, 0)); + if (m->opd_obd == NULL) { + CERROR("Cannot find obd with name %s\n", + lustre_cfg_string(cfg, 0)); + RETURN(-ENODEV); + } + + /* There is no record in the MDT configuration for the local disk + * device, so we have to extract this from elsewhere in the profile. + * The only information we get at setup is from the OSC records: + * setup 0:{fsname}-OSTxxxx-osc[-MDTxxxx] 1:lustre-OST0000_UUID 2:NID + * Note that 1.8 generated configs are missing the -MDTxxxx part. + * We need to reconstruct the name of the underlying OSD from this: + * {fsname}-{svname}-osd, for example "lustre-MDT0000-osd". We + * also need to determine the OST index from this - will be used + * to calculate the offset in shared lov_objids file later */ + + src = lustre_cfg_string(cfg, 0); + if (src == NULL) + RETURN(-EINVAL); + + ost = strstr(src, "-OST"); + if (ost == NULL) + RETURN(-EINVAL); + + idx = simple_strtol(ost + 4, &mdt, 16); + if (mdt[0] != '-' || idx > INT_MAX || idx < 0) { + CERROR("%s: invalid OST index in '%s'\n", + m->opd_obd->obd_name, src); + GOTO(out_fini, rc = -EINVAL); + } + m->opd_index = idx; + + idx = ost - src; + /* check the fsname length, and after this everything else will fit */ + if (idx > MTI_NAME_MAXLEN) { + CERROR("%s: fsname too long in '%s'\n", + m->opd_obd->obd_name, src); + GOTO(out_fini, rc = -EINVAL); + } + + OBD_ALLOC(osdname, MAX_OBD_NAME); + if (osdname == NULL) + GOTO(out_fini, rc = -ENOMEM); + + memcpy(osdname, src, idx); /* copy just the fsname part */ + osdname[idx] = '\0'; + + mdt = strstr(mdt, "-MDT"); + if (mdt == NULL) /* 1.8 configs don't have "-MDT0000" at the end */ + strcat(osdname, "-MDT0000"); + else + strcat(osdname, mdt); + strcat(osdname, "-osd"); + CDEBUG(D_HA, "%s: connect to %s (%s)\n", + m->opd_obd->obd_name, osdname, src); + + m->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops; + m->opd_dt_dev.dd_ops = &osp_dt_ops; + m->opd_obd->obd_lu_dev = &m->opd_dt_dev.dd_lu_dev; + + rc = osp_connect_to_osd(env, m, osdname); + if (rc) + GOTO(out_fini, rc); + + rc = ptlrpcd_addref(); + if (rc) + GOTO(out_disconnect, rc); + + rc = client_obd_setup(m->opd_obd, cfg); + if (rc) { + CERROR("%s: can't setup obd: %d\n", m->opd_obd->obd_name, rc); + GOTO(out_ref, rc); + } + + lprocfs_osp_init_vars(&lvars); + if (lprocfs_obd_setup(m->opd_obd, lvars.obd_vars) == 0) + ptlrpc_lprocfs_register_obd(m->opd_obd); + + /* for compatibility we link old procfs's OSC entries to osp ones */ + osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc"); + if (osc_proc_dir) { + cfs_proc_dir_entry_t *symlink = NULL; + char *name; + + OBD_ALLOC(name, strlen(m->opd_obd->obd_name) + 1); + if (name == NULL) + GOTO(out, rc = -ENOMEM); + + strcpy(name, m->opd_obd->obd_name); + if (strstr(name, "osc")) + symlink = lprocfs_add_symlink(name, osc_proc_dir, + "../osp/%s", + m->opd_obd->obd_name); + OBD_FREE(name, strlen(m->opd_obd->obd_name) + 1); + m->opd_symlink = symlink; + } + + /* + * Initiate connect to OST + */ + ll_generate_random_uuid(uuid); + class_uuid_unparse(uuid, &m->opd_cluuid); + + imp = m->opd_obd->u.cli.cl_import; + + rc = ptlrpc_init_import(imp); + if (rc) + GOTO(out, rc); + if (osdname) + OBD_FREE(osdname, MAX_OBD_NAME); + RETURN(0); + +out: + ptlrpc_lprocfs_unregister_obd(m->opd_obd); + lprocfs_obd_cleanup(m->opd_obd); + class_destroy_import(m->opd_obd->u.cli.cl_import); + client_obd_cleanup(m->opd_obd); +out_ref: + ptlrpcd_decref(); +out_disconnect: + obd_disconnect(m->opd_storage_exp); +out_fini: + if (osdname) + OBD_FREE(osdname, MAX_OBD_NAME); + RETURN(rc); +} + +static struct lu_device *osp_device_free(const struct lu_env *env, + struct lu_device *lu) +{ + struct osp_device *m = lu2osp_dev(lu); + + ENTRY; + + dt_device_fini(&m->opd_dt_dev); + OBD_FREE_PTR(m); + RETURN(NULL); +} + +static struct lu_device *osp_device_alloc(const struct lu_env *env, + struct lu_device_type *t, + struct lustre_cfg *lcfg) +{ + struct osp_device *m; + struct lu_device *l; + + OBD_ALLOC_PTR(m); + if (m == NULL) { + l = ERR_PTR(-ENOMEM); + } else { + int rc; + + l = osp2lu_dev(m); + dt_device_init(&m->opd_dt_dev, t); + rc = osp_init0(env, m, t, lcfg); + if (rc != 0) { + osp_device_free(env, l); + l = ERR_PTR(rc); + } + } + return l; +} + +static struct lu_device *osp_device_fini(const struct lu_env *env, + struct lu_device *d) +{ + struct osp_device *m = lu2osp_dev(d); + struct obd_import *imp; + int rc; + + ENTRY; + + LASSERT(m->opd_storage_exp); + obd_disconnect(m->opd_storage_exp); + + imp = m->opd_obd->u.cli.cl_import; + + if (imp->imp_rq_pool) { + ptlrpc_free_rq_pool(imp->imp_rq_pool); + imp->imp_rq_pool = NULL; + } + + obd_cleanup_client_import(m->opd_obd); + + if (m->opd_symlink) + lprocfs_remove(&m->opd_symlink); + + LASSERT(m->opd_obd); + ptlrpc_lprocfs_unregister_obd(m->opd_obd); + lprocfs_obd_cleanup(m->opd_obd); + + rc = client_obd_cleanup(m->opd_obd); + LASSERTF(rc == 0, "error %d\n", rc); + + ptlrpcd_decref(); + + RETURN(NULL); +} + +static int osp_reconnect(const struct lu_env *env, + struct obd_export *exp, struct obd_device *obd, + struct obd_uuid *cluuid, + struct obd_connect_data *data, + void *localdata) +{ + return 0; +} + +/* + * we use exports to track all LOD users + */ +static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp, + struct obd_device *obd, struct obd_uuid *cluuid, + struct obd_connect_data *data, void *localdata) +{ + struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev); + struct obd_connect_data *ocd; + struct obd_import *imp; + struct lustre_handle conn; + int rc; + + ENTRY; + + CDEBUG(D_CONFIG, "connect #%d\n", osp->opd_connects); + + rc = class_connect(&conn, obd, cluuid); + if (rc) + RETURN(rc); + + *exp = class_conn2export(&conn); + + /* Why should there ever be more than 1 connect? */ + osp->opd_connects++; + LASSERT(osp->opd_connects == 1); + + imp = osp->opd_obd->u.cli.cl_import; + imp->imp_dlm_handle = conn; + + ocd = &imp->imp_connect_data; + ocd->ocd_connect_flags = OBD_CONNECT_AT | + OBD_CONNECT_FULL20 | + OBD_CONNECT_INDEX | +#ifdef HAVE_LRU_RESIZE_SUPPORT + OBD_CONNECT_LRU_RESIZE | +#endif + OBD_CONNECT_MDS | + OBD_CONNECT_OSS_CAPA | + OBD_CONNECT_REQPORTAL | + OBD_CONNECT_SKIP_ORPHAN | + OBD_CONNECT_VERSION; + ocd->ocd_version = LUSTRE_VERSION_CODE; + LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX); + ocd->ocd_index = data->ocd_index; + imp->imp_connect_flags_orig = ocd->ocd_connect_flags; + + rc = ptlrpc_connect_import(imp); + if (rc) { + CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc); + GOTO(out, rc); + } + + ptlrpc_pinger_add_import(imp); + +out: + RETURN(rc); +} + +/* + * once last export (we don't count self-export) disappeared + * osp can be released + */ +static int osp_obd_disconnect(struct obd_export *exp) +{ + struct obd_device *obd = exp->exp_obd; + struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev); + int rc; + + ENTRY; + + /* Only disconnect the underlying layers on the final disconnect. */ + LASSERT(osp->opd_connects == 1); + osp->opd_connects--; + + rc = class_disconnect(exp); + + /* destroy the device */ + if (rc == 0) + class_manual_cleanup(obd); + + RETURN(rc); +} + +/* context key constructor/destructor: mdt_key_init, mdt_key_fini */ +LU_KEY_INIT_FINI(osp, struct osp_thread_info); +static void osp_key_exit(const struct lu_context *ctx, + struct lu_context_key *key, void *data) +{ + struct osp_thread_info *info = data; + + info->osi_attr.la_valid = 0; +} + +struct lu_context_key osp_thread_key = { + .lct_tags = LCT_MD_THREAD, + .lct_init = osp_key_init, + .lct_fini = osp_key_fini, + .lct_exit = osp_key_exit +}; + +/* context key constructor/destructor: mdt_txn_key_init, mdt_txn_key_fini */ +LU_KEY_INIT_FINI(osp_txn, struct osp_txn_info); + +struct lu_context_key osp_txn_key = { + .lct_tags = LCT_OSP_THREAD, + .lct_init = osp_txn_key_init, + .lct_fini = osp_txn_key_fini +}; +LU_TYPE_INIT_FINI(osp, &osp_thread_key, &osp_txn_key); + +static struct lu_device_type_operations osp_device_type_ops = { + .ldto_init = osp_type_init, + .ldto_fini = osp_type_fini, + + .ldto_start = osp_type_start, + .ldto_stop = osp_type_stop, + + .ldto_device_alloc = osp_device_alloc, + .ldto_device_free = osp_device_free, + + .ldto_device_fini = osp_device_fini +}; + +static struct lu_device_type osp_device_type = { + .ldt_tags = LU_DEVICE_DT, + .ldt_name = LUSTRE_OSP_NAME, + .ldt_ops = &osp_device_type_ops, + .ldt_ctx_tags = LCT_MD_THREAD +}; + +static struct obd_ops osp_obd_device_ops = { + .o_owner = THIS_MODULE, + .o_add_conn = client_import_add_conn, + .o_del_conn = client_import_del_conn, + .o_reconnect = osp_reconnect, + .o_connect = osp_obd_connect, + .o_disconnect = osp_obd_disconnect, +}; + +static int __init osp_mod_init(void) +{ + struct lprocfs_static_vars lvars; + cfs_proc_dir_entry_t *osc_proc_dir; + int rc; + + rc = lu_kmem_init(osp_caches); + if (rc) + return rc; + + lprocfs_osp_init_vars(&lvars); + + rc = class_register_type(&osp_obd_device_ops, NULL, lvars.module_vars, + LUSTRE_OSP_NAME, &osp_device_type); + + /* create "osc" entry in procfs for compatibility purposes */ + if (rc != 0) { + lu_kmem_fini(osp_caches); + return rc; + } + + osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc"); + if (osc_proc_dir == NULL) { + osc_proc_dir = lprocfs_register("osc", proc_lustre_root, NULL, + NULL); + if (IS_ERR(osc_proc_dir)) + CERROR("osp: can't create compat entry \"osc\": %d\n", + (int) PTR_ERR(osc_proc_dir)); + } + return rc; +} + +static void __exit osp_mod_exit(void) +{ + lprocfs_try_remove_proc_entry("osc", proc_lustre_root); + + class_unregister_type(LUSTRE_OSP_NAME); + lu_kmem_fini(osp_caches); +} + +MODULE_AUTHOR("Intel, Inc. "); +MODULE_DESCRIPTION("Lustre OST Proxy Device ("LUSTRE_OSP_NAME")"); +MODULE_LICENSE("GPL"); + +cfs_module(osp, LUSTRE_VERSION_STRING, osp_mod_init, osp_mod_exit); + diff --git a/lustre/osp/osp_internal.h b/lustre/osp/osp_internal.h new file mode 100644 index 0000000..f22e1b2 --- /dev/null +++ b/lustre/osp/osp_internal.h @@ -0,0 +1,176 @@ +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + * GPL HEADER END + */ +/* + * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. + * Use is subject to license terms. + * + * Copyright (c) 2011, 2012, Intel, Inc. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + * Lustre is a trademark of Sun Microsystems, Inc. + * + * lustre/osp/osp_internal.h + * + * Author: Alex Zhuravlev + */ + +#ifndef _OSP_INTERNAL_H +#define _OSP_INTERNAL_H + +#include +#include +#include + +struct osp_device { + struct dt_device opd_dt_dev; + /* corresponded OST index */ + int opd_index; + /* device used to store persistent state (llogs, last ids) */ + struct obd_export *opd_storage_exp; + struct dt_device *opd_storage; + /* connection to OST */ + struct obd_device *opd_obd; + struct obd_export *opd_exp; + struct obd_uuid opd_cluuid; + struct obd_connect_data *opd_connect_data; + int opd_connects; + cfs_proc_dir_entry_t *opd_proc_entry; + struct lprocfs_stats *opd_stats; + /* connection status. */ + int opd_new_connection; + int opd_got_disconnected; + int opd_imp_connected; + int opd_imp_active; + int opd_imp_seen_connected:1; + + /* whether local recovery is completed: + * reported via ->ldo_recovery_complete() */ + int opd_recovery_completed; + + cfs_proc_dir_entry_t *opd_symlink; +}; + +extern cfs_mem_cache_t *osp_object_kmem; + +/* this is a top object */ +struct osp_object { + struct lu_object_header opo_header; + struct dt_object opo_obj; + int opo_reserved; +}; + +extern struct lu_object_operations osp_lu_obj_ops; + +struct osp_thread_info { + struct lu_attr osi_attr; +}; + +extern struct lu_context_key osp_thread_key; + +static inline struct osp_thread_info *osp_env_info(const struct lu_env *env) +{ + struct osp_thread_info *info; + + info = lu_context_key_get(&env->le_ctx, &osp_thread_key); + if (info == NULL) { + lu_env_refill((struct lu_env *)env); + info = lu_context_key_get(&env->le_ctx, &osp_thread_key); + } + LASSERT(info); + return info; +} + +struct osp_txn_info { + __u32 oti_current_id; +}; + +extern struct lu_context_key osp_txn_key; + +static inline struct osp_txn_info *osp_txn_info(struct lu_context *ctx) +{ + struct osp_txn_info *info; + + info = lu_context_key_get(ctx, &osp_txn_key); + return info; +} + +extern const struct lu_device_operations osp_lu_ops; + +static inline int lu_device_is_osp(struct lu_device *d) +{ + return ergo(d != NULL && d->ld_ops != NULL, d->ld_ops == &osp_lu_ops); +} + +static inline struct osp_device *lu2osp_dev(struct lu_device *d) +{ + LASSERT(lu_device_is_osp(d)); + return container_of0(d, struct osp_device, opd_dt_dev.dd_lu_dev); +} + +static inline struct lu_device *osp2lu_dev(struct osp_device *d) +{ + return &d->opd_dt_dev.dd_lu_dev; +} + +static inline struct osp_device *dt2osp_dev(struct dt_device *d) +{ + LASSERT(lu_device_is_osp(&d->dd_lu_dev)); + return container_of0(d, struct osp_device, opd_dt_dev); +} + +static inline struct osp_object *lu2osp_obj(struct lu_object *o) +{ + LASSERT(ergo(o != NULL, lu_device_is_osp(o->lo_dev))); + return container_of0(o, struct osp_object, opo_obj.do_lu); +} + +static inline struct lu_object *osp2lu_obj(struct osp_object *obj) +{ + return &obj->opo_obj.do_lu; +} + +static inline struct osp_object *osp_obj(const struct lu_object *o) +{ + LASSERT(lu_device_is_osp(o->lo_dev)); + return container_of0(o, struct osp_object, opo_obj.do_lu); +} + +static inline struct osp_object *dt2osp_obj(const struct dt_object *d) +{ + return osp_obj(&d->do_lu); +} + +static inline struct dt_object *osp_object_child(struct osp_object *o) +{ + return container_of0(lu_object_next(osp2lu_obj(o)), + struct dt_object, do_lu); +} + +/* lproc_osp.c */ +void lprocfs_osp_init_vars(struct lprocfs_static_vars *lvars); + +#endif diff --git a/lustre/osp/osp_object.c b/lustre/osp/osp_object.c new file mode 100644 index 0000000..cb2f1b4 --- /dev/null +++ b/lustre/osp/osp_object.c @@ -0,0 +1,98 @@ +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + * GPL HEADER END + */ +/* + * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. + * Use is subject to license terms. + * + * Copyright (c) 2011, 2012, Intel, Inc. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + * Lustre is a trademark of Sun Microsystems, Inc. + * + * lustre/osp/osp_object.c + * + * Lustre OST Proxy Device + * + * Author: Alex Zhuravlev + * Author: Mikhail Pershin + */ + +#ifndef EXPORT_SYMTAB +# define EXPORT_SYMTAB +#endif +#define DEBUG_SUBSYSTEM S_MDS + +#include "osp_internal.h" + +struct dt_object_operations osp_obj_ops = { +}; + +static int osp_object_init(const struct lu_env *env, struct lu_object *o, + const struct lu_object_conf *unused) +{ + struct osp_object *po = lu2osp_obj(o); + + po->opo_obj.do_ops = &osp_obj_ops; + + return 0; +} + +static void osp_object_free(const struct lu_env *env, struct lu_object *o) +{ + struct osp_object *obj = lu2osp_obj(o); + struct lu_object_header *h = o->lo_header; + + dt_object_fini(&obj->opo_obj); + lu_object_header_fini(h); + OBD_SLAB_FREE_PTR(obj, osp_object_kmem); +} + +static void osp_object_release(const struct lu_env *env, struct lu_object *o) +{ + return; +} + +static int osp_object_print(const struct lu_env *env, void *cookie, + lu_printer_t p, const struct lu_object *l) +{ + const struct osp_object *o = lu2osp_obj((struct lu_object *)l); + + return (*p)(env, cookie, LUSTRE_OSP_NAME"-object@%p", o); +} + +static int osp_object_invariant(const struct lu_object *o) +{ + LBUG(); +} + +struct lu_object_operations osp_lu_obj_ops = { + .loo_object_init = osp_object_init, + .loo_object_free = osp_object_free, + .loo_object_release = osp_object_release, + .loo_object_print = osp_object_print, + .loo_object_invariant = osp_object_invariant +}; -- 1.8.3.1