1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
6 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 only,
10 * as published by the Free Software Foundation.
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License version 2 for more details (a copy is included
16 * in the LICENSE file that accompanied this code).
18 * You should have received a copy of the GNU General Public License
19 * version 2 along with this program; If not, see
20 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
22 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
23 * CA 95054 USA or visit www.sun.com if you need additional information or
29 * Copyright 2008 Sun Microsystems, Inc. All rights reserved
30 * Use is subject to license terms.
33 * This file is part of Lustre, http://www.lustre.org/
34 * Lustre is a trademark of Sun Microsystems, Inc.
36 * lustre/cmm/cmm_device.c
38 * Lustre Cluster Metadata Manager (cmm)
40 * Author: Mike Pershin <tappro@clusterfs.com>
44 # define EXPORT_SYMTAB
46 #define DEBUG_SUBSYSTEM S_MDS
48 #include <linux/module.h>
51 #include <obd_class.h>
52 #include <lprocfs_status.h>
53 #include <lustre_ver.h>
54 #include "cmm_internal.h"
55 #include "mdc_internal.h"
57 static struct obd_ops cmm_obd_device_ops = {
58 .o_owner = THIS_MODULE
61 static struct lu_device_operations cmm_lu_ops;
63 static inline int lu_device_is_cmm(struct lu_device *d)
65 return ergo(d != NULL && d->ld_ops != NULL, d->ld_ops == &cmm_lu_ops);
68 int cmm_root_get(const struct lu_env *env, struct md_device *md,
71 struct cmm_device *cmm_dev = md2cmm_dev(md);
72 /* valid only on master MDS */
73 if (cmm_dev->cmm_local_num == 0)
74 return cmm_child_ops(cmm_dev)->mdo_root_get(env,
75 cmm_dev->cmm_child, fid);
80 static int cmm_statfs(const struct lu_env *env, struct md_device *md,
83 struct cmm_device *cmm_dev = md2cmm_dev(md);
87 rc = cmm_child_ops(cmm_dev)->mdo_statfs(env,
88 cmm_dev->cmm_child, sfs);
92 static int cmm_maxsize_get(const struct lu_env *env, struct md_device *md,
93 int *md_size, int *cookie_size)
95 struct cmm_device *cmm_dev = md2cmm_dev(md);
98 rc = cmm_child_ops(cmm_dev)->mdo_maxsize_get(env, cmm_dev->cmm_child,
99 md_size, cookie_size);
103 static int cmm_init_capa_ctxt(const struct lu_env *env, struct md_device *md,
104 int mode , unsigned long timeout, __u32 alg,
105 struct lustre_capa_key *keys)
107 struct cmm_device *cmm_dev = md2cmm_dev(md);
110 LASSERT(cmm_child_ops(cmm_dev)->mdo_init_capa_ctxt);
111 rc = cmm_child_ops(cmm_dev)->mdo_init_capa_ctxt(env, cmm_dev->cmm_child,
117 static int cmm_update_capa_key(const struct lu_env *env,
118 struct md_device *md,
119 struct lustre_capa_key *key)
121 struct cmm_device *cmm_dev = md2cmm_dev(md);
124 rc = cmm_child_ops(cmm_dev)->mdo_update_capa_key(env,
130 static struct md_device_operations cmm_md_ops = {
131 .mdo_statfs = cmm_statfs,
132 .mdo_root_get = cmm_root_get,
133 .mdo_maxsize_get = cmm_maxsize_get,
134 .mdo_init_capa_ctxt = cmm_init_capa_ctxt,
135 .mdo_update_capa_key = cmm_update_capa_key,
138 extern struct lu_device_type mdc_device_type;
140 static int cmm_post_init_mdc(const struct lu_env *env,
141 struct cmm_device *cmm)
143 int max_mdsize, max_cookiesize, rc;
144 struct mdc_device *mc, *tmp;
146 /* get the max mdsize and cookiesize from lower layer */
147 rc = cmm_maxsize_get(env, &cmm->cmm_md_dev, &max_mdsize,
152 spin_lock(&cmm->cmm_tgt_guard);
153 list_for_each_entry_safe(mc, tmp, &cmm->cmm_targets,
155 cmm_mdc_init_ea_size(env, mc, max_mdsize, max_cookiesize);
157 spin_unlock(&cmm->cmm_tgt_guard);
161 /* --- cmm_lu_operations --- */
162 /* add new MDC to the CMM, create MDC lu_device and connect it to mdc_obd */
163 static int cmm_add_mdc(const struct lu_env *env,
164 struct cmm_device *cm, struct lustre_cfg *cfg)
166 struct lu_device_type *ldt = &mdc_device_type;
167 char *p, *num = lustre_cfg_string(cfg, 2);
168 struct mdc_device *mc, *tmp;
169 struct lu_fld_target target;
170 struct lu_device *ld;
175 /* find out that there is no such mdc */
177 mdc_num = simple_strtol(num, &p, 10);
179 CERROR("Invalid index in lustre_cgf, offset 2\n");
183 spin_lock(&cm->cmm_tgt_guard);
184 list_for_each_entry_safe(mc, tmp, &cm->cmm_targets,
186 if (mc->mc_num == mdc_num) {
187 spin_unlock(&cm->cmm_tgt_guard);
191 spin_unlock(&cm->cmm_tgt_guard);
192 ld = ldt->ldt_ops->ldto_device_alloc(env, ldt, cfg);
196 ld->ld_site = cmm2lu_dev(cm)->ld_site;
198 rc = ldt->ldt_ops->ldto_device_init(env, ld, NULL, NULL);
200 ldt->ldt_ops->ldto_device_free(env, ld);
203 /* pass config to the just created MDC */
204 rc = ld->ld_ops->ldo_process_config(env, ld, cfg);
206 ldt->ldt_ops->ldto_device_fini(env, ld);
207 ldt->ldt_ops->ldto_device_free(env, ld);
211 spin_lock(&cm->cmm_tgt_guard);
212 list_for_each_entry_safe(mc, tmp, &cm->cmm_targets,
214 if (mc->mc_num == mdc_num) {
215 spin_unlock(&cm->cmm_tgt_guard);
216 ldt->ldt_ops->ldto_device_fini(env, ld);
217 ldt->ldt_ops->ldto_device_free(env, ld);
222 list_add_tail(&mc->mc_linkage, &cm->cmm_targets);
224 spin_unlock(&cm->cmm_tgt_guard);
226 lu_device_get(cmm2lu_dev(cm));
228 target.ft_srv = NULL;
229 target.ft_idx = mc->mc_num;
230 target.ft_exp = mc->mc_desc.cl_exp;
231 fld_client_add_target(cm->cmm_fld, &target);
233 /* Set max md size for the mdc. */
234 rc = cmm_post_init_mdc(env, cm);
238 static void cmm_device_shutdown(const struct lu_env *env,
239 struct cmm_device *cm,
240 struct lustre_cfg *cfg)
242 struct mdc_device *mc, *tmp;
245 /* Remove local target from FLD. */
246 fld_client_del_target(cm->cmm_fld, cm->cmm_local_num);
248 /* Finish all mdc devices. */
249 spin_lock(&cm->cmm_tgt_guard);
250 list_for_each_entry_safe(mc, tmp, &cm->cmm_targets, mc_linkage) {
251 struct lu_device *ld_m = mdc2lu_dev(mc);
252 fld_client_del_target(cm->cmm_fld, mc->mc_num);
253 ld_m->ld_ops->ldo_process_config(env, ld_m, cfg);
255 spin_unlock(&cm->cmm_tgt_guard);
257 /* remove upcall device*/
258 md_upcall_fini(&cm->cmm_md_dev);
263 static int cmm_device_mount(const struct lu_env *env,
264 struct cmm_device *m, struct lustre_cfg *cfg)
266 const char *index = lustre_cfg_string(cfg, 2);
269 LASSERT(index != NULL);
271 m->cmm_local_num = simple_strtol(index, &p, 10);
273 CERROR("Invalid index in lustre_cgf\n");
280 static int cmm_process_config(const struct lu_env *env,
281 struct lu_device *d, struct lustre_cfg *cfg)
283 struct cmm_device *m = lu2cmm_dev(d);
284 struct lu_device *next = md2lu_dev(m->cmm_child);
288 switch(cfg->lcfg_command) {
290 /* On first ADD_MDC add also local target. */
291 if (!(m->cmm_flags & CMM_INITIALIZED)) {
292 struct lu_site *ls = cmm2lu_dev(m)->ld_site;
293 struct lu_fld_target target;
295 target.ft_srv = ls->ls_server_fld;
296 target.ft_idx = m->cmm_local_num;
297 target.ft_exp = NULL;
299 fld_client_add_target(m->cmm_fld, &target);
301 err = cmm_add_mdc(env, m, cfg);
303 /* The first ADD_MDC can be counted as setup is finished. */
304 if (!(m->cmm_flags & CMM_INITIALIZED))
305 m->cmm_flags |= CMM_INITIALIZED;
310 /* lower layers should be set up at first */
311 err = next->ld_ops->ldo_process_config(env, next, cfg);
313 err = cmm_device_mount(env, m, cfg);
318 cmm_device_shutdown(env, m, cfg);
321 err = next->ld_ops->ldo_process_config(env, next, cfg);
326 static int cmm_recovery_complete(const struct lu_env *env,
329 struct cmm_device *m = lu2cmm_dev(d);
330 struct lu_device *next = md2lu_dev(m->cmm_child);
333 rc = next->ld_ops->ldo_recovery_complete(env, next);
337 static struct lu_device_operations cmm_lu_ops = {
338 .ldo_object_alloc = cmm_object_alloc,
339 .ldo_process_config = cmm_process_config,
340 .ldo_recovery_complete = cmm_recovery_complete
343 /* --- lu_device_type operations --- */
344 int cmm_upcall(const struct lu_env *env, struct md_device *md,
345 enum md_upcall_event ev)
352 rc = cmm_post_init_mdc(env, md2cmm_dev(md));
354 CERROR("can not init md size %d\n", rc);
357 rc = md_do_upcall(env, md, ev);
362 static struct lu_device *cmm_device_alloc(const struct lu_env *env,
363 struct lu_device_type *t,
364 struct lustre_cfg *cfg)
367 struct cmm_device *m;
372 l = ERR_PTR(-ENOMEM);
374 md_device_init(&m->cmm_md_dev, t);
375 m->cmm_md_dev.md_ops = &cmm_md_ops;
376 md_upcall_init(&m->cmm_md_dev, cmm_upcall);
378 l->ld_ops = &cmm_lu_ops;
380 OBD_ALLOC_PTR(m->cmm_fld);
382 GOTO(out_free_cmm, l = ERR_PTR(-ENOMEM));
391 static struct lu_device *cmm_device_free(const struct lu_env *env,
394 struct cmm_device *m = lu2cmm_dev(d);
395 struct lu_device *next = md2lu_dev(m->cmm_child);
398 LASSERT(m->cmm_tgt_count == 0);
399 LASSERT(list_empty(&m->cmm_targets));
400 if (m->cmm_fld != NULL) {
401 OBD_FREE_PTR(m->cmm_fld);
404 md_device_fini(&m->cmm_md_dev);
409 /* context key constructor/destructor: cmm_key_init, cmm_key_fini */
410 LU_KEY_INIT_FINI(cmm, struct cmm_thread_info);
412 /* context key: cmm_thread_key */
413 LU_CONTEXT_KEY_DEFINE(cmm, LCT_MD_THREAD);
415 struct cmm_thread_info *cmm_env_info(const struct lu_env *env)
417 struct cmm_thread_info *info;
419 info = lu_context_key_get(&env->le_ctx, &cmm_thread_key);
420 LASSERT(info != NULL);
424 /* type constructor/destructor: cmm_type_init/cmm_type_fini */
425 LU_TYPE_INIT_FINI(cmm, &cmm_thread_key);
427 static int cmm_device_init(const struct lu_env *env, struct lu_device *d,
428 const char *name, struct lu_device *next)
430 struct cmm_device *m = lu2cmm_dev(d);
435 spin_lock_init(&m->cmm_tgt_guard);
436 CFS_INIT_LIST_HEAD(&m->cmm_targets);
437 m->cmm_tgt_count = 0;
438 m->cmm_child = lu2md_dev(next);
440 err = fld_client_init(m->cmm_fld, name,
441 LUSTRE_CLI_FLD_HASH_DHT);
443 CERROR("Can't init FLD, err %d\n", err);
447 /* Assign site's fld client ref, needed for asserts in osd. */
448 ls = cmm2lu_dev(m)->ld_site;
449 ls->ls_client_fld = m->cmm_fld;
450 err = cmm_procfs_init(m, name);
455 static struct lu_device *cmm_device_fini(const struct lu_env *env,
456 struct lu_device *ld)
458 struct cmm_device *cm = lu2cmm_dev(ld);
459 struct mdc_device *mc, *tmp;
463 /* Finish all mdc devices */
464 spin_lock(&cm->cmm_tgt_guard);
465 list_for_each_entry_safe(mc, tmp, &cm->cmm_targets, mc_linkage) {
466 struct lu_device *ld_m = mdc2lu_dev(mc);
468 list_del_init(&mc->mc_linkage);
469 lu_device_put(cmm2lu_dev(cm));
470 ld_m->ld_type->ldt_ops->ldto_device_fini(env, ld_m);
471 ld_m->ld_type->ldt_ops->ldto_device_free(env, ld_m);
474 spin_unlock(&cm->cmm_tgt_guard);
476 fld_client_fini(cm->cmm_fld);
477 ls = cmm2lu_dev(cm)->ld_site;
478 ls->ls_client_fld = NULL;
481 RETURN (md2lu_dev(cm->cmm_child));
484 static struct lu_device_type_operations cmm_device_type_ops = {
485 .ldto_init = cmm_type_init,
486 .ldto_fini = cmm_type_fini,
488 .ldto_device_alloc = cmm_device_alloc,
489 .ldto_device_free = cmm_device_free,
491 .ldto_device_init = cmm_device_init,
492 .ldto_device_fini = cmm_device_fini
495 static struct lu_device_type cmm_device_type = {
496 .ldt_tags = LU_DEVICE_MD,
497 .ldt_name = LUSTRE_CMM_NAME,
498 .ldt_ops = &cmm_device_type_ops,
499 .ldt_ctx_tags = LCT_MD_THREAD | LCT_DT_THREAD
502 struct lprocfs_vars lprocfs_cmm_obd_vars[] = {
506 struct lprocfs_vars lprocfs_cmm_module_vars[] = {
510 static void lprocfs_cmm_init_vars(struct lprocfs_static_vars *lvars)
512 lvars->module_vars = lprocfs_cmm_module_vars;
513 lvars->obd_vars = lprocfs_cmm_obd_vars;
516 static int __init cmm_mod_init(void)
518 struct lprocfs_static_vars lvars;
520 lprocfs_cmm_init_vars(&lvars);
521 return class_register_type(&cmm_obd_device_ops, NULL, lvars.module_vars,
522 LUSTRE_CMM_NAME, &cmm_device_type);
525 static void __exit cmm_mod_exit(void)
527 class_unregister_type(LUSTRE_CMM_NAME);
530 MODULE_AUTHOR("Sun Microsystems, Inc. <http://www.lustre.org/>");
531 MODULE_DESCRIPTION("Lustre Clustered Metadata Manager ("LUSTRE_CMM_NAME")");
532 MODULE_LICENSE("GPL");
534 cfs_module(cmm, "0.1.0", cmm_mod_init, cmm_mod_exit);