4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.gnu.org/licenses/gpl-2.0.html
23 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Use is subject to license terms.
26 * Copyright (c) 2012, 2017, Intel Corporation.
29 * This file is part of Lustre, http://www.lustre.org/
31 * Implementation of cl_device and cl_device_type for LOV layer.
33 * Author: Nikita Danilov <nikita.danilov@sun.com>
36 #define DEBUG_SUBSYSTEM S_LOV
38 /* class_name2obd() */
39 #include <obd_class.h>
41 #include "lov_cl_internal.h"
43 struct kmem_cache *lov_lock_kmem;
44 struct kmem_cache *lov_object_kmem;
45 struct kmem_cache *lov_thread_kmem;
46 struct kmem_cache *lov_session_kmem;
48 struct kmem_cache *lovsub_object_kmem;
50 struct lu_kmem_descr lov_caches[] = {
52 .ckd_cache = &lov_lock_kmem,
53 .ckd_name = "lov_lock_kmem",
54 .ckd_size = sizeof(struct lov_lock)
57 .ckd_cache = &lov_object_kmem,
58 .ckd_name = "lov_object_kmem",
59 .ckd_size = sizeof(struct lov_object)
62 .ckd_cache = &lov_thread_kmem,
63 .ckd_name = "lov_thread_kmem",
64 .ckd_size = sizeof(struct lov_thread_info)
67 .ckd_cache = &lov_session_kmem,
68 .ckd_name = "lov_session_kmem",
69 .ckd_size = sizeof(struct lov_session)
72 .ckd_cache = &lovsub_object_kmem,
73 .ckd_name = "lovsub_object_kmem",
74 .ckd_size = sizeof(struct lovsub_object)
82 * Lov device and device type functions.
86 * Initilize and associate key(per context data) with
87 * lu_context(execution context)
89 * \param[in] ctx Execution context
90 * \param[in] key Describes Data associated with this context
93 static void *lov_key_init(const struct lu_context *ctx,
94 struct lu_context_key *key)
96 struct lov_thread_info *info;
98 OBD_SLAB_ALLOC_PTR_GFP(info, lov_thread_kmem, GFP_NOFS);
100 info = ERR_PTR(-ENOMEM);
105 * Release execution context and disassociate key
107 * \param[in] ctx execution environment
109 * \param[in] data Data associated with this context
112 static void lov_key_fini(const struct lu_context *ctx,
113 struct lu_context_key *key, void *data)
115 struct lov_thread_info *info = data;
116 OBD_SLAB_FREE_PTR(info, lov_thread_kmem);
119 struct lu_context_key lov_key = {
120 .lct_tags = LCT_CL_THREAD,
121 .lct_init = lov_key_init,
122 .lct_fini = lov_key_fini
126 * Initilize and associate key(per context data) with
127 * lu_context(execution context) for a session
129 * \param[in] ctx Execution context
130 * \param[in] key Describes Data associated with this context
132 static void *lov_session_key_init(const struct lu_context *ctx,
133 struct lu_context_key *key)
135 struct lov_session *info;
137 OBD_SLAB_ALLOC_PTR_GFP(info, lov_session_kmem, GFP_NOFS);
139 info = ERR_PTR(-ENOMEM);
144 * Release execution context and disassociate key for a session
146 * \param[in] ctx execution environment
148 * \param[in] data Data associated with this context
150 static void lov_session_key_fini(const struct lu_context *ctx,
151 struct lu_context_key *key, void *data)
153 struct lov_session *info = data;
155 OBD_SLAB_FREE_PTR(info, lov_session_kmem);
158 struct lu_context_key lov_session_key = {
159 .lct_tags = LCT_SESSION,
160 .lct_init = lov_session_key_init,
161 .lct_fini = lov_session_key_fini
164 /* type constructor/destructor: lov_type_{init,fini,start,stop}() */
165 LU_TYPE_INIT_FINI(lov, &lov_key, &lov_session_key);
169 * Add new MDC target device in LOV.
171 * This function is part of the configuration log processing. It adds new MDC
172 * device to the MDC device array indexed by their indexes.
174 * \param[in] env local execution environment
175 * \param[in] ld LU device of LOV device
176 * \param[in] mdc_dev MDC device to add
177 * \param[in] idx MDC device index
178 * \param[in] nr MDC device index
180 * \retval 0 if successful
181 * \retval Non zero value on error
183 static int lov_mdc_dev_init(const struct lu_env *env, struct lov_device *ld,
184 struct lu_device *mdc_dev, __u32 idx, __u32 nr)
186 struct cl_device *cl;
189 cl = cl_type_setup(env, &ld->ld_site, &lovsub_device_type,
194 ld->ld_md_tgts[nr].ldm_mdc = cl;
195 ld->ld_md_tgts[nr].ldm_idx = idx;
200 * Free Resource associated with lov device context
202 * \param[in] env execution environment
203 * \param[in] d LU device of LOV device
205 * \retval Returns NULL
207 static struct lu_device *lov_device_fini(const struct lu_env *env,
210 struct lov_device *ld = lu2lov_dev(d);
213 LASSERT(ld->ld_lov != NULL);
216 class_decref(ld->ld_lmv, "lov", d);
220 if (ld->ld_md_tgts) {
221 for (i = 0; i < ld->ld_md_tgts_nr; i++) {
222 if (!ld->ld_md_tgts[i].ldm_mdc)
225 cl_stack_fini(env, ld->ld_md_tgts[i].ldm_mdc);
226 ld->ld_md_tgts[i].ldm_mdc = NULL;
227 ld->ld_lov->lov_mdc_tgts[i].lmtd_mdc = NULL;
232 lov_foreach_target(ld, i) {
233 struct lovsub_device *lsd;
235 lsd = ld->ld_target[i];
237 cl_stack_fini(env, lovsub2cl_dev(lsd));
238 ld->ld_target[i] = NULL;
246 * Initilize lov_device
248 * \param[in] env execution environment
249 * \param[in] d LU device of LOV device
250 * \param[in] name Device name
251 * \param[in] next Pointer to next lu_device
253 * \retval 0 if successful
254 * \retval negative value on error
256 static int lov_device_init(const struct lu_env *env, struct lu_device *d,
257 const char *name, struct lu_device *next)
259 struct lov_device *ld = lu2lov_dev(d);
263 /* check all added already MDC subdevices and initialize them */
264 for (i = 0; i < ld->ld_md_tgts_nr; i++) {
265 struct obd_device *mdc;
268 mdc = ld->ld_lov->lov_mdc_tgts[i].lmtd_mdc;
269 idx = ld->ld_lov->lov_mdc_tgts[i].lmtd_index;
274 rc = lov_mdc_dev_init(env, ld, mdc->obd_lu_dev, idx, i);
276 CERROR("%s: failed to add MDC %s as target: rc = %d\n",
278 obd_uuid2str(&mdc->obd_uuid), rc);
286 lov_foreach_target(ld, i) {
287 struct lovsub_device *lsd;
288 struct cl_device *cl;
289 struct lov_tgt_desc *desc;
291 desc = ld->ld_lov->lov_tgts[i];
295 cl = cl_type_setup(env, &ld->ld_site, &lovsub_device_type,
296 desc->ltd_obd->obd_lu_dev);
298 GOTO(out_err, rc = PTR_ERR(cl));
300 lsd = cl2lovsub_dev(cl);
301 ld->ld_target[i] = lsd;
303 ld->ld_flags |= LOV_DEV_INITIALIZED;
312 * Free the lov specific data created for the back end lu_device.
314 * \param[in] env execution environment
315 * \param[in] d Backend lu_device
317 * \retval Free data and return NULL
319 static struct lu_device *lov_device_free(const struct lu_env *env,
322 struct lov_device *ld = lu2lov_dev(d);
323 const int nr = ld->ld_target_nr;
325 lu_site_fini(&ld->ld_site);
327 cl_device_fini(lu2cl_dev(d));
329 OBD_FREE_PTR_ARRAY(ld->ld_target, nr);
330 ld->ld_target = NULL;
332 if (ld->ld_md_tgts) {
333 OBD_FREE_PTR_ARRAY(ld->ld_md_tgts, LOV_MDC_TGT_MAX);
334 ld->ld_md_tgts = NULL;
336 /* free array of MDCs */
337 if (ld->ld_lov->lov_mdc_tgts) {
338 OBD_FREE_PTR_ARRAY(ld->ld_lov->lov_mdc_tgts, LOV_MDC_TGT_MAX);
339 ld->ld_lov->lov_mdc_tgts = NULL;
347 * Delete cl_object(osc) target from lov
349 * \param[in] env execution environment
350 * \param[in] dev LU device of LOV device
351 * \param[in] index index of backend lu device
354 static void lov_cl_del_target(const struct lu_env *env, struct lu_device *dev,
357 struct lov_device *ld = lu2lov_dev(dev);
361 if (ld->ld_target[index]) {
362 cl_stack_fini(env, lovsub2cl_dev(ld->ld_target[index]));
363 ld->ld_target[index] = NULL;
368 static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev)
376 tgt_size = dev->ld_lov->lov_tgt_size;
377 sub_size = dev->ld_target_nr;
378 if (sub_size < tgt_size) {
379 struct lovsub_device **newd;
380 const size_t sz = sizeof(newd[0]);
382 OBD_ALLOC_PTR_ARRAY(newd, tgt_size);
385 memcpy(newd, dev->ld_target, sub_size * sz);
386 OBD_FREE(dev->ld_target, sub_size * sz);
389 dev->ld_target = newd;
390 dev->ld_target_nr = tgt_size;
400 * Add cl_object(osc) target to lov
402 * \param[in] env execution environment
403 * \param[in] dev LU device of LOV device
404 * \param[in] index index of lu backend device
406 * \retval 0 if successful
407 * \retval negative value on error
409 static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
412 struct obd_device *obd = dev->ld_obd;
413 struct lov_device *ld = lu2lov_dev(dev);
414 struct lov_tgt_desc *tgt;
415 struct lovsub_device *lsd;
416 struct cl_device *cl;
421 lov_tgts_getref(obd);
423 tgt = obd->u.lov.lov_tgts[index];
424 LASSERT(tgt != NULL);
425 LASSERT(tgt->ltd_obd != NULL);
427 if (!tgt->ltd_obd->obd_set_up) {
428 CERROR("Target %s not set up\n", obd_uuid2str(&tgt->ltd_uuid));
432 rc = lov_expand_targets(env, ld);
433 if (rc == 0 && ld->ld_flags & LOV_DEV_INITIALIZED) {
434 cl = cl_type_setup(env, &ld->ld_site, &lovsub_device_type,
435 tgt->ltd_obd->obd_lu_dev);
437 lsd = cl2lovsub_dev(cl);
438 ld->ld_target[index] = lsd;
440 CERROR("add failed (%d), deleting %s\n", rc,
441 obd_uuid2str(&tgt->ltd_uuid));
442 lov_cl_del_target(env, dev, index);
447 lov_tgts_putref(obd);
453 * Add new MDC target device in LOV.
455 * This function is part of the configuration log processing. It adds new MDC
456 * device to the MDC device array indexed by their indexes.
458 * \param[in] env execution environment
459 * \param[in] d LU device of LOV device
460 * \param[in] mdc MDC device to add
461 * \param[in] idx MDC device index
463 * \retval 0 if successful
464 * \retval negative value on error
466 static int lov_add_mdc_target(const struct lu_env *env, struct lu_device *d,
467 struct obd_device *mdc, __u32 idx)
469 struct lov_device *ld = lu2lov_dev(d);
470 struct obd_device *lov_obd = d->ld_obd;
471 struct obd_device *lmv_obd = NULL;
472 unsigned long dev_no = 0;
477 LASSERT(mdc != NULL);
478 if (ld->ld_md_tgts_nr == LOV_MDC_TGT_MAX) {
480 * If the maximum value of LOV_MDC_TGT_MAX will become too
481 * small then all MD target handling must be rewritten in LOD
482 * manner, check lod_add_device() and related functionality.
484 CERROR("%s: cannot serve more than %d MDC devices\n",
485 lov_obd->obd_name, LOV_MDC_TGT_MAX);
490 * grab FLD from lmv, do that here, when first MDC is added
491 * to be sure LMV is set up and can be found
495 obd_device_for_each_uuid(dev_no, lmv_obd,
496 &lov_obd->obd_uuid) {
497 if ((strncmp(lmv_obd->obd_type->typ_name,
499 strlen(LUSTRE_LMV_NAME)) == 0)) {
500 spin_lock(&lmv_obd->obd_dev_lock);
501 class_incref(lmv_obd, "lov", ld);
502 spin_unlock(&lmv_obd->obd_dev_lock);
509 CERROR("%s: cannot find LMV OBD by UUID (%s)\n",
511 obd_uuid2str(&lmv_obd->obd_uuid));
514 ld->ld_lmv = lmv_obd;
517 LASSERT(lov_obd->u.lov.lov_mdc_tgts[ld->ld_md_tgts_nr].lmtd_mdc ==
520 if (ld->ld_flags & LOV_DEV_INITIALIZED) {
521 rc = lov_mdc_dev_init(env, ld, mdc->obd_lu_dev, idx,
524 CERROR("%s: failed to add MDC %s as target: rc = %d\n",
525 lov_obd->obd_name, obd_uuid2str(&mdc->obd_uuid),
531 lov_obd->u.lov.lov_mdc_tgts[ld->ld_md_tgts_nr].lmtd_mdc = mdc;
532 lov_obd->u.lov.lov_mdc_tgts[ld->ld_md_tgts_nr].lmtd_index = idx;
539 * Called when lov configuration changes are needed
541 * \param[in] env execution environment
542 * \param[in] d LU device of LOV device
543 * \param[in] cfg setup configuration commands and arguments
545 * \retval Return a new lu_device on success
546 * \retval Error pointer on failure
548 static int lov_process_config(const struct lu_env *env,
549 struct lu_device *d, struct lustre_cfg *cfg)
551 struct obd_device *obd = d->ld_obd;
557 lov_tgts_getref(obd);
559 cmd = cfg->lcfg_command;
561 rc = lov_process_config_base(d->ld_obd, cfg, &index, &gen);
566 case LCFG_LOV_ADD_OBD:
567 case LCFG_LOV_ADD_INA:
568 rc = lov_cl_add_target(env, d, index);
570 lov_del_target(d->ld_obd, index, NULL, 0);
572 case LCFG_LOV_DEL_OBD:
573 lov_cl_del_target(env, d, index);
577 struct obd_device *mdc;
578 struct obd_uuid tgt_uuid;
581 * modify_mdc_tgts add 0:lustre-clilmv 1:lustre-MDT0000_UUID
582 * 2:0 3:1 4:lustre-MDT0000-mdc_UUID
584 if (LUSTRE_CFG_BUFLEN(cfg, 1) > sizeof(tgt_uuid.uuid))
585 GOTO(out, rc = -EINVAL);
587 obd_str2uuid(&tgt_uuid, lustre_cfg_buf(cfg, 1));
589 rc = kstrtou32(lustre_cfg_buf(cfg, 2), 10, &index);
593 mdc = class_find_client_obd(&tgt_uuid, LUSTRE_MDC_NAME,
596 GOTO(out, rc = -ENODEV);
597 rc = lov_add_mdc_target(env, d, mdc, index);
602 lov_tgts_putref(obd);
606 static const struct lu_device_operations lov_lu_ops = {
607 .ldo_object_alloc = lov_object_alloc,
608 .ldo_process_config = lov_process_config,
612 * Allocate a new lov lu_device
614 * \param[in] env execution environment
615 * \param[in] t Backend OSD device type (ldiskfs,zfs)
616 * \param[in] cfg setup configuration commands and arguments
618 * \retval Return a new lu_device on success
619 * \retval Error pointer on failure
621 static struct lu_device *lov_device_alloc(const struct lu_env *env,
622 struct lu_device_type *t,
623 struct lustre_cfg *cfg)
626 struct lov_device *ld;
627 struct obd_device *obd;
632 RETURN(ERR_PTR(-ENOMEM));
634 cl_device_init(&ld->ld_cl, t);
636 d->ld_ops = &lov_lu_ops;
638 /* setup the LOV OBD */
639 obd = class_name2obd(lustre_cfg_string(cfg, 0));
640 LASSERT(obd != NULL);
641 rc = lov_setup(obd, cfg);
645 /* Alloc MDC devices array */
646 /* XXX: need dynamic allocation at some moment */
647 OBD_ALLOC_PTR_ARRAY(ld->ld_md_tgts, LOV_MDC_TGT_MAX);
649 GOTO(out, rc = -ENOMEM);
651 ld->ld_md_tgts_nr = 0;
653 ld->ld_lov = &obd->u.lov;
654 OBD_ALLOC_PTR_ARRAY(ld->ld_lov->lov_mdc_tgts, LOV_MDC_TGT_MAX);
655 if (!ld->ld_lov->lov_mdc_tgts)
656 GOTO(out_md_tgts, rc = -ENOMEM);
658 rc = lu_site_init(&ld->ld_site, d);
660 GOTO(out_mdc_tgts, rc);
662 rc = lu_site_init_finish(&ld->ld_site);
668 lu_site_fini(&ld->ld_site);
670 OBD_FREE_PTR_ARRAY(ld->ld_lov->lov_mdc_tgts, LOV_MDC_TGT_MAX);
671 ld->ld_lov->lov_mdc_tgts = NULL;
673 OBD_FREE_PTR_ARRAY(ld->ld_md_tgts, LOV_MDC_TGT_MAX);
674 ld->ld_md_tgts = NULL;
681 static const struct lu_device_type_operations lov_device_type_ops = {
682 .ldto_init = lov_type_init,
683 .ldto_fini = lov_type_fini,
685 .ldto_start = lov_type_start,
686 .ldto_stop = lov_type_stop,
688 .ldto_device_alloc = lov_device_alloc,
689 .ldto_device_free = lov_device_free,
691 .ldto_device_init = lov_device_init,
692 .ldto_device_fini = lov_device_fini
695 struct lu_device_type lov_device_type = {
696 .ldt_tags = LU_DEVICE_CL,
697 .ldt_name = LUSTRE_LOV_NAME,
698 .ldt_ops = &lov_device_type_ops,
699 .ldt_ctx_tags = LCT_CL_THREAD