4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.gnu.org/licenses/gpl-2.0.html
23 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Use is subject to license terms.
26 * Copyright (c) 2012, 2017, Intel Corporation.
29 * This file is part of Lustre, http://www.lustre.org/
30 * Lustre is a trademark of Sun Microsystems, Inc.
32 * Implementation of cl_device and cl_device_type for LOV layer.
34 * Author: Nikita Danilov <nikita.danilov@sun.com>
37 #define DEBUG_SUBSYSTEM S_LOV
39 /* class_name2obd() */
40 #include <obd_class.h>
42 #include "lov_cl_internal.h"
44 struct kmem_cache *lov_lock_kmem;
45 struct kmem_cache *lov_object_kmem;
46 struct kmem_cache *lov_thread_kmem;
47 struct kmem_cache *lov_session_kmem;
49 struct kmem_cache *lovsub_object_kmem;
51 struct lu_kmem_descr lov_caches[] = {
53 .ckd_cache = &lov_lock_kmem,
54 .ckd_name = "lov_lock_kmem",
55 .ckd_size = sizeof (struct lov_lock)
58 .ckd_cache = &lov_object_kmem,
59 .ckd_name = "lov_object_kmem",
60 .ckd_size = sizeof (struct lov_object)
63 .ckd_cache = &lov_thread_kmem,
64 .ckd_name = "lov_thread_kmem",
65 .ckd_size = sizeof (struct lov_thread_info)
68 .ckd_cache = &lov_session_kmem,
69 .ckd_name = "lov_session_kmem",
70 .ckd_size = sizeof (struct lov_session)
73 .ckd_cache = &lovsub_object_kmem,
74 .ckd_name = "lovsub_object_kmem",
75 .ckd_size = sizeof (struct lovsub_object)
82 /*****************************************************************************
84 * Lov device and device type functions.
88 static void *lov_key_init(const struct lu_context *ctx,
89 struct lu_context_key *key)
91 struct lov_thread_info *info;
93 OBD_SLAB_ALLOC_PTR_GFP(info, lov_thread_kmem, GFP_NOFS);
95 info = ERR_PTR(-ENOMEM);
99 static void lov_key_fini(const struct lu_context *ctx,
100 struct lu_context_key *key, void *data)
102 struct lov_thread_info *info = data;
103 OBD_SLAB_FREE_PTR(info, lov_thread_kmem);
106 struct lu_context_key lov_key = {
107 .lct_tags = LCT_CL_THREAD,
108 .lct_init = lov_key_init,
109 .lct_fini = lov_key_fini
112 static void *lov_session_key_init(const struct lu_context *ctx,
113 struct lu_context_key *key)
115 struct lov_session *info;
117 OBD_SLAB_ALLOC_PTR_GFP(info, lov_session_kmem, GFP_NOFS);
119 info = ERR_PTR(-ENOMEM);
123 static void lov_session_key_fini(const struct lu_context *ctx,
124 struct lu_context_key *key, void *data)
126 struct lov_session *info = data;
127 OBD_SLAB_FREE_PTR(info, lov_session_kmem);
130 struct lu_context_key lov_session_key = {
131 .lct_tags = LCT_SESSION,
132 .lct_init = lov_session_key_init,
133 .lct_fini = lov_session_key_fini
136 /* type constructor/destructor: lov_type_{init,fini,start,stop}() */
137 LU_TYPE_INIT_FINI(lov, &lov_key, &lov_session_key);
140 static int lov_mdc_dev_init(const struct lu_env *env, struct lov_device *ld,
141 struct lu_device *mdc_dev, __u32 idx, __u32 nr)
143 struct cl_device *cl;
146 cl = cl_type_setup(env, &ld->ld_site, &lovsub_device_type,
151 ld->ld_md_tgts[nr].ldm_mdc = cl;
152 ld->ld_md_tgts[nr].ldm_idx = idx;
156 static struct lu_device *lov_device_fini(const struct lu_env *env,
159 struct lov_device *ld = lu2lov_dev(d);
162 LASSERT(ld->ld_lov != NULL);
164 if (ld->ld_lmv != NULL) {
165 class_decref(ld->ld_lmv, "lov", d);
169 if (ld->ld_md_tgts != NULL) {
170 for (i = 0; i < ld->ld_md_tgts_nr; i++) {
171 if (ld->ld_md_tgts[i].ldm_mdc == NULL)
174 cl_stack_fini(env, ld->ld_md_tgts[i].ldm_mdc);
175 ld->ld_md_tgts[i].ldm_mdc = NULL;
176 ld->ld_lov->lov_mdc_tgts[i].lmtd_mdc = NULL;
180 if (ld->ld_target != NULL) {
181 lov_foreach_target(ld, i) {
182 struct lovsub_device *lsd;
184 lsd = ld->ld_target[i];
186 cl_stack_fini(env, lovsub2cl_dev(lsd));
187 ld->ld_target[i] = NULL;
194 static int lov_device_init(const struct lu_env *env, struct lu_device *d,
195 const char *name, struct lu_device *next)
197 struct lov_device *ld = lu2lov_dev(d);
201 /* check all added already MDC subdevices and initialize them */
202 for (i = 0; i < ld->ld_md_tgts_nr; i++) {
203 struct obd_device *mdc;
206 mdc = ld->ld_lov->lov_mdc_tgts[i].lmtd_mdc;
207 idx = ld->ld_lov->lov_mdc_tgts[i].lmtd_index;
212 rc = lov_mdc_dev_init(env, ld, mdc->obd_lu_dev, idx, i);
214 CERROR("%s: failed to add MDC %s as target: rc = %d\n",
216 obd_uuid2str(&mdc->obd_uuid), rc);
221 if (ld->ld_target == NULL)
224 lov_foreach_target(ld, i) {
225 struct lovsub_device *lsd;
226 struct cl_device *cl;
227 struct lov_tgt_desc *desc;
229 desc = ld->ld_lov->lov_tgts[i];
233 cl = cl_type_setup(env, &ld->ld_site, &lovsub_device_type,
234 desc->ltd_obd->obd_lu_dev);
236 GOTO(out_err, rc = PTR_ERR(cl));
238 lsd = cl2lovsub_dev(cl);
239 ld->ld_target[i] = lsd;
241 ld->ld_flags |= LOV_DEV_INITIALIZED;
249 /* Free the lov specific data created for the back end lu_device. */
250 static struct lu_device *lov_device_free(const struct lu_env *env,
253 struct lov_device *ld = lu2lov_dev(d);
254 const int nr = ld->ld_target_nr;
256 lu_site_fini(&ld->ld_site);
258 cl_device_fini(lu2cl_dev(d));
260 OBD_FREE(ld->ld_target, nr * sizeof ld->ld_target[0]);
261 ld->ld_target = NULL;
263 if (ld->ld_md_tgts) {
264 OBD_FREE(ld->ld_md_tgts,
265 sizeof(*ld->ld_md_tgts) * LOV_MDC_TGT_MAX);
266 ld->ld_md_tgts = NULL;
268 /* free array of MDCs */
269 if (ld->ld_lov->lov_mdc_tgts) {
270 OBD_FREE(ld->ld_lov->lov_mdc_tgts,
271 sizeof(*ld->ld_lov->lov_mdc_tgts) * LOV_MDC_TGT_MAX);
272 ld->ld_lov->lov_mdc_tgts = NULL;
279 static void lov_cl_del_target(const struct lu_env *env, struct lu_device *dev,
282 struct lov_device *ld = lu2lov_dev(dev);
285 if (ld->ld_target[index] != NULL) {
286 cl_stack_fini(env, lovsub2cl_dev(ld->ld_target[index]));
287 ld->ld_target[index] = NULL;
292 static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev)
300 tgt_size = dev->ld_lov->lov_tgt_size;
301 sub_size = dev->ld_target_nr;
302 if (sub_size < tgt_size) {
303 struct lovsub_device **newd;
304 const size_t sz = sizeof(newd[0]);
306 OBD_ALLOC(newd, tgt_size * sz);
309 memcpy(newd, dev->ld_target, sub_size * sz);
310 OBD_FREE(dev->ld_target, sub_size * sz);
313 dev->ld_target = newd;
314 dev->ld_target_nr = tgt_size;
323 static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
326 struct obd_device *obd = dev->ld_obd;
327 struct lov_device *ld = lu2lov_dev(dev);
328 struct lov_tgt_desc *tgt;
329 struct lovsub_device *lsd;
330 struct cl_device *cl;
336 tgt = obd->u.lov.lov_tgts[index];
337 LASSERT(tgt != NULL);
338 LASSERT(tgt->ltd_obd != NULL);
340 if (!tgt->ltd_obd->obd_set_up) {
341 CERROR("Target %s not set up\n", obd_uuid2str(&tgt->ltd_uuid));
345 rc = lov_expand_targets(env, ld);
346 if (rc == 0 && ld->ld_flags & LOV_DEV_INITIALIZED) {
347 cl = cl_type_setup(env, &ld->ld_site, &lovsub_device_type,
348 tgt->ltd_obd->obd_lu_dev);
350 lsd = cl2lovsub_dev(cl);
351 ld->ld_target[index] = lsd;
353 CERROR("add failed (%d), deleting %s\n", rc,
354 obd_uuid2str(&tgt->ltd_uuid));
355 lov_cl_del_target(env, dev, index);
364 * Add new MDC target device in LOV.
366 * This function is part of the configuration log processing. It adds new MDC
367 * device to the MDC device array indexed by their indexes.
369 * \param[in] env execution environment
370 * \param[in] d LU device of LOV device
371 * \param[in] mdc MDC device to add
372 * \param[in] idx MDC device index
374 * \retval 0 if successful
375 * \retval negative value on error
377 static int lov_add_mdc_target(const struct lu_env *env, struct lu_device *d,
378 struct obd_device *mdc, __u32 idx)
380 struct lov_device *ld = lu2lov_dev(d);
381 struct obd_device *lov_obd = d->ld_obd;
382 struct obd_device *lmv_obd;
388 LASSERT(mdc != NULL);
389 if (ld->ld_md_tgts_nr == LOV_MDC_TGT_MAX) {
390 /* If the maximum value of LOV_MDC_TGT_MAX will become too
391 * small then all MD target handling must be rewritten in LOD
392 * manner, check lod_add_device() and related functionality.
394 CERROR("%s: cannot serve more than %d MDC devices\n",
395 lov_obd->obd_name, LOV_MDC_TGT_MAX);
399 /* grab FLD from lmv, do that here, when first MDC is added
400 * to be sure LMV is set up and can be found */
401 if (ld->ld_lmv == NULL) {
403 while ((lmv_obd = class_devices_in_group(&lov_obd->obd_uuid,
405 if ((strncmp(lmv_obd->obd_type->typ_name,
407 strlen(LUSTRE_LMV_NAME)) == 0))
410 if (lmv_obd == NULL) {
411 CERROR("%s: cannot find LMV OBD by UUID (%s)\n",
413 obd_uuid2str(&lmv_obd->obd_uuid));
416 spin_lock(&lmv_obd->obd_dev_lock);
417 class_incref(lmv_obd, "lov", ld);
418 spin_unlock(&lmv_obd->obd_dev_lock);
419 ld->ld_lmv = lmv_obd;
422 LASSERT(lov_obd->u.lov.lov_mdc_tgts[ld->ld_md_tgts_nr].lmtd_mdc ==
425 if (ld->ld_flags & LOV_DEV_INITIALIZED) {
426 rc = lov_mdc_dev_init(env, ld, mdc->obd_lu_dev, idx,
429 CERROR("%s: failed to add MDC %s as target: rc = %d\n",
430 lov_obd->obd_name, obd_uuid2str(&mdc->obd_uuid),
436 lov_obd->u.lov.lov_mdc_tgts[ld->ld_md_tgts_nr].lmtd_mdc = mdc;
437 lov_obd->u.lov.lov_mdc_tgts[ld->ld_md_tgts_nr].lmtd_index = idx;
443 static int lov_process_config(const struct lu_env *env,
444 struct lu_device *d, struct lustre_cfg *cfg)
446 struct obd_device *obd = d->ld_obd;
454 cmd = cfg->lcfg_command;
456 rc = lov_process_config_base(d->ld_obd, cfg, &index, &gen);
461 case LCFG_LOV_ADD_OBD:
462 case LCFG_LOV_ADD_INA:
463 rc = lov_cl_add_target(env, d, index);
465 lov_del_target(d->ld_obd, index, NULL, 0);
467 case LCFG_LOV_DEL_OBD:
468 lov_cl_del_target(env, d, index);
472 struct obd_device *mdc;
473 struct obd_uuid tgt_uuid;
475 /* modify_mdc_tgts add 0:lustre-clilmv 1:lustre-MDT0000_UUID
476 * 2:0 3:1 4:lustre-MDT0000-mdc_UUID */
477 if (LUSTRE_CFG_BUFLEN(cfg, 1) > sizeof(tgt_uuid.uuid))
478 GOTO(out, rc = -EINVAL);
480 obd_str2uuid(&tgt_uuid, lustre_cfg_buf(cfg, 1));
482 if (sscanf(lustre_cfg_buf(cfg, 2), "%d", &index) != 1)
483 GOTO(out, rc = -EINVAL);
485 mdc = class_find_client_obd(&tgt_uuid, LUSTRE_MDC_NAME,
488 GOTO(out, rc = -ENODEV);
489 rc = lov_add_mdc_target(env, d, mdc, index);
498 static const struct lu_device_operations lov_lu_ops = {
499 .ldo_object_alloc = lov_object_alloc,
500 .ldo_process_config = lov_process_config,
503 static struct lu_device *lov_device_alloc(const struct lu_env *env,
504 struct lu_device_type *t,
505 struct lustre_cfg *cfg)
508 struct lov_device *ld;
509 struct obd_device *obd;
514 RETURN(ERR_PTR(-ENOMEM));
516 cl_device_init(&ld->ld_cl, t);
518 d->ld_ops = &lov_lu_ops;
520 /* setup the LOV OBD */
521 obd = class_name2obd(lustre_cfg_string(cfg, 0));
522 LASSERT(obd != NULL);
523 rc = lov_setup(obd, cfg);
527 /* Alloc MDC devices array */
528 /* XXX: need dynamic allocation at some moment */
529 OBD_ALLOC(ld->ld_md_tgts, sizeof(*ld->ld_md_tgts) * LOV_MDC_TGT_MAX);
530 if (ld->ld_md_tgts == NULL)
531 GOTO(out, rc = -ENOMEM);
533 ld->ld_md_tgts_nr = 0;
535 ld->ld_lov = &obd->u.lov;
536 OBD_ALLOC(ld->ld_lov->lov_mdc_tgts,
537 sizeof(*ld->ld_lov->lov_mdc_tgts) * LOV_MDC_TGT_MAX);
538 if (ld->ld_lov->lov_mdc_tgts == NULL)
539 GOTO(out_md_tgts, rc = -ENOMEM);
541 rc = lu_site_init(&ld->ld_site, d);
543 GOTO(out_mdc_tgts, rc);
545 rc = lu_site_init_finish(&ld->ld_site);
551 lu_site_fini(&ld->ld_site);
553 OBD_FREE(ld->ld_lov->lov_mdc_tgts,
554 sizeof(*ld->ld_lov->lov_mdc_tgts) * LOV_MDC_TGT_MAX);
555 ld->ld_lov->lov_mdc_tgts = NULL;
557 OBD_FREE(ld->ld_md_tgts, sizeof(*ld->ld_md_tgts) * LOV_MDC_TGT_MAX);
558 ld->ld_md_tgts = NULL;
565 static const struct lu_device_type_operations lov_device_type_ops = {
566 .ldto_init = lov_type_init,
567 .ldto_fini = lov_type_fini,
569 .ldto_start = lov_type_start,
570 .ldto_stop = lov_type_stop,
572 .ldto_device_alloc = lov_device_alloc,
573 .ldto_device_free = lov_device_free,
575 .ldto_device_init = lov_device_init,
576 .ldto_device_fini = lov_device_fini
579 struct lu_device_type lov_device_type = {
580 .ldt_tags = LU_DEVICE_CL,
581 .ldt_name = LUSTRE_LOV_NAME,
582 .ldt_ops = &lov_device_type_ops,
583 .ldt_ctx_tags = LCT_CL_THREAD