lustre/lov/lov_dev.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, 2015, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * Implementation of cl_device and cl_device_type for LOV layer.
  37  *
  38  *   Author: Nikita Danilov <nikita.danilov@sun.com>
  39  */
  40
  41 #define DEBUG_SUBSYSTEM S_LOV
  42
  43 /* class_name2obd() */
  44 #include <obd_class.h>
  45
  46 #include "lov_cl_internal.h"
  47
  48 struct kmem_cache *lov_lock_kmem;
  49 struct kmem_cache *lov_object_kmem;
  50 struct kmem_cache *lov_thread_kmem;
  51 struct kmem_cache *lov_session_kmem;
  52
  53 struct kmem_cache *lovsub_lock_kmem;
  54 struct kmem_cache *lovsub_object_kmem;
  55
  56 struct kmem_cache *lov_lock_link_kmem;
  57
  58 /** Lock class of lov_device::ld_mutex. */
  59 static struct lock_class_key cl_lov_device_mutex_class;
  60
  61 struct lu_kmem_descr lov_caches[] = {
  62         {
  63                 .ckd_cache = &lov_lock_kmem,
  64                 .ckd_name  = "lov_lock_kmem",
  65                 .ckd_size  = sizeof (struct lov_lock)
  66         },
  67         {
  68                 .ckd_cache = &lov_object_kmem,
  69                 .ckd_name  = "lov_object_kmem",
  70                 .ckd_size  = sizeof (struct lov_object)
  71         },
  72         {
  73                 .ckd_cache = &lov_thread_kmem,
  74                 .ckd_name  = "lov_thread_kmem",
  75                 .ckd_size  = sizeof (struct lov_thread_info)
  76         },
  77         {
  78                 .ckd_cache = &lov_session_kmem,
  79                 .ckd_name  = "lov_session_kmem",
  80                 .ckd_size  = sizeof (struct lov_session)
  81         },
  82         {
  83                 .ckd_cache = &lovsub_lock_kmem,
  84                 .ckd_name  = "lovsub_lock_kmem",
  85                 .ckd_size  = sizeof (struct lovsub_lock)
  86         },
  87         {
  88                 .ckd_cache = &lovsub_object_kmem,
  89                 .ckd_name  = "lovsub_object_kmem",
  90                 .ckd_size  = sizeof (struct lovsub_object)
  91         },
  92         {
  93                 .ckd_cache = &lov_lock_link_kmem,
  94                 .ckd_name  = "lov_lock_link_kmem",
  95                 .ckd_size  = sizeof (struct lov_lock_link)
  96         },
  97         {
  98                 .ckd_cache = NULL
  99         }
 100 };
 101
 102 /*****************************************************************************
 103  *
 104  * Lov device and device type functions.
 105  *
 106  */
 107
 108 static void *lov_key_init(const struct lu_context *ctx,
 109                           struct lu_context_key *key)
 110 {
 111         struct lov_thread_info *info;
 112
 113         OBD_SLAB_ALLOC_PTR_GFP(info, lov_thread_kmem, GFP_NOFS);
 114         if (info == NULL)
 115                 info = ERR_PTR(-ENOMEM);
 116         return info;
 117 }
 118
 119 static void lov_key_fini(const struct lu_context *ctx,
 120                          struct lu_context_key *key, void *data)
 121 {
 122         struct lov_thread_info *info = data;
 123         OBD_SLAB_FREE_PTR(info, lov_thread_kmem);
 124 }
 125
 126 struct lu_context_key lov_key = {
 127         .lct_tags = LCT_CL_THREAD,
 128         .lct_init = lov_key_init,
 129         .lct_fini = lov_key_fini
 130 };
 131
 132 static void *lov_session_key_init(const struct lu_context *ctx,
 133                                   struct lu_context_key *key)
 134 {
 135         struct lov_session *info;
 136
 137         OBD_SLAB_ALLOC_PTR_GFP(info, lov_session_kmem, GFP_NOFS);
 138         if (info == NULL)
 139                 info = ERR_PTR(-ENOMEM);
 140         return info;
 141 }
 142
 143 static void lov_session_key_fini(const struct lu_context *ctx,
 144                                  struct lu_context_key *key, void *data)
 145 {
 146         struct lov_session *info = data;
 147         OBD_SLAB_FREE_PTR(info, lov_session_kmem);
 148 }
 149
 150 struct lu_context_key lov_session_key = {
 151         .lct_tags = LCT_SESSION,
 152         .lct_init = lov_session_key_init,
 153         .lct_fini = lov_session_key_fini
 154 };
 155
 156 /* type constructor/destructor: lov_type_{init,fini,start,stop}() */
 157 LU_TYPE_INIT_FINI(lov, &lov_key, &lov_session_key);
 158
 159 static struct lu_device *lov_device_fini(const struct lu_env *env,
 160                                          struct lu_device *d)
 161 {
 162         int i;
 163         struct lov_device *ld = lu2lov_dev(d);
 164
 165         LASSERT(ld->ld_lov != NULL);
 166         if (ld->ld_target == NULL)
 167                 RETURN(NULL);
 168
 169         lov_foreach_target(ld, i) {
 170                 struct lovsub_device *lsd;
 171
 172                 lsd = ld->ld_target[i];
 173                 if (lsd != NULL) {
 174                         cl_stack_fini(env, lovsub2cl_dev(lsd));
 175                         ld->ld_target[i] = NULL;
 176                 }
 177         }
 178         RETURN(NULL);
 179 }
 180
 181 static int lov_device_init(const struct lu_env *env, struct lu_device *d,
 182                            const char *name, struct lu_device *next)
 183 {
 184         struct lov_device *ld = lu2lov_dev(d);
 185         int i;
 186         int rc = 0;
 187
 188         LASSERT(d->ld_site != NULL);
 189         if (ld->ld_target == NULL)
 190                 RETURN(rc);
 191
 192         lov_foreach_target(ld, i) {
 193                 struct lovsub_device *lsd;
 194                 struct cl_device     *cl;
 195                 struct lov_tgt_desc  *desc;
 196
 197                 desc = ld->ld_lov->lov_tgts[i];
 198                 if (desc == NULL)
 199                         continue;
 200
 201                 cl = cl_type_setup(env, d->ld_site, &lovsub_device_type,
 202                                    desc->ltd_obd->obd_lu_dev);
 203                 if (IS_ERR(cl)) {
 204                         rc = PTR_ERR(cl);
 205                         break;
 206                 }
 207                 lsd = cl2lovsub_dev(cl);
 208                 lsd->acid_idx = i;
 209                 lsd->acid_super = ld;
 210                 ld->ld_target[i] = lsd;
 211         }
 212
 213         if (rc)
 214                 lov_device_fini(env, d);
 215         else
 216                 ld->ld_flags |= LOV_DEV_INITIALIZED;
 217
 218         RETURN(rc);
 219 }
 220
 221 static void lov_emerg_free(struct lov_device_emerg **emrg, int nr)
 222 {
 223         int i;
 224
 225         for (i = 0; i < nr; ++i) {
 226                 struct lov_device_emerg *em;
 227
 228                 em = emrg[i];
 229                 if (em != NULL) {
 230                         LASSERT(em->emrg_page_list.pl_nr == 0);
 231                         if (em->emrg_env != NULL)
 232                                 cl_env_put(em->emrg_env, &em->emrg_refcheck);
 233                         OBD_FREE_PTR(em);
 234                 }
 235         }
 236         OBD_FREE(emrg, nr * sizeof emrg[0]);
 237 }
 238
 239 static struct lu_device *lov_device_free(const struct lu_env *env,
 240                                          struct lu_device *d)
 241 {
 242         struct lov_device *ld = lu2lov_dev(d);
 243         const int          nr = ld->ld_target_nr;
 244
 245         cl_device_fini(lu2cl_dev(d));
 246         if (ld->ld_target != NULL)
 247                 OBD_FREE(ld->ld_target, nr * sizeof ld->ld_target[0]);
 248         if (ld->ld_emrg != NULL)
 249                 lov_emerg_free(ld->ld_emrg, nr);
 250         OBD_FREE_PTR(ld);
 251         return NULL;
 252 }
 253
 254 static void lov_cl_del_target(const struct lu_env *env, struct lu_device *dev,
 255                               __u32 index)
 256 {
 257         struct lov_device *ld = lu2lov_dev(dev);
 258         ENTRY;
 259
 260         if (ld->ld_target[index] != NULL) {
 261                 cl_stack_fini(env, lovsub2cl_dev(ld->ld_target[index]));
 262                 ld->ld_target[index] = NULL;
 263         }
 264         EXIT;
 265 }
 266
 267 static struct lov_device_emerg **lov_emerg_alloc(int nr)
 268 {
 269         struct lov_device_emerg **emerg;
 270         int i;
 271         int result;
 272
 273         OBD_ALLOC(emerg, nr * sizeof emerg[0]);
 274         if (emerg == NULL)
 275                 return ERR_PTR(-ENOMEM);
 276         for (result = i = 0; i < nr && result == 0; i++) {
 277                 struct lov_device_emerg *em;
 278
 279                 OBD_ALLOC_PTR(em);
 280                 if (em != NULL) {
 281                         emerg[i] = em;
 282                         cl_page_list_init(&em->emrg_page_list);
 283                         em->emrg_env = cl_env_alloc(&em->emrg_refcheck,
 284                                                     LCT_REMEMBER|LCT_NOREF);
 285                         if (!IS_ERR(em->emrg_env))
 286                                 em->emrg_env->le_ctx.lc_cookie = 0x2;
 287                         else {
 288                                 result = PTR_ERR(em->emrg_env);
 289                                 em->emrg_env = NULL;
 290                         }
 291                 } else
 292                         result = -ENOMEM;
 293         }
 294         if (result != 0) {
 295                 lov_emerg_free(emerg, nr);
 296                 emerg = ERR_PTR(result);
 297         }
 298         return emerg;
 299 }
 300
 301 static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev)
 302 {
 303         int   result;
 304         __u32 tgt_size;
 305         __u32 sub_size;
 306
 307         ENTRY;
 308         result = 0;
 309         tgt_size = dev->ld_lov->lov_tgt_size;
 310         sub_size = dev->ld_target_nr;
 311         if (sub_size < tgt_size) {
 312                 struct lovsub_device    **newd;
 313                 struct lov_device_emerg **emerg;
 314                 const size_t              sz   = sizeof newd[0];
 315
 316                 emerg = lov_emerg_alloc(tgt_size);
 317                 if (IS_ERR(emerg))
 318                         RETURN(PTR_ERR(emerg));
 319
 320                 OBD_ALLOC(newd, tgt_size * sz);
 321                 if (newd != NULL) {
 322                         mutex_lock(&dev->ld_mutex);
 323                         if (sub_size > 0) {
 324                                 memcpy(newd, dev->ld_target, sub_size * sz);
 325                                 OBD_FREE(dev->ld_target, sub_size * sz);
 326                         }
 327                         dev->ld_target    = newd;
 328                         dev->ld_target_nr = tgt_size;
 329
 330                         if (dev->ld_emrg != NULL)
 331                                 lov_emerg_free(dev->ld_emrg, sub_size);
 332                         dev->ld_emrg = emerg;
 333                         mutex_unlock(&dev->ld_mutex);
 334                 } else {
 335                         lov_emerg_free(emerg, tgt_size);
 336                         result = -ENOMEM;
 337                 }
 338         }
 339         RETURN(result);
 340 }
 341
 342 static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
 343                              __u32 index)
 344 {
 345         struct obd_device    *obd = dev->ld_obd;
 346         struct lov_device    *ld  = lu2lov_dev(dev);
 347         struct lov_tgt_desc  *tgt;
 348         struct lovsub_device *lsd;
 349         struct cl_device     *cl;
 350         int rc;
 351         ENTRY;
 352
 353         obd_getref(obd);
 354
 355         tgt = obd->u.lov.lov_tgts[index];
 356         LASSERT(tgt != NULL);
 357         LASSERT(tgt->ltd_obd != NULL);
 358
 359         if (!tgt->ltd_obd->obd_set_up) {
 360                 CERROR("Target %s not set up\n", obd_uuid2str(&tgt->ltd_uuid));
 361                 RETURN(-EINVAL);
 362         }
 363
 364         rc = lov_expand_targets(env, ld);
 365         if (rc == 0 && ld->ld_flags & LOV_DEV_INITIALIZED) {
 366                 LASSERT(dev->ld_site != NULL);
 367
 368                 cl = cl_type_setup(env, dev->ld_site, &lovsub_device_type,
 369                                    tgt->ltd_obd->obd_lu_dev);
 370                 if (!IS_ERR(cl)) {
 371                         lsd = cl2lovsub_dev(cl);
 372                         lsd->acid_idx = index;
 373                         lsd->acid_super = ld;
 374                         ld->ld_target[index] = lsd;
 375                 } else {
 376                         CERROR("add failed (%d), deleting %s\n", rc,
 377                                obd_uuid2str(&tgt->ltd_uuid));
 378                         lov_cl_del_target(env, dev, index);
 379                         rc = PTR_ERR(cl);
 380                 }
 381         }
 382         obd_putref(obd);
 383         RETURN(rc);
 384 }
 385
 386 static int lov_process_config(const struct lu_env *env,
 387                               struct lu_device *d, struct lustre_cfg *cfg)
 388 {
 389         struct obd_device *obd = d->ld_obd;
 390         int cmd;
 391         int rc;
 392         int gen;
 393         __u32 index;
 394
 395         obd_getref(obd);
 396
 397         cmd = cfg->lcfg_command;
 398         rc = lov_process_config_base(d->ld_obd, cfg, &index, &gen);
 399         if (rc == 0) {
 400                 switch(cmd) {
 401                 case LCFG_LOV_ADD_OBD:
 402                 case LCFG_LOV_ADD_INA:
 403                         rc = lov_cl_add_target(env, d, index);
 404                         if (rc != 0)
 405                                 lov_del_target(d->ld_obd, index, NULL, 0);
 406                         break;
 407                 case LCFG_LOV_DEL_OBD:
 408                         lov_cl_del_target(env, d, index);
 409                         break;
 410                 }
 411         }
 412         obd_putref(obd);
 413         RETURN(rc);
 414 }
 415
 416 static const struct lu_device_operations lov_lu_ops = {
 417         .ldo_object_alloc      = lov_object_alloc,
 418         .ldo_process_config    = lov_process_config,
 419 };
 420
 421 static struct lu_device *lov_device_alloc(const struct lu_env *env,
 422                                           struct lu_device_type *t,
 423                                           struct lustre_cfg *cfg)
 424 {
 425         struct lu_device *d;
 426         struct lov_device *ld;
 427         struct obd_device *obd;
 428         int rc;
 429
 430         OBD_ALLOC_PTR(ld);
 431         if (ld == NULL)
 432                 RETURN(ERR_PTR(-ENOMEM));
 433
 434         cl_device_init(&ld->ld_cl, t);
 435         d = lov2lu_dev(ld);
 436         d->ld_ops        = &lov_lu_ops;
 437
 438         mutex_init(&ld->ld_mutex);
 439         lockdep_set_class(&ld->ld_mutex, &cl_lov_device_mutex_class);
 440
 441         /* setup the LOV OBD */
 442         obd = class_name2obd(lustre_cfg_string(cfg, 0));
 443         LASSERT(obd != NULL);
 444         rc = lov_setup(obd, cfg);
 445         if (rc) {
 446                 lov_device_free(env, d);
 447                 RETURN(ERR_PTR(rc));
 448         }
 449
 450         ld->ld_lov = &obd->u.lov;
 451         RETURN(d);
 452 }
 453
 454 static const struct lu_device_type_operations lov_device_type_ops = {
 455         .ldto_init = lov_type_init,
 456         .ldto_fini = lov_type_fini,
 457
 458         .ldto_start = lov_type_start,
 459         .ldto_stop  = lov_type_stop,
 460
 461         .ldto_device_alloc = lov_device_alloc,
 462         .ldto_device_free  = lov_device_free,
 463
 464         .ldto_device_init    = lov_device_init,
 465         .ldto_device_fini    = lov_device_fini
 466 };
 467
 468 struct lu_device_type lov_device_type = {
 469         .ldt_tags     = LU_DEVICE_CL,
 470         .ldt_name     = LUSTRE_LOV_NAME,
 471         .ldt_ops      = &lov_device_type_ops,
 472         .ldt_ctx_tags = LCT_CL_THREAD
 473 };
 474
 475 /** @} lov */