lustre/lov/lov_dev.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  */
  30 /*
  31  * This file is part of Lustre, http://www.lustre.org/
  32  * Lustre is a trademark of Sun Microsystems, Inc.
  33  *
  34  * Implementation of cl_device and cl_device_type for LOV layer.
  35  *
  36  *   Author: Nikita Danilov <nikita.danilov@sun.com>
  37  */
  38
  39 #define DEBUG_SUBSYSTEM S_LOV
  40
  41 /* class_name2obd() */
  42 #include <obd_class.h>
  43
  44 #include "lov_cl_internal.h"
  45
  46 cfs_mem_cache_t *lov_page_kmem;
  47 cfs_mem_cache_t *lov_lock_kmem;
  48 cfs_mem_cache_t *lov_object_kmem;
  49 cfs_mem_cache_t *lov_thread_kmem;
  50 cfs_mem_cache_t *lov_session_kmem;
  51 cfs_mem_cache_t *lov_req_kmem;
  52
  53 cfs_mem_cache_t *lovsub_page_kmem;
  54 cfs_mem_cache_t *lovsub_lock_kmem;
  55 cfs_mem_cache_t *lovsub_object_kmem;
  56 cfs_mem_cache_t *lovsub_req_kmem;
  57
  58 cfs_mem_cache_t *lov_lock_link_kmem;
  59
  60 /** Lock class of lov_device::ld_mutex. */
  61 cfs_lock_class_key_t cl_lov_device_mutex_class;
  62
  63 struct lu_kmem_descr lov_caches[] = {
  64         {
  65                 .ckd_cache = &lov_page_kmem,
  66                 .ckd_name  = "lov_page_kmem",
  67                 .ckd_size  = sizeof (struct lov_page)
  68         },
  69         {
  70                 .ckd_cache = &lov_lock_kmem,
  71                 .ckd_name  = "lov_lock_kmem",
  72                 .ckd_size  = sizeof (struct lov_lock)
  73         },
  74         {
  75                 .ckd_cache = &lov_object_kmem,
  76                 .ckd_name  = "lov_object_kmem",
  77                 .ckd_size  = sizeof (struct lov_object)
  78         },
  79         {
  80                 .ckd_cache = &lov_thread_kmem,
  81                 .ckd_name  = "lov_thread_kmem",
  82                 .ckd_size  = sizeof (struct lov_thread_info)
  83         },
  84         {
  85                 .ckd_cache = &lov_session_kmem,
  86                 .ckd_name  = "lov_session_kmem",
  87                 .ckd_size  = sizeof (struct lov_session)
  88         },
  89         {
  90                 .ckd_cache = &lov_req_kmem,
  91                 .ckd_name  = "lov_req_kmem",
  92                 .ckd_size  = sizeof (struct lov_req)
  93         },
  94         {
  95                 .ckd_cache = &lovsub_page_kmem,
  96                 .ckd_name  = "lovsub_page_kmem",
  97                 .ckd_size  = sizeof (struct lovsub_page)
  98         },
  99         {
 100                 .ckd_cache = &lovsub_lock_kmem,
 101                 .ckd_name  = "lovsub_lock_kmem",
 102                 .ckd_size  = sizeof (struct lovsub_lock)
 103         },
 104         {
 105                 .ckd_cache = &lovsub_object_kmem,
 106                 .ckd_name  = "lovsub_object_kmem",
 107                 .ckd_size  = sizeof (struct lovsub_object)
 108         },
 109         {
 110                 .ckd_cache = &lovsub_req_kmem,
 111                 .ckd_name  = "lovsub_req_kmem",
 112                 .ckd_size  = sizeof (struct lovsub_req)
 113         },
 114         {
 115                 .ckd_cache = &lov_lock_link_kmem,
 116                 .ckd_name  = "lov_lock_link_kmem",
 117                 .ckd_size  = sizeof (struct lov_lock_link)
 118         },
 119         {
 120                 .ckd_cache = NULL
 121         }
 122 };
 123
 124 /*****************************************************************************
 125  *
 126  * Lov transfer operations.
 127  *
 128  */
 129
 130 static void lov_req_completion(const struct lu_env *env,
 131                                const struct cl_req_slice *slice, int ioret)
 132 {
 133         struct lov_req *lr;
 134
 135         ENTRY;
 136         lr = cl2lov_req(slice);
 137         OBD_SLAB_FREE_PTR(lr, lov_req_kmem);
 138         EXIT;
 139 }
 140
 141 static const struct cl_req_operations lov_req_ops = {
 142         .cro_completion = lov_req_completion
 143 };
 144
 145 /*****************************************************************************
 146  *
 147  * Lov device and device type functions.
 148  *
 149  */
 150
 151 static void *lov_key_init(const struct lu_context *ctx,
 152                           struct lu_context_key *key)
 153 {
 154         struct lov_thread_info *info;
 155
 156         OBD_SLAB_ALLOC_PTR_GFP(info, lov_thread_kmem, CFS_ALLOC_IO);
 157         if (info != NULL)
 158                 CFS_INIT_LIST_HEAD(&info->lti_closure.clc_list);
 159         else
 160                 info = ERR_PTR(-ENOMEM);
 161         return info;
 162 }
 163
 164 static void lov_key_fini(const struct lu_context *ctx,
 165                          struct lu_context_key *key, void *data)
 166 {
 167         struct lov_thread_info *info = data;
 168         LINVRNT(cfs_list_empty(&info->lti_closure.clc_list));
 169         OBD_SLAB_FREE_PTR(info, lov_thread_kmem);
 170 }
 171
 172 struct lu_context_key lov_key = {
 173         .lct_tags = LCT_CL_THREAD,
 174         .lct_init = lov_key_init,
 175         .lct_fini = lov_key_fini
 176 };
 177
 178 static void *lov_session_key_init(const struct lu_context *ctx,
 179                                   struct lu_context_key *key)
 180 {
 181         struct lov_session *info;
 182
 183         OBD_SLAB_ALLOC_PTR_GFP(info, lov_session_kmem, CFS_ALLOC_IO);
 184         if (info == NULL)
 185                 info = ERR_PTR(-ENOMEM);
 186         return info;
 187 }
 188
 189 static void lov_session_key_fini(const struct lu_context *ctx,
 190                                  struct lu_context_key *key, void *data)
 191 {
 192         struct lov_session *info = data;
 193         OBD_SLAB_FREE_PTR(info, lov_session_kmem);
 194 }
 195
 196 struct lu_context_key lov_session_key = {
 197         .lct_tags = LCT_SESSION,
 198         .lct_init = lov_session_key_init,
 199         .lct_fini = lov_session_key_fini
 200 };
 201
 202 /* type constructor/destructor: lov_type_{init,fini,start,stop}() */
 203 LU_TYPE_INIT_FINI(lov, &lov_key, &lov_session_key);
 204
 205 static struct lu_device *lov_device_fini(const struct lu_env *env,
 206                                          struct lu_device *d)
 207 {
 208         int i;
 209         struct lov_device *ld = lu2lov_dev(d);
 210
 211         LASSERT(ld->ld_lov != NULL);
 212         if (ld->ld_target == NULL)
 213                 RETURN(NULL);
 214
 215         lov_foreach_target(ld, i) {
 216                 struct lovsub_device *lsd;
 217
 218                 lsd = ld->ld_target[i];
 219                 if (lsd != NULL) {
 220                         cl_stack_fini(env, lovsub2cl_dev(lsd));
 221                         ld->ld_target[i] = NULL;
 222                 }
 223         }
 224         RETURN(NULL);
 225 }
 226
 227 static int lov_device_init(const struct lu_env *env, struct lu_device *d,
 228                            const char *name, struct lu_device *next)
 229 {
 230         struct lov_device *ld = lu2lov_dev(d);
 231         int i;
 232         int rc = 0;
 233
 234         LASSERT(d->ld_site != NULL);
 235         if (ld->ld_target == NULL)
 236                 RETURN(rc);
 237
 238         lov_foreach_target(ld, i) {
 239                 struct lovsub_device *lsd;
 240                 struct cl_device     *cl;
 241                 struct lov_tgt_desc  *desc;
 242
 243                 desc = ld->ld_lov->lov_tgts[i];
 244                 if (desc == NULL)
 245                         continue;
 246
 247                 cl = cl_type_setup(env, d->ld_site, &lovsub_device_type,
 248                                    desc->ltd_obd->obd_lu_dev);
 249                 if (IS_ERR(cl)) {
 250                         rc = PTR_ERR(cl);
 251                         break;
 252                 }
 253                 lsd = cl2lovsub_dev(cl);
 254                 lsd->acid_idx = i;
 255                 lsd->acid_super = ld;
 256                 ld->ld_target[i] = lsd;
 257         }
 258
 259         if (rc)
 260                 lov_device_fini(env, d);
 261         else
 262                 ld->ld_flags |= LOV_DEV_INITIALIZED;
 263
 264         RETURN(rc);
 265 }
 266
 267 static int lov_req_init(const struct lu_env *env, struct cl_device *dev,
 268                         struct cl_req *req)
 269 {
 270         struct lov_req *lr;
 271         int result;
 272
 273         ENTRY;
 274         OBD_SLAB_ALLOC_PTR_GFP(lr, lov_req_kmem, CFS_ALLOC_IO);
 275         if (lr != NULL) {
 276                 cl_req_slice_add(req, &lr->lr_cl, dev, &lov_req_ops);
 277                 result = 0;
 278         } else
 279                 result = -ENOMEM;
 280         RETURN(result);
 281 }
 282
 283 static const struct cl_device_operations lov_cl_ops = {
 284         .cdo_req_init = lov_req_init
 285 };
 286
 287 static void lov_emerg_free(struct lov_device_emerg **emrg, int nr)
 288 {
 289         int i;
 290
 291         for (i = 0; i < nr; ++i) {
 292                 struct lov_device_emerg *em;
 293
 294                 em = emrg[i];
 295                 if (em != NULL) {
 296                         LASSERT(em->emrg_page_list.pl_nr == 0);
 297                         if (em->emrg_env != NULL)
 298                                 cl_env_put(em->emrg_env, &em->emrg_refcheck);
 299                         OBD_FREE_PTR(em);
 300                 }
 301         }
 302         OBD_FREE(emrg, nr * sizeof emrg[0]);
 303 }
 304
 305 static struct lu_device *lov_device_free(const struct lu_env *env,
 306                                          struct lu_device *d)
 307 {
 308         struct lov_device *ld = lu2lov_dev(d);
 309         const int          nr = ld->ld_target_nr;
 310
 311         cl_device_fini(lu2cl_dev(d));
 312         if (ld->ld_target != NULL)
 313                 OBD_FREE(ld->ld_target, nr * sizeof ld->ld_target[0]);
 314         if (ld->ld_emrg != NULL)
 315                 lov_emerg_free(ld->ld_emrg, nr);
 316         OBD_FREE_PTR(ld);
 317         return NULL;
 318 }
 319
 320 static void lov_cl_del_target(const struct lu_env *env, struct lu_device *dev,
 321                               __u32 index)
 322 {
 323         struct lov_device *ld = lu2lov_dev(dev);
 324         ENTRY;
 325
 326         if (ld->ld_target[index] != NULL) {
 327                 cl_stack_fini(env, lovsub2cl_dev(ld->ld_target[index]));
 328                 ld->ld_target[index] = NULL;
 329         }
 330         EXIT;
 331 }
 332
 333 static struct lov_device_emerg **lov_emerg_alloc(int nr)
 334 {
 335         struct lov_device_emerg **emerg;
 336         int i;
 337         int result;
 338
 339         OBD_ALLOC(emerg, nr * sizeof emerg[0]);
 340         if (emerg == NULL)
 341                 return ERR_PTR(-ENOMEM);
 342         for (result = i = 0; i < nr && result == 0; i++) {
 343                 struct lov_device_emerg *em;
 344
 345                 OBD_ALLOC_PTR(em);
 346                 if (em != NULL) {
 347                         emerg[i] = em;
 348                         cl_page_list_init(&em->emrg_page_list);
 349                         em->emrg_env = cl_env_alloc(&em->emrg_refcheck,
 350                                                     LCT_REMEMBER|LCT_NOREF);
 351                         if (!IS_ERR(em->emrg_env))
 352                                 em->emrg_env->le_ctx.lc_cookie = 0x2;
 353                         else {
 354                                 result = PTR_ERR(em->emrg_env);
 355                                 em->emrg_env = NULL;
 356                         }
 357                 } else
 358                         result = -ENOMEM;
 359         }
 360         if (result != 0) {
 361                 lov_emerg_free(emerg, nr);
 362                 emerg = ERR_PTR(result);
 363         }
 364         return emerg;
 365 }
 366
 367 static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev)
 368 {
 369         int   result;
 370         __u32 tgt_size;
 371         __u32 sub_size;
 372
 373         ENTRY;
 374         result = 0;
 375         tgt_size = dev->ld_lov->lov_tgt_size;
 376         sub_size = dev->ld_target_nr;
 377         if (sub_size < tgt_size) {
 378                 struct lovsub_device    **newd;
 379                 struct lov_device_emerg **emerg;
 380                 const size_t              sz   = sizeof newd[0];
 381
 382                 emerg = lov_emerg_alloc(tgt_size);
 383                 if (IS_ERR(emerg))
 384                         RETURN(PTR_ERR(emerg));
 385
 386                 OBD_ALLOC(newd, tgt_size * sz);
 387                 if (newd != NULL) {
 388                         cfs_mutex_lock(&dev->ld_mutex);
 389                         if (sub_size > 0) {
 390                                 memcpy(newd, dev->ld_target, sub_size * sz);
 391                                 OBD_FREE(dev->ld_target, sub_size * sz);
 392                         }
 393                         dev->ld_target    = newd;
 394                         dev->ld_target_nr = tgt_size;
 395
 396                         if (dev->ld_emrg != NULL)
 397                                 lov_emerg_free(dev->ld_emrg, sub_size);
 398                         dev->ld_emrg = emerg;
 399                         cfs_mutex_unlock(&dev->ld_mutex);
 400                 } else {
 401                         lov_emerg_free(emerg, tgt_size);
 402                         result = -ENOMEM;
 403                 }
 404         }
 405         RETURN(result);
 406 }
 407
 408 static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
 409                              __u32 index)
 410 {
 411         struct obd_device    *obd = dev->ld_obd;
 412         struct lov_device    *ld  = lu2lov_dev(dev);
 413         struct lov_tgt_desc  *tgt;
 414         struct lovsub_device *lsd;
 415         struct cl_device     *cl;
 416         int rc;
 417         ENTRY;
 418
 419         obd_getref(obd);
 420
 421         tgt = obd->u.lov.lov_tgts[index];
 422         LASSERT(tgt != NULL);
 423         LASSERT(tgt->ltd_obd != NULL);
 424
 425         if (!tgt->ltd_obd->obd_set_up) {
 426                 CERROR("Target %s not set up\n", obd_uuid2str(&tgt->ltd_uuid));
 427                 RETURN(-EINVAL);
 428         }
 429
 430         rc = lov_expand_targets(env, ld);
 431         if (rc == 0 && ld->ld_flags & LOV_DEV_INITIALIZED) {
 432                 LASSERT(dev->ld_site != NULL);
 433
 434                 cl = cl_type_setup(env, dev->ld_site, &lovsub_device_type,
 435                                    tgt->ltd_obd->obd_lu_dev);
 436                 if (!IS_ERR(cl)) {
 437                         lsd = cl2lovsub_dev(cl);
 438                         lsd->acid_idx = index;
 439                         lsd->acid_super = ld;
 440                         ld->ld_target[index] = lsd;
 441                 } else {
 442                         CERROR("add failed (%d), deleting %s\n", rc,
 443                                obd_uuid2str(&tgt->ltd_uuid));
 444                         lov_cl_del_target(env, dev, index);
 445                         rc = PTR_ERR(cl);
 446                 }
 447         }
 448         obd_putref(obd);
 449         RETURN(rc);
 450 }
 451
 452 static int lov_process_config(const struct lu_env *env,
 453                               struct lu_device *d, struct lustre_cfg *cfg)
 454 {
 455         struct obd_device *obd = d->ld_obd;
 456         int cmd;
 457         int rc;
 458         int gen;
 459         __u32 index;
 460
 461         obd_getref(obd);
 462
 463         cmd = cfg->lcfg_command;
 464         rc = lov_process_config_base(d->ld_obd, cfg, &index, &gen);
 465         if (rc == 0) {
 466                 switch(cmd) {
 467                 case LCFG_LOV_ADD_OBD:
 468                 case LCFG_LOV_ADD_INA:
 469                         rc = lov_cl_add_target(env, d, index);
 470                         if (rc != 0)
 471                                 lov_del_target(d->ld_obd, index, 0, 0);
 472                         break;
 473                 case LCFG_LOV_DEL_OBD:
 474                         lov_cl_del_target(env, d, index);
 475                         break;
 476                 }
 477         }
 478         obd_putref(obd);
 479         RETURN(rc);
 480 }
 481
 482 static const struct lu_device_operations lov_lu_ops = {
 483         .ldo_object_alloc      = lov_object_alloc,
 484         .ldo_process_config    = lov_process_config,
 485 };
 486
 487 static struct lu_device *lov_device_alloc(const struct lu_env *env,
 488                                           struct lu_device_type *t,
 489                                           struct lustre_cfg *cfg)
 490 {
 491         struct lu_device *d;
 492         struct lov_device *ld;
 493         struct obd_device *obd;
 494         int rc;
 495
 496         OBD_ALLOC_PTR(ld);
 497         if (ld == NULL)
 498                 RETURN(ERR_PTR(-ENOMEM));
 499
 500         cl_device_init(&ld->ld_cl, t);
 501         d = lov2lu_dev(ld);
 502         d->ld_ops        = &lov_lu_ops;
 503         ld->ld_cl.cd_ops = &lov_cl_ops;
 504
 505         cfs_mutex_init(&ld->ld_mutex);
 506         cfs_lockdep_set_class(&ld->ld_mutex, &cl_lov_device_mutex_class);
 507
 508         /* setup the LOV OBD */
 509         obd = class_name2obd(lustre_cfg_string(cfg, 0));
 510         LASSERT(obd != NULL);
 511         rc = lov_setup(obd, cfg);
 512         if (rc) {
 513                 lov_device_free(env, d);
 514                 RETURN(ERR_PTR(rc));
 515         }
 516
 517         ld->ld_lov = &obd->u.lov;
 518         RETURN(d);
 519 }
 520
 521 static const struct lu_device_type_operations lov_device_type_ops = {
 522         .ldto_init = lov_type_init,
 523         .ldto_fini = lov_type_fini,
 524
 525         .ldto_start = lov_type_start,
 526         .ldto_stop  = lov_type_stop,
 527
 528         .ldto_device_alloc = lov_device_alloc,
 529         .ldto_device_free  = lov_device_free,
 530
 531         .ldto_device_init    = lov_device_init,
 532         .ldto_device_fini    = lov_device_fini
 533 };
 534
 535 struct lu_device_type lov_device_type = {
 536         .ldt_tags     = LU_DEVICE_CL,
 537         .ldt_name     = LUSTRE_LOV_NAME,
 538         .ldt_ops      = &lov_device_type_ops,
 539         .ldt_ctx_tags = LCT_CL_THREAD
 540 };
 541 EXPORT_SYMBOL(lov_device_type);
 542
 543 /** @} lov */