lustre/lov/lov_dev.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * Implementation of cl_device and cl_device_type for LOV layer.
  37  *
  38  *   Author: Nikita Danilov <nikita.danilov@sun.com>
  39  */
  40
  41 #define DEBUG_SUBSYSTEM S_LOV
  42
  43 /* class_name2obd() */
  44 #include <obd_class.h>
  45
  46 #include "lov_cl_internal.h"
  47
  48 cfs_mem_cache_t *lov_page_kmem;
  49 cfs_mem_cache_t *lov_lock_kmem;
  50 cfs_mem_cache_t *lov_object_kmem;
  51 cfs_mem_cache_t *lov_thread_kmem;
  52 cfs_mem_cache_t *lov_session_kmem;
  53 cfs_mem_cache_t *lov_req_kmem;
  54
  55 cfs_mem_cache_t *lovsub_page_kmem;
  56 cfs_mem_cache_t *lovsub_lock_kmem;
  57 cfs_mem_cache_t *lovsub_object_kmem;
  58 cfs_mem_cache_t *lovsub_req_kmem;
  59
  60 cfs_mem_cache_t *lov_lock_link_kmem;
  61
  62 /** Lock class of lov_device::ld_mutex. */
  63 struct lock_class_key cl_lov_device_mutex_class;
  64
  65 struct lu_kmem_descr lov_caches[] = {
  66         {
  67                 .ckd_cache = &lov_page_kmem,
  68                 .ckd_name  = "lov_page_kmem",
  69                 .ckd_size  = sizeof (struct lov_page)
  70         },
  71         {
  72                 .ckd_cache = &lov_lock_kmem,
  73                 .ckd_name  = "lov_lock_kmem",
  74                 .ckd_size  = sizeof (struct lov_lock)
  75         },
  76         {
  77                 .ckd_cache = &lov_object_kmem,
  78                 .ckd_name  = "lov_object_kmem",
  79                 .ckd_size  = sizeof (struct lov_object)
  80         },
  81         {
  82                 .ckd_cache = &lov_thread_kmem,
  83                 .ckd_name  = "lov_thread_kmem",
  84                 .ckd_size  = sizeof (struct lov_thread_info)
  85         },
  86         {
  87                 .ckd_cache = &lov_session_kmem,
  88                 .ckd_name  = "lov_session_kmem",
  89                 .ckd_size  = sizeof (struct lov_session)
  90         },
  91         {
  92                 .ckd_cache = &lov_req_kmem,
  93                 .ckd_name  = "lov_req_kmem",
  94                 .ckd_size  = sizeof (struct lov_req)
  95         },
  96         {
  97                 .ckd_cache = &lovsub_page_kmem,
  98                 .ckd_name  = "lovsub_page_kmem",
  99                 .ckd_size  = sizeof (struct lovsub_page)
 100         },
 101         {
 102                 .ckd_cache = &lovsub_lock_kmem,
 103                 .ckd_name  = "lovsub_lock_kmem",
 104                 .ckd_size  = sizeof (struct lovsub_lock)
 105         },
 106         {
 107                 .ckd_cache = &lovsub_object_kmem,
 108                 .ckd_name  = "lovsub_object_kmem",
 109                 .ckd_size  = sizeof (struct lovsub_object)
 110         },
 111         {
 112                 .ckd_cache = &lovsub_req_kmem,
 113                 .ckd_name  = "lovsub_req_kmem",
 114                 .ckd_size  = sizeof (struct lovsub_req)
 115         },
 116         {
 117                 .ckd_cache = &lov_lock_link_kmem,
 118                 .ckd_name  = "lov_lock_link_kmem",
 119                 .ckd_size  = sizeof (struct lov_lock_link)
 120         },
 121         {
 122                 .ckd_cache = NULL
 123         }
 124 };
 125
 126 /*****************************************************************************
 127  *
 128  * Lov transfer operations.
 129  *
 130  */
 131
 132 static void lov_req_completion(const struct lu_env *env,
 133                                const struct cl_req_slice *slice, int ioret)
 134 {
 135         struct lov_req *lr;
 136
 137         ENTRY;
 138         lr = cl2lov_req(slice);
 139         OBD_SLAB_FREE_PTR(lr, lov_req_kmem);
 140         EXIT;
 141 }
 142
 143 static const struct cl_req_operations lov_req_ops = {
 144         .cro_completion = lov_req_completion
 145 };
 146
 147 /*****************************************************************************
 148  *
 149  * Lov device and device type functions.
 150  *
 151  */
 152
 153 static void *lov_key_init(const struct lu_context *ctx,
 154                           struct lu_context_key *key)
 155 {
 156         struct lov_thread_info *info;
 157
 158         OBD_SLAB_ALLOC_PTR_GFP(info, lov_thread_kmem, CFS_ALLOC_IO);
 159         if (info != NULL)
 160                 CFS_INIT_LIST_HEAD(&info->lti_closure.clc_list);
 161         else
 162                 info = ERR_PTR(-ENOMEM);
 163         return info;
 164 }
 165
 166 static void lov_key_fini(const struct lu_context *ctx,
 167                          struct lu_context_key *key, void *data)
 168 {
 169         struct lov_thread_info *info = data;
 170         LINVRNT(list_empty(&info->lti_closure.clc_list));
 171         OBD_SLAB_FREE_PTR(info, lov_thread_kmem);
 172 }
 173
 174 struct lu_context_key lov_key = {
 175         .lct_tags = LCT_CL_THREAD,
 176         .lct_init = lov_key_init,
 177         .lct_fini = lov_key_fini
 178 };
 179
 180 static void *lov_session_key_init(const struct lu_context *ctx,
 181                                   struct lu_context_key *key)
 182 {
 183         struct lov_session *info;
 184
 185         OBD_SLAB_ALLOC_PTR_GFP(info, lov_session_kmem, CFS_ALLOC_IO);
 186         if (info == NULL)
 187                 info = ERR_PTR(-ENOMEM);
 188         return info;
 189 }
 190
 191 static void lov_session_key_fini(const struct lu_context *ctx,
 192                                  struct lu_context_key *key, void *data)
 193 {
 194         struct lov_session *info = data;
 195         OBD_SLAB_FREE_PTR(info, lov_session_kmem);
 196 }
 197
 198 struct lu_context_key lov_session_key = {
 199         .lct_tags = LCT_SESSION,
 200         .lct_init = lov_session_key_init,
 201         .lct_fini = lov_session_key_fini
 202 };
 203
 204 /* type constructor/destructor: lov_type_{init,fini,start,stop}() */
 205 LU_TYPE_INIT_FINI(lov, &lov_key, &lov_session_key);
 206
 207 static struct lu_device *lov_device_fini(const struct lu_env *env,
 208                                          struct lu_device *d)
 209 {
 210         int i;
 211         struct lov_device *ld = lu2lov_dev(d);
 212
 213         LASSERT(ld->ld_lov != NULL);
 214         if (ld->ld_target == NULL)
 215                 RETURN(NULL);
 216
 217         lov_foreach_target(ld, i) {
 218                 struct lovsub_device *lsd;
 219
 220                 lsd = ld->ld_target[i];
 221                 if (lsd != NULL) {
 222                         cl_stack_fini(env, lovsub2cl_dev(lsd));
 223                         ld->ld_target[i] = NULL;
 224                 }
 225         }
 226         RETURN(NULL);
 227 }
 228
 229 static int lov_device_init(const struct lu_env *env, struct lu_device *d,
 230                            const char *name, struct lu_device *next)
 231 {
 232         struct lov_device *ld = lu2lov_dev(d);
 233         int i;
 234         int rc = 0;
 235
 236         LASSERT(d->ld_site != NULL);
 237         if (ld->ld_target == NULL)
 238                 RETURN(rc);
 239
 240         lov_foreach_target(ld, i) {
 241                 struct lovsub_device *lsd;
 242                 struct cl_device     *cl;
 243                 struct lov_tgt_desc  *desc;
 244
 245                 desc = ld->ld_lov->lov_tgts[i];
 246                 if (desc == NULL)
 247                         continue;
 248
 249                 cl = cl_type_setup(env, d->ld_site, &lovsub_device_type,
 250                                    desc->ltd_obd->obd_lu_dev);
 251                 if (IS_ERR(cl)) {
 252                         rc = PTR_ERR(cl);
 253                         break;
 254                 }
 255                 lsd = cl2lovsub_dev(cl);
 256                 lsd->acid_idx = i;
 257                 lsd->acid_super = ld;
 258                 ld->ld_target[i] = lsd;
 259         }
 260
 261         if (rc)
 262                 lov_device_fini(env, d);
 263         else
 264                 ld->ld_flags |= LOV_DEV_INITIALIZED;
 265
 266         RETURN(rc);
 267 }
 268
 269 static int lov_req_init(const struct lu_env *env, struct cl_device *dev,
 270                         struct cl_req *req)
 271 {
 272         struct lov_req *lr;
 273         int result;
 274
 275         ENTRY;
 276         OBD_SLAB_ALLOC_PTR_GFP(lr, lov_req_kmem, CFS_ALLOC_IO);
 277         if (lr != NULL) {
 278                 cl_req_slice_add(req, &lr->lr_cl, dev, &lov_req_ops);
 279                 result = 0;
 280         } else
 281                 result = -ENOMEM;
 282         RETURN(result);
 283 }
 284
 285 static const struct cl_device_operations lov_cl_ops = {
 286         .cdo_req_init = lov_req_init
 287 };
 288
 289 static void lov_emerg_free(struct lov_device_emerg **emrg, int nr)
 290 {
 291         int i;
 292
 293         for (i = 0; i < nr; ++i) {
 294                 struct lov_device_emerg *em;
 295
 296                 em = emrg[i];
 297                 if (em != NULL) {
 298                         LASSERT(em->emrg_page_list.pl_nr == 0);
 299                         if (em->emrg_env != NULL)
 300                                 cl_env_put(em->emrg_env, &em->emrg_refcheck);
 301                         OBD_FREE_PTR(em);
 302                 }
 303         }
 304         OBD_FREE(emrg, nr * sizeof emrg[0]);
 305 }
 306
 307 static struct lu_device *lov_device_free(const struct lu_env *env,
 308                                          struct lu_device *d)
 309 {
 310         struct lov_device *ld = lu2lov_dev(d);
 311         const int          nr = ld->ld_target_nr;
 312
 313         cl_device_fini(lu2cl_dev(d));
 314         if (ld->ld_target != NULL)
 315                 OBD_FREE(ld->ld_target, nr * sizeof ld->ld_target[0]);
 316         if (ld->ld_emrg != NULL)
 317                 lov_emerg_free(ld->ld_emrg, nr);
 318         OBD_FREE_PTR(ld);
 319         return NULL;
 320 }
 321
 322 static void lov_cl_del_target(const struct lu_env *env, struct lu_device *dev,
 323                               __u32 index)
 324 {
 325         struct lov_device *ld = lu2lov_dev(dev);
 326         ENTRY;
 327
 328         if (ld->ld_target[index] != NULL) {
 329                 cl_stack_fini(env, lovsub2cl_dev(ld->ld_target[index]));
 330                 ld->ld_target[index] = NULL;
 331         }
 332         EXIT;
 333 }
 334
 335 static struct lov_device_emerg **lov_emerg_alloc(int nr)
 336 {
 337         struct lov_device_emerg **emerg;
 338         int i;
 339         int result;
 340
 341         OBD_ALLOC(emerg, nr * sizeof emerg[0]);
 342         if (emerg == NULL)
 343                 return ERR_PTR(-ENOMEM);
 344         for (result = i = 0; i < nr && result == 0; i++) {
 345                 struct lov_device_emerg *em;
 346                 void *cookie;
 347
 348                 OBD_ALLOC_PTR(em);
 349                 if (em != NULL) {
 350                         emerg[i] = em;
 351                         cl_page_list_init(&em->emrg_page_list);
 352                         cookie = cl_env_reenter();
 353                         em->emrg_env = cl_env_alloc(&em->emrg_refcheck,
 354                                                     LCT_REMEMBER|LCT_NOREF);
 355                         cl_env_reexit(cookie);
 356                         if (!IS_ERR(em->emrg_env))
 357                                 em->emrg_env->le_ctx.lc_cookie = 0x2;
 358                         else {
 359                                 result = PTR_ERR(em->emrg_env);
 360                                 em->emrg_env = NULL;
 361                         }
 362                 } else
 363                         result = -ENOMEM;
 364         }
 365         if (result != 0) {
 366                 lov_emerg_free(emerg, nr);
 367                 emerg = ERR_PTR(result);
 368         }
 369         return emerg;
 370 }
 371
 372 static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev)
 373 {
 374         int   result;
 375         __u32 tgt_size;
 376         __u32 sub_size;
 377
 378         ENTRY;
 379         result = 0;
 380         tgt_size = dev->ld_lov->lov_tgt_size;
 381         sub_size = dev->ld_target_nr;
 382         if (sub_size < tgt_size) {
 383                 struct lovsub_device    **newd;
 384                 struct lov_device_emerg **emerg;
 385                 const size_t              sz   = sizeof newd[0];
 386
 387                 emerg = lov_emerg_alloc(tgt_size);
 388                 if (IS_ERR(emerg))
 389                         RETURN(PTR_ERR(emerg));
 390
 391                 OBD_ALLOC(newd, tgt_size * sz);
 392                 if (newd != NULL) {
 393                         mutex_lock(&dev->ld_mutex);
 394                         if (sub_size > 0) {
 395                                 memcpy(newd, dev->ld_target, sub_size * sz);
 396                                 OBD_FREE(dev->ld_target, sub_size * sz);
 397                         }
 398                         dev->ld_target    = newd;
 399                         dev->ld_target_nr = tgt_size;
 400
 401                         if (dev->ld_emrg != NULL)
 402                                 lov_emerg_free(dev->ld_emrg, sub_size);
 403                         dev->ld_emrg = emerg;
 404                         mutex_unlock(&dev->ld_mutex);
 405                 } else {
 406                         lov_emerg_free(emerg, tgt_size);
 407                         result = -ENOMEM;
 408                 }
 409         }
 410         RETURN(result);
 411 }
 412
 413 static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
 414                              __u32 index)
 415 {
 416         struct obd_device    *obd = dev->ld_obd;
 417         struct lov_device    *ld  = lu2lov_dev(dev);
 418         struct lov_tgt_desc  *tgt;
 419         struct lovsub_device *lsd;
 420         struct cl_device     *cl;
 421         int rc;
 422         ENTRY;
 423
 424         obd_getref(obd);
 425
 426         tgt = obd->u.lov.lov_tgts[index];
 427         LASSERT(tgt != NULL);
 428         LASSERT(tgt->ltd_obd != NULL);
 429
 430         if (!tgt->ltd_obd->obd_set_up) {
 431                 CERROR("Target %s not set up\n", obd_uuid2str(&tgt->ltd_uuid));
 432                 RETURN(-EINVAL);
 433         }
 434
 435         rc = lov_expand_targets(env, ld);
 436         if (rc == 0 && ld->ld_flags & LOV_DEV_INITIALIZED) {
 437                 LASSERT(dev->ld_site != NULL);
 438
 439                 cl = cl_type_setup(env, dev->ld_site, &lovsub_device_type,
 440                                    tgt->ltd_obd->obd_lu_dev);
 441                 if (!IS_ERR(cl)) {
 442                         lsd = cl2lovsub_dev(cl);
 443                         lsd->acid_idx = index;
 444                         lsd->acid_super = ld;
 445                         ld->ld_target[index] = lsd;
 446                 } else {
 447                         CERROR("add failed (%d), deleting %s\n", rc,
 448                                obd_uuid2str(&tgt->ltd_uuid));
 449                         lov_cl_del_target(env, dev, index);
 450                         rc = PTR_ERR(cl);
 451                 }
 452         }
 453         obd_putref(obd);
 454         RETURN(rc);
 455 }
 456
 457 static int lov_process_config(const struct lu_env *env,
 458                               struct lu_device *d, struct lustre_cfg *cfg)
 459 {
 460         struct obd_device *obd = d->ld_obd;
 461         int cmd;
 462         int rc;
 463         int gen;
 464         __u32 index;
 465
 466         obd_getref(obd);
 467
 468         cmd = cfg->lcfg_command;
 469         rc = lov_process_config_base(d->ld_obd, cfg, &index, &gen);
 470         if (rc == 0) {
 471                 switch(cmd) {
 472                 case LCFG_LOV_ADD_OBD:
 473                 case LCFG_LOV_ADD_INA:
 474                         rc = lov_cl_add_target(env, d, index);
 475                         if (rc != 0)
 476                                 lov_del_target(d->ld_obd, index, 0, 0);
 477                         break;
 478                 case LCFG_LOV_DEL_OBD:
 479                         lov_cl_del_target(env, d, index);
 480                         break;
 481                 }
 482         }
 483         obd_putref(obd);
 484         RETURN(rc);
 485 }
 486
 487 static const struct lu_device_operations lov_lu_ops = {
 488         .ldo_object_alloc      = lov_object_alloc,
 489         .ldo_process_config    = lov_process_config,
 490 };
 491
 492 static struct lu_device *lov_device_alloc(const struct lu_env *env,
 493                                           struct lu_device_type *t,
 494                                           struct lustre_cfg *cfg)
 495 {
 496         struct lu_device *d;
 497         struct lov_device *ld;
 498         struct obd_device *obd;
 499         int rc;
 500
 501         OBD_ALLOC_PTR(ld);
 502         if (ld == NULL)
 503                 RETURN(ERR_PTR(-ENOMEM));
 504
 505         cl_device_init(&ld->ld_cl, t);
 506         d = lov2lu_dev(ld);
 507         d->ld_ops        = &lov_lu_ops;
 508         ld->ld_cl.cd_ops = &lov_cl_ops;
 509
 510         mutex_init(&ld->ld_mutex);
 511         lockdep_set_class(&ld->ld_mutex, &cl_lov_device_mutex_class);
 512
 513         /* setup the LOV OBD */
 514         obd = class_name2obd(lustre_cfg_string(cfg, 0));
 515         LASSERT(obd != NULL);
 516         rc = lov_setup(obd, cfg);
 517         if (rc) {
 518                 lov_device_free(env, d);
 519                 RETURN(ERR_PTR(rc));
 520         }
 521
 522         ld->ld_lov = &obd->u.lov;
 523         RETURN(d);
 524 }
 525
 526 static const struct lu_device_type_operations lov_device_type_ops = {
 527         .ldto_init = lov_type_init,
 528         .ldto_fini = lov_type_fini,
 529
 530         .ldto_start = lov_type_start,
 531         .ldto_stop  = lov_type_stop,
 532
 533         .ldto_device_alloc = lov_device_alloc,
 534         .ldto_device_free  = lov_device_free,
 535
 536         .ldto_device_init    = lov_device_init,
 537         .ldto_device_fini    = lov_device_fini
 538 };
 539
 540 struct lu_device_type lov_device_type = {
 541         .ldt_tags     = LU_DEVICE_CL,
 542         .ldt_name     = LUSTRE_LOV_NAME,
 543         .ldt_ops      = &lov_device_type_ops,
 544         .ldt_ctx_tags = LCT_CL_THREAD
 545 };
 546 EXPORT_SYMBOL(lov_device_type);
 547
 548 /** @} lov */