lustre/lov/lov_pool.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2017, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * lustre/lov/lov_pool.c
  33  *
  34  * OST pool methods
  35  *
  36  * Author: Jacques-Charles LAFOUCRIERE <jc.lafoucriere@cea.fr>
  37  * Author: Alex Lyashkov <Alexey.Lyashkov@Sun.COM>
  38  * Author: Nathaniel Rutman <Nathan.Rutman@Sun.COM>
  39  */
  40
  41 #define DEBUG_SUBSYSTEM S_LOV
  42
  43 #include <libcfs/libcfs.h>
  44 #include <libcfs/linux/linux-hash.h>
  45
  46 #include <obd.h>
  47 #include "lov_internal.h"
  48
  49 #define pool_tgt(_p, _i) \
  50                 _p->pool_lobd->u.lov.lov_tgts[_p->pool_obds.op_array[_i]]
  51
  52 static u32 pool_hashfh(const void *data, u32 len, u32 seed)
  53 {
  54         const char *pool_name = data;
  55
  56         return hashlen_hash(cfs_hashlen_string((void *)(unsigned long)seed,
  57                                                pool_name));
  58 }
  59
  60 static int pool_cmpfn(struct rhashtable_compare_arg *arg, const void *obj)
  61 {
  62         const struct pool_desc *pool = obj;
  63         const char *pool_name = arg->key;
  64
  65         return strcmp(pool_name, pool->pool_name);
  66 }
  67
  68 static const struct rhashtable_params pools_hash_params = {
  69         .key_len        = 1, /* actually variable */
  70         .key_offset     = offsetof(struct pool_desc, pool_name),
  71         .head_offset    = offsetof(struct pool_desc, pool_hash),
  72         .hashfn         = pool_hashfh,
  73         .obj_cmpfn      = pool_cmpfn,
  74         .automatic_shrinking = true,
  75 };
  76
  77 static void lov_pool_getref(struct pool_desc *pool)
  78 {
  79         CDEBUG(D_INFO, "pool %p\n", pool);
  80         atomic_inc(&pool->pool_refcount);
  81 }
  82
  83 void lov_pool_putref(struct pool_desc *pool)
  84 {
  85         CDEBUG(D_INFO, "pool %p\n", pool);
  86         if (atomic_dec_and_test(&pool->pool_refcount)) {
  87                 LASSERT(list_empty(&pool->pool_list));
  88                 LASSERT(pool->pool_proc_entry == NULL);
  89                 lov_ost_pool_free(&(pool->pool_obds));
  90                 kfree_rcu(pool, pool_rcu);
  91                 EXIT;
  92         }
  93 }
  94
  95 #ifdef CONFIG_PROC_FS
  96 /*
  97  * pool /proc seq_file methods
  98  */
  99 /*
 100  * iterator is used to go through the target pool entries
 101  * index is the current entry index in the lp_array[] array
 102  * index >= pos returned to the seq_file interface
 103  * pos is from 0 to (pool->pool_obds.op_count - 1)
 104  */
 105 #define POOL_IT_MAGIC 0xB001CEA0
 106 struct pool_iterator {
 107         int magic;
 108         struct pool_desc *pool;
 109         int idx;        /* from 0 to pool_tgt_size - 1 */
 110 };
 111
 112 static void *pool_proc_next(struct seq_file *s, void *v, loff_t *pos)
 113 {
 114         struct pool_iterator *iter = (struct pool_iterator *)s->private;
 115         int prev_idx;
 116
 117         LASSERTF(iter->magic == POOL_IT_MAGIC, "%08X\n", iter->magic);
 118
 119         /* test if end of file */
 120         if (*pos >= pool_tgt_count(iter->pool))
 121                 return NULL;
 122
 123         /* iterate to find a non empty entry */
 124         prev_idx = iter->idx;
 125         iter->idx++;
 126         if (iter->idx >= pool_tgt_count(iter->pool)) {
 127                 iter->idx = prev_idx; /* we stay on the last entry */
 128                 return NULL;
 129         }
 130         (*pos)++;
 131         /* return != NULL to continue */
 132         return iter;
 133 }
 134
 135 static void *pool_proc_start(struct seq_file *s, loff_t *pos)
 136 {
 137         struct pool_desc *pool = (struct pool_desc *)s->private;
 138         struct pool_iterator *iter;
 139
 140         lov_pool_getref(pool);
 141         if ((pool_tgt_count(pool) == 0) ||
 142             (*pos >= pool_tgt_count(pool))) {
 143                 /* iter is not created, so stop() has no way to
 144                  * find pool to dec ref */
 145                 lov_pool_putref(pool);
 146                 return NULL;
 147         }
 148
 149         OBD_ALLOC_PTR(iter);
 150         if (!iter)
 151                 return ERR_PTR(-ENOMEM);
 152         iter->magic = POOL_IT_MAGIC;
 153         iter->pool = pool;
 154         iter->idx = 0;
 155
 156         /* we use seq_file private field to memorized iterator so
 157          * we can free it at stop() */
 158         /* /!\ do not forget to restore it to pool before freeing it */
 159         s->private = iter;
 160         down_read(&pool_tgt_rw_sem(pool));
 161         if (*pos > 0) {
 162                 loff_t i;
 163                 void *ptr;
 164
 165                 i = 0;
 166                 do {
 167                      ptr = pool_proc_next(s, &iter, &i);
 168                 } while ((i < *pos) && (ptr != NULL));
 169                 return ptr;
 170         }
 171         return iter;
 172 }
 173
 174 static void pool_proc_stop(struct seq_file *s, void *v)
 175 {
 176         struct pool_iterator *iter = (struct pool_iterator *)s->private;
 177
 178         /* in some cases stop() method is called 2 times, without
 179          * calling start() method (see seq_read() from fs/seq_file.c)
 180          * we have to free only if s->private is an iterator */
 181         if ((iter) && (iter->magic == POOL_IT_MAGIC)) {
 182                 up_read(&pool_tgt_rw_sem(iter->pool));
 183                 /* we restore s->private so next call to pool_proc_start()
 184                  * will work */
 185                 s->private = iter->pool;
 186                 lov_pool_putref(iter->pool);
 187                 OBD_FREE_PTR(iter);
 188         }
 189 }
 190
 191 static int pool_proc_show(struct seq_file *s, void *v)
 192 {
 193         struct pool_iterator *iter = (struct pool_iterator *)v;
 194         struct lov_tgt_desc *tgt;
 195
 196         LASSERTF(iter->magic == POOL_IT_MAGIC, "%08X\n", iter->magic);
 197         LASSERT(iter->pool != NULL);
 198         LASSERT(iter->idx <= pool_tgt_count(iter->pool));
 199
 200         tgt = pool_tgt(iter->pool, iter->idx);
 201         if (tgt)
 202                 seq_printf(s, "%s\n", obd_uuid2str(&(tgt->ltd_uuid)));
 203
 204         return 0;
 205 }
 206
 207 static struct seq_operations pool_proc_ops = {
 208         .start          = pool_proc_start,
 209         .next           = pool_proc_next,
 210         .stop           = pool_proc_stop,
 211         .show           = pool_proc_show,
 212 };
 213
 214 static int pool_proc_open(struct inode *inode, struct file *file)
 215 {
 216         int rc;
 217
 218         rc = seq_open(file, &pool_proc_ops);
 219         if (!rc) {
 220                 struct seq_file *s = file->private_data;
 221                 s->private = PDE_DATA(inode);
 222         }
 223         return rc;
 224 }
 225
 226 static struct file_operations pool_proc_operations = {
 227         .open           = pool_proc_open,
 228         .read           = seq_read,
 229         .llseek         = seq_lseek,
 230         .release        = seq_release,
 231 };
 232 #endif /* CONFIG_PROC_FS */
 233
 234 void lov_dump_pool(int level, struct pool_desc *pool)
 235 {
 236         int i;
 237
 238         lov_pool_getref(pool);
 239
 240         CDEBUG(level, "pool "LOV_POOLNAMEF" has %d members\n",
 241                pool->pool_name, pool->pool_obds.op_count);
 242         down_read(&pool_tgt_rw_sem(pool));
 243
 244         for (i = 0; i < pool_tgt_count(pool) ; i++) {
 245                 if (!pool_tgt(pool, i) || !(pool_tgt(pool, i))->ltd_exp)
 246                         continue;
 247                 CDEBUG(level, "pool "LOV_POOLNAMEF"[%d] = %s\n",
 248                        pool->pool_name, i,
 249                        obd_uuid2str(&((pool_tgt(pool, i))->ltd_uuid)));
 250         }
 251
 252         up_read(&pool_tgt_rw_sem(pool));
 253         lov_pool_putref(pool);
 254 }
 255
 256 #define LOV_POOL_INIT_COUNT 2
 257 int lov_ost_pool_init(struct lu_tgt_pool *op, unsigned int count)
 258 {
 259         ENTRY;
 260
 261         if (count == 0)
 262                 count = LOV_POOL_INIT_COUNT;
 263         op->op_array = NULL;
 264         op->op_count = 0;
 265         init_rwsem(&op->op_rw_sem);
 266         op->op_size = count * sizeof(op->op_array[0]);
 267         OBD_ALLOC(op->op_array, op->op_size);
 268         if (op->op_array == NULL) {
 269                 op->op_size = 0;
 270                 RETURN(-ENOMEM);
 271         }
 272         EXIT;
 273         return 0;
 274 }
 275
 276 /* Caller must hold write op_rwlock */
 277 int lov_ost_pool_extend(struct lu_tgt_pool *op, unsigned int min_count)
 278 {
 279         __u32 *new;
 280         __u32 new_size;
 281
 282         LASSERT(min_count != 0);
 283
 284         if (op->op_count * sizeof(op->op_array[0]) < op->op_size)
 285                 return 0;
 286
 287         new_size = max_t(__u32, min_count * sizeof(op->op_array[0]),
 288                          2 * op->op_size);
 289         OBD_ALLOC(new, new_size);
 290         if (new == NULL)
 291                 return -ENOMEM;
 292
 293         /* copy old array to new one */
 294         memcpy(new, op->op_array, op->op_size);
 295         OBD_FREE(op->op_array, op->op_size);
 296         op->op_array = new;
 297         op->op_size = new_size;
 298         return 0;
 299 }
 300
 301 int lov_ost_pool_add(struct lu_tgt_pool *op, __u32 idx, unsigned int min_count)
 302 {
 303         int rc = 0, i;
 304         ENTRY;
 305
 306         down_write(&op->op_rw_sem);
 307
 308         rc = lov_ost_pool_extend(op, min_count);
 309         if (rc)
 310                 GOTO(out, rc);
 311
 312         /* search ost in pool array */
 313         for (i = 0; i < op->op_count; i++) {
 314                 if (op->op_array[i] == idx)
 315                         GOTO(out, rc = -EEXIST);
 316         }
 317         /* ost not found we add it */
 318         op->op_array[op->op_count] = idx;
 319         op->op_count++;
 320         EXIT;
 321 out:
 322         up_write(&op->op_rw_sem);
 323         return rc;
 324 }
 325
 326 int lov_ost_pool_remove(struct lu_tgt_pool *op, __u32 idx)
 327 {
 328         int i;
 329         ENTRY;
 330
 331         down_write(&op->op_rw_sem);
 332
 333         for (i = 0; i < op->op_count; i++) {
 334                 if (op->op_array[i] == idx) {
 335                         memmove(&op->op_array[i], &op->op_array[i + 1],
 336                                 (op->op_count - i - 1) * sizeof(op->op_array[0]));
 337                         op->op_count--;
 338                         up_write(&op->op_rw_sem);
 339                         EXIT;
 340                         return 0;
 341                 }
 342         }
 343
 344         up_write(&op->op_rw_sem);
 345         RETURN(-EINVAL);
 346 }
 347
 348 int lov_ost_pool_free(struct lu_tgt_pool *op)
 349 {
 350         ENTRY;
 351
 352         if (op->op_size == 0)
 353                 RETURN(0);
 354
 355         down_write(&op->op_rw_sem);
 356
 357         OBD_FREE(op->op_array, op->op_size);
 358         op->op_array = NULL;
 359         op->op_count = 0;
 360         op->op_size = 0;
 361
 362         up_write(&op->op_rw_sem);
 363         RETURN(0);
 364 }
 365
 366 static void pools_hash_exit(void *vpool, void *data)
 367 {
 368         struct pool_desc *pool = vpool;
 369
 370         lov_pool_putref(pool);
 371 }
 372
 373 int lov_pool_hash_init(struct rhashtable *tbl)
 374 {
 375         return rhashtable_init(tbl, &pools_hash_params);
 376 }
 377
 378 void lov_pool_hash_destroy(struct rhashtable *tbl)
 379 {
 380         rhashtable_free_and_destroy(tbl, pools_hash_exit, NULL);
 381 }
 382
 383 int lov_pool_new(struct obd_device *obd, char *poolname)
 384 {
 385         struct lov_obd *lov;
 386         struct pool_desc *new_pool;
 387         int rc;
 388         ENTRY;
 389
 390         lov = &(obd->u.lov);
 391
 392         if (strlen(poolname) > LOV_MAXPOOLNAME)
 393                 RETURN(-ENAMETOOLONG);
 394
 395         /* OBD_ALLOC doesn't work with direct use of kfree_rcu */
 396         new_pool = kmalloc(sizeof(*new_pool), GFP_KERNEL);
 397         if (new_pool == NULL)
 398                 RETURN(-ENOMEM);
 399
 400         strlcpy(new_pool->pool_name, poolname, sizeof(new_pool->pool_name));
 401         new_pool->pool_lobd = obd;
 402         /* ref count init to 1 because when created a pool is always used
 403          * up to deletion
 404          */
 405         atomic_set(&new_pool->pool_refcount, 1);
 406         rc = lov_ost_pool_init(&new_pool->pool_obds, 0);
 407         if (rc)
 408                 GOTO(out_err, rc);
 409
 410 #ifdef CONFIG_PROC_FS
 411         /* get ref for /proc file */
 412         lov_pool_getref(new_pool);
 413         new_pool->pool_proc_entry = lprocfs_add_simple(lov->lov_pool_proc_entry,
 414                                                        poolname, new_pool,
 415                                                        &pool_proc_operations);
 416         if (IS_ERR(new_pool->pool_proc_entry)) {
 417                 CWARN("Cannot add proc pool entry "LOV_POOLNAMEF"\n", poolname);
 418                 new_pool->pool_proc_entry = NULL;
 419                 lov_pool_putref(new_pool);
 420         }
 421         CDEBUG(D_INFO, "pool %p - proc %p\n",
 422                new_pool, new_pool->pool_proc_entry);
 423 #endif
 424
 425         spin_lock(&obd->obd_dev_lock);
 426         list_add_tail(&new_pool->pool_list, &lov->lov_pool_list);
 427         lov->lov_pool_count++;
 428         spin_unlock(&obd->obd_dev_lock);
 429
 430         /* Add to hash table only when it is fully ready. */
 431         rc = rhashtable_lookup_insert_fast(&lov->lov_pools_hash_body,
 432                                            &new_pool->pool_hash,
 433                                            pools_hash_params);
 434         if (rc) {
 435                 if (rc != -EEXIST)
 436                         /*
 437                          * Hide -E2BIG and -EBUSY which
 438                          * are not helpful.
 439                          */
 440                         rc = -ENOMEM;
 441                 GOTO(out_err, rc);
 442         }
 443
 444         CDEBUG(D_CONFIG, LOV_POOLNAMEF" is pool #%d\n",
 445                poolname, lov->lov_pool_count);
 446
 447         RETURN(0);
 448
 449 out_err:
 450         spin_lock(&obd->obd_dev_lock);
 451         list_del_init(&new_pool->pool_list);
 452         lov->lov_pool_count--;
 453         spin_unlock(&obd->obd_dev_lock);
 454         lprocfs_remove(&new_pool->pool_proc_entry);
 455         lov_ost_pool_free(&new_pool->pool_obds);
 456         OBD_FREE_PTR(new_pool);
 457
 458         return rc;
 459 }
 460
 461 struct pool_desc *lov_pool_find(struct obd_device *obd, char *poolname)
 462 {
 463         struct pool_desc *pool;
 464         struct lov_obd *lov = &obd->u.lov;
 465
 466         rcu_read_lock();
 467         pool = rhashtable_lookup(&lov->lov_pools_hash_body,
 468                                  poolname,
 469                                  pools_hash_params);
 470         if (pool && !atomic_inc_not_zero(&pool->pool_refcount))
 471                 pool = NULL;
 472         rcu_read_unlock();
 473
 474         return pool;
 475 }
 476
 477 int lov_pool_del(struct obd_device *obd, char *poolname)
 478 {
 479         struct lov_obd *lov;
 480         struct pool_desc *pool;
 481         ENTRY;
 482
 483         lov = &(obd->u.lov);
 484
 485         /* lookup and kill hash reference */
 486         rcu_read_lock();
 487         pool = rhashtable_lookup(&lov->lov_pools_hash_body, poolname,
 488                                  pools_hash_params);
 489         if (pool && rhashtable_remove_fast(&lov->lov_pools_hash_body,
 490                                            &pool->pool_hash,
 491                                            pools_hash_params) != 0)
 492                 pool = NULL;
 493         rcu_read_unlock();
 494         if (!pool)
 495                 RETURN(-ENOENT);
 496
 497         if (pool->pool_proc_entry != NULL) {
 498                 CDEBUG(D_INFO, "proc entry %p\n", pool->pool_proc_entry);
 499                 lprocfs_remove(&pool->pool_proc_entry);
 500                 lov_pool_putref(pool);
 501         }
 502
 503         spin_lock(&obd->obd_dev_lock);
 504         list_del_init(&pool->pool_list);
 505         lov->lov_pool_count--;
 506         spin_unlock(&obd->obd_dev_lock);
 507
 508         /* release last reference */
 509         lov_pool_putref(pool);
 510
 511         RETURN(0);
 512 }
 513
 514
 515 int lov_pool_add(struct obd_device *obd, char *poolname, char *ostname)
 516 {
 517         struct obd_uuid ost_uuid;
 518         struct lov_obd *lov;
 519         struct pool_desc *pool;
 520         unsigned int lov_idx;
 521         int rc;
 522         ENTRY;
 523
 524         lov = &(obd->u.lov);
 525
 526         rcu_read_lock();
 527         pool = rhashtable_lookup(&lov->lov_pools_hash_body, poolname,
 528                                  pools_hash_params);
 529         if (pool && !atomic_inc_not_zero(&pool->pool_refcount))
 530                 pool = NULL;
 531         rcu_read_unlock();
 532         if (!pool)
 533                 RETURN(-ENOENT);
 534
 535         obd_str2uuid(&ost_uuid, ostname);
 536
 537
 538         /* search ost in lov array */
 539         lov_tgts_getref(obd);
 540         for (lov_idx = 0; lov_idx < lov->desc.ld_tgt_count; lov_idx++) {
 541                 if (!lov->lov_tgts[lov_idx])
 542                         continue;
 543                 if (obd_uuid_equals(&ost_uuid,
 544                                     &(lov->lov_tgts[lov_idx]->ltd_uuid)))
 545                         break;
 546         }
 547         /* test if ost found in lov */
 548         if (lov_idx == lov->desc.ld_tgt_count)
 549                 GOTO(out, rc = -EINVAL);
 550
 551         rc = lov_ost_pool_add(&pool->pool_obds, lov_idx, lov->lov_tgt_size);
 552         if (rc)
 553                 GOTO(out, rc);
 554
 555         CDEBUG(D_CONFIG, "Added %s to "LOV_POOLNAMEF" as member %d\n",
 556                ostname, poolname,  pool_tgt_count(pool));
 557
 558         EXIT;
 559 out:
 560         lov_tgts_putref(obd);
 561         lov_pool_putref(pool);
 562
 563         return rc;
 564 }
 565
 566 int lov_pool_remove(struct obd_device *obd, char *poolname, char *ostname)
 567 {
 568         struct obd_uuid ost_uuid;
 569         struct lov_obd *lov;
 570         struct pool_desc *pool;
 571         unsigned int lov_idx;
 572         int rc = 0;
 573         ENTRY;
 574
 575         lov = &(obd->u.lov);
 576
 577         /* lookup and kill hash reference */
 578         rcu_read_lock();
 579         pool = rhashtable_lookup(&lov->lov_pools_hash_body, poolname,
 580                                  pools_hash_params);
 581         if (pool && !atomic_inc_not_zero(&pool->pool_refcount))
 582                 pool = NULL;
 583         rcu_read_unlock();
 584         if (!pool)
 585                 RETURN(-ENOENT);
 586
 587         obd_str2uuid(&ost_uuid, ostname);
 588
 589         lov_tgts_getref(obd);
 590         /* search ost in lov array, to get index */
 591         for (lov_idx = 0; lov_idx < lov->desc.ld_tgt_count; lov_idx++) {
 592                 if (!lov->lov_tgts[lov_idx])
 593                         continue;
 594
 595                 if (obd_uuid_equals(&ost_uuid,
 596                                     &(lov->lov_tgts[lov_idx]->ltd_uuid)))
 597                         break;
 598         }
 599
 600         /* test if ost found in lov */
 601         if (lov_idx == lov->desc.ld_tgt_count)
 602                 GOTO(out, rc = -EINVAL);
 603
 604         lov_ost_pool_remove(&pool->pool_obds, lov_idx);
 605
 606         CDEBUG(D_CONFIG, "%s removed from "LOV_POOLNAMEF"\n", ostname,
 607                poolname);
 608
 609         EXIT;
 610 out:
 611         lov_tgts_putref(obd);
 612         lov_pool_putref(pool);
 613
 614         return rc;
 615 }