lustre/lod/lod_lov.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License version 2 for more details.  A copy is
  14  * included in the COPYING file that accompanied this code.
  15
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright  2009 Sun Microsystems, Inc. All rights reserved
  24  * Use is subject to license terms.
  25  */
  26 /*
  27  * lustre/lod/lod_lov.c
  28  *
  29  * Author: Alex Zhuravlev <alexey.zhuravlev@intel.com>
  30  */
  31
  32 #ifndef EXPORT_SYMTAB
  33 # define EXPORT_SYMTAB
  34 #endif
  35 #define DEBUG_SUBSYSTEM S_MDS
  36
  37 #include <obd_class.h>
  38 #include <obd_lov.h>
  39
  40 #include "lod_internal.h"
  41
  42 /*
  43  * Keep a refcount of lod->lod_osts usage to prevent racing with
  44  * addition/deletion. Any function that expects lov_tgts to remain stationary
  45  * must take a ref.
  46  *
  47  * \param lod - is the lod device from which we want to grab a reference
  48  */
  49 void lod_getref(struct lod_device *lod)
  50 {
  51         cfs_down_read(&lod->lod_rw_sem);
  52         cfs_mutex_lock(&lod->lod_mutex);
  53         lod->lod_refcount++;
  54         cfs_mutex_unlock(&lod->lod_mutex);
  55 }
  56
  57 /*
  58  * Companion of lod_getref() to release a reference on the lod table.
  59  * If this is the last reference and the ost entry was scheduled for deletion,
  60  * the descriptor is removed from the array.
  61  *
  62  * \param lod - is the lod device from which we release a reference
  63  */
  64 void lod_putref(struct lod_device *lod)
  65 {
  66         cfs_mutex_lock(&lod->lod_mutex);
  67         lod->lod_refcount--;
  68         if (lod->lod_refcount == 0 && lod->lod_death_row) {
  69                 struct lod_ost_desc *ost_desc, *tmp;
  70                 int                  idx;
  71                 CFS_LIST_HEAD(kill);
  72
  73                 CDEBUG(D_CONFIG, "destroying %d lod desc\n",
  74                        lod->lod_death_row);
  75
  76                 cfs_foreach_bit(lod->lod_ost_bitmap, idx) {
  77                         ost_desc = OST_TGT(lod, idx);
  78                         LASSERT(ost_desc);
  79
  80                         if (!ost_desc->ltd_reap)
  81                                 continue;
  82
  83                         cfs_list_add(&ost_desc->ltd_kill, &kill);
  84
  85                         lod_ost_pool_remove(&lod->lod_pool_info, idx);
  86                         OST_TGT(lod, idx) = NULL;
  87                         lod->lod_ostnr--;
  88                         cfs_bitmap_clear(lod->lod_ost_bitmap, idx);
  89                         if (ost_desc->ltd_active)
  90                                 lod->lod_desc.ld_active_tgt_count--;
  91                         lod->lod_death_row--;
  92                 }
  93                 cfs_mutex_unlock(&lod->lod_mutex);
  94                 cfs_up_read(&lod->lod_rw_sem);
  95
  96                 cfs_list_for_each_entry_safe(ost_desc, tmp, &kill, ltd_kill) {
  97                         int rc;
  98                         cfs_list_del(&ost_desc->ltd_kill);
  99                         /* XXX: remove from QoS structures */
 100                         /* disconnect from OSP */
 101                         rc = obd_disconnect(ost_desc->ltd_exp);
 102                         if (rc)
 103                                 CERROR("%s: failed to disconnect %s (%d)\n",
 104                                        lod2obd(lod)->obd_name,
 105                                        obd_uuid2str(&ost_desc->ltd_uuid), rc);
 106                         OBD_FREE_PTR(ost_desc);
 107                 }
 108         } else {
 109                 cfs_mutex_unlock(&lod->lod_mutex);
 110                 cfs_up_read(&lod->lod_rw_sem);
 111         }
 112 }
 113
 114 static int lod_bitmap_resize(struct lod_device *lod, __u32 newsize)
 115 {
 116         cfs_bitmap_t *new_bitmap, *old_bitmap = NULL;
 117         int           rc = 0;
 118         ENTRY;
 119
 120         /* grab write reference on the lod. Relocating the array requires
 121          * exclusive access */
 122         cfs_down_write(&lod->lod_rw_sem);
 123
 124         if (newsize <= lod->lod_osts_size)
 125                 /* someone else has already resize the array */
 126                 GOTO(out, rc = 0);
 127
 128         /* allocate new bitmap */
 129         new_bitmap = CFS_ALLOCATE_BITMAP(newsize);
 130         if (!new_bitmap)
 131                 GOTO(out, rc = -ENOMEM);
 132
 133         if (lod->lod_osts_size > 0) {
 134                 /* the bitmap already exists, we need
 135                  * to copy data from old one */
 136                 cfs_bitmap_copy(new_bitmap, lod->lod_ost_bitmap);
 137                 old_bitmap = lod->lod_ost_bitmap;
 138         }
 139
 140         lod->lod_osts_size  = newsize;
 141         lod->lod_ost_bitmap = new_bitmap;
 142
 143         if (old_bitmap)
 144                 CFS_FREE_BITMAP(old_bitmap);
 145
 146         CDEBUG(D_CONFIG, "ost size: %d\n", lod->lod_osts_size);
 147
 148         EXIT;
 149 out:
 150         cfs_up_write(&lod->lod_rw_sem);
 151         return rc;
 152 }
 153
 154 /*
 155  * Connect LOD to a new OSP and add it to the device table.
 156  *
 157  * \param env - is the environment passed by the caller
 158  * \param lod - is the LOD device to be connected to the new OSP
 159  * \param osp - is the name of OSP device name about to be added
 160  * \param index - is the OSP index
 161  * \param gen - is the generation number
 162  */
 163 int lod_add_device(const struct lu_env *env, struct lod_device *lod,
 164                    char *osp, unsigned index, unsigned gen, int active)
 165 {
 166         struct obd_connect_data *data = NULL;
 167         struct obd_export       *exp = NULL;
 168         struct obd_device       *obd;
 169         struct lu_device        *ldev;
 170         struct dt_device        *d;
 171         int                      rc;
 172         struct lod_ost_desc     *ost_desc;
 173         struct obd_uuid          obd_uuid;
 174
 175         ENTRY;
 176
 177         CDEBUG(D_CONFIG, "osp:%s idx:%d gen:%d\n", osp, index, gen);
 178
 179         if (gen <= 0) {
 180                 CERROR("request to add OBD %s with invalid generation: %d\n",
 181                        osp, gen);
 182                 RETURN(-EINVAL);
 183         }
 184
 185         obd_str2uuid(&obd_uuid, osp);
 186
 187         obd = class_find_client_obd(&obd_uuid, LUSTRE_OSP_NAME,
 188                                 &lod->lod_dt_dev.dd_lu_dev.ld_obd->obd_uuid);
 189         if (obd == NULL) {
 190                 CERROR("can't find %s device\n", osp);
 191                 RETURN(-EINVAL);
 192         }
 193
 194         OBD_ALLOC_PTR(data);
 195         if (data == NULL)
 196                 RETURN(-ENOMEM);
 197
 198         data->ocd_connect_flags = OBD_CONNECT_INDEX | OBD_CONNECT_VERSION;
 199         data->ocd_version = LUSTRE_VERSION_CODE;
 200         data->ocd_index = index;
 201
 202         rc = obd_connect(env, &exp, obd, &obd->obd_uuid, data, NULL);
 203         OBD_FREE_PTR(data);
 204         if (rc) {
 205                 CERROR("%s: cannot connect to next dev %s (%d)\n",
 206                        obd->obd_name, osp, rc);
 207                 GOTO(out_free, rc);
 208         }
 209
 210         LASSERT(obd->obd_lu_dev);
 211         LASSERT(obd->obd_lu_dev->ld_site = lod->lod_dt_dev.dd_lu_dev.ld_site);
 212
 213         ldev = obd->obd_lu_dev;
 214         d = lu2dt_dev(ldev);
 215
 216         /* Allocate ost descriptor and fill it */
 217         OBD_ALLOC_PTR(ost_desc);
 218         if (!ost_desc)
 219                 GOTO(out_conn, rc = -ENOMEM);
 220
 221         ost_desc->ltd_ost    = d;
 222         ost_desc->ltd_exp    = exp;
 223         ost_desc->ltd_uuid   = obd->u.cli.cl_target_uuid;
 224         ost_desc->ltd_gen    = gen;
 225         ost_desc->ltd_index  = index;
 226         ost_desc->ltd_active = active;
 227
 228         lod_getref(lod);
 229         if (index >= lod->lod_osts_size) {
 230                 /* we have to increase the size of the lod_osts array */
 231                 __u32  newsize;
 232
 233                 newsize = max(lod->lod_osts_size, (__u32)2);
 234                 while (newsize < index + 1)
 235                         newsize = newsize << 1;
 236
 237                 /* lod_bitmap_resize() needs lod_rw_sem
 238                  * which we hold with th reference */
 239                 lod_putref(lod);
 240
 241                 rc = lod_bitmap_resize(lod, newsize);
 242                 if (rc)
 243                         GOTO(out_desc, rc);
 244
 245                 lod_getref(lod);
 246         }
 247
 248         cfs_mutex_lock(&lod->lod_mutex);
 249         if (cfs_bitmap_check(lod->lod_ost_bitmap, index)) {
 250                 CERROR("%s: device %d is registered already\n", obd->obd_name,
 251                        index);
 252                 GOTO(out_mutex, rc = -EEXIST);
 253         }
 254
 255         if (lod->lod_ost_idx[index / OST_PTRS_PER_BLOCK] == NULL) {
 256                 OBD_ALLOC_PTR(lod->lod_ost_idx[index / OST_PTRS_PER_BLOCK]);
 257                 if (lod->lod_ost_idx[index / OST_PTRS_PER_BLOCK] == NULL) {
 258                         CERROR("can't allocate index to add %s\n",
 259                                obd->obd_name);
 260                         GOTO(out_mutex, rc = -ENOMEM);
 261                 }
 262         }
 263
 264         rc = lod_ost_pool_add(&lod->lod_pool_info, index, lod->lod_osts_size);
 265         if (rc) {
 266                 CERROR("%s: can't set up pool, failed with %d\n",
 267                        obd->obd_name, rc);
 268                 GOTO(out_mutex, rc);
 269         }
 270
 271         /* The new OST is now a full citizen */
 272         if (index >= lod->lod_desc.ld_tgt_count)
 273                 lod->lod_desc.ld_tgt_count = index + 1;
 274         if (active)
 275                 lod->lod_desc.ld_active_tgt_count++;
 276         OST_TGT(lod, index) = ost_desc;
 277         cfs_bitmap_set(lod->lod_ost_bitmap, index);
 278         lod->lod_ostnr++;
 279         cfs_mutex_unlock(&lod->lod_mutex);
 280         lod_putref(lod);
 281
 282         if (lod->lod_recovery_completed)
 283                 ldev->ld_ops->ldo_recovery_complete(env, ldev);
 284
 285         RETURN(0);
 286
 287         lod_ost_pool_remove(&lod->lod_pool_info, index);
 288 out_mutex:
 289         cfs_mutex_unlock(&lod->lod_mutex);
 290         lod_putref(lod);
 291 out_desc:
 292         OBD_FREE_PTR(ost_desc);
 293 out_conn:
 294         obd_disconnect(exp);
 295 out_free:
 296         return rc;
 297 }
 298
 299 /*
 300  * helper function to schedule OST removal from the device table
 301  */
 302 static void __lod_del_device(struct lod_device *lod, unsigned idx)
 303 {
 304         LASSERT(OST_TGT(lod,idx));
 305         if (OST_TGT(lod,idx)->ltd_reap == 0) {
 306                 OST_TGT(lod,idx)->ltd_reap = 1;
 307                 lod->lod_death_row++;
 308         }
 309 }
 310
 311 /*
 312  * Add support for administratively disabled OST (through the MGS).
 313  * Schedule a target for deletion.  Disconnection and real removal from the
 314  * table takes place in lod_putref() once the last table user release its
 315  * reference.
 316  *
 317  * \param env - is the environment passed by the caller
 318  * \param lod - is the lod device currently connected to the OSP about to be
 319  *              removed
 320  * \param osp - is the name of OSP device about to be removed
 321  * \param idx - is the OSP index
 322  * \param gen - is the generation number, not used currently
 323  */
 324 int lod_del_device(const struct lu_env *env, struct lod_device *lod,
 325                    char *osp, unsigned idx, unsigned gen)
 326 {
 327         struct obd_device *obd;
 328         int                rc = 0;
 329         struct obd_uuid    uuid;
 330         ENTRY;
 331
 332         CDEBUG(D_CONFIG, "osp:%s idx:%d gen:%d\n", osp, idx, gen);
 333
 334         obd = class_name2obd(osp);
 335         if (obd == NULL) {
 336                 CERROR("can't find %s device\n", osp);
 337                 RETURN(-EINVAL);
 338         }
 339
 340         if (gen <= 0) {
 341                 CERROR("%s: request to remove OBD %s with invalid generation %d"
 342                        "\n", obd->obd_name, osp, gen);
 343                 RETURN(-EINVAL);
 344         }
 345
 346         obd_str2uuid(&uuid,  osp);
 347
 348         lod_getref(lod);
 349         cfs_mutex_lock(&lod->lod_mutex);
 350         /* check that the index is allocated in the bitmap */
 351         if (!cfs_bitmap_check(lod->lod_ost_bitmap, idx) || !OST_TGT(lod,idx)) {
 352                 CERROR("%s: device %d is not set up\n", obd->obd_name, idx);
 353                 GOTO(out, rc = -EINVAL);
 354         }
 355
 356         /* check that the UUID matches */
 357         if (!obd_uuid_equals(&uuid, &OST_TGT(lod,idx)->ltd_uuid)) {
 358                 CERROR("%s: LOD target UUID %s at index %d does not match %s\n",
 359                        obd->obd_name, obd_uuid2str(&OST_TGT(lod,idx)->ltd_uuid),
 360                        idx, osp);
 361                 GOTO(out, rc = -EINVAL);
 362         }
 363
 364         __lod_del_device(lod, idx);
 365         EXIT;
 366 out:
 367         cfs_mutex_unlock(&lod->lod_mutex);
 368         lod_putref(lod);
 369         return(rc);
 370 }
 371
 372 void lod_fix_desc_stripe_size(__u64 *val)
 373 {
 374         if (*val < PTLRPC_MAX_BRW_SIZE) {
 375                 LCONSOLE_WARN("Increasing default stripe size to min %u\n",
 376                               PTLRPC_MAX_BRW_SIZE);
 377                 *val = PTLRPC_MAX_BRW_SIZE;
 378         } else if (*val & (LOV_MIN_STRIPE_SIZE - 1)) {
 379                 *val &= ~(LOV_MIN_STRIPE_SIZE - 1);
 380                 LCONSOLE_WARN("Changing default stripe size to "LPU64" (a "
 381                               "multiple of %u)\n",
 382                               *val, LOV_MIN_STRIPE_SIZE);
 383         }
 384 }
 385
 386 void lod_fix_desc_stripe_count(__u32 *val)
 387 {
 388         if (*val == 0)
 389                 *val = 1;
 390 }
 391
 392 void lod_fix_desc_pattern(__u32 *val)
 393 {
 394         /* from lov_setstripe */
 395         if ((*val != 0) && (*val != LOV_PATTERN_RAID0)) {
 396                 LCONSOLE_WARN("Unknown stripe pattern: %#x\n", *val);
 397                 *val = 0;
 398         }
 399 }
 400
 401 void lod_fix_desc_qos_maxage(__u32 *val)
 402 {
 403         /* fix qos_maxage */
 404         if (*val == 0)
 405                 *val = QOS_DEFAULT_MAXAGE;
 406 }
 407
 408 void lod_fix_desc(struct lov_desc *desc)
 409 {
 410         lod_fix_desc_stripe_size(&desc->ld_default_stripe_size);
 411         lod_fix_desc_stripe_count(&desc->ld_default_stripe_count);
 412         lod_fix_desc_pattern(&desc->ld_pattern);
 413         lod_fix_desc_qos_maxage(&desc->ld_qos_maxage);
 414 }
 415
 416 int lod_pools_init(struct lod_device *lod, struct lustre_cfg *lcfg)
 417 {
 418         struct lprocfs_static_vars  lvars = { 0 };
 419         struct obd_device          *obd;
 420         struct lov_desc            *desc;
 421         int                         rc;
 422         ENTRY;
 423
 424         obd = class_name2obd(lustre_cfg_string(lcfg, 0));
 425         LASSERT(obd != NULL);
 426         obd->obd_lu_dev = &lod->lod_dt_dev.dd_lu_dev;
 427
 428         if (LUSTRE_CFG_BUFLEN(lcfg, 1) < 1) {
 429                 CERROR("LOD setup requires a descriptor\n");
 430                 RETURN(-EINVAL);
 431         }
 432
 433         desc = (struct lov_desc *)lustre_cfg_buf(lcfg, 1);
 434
 435         if (sizeof(*desc) > LUSTRE_CFG_BUFLEN(lcfg, 1)) {
 436                 CERROR("descriptor size wrong: %d > %d\n",
 437                        (int)sizeof(*desc), LUSTRE_CFG_BUFLEN(lcfg, 1));
 438                 RETURN(-EINVAL);
 439         }
 440
 441         if (desc->ld_magic != LOV_DESC_MAGIC) {
 442                 if (desc->ld_magic == __swab32(LOV_DESC_MAGIC)) {
 443                         CDEBUG(D_OTHER, "%s: Swabbing lov desc %p\n",
 444                                obd->obd_name, desc);
 445                         lustre_swab_lov_desc(desc);
 446                 } else {
 447                         CERROR("%s: Bad lov desc magic: %#x\n",
 448                                obd->obd_name, desc->ld_magic);
 449                         RETURN(-EINVAL);
 450                 }
 451         }
 452
 453         lod_fix_desc(desc);
 454
 455         desc->ld_active_tgt_count = 0;
 456         lod->lod_desc = *desc;
 457
 458         lod->lod_sp_me = LUSTRE_SP_CLI;
 459
 460         /* Set up allocation policy (QoS and RR) */
 461         CFS_INIT_LIST_HEAD(&lod->lod_qos.lq_oss_list);
 462         cfs_init_rwsem(&lod->lod_qos.lq_rw_sem);
 463         lod->lod_qos.lq_dirty = 1;
 464         lod->lod_qos.lq_rr.lqr_dirty = 1;
 465         lod->lod_qos.lq_reset = 1;
 466         /* Default priority is toward free space balance */
 467         lod->lod_qos.lq_prio_free = 232;
 468         /* Default threshold for rr (roughly 17%) */
 469         lod->lod_qos.lq_threshold_rr = 43;
 470         /* Init statfs fields */
 471         OBD_ALLOC_PTR(lod->lod_qos.lq_statfs_data);
 472         if (NULL == lod->lod_qos.lq_statfs_data)
 473                 RETURN(-ENOMEM);
 474         cfs_waitq_init(&lod->lod_qos.lq_statfs_waitq);
 475
 476         /* Set up OST pool environment */
 477         lod->lod_pools_hash_body = cfs_hash_create("POOLS", HASH_POOLS_CUR_BITS,
 478                                                    HASH_POOLS_MAX_BITS,
 479                                                    HASH_POOLS_BKT_BITS, 0,
 480                                                    CFS_HASH_MIN_THETA,
 481                                                    CFS_HASH_MAX_THETA,
 482                                                    &pool_hash_operations,
 483                                                    CFS_HASH_DEFAULT);
 484         if (!lod->lod_pools_hash_body)
 485                 GOTO(out_statfs, rc = -ENOMEM);
 486         CFS_INIT_LIST_HEAD(&lod->lod_pool_list);
 487         lod->lod_pool_count = 0;
 488         rc = lod_ost_pool_init(&lod->lod_pool_info, 0);
 489         if (rc)
 490                 GOTO(out_hash, rc);
 491         rc = lod_ost_pool_init(&lod->lod_qos.lq_rr.lqr_pool, 0);
 492         if (rc)
 493                 GOTO(out_pool_info, rc);
 494
 495         /* the OST array and bitmap are allocated/grown dynamically as OSTs are
 496          * added to the LOD, see lod_add_device() */
 497         lod->lod_ost_bitmap = NULL;
 498         lod->lod_osts_size  = 0;
 499         lod->lod_ostnr      = 0;
 500
 501         lod->lod_death_row = 0;
 502         lod->lod_refcount  = 0;
 503
 504         lprocfs_lod_init_vars(&lvars);
 505         lprocfs_obd_setup(obd, lvars.obd_vars);
 506
 507 #ifdef LPROCFS
 508         rc = lprocfs_seq_create(obd->obd_proc_entry, "target_obd",
 509                                 0444, &lod_proc_target_fops, obd);
 510         if (rc) {
 511                 CWARN("%s: Error adding the target_obd file %d\n",
 512                       obd->obd_name, rc);
 513                 GOTO(out_lproc, rc);
 514         }
 515         lod->lod_pool_proc_entry = lprocfs_register("pools",
 516                                                     obd->obd_proc_entry,
 517                                                     NULL, NULL);
 518         if (IS_ERR(lod->lod_pool_proc_entry)) {
 519                 int ret = PTR_ERR(lod->lod_pool_proc_entry);
 520                 lod->lod_pool_proc_entry = NULL;
 521                 CWARN("%s: Failed to create pool proc file %d\n",
 522                       obd->obd_name, ret);
 523                 rc = lod_pools_fini(lod);
 524                 RETURN(ret);
 525         }
 526 #endif
 527
 528         RETURN(0);
 529
 530 out_lproc:
 531         lprocfs_obd_cleanup(obd);
 532         lod_ost_pool_free(&lod->lod_qos.lq_rr.lqr_pool);
 533 out_pool_info:
 534         lod_ost_pool_free(&lod->lod_pool_info);
 535 out_hash:
 536         cfs_hash_putref(lod->lod_pools_hash_body);
 537 out_statfs:
 538         OBD_FREE_PTR(lod->lod_qos.lq_statfs_data);
 539         return rc;
 540 }
 541
 542 int lod_pools_fini(struct lod_device *lod)
 543 {
 544         struct obd_device   *obd = lod2obd(lod);
 545         cfs_list_t          *pos, *tmp;
 546         struct pool_desc    *pool;
 547         ENTRY;
 548
 549         cfs_list_for_each_safe(pos, tmp, &lod->lod_pool_list) {
 550                 pool = cfs_list_entry(pos, struct pool_desc, pool_list);
 551                 /* free pool structs */
 552                 CDEBUG(D_INFO, "delete pool %p\n", pool);
 553                 lod_pool_del(obd, pool->pool_name);
 554         }
 555
 556         if (lod->lod_osts_size > 0) {
 557                 int idx;
 558                 lod_getref(lod);
 559                 cfs_mutex_lock(&lod->lod_mutex);
 560                 cfs_foreach_bit(lod->lod_ost_bitmap, idx)
 561                         __lod_del_device(lod, idx);
 562                 cfs_mutex_unlock(&lod->lod_mutex);
 563                 lod_putref(lod);
 564                 CFS_FREE_BITMAP(lod->lod_ost_bitmap);
 565                 for (idx = 0; idx < OST_PTRS; idx++) {
 566                         if (lod->lod_ost_idx[idx])
 567                                 OBD_FREE_PTR(lod->lod_ost_idx[idx]);
 568                 }
 569                 lod->lod_osts_size = 0;
 570         }
 571
 572         cfs_hash_putref(lod->lod_pools_hash_body);
 573         lod_ost_pool_free(&(lod->lod_qos.lq_rr.lqr_pool));
 574         lod_ost_pool_free(&lod->lod_pool_info);
 575
 576         /* clear pools parent proc entry only after all pools are killed */
 577         if (lod->lod_pool_proc_entry) {
 578                 lprocfs_remove(&lod->lod_pool_proc_entry);
 579                 lod->lod_pool_proc_entry = NULL;
 580         }
 581
 582         lprocfs_obd_cleanup(obd);
 583
 584         OBD_FREE_PTR(lod->lod_qos.lq_statfs_data);
 585         RETURN(0);
 586 }
 587