lustre/obdclass/class_obd.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2011, 2017, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  */
  32
  33 #define DEBUG_SUBSYSTEM S_CLASS
  34
  35 #include <linux/miscdevice.h>
  36 #include <linux/user_namespace.h>
  37 #ifdef HAVE_UIDGID_HEADER
  38 # include <linux/uidgid.h>
  39 #endif
  40 #include <linux/atomic.h>
  41 #include <linux/list.h>
  42
  43 #include <obd_support.h>
  44 #include <obd_class.h>
  45 #include <uapi/linux/lnet/lnetctl.h>
  46 #include <lustre_debug.h>
  47 #include <lustre_kernelcomm.h>
  48 #include <lprocfs_status.h>
  49 #include <cl_object.h>
  50 #ifdef HAVE_SERVER_SUPPORT
  51 # include <dt_object.h>
  52 # include <md_object.h>
  53 #endif /* HAVE_SERVER_SUPPORT */
  54 #include <uapi/linux/lustre/lustre_ioctl.h>
  55 #include "llog_internal.h"
  56
  57 #ifdef CONFIG_PROC_FS
  58 static __u64 obd_max_alloc;
  59 #else
  60 __u64 obd_max_alloc;
  61 #endif
  62
  63 static DEFINE_SPINLOCK(obd_updatemax_lock);
  64
  65 /* The following are visible and mutable through /proc/sys/lustre/. */
  66 unsigned int obd_debug_peer_on_timeout;
  67 EXPORT_SYMBOL(obd_debug_peer_on_timeout);
  68 unsigned int obd_dump_on_timeout;
  69 EXPORT_SYMBOL(obd_dump_on_timeout);
  70 unsigned int obd_dump_on_eviction;
  71 EXPORT_SYMBOL(obd_dump_on_eviction);
  72 unsigned long obd_max_dirty_pages;
  73 EXPORT_SYMBOL(obd_max_dirty_pages);
  74 atomic_long_t obd_dirty_pages;
  75 EXPORT_SYMBOL(obd_dirty_pages);
  76 unsigned int obd_timeout = OBD_TIMEOUT_DEFAULT;   /* seconds */
  77 EXPORT_SYMBOL(obd_timeout);
  78 unsigned int ldlm_timeout = LDLM_TIMEOUT_DEFAULT; /* seconds */
  79 EXPORT_SYMBOL(ldlm_timeout);
  80 unsigned int obd_timeout_set;
  81 EXPORT_SYMBOL(obd_timeout_set);
  82 unsigned int ldlm_timeout_set;
  83 EXPORT_SYMBOL(ldlm_timeout_set);
  84 /* bulk transfer timeout, give up after 100s by default */
  85 unsigned int bulk_timeout = 100; /* seconds */
  86 EXPORT_SYMBOL(bulk_timeout);
  87 /* Adaptive timeout defs here instead of ptlrpc module for /proc/sys/ access */
  88 unsigned int at_min = 0;
  89 EXPORT_SYMBOL(at_min);
  90 unsigned int at_max = 600;
  91 EXPORT_SYMBOL(at_max);
  92 unsigned int at_history = 600;
  93 EXPORT_SYMBOL(at_history);
  94 int at_early_margin = 5;
  95 EXPORT_SYMBOL(at_early_margin);
  96 int at_extra = 30;
  97 EXPORT_SYMBOL(at_extra);
  98
  99 atomic_long_t obd_dirty_transit_pages;
 100 EXPORT_SYMBOL(obd_dirty_transit_pages);
 101
 102 #ifdef CONFIG_PROC_FS
 103 struct lprocfs_stats *obd_memory = NULL;
 104 EXPORT_SYMBOL(obd_memory);
 105 #endif
 106
 107 static int class_resolve_dev_name(__u32 len, const char *name)
 108 {
 109         int rc;
 110         int dev;
 111
 112         ENTRY;
 113         if (!len || !name) {
 114                 CERROR("No name passed,!\n");
 115                 GOTO(out, rc = -EINVAL);
 116         }
 117         if (name[len - 1] != 0) {
 118                 CERROR("Name not nul terminated!\n");
 119                 GOTO(out, rc = -EINVAL);
 120         }
 121
 122         CDEBUG(D_IOCTL, "device name %s\n", name);
 123         dev = class_name2dev(name);
 124         if (dev == -1) {
 125                 CDEBUG(D_IOCTL, "No device for name %s!\n", name);
 126                 GOTO(out, rc = -EINVAL);
 127         }
 128
 129         CDEBUG(D_IOCTL, "device name %s, dev %d\n", name, dev);
 130         rc = dev;
 131
 132 out:
 133         RETURN(rc);
 134 }
 135
 136 int class_handle_ioctl(unsigned int cmd, unsigned long arg)
 137 {
 138         char *buf = NULL;
 139         struct obd_ioctl_data *data;
 140         struct libcfs_debug_ioctl_data *debug_data;
 141         struct obd_device *obd = NULL;
 142         int err = 0, len = 0;
 143         ENTRY;
 144
 145         /* only for debugging */
 146         if (cmd == LIBCFS_IOC_DEBUG_MASK) {
 147                 debug_data = (struct libcfs_debug_ioctl_data*)arg;
 148                 libcfs_subsystem_debug = debug_data->subs;
 149                 libcfs_debug = debug_data->debug;
 150                 return 0;
 151         }
 152
 153         CDEBUG(D_IOCTL, "cmd = %x\n", cmd);
 154         if (obd_ioctl_getdata(&buf, &len, (void __user *)arg)) {
 155                 CERROR("OBD ioctl: data error\n");
 156                 RETURN(-EINVAL);
 157         }
 158         data = (struct obd_ioctl_data *)buf;
 159
 160         switch (cmd) {
 161         case OBD_IOC_PROCESS_CFG: {
 162                 struct lustre_cfg *lcfg;
 163
 164                 if (!data->ioc_plen1 || !data->ioc_pbuf1) {
 165                         CERROR("No config buffer passed!\n");
 166                         GOTO(out, err = -EINVAL);
 167                 }
 168                 OBD_ALLOC(lcfg, data->ioc_plen1);
 169                 if (lcfg == NULL)
 170                         GOTO(out, err = -ENOMEM);
 171                 err = copy_from_user(lcfg, data->ioc_pbuf1,
 172                                          data->ioc_plen1);
 173                 if (!err)
 174                         err = lustre_cfg_sanity_check(lcfg, data->ioc_plen1);
 175                 if (!err)
 176                         err = class_process_config(lcfg);
 177
 178                 OBD_FREE(lcfg, data->ioc_plen1);
 179                 GOTO(out, err);
 180         }
 181
 182 #if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(3, 0, 53, 0)
 183         case OBD_GET_VERSION: {
 184                 static bool warned;
 185
 186                 if (!data->ioc_inlbuf1) {
 187                         CERROR("No buffer passed in ioctl\n");
 188                         GOTO(out, err = -EINVAL);
 189                 }
 190
 191                 if (strlen(LUSTRE_VERSION_STRING) + 1 > data->ioc_inllen1) {
 192                         CERROR("ioctl buffer too small to hold version\n");
 193                         GOTO(out, err = -EINVAL);
 194                 }
 195
 196                 if (!warned) {
 197                         warned = true;
 198                         CWARN("%s: ioctl(OBD_GET_VERSION) is deprecated, "
 199                               "use llapi_get_version_string() and/or relink\n",
 200                               current->comm);
 201                 }
 202                 memcpy(data->ioc_bulk, LUSTRE_VERSION_STRING,
 203                        strlen(LUSTRE_VERSION_STRING) + 1);
 204
 205                 if (copy_to_user((void __user *)arg, data, len))
 206                         err = -EFAULT;
 207                 GOTO(out, err);
 208         }
 209 #endif
 210         case OBD_IOC_NAME2DEV: {
 211                 /* Resolve a device name.  This does not change the
 212                  * currently selected device.
 213                  */
 214                 int dev;
 215
 216                 dev = class_resolve_dev_name(data->ioc_inllen1,
 217                                              data->ioc_inlbuf1);
 218                 data->ioc_dev = dev;
 219                 if (dev < 0)
 220                         GOTO(out, err = -EINVAL);
 221
 222                 if (copy_to_user((void __user *)arg, data, sizeof(*data)))
 223                         err = -EFAULT;
 224                 GOTO(out, err);
 225         }
 226
 227         case OBD_IOC_UUID2DEV: {
 228                 /* Resolve a device uuid.  This does not change the
 229                  * currently selected device.
 230                  */
 231                 int dev;
 232                 struct obd_uuid uuid;
 233
 234                 if (!data->ioc_inllen1 || !data->ioc_inlbuf1) {
 235                         CERROR("No UUID passed!\n");
 236                         GOTO(out, err = -EINVAL);
 237                 }
 238                 if (data->ioc_inlbuf1[data->ioc_inllen1 - 1] != 0) {
 239                         CERROR("UUID not NUL terminated!\n");
 240                         GOTO(out, err = -EINVAL);
 241                 }
 242
 243                 CDEBUG(D_IOCTL, "device name %s\n", data->ioc_inlbuf1);
 244                 obd_str2uuid(&uuid, data->ioc_inlbuf1);
 245                 dev = class_uuid2dev(&uuid);
 246                 data->ioc_dev = dev;
 247                 if (dev == -1) {
 248                         CDEBUG(D_IOCTL, "No device for UUID %s!\n",
 249                                data->ioc_inlbuf1);
 250                         GOTO(out, err = -EINVAL);
 251                 }
 252
 253                 CDEBUG(D_IOCTL, "device name %s, dev %d\n", data->ioc_inlbuf1,
 254                        dev);
 255                 if (copy_to_user((void __user *)arg, data, sizeof(*data)))
 256                         err = -EFAULT;
 257                 GOTO(out, err);
 258         }
 259
 260         case OBD_IOC_GETDEVICE: {
 261                 int     index = data->ioc_count;
 262                 char    *status, *str;
 263
 264                 if (!data->ioc_inlbuf1) {
 265                         CERROR("No buffer passed in ioctl\n");
 266                         GOTO(out, err = -EINVAL);
 267                 }
 268                 if (data->ioc_inllen1 < 128) {
 269                         CERROR("ioctl buffer too small to hold version\n");
 270                         GOTO(out, err = -EINVAL);
 271                 }
 272
 273                 obd = class_num2obd(index);
 274                 if (!obd)
 275                         GOTO(out, err = -ENOENT);
 276
 277                 if (obd->obd_stopping)
 278                         status = "ST";
 279                 else if (obd->obd_set_up)
 280                         status = "UP";
 281                 else if (obd->obd_attached)
 282                         status = "AT";
 283                 else
 284                         status = "--";
 285                 str = (char *)data->ioc_bulk;
 286                 snprintf(str, len - sizeof(*data), "%3d %s %s %s %s %d",
 287                          (int)index, status, obd->obd_type->typ_name,
 288                          obd->obd_name, obd->obd_uuid.uuid,
 289                          atomic_read(&obd->obd_refcount));
 290
 291                 if (copy_to_user((void __user *)arg, data, len))
 292                         err = -EFAULT;
 293
 294                 GOTO(out, err);
 295         }
 296
 297         }
 298
 299         if (data->ioc_dev == OBD_DEV_BY_DEVNAME) {
 300                 if (data->ioc_inllen4 <= 0 || data->ioc_inlbuf4 == NULL)
 301                         GOTO(out, err = -EINVAL);
 302                 if (strnlen(data->ioc_inlbuf4, MAX_OBD_NAME) >= MAX_OBD_NAME)
 303                         GOTO(out, err = -EINVAL);
 304                 obd = class_name2obd(data->ioc_inlbuf4);
 305         } else if (data->ioc_dev < class_devno_max()) {
 306                 obd = class_num2obd(data->ioc_dev);
 307         } else {
 308                 CERROR("OBD ioctl: No device\n");
 309                 GOTO(out, err = -EINVAL);
 310         }
 311
 312         if (obd == NULL) {
 313                 CERROR("OBD ioctl : No Device %d\n", data->ioc_dev);
 314                 GOTO(out, err = -EINVAL);
 315         }
 316         LASSERT(obd->obd_magic == OBD_DEVICE_MAGIC);
 317
 318         if (!obd->obd_set_up || obd->obd_stopping) {
 319                 CERROR("OBD ioctl: device not setup %d \n", data->ioc_dev);
 320                 GOTO(out, err = -EINVAL);
 321         }
 322
 323         switch(cmd) {
 324         case OBD_IOC_NO_TRANSNO: {
 325                 if (!obd->obd_attached) {
 326                         CERROR("Device %d not attached\n", obd->obd_minor);
 327                         GOTO(out, err = -ENODEV);
 328                 }
 329                 CDEBUG(D_HA, "%s: disabling committed-transno notification\n",
 330                        obd->obd_name);
 331                 obd->obd_no_transno = 1;
 332                 GOTO(out, err = 0);
 333         }
 334
 335         default: {
 336                 err = obd_iocontrol(cmd, obd->obd_self_export, len, data, NULL);
 337                 if (err)
 338                         GOTO(out, err);
 339
 340                 if (copy_to_user((void __user *)arg, data, len))
 341                         err = -EFAULT;
 342                 GOTO(out, err);
 343         }
 344         }
 345
 346 out:
 347         OBD_FREE_LARGE(buf, len);
 348         RETURN(err);
 349 } /* class_handle_ioctl */
 350
 351 static int obd_init_checks(void)
 352 {
 353         __u64 u64val, div64val;
 354         char buf[64];
 355         int len, ret = 0;
 356
 357         CDEBUG(D_INFO, "OBD_OBJECT_EOF = %#llx\n", (__u64)OBD_OBJECT_EOF);
 358
 359         u64val = OBD_OBJECT_EOF;
 360         CDEBUG(D_INFO, "u64val OBD_OBJECT_EOF = %#llx\n", u64val);
 361         if (u64val != OBD_OBJECT_EOF) {
 362                 CERROR("__u64 %#llx(%d) != 0xffffffffffffffff\n",
 363                        u64val, (int)sizeof(u64val));
 364                 ret = -EINVAL;
 365         }
 366         len = snprintf(buf, sizeof(buf), "%#llx", u64val);
 367         if (len != 18) {
 368                 CWARN("u64 hex wrong length! strlen(%s)=%d != 18\n", buf, len);
 369                 ret = -EINVAL;
 370         }
 371
 372         div64val = OBD_OBJECT_EOF;
 373         CDEBUG(D_INFO, "u64val OBD_OBJECT_EOF = %#llx\n", u64val);
 374         if (u64val != OBD_OBJECT_EOF) {
 375                 CERROR("__u64 %#llx(%d) != 0xffffffffffffffff\n",
 376                        u64val, (int)sizeof(u64val));
 377                 ret = -EOVERFLOW;
 378         }
 379         if (u64val >> 8 != OBD_OBJECT_EOF >> 8) {
 380                 CERROR("__u64 %#llx(%d) != 0xffffffffffffffff\n",
 381                        u64val, (int)sizeof(u64val));
 382                 return -EOVERFLOW;
 383         }
 384         if (do_div(div64val, 256) != (u64val & 255)) {
 385                 CERROR("do_div(%#llx,256) != %llu\n", u64val, u64val & 255);
 386                 return -EOVERFLOW;
 387         }
 388         if (u64val >> 8 != div64val) {
 389                 CERROR("do_div(%#llx,256) %llu != %llu\n",
 390                        u64val, div64val, u64val >> 8);
 391                 return -EOVERFLOW;
 392         }
 393         len = snprintf(buf, sizeof(buf), "%#llx", u64val);
 394         if (len != 18) {
 395                 CWARN("u64 hex wrong length! strlen(%s)=%d != 18\n", buf, len);
 396                 ret = -EINVAL;
 397         }
 398         len = snprintf(buf, sizeof(buf), "%llu", u64val);
 399         if (len != 20) {
 400                 CWARN("u64 wrong length! strlen(%s)=%d != 20\n", buf, len);
 401                 ret = -EINVAL;
 402         }
 403         len = snprintf(buf, sizeof(buf), "%lld", u64val);
 404         if (len != 2) {
 405                 CWARN("s64 wrong length! strlen(%s)=%d != 2\n", buf, len);
 406                 ret = -EINVAL;
 407         }
 408         if ((u64val & ~PAGE_MASK) >= PAGE_SIZE) {
 409                 CWARN("mask failed: u64val %llu >= %llu\n", u64val,
 410                       (__u64)PAGE_SIZE);
 411                 ret = -EINVAL;
 412         }
 413
 414         return ret;
 415 }
 416
 417 static int __init obdclass_init(void)
 418 {
 419         int err;
 420
 421         LCONSOLE_INFO("Lustre: Build Version: "LUSTRE_VERSION_STRING"\n");
 422
 423         libcfs_kkuc_init();
 424
 425         err = obd_init_checks();
 426         if (err == -EOVERFLOW)
 427                 return err;
 428
 429 #ifdef CONFIG_PROC_FS
 430         obd_memory = lprocfs_alloc_stats(OBD_STATS_NUM,
 431                                          LPROCFS_STATS_FLAG_NONE |
 432                                          LPROCFS_STATS_FLAG_IRQ_SAFE);
 433         if (obd_memory == NULL) {
 434                 CERROR("kmalloc of 'obd_memory' failed\n");
 435                 return -ENOMEM;
 436         }
 437
 438         lprocfs_counter_init(obd_memory, OBD_MEMORY_STAT,
 439                              LPROCFS_CNTR_AVGMINMAX,
 440                              "memused", "bytes");
 441 #endif
 442         err = obd_zombie_impexp_init();
 443         if (err)
 444                 goto cleanup_obd_memory;
 445
 446         err = class_handle_init();
 447         if (err)
 448                 goto cleanup_zombie_impexp;
 449
 450         err = misc_register(&obd_psdev);
 451         if (err) {
 452                 CERROR("cannot register OBD miscdevice: err = %d\n", err);
 453                 goto cleanup_class_handle;
 454         }
 455
 456         /* Default the dirty page cache cap to 1/2 of system memory.
 457          * For clients with less memory, a larger fraction is needed
 458          * for other purposes (mostly for BGL). */
 459         if (totalram_pages <= 512 << (20 - PAGE_SHIFT))
 460                 obd_max_dirty_pages = totalram_pages / 4;
 461         else
 462                 obd_max_dirty_pages = totalram_pages / 2;
 463
 464         err = obd_init_caches();
 465         if (err)
 466                 goto cleanup_deregister;
 467
 468         err = class_procfs_init();
 469         if (err)
 470                 goto cleanup_caches;
 471
 472         err = lu_global_init();
 473         if (err)
 474                 goto cleanup_class_procfs;
 475
 476         err = cl_global_init();
 477         if (err != 0)
 478                 goto cleanup_lu_global;
 479
 480 #ifdef HAVE_SERVER_SUPPORT
 481         err = dt_global_init();
 482         if (err != 0)
 483                 goto cleanup_cl_global;
 484
 485         err = lu_ucred_global_init();
 486         if (err != 0)
 487                 goto cleanup_dt_global;
 488 #endif /* HAVE_SERVER_SUPPORT */
 489
 490         err = llog_info_init();
 491         if (err)
 492 #ifdef HAVE_SERVER_SUPPORT
 493                 goto cleanup_lu_ucred_global;
 494 #else /* !HAVE_SERVER_SUPPORT */
 495                 goto cleanup_cl_global;
 496 #endif /* HAVE_SERVER_SUPPORT */
 497
 498         err = lustre_register_fs();
 499
 500         /* simulate a late OOM situation now to require all
 501          * alloc'ed/initialized resources to be freed */
 502         if (OBD_FAIL_CHECK(OBD_FAIL_OBDCLASS_MODULE_LOAD)) {
 503                 /* fake error but filesystem has been registered */
 504                 lustre_unregister_fs();
 505                 /* force error to ensure module will be unloaded/cleaned */
 506                 err = -ENOMEM;
 507         }
 508
 509         if (err)
 510                 goto cleanup_llog_info;
 511
 512         return 0;
 513
 514 cleanup_llog_info:
 515         llog_info_fini();
 516
 517 #ifdef HAVE_SERVER_SUPPORT
 518 cleanup_lu_ucred_global:
 519         lu_ucred_global_fini();
 520
 521 cleanup_dt_global:
 522         dt_global_fini();
 523 #endif /* HAVE_SERVER_SUPPORT */
 524
 525 cleanup_cl_global:
 526         cl_global_fini();
 527
 528 cleanup_lu_global:
 529         lu_global_fini();
 530
 531 cleanup_class_procfs:
 532         class_procfs_clean();
 533
 534 cleanup_caches:
 535         obd_cleanup_caches();
 536
 537 cleanup_deregister:
 538         misc_deregister(&obd_psdev);
 539
 540 cleanup_class_handle:
 541         class_handle_cleanup();
 542
 543 cleanup_zombie_impexp:
 544         obd_zombie_impexp_stop();
 545
 546 cleanup_obd_memory:
 547 #ifdef CONFIG_PROC_FS
 548         lprocfs_free_stats(&obd_memory);
 549 #endif
 550
 551         return err;
 552 }
 553
 554 void obd_update_maxusage(void)
 555 {
 556         __u64 max;
 557
 558         max = obd_memory_sum();
 559
 560         spin_lock(&obd_updatemax_lock);
 561         if (max > obd_max_alloc)
 562                 obd_max_alloc = max;
 563         spin_unlock(&obd_updatemax_lock);
 564 }
 565 EXPORT_SYMBOL(obd_update_maxusage);
 566
 567 #ifdef CONFIG_PROC_FS
 568 __u64 obd_memory_max(void)
 569 {
 570         __u64 ret;
 571
 572         obd_update_maxusage();
 573         spin_lock(&obd_updatemax_lock);
 574         ret = obd_max_alloc;
 575         spin_unlock(&obd_updatemax_lock);
 576
 577         return ret;
 578 }
 579 #endif /* CONFIG_PROC_FS */
 580
 581 static void __exit obdclass_exit(void)
 582 {
 583 #ifdef CONFIG_PROC_FS
 584         __u64 memory_leaked;
 585         __u64 memory_max;
 586 #endif /* CONFIG_PROC_FS */
 587         ENTRY;
 588
 589         lustre_unregister_fs();
 590
 591         misc_deregister(&obd_psdev);
 592         llog_info_fini();
 593 #ifdef HAVE_SERVER_SUPPORT
 594         lu_ucred_global_fini();
 595         dt_global_fini();
 596 #endif /* HAVE_SERVER_SUPPORT */
 597         cl_global_fini();
 598         lu_global_fini();
 599
 600         obd_cleanup_caches();
 601
 602         class_procfs_clean();
 603
 604         class_handle_cleanup();
 605         class_del_uuid(NULL); /* Delete all UUIDs. */
 606         obd_zombie_impexp_stop();
 607
 608 #ifdef CONFIG_PROC_FS
 609         memory_leaked = obd_memory_sum();
 610         memory_max = obd_memory_max();
 611
 612         lprocfs_free_stats(&obd_memory);
 613         CDEBUG((memory_leaked) ? D_ERROR : D_INFO,
 614                "obd_memory max: %llu, leaked: %llu\n",
 615                memory_max, memory_leaked);
 616 #endif /* CONFIG_PROC_FS */
 617
 618         EXIT;
 619 }
 620
 621 MODULE_AUTHOR("OpenSFS, Inc. <http://www.lustre.org/>");
 622 MODULE_DESCRIPTION("Lustre Class Driver");
 623 MODULE_VERSION(LUSTRE_VERSION_STRING);
 624 MODULE_LICENSE("GPL");
 625
 626 module_init(obdclass_init);
 627 module_exit(obdclass_exit);