LU-9859 libcfs: refactor libcfs initialization.

Many lustre modules depend on libcfs having initialized
properly, but do not explicit check that it did.
When lustre is built as discrete modules, this does not
cause a problem because if the libcfs module fails
initialization, the other modules don't even get loaded.

When lustre is compiled into the kernel, all module_init()
routines get run, so they need to check the required initialization
succeeded.

This patch splits out the initialization of libcfs into a new
libcfs_setup(), and has all modules call that.

The misc_register() call is kept separate as it does not allocate any
resources and if it fails, it fails hard - no point in retrying.
Other set-up allocates resources and so is best delayed until they
are needed, and can be worth retrying.

Ideally, the initialization would happen at mount time (or similar)
rather than at load time.  Doing this requires each module to
check dependencies when they are activated rather than when
they are loaded.  Achieving that is a much larger job that would
have to progress in stages.

For now, this change ensures that if some initialization in libcfs
fails, other modules will fail-safe.

Linux-commit: 64bf0b1a079d61e9e059b9dc7a58e064c7d994ae

Change-Id: I6b5ecdba0defc6e033f78d8fc2b9be9e26c7f720
Signed-off-by: Mr. NeilBrown <neilb@suse.de>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/52700
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Timothy Day <timday@amazon.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/libcfs/include/libcfs/libcfs.h b/libcfs/include/libcfs/libcfs.h
index 12d9dfd..0c646f9 100644 (file)
--- a/libcfs/include/libcfs/libcfs.h
+++ b/libcfs/include/libcfs/libcfs.h
@@ -67,6 +67,7 @@
 
 typedef s32 timeout_t;
 
+int libcfs_setup(void);
 int libcfs_ioctl(unsigned int cmd, struct libcfs_ioctl_data *data);
 
 extern struct workqueue_struct *cfs_rehash_wq;

diff --git a/libcfs/include/libcfs/linux/linux-misc.h b/libcfs/include/libcfs/linux/linux-misc.h
index 2f285a5..9e4c787 100644 (file)
--- a/libcfs/include/libcfs/linux/linux-misc.h
+++ b/libcfs/include/libcfs/linux/linux-misc.h
@@ -119,7 +119,7 @@ static inline int kref_read(const struct kref *kref)
 }
 #endif /* HAVE_KREF_READ */
 
-void cfs_arch_init(void);
+int cfs_arch_init(void);
 void cfs_arch_exit(void);
 
 #ifndef container_of_safe

diff --git a/libcfs/libcfs/linux/linux-prim.c b/libcfs/libcfs/linux/linux-prim.c
index 8e0ee07..660d328 100644 (file)
--- a/libcfs/libcfs/linux/linux-prim.c
+++ b/libcfs/libcfs/linux/linux-prim.c
@@ -47,6 +47,7 @@
 #endif
 
 #include <lustre_compat.h>
+#include <libcfs/crypto/llcrypt.h>
 #include <libcfs/linux/linux-time.h>
 #include <libcfs/linux/linux-wait.h>
 #include <libcfs/linux/linux-misc.h>
@@ -202,7 +203,7 @@ void __init init_libcfs_vfree_atomic(void)
        }
 }
 
-void __init cfs_arch_init(void)
+int __init cfs_arch_init(void)
 {
        init_libcfs_vfree_atomic();
 
@@ -221,12 +222,15 @@ void __init cfs_arch_init(void)
                                          SLAB_PANIC | SLAB_RECLAIM_ACCOUNT,
                                          xarray_node_ctor);
 #endif
+       return llcrypt_init();
 }
 
 void __exit cfs_arch_exit(void)
 {
        /* exit_libcfs_vfree_atomic */
        flush_scheduled_work();
+
+       llcrypt_exit();
 }
 
 int cfs_kernel_write(struct file *filp, const void *buf, size_t count,

diff --git a/libcfs/libcfs/module.c b/libcfs/libcfs/module.c
index 201b5e3..fb7e861 100644 (file)
--- a/libcfs/libcfs/module.c
+++ b/libcfs/libcfs/module.c
@@ -660,17 +660,23 @@ void lnet_remove_debugfs(struct ctl_table *table)
 }
 EXPORT_SYMBOL_GPL(lnet_remove_debugfs);
 
+static DEFINE_MUTEX(libcfs_startup);
+static int libcfs_active;
+
 static void *debugfs_state;
-static int __init libcfs_init(void)
+
+int libcfs_setup(void)
 {
-       int rc;
+       int rc = -EINVAL;
 
-       cfs_arch_init();
+       mutex_lock(&libcfs_startup);
+       if (libcfs_active)
+               goto out;
 
        rc = libcfs_debug_init(5 * 1024 * 1024);
        if (rc < 0) {
                pr_err("LustreError: libcfs_debug_init: rc = %d\n", rc);
-               return (rc);
+               goto cleanup_lock;
        }
 
        rc = cfs_cpu_init();
@@ -687,30 +693,16 @@ static int __init libcfs_init(void)
 
        rc = cfs_crypto_register();
        if (rc) {
-               CERROR("cfs_crypto_regster: error %d\n", rc);
+               CERROR("cfs_crypto_register: error %d\n", rc);
                goto cleanup_wq;
        }
 
-       lnet_insert_debugfs(lnet_table, THIS_MODULE, &debugfs_state);
-       if (!IS_ERR_OR_NULL(lnet_debugfs_root))
-               lnet_insert_debugfs_links(lnet_debugfs_symlinks);
-
-       rc = llcrypt_init();
-       if (rc) {
-               CERROR("llcrypt_init: error %d\n", rc);
-               goto cleanup_lnet;
-       }
-
-       CDEBUG(D_OTHER, "portals setup OK\n");
+       CDEBUG(D_OTHER, "libcfs setup OK\n");
+out:
+       libcfs_active = 1;
+       mutex_unlock(&libcfs_startup);
        return 0;
 
-cleanup_lnet:
-       if (!IS_ERR_OR_NULL(lnet_debugfs_root)) {
-               debugfs_remove_recursive(lnet_debugfs_root);
-               lnet_debugfs_root = NULL;
-               lnet_debugfs_fini(&debugfs_state);
-       }
-       cfs_crypto_unregister();
 cleanup_wq:
        destroy_workqueue(cfs_rehash_wq);
        cfs_rehash_wq = NULL;
@@ -718,6 +710,26 @@ cleanup_cpu:
        cfs_cpu_fini();
 cleanup_debug:
        libcfs_debug_cleanup();
+cleanup_lock:
+       mutex_unlock(&libcfs_startup);
+       return rc;
+}
+EXPORT_SYMBOL(libcfs_setup);
+
+static int __init libcfs_init(void)
+{
+       int rc;
+
+       rc = cfs_arch_init();
+       if (rc < 0) {
+               CERROR("cfs_arch_init: error %d\n", rc);
+               return rc;
+       }
+
+       lnet_insert_debugfs(lnet_table, THIS_MODULE, &debugfs_state);
+       if (!IS_ERR_OR_NULL(lnet_debugfs_root))
+               lnet_insert_debugfs_links(lnet_debugfs_symlinks);
+
        return rc;
 }
 
@@ -734,12 +746,8 @@ static void __exit libcfs_exit(void)
        CDEBUG(D_MALLOC, "before Portals cleanup: kmem %lld\n",
               libcfs_kmem_read());
 
-       llcrypt_exit();
-
-       if (cfs_rehash_wq) {
+       if (cfs_rehash_wq)
                destroy_workqueue(cfs_rehash_wq);
-               cfs_rehash_wq = NULL;
-       }
 
        cfs_crypto_unregister();
 
@@ -753,6 +761,8 @@ static void __exit libcfs_exit(void)
        rc = libcfs_debug_cleanup();
        if (rc)
                pr_err("LustreError: libcfs_debug_cleanup: rc = %d\n", rc);
+
+       cfs_arch_exit();
 }
 
 MODULE_AUTHOR("OpenSFS, Inc. <http://www.lustre.org/>");

diff --git a/lnet/klnds/gnilnd/gnilnd.c b/lnet/klnds/gnilnd/gnilnd.c
index 375b87d..cd6fd8a 100644 (file)
--- a/lnet/klnds/gnilnd/gnilnd.c
+++ b/lnet/klnds/gnilnd/gnilnd.c
@@ -2840,6 +2840,10 @@ static int __init kgnilnd_init(void)
        kgnilnd_insert_sysctl();
        kgnilnd_proc_init();
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        lnet_register_lnd(&the_kgnilnd);
 
        return 0;

diff --git a/lnet/klnds/kfilnd/kfilnd.c b/lnet/klnds/kfilnd/kfilnd.c
index e3d7732..69cc3ce 100644 (file)
--- a/lnet/klnds/kfilnd/kfilnd.c
+++ b/lnet/klnds/kfilnd/kfilnd.c
@@ -511,6 +511,10 @@ static int __init kfilnd_init(void)
        if (rc)
                goto err;
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        /* Do any initialization of the transaction system */
        rc = kfilnd_tn_init();
        if (rc) {

diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c
index 5df62c8..1145ac1 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/lnet/klnds/o2iblnd/o2iblnd.c
@@ -3935,6 +3935,10 @@ static int __init ko2iblnd_init(void)
        if (rc != 0)
                return rc;
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        lnet_register_lnd(&the_o2iblnd);
 
        return 0;

diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c
index 53102e1..23a23ee 100644 (file)
--- a/lnet/klnds/socklnd/socklnd.c
+++ b/lnet/klnds/socklnd/socklnd.c
@@ -2650,6 +2650,10 @@ static int __init ksocklnd_init(void)
        if (rc != 0)
                return rc;
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        lnet_register_lnd(&the_ksocklnd);
 
        return 0;

diff --git a/lnet/lnet/module.c b/lnet/lnet/module.c
index aac5522..b0fe719 100644 (file)
--- a/lnet/lnet/module.c
+++ b/lnet/lnet/module.c
@@ -429,7 +429,11 @@ static struct miscdevice lnet_dev = {
 static int __init lnet_init(void)
 {
        int rc;
+
        ENTRY;
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
 
        rc = lnet_lib_init();
        if (rc != 0) {

diff --git a/lnet/selftest/module.c b/lnet/selftest/module.c
index 93cb266..ee958d8 100644 (file)
--- a/lnet/selftest/module.c
+++ b/lnet/selftest/module.c
@@ -92,6 +92,10 @@ lnet_selftest_init(void)
        int rc = -ENOMEM;
        int i;
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        lst_serial_wq = alloc_ordered_workqueue("lst_s", 0);
        if (!lst_serial_wq) {
                CERROR("Failed to create serial WI scheduler for LST\n");

diff --git a/lustre/fid/fid_request.c b/lustre/fid/fid_request.c
index ed6cd85..3fe55eb 100644 (file)
--- a/lustre/fid/fid_request.c
+++ b/lustre/fid/fid_request.c
@@ -493,8 +493,13 @@ EXPORT_SYMBOL(client_fid_fini);
 static int __init fid_init(void)
 {
        struct dentry *de;
+       int rc;
+
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
 #ifdef HAVE_SERVER_SUPPORT
-       int rc = fid_server_mod_init();
+       rc = fid_server_mod_init();
 
        if (rc)
                return rc;

diff --git a/lustre/fld/fld_request.c b/lustre/fld/fld_request.c
index 5210efe..f5460e0 100644 (file)
--- a/lustre/fld/fld_request.c
+++ b/lustre/fld/fld_request.c
@@ -515,9 +515,13 @@ void fld_client_flush(struct lu_client_fld *fld)
 
 static int __init fld_init(void)
 {
-#ifdef HAVE_SERVER_SUPPORT
        int rc;
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
+#ifdef HAVE_SERVER_SUPPORT
        rc = fld_server_mod_init();
        if (rc)
                return rc;

diff --git a/lustre/lfsck/lfsck_lib.c b/lustre/lfsck/lfsck_lib.c
index 4126509..7617aa4 100644 (file)
--- a/lustre/lfsck/lfsck_lib.c
+++ b/lustre/lfsck/lfsck_lib.c
@@ -3964,6 +3964,10 @@ static int __init lfsck_init(void)
 {
        int rc;
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        lfsck_key_init_generic(&lfsck_thread_key, NULL);
        rc = lu_context_key_register(&lfsck_thread_key);
        if (!rc) {

diff --git a/lustre/llite/super25.c b/lustre/llite/super25.c
index 6622625..367b34f 100644 (file)
--- a/lustre/llite/super25.c
+++ b/lustre/llite/super25.c
@@ -230,6 +230,10 @@ static int __init lustre_init(void)
        BUILD_BUG_ON(sizeof(LUSTRE_VOLATILE_HDR) !=
                     LUSTRE_VOLATILE_HDR_LEN + 1);
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        /* print an address of _any_ initialized kernel symbol from this
         * module, to allow debugging with gdb that doesn't support data
         * symbols from modules.*/

diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c
index f295f77..1147316 100644 (file)
--- a/lustre/lmv/lmv_obd.c
+++ b/lustre/lmv/lmv_obd.c
@@ -4287,6 +4287,12 @@ static const struct md_ops lmv_md_ops = {
 
 static int __init lmv_init(void)
 {
+       int rc;
+
+       rc  = libcfs_setup();
+       if (rc)
+               return rc;
+
        return class_register_type(&lmv_obd_ops, &lmv_md_ops, true,
                                   LUSTRE_LMV_NAME, NULL);
 }

diff --git a/lustre/lod/lod_dev.c b/lustre/lod/lod_dev.c
index 25bfb80..b70245f 100644 (file)
--- a/lustre/lod/lod_dev.c
+++ b/lustre/lod/lod_dev.c
@@ -2799,6 +2799,10 @@ static int __init lod_init(void)
        struct obd_type *sym;
        int rc;
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        rc = lu_kmem_init(lod_caches);
        if (rc)
                return rc;

diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c
index a0c0cf7..8ef710d 100644 (file)
--- a/lustre/lov/lov_obd.c
+++ b/lustre/lov/lov_obd.c
@@ -1357,6 +1357,10 @@ static int __init lov_init(void)
         * symbols from modules.*/
        CDEBUG(D_INFO, "Lustre LOV module (%p).\n", &lov_caches);
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        rc = lu_kmem_init(lov_caches);
        if (rc)
                return rc;

diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c
index f17744a..a54260b 100644 (file)
--- a/lustre/mdc/mdc_request.c
+++ b/lustre/mdc/mdc_request.c
@@ -3065,6 +3065,11 @@ struct class *mdc_changelog_class;
 static int __init mdc_init(void)
 {
        int rc = 0;
+
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        rc = alloc_chrdev_region(&mdc_changelog_dev, 0,
                                 MDC_CHANGELOG_DEV_COUNT,
                                 MDC_CHANGELOG_DEV_NAME);

diff --git a/lustre/mdd/mdd_device.c b/lustre/mdd/mdd_device.c
index f237468..21b5f57 100644 (file)
--- a/lustre/mdd/mdd_device.c
+++ b/lustre/mdd/mdd_device.c
@@ -2426,6 +2426,10 @@ static int __init mdd_init(void)
 {
        int rc;
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        rc = lu_kmem_init(mdd_caches);
        if (rc)
                return rc;

diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index 6d4fe1e..236eaac 100644 (file)
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -7999,6 +7999,11 @@ static int __init mdt_init(void)
                     FID_NOBRACE_LEN + 1);
        BUILD_BUG_ON(sizeof("[0x0123456789ABCDEF:0x01234567:0x01234567]") !=
                     FID_LEN + 1);
+
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        rc = lu_kmem_init(mdt_caches);
        if (rc)
                return rc;

diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c
index 2814b7b..8722dda 100644 (file)
--- a/lustre/mgc/mgc_request.c
+++ b/lustre/mgc/mgc_request.c
@@ -1977,6 +1977,12 @@ MODULE_PARM_DESC(mgc_requeue_timeout_min, "Minimal requeue time to refresh logs"
 
 static int __init mgc_init(void)
 {
+       int rc;
+
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        return class_register_type(&mgc_obd_ops, NULL, false,
                                   LUSTRE_MGC_NAME, NULL);
 }

diff --git a/lustre/mgs/mgs_handler.c b/lustre/mgs/mgs_handler.c
index e8452e0..2c1ed34 100644 (file)
--- a/lustre/mgs/mgs_handler.c
+++ b/lustre/mgs/mgs_handler.c
@@ -1756,6 +1756,12 @@ static const struct obd_ops mgs_obd_device_ops = {
 
 static int __init mgs_init(void)
 {
+       int rc;
+
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        return class_register_type(&mgs_obd_device_ops, NULL, true,
                                   LUSTRE_MGS_NAME, &mgs_device_type);
 }

diff --git a/lustre/obdclass/class_obd.c b/lustre/obdclass/class_obd.c
index 8d2ebc2..43deebd 100644 (file)
--- a/lustre/obdclass/class_obd.c
+++ b/lustre/obdclass/class_obd.c
@@ -707,6 +707,10 @@ static int __init obdclass_init(void)
 
        register_oom_notifier(&obdclass_oom);
 
+       err = libcfs_setup();
+       if (err)
+               return err;
+
        err = obd_init_checks();
        if (err)
                return err;

diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c
index 6c2f925..1cf1a75 100644 (file)
--- a/lustre/obdecho/echo_client.c
+++ b/lustre/obdecho/echo_client.c
@@ -2585,6 +2585,10 @@ static int __init obdecho_init(void)
 
        LASSERT(PAGE_SIZE % OBD_ECHO_BLOCK_SIZE == 0);
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
 # ifdef HAVE_SERVER_SUPPORT
        rc = echo_persistent_pages_init();
        if (rc != 0)

diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c
index 4eb489a..b495b5c 100644 (file)
--- a/lustre/ofd/ofd_dev.c
+++ b/lustre/ofd/ofd_dev.c
@@ -3263,6 +3263,10 @@ static int __init ofd_init(void)
 {
        int rc;
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        rc = lu_kmem_init(ofd_caches);
        if (rc)
                return rc;

diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index 2d5afc8..d64e54c 100644 (file)
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -3947,6 +3947,10 @@ static int __init osc_init(void)
         * symbols from modules.*/
        CDEBUG(D_INFO, "Lustre OSC module (%p).\n", &osc_caches);
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        rc = lu_kmem_init(osc_caches);
        if (rc)
                RETURN(rc);

diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c
index 68ae68a..e42fff1 100644 (file)
--- a/lustre/osd-ldiskfs/osd_handler.c
+++ b/lustre/osd-ldiskfs/osd_handler.c
@@ -9000,6 +9000,10 @@ static int __init osd_init(void)
        BUILD_BUG_ON(sizeof(struct osd_thread_info) > PAGE_SIZE);
 #endif
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        osd_oi_mod_init();
 
        rc = lu_kmem_init(ldiskfs_caches);

diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c
index c29bae6..9c24dd4 100644 (file)
--- a/lustre/osd-zfs/osd_handler.c
+++ b/lustre/osd-zfs/osd_handler.c
@@ -1655,6 +1655,10 @@ static int __init osd_init(void)
 {
        int rc;
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        rc = osd_options_init();
        if (rc)
                return rc;

diff --git a/lustre/osp/osp_dev.c b/lustre/osp/osp_dev.c
index 5460ff7..69c9068 100644 (file)
--- a/lustre/osp/osp_dev.c
+++ b/lustre/osp/osp_dev.c
@@ -1892,6 +1892,10 @@ static int __init osp_init(void)
        struct obd_type *sym;
        int rc;
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        rc = lu_kmem_init(osp_caches);
        if (rc)
                return rc;

diff --git a/lustre/ost/ost_handler.c b/lustre/ost/ost_handler.c
index 8793404..b9e8ff8 100644 (file)
--- a/lustre/ost/ost_handler.c
+++ b/lustre/ost/ost_handler.c
@@ -424,6 +424,9 @@ static int __init ost_init(void)
        int rc;
 
        ENTRY;
+       rc = libcfs_setup();
+       if (rc)
+               RETURN(rc);
 
        rc = class_register_type(&ost_obd_ops, NULL, false,
                                 LUSTRE_OSS_NAME, NULL);

diff --git a/lustre/ptlrpc/ptlrpc_module.c b/lustre/ptlrpc/ptlrpc_module.c
index c21fa62..21a1ae0 100644 (file)
--- a/lustre/ptlrpc/ptlrpc_module.c
+++ b/lustre/ptlrpc/ptlrpc_module.c
@@ -55,6 +55,10 @@ static __init int ptlrpc_init(void)
        ptlrpc_init_xid();
        lustre_msg_early_size_init();
 
+       rc = libcfs_setup();
+       if (rc)
+               RETURN(rc);
+
        rc = req_layout_init();
        if (rc)
                RETURN(rc);

diff --git a/lustre/quota/lquota_lib.c b/lustre/quota/lquota_lib.c
index 8bbc58b..12f175d 100644 (file)
--- a/lustre/quota/lquota_lib.c
+++ b/lustre/quota/lquota_lib.c
@@ -356,6 +356,10 @@ static int __init lquota_init(void)
        int     rc;
        ENTRY;
 
+       rc = libcfs_setup();
+       if (rc)
+               return rc;
+
        lquota_key_init_generic(&lquota_thread_key, NULL);
        lu_context_key_register(&lquota_thread_key);