From: Mr NeilBrown Date: Mon, 28 Dec 2020 20:56:12 +0000 (-0500) Subject: LU-12514 llite: move client mounting from obdclass to llite X-Git-Tag: 2.14.51~61 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=53fa81765750e38f7879ed5092fd729c1bdc8a0f LU-12514 llite: move client mounting from obdclass to llite Mounting a lustre client is currently handled in obdclass, using services from llite. This requires obdclass to load the llite module and set up inter-module linkage. The purpose of this was for common code to support both client and server mounts. This isn't really a good idea and need to go. For lustre servers we already use a separate filesystem type. So move the mounting code from obdclass/obd_mount to llite/super25 and remove the inter-module linkages. Add some EXPORT_SYMBOL() so that llite can access some helpers that remain in obdclass. Linux-commit: a989830c88149511ee840356d9c1b34304bac576 Change-Id: Ia33bd55a042f90b178156c745a8072b516f00568 Signed-off-by: Mr NeilBrown Reviewed-on: https://review.whamcloud.com/37693 Tested-by: Maloo Tested-by: jenkins Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin --- diff --git a/lustre.spec.in b/lustre.spec.in index 225880d..486c3a4 100644 --- a/lustre.spec.in +++ b/lustre.spec.in @@ -646,7 +646,9 @@ fi %{_bindir}/lfs %{_bindir}/lfs_migrate /sbin/mount.lustre +%if %{with servers} /sbin/mount.lustre_tgt +%endif @BASH_COMPLETION_DIR@ %endif diff --git a/lustre/include/lustre_disk.h b/lustre/include/lustre_disk.h index 66ec909..3a071d4 100644 --- a/lustre/include/lustre_disk.h +++ b/lustre/include/lustre_disk.h @@ -350,16 +350,18 @@ int target_name2index(const char *svname, u32 *idx, const char **endptr); int lustre_put_lsi(struct super_block *sb); int lustre_start_simple(char *obdname, char *type, char *uuid, char *s1, char *s2, char *s3, char *s4); -int lustre_start_mgc(struct super_block *sb); #endif /* HAVE_SERVER_SUPPORT */ int server_name2fsname(const char *svname, char *fsname, const char **endptr); void obdname2fsname(const char *tgt, char *fsname, size_t fslen); -void lustre_register_super_ops(struct module *mod, - int (*cfs)(struct super_block *sb), - void (*ksc)(struct super_block *sb)); +int lustre_start_mgc(struct super_block *sb); int lustre_common_put_super(struct super_block *sb); +struct lustre_sb_info *lustre_init_lsi(struct super_block *sb); +int lustre_put_lsi(struct super_block *sb); +int lmd_parse(char *options, struct lustre_mount_data *lmd); + +/* mgc_request.c */ int mgc_fsname2resid(char *fsname, struct ldlm_res_id *res_id, int type); int mgc_logname2resid(char *fsname, struct ldlm_res_id *res_id, int type); diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 8114658..cdf9e9a 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -1829,9 +1829,9 @@ void lustre_deregister_lwp_item(struct obd_export **exp); struct obd_export *lustre_find_lwp_by_index(const char *dev, __u32 idx); void lustre_notify_lwp_list(struct obd_export *exp); int tgt_name2lwp_name(const char *tgt_name, char *lwp_name, int len, __u32 idx); +int lustre_tgt_register_fs(void); +void lustre_tgt_unregister_fs(void); #endif /* HAVE_SERVER_SUPPORT */ -int lustre_register_fs(void); -int lustre_unregister_fs(void); int lustre_check_exclusion(struct super_block *sb, char *svname); /* lustre_peer.c */ diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c index 5bdef51..f7a7cb3 100644 --- a/lustre/llite/llite_lib.c +++ b/lustre/llite/llite_lib.c @@ -1428,8 +1428,6 @@ out_no_sbi: cl_env_cache_purge(~0); - module_put(THIS_MODULE); - EXIT; } /* client_put_super */ diff --git a/lustre/llite/super25.c b/lustre/llite/super25.c index 272cac9..8f8d93b 100644 --- a/lustre/llite/super25.c +++ b/lustre/llite/super25.c @@ -32,6 +32,8 @@ #define DEBUG_SUBSYSTEM S_LLITE +#define D_MOUNT (D_SUPER | D_CONFIG/*|D_WARNING */) + #include #include #include @@ -94,6 +96,121 @@ const struct super_operations lustre_super_operations = .show_options = ll_show_options, }; +/** + * This is the entry point for the mount call into Lustre. + * This is called when a client is mounted, and this is + * where we start setting things up. + * + * @lmd2data data Mount options (e.g. -o flock,abort_recov) + */ +static int lustre_fill_super(struct super_block *sb, void *lmd2_data, + int silent) +{ + struct lustre_mount_data *lmd; + struct lustre_sb_info *lsi; + int rc; + + ENTRY; + + CDEBUG(D_MOUNT|D_VFSTRACE, "VFS Op: sb %p\n", sb); + + lsi = lustre_init_lsi(sb); + if (!lsi) + RETURN(-ENOMEM); + lmd = lsi->lsi_lmd; + + /* + * Disable lockdep during mount, because mount locking patterns are + * 'special'. + */ + lockdep_off(); + + /* + * LU-639: the OBD cleanup of last mount may not finish yet, wait here. + */ + obd_zombie_barrier(); + + /* Figure out the lmd from the mount options */ + if (lmd_parse(lmd2_data, lmd)) { + lustre_put_lsi(sb); + GOTO(out, rc = -EINVAL); + } + + if (!lmd_is_client(lmd)) { +#ifdef HAVE_SERVER_SUPPORT +#if LUSTRE_VERSION_CODE > OBD_OCD_VERSION(2, 15, 51, 0) + static bool printed; + + if (!printed) { + LCONSOLE_WARN("%s: mounting server target with '-t lustre' deprecated, use '-t lustre_tgt'\n", + lmd->lmd_profile); + printed = true; + } +#endif + rc = server_fill_super(sb); +#else + rc = -ENODEV; + CERROR("%s: This is client-side-only module, cannot handle server mount: rc = %d\n", + lmd->lmd_profile, rc); + lustre_put_lsi(sb); +#endif + GOTO(out, rc); + } + + CDEBUG(D_MOUNT, "Mounting client %s\n", lmd->lmd_profile); + rc = lustre_start_mgc(sb); + if (rc) { + lustre_common_put_super(sb); + GOTO(out, rc); + } + /* Connect and start */ + rc = ll_fill_super(sb); + /* ll_file_super will call lustre_common_put_super on failure, + * which takes care of the module reference. + * + * If error happens in fill_super() call, @lsi will be killed there. + * This is why we do not put it here. + */ +out: + if (rc) { + CERROR("llite: Unable to mount %s: rc = %d\n", + s2lsi(sb) ? lmd->lmd_dev : "", rc); + } else { + CDEBUG(D_SUPER, "%s: Mount complete\n", + lmd->lmd_dev); + } + lockdep_on(); + return rc; +} + +/***************** FS registration ******************/ +static struct dentry *lustre_mount(struct file_system_type *fs_type, int flags, + const char *devname, void *data) +{ + return mount_nodev(fs_type, flags, data, lustre_fill_super); +} + +static void lustre_kill_super(struct super_block *sb) +{ + struct lustre_sb_info *lsi = s2lsi(sb); + + if (lsi && !IS_SERVER(lsi)) + ll_kill_super(sb); + + kill_anon_super(sb); +} + +/** Register the "lustre" fs type + */ +static struct file_system_type lustre_fs_type = { + .owner = THIS_MODULE, + .name = "lustre", + .mount = lustre_mount, + .kill_sb = lustre_kill_super, + .fs_flags = FS_RENAME_DOES_D_MOVE, +}; +MODULE_ALIAS_FS("lustre"); + static int __init lustre_init(void) { struct lnet_process_id lnet_id; @@ -161,10 +278,14 @@ static int __init lustre_init(void) if (rc != 0) GOTO(out_inode_fini_env, rc); - lustre_register_super_ops(THIS_MODULE, ll_fill_super, ll_kill_super); + rc = register_filesystem(&lustre_fs_type); + if (rc) + GOTO(out_xattr, rc); RETURN(0); +out_xattr: + ll_xattr_fini(); out_inode_fini_env: cl_env_put(cl_inode_fini_env, &cl_inode_fini_refcheck); out_vvp: @@ -180,7 +301,7 @@ out_cache: static void __exit lustre_exit(void) { - lustre_register_super_ops(NULL, NULL, NULL); + unregister_filesystem(&lustre_fs_type); llite_tunables_unregister(); diff --git a/lustre/obdclass/class_obd.c b/lustre/obdclass/class_obd.c index a47d434..7fab38a 100644 --- a/lustre/obdclass/class_obd.c +++ b/lustre/obdclass/class_obd.c @@ -717,50 +717,52 @@ static int __init obdclass_init(void) if (err != 0) goto cleanup_lu_global; + err = llog_info_init(); + if (err) + goto cleanup_cl_global; + #ifdef HAVE_SERVER_SUPPORT err = dt_global_init(); if (err != 0) - goto cleanup_cl_global; + goto cleanup_llog_info; err = lu_ucred_global_init(); if (err != 0) goto cleanup_dt_global; -#endif /* HAVE_SERVER_SUPPORT */ - err = llog_info_init(); - if (err) -#ifdef HAVE_SERVER_SUPPORT + err = lustre_tgt_register_fs(); + if (err && err != -EBUSY) { + /* Don't fail if server code also registers "lustre_tgt" */ + CERROR("obdclass: register fstype 'lustre_tgt' failed: rc = %d\n", + err); goto cleanup_lu_ucred_global; -#else /* !HAVE_SERVER_SUPPORT */ - goto cleanup_cl_global; + } #endif /* HAVE_SERVER_SUPPORT */ - err = lustre_register_fs(); - /* simulate a late OOM situation now to require all - * alloc'ed/initialized resources to be freed */ + * alloc'ed/initialized resources to be freed + */ if (OBD_FAIL_CHECK(OBD_FAIL_OBDCLASS_MODULE_LOAD)) { - /* fake error but filesystem has been registered */ - lustre_unregister_fs(); /* force error to ensure module will be unloaded/cleaned */ err = -ENOMEM; + goto cleanup_all; } - - if (err) - goto cleanup_llog_info; - return 0; -cleanup_llog_info: - llog_info_fini(); - +cleanup_all: #ifdef HAVE_SERVER_SUPPORT + /* fake error but filesystem has been registered */ + lustre_tgt_unregister_fs(); + cleanup_lu_ucred_global: lu_ucred_global_fini(); cleanup_dt_global: dt_global_fini(); + +cleanup_llog_info: #endif /* HAVE_SERVER_SUPPORT */ + llog_info_fini(); cleanup_cl_global: cl_global_fini(); @@ -826,14 +828,13 @@ static void __exit obdclass_exit(void) #endif /* CONFIG_PROC_FS */ ENTRY; - lustre_unregister_fs(); - misc_deregister(&obd_psdev); - llog_info_fini(); #ifdef HAVE_SERVER_SUPPORT + lustre_tgt_unregister_fs(); lu_ucred_global_fini(); dt_global_fini(); #endif /* HAVE_SERVER_SUPPORT */ + llog_info_fini(); cl_global_fini(); lu_global_fini(); diff --git a/lustre/obdclass/obd_mount.c b/lustre/obdclass/obd_mount.c index 3ec7d9c..37165a2 100644 --- a/lustre/obdclass/obd_mount.c +++ b/lustre/obdclass/obd_mount.c @@ -51,12 +51,6 @@ #include #include -static DEFINE_SPINLOCK(client_lock); -static struct module *client_mod; -static int (*client_fill_super)(struct super_block *sb); - -static void (*kill_super_cb)(struct super_block *sb); - /**************** config llog ********************/ /** @@ -501,6 +495,7 @@ out_free: OBD_FREE(niduuid, len + 2); RETURN(rc); } +EXPORT_SYMBOL(lustre_start_mgc); static int lustre_stop_mgc(struct super_block *sb) { @@ -584,7 +579,7 @@ out: /***************** lustre superblock **************/ -static struct lustre_sb_info *lustre_init_lsi(struct super_block *sb) +struct lustre_sb_info *lustre_init_lsi(struct super_block *sb) { struct lustre_sb_info *lsi; @@ -610,6 +605,7 @@ static struct lustre_sb_info *lustre_init_lsi(struct super_block *sb) RETURN(lsi); } +EXPORT_SYMBOL(lustre_init_lsi); static int lustre_free_lsi(struct super_block *sb) { @@ -1296,7 +1292,7 @@ static int lmd_parse_nidlist(char *buf, char **endh) * e.g. mount -v -t lustre -o abort_recov uml1:uml2:/lustre-client /mnt/lustre * dev is passed as device=uml1:/lustre by mount.lustre_tgt */ -static int lmd_parse(char *options, struct lustre_mount_data *lmd) +int lmd_parse(char *options, struct lustre_mount_data *lmd) { char *s1, *s2, *devname = NULL; struct lustre_mount_data *raw = (struct lustre_mount_data *)options; @@ -1585,15 +1581,17 @@ invalid: CERROR("Bad mount options %s\n", options); RETURN(-EINVAL); } +EXPORT_SYMBOL(lmd_parse); +#ifdef HAVE_SERVER_SUPPORT /** * This is the entry point for the mount call into Lustre. - * This is called when a server or client is mounted, + * This is called when a server target is mounted, * and this is where we start setting things up. * @param data Mount options (e.g. -o flock,abort_recov) */ -static int lustre_fill_super(struct super_block *sb, void *lmd2_data, - int silent) +static int lustre_tgt_fill_super(struct super_block *sb, void *lmd2_data, + int silent) { struct lustre_mount_data *lmd; struct lustre_sb_info *lsi; @@ -1626,54 +1624,25 @@ static int lustre_fill_super(struct super_block *sb, void *lmd2_data, } if (lmd_is_client(lmd)) { - bool have_client = false; - - CDEBUG(D_MOUNT, "Mounting client %s\n", lmd->lmd_profile); - if (!client_fill_super) - request_module("lustre"); - spin_lock(&client_lock); - if (client_fill_super && try_module_get(client_mod)) - have_client = true; - spin_unlock(&client_lock); - if (!have_client) { - LCONSOLE_ERROR_MSG(0x165, - "Nothing registered for client mount! Is the 'lustre' module loaded?\n"); - lustre_put_lsi(sb); - rc = -ENODEV; - } else { - rc = lustre_start_mgc(sb); - if (rc) { - lustre_common_put_super(sb); - GOTO(out, rc); - } - /* Connect and start */ - /* (should always be ll_fill_super) */ - rc = (*client_fill_super)(sb); - /* c_f_s will call lustre_common_put_super on failure, - * which takes care of the module reference. - */ - } - } else { -#ifdef HAVE_SERVER_SUPPORT - CDEBUG(D_MOUNT, "Mounting server from %s\n", lmd->lmd_dev); - rc = server_fill_super(sb); - /* - * s_f_s calls lustre_start_mgc after the mount because we need - * the MGS NIDs which are stored on disk. Plus, we may - * need to start the MGS first. - */ - /* s_f_s will call server_put_super on failure */ -#else - CERROR("client-side-only module, cannot handle server mount\n"); - rc = -EINVAL; -#endif + rc = -ENODEV; + CERROR("%s: attempting to mount a client with -t lustre_tgt' which is only for server-side mounts: rc = %d\n", + lmd->lmd_dev, rc); + lustre_put_lsi(sb); + GOTO(out, rc); } + CDEBUG(D_MOUNT, "Mounting server from %s\n", lmd->lmd_dev); + rc = server_fill_super(sb); /* + * server_fill_super calls lustre_start_mgc after the mount + * because we need the MGS NIDs which are stored on disk. + * Plus, we may need to start the MGS first. + * + * server_fill_super will call server_put_super on failure + * * If error happens in fill_super() call, @lsi will be killed there. * This is why we do not put it here. */ - GOTO(out, rc); out: if (rc) { CERROR("Unable to mount %s (%d)\n", @@ -1686,41 +1655,14 @@ out: return rc; } - -/* - * We can't call ll_fill_super by name because it lives in a module that - * must be loaded after this one. - */ -void lustre_register_super_ops(struct module *mod, - int (*cfs)(struct super_block *sb), - void (*ksc)(struct super_block *sb)) -{ - spin_lock(&client_lock); - client_mod = mod; - client_fill_super = cfs; - kill_super_cb = ksc; - spin_unlock(&client_lock); -} -EXPORT_SYMBOL(lustre_register_super_ops); - /***************** FS registration ******************/ -static struct dentry *lustre_mount(struct file_system_type *fs_type, int flags, - const char *devname, void *data) +static struct dentry *lustre_tgt_mount(struct file_system_type *fs_type, + int flags, const char *devname, + void *data) { - return mount_nodev(fs_type, flags, data, lustre_fill_super); + return mount_nodev(fs_type, flags, data, lustre_tgt_fill_super); } -static void lustre_kill_super(struct super_block *sb) -{ - struct lustre_sb_info *lsi = s2lsi(sb); - - if (kill_super_cb && lsi && !IS_SERVER(lsi)) - (*kill_super_cb)(sb); - - kill_anon_super(sb); -} - -#ifdef HAVE_SERVER_SUPPORT /* Register the "lustre_tgt" fs type. * * Right now this isn't any different than the normal "lustre" filesystem @@ -1731,53 +1673,23 @@ static void lustre_kill_super(struct super_block *sb) * * The long-term goal is to disentangle the client and server mount code. */ -static struct file_system_type lustre_fs_type_tgt = { +static struct file_system_type lustre_tgt_fstype = { .owner = THIS_MODULE, .name = "lustre_tgt", - .mount = lustre_mount, - .kill_sb = lustre_kill_super, + .mount = lustre_tgt_mount, + .kill_sb = kill_anon_super, .fs_flags = FS_REQUIRES_DEV | FS_RENAME_DOES_D_MOVE, }; MODULE_ALIAS_FS("lustre_tgt"); -#define register_filesystem_tgt(fstype) \ -do { \ - int _rc; \ - \ - _rc = register_filesystem(fstype); \ - if (_rc && _rc != -EBUSY) { \ - /* Don't fail if server code also registers "lustre_tgt" */ \ - CERROR("obdclass: register fstype '%s' failed: rc = %d\n", \ - (fstype)->name, _rc); \ - return _rc; \ - } \ -} while (0) -#define unregister_filesystem_tgt(fstype) unregister_filesystem(fstype) -#else -#define register_filesystem_tgt(fstype) do {} while (0) -#define unregister_filesystem_tgt(fstype) do {} while (0) -#endif - -/* Register the "lustre" fs type */ -static struct file_system_type lustre_fs_type = { - .owner = THIS_MODULE, - .name = "lustre", - .mount = lustre_mount, - .kill_sb = lustre_kill_super, - .fs_flags = FS_RENAME_DOES_D_MOVE, -}; -MODULE_ALIAS_FS("lustre"); - -int lustre_register_fs(void) +int lustre_tgt_register_fs(void) { - register_filesystem_tgt(&lustre_fs_type_tgt); - - return register_filesystem(&lustre_fs_type); + return register_filesystem(&lustre_tgt_fstype); } -int lustre_unregister_fs(void) +void lustre_tgt_unregister_fs(void) { - unregister_filesystem_tgt(&lustre_fs_type_tgt); - - return unregister_filesystem(&lustre_fs_type); + unregister_filesystem(&lustre_tgt_fstype); } + +#endif /* HAVE_SERVER_SUPPORT */ diff --git a/lustre/obdclass/obd_mount_server.c b/lustre/obdclass/obd_mount_server.c index d9166d3..acd9222 100644 --- a/lustre/obdclass/obd_mount_server.c +++ b/lustre/obdclass/obd_mount_server.c @@ -2048,6 +2048,7 @@ out_mnt: server_put_super(sb); return rc; } +EXPORT_SYMBOL(server_fill_super); /* * Calculate timeout value for a target. diff --git a/lustre/utils/Makefile.am b/lustre/utils/Makefile.am index 1d5a3ea..06d9b60 100644 --- a/lustre/utils/Makefile.am +++ b/lustre/utils/Makefile.am @@ -23,7 +23,10 @@ GSSLIB = endif # mount only finds helpers in /sbin -rootsbin_PROGRAMS = mount.lustre mount.lustre_tgt +rootsbin_PROGRAMS = mount.lustre +if SERVER +rootsbin_PROGRAMS += mount.lustre_tgt +endif bin_SCRIPTS = llstat llobdstat plot-llstat bin_PROGRAMS = lfs sbin_SCRIPTS = ldlm_debug_upcall