From 015f0b31e9ceb50cb317d77e5564a978a55986be Mon Sep 17 00:00:00 2001 From: Frederick Dilger Date: Wed, 12 Jun 2024 16:43:08 -0400 Subject: [PATCH] LU-11077 utils: mount with client-only params Client-specific parameters can be added manually to either '/etc/lustre/mount.client.params' or '/etc/lustre/mount.FSNAME.params' to be set upon the next client mount. Signed-off-by: Frederick Dilger Change-Id: Ided377fabaa294572c046a4a358c57c7b6cd3ca1 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/55414 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Oleg Drokin Reviewed-by: Andreas Dilger Reviewed-by: Timothy Day Reviewed-by: James Simmons --- lustre.spec.in | 1 + lustre/conf/Makefile.am | 4 +- lustre/conf/mount.client.params | 13 ++++++ lustre/doc/mount.lustre.8 | 12 +++++- lustre/tests/conf-sanity.sh | 90 +++++++++++++++++++++++++++++++++++++++++ lustre/tests/test-framework.sh | 15 ++++--- lustre/utils/Makefile.am | 8 ++-- lustre/utils/lustre_param.c | 30 +++++++++++--- lustre/utils/mount_lustre.c | 88 +++++++++++++++++++++++++++++++++++++++- lustre/utils/mount_utils.c | 33 +++++++++++++++ lustre/utils/mount_utils.h | 5 +++ lustre/utils/obdctl.h | 1 - 12 files changed, 281 insertions(+), 19 deletions(-) create mode 100644 lustre/conf/mount.client.params diff --git a/lustre.spec.in b/lustre.spec.in index 1da7fac..0f4e4e3 100644 --- a/lustre.spec.in +++ b/lustre.spec.in @@ -1006,6 +1006,7 @@ echo '%{_libdir}/lustre/tests/lutf/*' >>lustre-tests.files %config(noreplace) %{_sysconfdir}/modprobe.d/ko2iblnd.conf %if %{with lustre_utils} %config(noreplace) %{_sysconfdir}/lnet_routes.conf +%config(noreplace) %{_sysconfdir}/lustre/mount.client.params %endif %if %{with lustre_modules} %exclude /Module.symvers diff --git a/lustre/conf/Makefile.am b/lustre/conf/Makefile.am index 6cec5b5..2708a04 100644 --- a/lustre/conf/Makefile.am +++ b/lustre/conf/Makefile.am @@ -36,12 +36,14 @@ DIST_SUBDIRS = resource EXTRA_DIST = lustre.xml 99-lustre.rules lustre ldev.conf ko2iblnd.conf \ lsvcgss lnet_routes.conf lnet.conf lgssc.conf \ - lnet-sysctl.conf 99-lustre-server.rules + lnet-sysctl.conf 99-lustre-server.rules mount.client.params sysconf_DATA = lnet.conf if UTILS sysconf_DATA += lnet_routes.conf +lustredir = $(sysconfdir)/lustre +lustre_DATA = mount.client.params endif firewallddir = ${prefix}/lib/firewalld/services diff --git a/lustre/conf/mount.client.params b/lustre/conf/mount.client.params new file mode 100644 index 0000000..b9aca77 --- /dev/null +++ b/lustre/conf/mount.client.params @@ -0,0 +1,13 @@ +# This is the per-client lustre configuration file. +# The parameters declared in this file will be set after the system has mounted, +# but only on this node. + +# if mount.$FS_NAME.params exists, the parameters declared in +# that file will be loaded afterwards, meaning the filesystem +# specific parameters will override the client specific ones. + +# Examples: +# max_dirty_mb=7873 +# mdc.*.max_rpcs_in_flight=16 +# jobid_var=nodelocal +# jobid_name=%H:%e:%u \ No newline at end of file diff --git a/lustre/doc/mount.lustre.8 b/lustre/doc/mount.lustre.8 index 80f5ddc..d1d88e3 100644 --- a/lustre/doc/mount.lustre.8 +++ b/lustre/doc/mount.lustre.8 @@ -33,8 +33,16 @@ or .B lustre_tgt filesystem type is used. Lustre clients and targets are stopped by using the .BR umount (8) -command. -.PP +command. When mounted, Lustre clients will have parameters set according to +what is described in +.BR /etc/lustre/mount.client.params , +and/or the +.BI /etc/lustre/mount. FSNAME .params +file, with lines of the same style as +.BR lctl-set_param (8), +with lines of the form +.IR PARAM = VALUE . +.P There are two forms for the device option passed to the .I mount command, depending on whether a client or a target service is started: diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index ccdc71d..d76268d 100755 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -1206,6 +1206,96 @@ test_28a() { # LU-4221 } run_test 28a "set symlink parameters permanently with lctl" +test_28b() { + setup_noconfig + stack_trap "cleanup" + + # jobid_this_session is not available so use jobid_name when possible + if $LCTL list_param jobid_name &> /dev/null; then + local jobvarname=jobid_name test_val="$TESTNAME-$RANDOM%p" + else + local jobvarname=jobid_var test_val=$TESTNAME-$RANDOM + fi + + local jobid_var_new + local jobid_var_old=$($LCTL get_param -n $jobvarname) + local paramdir=/etc/lustre + local paramfile=$paramdir/mount.client.params + + [[ -d $paramdir ]] || mkdir $paramdir + + stack_trap "$LCTL set_param $jobvarname=$jobid_var_old" + + if [[ -e $paramfile ]]; then + mv $paramfile $paramfile.$TESTNAME + stack_trap "mv $paramfile.$TESTNAME $paramfile" + else + # leave no traces behind + stack_trap "rm $paramfile" + fi + + # if the test param is set in the filesystem params file + # it will override the more general client params + if [[ -e $paramdir/mount.$FSNAME.params ]]; then + echo "disabling $FSNAME specific params file" + local clientfile=$paramdir/mount.$FSNAME.params + + mv $clientfile $clientfile.disabled + stack_trap "mv $clientfile.disabled $clientfile" + fi + echo "$jobvarname=$test_val" > $paramfile + + remount_client $MOUNT + + jobid_var_new=$($LCTL get_param -n $jobvarname) + + echo "before mount: $jobid_var_old" + echo "after mount: $jobid_var_new" + [[ "$jobid_var_new" == "$test_val" ]] || + error "$jobvarname was not $test_val: got $jobid_var_new" +} +run_test 28b "verify client-side parameters are set upon mount" + +test_28c() { + setup_noconfig + stack_trap "cleanup" + + # jobid_this_session is not available so use jobid_name when possible + if $LCTL list_param jobid_name > /dev/null 2>&1; then + local jobvarname=jobid_name test_val="$TESTNAME-$RANDOM%p" + else + local jobvarname=jobid_var test_val=$TESTNAME-$RANDOM + fi + + local jobid_var_new + local jobid_var_old=$($LCTL get_param -n $jobvarname) + local paramdir=/etc/lustre + local paramfile=$paramdir/mount.$FSNAME.params + + [[ -d $paramdir ]] || mkdir $paramdir + + stack_trap "$LCTL set_param $jobvarname=$jobid_var_old" + + if [[ -e $paramfile ]]; then + mv $paramfile $paramfile.$testname + stack_trap "mv $paramfile.$testname $paramfile" + else + # leave no traces behind + stack_trap "rm $paramfile" + fi + echo "$jobvarname=$test_val" > $paramfile + + remount_client $MOUNT + + jobid_var_new=$($LCTL get_param -n $jobvarname) + + echo "before mount: $jobid_var_old" + echo "after mount: $jobid_var_new" + [[ "$jobid_var_new" == "$test_val" ]] || + error "$jobvarname was not $test_val: got $jobid_var_new" +} +run_test 28c "verify filesystem parameters are set upon mount" + test_29() { [ "$OSTCOUNT" -lt "2" ] && skip_env "needs >= 2 OSTs" setup_noconfig > /dev/null 2>&1 diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 73d6d60..2aac1d0 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1155,6 +1155,9 @@ load_modules_local() { fi mount --bind $mount_lustre $sbin_mount || error "can't bind $mount_lustre to $sbin_mount" + # ignore errors to symlink .libs for read-only /sbin + [[ -e /sbin/.libs ]] || + ln -sf $LUSTRE/utils/.libs /sbin/.libs || true fi fi } @@ -1252,6 +1255,8 @@ unload_modules() { rm -f $sbin_mount fi + [ -L /sbin/.libs ] && rm /sbin/.libs + [[ $rc -eq 0 ]] && echo "modules unloaded." return $rc @@ -2795,7 +2800,7 @@ zconf_mount() { do_node $client mkdir -p $mnt if [ -n "$FILESET" -a -z "$SKIP_FILESET" ];then do_node $client $MOUNT_CMD $flags $opts $MGSNID:/$FSNAME \ - $mnt || return 1 + $mnt || return $? #disable FILESET if not supported do_nodes $client lctl get_param -n \ mdc.$FSNAME-MDT0000*.import | grep -q subtree || @@ -2813,10 +2818,10 @@ zconf_mount() { local prunedopts=$(echo $opts | sed -e "s#skpath=[^,^ ]*#skpath=$mountkey#g") do_node $client $MOUNT_CMD $flags $prunedopts $device $mnt || - return 1 + return $? else do_node $client $MOUNT_CMD $flags $opts $device $mnt || - return 1 + return $? fi set_default_debug_nodes $client @@ -2858,7 +2863,7 @@ mount_mds_client() { local host=$(facet_active_host $SINGLEMDS) echo $host zconf_mount $host $MOUNT2 $MOUNT_OPTS || - error "unable to mount $MOUNT2 on $host" + error "unable to mount $MOUNT2 on $host with ($?)" } # Unmount the file system on the MDS @@ -5621,7 +5626,7 @@ switch_identity() { remount_client() { zconf_umount $HOSTNAME $1 || error "umount failed" - zconf_mount $HOSTNAME $1 || error "mount failed" + zconf_mount $HOSTNAME $1 || error "mount failed with ($?)" } writeconf_facet() { diff --git a/lustre/utils/Makefile.am b/lustre/utils/Makefile.am index 21dbd1c..500bdba 100644 --- a/lustre/utils/Makefile.am +++ b/lustre/utils/Makefile.am @@ -214,11 +214,13 @@ PLUGIN_LIB += libmount_utils_ldiskfs.a endif # PLUGINS endif # LDISKFS_ENABLED -mount_lustre_SOURCES = mount_lustre.c mount_utils.c mount_utils.h $(GSSSRC) +mount_lustre_SOURCES = mount_lustre.c mount_utils.c mount_utils.h $(GSSSRC) \ + lustre_param.c mount_lustre_CPPFLAGS := ${MNTMODCFLAGS} mount_lustre_LDFLAGS := ${MNTMODLDFLAGS} -mount_lustre_LDADD := $(SELINUX) $(LDLIBMOUNT) $(PLUGIN_LIB) $(GSSLIB) \ - $(top_builddir)/libcfs/libcfs/libcfs.la $(MNTMODLIBS) +mount_lustre_LDADD := $(SELINUX) $(LDLIBMOUNT) $(PLUGIN_LIB) $(GSSLIB) +mount_lustre_LDADD += $(top_builddir)/libcfs/libcfs/libcfs.la $(MNTMODLIBS) +mount_lustre_LDADD += liblustreapi.la mount_lustre_tgt_SOURCES = ${mount_lustre_SOURCES} mount_lustre_tgt_CPPFLAGS = ${mount_lustre_CPPFLAGS} mount_lustre_tgt_LDFLAGS = ${mount_lustre_LDFLAGS} diff --git a/lustre/utils/lustre_param.c b/lustre/utils/lustre_param.c index 6a65e32..c38bc8d 100644 --- a/lustre/utils/lustre_param.c +++ b/lustre/utils/lustre_param.c @@ -72,6 +72,20 @@ #include #include +/** + * Parse the arguments to set_param and return the first parameter and value + * pair and the number of arguments consumed. + * + * \param[in] argc number of arguments remaining in argv + * \param[in] argv list of param-value arguments to set_param (this function + * will modify the strings by overwriting '=' with '\0') + * \param[out] param the parameter name + * \param[out] value the parameter value + * + * \retval the number of args consumed from argv (1 for "param=value" format, 2 + * for "param value" format) + * \retval -errno if unsuccessful + */ static int sp_parse_param_value(int argc, char **argv, char **param, char **value) { @@ -626,6 +640,8 @@ static int listparam_cmdline(int argc, char **argv, struct param_opts *popt) popt->po_only_name = 1; popt->po_follow_symlinks = 1; + /* reset optind for each getopt_long() in case of multiple calls */ + optind = 0; while ((ch = getopt_long(argc, argv, "DFlLpR", long_opts, NULL)) != -1) { switch (ch) { @@ -720,6 +736,8 @@ static int getparam_cmdline(int argc, char **argv, struct param_opts *popt) popt->po_show_name = 1; popt->po_follow_symlinks = 1; + /* reset optind for each getopt_long() in case of multiple calls */ + optind = 0; while ((ch = getopt_long(argc, argv, "FHlLnNRy", long_opts, NULL)) != -1) { switch (ch) { @@ -860,13 +878,13 @@ static int setparam_cmdline(int argc, char **argv, struct param_opts *popt) return -EINVAL; #else { - static bool printed; + static bool printed; - if (!printed) { - printed = true; - fprintf(stderr, - "warning: set_param: no pthread support, proceeding serially.\n"); - } + if (!printed) { + printed = true; + fprintf(stderr, + "warning: set_param: no pthread support, proceeding serially.\n"); + } } #endif break; diff --git a/lustre/utils/mount_lustre.c b/lustre/utils/mount_lustre.c index a095940..d440644 100644 --- a/lustre/utils/mount_lustre.c +++ b/lustre/utils/mount_lustre.c @@ -61,6 +61,7 @@ #include "mount_utils.h" #define MAX_RETRIES 99 +#define PATH_FORMAT "/etc/lustre/mount.%s.params" int verbose; int version; @@ -754,6 +755,7 @@ static int parse_opts(int argc, char *const argv[], struct mount_opts *mop) mop->mo_source = convert_hostnames(mop->mo_usource, true); if (!mop->mo_source) usage(stderr); + mop->mo_fsname = convert_fsname(mop->mo_usource); } else { mop->mo_source = strdup(mop->mo_usource); } @@ -804,6 +806,84 @@ static void label_lustre(struct mount_opts *mop) } #endif /* HAVE_SERVER_SUPPORT */ +/* no-op version for mount, since it only needs temporary parameters */ +int jt_lcfg_setparam_perm(int argc, char **argv, struct param_opts *popt) +{ + return 0; +} + +char *jt_cmdname(char *func) +{ + return func; +} +struct sp_workq { int unused; }; +int spwq_init(struct sp_workq *wq, struct param_opts *popt) +{ return 0; } +int spwq_destroy(struct sp_workq *wq) +{ return 0; } +int spwq_expand(struct sp_workq *wq, size_t num_items) +{ return 0; } +int spwq_add_item(struct sp_workq *wq, char *path, + char *param_name, char *value) +{ return 0; } +int sp_run_threads(struct sp_workq *wq) +{ return 0; } + +int parse_param_file(char *path) +{ + int rc = 0; + + FILE *file = fopen(path, "r"); + + if (file) { + char *param = NULL; + size_t len = 0; + + while (getline(¶m, &len, file) != -1) { + char *tmp; + + /* skip any comments on lines */ + tmp = strchr(param, '#'); + if (tmp) { + if (tmp == param) + continue; + *tmp = '\0'; + } + /* remove trailing newline/whitespace. embedded OK */ + tmp = strchr(param, '\n'); + if (tmp) + *tmp = '\0'; + + if (!*param) + continue; + + rc = jt_lcfg_setparam(2, (char*[3]) + { "mount.params", param, NULL }); + } + free(param); + } + + return rc; +} + +int set_client_params(char *fsname) +{ + char path[PATH_MAX]; + int rc, rc1; + + snprintf(path, sizeof(path), PATH_FORMAT, "client"); + rc = parse_param_file(path); + + if (fsname) { + snprintf(path, sizeof(path), PATH_FORMAT, fsname); + rc1 = parse_param_file(path); + if (rc1 && !rc) + rc = rc1; + } + + return 0; +} + int main(int argc, char *const argv[]) { struct mount_opts mop; @@ -1072,9 +1152,14 @@ int main(int argc, char *const argv[]) * Deal with utab just for client. Note that we ignore * the return value here since it is not worth to fail * mount by prevent some rare cases + * Client specific parameters are stored in either + * '/etc/lustre/mount.params' or '/etc/lustre/FSNAME.params' + * and are set here. */ - if (strstr(mop.mo_usource, ":/") != NULL) + if (strstr(mop.mo_usource, ":/") != NULL) { update_utab_entry(&mop); + rc = set_client_params(mop.mo_fsname); + } if (!mop.mo_nomtab) { rc = update_mtab_entry(mop.mo_usource, mop.mo_target, "lustre", mop.mo_orig_options, @@ -1093,5 +1178,6 @@ out_options: out_mo_source: /* mo_usource should be freed, but we can rely on the kernel */ free(mop.mo_source); + free(mop.mo_fsname); return rc; } diff --git a/lustre/utils/mount_utils.c b/lustre/utils/mount_utils.c index d0095b9..99652b6 100644 --- a/lustre/utils/mount_utils.c +++ b/lustre/utils/mount_utils.c @@ -997,6 +997,39 @@ out_bad_mnt_str: return NULL; } +char *convert_fsname(char *devname) +{ + char *fsname, *start, *end; + int len = 0; + + start = strstr(devname, ":/"); + if (!start) + goto out_bad_name; + start += 2; /* skip ":/" */ + + end = strchr(start, '/'); + if (!end) + end = start + strlen(start); + + len = end - start + 1; + + fsname = calloc(len, sizeof(char)); + if (!fsname) { + fprintf(stderr, "%s: cannot allocate %u bytes for MOUNT: %s\n", + progname, len, strerror(ENOMEM)); + return NULL; + } + + memcpy(fsname, start, len); + fsname[len - 1] = '\0'; + return fsname; + +out_bad_name: + fprintf(stderr, "%s: Can't parse filesystem name: %s\n", + progname, devname); + return NULL; +} + #ifdef HAVE_SERVER_SUPPORT struct lustre_cfg_entry { struct list_head lce_list; diff --git a/lustre/utils/mount_utils.h b/lustre/utils/mount_utils.h index 3487c77..2661f18 100644 --- a/lustre/utils/mount_utils.h +++ b/lustre/utils/mount_utils.h @@ -59,6 +59,7 @@ #include #include #endif +#include extern char *progname; extern int verbose; @@ -109,6 +110,7 @@ struct mount_opts { char *mo_orig_options; char *mo_usource; /* user-specified mount device */ char *mo_source; /* our mount device name */ + char *mo_fsname; /* file system name */ char mo_target[PATH_MAX]; /* mount directory */ #ifdef HAVE_GSS char mo_skpath[PATH_MAX]; /* shared key file/directory */ @@ -178,6 +180,9 @@ int update_utab_entry(struct mount_opts *mop); int check_mountfsoptions(char *mountopts, char *wanted_mountopts); void trim_mountfsoptions(char *s); char *convert_hostnames(char *buf, bool mount); +char *convert_fsname(char *s1); +int set_client_params(char *fs_name); +int parse_param_file(char *path); #ifdef HAVE_SERVER_SUPPORT __u64 get_device_size(char* device); int lustre_rename_fsname(struct mkfs_opts *mop, const char *mntpt, diff --git a/lustre/utils/obdctl.h b/lustre/utils/obdctl.h index 1c497c9..e1c17ef 100644 --- a/lustre/utils/obdctl.h +++ b/lustre/utils/obdctl.h @@ -137,7 +137,6 @@ int lcfg_mgs_ioctl(char *func, int dev_id, struct lustre_cfg *lcfg); int parse_devname(char *func, char *name, int dev_id); char *jt_cmdname(char *func); - /* lustre_param.c */ struct param_opts; int jt_lcfg_getparam(int argc, char **argv); -- 1.8.3.1