From: Frederick Dilger Date: Wed, 12 Jun 2024 20:43:08 +0000 (-0400) Subject: LU-11077 utils: --client option for set_param X-Git-Tag: 2.16.53~22 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=refs%2Fchanges%2F58%2F55858%2F20;p=fs%2Flustre-release.git LU-11077 utils: --client option for set_param Added new [--client|-C[FSNAME]] option for 'lctl set_param' which writes the parameter to the local /etc/lustre/mount.client.params config file. Upon each Lustre client mount those parameters will be set on the local node. If FSNAME was provided, the parameters will be saved in the mount-specific /etc/lustre/mount.FSNAME.params config file, and will be set (and override) the more generic client mount parameters on that node when that filesystem is mounted. However only parameters containing FSNAME can be set to their respective params config file to avoid generic parameters that are only supposed to affect a single filesystem, actually affecting all of them. Can be used together with [--delete|-d] to remove the parameter from the given log file. If [--delete|-d] is specified without -C or -P it will enable -P by default. A warning message will be printed when this happens so users are aware of what's going on. Signed-off-by: Frederick Dilger Change-Id: Iec0b9bfb5e259154ed2439e6e505b826a888905f Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/55858 Reviewed-by: Andreas Dilger Reviewed-by: Timothy Day Reviewed-by: Oleg Drokin Tested-by: jenkins Tested-by: Maloo --- diff --git a/lustre/doc/lctl-set_param.8 b/lustre/doc/lctl-set_param.8 index fff4794..0e3d675 100644 --- a/lustre/doc/lctl-set_param.8 +++ b/lustre/doc/lctl-set_param.8 @@ -3,6 +3,8 @@ lctl-set_param \- Lustre filesystem set parameter utility .SH SYNOPSIS .SY "lctl set_param" +.RB [ --client | -C\c +.RI [ FSNAME ]] .RB [ --delete | -d ] .RB [ --file | -F ] .RB [ --no-name | -n ] @@ -41,6 +43,18 @@ The various options supported by .B lctl list_param are listed and explained below: .TP +.BR -C ", " --client +Write parameters to the local +.B /etc/lustre/mount.client.params +config file. Upon all future Lustre client mounts +those parameters will be set on the local node. If +.I FSNAME +was specified, the parameters will be instead written to the mount-specific +.BI /etc/lustre/mount. FSNAME .params +config file. The filesystem specific params will be set only when that +filesystem is mounted and will override the more generic client mount +parameters. +.TP .BR -d ", " --delete Remove the permanent setting (only for parameters set with the .B -P @@ -108,6 +122,18 @@ osc.testfs-OST0001-osc-ffff8803c9c0f000.max_dirty_mb=512 osc.testfs-OST0002-osc-ffff8803c9c0f000.max_dirty_mb=512 osc.testfs-OST0003-osc-ffff8803c9c0f000.max_dirty_mb=512 osc.testfs-OST0004-osc-ffff8803c9c0f000.max_dirty_mb=512 +.PP +Set +.B osc.testfs-*.max_dirty_mb=2000 +when mounting 'testfs' and +.B osc.testfs-*.max_dirty_mb=1024 +for other Lustre mountpoints on this client node +.EX +.RS +.B # lctl set_param -C=test_fs osc.testfs-*.max_dirty_mb=2000 +.B # lctl set_param -C osc.testfs-*.max_dirty_mb=1024 +.RE +.EE .SH AVAILABILITY .B lctl set_param is part of the diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 3091244..133985e 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -29913,6 +29913,139 @@ test_401f() { } run_test 401f "check 'lctl list_param' doesn't follow symlinks with --no-links" +test_401ga() { + local paramdir=/etc/lustre + local fsnamefile=$paramdir/mount.$FSNAME.params + local clientfile=$paramdir/mount.client.params + + local param1="osc.$FSNAME-OST0000-*.max_dirty_mb" + local param2="at_max" + + local value1_new value2_new + local value1_old=$($LCTL get_param -n $param1) + local value2_old=$($LCTL get_param -n $param2) + + local value1_client=$(( value1_old - 1 )) + local value1_fsname=$(( value1_old - 3 )) + local value2_client=$(( value2_old - 10 )) + local value2_fsname=$(( value2_old - 13 )) + + stack_trap "$LCTL set_param $param1=$value1_old" + stack_trap "$LCTL set_param $param2=$value2_old" + + if [[ -e $fsnamefile ]]; then + mv $fsnamefile $fsnamefile.$TESTNAME + stack_trap "mv $fsnamefile.$TESTNAME $fsnamefile" + else + stack_trap "rm -f $fsnamefile" + fi + + if [[ -e $clientfile ]]; then + mv $clientfile $clientfile.$TESTNAME + stack_trap "mv $clientfile.$TESTNAME $clientfile" + else + stack_trap "rm -f $clientfile" + fi + + # do not allow bad params to be set + $LCTL set_param --client $TESTNAME=$TESTNAME && + error "set $TESTNAME as param in $clientfile" || true + + # set garbage val to be overwritten + $LCTL set_param --client $param1=$TESTNAME || + error "failed to set $param1 in $clientfile" + + $LCTL set_param --client $param1=$value1_client || + error "failed to overwrite $param1 in $clientfile" + + # nothing should be changed + $LCTL set_param --client $param1=$value1_client || + error "error setting duplicate param $param1 in $clientfile" + + $LCTL set_param --client=$FSNAME $param1=$value1_fsname || + error "failed to set $param1 in $fsnamefile" + + $LCTL set_param --client=$FSNAME $param2=$value2_fsname && + error "set $param2 in $fsnamefile without containing $FSNAME" || + true + + $LCTL set_param --client $TESTNAME= $param2=$value2_client || true + grep -q "$param2=$value2_client" $clientfile || + error "set_param -C did not continue after error" + + remount_client $MOUNT + + value1_new=$($LCTL get_param -n $param1) + value2_new=$($LCTL get_param -n $param2) + + [[ "$value1_new" == "$value1_client" ]] && + error "$param1 not $value1_fsname, got client param $value1_new" + [[ "$value1_new" == "$value1_fsname" ]] || + error "$param1 not $FSNAME param $value1_fsname, got $value1_new" + [[ "$value2_new" == "$value2_client" ]] || + error "$param2 not client param $value2_client, got $value2_new" +} +run_test 401ga "check 'set_param -C' sets params upon mount" + +test_401gb() { + local paramdir=/etc/lustre + local fsnamefile=$paramdir/mount.$FSNAME.params + local clientfile=$paramdir/mount.client.params + + local param1="osc.$FSNAME-OST0000-*.max_dirty_mb" + local param2="at_max" + + local value1_new value2_new + local value1_old=$($LCTL get_param -n $param1) + local value2_old=$($LCTL get_param -n $param2) + + local value1_client=$(( value1_old - 1 )) + local value1_fsname=$(( value1_old - 3 )) + local value2_client=$(( value2_old - 10 )) + + stack_trap "$LCTL set_param $param1=$value1_old" + stack_trap "$LCTL set_param $param2=$value2_old" + + if [[ -e $fsnamefile ]]; then + mv $fsnamefile $fsnamefile.$TESTNAME + stack_trap "mv $fsnamefile.$TESTNAME $fsnamefile" + else + stack_trap "rm $fsnamefile" + fi + + if [[ -e $clientfile ]]; then + mv $clientfile $clientfile.$TESTNAME + stack_trap "mv $clientfile.$TESTNAME $clientfile" + else + stack_trap "rm $clientfile" + fi + + $LCTL set_param --client $param1=$value1_client $param2=$value2_client + $LCTL set_param --client=$FSNAME $param1=$value1_fsname + + $LCTL set_param -d --client $TESTNAME=$TESTNAME || + error "error deleting bad param $TESTNAME from $clientfile" + + $LCTL set_param -d --client $param2 || + error "error deleting $param2 from $clientfile" + + $LCTL set_param -d --client=$FSNAME $param1 $param2 || + error "error deleting $param1 and $param2 from $fsnamefile" + + remount_client $MOUNT + + value1_new=$($LCTL get_param -n $param1) + value2_new=$($LCTL get_param -n $param2) + + [[ "$value1_new" == "$value1_fsname" ]] && + error "$param1 not $value1_client, got $FSNAME param $value1_new" + [[ "$value1_new" == "$value1_client" ]] || + error "$param1 not client param $value1_client, got $value1_new" + [[ "$value2_new" == "$value2_old" ]] || + error "$param2 not $value2_old, got $value2_new" +} +run_test 401gb "check 'set_param -d -C' removes client params" + test_402() { [[ $MDS1_VERSION -ge $(version_code 2.7.66) ]] || [[ $MDS1_VERSION -ge $(version_code 2.7.18.4) && diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index a3bb1ac..672a61d 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -2534,7 +2534,7 @@ mount_facet() { health=$(do_facet ${facet} "$LCTL get_param -n health_check") if [[ "$health" != "healthy" ]]; then - error "$facet is in a unhealthy state" + error "$facet is in a unhealthy state, got: '$health'" fi set_default_debug_facet $facet diff --git a/lustre/utils/lctl.c b/lustre/utils/lctl.c index 827045a..c30fd8b 100644 --- a/lustre/utils/lctl.c +++ b/lustre/utils/lctl.c @@ -428,10 +428,10 @@ command_t cmdlist[] = { "Get the value of Lustre or LNET parameter from the specified path.\n" "The path can contain shell-style filename patterns.\n"}, {"set_param", jt_lcfg_setparam, 0, "set the Lustre or LNET parameter\n" - "usage: set_param [--delete|-d] [--file|-F] [--no-name|-n]\n" - " [--permanent|-P]" + "usage: set_param [--client|-C[FSNAME]] [--delete|-d] [--file|-F]\n" + " [--no-name|-n] [--permanent|-P]" #ifdef HAVE_LIBPTHREAD - " [--thread|-t [THREAD_COUNT]]" + " [--thread|-t[THREAD_COUNT]]" #endif "\n" " PARAM1=VALUE1 [PARAM2=VALUE2 ...]\n" diff --git a/lustre/utils/lctl_thread.h b/lustre/utils/lctl_thread.h index 8890ff5..7abc35e 100644 --- a/lustre/utils/lctl_thread.h +++ b/lustre/utils/lctl_thread.h @@ -50,8 +50,10 @@ struct param_opts { unsigned int po_header:1; unsigned int po_follow_symlinks:1; unsigned int po_tunable:1; + unsigned int po_client:1; unsigned int po_parallel_threads; unsigned int po_permissions; + char *po_fsname; }; #ifdef HAVE_LIBPTHREAD diff --git a/lustre/utils/lustre_cfg.c b/lustre/utils/lustre_cfg.c index 29d235a..9c72493 100644 --- a/lustre/utils/lustre_cfg.c +++ b/lustre/utils/lustre_cfg.c @@ -461,8 +461,7 @@ static int lcfg_setparam_perm(char *func, char *buf) * This should be loaded after the individual config logs. * Called from set param with -P option. */ -int jt_lcfg_setparam_perm(int argc, char **argv, - struct param_opts *popt) +int jt_lcfg_setparam_perm(int argc, char **argv, struct param_opts *popt) { int rc; int i; diff --git a/lustre/utils/lustre_param.c b/lustre/utils/lustre_param.c index 8b35e8a..b253b71 100644 --- a/lustre/utils/lustre_param.c +++ b/lustre/utils/lustre_param.c @@ -189,7 +189,7 @@ static char *display_name(const char *filename, struct stat *st, * * \retval -errno on error. */ -static int clean_path(struct param_opts *popt, char *path) +int jt_clean_path(struct param_opts *popt, char *path) { char *nidstart = NULL; char *nidend = NULL; @@ -579,7 +579,7 @@ static int do_param_op(struct param_opts *popt, char *pattern, char *value, } /* Turn param_name into file path format */ - rc2 = clean_path(popt, param_name); + rc2 = jt_clean_path(popt, param_name); if (rc2 < 0) { fprintf(stderr, "error: %s: cleaning '%s': %s\n", opname, param_name, strerror(-rc2)); @@ -716,7 +716,7 @@ int jt_lcfg_listparam(int argc, char **argv) path = argv[i]; - rc2 = clean_path(&popt, path); + rc2 = jt_clean_path(&popt, path); if (rc2 < 0) { fprintf(stderr, "error: %s: cleaning '%s': %s\n", jt_cmdname(argv[0]), path, strerror(-rc2)); @@ -842,7 +842,7 @@ int jt_lcfg_getparam(int argc, char **argv) path = argv[i]; - rc2 = clean_path(&popt, path); + rc2 = jt_clean_path(&popt, path); if (rc2 < 0) { fprintf(stderr, "error: %s: cleaning '%s': %s\n", jt_cmdname(argv[0]), path, strerror(-rc2)); @@ -920,6 +920,7 @@ static void setparam_check_deprecated(const char *path) static int setparam_cmdline(int argc, char **argv, struct param_opts *popt) { struct option long_opts[] = { + { .val = 'C', .name = "client", .has_arg = optional_argument}, { .val = 'd', .name = "delete", .has_arg = no_argument}, { .val = 'F', .name = "file", .has_arg = no_argument}, { .val = 'n', .name = "noname", .has_arg = no_argument}, @@ -939,16 +940,45 @@ static int setparam_cmdline(int argc, char **argv, struct param_opts *popt) popt->po_file = 0; popt->po_parallel_threads = 0; popt->po_follow_symlinks = 1; + popt->po_client = 0; opterr = 0; /* reset optind for each getopt_long() in case of multiple calls */ optind = 0; - while ((ch = getopt_long(argc, argv, "dFnPt::", + while ((ch = getopt_long(argc, argv, "C::dFnPt::", long_opts, NULL)) != -1) { switch (ch) { + case 'C': + if (popt->po_perm) { + fprintf(stderr, + "error: %s: -C cannot be used with -P\n", + argv[0]); + return -1; + } + popt->po_client = 1; + if (optarg) + /* remove leading '=' from fsname if present */ + popt->po_fsname = strdup(optarg + + (optarg[0] == '=')); + break; + case 'd': + popt->po_delete = 1; + break; + case 'F': + popt->po_file = 1; + break; case 'n': popt->po_show_name = 0; break; + case 'P': + if (popt->po_client) { + fprintf(stderr, + "error: %s: -P cannot be used with -C\n", + argv[0]); + return -1; + } + popt->po_perm = 1; + break; case 't': #if HAVE_LIBPTHREAD if (optarg) @@ -959,25 +989,16 @@ static int setparam_cmdline(int argc, char **argv, struct param_opts *popt) return -EINVAL; #else { - static bool printed; + static bool printed; - if (!printed) { - printed = true; - fprintf(stderr, - "warning: set_param: no pthread support, proceeding serially.\n"); - } + if (!printed) { + printed = true; + fprintf(stderr, + "warning: set_param: no pthread support, proceeding serially.\n"); + } } #endif break; - case 'P': - popt->po_perm = 1; - break; - case 'd': - popt->po_delete = 1; - break; - case 'F': - popt->po_file = 1; - break; default: return -1; } @@ -986,8 +1007,10 @@ static int setparam_cmdline(int argc, char **argv, struct param_opts *popt) fprintf(stderr, "warning: ignoring -P option\n"); popt->po_perm = 0; } - if (popt->po_delete && !popt->po_perm) + if (popt->po_delete && !popt->po_perm && !popt->po_client) { + fprintf(stderr, "warning: setting -P option\n"); popt->po_perm = 1; + } return optind; } @@ -1020,6 +1043,9 @@ int jt_lcfg_setparam(int argc, char **argv) */ return jt_lcfg_setparam_perm(argc, argv, &popt); + if (popt.po_client) + return jt_lcfg_setparam_client(argc, argv, &popt); + if (popt.po_file) { fprintf(stderr, "warning: 'lctl set_param -F' is deprecated, use 'lctl apply_yaml' instead\n"); @@ -1053,7 +1079,7 @@ int jt_lcfg_setparam(int argc, char **argv) /* Increment index by the number of arguments consumed. */ index += rc; - rc = clean_path(&popt, path); + rc = jt_clean_path(&popt, path); if (rc < 0) break; @@ -1100,3 +1126,245 @@ int jt_lcfg_setparam(int argc, char **argv) return rc; } + +/* + * Param set to single client file, used by all mounts on a client or specific + * filesystem if FSNAME is specified. + * These params should be loaded directly after mounting. + * Called from set param with -C option. + */ +static int lcfg_setparam_client(char *func, char *buf, struct param_opts *popt) +{ + glob_t paths; + char path[NAME_MAX]; + char *param_name, *param, *tmp; + char *dir_path = "/etc/lustre"; + char *line = NULL; + bool found_param_name = false; + bool found_param_value = false; + size_t len = 0; + size_t buf_len; + FILE *file = NULL; + int fd = -1; + int rc, rc1; + + buf_len = strlen(buf); + if (buf && buf[buf_len - 1] == '\n') { + param = buf; + } else { + param = malloc(++buf_len + 1); + snprintf(param, buf_len + 1, "%s\n", buf); + } + + param_name = strdup(buf); + tmp = strchr(param_name, '='); + if (tmp) { + *tmp = '\0'; + } else if (!popt->po_delete) { + rc = -EINVAL; + fprintf(stderr, "error: %s: client: argument '%s' does not contain '=': %s\n", + jt_cmdname(func), param, strerror(-rc)); + goto out; + } + + if (!popt->po_delete) { + if (popt->po_fsname && !strstr(buf, popt->po_fsname)) { + rc = -EINVAL; + fprintf(stderr, + "error: %s: client: argument '%s' must contain '%s' to be written to "PATH_FORMAT": %s\n", + jt_cmdname(func), buf, popt->po_fsname, + popt->po_fsname, strerror(-rc)); + goto out; + } + char *tmp_path = strdup(param_name); + + rc = jt_clean_path(popt, tmp_path); + if (rc < 0) { + fprintf(stderr, + "error: %s: client: cleaning '%s': %s\n", + jt_cmdname(func), param_name, strerror(-rc)); + goto out; + } + rc = llapi_param_get_paths(tmp_path, &paths); + if (rc) { + rc = -errno; + fprintf(stderr, + "error: %s: client: param_paths '%s': %s\n", + jt_cmdname(func), param_name, strerror(errno)); + goto out; + } + free(tmp_path); + } + + snprintf(path, sizeof(path), PATH_FORMAT, + popt->po_fsname ? popt->po_fsname : "client"); + + file = fopen(path, "r"); + + if (file) { + while (getline(&line, &len, file) != -1) { + if (strstr(line, param_name)) { + found_param_name = true; + if (!popt->po_delete && strstr(line, param)) + found_param_value = true; + break; + } + } + if (found_param_value && !popt->po_delete) + goto out_file; /* nothing to change */ + } + + if (!found_param_name) { + if (popt->po_delete) + goto out_file; /* nothing to delete */ + mkdir(dir_path, 0644); + fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0644); + if (fd < 0) { + rc = -errno; + fprintf(stderr, + "error: %s: client: failed open file %s: %s\n", + jt_cmdname(func), path, strerror(-rc)); + goto out_fd; + } + rc1 = write(fd, param, strlen(param)); + if (rc1 < strlen(param)) { + rc = -ENOMEM; + fprintf(stderr, + "error: %s: client: failed to write '%s': %s\n", + jt_cmdname(func), param, strerror(-rc)); + goto out_fd; + } + } else { + struct stat st; + struct timeval now; + ssize_t line_len; + size_t tmp_len; + char *tmp_path; + char *bak_path; + + line = NULL; + len = 0; + + tmp_len = strlen(path) + 8; + tmp_path = malloc(tmp_len); + snprintf(tmp_path, tmp_len, "%s.XXXXXX", path); + + rewind(file); + + fd = mkstemp(tmp_path); + if (fd < 0) { + rc = -errno; + fprintf(stderr, + "error: %s: client: failed open file %s: %s\n", + jt_cmdname(func), tmp, strerror(-rc)); + goto out_fd; + } + + bak_path = malloc(strlen(path) + 5); + snprintf(bak_path, strlen(path) + 5, "%s.bak", path); + gettimeofday(&now, NULL); + if (stat(bak_path, &st) == -1 || + st.st_atim.tv_sec < now.tv_sec - 100) { + rc = rename(path, bak_path); + free(bak_path); + } + if (rc) { + fprintf(stderr, + "error: %s: client: failed to backup %s: %s\n", + jt_cmdname(func), path, strerror(-rc)); + goto out_fd; + } + + while ((line_len = getline(&line, &len, file)) != -1) { + if (strstr(line, param_name)) { + if (popt->po_delete) + continue; /* do not write param */ + rc = write(fd, param, strlen(param)); + if (rc < strlen(param)) { + fprintf(stderr, + "error: %s: client: failed to write '%s': %s\n", + jt_cmdname(func), param, + strerror(-rc)); + goto out; + } + } else { + rc1 = write(fd, line, line_len); + if (rc1 < line_len) { + rc = -ENOMEM; + fprintf(stderr, + "error: %s: client: failed to write '%s': %s\n", + jt_cmdname(func), line, + strerror(-rc)); + goto out; + } + } + } + + rc = fsync(fd); + if (rc && errno != EEXIST && errno != ENOENT) { + rc = -errno; + fprintf(stderr, + "error: %s: client: failed to sync %s: %s\n", + jt_cmdname(func), tmp_path, strerror(-rc)); + goto out_fd; + } + + rc = rename(tmp_path, path); + if (rc) { + fprintf(stderr, + "error: %s: client: failed to rename %s: %s\n", + jt_cmdname(func), tmp_path, strerror(-rc)); + goto out_fd; + } + } + +out_fd: + close(fd); +out_file: + if (file) + fclose(file); + free(line); +out: + if (param != buf) + free(param); + free(param_name); + + return rc; +} + +int jt_lcfg_setparam_client(int argc, char **argv, struct param_opts *popt) +{ + int rc, rc1; + int i; + int first_param; + char *buf = NULL; + char *tmp; + + first_param = optind; + if (first_param < 0 || first_param >= argc) + return CMD_HELP; + + if (popt->po_show_name) + printf("params %s /etc/lustre/mount.%s.params:\n", + popt->po_delete ? "deleted from" : "written to", + popt->po_fsname ? popt->po_fsname : "client"); + + for (i = first_param, rc = 0; i < argc; i++) { + buf = argv[i]; + + rc1 = lcfg_setparam_client(argv[0], buf, popt); + if (popt->po_show_name && !rc1) { + tmp = strchr(buf, '='); + if (popt->po_delete) + printf("%.*s=\n", tmp ? (int) (tmp - buf) : + (int) strlen(buf), buf); + else + printf("%s\n", buf); + } + if (!rc && rc1) + rc = rc1; + } + + free(popt->po_fsname); + return rc; +} diff --git a/lustre/utils/obdctl.h b/lustre/utils/obdctl.h index e1c17ef..78a26ed 100644 --- a/lustre/utils/obdctl.h +++ b/lustre/utils/obdctl.h @@ -36,6 +36,8 @@ #include #endif +#define PATH_FORMAT "/etc/lustre/mount.%s.params" + /* ptlctl.a */ int ptl_initialize(int argc, char **argv); int jt_ptl_network(int argc, char **argv); @@ -139,9 +141,11 @@ char *jt_cmdname(char *func); /* lustre_param.c */ struct param_opts; +int jt_clean_path(struct param_opts *popt, char *path); int jt_lcfg_getparam(int argc, char **argv); int jt_lcfg_setparam(int argc, char **argv); int jt_lcfg_listparam(int argc, char **argv); +int jt_lcfg_setparam_client(int argc, char **argv, struct param_opts *popt); /* lustre_cfg.c */ int lcfg_set_devname(char *name);