From 5e546603cbff0939c9f6335d9daddc641530dd7e Mon Sep 17 00:00:00 2001 From: Nathan Rutman Date: Fri, 14 May 2010 09:18:43 -0700 Subject: [PATCH] b=15253 add conf_param -d to remove permanent settings i=adilger i=rread --- lustre/doc/lctl.8 | 43 ++++++++++-- lustre/include/lustre_cfg.h | 3 - lustre/include/lustre_param.h | 12 ++-- lustre/mgs/mgs_handler.c | 2 +- lustre/mgs/mgs_llog.c | 159 +++++++++++++++++++++++++++--------------- lustre/tests/conf-sanity.sh | 15 +++- lustre/utils/lctl.c | 7 +- lustre/utils/lustre_cfg.c | 41 +++++++++-- 8 files changed, 201 insertions(+), 81 deletions(-) diff --git a/lustre/doc/lctl.8 b/lustre/doc/lctl.8 index fca7631..9e2b52a 100644 --- a/lustre/doc/lctl.8 +++ b/lustre/doc/lctl.8 @@ -92,12 +92,45 @@ Show all the local Lustre OBDs. AKA .PP .SS Device Operations .TP -.BI conf_param " " +.BI conf_param " [-d] .=" Set a permanent configuration parameter for any device via the MGS. This -command must be run on the MGS node. +command must be run on the MGS node. +.br +.B -d . +Delete a parameter setting (use the default value at the next restart). A null value for also deletes the parameter setting. +.br +.B Parameters: +.br +All of the writable parameters under +.B lctl list_param +(e.g. +.I lctl list_param -F osc.*.* | grep = +) can be permanently set using +.B lctl conf_param +, but the format is slightly different. For conf_param, the device is specified first, then the obdtype. (See examples below.) Wildcards are not supported. +.br +Additionally, failover nodes may be added (or removed), and some system-wide parameters may be set as well (sys.at_max, sys.at_min, sys.at_extra, sys.at_early_margin, sys.at_history, sys.timeout, sys.ldlm_timeout.) is ignored for system wide parameters. +.br +.B Examples: +.br +# lctl conf_param testfs.sys.at_max=1200 +.br +# lctl conf_param testfs.llite.max_read_ahead_mb=16 +.br +# lctl conf_param testfs-MDT0000.lov.stripesize=2M +.br +# lctl conf_param lustre-OST0001.osc.active=0 +.br +# lctl conf_param testfs-OST0000.osc.max_dirty_mb=29.15 +.br +# lctl conf_param testfs-OST0000.ost.client_cache_seconds=15 +.br +# lctl conf_param testfs-OST0000.failover.node=1.2.3.4@tcp1 .TP .BI activate -Reactivate an import after deactivating, below. +Reactivate an import after deactivating, below. This setting is only effective until the next restart (see +.B conf_param +). .TP .BI deactivate Deactivate an import, in particular meaning do not assign new file stripes @@ -170,7 +203,7 @@ number. See .TP .B --ignore_errors | ignore_errors Ignore errors during script processing -.TP + .SH EXAMPLES # lctl .br @@ -183,8 +216,6 @@ lctl > dk /tmp/log Debug log: 87 lines, 87 kept, 0 dropped. .br lctl > quit -.PP -# lctl conf_param testfs-MDT0000 sys.timeout=40 .SH BUGS Please report all bugs to Sun Microsystems, Inc. http://bugzilla.lustre.org/ diff --git a/lustre/include/lustre_cfg.h b/lustre/include/lustre_cfg.h index cf3c120..f1b103a 100644 --- a/lustre/include/lustre_cfg.h +++ b/lustre/include/lustre_cfg.h @@ -94,9 +94,6 @@ struct lustre_cfg_bufs { __u32 lcfg_bufcount; }; -/* Mountconf transitional hack, should go away after 1.6 */ -#define LCFG_FLG_MOUNTCONF 0x400 - struct lustre_cfg { __u32 lcfg_version; __u32 lcfg_command; diff --git a/lustre/include/lustre_param.h b/lustre/include/lustre_param.h index adeabc6..ffa5eb9 100644 --- a/lustre/include/lustre_param.h +++ b/lustre/include/lustre_param.h @@ -69,7 +69,9 @@ int do_lcfg(char *cfgname, lnet_nid_t nid, int cmd, ... testfs.llite.max_read_ahead_mb=16 */ -/* System global or special params not handled in obd's proc */ +/* System global or special params not handled in obd's proc + * See mgs_write_log_sys() + */ #define PARAM_TIMEOUT "timeout=" /* global */ #define PARAM_LDLM_TIMEOUT "ldlm_timeout=" /* global */ #define PARAM_AT_MIN "at_min=" /* global */ @@ -77,10 +79,10 @@ int do_lcfg(char *cfgname, lnet_nid_t nid, int cmd, #define PARAM_AT_EXTRA "at_extra=" /* global */ #define PARAM_AT_EARLY_MARGIN "at_early_margin=" /* global */ #define PARAM_AT_HISTORY "at_history=" /* global */ -#define PARAM_MGSNODE "mgsnode=" /* during mount */ -#define PARAM_FAILNODE "failover.node=" /* llog generation */ -#define PARAM_FAILMODE "failover.mode=" /* llog generation */ -#define PARAM_ACTIVE "active=" /* llog generation */ +#define PARAM_MGSNODE "mgsnode=" /* only at mounttime */ +#define PARAM_FAILNODE "failover.node=" /* add failover nid */ +#define PARAM_FAILMODE "failover.mode=" /* initial mount only */ +#define PARAM_ACTIVE "active=" /* activate/deactivate */ /* Prefixes for parameters handled by obd's proc methods (XXX_process_config) */ #define PARAM_OST "ost." diff --git a/lustre/mgs/mgs_handler.c b/lustre/mgs/mgs_handler.c index deb9b1b..1d687e9 100644 --- a/lustre/mgs/mgs_handler.c +++ b/lustre/mgs/mgs_handler.c @@ -42,7 +42,7 @@ # define EXPORT_SYMTAB #endif #define DEBUG_SUBSYSTEM S_MGS -#define D_MGS D_CONFIG/*|D_WARNING*/ +#define D_MGS D_CONFIG #ifdef __KERNEL__ # include diff --git a/lustre/mgs/mgs_llog.c b/lustre/mgs/mgs_llog.c index 4848990..46b9569 100644 --- a/lustre/mgs/mgs_llog.c +++ b/lustre/mgs/mgs_llog.c @@ -44,7 +44,7 @@ #define EXPORT_SYMTAB #endif #define DEBUG_SUBSYSTEM S_MGS -#define D_MGS D_CONFIG /*|D_WARNING*/ +#define D_MGS D_CONFIG #ifdef __KERNEL__ #include @@ -662,7 +662,8 @@ static int mgs_modify(struct obd_device *obd, struct fs_db *fsdb, int rc, rc2; ENTRY; - CDEBUG(D_MGS, "modify %s/%s/%s\n", logname, devname, comment); + CDEBUG(D_MGS, "modify %s/%s/%s fl=%x\n", logname, devname, comment, + flags); push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); @@ -1037,11 +1038,15 @@ int mgs_write_log_direct_all(struct obd_device *obd, struct fs_db *fsdb, mgs_modify(obd, fsdb, mti, dirent->lld_name, devname, comment, CM_SKIP); /* Write the new one */ - rc = mgs_write_log_direct(obd, fsdb, dirent->lld_name, - lcfg, devname, comment); - if (rc) - CERROR("err %d writing log %s\n", rc, - dirent->lld_name); + if (lcfg) { + rc = mgs_write_log_direct(obd, fsdb, + dirent->lld_name, + lcfg, devname, + comment); + if (rc) + CERROR("err %d writing log %s\n", rc, + dirent->lld_name); + } } OBD_FREE(dirent, sizeof(*dirent)); } @@ -1810,6 +1815,46 @@ static int mgs_write_log_ost(struct obd_device *obd, struct fs_db *fsdb, RETURN(rc); } +static __inline__ int mgs_param_empty(char *ptr) +{ + char *tmp; + + if ((tmp = strchr(ptr, '=')) && (*(++tmp) == '\0')) + return 1; + return 0; +} + +static int mgs_write_log_failnid_internal(struct obd_device *obd, + struct fs_db *fsdb, + struct mgs_target_info *mti, + char *logname, char *cliname) +{ + int rc; + struct llog_handle *llh = NULL; + + if (mgs_param_empty(mti->mti_params)) { + /* Remove _all_ failnids */ + rc = mgs_modify(obd, fsdb, mti, logname, + mti->mti_svname, "add failnid", CM_SKIP); + return rc; + } + + /* Otherwise failover nids are additive */ + rc = record_start_log(obd, &llh, logname); + if (!rc) { + /* FIXME this should be a single journal transaction */ + rc = record_marker(obd, llh, fsdb, CM_START, + mti->mti_svname, "add failnid"); + rc = mgs_write_log_failnids(obd, mti, llh, cliname); + rc = record_marker(obd, llh, fsdb, CM_END, + mti->mti_svname, "add failnid"); + rc = record_end_log(obd, &llh); + } + + return rc; +} + + /* Add additional failnids to an existing log. The mdc/osc must have been added to logs first */ /* tcp nids must be in dotted-quad ascii - @@ -1818,15 +1863,12 @@ static int mgs_write_log_add_failnid(struct obd_device *obd, struct fs_db *fsdb, struct mgs_target_info *mti) { char *logname, *cliname; - struct llog_handle *llh = NULL; int rc; ENTRY; - /* FIXME how do we delete a failnid? Currently --writeconf is the - only way. Maybe make --erase-params pass a flag to really - erase all params from logs - except it can't erase the failnids - given when a target first registers, since they aren't processed - as params... */ + /* FIXME we currently can't erase the failnids + * given when a target first registers, since they aren't part of + * an "add uuid" stanza */ /* Verify that we know about this target */ if (mgs_log_is_empty(obd, mti->mti_svname)) { @@ -1845,18 +1887,9 @@ static int mgs_write_log_add_failnid(struct obd_device *obd, struct fs_db *fsdb, RETURN(-EINVAL); } - /* Add failover nids to client log */ + /* Add failover nids to the client log */ name_create(&logname, mti->mti_fsname, "-client"); - rc = record_start_log(obd, &llh, logname); - if (!rc) { - /* FIXME this fn should be a single journal transaction */ - rc = record_marker(obd, llh, fsdb, CM_START, mti->mti_svname, - "add failnid"); - rc = mgs_write_log_failnids(obd, mti, llh, cliname); - rc = record_marker(obd, llh, fsdb, CM_END, mti->mti_svname, - "add failnid"); - rc = record_end_log(obd, &llh); - } + rc = mgs_write_log_failnid_internal(obd, fsdb, mti, logname, cliname); name_destroy(&logname); name_destroy(&cliname); @@ -1869,19 +1902,8 @@ static int mgs_write_log_add_failnid(struct obd_device *obd, struct fs_db *fsdb, continue; name_create_mdt(&logname, mti->mti_fsname, i); name_create_mdt_osc(&cliname, mti->mti_svname, fsdb, i); - - rc = record_start_log(obd, &llh, logname); - if (!rc) { - rc = record_marker(obd, llh, fsdb, CM_START, - mti->mti_svname, - "add failnid"); - rc = mgs_write_log_failnids(obd, mti, llh, - cliname); - rc = record_marker(obd, llh, fsdb, CM_END, - mti->mti_svname, - "add failnid"); - rc = record_end_log(obd, &llh); - } + rc = mgs_write_log_failnid_internal(obd, fsdb, mti, + logname, cliname); name_destroy(&cliname); name_destroy(&logname); } @@ -1898,7 +1920,7 @@ static int mgs_wlp_lcfg(struct obd_device *obd, struct fs_db *fsdb, char comment[MTI_NAME_MAXLEN]; char *tmp; struct lustre_cfg *lcfg; - int rc; + int rc, del; /* Erase any old settings of this same parameter */ memcpy(comment, ptr, MTI_NAME_MAXLEN); @@ -1908,8 +1930,12 @@ static int mgs_wlp_lcfg(struct obd_device *obd, struct fs_db *fsdb, *tmp = 0; /* FIXME we should skip settings that are the same as old values */ rc = mgs_modify(obd, fsdb, mti, logname, tgtname, comment, CM_SKIP); - LCONSOLE_INFO("%sing parameter %s.%s in log %s\n", rc ? + del = mgs_param_empty(ptr); + + LCONSOLE_INFO("%sing parameter %s.%s in log %s\n", del ? "Disabl" : rc ? "Sett" : "Modify", tgtname, comment, logname); + if (del) + return rc; lustre_cfg_bufs_reset(bufs, tgtname); lustre_cfg_bufs_set_string(bufs, 1, ptr); @@ -1928,6 +1954,7 @@ static int mgs_write_log_sys(struct obd_device *obd, struct fs_db *fsdb, struct lustre_cfg_bufs bufs; struct lustre_cfg *lcfg; char *tmp; + char sep; int cmd, val; int rc; @@ -1945,16 +1972,26 @@ static int mgs_write_log_sys(struct obd_device *obd, struct fs_db *fsdb, else return -EINVAL; + /* separate the value */ val = simple_strtoul(tmp, NULL, 0); - CDEBUG(D_MGS, "global %s = %d\n", ptr, val); + if (*tmp == '\0') + CDEBUG(D_MGS, "global '%s' removed\n", sys); + else + CDEBUG(D_MGS, "global '%s' val=%d\n", sys, val); lustre_cfg_bufs_reset(&bufs, NULL); lustre_cfg_bufs_set_string(&bufs, 1, sys); lcfg = lustre_cfg_new(cmd, &bufs); lcfg->lcfg_num = val; + /* truncate the comment to the parameter name */ + ptr = tmp - 1; + sep = *ptr; + *ptr = '\0'; /* modify all servers and clients */ - rc = mgs_write_log_direct_all(obd, fsdb, mti, lcfg, mti->mti_fsname, - ptr); + rc = mgs_write_log_direct_all(obd, fsdb, mti, + *tmp == '\0' ? NULL : lcfg, + mti->mti_fsname, sys); + *ptr = sep; lustre_cfg_free(lcfg); return rc; } @@ -2005,12 +2042,13 @@ static int mgs_srpc_set_param_disk(struct obd_device *obd, /* obsolete old one */ mgs_modify(obd, fsdb, mti, logname, mti->mti_svname, comment, CM_SKIP); - /* write the new one */ - rc = mgs_write_log_direct(obd, fsdb, logname, lcfg, - mti->mti_svname, comment); - if (rc) - CERROR("err %d writing log %s\n", rc, logname); - + if (!mgs_param_empty(param)) { + /* write the new one */ + rc = mgs_write_log_direct(obd, fsdb, logname, lcfg, + mti->mti_svname, comment); + if (rc) + CERROR("err %d writing log %s\n", rc, logname); + } out: name_destroy(&logname); out_lcfg: @@ -2146,7 +2184,7 @@ static int mgs_srpc_set_param(struct obd_device *obd, char *param) { char *copy; - int rc, copy_size; + int rc, copy_size, del; ENTRY; #ifndef HAVE_GSS @@ -2160,9 +2198,12 @@ static int mgs_srpc_set_param(struct obd_device *obd, return -ENOMEM; memcpy(copy, param, copy_size); - rc = mgs_srpc_set_param_mem(fsdb, mti->mti_svname, param); - if (rc) - goto out_free; + del = mgs_param_empty(param); + if (!del) { + rc = mgs_srpc_set_param_mem(fsdb, mti->mti_svname, param); + if (rc) + goto out_free; + } /* previous steps guaranteed the syntax is correct */ rc = mgs_srpc_set_param_disk(obd, fsdb, mti, copy); @@ -2304,6 +2345,11 @@ out: RETURN(rc); } +/* Permanent settings of all parameters by writing into the appropriate + * configuration logs. + * A parameter with null value ("='\0'") means to erase it out of + * the logs. + */ static int mgs_write_log_param(struct obd_device *obd, struct fs_db *fsdb, struct mgs_target_info *mti, char *ptr) { @@ -2875,7 +2921,10 @@ static void print_lustre_cfg(struct lustre_cfg *lcfg) EXIT; } -/* Set a permanent (config log) param for a target or fs */ +/* Set a permanent (config log) param for a target or fs + * \param lcfg buf0 may contain the device (testfs-MDT0000) name + * buf1 contains the single parameter + */ int mgs_setparam(struct obd_device *obd, struct lustre_cfg *lcfg, char *fsname) { struct fs_db *fsdb; @@ -2920,7 +2969,7 @@ int mgs_setparam(struct obd_device *obd, struct lustre_cfg *lcfg, char *fsname) strncpy(fsname, devname, MTI_NAME_MAXLEN); } fsname[MTI_NAME_MAXLEN - 1] = 0; - CDEBUG(D_MGS, "setparam on fs %s device %s\n", fsname, devname); + CDEBUG(D_MGS, "setparam fs='%s' device='%s'\n", fsname, devname); rc = mgs_find_or_make_fsdb(obd, fsname, &fsdb); if (rc) @@ -2952,8 +3001,6 @@ int mgs_setparam(struct obd_device *obd, struct lustre_cfg *lcfg, char *fsname) mti->mti_flags = rc | LDD_F_PARAM; cfs_down(&fsdb->fsdb_sem); - /* this is lctl conf_param's single param path, there is not - need to loop through parameters */ rc = mgs_write_log_param(obd, fsdb, mti, mti->mti_params); cfs_up(&fsdb->fsdb_sem); diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 54f27ba..5587acf 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -929,6 +929,7 @@ run_test 29 "permanently remove an OST" test_30() { setup + echo Big config llog TEST="lctl get_param -n llite.$FSNAME-*.max_read_ahead_whole_mb" ORIG=$($TEST) LIST=(1 2 3 4 5 4 3 2 1 2 3 4 5 4 3 2 1 2 3 4 5) @@ -939,10 +940,20 @@ test_30() { umount_client $MOUNT mount_client $MOUNT || return 4 [ "$($TEST)" -ne "$i" ] && return 5 - set_and_check client "$TEST" "$FSNAME.llite.max_read_ahead_whole_mb" $ORIG || return 6 + pass + + echo Erase parameter setting + do_facet mgs "$LCTL conf_param -d $FSNAME.llite.max_read_ahead_whole_mb" || return 6 + umount_client $MOUNT + mount_client $MOUNT || return 6 + FINAL=$($TEST) + echo "deleted (default) value=$FINAL, orig=$ORIG" + # assumes this parameter started at the default value + [ "$FINAL" -eq "$ORIG" ] || fail "Deleted value=$FINAL, orig=$ORIG" + cleanup } -run_test 30 "Big config llog" +run_test 30 "Big config llog and conf_param deletion" test_31() { # bug 10734 # ipaddr must not exist diff --git a/lustre/utils/lctl.c b/lustre/utils/lctl.c index f1c7c31..369dab4 100644 --- a/lustre/utils/lctl.c +++ b/lustre/utils/lctl.c @@ -131,11 +131,12 @@ command_t cmdlist[] = { "abort recovery on a restarting MDT or OST device\n"}, {"set_timeout", jt_lcfg_set_timeout, 0, "usage: conf_param obd_timeout=\n"}, - {"conf_param", jt_lcfg_mgsparam, 0, "set a permanent config param. " + {"conf_param", jt_lcfg_mgsparam, 0,"set a permanent config parameter.\n" "This command must be run on the MGS node\n" - "usage: conf_param ...\n"}, + "usage: conf_param [-d] \n" + " -d Remove the permanent setting."}, {"local_param", jt_lcfg_param, 0, "set a temporary, local param\n" - "usage: local_param ...\n"}, + "usage: local_param \n"}, {"get_param", jt_lcfg_getparam, 0, "get the Lustre or LNET parameter\n" "usage: get_param [-n|-N|-F] \n" "Get the value of Lustre or LNET parameter from the specified path.\n" diff --git a/lustre/utils/lustre_cfg.c b/lustre/utils/lustre_cfg.c index c5d825b..6860502 100644 --- a/lustre/utils/lustre_cfg.c +++ b/lustre/utils/lustre_cfg.c @@ -491,19 +491,48 @@ int jt_lcfg_param(int argc, char **argv) } /* Param set in config log on MGS */ -/* conf_param key1=value1 [key2=value2...] */ +/* conf_param key=value */ +/* Note we can actually send mgc conf_params from clients, but currently + * that's only done for default file striping (see ll_send_mgc_param), + * and not here. */ +/* After removal of a parameter (-d) Lustre will use the default + * AT NEXT REBOOT, not immediately. */ int jt_lcfg_mgsparam(int argc, char **argv) { - int i, rc; + int rc; + int del = 0; struct lustre_cfg_bufs bufs; struct lustre_cfg *lcfg; + char *buf = NULL; - if ((argc >= LUSTRE_CFG_MAX_BUFCOUNT) || (argc <= 1)) + /* mgs_setparam processes only lctl buf #1 */ + if ((argc > 3) || (argc <= 1)) return CMD_HELP; + while ((rc = getopt(argc, argv, "d")) != -1) { + switch (rc) { + case 'd': + del = 1; + break; + default: + return CMD_HELP; + } + } + lustre_cfg_bufs_reset(&bufs, NULL); - for (i = 1; i < argc; i++) { - lustre_cfg_bufs_set_string(&bufs, i, argv[i]); + if (del) { + char *ptr; + + /* for delete, make it "=\0" */ + buf = malloc(strlen(argv[optind]) + 2); + /* put an '=' on the end in case it doesn't have one */ + sprintf(buf, "%s=", argv[optind]); + /* then truncate after the first '=' */ + ptr = strchr(buf, '='); + *(++ptr) = '\0'; + lustre_cfg_bufs_set_string(&bufs, 1, buf); + } else { + lustre_cfg_bufs_set_string(&bufs, 1, argv[optind]); } /* We could put other opcodes here. */ @@ -511,6 +540,8 @@ int jt_lcfg_mgsparam(int argc, char **argv) rc = lcfg_mgs_ioctl(argv[0], OBD_DEV_ID, lcfg); lustre_cfg_free(lcfg); + if (buf) + free(buf); if (rc < 0) { fprintf(stderr, "error: %s: %s\n", jt_cmdname(argv[0]), strerror(rc = errno)); -- 1.8.3.1