From: Vitaliy Kuznetsov Date: Wed, 8 Feb 2023 21:34:38 +0000 (+0300) Subject: LU-16454 mdt: Add a per-MDT "max_mod_rpcs_in_flight" X-Git-Tag: 2.15.55~163 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=f16c31ccd91d66caba69d3ceea6a61c1682df59e;p=fs%2Flustre-release.git LU-16454 mdt: Add a per-MDT "max_mod_rpcs_in_flight" Value max_mod_rpcs_per_client doesn't define a static number of slots for the per-client replies or anything, and the only thing it is used for is to pass the limit to the client. For the same reason, there also doesn't appear to be a particularly hard limitation why the client cannot change and exceed the server-provided parameter, except to avoid overloading the server with too many RPCs at once, but that may also be true of the current limit with a larger number of clients, no different than "max_rpcs_in_flight". This fix adds a tunable parameter "max_mod_rpcs_in_flight" per MDT to lustre/mdt/mdt_lproc.c so that it can be set with "lctl set_param" at runtime. The max_mod_rpcs_per_client global setting is marked "deprecated" but is still used as the default value when creating an MDT. Signed-off-by: Vitaliy Kuznetsov Change-Id: I27cfcb68e1a534e80e6a2dbf2e1affc430803b49 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49749 Tested-by: jenkins Tested-by: Maloo Reviewed-by: jsimmons Reviewed-by: Oleg Drokin Reviewed-by: Andreas Dilger Reviewed-by: Jian Yu --- diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c index 7a17d54..644441e 100644 --- a/lustre/mdt/mdt_handler.c +++ b/lustre/mdt/mdt_handler.c @@ -67,9 +67,41 @@ #include "mdt_internal.h" -static unsigned int max_mod_rpcs_per_client = 8; -module_param(max_mod_rpcs_per_client, uint, 0644); -MODULE_PARM_DESC(max_mod_rpcs_per_client, "maximum number of modify RPCs in flight allowed per client"); +#if OBD_OCD_VERSION(3, 0, 53, 0) > LUSTRE_VERSION_CODE +static int mdt_max_mod_rpcs_per_client_set(const char *val, + cfs_kernel_param_arg_t *kp) +{ + unsigned int num; + int rc; + + rc = kstrtouint(val, 0, &num); + if (rc < 0) + return rc; + + if (num < 1 || num > OBD_MAX_RIF_MAX) + return -EINVAL; + + CWARN("max_mod_rpcs_per_client is deprecated, set mdt.*.max_mod_rpcs_in_flight parameter instead\n"); + + max_mod_rpcs_per_client = num; + return 0; +} +static const struct kernel_param_ops + param_ops_max_mod_rpcs_per_client = { + .set = mdt_max_mod_rpcs_per_client_set, + .get = param_get_uint, +}; + +#define param_check_max_mod_rpcs_per_client(name, p) \ + __param_check(name, p, unsigned int) + +module_param_cb(max_mod_rpcs_per_client, + ¶m_ops_max_mod_rpcs_per_client, + &max_mod_rpcs_per_client, 0644); + +MODULE_PARM_DESC(max_mod_rpcs_per_client, + "maximum number of modify RPCs in flight allowed per client (Deprecated)"); +#endif mdl_mode_t mdt_mdl_lock_modes[] = { [LCK_MINMODE] = MDL_MINMODE, @@ -6000,6 +6032,7 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m, m->mdt_enable_remote_rename = 1; m->mdt_enable_striped_dir = 1; m->mdt_dir_restripe_nsonly = 1; + m->mdt_max_mod_rpcs_in_flight = OBD_MAX_RIF_DEFAULT; atomic_set(&m->mdt_mds_mds_conns, 0); atomic_set(&m->mdt_async_commit_count, 0); @@ -6615,7 +6648,15 @@ static int mdt_connect_internal(const struct lu_env *env, * multiple modify RPCs, and it is safe to expose this flag before * connection processing completes. */ if (data->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS) { - data->ocd_maxmodrpcs = max_mod_rpcs_per_client; + if (mdt_max_mod_rpcs_changed(mdt)) + /* The new mdt.*.max_mod_rpcs_in_flight parameter + * has not changed since initialization, but the + * deprecated module parameter was changed, + * so use that instead. + */ + data->ocd_maxmodrpcs = max_mod_rpcs_per_client; + else + data->ocd_maxmodrpcs = mdt->mdt_max_mod_rpcs_in_flight; spin_lock(&exp->exp_lock); *exp_connect_flags_ptr(exp) |= OBD_CONNECT_MULTIMODRPCS; spin_unlock(&exp->exp_lock); diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h index 6866402..60d6d30 100644 --- a/lustre/mdt/mdt_internal.h +++ b/lustre/mdt/mdt_internal.h @@ -288,6 +288,9 @@ struct mdt_device { int mdt_max_ea_size; + /* Max modify RPCs, replaces max_mod_rpcs_per_client module param */ + unsigned int mdt_max_mod_rpcs_in_flight; + /* preferred BRW size, decided by storage type and capability */ __u32 mdt_brw_size; @@ -1515,4 +1518,15 @@ int mdt_is_remote_object(struct mdt_thread_info *info, struct mdt_object *parent, struct mdt_object *child); +static unsigned int max_mod_rpcs_per_client = OBD_MAX_RIF_DEFAULT; +#if OBD_OCD_VERSION(3, 0, 53, 0) > LUSTRE_VERSION_CODE +static inline bool mdt_max_mod_rpcs_changed(struct mdt_device *mdt) +{ + return max_mod_rpcs_per_client != OBD_MAX_RIF_DEFAULT && + mdt->mdt_max_mod_rpcs_in_flight == OBD_MAX_RIF_DEFAULT; +} +#else +#define mdt_max_mod_rpcs_changed(mdt) false +#endif + #endif /* _MDT_INTERNAL_H */ diff --git a/lustre/mdt/mdt_lproc.c b/lustre/mdt/mdt_lproc.c index 6e6cd46..5e71039 100644 --- a/lustre/mdt/mdt_lproc.c +++ b/lustre/mdt/mdt_lproc.c @@ -1428,6 +1428,53 @@ static ssize_t checksum_t10pi_enforce_store(struct kobject *kobj, } LUSTRE_RW_ATTR(checksum_t10pi_enforce); +/** + * Show MDT Maximum modify RPCs in flight. + * + * @m seq_file handle + * @data unused for single entry + * + * Return: value on success or negative number on error + */ +static ssize_t max_mod_rpcs_in_flight_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct obd_device *obd = container_of(kobj, struct obd_device, + obd_kset.kobj); + struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev); + + return scnprintf(buf, PAGE_SIZE, "%u\n", + mdt->mdt_max_mod_rpcs_in_flight); +} + +static ssize_t max_mod_rpcs_in_flight_store(struct kobject *kobj, + struct attribute *attr, + const char *buffer, size_t count) +{ + struct obd_device *obd = container_of(kobj, struct obd_device, + obd_kset.kobj); + struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev); + unsigned int val; + int rc; + + rc = kstrtouint(buffer, 0, &val); + if (rc) + return rc; + + if (val < 1 || val > OBD_MAX_RIF_DEFAULT) + return -ERANGE; + + if (mdt_max_mod_rpcs_changed(mdt)) { + CWARN("%s: deprecated 'max_mod_rpcs_in_flight' module parameter has also been modified\n", + obd->obd_name); + max_mod_rpcs_per_client = val; + } + mdt->mdt_max_mod_rpcs_in_flight = val; + + return count; +} +LUSTRE_RW_ATTR(max_mod_rpcs_in_flight); + /* * mdt_checksum_type(server) proc handling */ @@ -1528,6 +1575,7 @@ static struct attribute *mdt_attrs[] = { &lustre_attr_dir_restripe_nsonly.attr, &lustre_attr_checksum_t10pi_enforce.attr, &lustre_attr_enable_remote_subdir_mount.attr, + &lustre_attr_max_mod_rpcs_in_flight.attr, NULL, }; diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c index 7fc8971..296bc28 100644 --- a/lustre/obdclass/genops.c +++ b/lustre/obdclass/genops.c @@ -2173,7 +2173,7 @@ int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, __u16 max) maxmodrpcs = 1; } if (max > maxmodrpcs) { - CERROR("%s: can't set max_mod_rpcs_in_flight=%hu higher than ocd_maxmodrpcs=%hu returned by the server at connection\n", + CERROR("%s: can't set max_mod_rpcs_in_flight=%hu higher than mdt.*.max_mod_rpcs_in_flight=%hu returned by the MDT server at connection.\n", cli->cl_import->imp_obd->obd_name, max, maxmodrpcs); return -ERANGE; diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 0e88560..ffa9929 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -7195,6 +7195,40 @@ check_max_mod_rpcs_in_flight() { wait } +get_mdt_max_mod_rpcs_in_flight_val() { + local tmp_value + local host_d="$1" + + tmp_value=$($LCTL get_param -n \ + mdt.*.max_mod_rpcs_in_flight) + if [[ $tmp_value ]]; then + echo $tmp_value + else + tmp_value=$(do_facet $host_d \ + cat /sys/module/mdt/parameters/max_mod_rpcs_per_client) + echo $tmp_value + fi +} + +set_mdt_max_mod_rpcs_in_flight() { + local lctl_op + local value_d="$1" + local host_d="$2" + + lctl_op=$($LCTL get_param \ + mdt.*.max_mod_rpcs_in_flight) + if [[ $lctl_op ]]; then + $LCTL set_param \ + mdt.$FSNAME-MDT0000.max_mod_rpcs_in_flight=$value_d + else + do_facet $host_d \ + "echo $value_d > \ + /sys/module/mdt/parameters/max_mod_rpcs_per_client" + echo "the deprecated max_mod_rpcs_per_client \ + parameter was involved" + fi +} + test_90a() { setup @@ -7245,21 +7279,17 @@ test_90b() { idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/${tdir}3)) facet="mds$((0x$idx + 1))" - # save MDT max_mod_rpcs_per_client - mmrpc=$(do_facet $facet \ - cat /sys/module/mdt/parameters/max_mod_rpcs_per_client) - + mmrpc=$(get_mdt_max_mod_rpcs_in_flight_val $facet) + echo "mdt_max_mod_rpcs_in_flight is $mmrpc" # update max_mod_rpcs_in_flight umount_client $MOUNT - do_facet $facet \ - "echo 16 > /sys/module/mdt/parameters/max_mod_rpcs_per_client" + set_mdt_max_mod_rpcs_in_flight 16 $facet mount_client $MOUNT $LCTL set_param mdc.$FSNAME-MDT$idx-mdc-*.max_rpcs_in_flight=17 check_max_mod_rpcs_in_flight $DIR/${tdir}3 16 - # restore MDT max_mod_rpcs_per_client initial value - do_facet $facet \ - "echo $mmrpc > /sys/module/mdt/parameters/max_mod_rpcs_per_client" + # restore MDT max_mod_rpcs_in_flight initial value + set_mdt_max_mod_rpcs_in_flight $mmrpc $facet rm -rf $DIR/${tdir}? cleanup @@ -7272,15 +7302,14 @@ save_params_90c() { mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight) echo "max_rpcs_in_flight is $mrif_90c" - # get max_mod_rpcs_in_flight value + # get MDC max_mod_rpcs_in_flight value mmrif_90c=$($LCTL get_param -n \ mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight) - echo "max_mod_rpcs_in_flight is $mmrif_90c" + echo "MDC max_mod_rpcs_in_flight is $mmrif_90c" - # get MDT max_mod_rpcs_per_client value - mmrpc_90c=$(do_facet mds1 \ - cat /sys/module/mdt/parameters/max_mod_rpcs_per_client) - echo "max_mod_rpcs_per_client is $mmrpc_90c" + # get MDT max_mod_rpcs_in_flight value + mmrpc_90c=$(get_mdt_max_mod_rpcs_in_flight_val "mds1") + echo "mdt_max_mod_rpcs_in_flight is $mmrpc_90c" } restore_params_90c() { @@ -7294,9 +7323,8 @@ restore_params_90c() { do_facet mgs $LCTL set_param -P \ mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mmrif_90c - # restore MDT max_mod_rpcs_per_client value - do_facet mds1 "echo $mmrpc_90c > \ - /sys/module/mdt/parameters/max_mod_rpcs_per_client" + # restore MDT max_mod_rpcs_in_flight value + set_mdt_max_mod_rpcs_in_flight $mmrpc_90c "mds1" } test_90c() { @@ -7321,9 +7349,9 @@ test_90c() { # testcase 1 # attempt to set max_mod_rpcs_in_flight to max_rpcs_in_flight value - # prerequisite: set max_mod_rpcs_per_client to max_rpcs_in_flight value - do_facet mds1 "echo $mrif_90c > \ - /sys/module/mdt/parameters/max_mod_rpcs_per_client" + # prerequisite: set MDT max_mod_rpcs_in_flight to + # max_rpcs_in_flight value + set_mdt_max_mod_rpcs_in_flight $mrif_90c "mds1" # if max_mod_rpcs_in_flight is set to be equal to or larger than # max_rpcs_in_flight, then max_rpcs_in_flight will be increased @@ -7339,13 +7367,12 @@ test_90c() { fi umount_client $MOUNT - do_facet mds1 "echo $mmrpc_90c > \ - /sys/module/mdt/parameters/max_mod_rpcs_per_client" + set_mdt_max_mod_rpcs_in_flight $mmrpc_90c "mds1" mount_client $MOUNT # testcase 2 - # attempt to set max_mod_rpcs_in_flight to max_mod_rpcs_per_client+1 - # prerequisite: set max_rpcs_in_flight to max_mod_rpcs_per_client+2 + # attempt to set max_mod_rpcs_in_flight to MDT max_mod_rpcs_in_flight+1 + # prerequisite: set max_rpcs_in_flight to MDT max_mod_rpcs_in_flight+2 $LCTL set_param \ mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$((mmrpc_90c + 2))