LU-16454 mdt: Add a per-MDT "max_mod_rpcs_in_flight"

Value max_mod_rpcs_per_client doesn't define a static number of
slots for the per-client replies or anything, and the only
thing it is used for is to pass the limit to the client. For the
same reason, there also doesn't appear to be a particularly hard
limitation why the client cannot change and exceed the
server-provided parameter, except to avoid overloading the server
with too many RPCs at once, but that may also be true of the
current limit with a larger number of clients, no different than
"max_rpcs_in_flight".

This fix adds a tunable parameter "max_mod_rpcs_in_flight" per MDT
to lustre/mdt/mdt_lproc.c so that it can be set
with "lctl set_param" at runtime. The max_mod_rpcs_per_client global
setting is marked "deprecated" but is still used as the default
value when creating an MDT.

Signed-off-by: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
Change-Id: I27cfcb68e1a534e80e6a2dbf2e1affc430803b49
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49749
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: jsimmons <jsimmons@infradead.org>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Jian Yu <yujian@whamcloud.com>

diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index 7a17d54..644441e 100644 (file)
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -67,9 +67,41 @@
 
 #include "mdt_internal.h"
 
-static unsigned int max_mod_rpcs_per_client = 8;
-module_param(max_mod_rpcs_per_client, uint, 0644);
-MODULE_PARM_DESC(max_mod_rpcs_per_client, "maximum number of modify RPCs in flight allowed per client");
+#if OBD_OCD_VERSION(3, 0, 53, 0) > LUSTRE_VERSION_CODE
+static int mdt_max_mod_rpcs_per_client_set(const char *val,
+                               cfs_kernel_param_arg_t *kp)
+{
+       unsigned int num;
+       int rc;
+
+       rc = kstrtouint(val, 0, &num);
+       if (rc < 0)
+               return rc;
+
+       if (num < 1 || num > OBD_MAX_RIF_MAX)
+               return -EINVAL;
+
+       CWARN("max_mod_rpcs_per_client is deprecated, set mdt.*.max_mod_rpcs_in_flight parameter instead\n");
+
+       max_mod_rpcs_per_client = num;
+       return 0;
+}
+static const struct kernel_param_ops
+                               param_ops_max_mod_rpcs_per_client = {
+       .set = mdt_max_mod_rpcs_per_client_set,
+       .get = param_get_uint,
+};
+
+#define param_check_max_mod_rpcs_per_client(name, p) \
+               __param_check(name, p, unsigned int)
+
+module_param_cb(max_mod_rpcs_per_client,
+                               &param_ops_max_mod_rpcs_per_client,
+                               &max_mod_rpcs_per_client, 0644);
+
+MODULE_PARM_DESC(max_mod_rpcs_per_client,
+       "maximum number of modify RPCs in flight allowed per client (Deprecated)");
+#endif
 
 mdl_mode_t mdt_mdl_lock_modes[] = {
         [LCK_MINMODE] = MDL_MINMODE,
@@ -6000,6 +6032,7 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m,
        m->mdt_enable_remote_rename = 1;
        m->mdt_enable_striped_dir = 1;
        m->mdt_dir_restripe_nsonly = 1;
+       m->mdt_max_mod_rpcs_in_flight = OBD_MAX_RIF_DEFAULT;
 
        atomic_set(&m->mdt_mds_mds_conns, 0);
        atomic_set(&m->mdt_async_commit_count, 0);
@@ -6615,7 +6648,15 @@ static int mdt_connect_internal(const struct lu_env *env,
         * multiple modify RPCs, and it is safe to expose this flag before
         * connection processing completes. */
        if (data->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS) {
-               data->ocd_maxmodrpcs = max_mod_rpcs_per_client;
+               if (mdt_max_mod_rpcs_changed(mdt))
+                       /* The new mdt.*.max_mod_rpcs_in_flight parameter
+                        * has not changed since initialization, but the
+                        * deprecated module parameter was changed,
+                        * so use that instead.
+                        */
+                       data->ocd_maxmodrpcs = max_mod_rpcs_per_client;
+               else
+                       data->ocd_maxmodrpcs = mdt->mdt_max_mod_rpcs_in_flight;
                spin_lock(&exp->exp_lock);
                *exp_connect_flags_ptr(exp) |= OBD_CONNECT_MULTIMODRPCS;
                spin_unlock(&exp->exp_lock);

diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h
index 6866402..60d6d30 100644 (file)
--- a/lustre/mdt/mdt_internal.h
+++ b/lustre/mdt/mdt_internal.h
@@ -288,6 +288,9 @@ struct mdt_device {
 
        int                        mdt_max_ea_size;
 
+       /* Max modify RPCs, replaces max_mod_rpcs_per_client module param */
+       unsigned int                       mdt_max_mod_rpcs_in_flight;
+
        /* preferred BRW size, decided by storage type and capability */
        __u32                      mdt_brw_size;
 
@@ -1515,4 +1518,15 @@ int mdt_is_remote_object(struct mdt_thread_info *info,
                         struct mdt_object *parent,
                         struct mdt_object *child);
 
+static unsigned int max_mod_rpcs_per_client = OBD_MAX_RIF_DEFAULT;
+#if OBD_OCD_VERSION(3, 0, 53, 0) > LUSTRE_VERSION_CODE
+static inline bool mdt_max_mod_rpcs_changed(struct mdt_device *mdt)
+{
+       return max_mod_rpcs_per_client != OBD_MAX_RIF_DEFAULT &&
+               mdt->mdt_max_mod_rpcs_in_flight == OBD_MAX_RIF_DEFAULT;
+}
+#else
+#define mdt_max_mod_rpcs_changed(mdt) false
+#endif
+
 #endif /* _MDT_INTERNAL_H */

diff --git a/lustre/mdt/mdt_lproc.c b/lustre/mdt/mdt_lproc.c
index 6e6cd46..5e71039 100644 (file)
--- a/lustre/mdt/mdt_lproc.c
+++ b/lustre/mdt/mdt_lproc.c
@@ -1428,6 +1428,53 @@ static ssize_t checksum_t10pi_enforce_store(struct kobject *kobj,
 }
 LUSTRE_RW_ATTR(checksum_t10pi_enforce);
 
+/**
+ * Show MDT Maximum modify RPCs in flight.
+ *
+ * @m          seq_file handle
+ * @data       unused for single entry
+ *
+ * Return:     value on success or negative number on error
+ */
+static ssize_t max_mod_rpcs_in_flight_show(struct kobject *kobj,
+                                      struct attribute *attr, char *buf)
+{
+       struct obd_device *obd = container_of(kobj, struct obd_device,
+                                             obd_kset.kobj);
+       struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev);
+
+       return scnprintf(buf, PAGE_SIZE, "%u\n",
+                        mdt->mdt_max_mod_rpcs_in_flight);
+}
+
+static ssize_t max_mod_rpcs_in_flight_store(struct kobject *kobj,
+                                       struct attribute *attr,
+                                       const char *buffer, size_t count)
+{
+       struct obd_device *obd = container_of(kobj, struct obd_device,
+                                             obd_kset.kobj);
+       struct mdt_device *mdt = mdt_dev(obd->obd_lu_dev);
+       unsigned int val;
+       int rc;
+
+       rc = kstrtouint(buffer, 0, &val);
+       if (rc)
+               return rc;
+
+       if (val < 1 || val > OBD_MAX_RIF_DEFAULT)
+               return -ERANGE;
+
+       if (mdt_max_mod_rpcs_changed(mdt)) {
+               CWARN("%s: deprecated 'max_mod_rpcs_in_flight' module parameter has also been modified\n",
+                               obd->obd_name);
+               max_mod_rpcs_per_client = val;
+       }
+       mdt->mdt_max_mod_rpcs_in_flight = val;
+
+       return count;
+}
+LUSTRE_RW_ATTR(max_mod_rpcs_in_flight);
+
 /*
  * mdt_checksum_type(server) proc handling
  */
@@ -1528,6 +1575,7 @@ static struct attribute *mdt_attrs[] = {
        &lustre_attr_dir_restripe_nsonly.attr,
        &lustre_attr_checksum_t10pi_enforce.attr,
        &lustre_attr_enable_remote_subdir_mount.attr,
+       &lustre_attr_max_mod_rpcs_in_flight.attr,
        NULL,
 };
 

diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c
index 7fc8971..296bc28 100644 (file)
--- a/lustre/obdclass/genops.c
+++ b/lustre/obdclass/genops.c
@@ -2173,7 +2173,7 @@ int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, __u16 max)
                maxmodrpcs = 1;
        }
        if (max > maxmodrpcs) {
-               CERROR("%s: can't set max_mod_rpcs_in_flight=%hu higher than ocd_maxmodrpcs=%hu returned by the server at connection\n",
+               CERROR("%s: can't set max_mod_rpcs_in_flight=%hu higher than mdt.*.max_mod_rpcs_in_flight=%hu returned by the MDT server at connection.\n",
                       cli->cl_import->imp_obd->obd_name,
                       max, maxmodrpcs);
                return -ERANGE;

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 0e88560..ffa9929 100644 (file)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -7195,6 +7195,40 @@ check_max_mod_rpcs_in_flight() {
        wait
 }
 
+get_mdt_max_mod_rpcs_in_flight_val() {
+       local tmp_value
+       local host_d="$1"
+
+       tmp_value=$($LCTL get_param -n \
+               mdt.*.max_mod_rpcs_in_flight)
+       if [[ $tmp_value ]]; then
+               echo $tmp_value
+       else
+               tmp_value=$(do_facet $host_d \
+               cat /sys/module/mdt/parameters/max_mod_rpcs_per_client)
+               echo $tmp_value
+       fi
+}
+
+set_mdt_max_mod_rpcs_in_flight() {
+       local lctl_op
+       local value_d="$1"
+       local host_d="$2"
+
+       lctl_op=$($LCTL get_param \
+               mdt.*.max_mod_rpcs_in_flight)
+       if [[ $lctl_op ]]; then
+               $LCTL set_param \
+                       mdt.$FSNAME-MDT0000.max_mod_rpcs_in_flight=$value_d
+       else
+               do_facet $host_d \
+                       "echo $value_d > \
+                       /sys/module/mdt/parameters/max_mod_rpcs_per_client"
+               echo "the deprecated max_mod_rpcs_per_client \
+                               parameter was involved"
+       fi
+}
+
 test_90a() {
        setup
 
@@ -7245,21 +7279,17 @@ test_90b() {
        idx=$(printf "%04x" $($LFS getdirstripe -i $DIR/${tdir}3))
        facet="mds$((0x$idx + 1))"
 
-       # save MDT max_mod_rpcs_per_client
-       mmrpc=$(do_facet $facet \
-                   cat /sys/module/mdt/parameters/max_mod_rpcs_per_client)
-
+       mmrpc=$(get_mdt_max_mod_rpcs_in_flight_val $facet)
+       echo "mdt_max_mod_rpcs_in_flight is $mmrpc"
        # update max_mod_rpcs_in_flight
        umount_client $MOUNT
-       do_facet $facet \
-               "echo 16 > /sys/module/mdt/parameters/max_mod_rpcs_per_client"
+       set_mdt_max_mod_rpcs_in_flight 16 $facet
        mount_client $MOUNT
        $LCTL set_param mdc.$FSNAME-MDT$idx-mdc-*.max_rpcs_in_flight=17
        check_max_mod_rpcs_in_flight $DIR/${tdir}3 16
 
-       # restore MDT max_mod_rpcs_per_client initial value
-       do_facet $facet \
-               "echo $mmrpc > /sys/module/mdt/parameters/max_mod_rpcs_per_client"
+       # restore MDT max_mod_rpcs_in_flight initial value
+       set_mdt_max_mod_rpcs_in_flight $mmrpc $facet
 
        rm -rf $DIR/${tdir}?
        cleanup
@@ -7272,15 +7302,14 @@ save_params_90c() {
                   mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight)
        echo "max_rpcs_in_flight is $mrif_90c"
 
-       # get max_mod_rpcs_in_flight value
+       # get MDC max_mod_rpcs_in_flight value
        mmrif_90c=$($LCTL get_param -n \
                    mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight)
-       echo "max_mod_rpcs_in_flight is $mmrif_90c"
+       echo "MDC max_mod_rpcs_in_flight is $mmrif_90c"
 
-       # get MDT max_mod_rpcs_per_client value
-       mmrpc_90c=$(do_facet mds1 \
-                   cat /sys/module/mdt/parameters/max_mod_rpcs_per_client)
-       echo "max_mod_rpcs_per_client is $mmrpc_90c"
+       # get MDT max_mod_rpcs_in_flight value
+       mmrpc_90c=$(get_mdt_max_mod_rpcs_in_flight_val "mds1")
+       echo "mdt_max_mod_rpcs_in_flight is $mmrpc_90c"
 }
 
 restore_params_90c() {
@@ -7294,9 +7323,8 @@ restore_params_90c() {
        do_facet mgs $LCTL set_param -P \
                mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mmrif_90c
 
-       # restore MDT max_mod_rpcs_per_client value
-       do_facet mds1 "echo $mmrpc_90c > \
-                      /sys/module/mdt/parameters/max_mod_rpcs_per_client"
+       # restore MDT max_mod_rpcs_in_flight value
+       set_mdt_max_mod_rpcs_in_flight $mmrpc_90c "mds1"
 }
 
 test_90c() {
@@ -7321,9 +7349,9 @@ test_90c() {
 
        # testcase 1
        # attempt to set max_mod_rpcs_in_flight to max_rpcs_in_flight value
-       # prerequisite: set max_mod_rpcs_per_client to max_rpcs_in_flight value
-       do_facet mds1 "echo $mrif_90c > \
-                      /sys/module/mdt/parameters/max_mod_rpcs_per_client"
+       # prerequisite: set MDT max_mod_rpcs_in_flight to
+       # max_rpcs_in_flight value
+       set_mdt_max_mod_rpcs_in_flight $mrif_90c "mds1"
 
        # if max_mod_rpcs_in_flight is set to be equal to or larger than
        # max_rpcs_in_flight, then max_rpcs_in_flight will be increased
@@ -7339,13 +7367,12 @@ test_90c() {
        fi
 
        umount_client $MOUNT
-       do_facet mds1 "echo $mmrpc_90c > \
-                      /sys/module/mdt/parameters/max_mod_rpcs_per_client"
+       set_mdt_max_mod_rpcs_in_flight $mmrpc_90c "mds1"
        mount_client $MOUNT
 
        # testcase 2
-       # attempt to set max_mod_rpcs_in_flight to max_mod_rpcs_per_client+1
-       # prerequisite: set max_rpcs_in_flight to max_mod_rpcs_per_client+2
+       # attempt to set max_mod_rpcs_in_flight to MDT max_mod_rpcs_in_flight+1
+       # prerequisite: set max_rpcs_in_flight to MDT max_mod_rpcs_in_flight+2
        $LCTL set_param \
                mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$((mmrpc_90c + 2))