LU-13503 mdc: allow setting max_mod_rpcs_in_flight larger

Allow setting mdc.*.max_mod_rpcs_in_flight > mdc.*.max_rpcs_in_flight
by increasing the latter value, rather than returning an error and
telling the user to do that.  This matches the similar behavior if
mdc.*.max_rpcs_in_flight is reduced lower than max_mod_rpcs_in_flight.

If there are multiple MDTs, the "mdc.*.max_mod_rpcs_in_flight" param
may be set from e.g. the MDT0000 config log before MDT0001 is fully
configured, catching MDT0001 with ocd_maxmodrpcs = 0 before the OCD
from the MDT has been filled in, and incorrectly trigger an error.
If seen during setup, allow ocd_maxmodrpcs = (max_rpcs_in_flight - 1),
since this will be fixed up later if mdc.*.max_rpcs_in_flight is set
smaller in the config log (if set larger it doesn't matter).

Test-Parameters: env=ONLY=90 testlist=conf-sanity

Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Signed-off-by: Jian Yu <yujian@whamcloud.com>
Change-Id: I4b20163e9e212db451738169ebdc361ab8c1c15e
Reviewed-on: https://review.whamcloud.com/38455
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c
index 0b2612c..d4feb7c 100644 (file)
--- a/lustre/obdclass/genops.c
+++ b/lustre/obdclass/genops.c
@@ -2062,20 +2062,21 @@ int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max)
        __u32                           old;
        int                             diff;
        int                             i;
        __u32                           old;
        int                             diff;
        int                             i;

-       const char *type_name;

        int                             rc;
 
        if (max > OBD_MAX_RIF_MAX || max < 1)
                return -ERANGE;
 
        int                             rc;
 
        if (max > OBD_MAX_RIF_MAX || max < 1)
                return -ERANGE;
 

-       type_name = cli->cl_import->imp_obd->obd_type->typ_name;
-       if (strcmp(type_name, LUSTRE_MDC_NAME) == 0) {
+       CDEBUG(D_INFO, "%s: max = %hu max_mod = %u rif = %u\n",
+              cli->cl_import->imp_obd->obd_name, max,
+              cli->cl_max_mod_rpcs_in_flight, cli->cl_max_rpcs_in_flight);
+
+       if (strcmp(cli->cl_import->imp_obd->obd_type->typ_name,
+                  LUSTRE_MDC_NAME) == 0) {

                /* adjust max_mod_rpcs_in_flight to ensure it is always
                 * strictly lower that max_rpcs_in_flight */
                if (max < 2) {
                /* adjust max_mod_rpcs_in_flight to ensure it is always
                 * strictly lower that max_rpcs_in_flight */
                if (max < 2) {

-                       CERROR("%s: cannot set max_rpcs_in_flight to 1 "
-                              "because it must be higher than "
-                              "max_mod_rpcs_in_flight value",
+                       CERROR("%s: cannot set mdc.*.max_rpcs_in_flight=1\n",

                               cli->cl_import->imp_obd->obd_name);
                        return -ERANGE;
                }
                               cli->cl_import->imp_obd->obd_name);
                        return -ERANGE;
                }


@@ -2119,32 +2120,50 @@ EXPORT_SYMBOL(obd_get_max_mod_rpcs_in_flight);
 
 int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, __u16 max)
 {
 
 int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, __u16 max)
 {

-       struct obd_connect_data *ocd;
+       struct obd_connect_data *ocd;

        __u16 maxmodrpcs;
        __u16 prev;
 
        if (max > OBD_MAX_RIF_MAX || max < 1)
                return -ERANGE;
 
        __u16 maxmodrpcs;
        __u16 prev;
 
        if (max > OBD_MAX_RIF_MAX || max < 1)
                return -ERANGE;
 

-       /* cannot exceed or equal max_rpcs_in_flight */
+       ocd = &cli->cl_import->imp_connect_data;
+       CDEBUG(D_INFO, "%s: max = %hu flags = %llx, max_mod = %u rif = %u\n",
+              cli->cl_import->imp_obd->obd_name, max, ocd->ocd_connect_flags,
+              ocd->ocd_maxmodrpcs, cli->cl_max_rpcs_in_flight);
+
+       if (max == OBD_MAX_RIF_MAX)
+               max = OBD_MAX_RIF_MAX - 1;
+
+       /* Cannot exceed or equal max_rpcs_in_flight.  If we are asked to
+        * increase this value, also bump up max_rpcs_in_flight to match.
+        */

        if (max >= cli->cl_max_rpcs_in_flight) {
        if (max >= cli->cl_max_rpcs_in_flight) {

-               CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) "
-                      "higher or equal to max_rpcs_in_flight value (%u)\n",
-                      cli->cl_import->imp_obd->obd_name,
-                      max, cli->cl_max_rpcs_in_flight);
-               return -ERANGE;
+               CDEBUG(D_INFO,
+                      "%s: increasing max_rpcs_in_flight=%hu to allow larger max_mod_rpcs_in_flight=%u\n",
+                      cli->cl_import->imp_obd->obd_name, max + 1, max);
+               obd_set_max_rpcs_in_flight(cli, max + 1);

        }
 
        }
 

-       /* cannot exceed max modify RPCs in flight supported by the server */
-       ocd = &cli->cl_import->imp_connect_data;
-       if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS)
+       /* cannot exceed max modify RPCs in flight supported by the server,
+        * but verify ocd_connect_flags is at least initialized first.  If
+        * not, allow it and fix value later in ptlrpc_connect_set_flags().
+        */
+       if (!ocd->ocd_connect_flags) {
+               maxmodrpcs = cli->cl_max_rpcs_in_flight - 1;
+       } else if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS) {

                maxmodrpcs = ocd->ocd_maxmodrpcs;
                maxmodrpcs = ocd->ocd_maxmodrpcs;

-       else
+               if (maxmodrpcs == 0) { /* connection not finished yet */
+                       maxmodrpcs = cli->cl_max_rpcs_in_flight - 1;
+                       CDEBUG(D_INFO,
+                              "%s: partial connect, assume maxmodrpcs=%hu\n",
+                              cli->cl_import->imp_obd->obd_name, maxmodrpcs);
+               }
+       } else {

                maxmodrpcs = 1;
                maxmodrpcs = 1;

+       }

        if (max > maxmodrpcs) {
        if (max > maxmodrpcs) {

-               CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) "
-                      "higher than max_mod_rpcs_per_client value (%hu) "
-                      "returned by the server at connection\n",
+               CERROR("%s: can't set max_mod_rpcs_in_flight=%hu higher than ocd_maxmodrpcs=%hu returned by the server at connection\n",

                       cli->cl_import->imp_obd->obd_name,
                       max, maxmodrpcs);
                return -ERANGE;
                       cli->cl_import->imp_obd->obd_name,
                       max, maxmodrpcs);
                return -ERANGE;

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index d86fe47..7cea4cb 100644 (file)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -6660,10 +6660,41 @@ test_90b() {
 }
 run_test 90b "check max_mod_rpcs_in_flight is enforced after update"
 
 }
 run_test 90b "check max_mod_rpcs_in_flight is enforced after update"
 

+save_params_90c() {
+       # get max_rpcs_in_flight value
+       mrif_90c=$($LCTL get_param -n \
+                  mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight)
+       echo "max_rpcs_in_flight is $mrif_90c"
+
+       # get max_mod_rpcs_in_flight value
+       mmrif_90c=$($LCTL get_param -n \
+                   mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight)
+       echo "max_mod_rpcs_in_flight is $mmrif_90c"
+
+       # get MDT max_mod_rpcs_per_client value
+       mmrpc_90c=$(do_facet mds1 \
+                   cat /sys/module/mdt/parameters/max_mod_rpcs_per_client)
+       echo "max_mod_rpcs_per_client is $mmrpc_90c"
+}
+
+restore_params_90c() {
+       trap 0
+
+       # restore max_rpcs_in_flight value
+       do_facet mgs $LCTL set_param -P \
+               mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$mrif_90c
+
+       # restore max_mod_rpcs_in_flight value
+       do_facet mgs $LCTL set_param -P \
+               mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mmrif_90c
+
+       # restore MDT max_mod_rpcs_per_client value
+       do_facet mds1 "echo $mmrpc_90c > \
+                      /sys/module/mdt/parameters/max_mod_rpcs_per_client"
+}
+

 test_90c() {
        local tmp
 test_90c() {
        local tmp

-       local mrif
-       local mmrpc

 
        setup
 
 
        setup
 


@@ -6679,42 +6710,64 @@ test_90c() {
                skip "Client not able to send multiple modify RPCs in parallel"
        fi
 
                skip "Client not able to send multiple modify RPCs in parallel"
        fi
 

-       # get max_rpcs_in_flight value
-       mrif=$($LCTL get_param -n mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight)
-       echo "max_rpcs_in_flight is $mrif"
-
-       # get MDT max_mod_rpcs_per_client
-       mmrpc=$(do_facet mds1 \
-                   cat /sys/module/mdt/parameters/max_mod_rpcs_per_client)
-       echo "max_mod_rpcs_per_client is $mmrpc"
+       save_params_90c
+       stack_trap restore_params_90c

 
        # testcase 1
        # attempt to set max_mod_rpcs_in_flight to max_rpcs_in_flight value
        # prerequisite: set max_mod_rpcs_per_client to max_rpcs_in_flight value
 
        # testcase 1
        # attempt to set max_mod_rpcs_in_flight to max_rpcs_in_flight value
        # prerequisite: set max_mod_rpcs_per_client to max_rpcs_in_flight value

-       umount_client $MOUNT
-       do_facet mds1 \
-               "echo $mrif > /sys/module/mdt/parameters/max_mod_rpcs_per_client"
-       mount_client $MOUNT
+       do_facet mds1 "echo $mrif_90c > \
+                      /sys/module/mdt/parameters/max_mod_rpcs_per_client"

 
 

-       $LCTL set_param \
-           mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mrif &&
-           error "set max_mod_rpcs_in_flight to $mrif should fail"
+       # if max_mod_rpcs_in_flight is set to be equal to or larger than
+       # max_rpcs_in_flight, then max_rpcs_in_flight will be increased
+       if [[ "$CLIENT_VERSION" -ge $(version_code 2.13.53) ]]; then
+               $LCTL set_param \
+               mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mrif_90c ||
+                       error "set max_mod_rpcs_in_flight to $mrif_90c failed"
+
+               local new_mrif=$($LCTL get_param -n \
+                                mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight)
+               ((new_mrif == mrif_90c + 1)) ||
+                       error "max_rpcs_in_flight was not increased"
+       fi

 
        umount_client $MOUNT
 
        umount_client $MOUNT

-       do_facet mds1 \
-               "echo $mmrpc > /sys/module/mdt/parameters/max_mod_rpcs_per_client"
+       do_facet mds1 "echo $mmrpc_90c > \
+                      /sys/module/mdt/parameters/max_mod_rpcs_per_client"

        mount_client $MOUNT
 
        # testcase 2
        # attempt to set max_mod_rpcs_in_flight to max_mod_rpcs_per_client+1
        # prerequisite: set max_rpcs_in_flight to max_mod_rpcs_per_client+2
        $LCTL set_param \
        mount_client $MOUNT
 
        # testcase 2
        # attempt to set max_mod_rpcs_in_flight to max_mod_rpcs_per_client+1
        # prerequisite: set max_rpcs_in_flight to max_mod_rpcs_per_client+2
        $LCTL set_param \

-           mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$((mmrpc + 2))
+               mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$((mmrpc_90c + 2))

 
        $LCTL set_param \
 
        $LCTL set_param \

-           mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$((mmrpc + 1)) &&
-           error "set max_mod_rpcs_in_flight to $((mmrpc + 1)) should fail"
+       mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$((mmrpc_90c + 1)) &&
+       error "set max_mod_rpcs_in_flight to $((mmrpc_90c + 1)) should fail"
+
+       # testcase 3
+       # attempt to set max_mod_rpcs_in_flight permanently
+       do_facet mgs $LCTL set_param -P \
+               mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$mrif_90c
+
+       do_facet mgs $LCTL set_param -P \
+               mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mrif_90c
+
+       remount_client $MOUNT
+
+       wait_update_facet --verbose client "$LCTL get_param -n \
+               mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight" \
+               "$((mrif_90c + 1))" ||
+               error "expected '$((mrif_90c + 1))' for max_rpcs_in_flight"
+
+       wait_update_facet --verbose client "$LCTL get_param -n \
+               mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight" \
+               "$mrif_90c" ||
+               error "expected '$mrif_90c' for max_mod_rpcs_in_flight"

 
 

+       restore_params_90c

        cleanup
 }
 run_test 90c "check max_mod_rpcs_in_flight update limits"
        cleanup
 }
 run_test 90c "check max_mod_rpcs_in_flight update limits"