From 6d314902e6d19229379577aab60d4b20a5b4d2ea Mon Sep 17 00:00:00 2001 From: Andreas Dilger Date: Wed, 27 May 2020 12:22:12 -0700 Subject: [PATCH] LU-13503 mdc: allow setting max_mod_rpcs_in_flight larger Allow setting mdc.*.max_mod_rpcs_in_flight > mdc.*.max_rpcs_in_flight by increasing the latter value, rather than returning an error and telling the user to do that. This matches the similar behavior if mdc.*.max_rpcs_in_flight is reduced lower than max_mod_rpcs_in_flight. If there are multiple MDTs, the "mdc.*.max_mod_rpcs_in_flight" param may be set from e.g. the MDT0000 config log before MDT0001 is fully configured, catching MDT0001 with ocd_maxmodrpcs = 0 before the OCD from the MDT has been filled in, and incorrectly trigger an error. If seen during setup, allow ocd_maxmodrpcs = (max_rpcs_in_flight - 1), since this will be fixed up later if mdc.*.max_rpcs_in_flight is set smaller in the config log (if set larger it doesn't matter). Test-Parameters: env=ONLY=90 testlist=conf-sanity Signed-off-by: Andreas Dilger Signed-off-by: Jian Yu Change-Id: I4b20163e9e212db451738169ebdc361ab8c1c15e Reviewed-on: https://review.whamcloud.com/38455 Reviewed-by: John L. Hammond Tested-by: jenkins Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- lustre/obdclass/genops.c | 59 +++++++++++++++++---------- lustre/tests/conf-sanity.sh | 97 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 114 insertions(+), 42 deletions(-) diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c index 0b2612c..d4feb7c 100644 --- a/lustre/obdclass/genops.c +++ b/lustre/obdclass/genops.c @@ -2062,20 +2062,21 @@ int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max) __u32 old; int diff; int i; - const char *type_name; int rc; if (max > OBD_MAX_RIF_MAX || max < 1) return -ERANGE; - type_name = cli->cl_import->imp_obd->obd_type->typ_name; - if (strcmp(type_name, LUSTRE_MDC_NAME) == 0) { + CDEBUG(D_INFO, "%s: max = %hu max_mod = %u rif = %u\n", + cli->cl_import->imp_obd->obd_name, max, + cli->cl_max_mod_rpcs_in_flight, cli->cl_max_rpcs_in_flight); + + if (strcmp(cli->cl_import->imp_obd->obd_type->typ_name, + LUSTRE_MDC_NAME) == 0) { /* adjust max_mod_rpcs_in_flight to ensure it is always * strictly lower that max_rpcs_in_flight */ if (max < 2) { - CERROR("%s: cannot set max_rpcs_in_flight to 1 " - "because it must be higher than " - "max_mod_rpcs_in_flight value", + CERROR("%s: cannot set mdc.*.max_rpcs_in_flight=1\n", cli->cl_import->imp_obd->obd_name); return -ERANGE; } @@ -2119,32 +2120,50 @@ EXPORT_SYMBOL(obd_get_max_mod_rpcs_in_flight); int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, __u16 max) { - struct obd_connect_data *ocd; + struct obd_connect_data *ocd; __u16 maxmodrpcs; __u16 prev; if (max > OBD_MAX_RIF_MAX || max < 1) return -ERANGE; - /* cannot exceed or equal max_rpcs_in_flight */ + ocd = &cli->cl_import->imp_connect_data; + CDEBUG(D_INFO, "%s: max = %hu flags = %llx, max_mod = %u rif = %u\n", + cli->cl_import->imp_obd->obd_name, max, ocd->ocd_connect_flags, + ocd->ocd_maxmodrpcs, cli->cl_max_rpcs_in_flight); + + if (max == OBD_MAX_RIF_MAX) + max = OBD_MAX_RIF_MAX - 1; + + /* Cannot exceed or equal max_rpcs_in_flight. If we are asked to + * increase this value, also bump up max_rpcs_in_flight to match. + */ if (max >= cli->cl_max_rpcs_in_flight) { - CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) " - "higher or equal to max_rpcs_in_flight value (%u)\n", - cli->cl_import->imp_obd->obd_name, - max, cli->cl_max_rpcs_in_flight); - return -ERANGE; + CDEBUG(D_INFO, + "%s: increasing max_rpcs_in_flight=%hu to allow larger max_mod_rpcs_in_flight=%u\n", + cli->cl_import->imp_obd->obd_name, max + 1, max); + obd_set_max_rpcs_in_flight(cli, max + 1); } - /* cannot exceed max modify RPCs in flight supported by the server */ - ocd = &cli->cl_import->imp_connect_data; - if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS) + /* cannot exceed max modify RPCs in flight supported by the server, + * but verify ocd_connect_flags is at least initialized first. If + * not, allow it and fix value later in ptlrpc_connect_set_flags(). + */ + if (!ocd->ocd_connect_flags) { + maxmodrpcs = cli->cl_max_rpcs_in_flight - 1; + } else if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS) { maxmodrpcs = ocd->ocd_maxmodrpcs; - else + if (maxmodrpcs == 0) { /* connection not finished yet */ + maxmodrpcs = cli->cl_max_rpcs_in_flight - 1; + CDEBUG(D_INFO, + "%s: partial connect, assume maxmodrpcs=%hu\n", + cli->cl_import->imp_obd->obd_name, maxmodrpcs); + } + } else { maxmodrpcs = 1; + } if (max > maxmodrpcs) { - CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) " - "higher than max_mod_rpcs_per_client value (%hu) " - "returned by the server at connection\n", + CERROR("%s: can't set max_mod_rpcs_in_flight=%hu higher than ocd_maxmodrpcs=%hu returned by the server at connection\n", cli->cl_import->imp_obd->obd_name, max, maxmodrpcs); return -ERANGE; diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index d86fe47..7cea4cb 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -6660,10 +6660,41 @@ test_90b() { } run_test 90b "check max_mod_rpcs_in_flight is enforced after update" +save_params_90c() { + # get max_rpcs_in_flight value + mrif_90c=$($LCTL get_param -n \ + mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight) + echo "max_rpcs_in_flight is $mrif_90c" + + # get max_mod_rpcs_in_flight value + mmrif_90c=$($LCTL get_param -n \ + mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight) + echo "max_mod_rpcs_in_flight is $mmrif_90c" + + # get MDT max_mod_rpcs_per_client value + mmrpc_90c=$(do_facet mds1 \ + cat /sys/module/mdt/parameters/max_mod_rpcs_per_client) + echo "max_mod_rpcs_per_client is $mmrpc_90c" +} + +restore_params_90c() { + trap 0 + + # restore max_rpcs_in_flight value + do_facet mgs $LCTL set_param -P \ + mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$mrif_90c + + # restore max_mod_rpcs_in_flight value + do_facet mgs $LCTL set_param -P \ + mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mmrif_90c + + # restore MDT max_mod_rpcs_per_client value + do_facet mds1 "echo $mmrpc_90c > \ + /sys/module/mdt/parameters/max_mod_rpcs_per_client" +} + test_90c() { local tmp - local mrif - local mmrpc setup @@ -6679,42 +6710,64 @@ test_90c() { skip "Client not able to send multiple modify RPCs in parallel" fi - # get max_rpcs_in_flight value - mrif=$($LCTL get_param -n mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight) - echo "max_rpcs_in_flight is $mrif" - - # get MDT max_mod_rpcs_per_client - mmrpc=$(do_facet mds1 \ - cat /sys/module/mdt/parameters/max_mod_rpcs_per_client) - echo "max_mod_rpcs_per_client is $mmrpc" + save_params_90c + stack_trap restore_params_90c # testcase 1 # attempt to set max_mod_rpcs_in_flight to max_rpcs_in_flight value # prerequisite: set max_mod_rpcs_per_client to max_rpcs_in_flight value - umount_client $MOUNT - do_facet mds1 \ - "echo $mrif > /sys/module/mdt/parameters/max_mod_rpcs_per_client" - mount_client $MOUNT + do_facet mds1 "echo $mrif_90c > \ + /sys/module/mdt/parameters/max_mod_rpcs_per_client" - $LCTL set_param \ - mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mrif && - error "set max_mod_rpcs_in_flight to $mrif should fail" + # if max_mod_rpcs_in_flight is set to be equal to or larger than + # max_rpcs_in_flight, then max_rpcs_in_flight will be increased + if [[ "$CLIENT_VERSION" -ge $(version_code 2.13.53) ]]; then + $LCTL set_param \ + mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mrif_90c || + error "set max_mod_rpcs_in_flight to $mrif_90c failed" + + local new_mrif=$($LCTL get_param -n \ + mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight) + ((new_mrif == mrif_90c + 1)) || + error "max_rpcs_in_flight was not increased" + fi umount_client $MOUNT - do_facet mds1 \ - "echo $mmrpc > /sys/module/mdt/parameters/max_mod_rpcs_per_client" + do_facet mds1 "echo $mmrpc_90c > \ + /sys/module/mdt/parameters/max_mod_rpcs_per_client" mount_client $MOUNT # testcase 2 # attempt to set max_mod_rpcs_in_flight to max_mod_rpcs_per_client+1 # prerequisite: set max_rpcs_in_flight to max_mod_rpcs_per_client+2 $LCTL set_param \ - mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$((mmrpc + 2)) + mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$((mmrpc_90c + 2)) $LCTL set_param \ - mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$((mmrpc + 1)) && - error "set max_mod_rpcs_in_flight to $((mmrpc + 1)) should fail" + mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$((mmrpc_90c + 1)) && + error "set max_mod_rpcs_in_flight to $((mmrpc_90c + 1)) should fail" + + # testcase 3 + # attempt to set max_mod_rpcs_in_flight permanently + do_facet mgs $LCTL set_param -P \ + mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight=$mrif_90c + + do_facet mgs $LCTL set_param -P \ + mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight=$mrif_90c + + remount_client $MOUNT + + wait_update_facet --verbose client "$LCTL get_param -n \ + mdc.$FSNAME-MDT0000-mdc-*.max_rpcs_in_flight" \ + "$((mrif_90c + 1))" || + error "expected '$((mrif_90c + 1))' for max_rpcs_in_flight" + + wait_update_facet --verbose client "$LCTL get_param -n \ + mdc.$FSNAME-MDT0000-mdc-*.max_mod_rpcs_in_flight" \ + "$mrif_90c" || + error "expected '$mrif_90c' for max_mod_rpcs_in_flight" + restore_params_90c cleanup } run_test 90c "check max_mod_rpcs_in_flight update limits" -- 1.8.3.1