From 60c05ea9f66f9bd3f5fd35942a12edb1e311c455 Mon Sep 17 00:00:00 2001 From: Gregoire Pichon Date: Wed, 18 Mar 2015 10:14:47 +0100 Subject: [PATCH] LU-5319 mdc: add max modify RPCs in flight variable This patch introduces the maximum modify RPCs in flight variable of a mdc client obd device. Its value is set from connection flag and and connection data. It can later be tuned through the max_mod_rpcs_in_flight procfs file. Signed-off-by: Gregoire Pichon Change-Id: I51600f151fd508450d1898647a94856f16d2b245 Reviewed-on: http://review.whamcloud.com/14153 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Dmitry Eremin Reviewed-by: Mike Pershin Reviewed-by: Alex Zhuravlev Reviewed-by: Oleg Drokin --- lustre/include/obd.h | 5 ++++ lustre/include/obd_class.h | 2 ++ lustre/mdc/lproc_mdc.c | 40 +++++++++++++++++++++++++++ lustre/mdc/mdc_request.c | 4 +++ lustre/obdclass/genops.c | 67 ++++++++++++++++++++++++++++++++++++++++++++++ lustre/ptlrpc/import.c | 8 ++++++ 6 files changed, 126 insertions(+) diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 7568dec..ff5e8c5 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -320,6 +320,11 @@ struct client_obd { struct mdc_rpc_lock *cl_rpc_lock; struct mdc_rpc_lock *cl_close_lock; + /* modify rpcs in flight + * currently used for metadata only */ + spinlock_t cl_mod_rpcs_lock; + __u16 cl_max_mod_rpcs_in_flight; + /* mgc datastruct */ struct mutex cl_mgc_mutex; struct local_oid_storage *cl_mgc_los; diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index c72a60e..9e05ebc 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -117,6 +117,8 @@ int obd_get_request_slot(struct client_obd *cli); void obd_put_request_slot(struct client_obd *cli); __u32 obd_get_max_rpcs_in_flight(struct client_obd *cli); int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max); +__u16 obd_get_max_mod_rpcs_in_flight(struct client_obd *cli); +int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, __u16 max); struct llog_handle; struct llog_rec_hdr; diff --git a/lustre/mdc/lproc_mdc.c b/lustre/mdc/lproc_mdc.c index 10eaf93..c5cb9d8 100644 --- a/lustre/mdc/lproc_mdc.c +++ b/lustre/mdc/lproc_mdc.c @@ -76,6 +76,44 @@ static ssize_t mdc_max_rpcs_in_flight_seq_write(struct file *file, } LPROC_SEQ_FOPS(mdc_max_rpcs_in_flight); + +static int mdc_max_mod_rpcs_in_flight_seq_show(struct seq_file *m, void *v) +{ + struct obd_device *dev = m->private; + __u16 max; + int rc; + + max = obd_get_max_mod_rpcs_in_flight(&dev->u.cli); + rc = seq_printf(m, "%hu\n", max); + + return rc; +} + +static ssize_t mdc_max_mod_rpcs_in_flight_seq_write(struct file *file, + const char *buffer, + size_t count, + loff_t *off) +{ + struct obd_device *dev = + ((struct seq_file *)file->private_data)->private; + int val; + int rc; + + rc = lprocfs_write_helper(buffer, count, &val); + if (rc != 0) + return rc; + + if (val < 0 || val > USHRT_MAX) + return -ERANGE; + + rc = obd_set_max_mod_rpcs_in_flight(&dev->u.cli, val); + if (rc != 0) + count = rc; + + return count; +} +LPROC_SEQ_FOPS(mdc_max_mod_rpcs_in_flight); + LPROC_SEQ_FOPS_WO_TYPE(mdc, ping); LPROC_SEQ_FOPS_RO_TYPE(mdc, uuid); @@ -134,6 +172,8 @@ struct lprocfs_vars lprocfs_mdc_obd_vars[] = { .fops = &mdc_obd_max_pages_per_rpc_fops }, { .name = "max_rpcs_in_flight", .fops = &mdc_max_rpcs_in_flight_fops }, + { .name = "max_mod_rpcs_in_flight", + .fops = &mdc_max_mod_rpcs_in_flight_fops }, { .name = "timeouts", .fops = &mdc_timeouts_fops }, { .name = "import", diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index 82bc3c1..999e81a 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -2756,8 +2756,12 @@ static int mdc_setup(struct obd_device *obd, struct lustre_cfg *cfg) if (rc) { mdc_cleanup(obd); CERROR("failed to setup llogging subsystems\n"); + RETURN(rc); } + spin_lock_init(&cli->cl_mod_rpcs_lock); + cli->cl_max_mod_rpcs_in_flight = OBD_MAX_RIF_DEFAULT - 1; + RETURN(rc); err_close_lock: diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c index 784082a..4023a7d 100644 --- a/lustre/obdclass/genops.c +++ b/lustre/obdclass/genops.c @@ -1965,10 +1965,30 @@ int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max) __u32 old; int diff; int i; + char *typ_name; + int rc; if (max > OBD_MAX_RIF_MAX || max < 1) return -ERANGE; + typ_name = cli->cl_import->imp_obd->obd_type->typ_name; + if (strcmp(typ_name, LUSTRE_MDC_NAME) == 0) { + /* adjust max_mod_rpcs_in_flight to ensure it is always + * strictly lower that max_rpcs_in_flight */ + if (max < 2) { + CERROR("%s: cannot set max_rpcs_in_flight to 1 " + "because it must be higher than " + "max_mod_rpcs_in_flight value", + cli->cl_import->imp_obd->obd_name); + return -ERANGE; + } + if (max <= cli->cl_max_mod_rpcs_in_flight) { + rc = obd_set_max_mod_rpcs_in_flight(cli, max - 1); + if (rc != 0) + return rc; + } + } + spin_lock(&cli->cl_loi_list_lock); old = cli->cl_max_rpcs_in_flight; cli->cl_max_rpcs_in_flight = max; @@ -1990,3 +2010,50 @@ int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max) return 0; } EXPORT_SYMBOL(obd_set_max_rpcs_in_flight); + +__u16 obd_get_max_mod_rpcs_in_flight(struct client_obd *cli) +{ + return cli->cl_max_mod_rpcs_in_flight; +} +EXPORT_SYMBOL(obd_get_max_mod_rpcs_in_flight); + +int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, __u16 max) +{ + struct obd_connect_data *ocd; + __u16 maxmodrpcs; + + if (max > OBD_MAX_RIF_MAX || max < 1) + return -ERANGE; + + /* cannot exceed or equal max_rpcs_in_flight */ + if (max >= cli->cl_max_rpcs_in_flight) { + CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) " + "higher or equal to max_rpcs_in_flight value (%u)\n", + cli->cl_import->imp_obd->obd_name, + max, cli->cl_max_rpcs_in_flight); + return -ERANGE; + } + + /* cannot exceed max modify RPCs in flight supported by the server */ + ocd = &cli->cl_import->imp_connect_data; + if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS) + maxmodrpcs = ocd->ocd_maxmodrpcs; + else + maxmodrpcs = 1; + if (max > maxmodrpcs) { + CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) " + "higher than max_mod_rpcs_per_client value (%hu) " + "returned by the server at connection\n", + cli->cl_import->imp_obd->obd_name, + max, maxmodrpcs); + return -ERANGE; + } + + cli->cl_max_mod_rpcs_in_flight = max; + + /* will have to wakeup waiters if max has been increased */ + + return 0; +} +EXPORT_SYMBOL(obd_set_max_mod_rpcs_in_flight); + diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c index e4d1b9a..aa07573 100644 --- a/lustre/ptlrpc/import.c +++ b/lustre/ptlrpc/import.c @@ -897,6 +897,14 @@ static int ptlrpc_connect_set_flags(struct obd_import *imp, client_adjust_max_dirty(cli); + /* Update client max modify RPCs in flight with value returned + * by the server */ + if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS) + cli->cl_max_mod_rpcs_in_flight = min( + cli->cl_max_mod_rpcs_in_flight, + ocd->ocd_maxmodrpcs); + else + cli->cl_max_mod_rpcs_in_flight = 1; /* Reset ns_connect_flags only for initial connect. It might be * changed in while using FS and if we reset it in reconnect -- 1.8.3.1