LU-5319 mdc: add max modify RPCs in flight variable

This patch introduces the maximum modify RPCs in flight variable of
a mdc client obd device. Its value is set from connection flag and
and connection data. It can later be tuned through the
max_mod_rpcs_in_flight procfs file.

Signed-off-by: Gregoire Pichon <gregoire.pichon@bull.net>
Change-Id: I51600f151fd508450d1898647a94856f16d2b245
Reviewed-on: http://review.whamcloud.com/14153
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Dmitry Eremin <dmitry.eremin@intel.com>
Reviewed-by: Mike Pershin <mike.pershin@intel.com>
Reviewed-by: Alex Zhuravlev <alexey.zhuravlev@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/include/obd.h b/lustre/include/obd.h
index 7568dec..ff5e8c5 100644 (file)
--- a/lustre/include/obd.h
+++ b/lustre/include/obd.h
@@ -320,6 +320,11 @@ struct client_obd {
         struct mdc_rpc_lock     *cl_rpc_lock;
         struct mdc_rpc_lock     *cl_close_lock;
 
         struct mdc_rpc_lock     *cl_rpc_lock;
         struct mdc_rpc_lock     *cl_close_lock;
 

+       /* modify rpcs in flight
+        * currently used for metadata only */
+       spinlock_t               cl_mod_rpcs_lock;
+       __u16                    cl_max_mod_rpcs_in_flight;
+

         /* mgc datastruct */
        struct mutex              cl_mgc_mutex;
        struct local_oid_storage *cl_mgc_los;
         /* mgc datastruct */
        struct mutex              cl_mgc_mutex;
        struct local_oid_storage *cl_mgc_los;

diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h
index c72a60e..9e05ebc 100644 (file)
--- a/lustre/include/obd_class.h
+++ b/lustre/include/obd_class.h
@@ -117,6 +117,8 @@ int obd_get_request_slot(struct client_obd *cli);
 void obd_put_request_slot(struct client_obd *cli);
 __u32 obd_get_max_rpcs_in_flight(struct client_obd *cli);
 int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max);
 void obd_put_request_slot(struct client_obd *cli);
 __u32 obd_get_max_rpcs_in_flight(struct client_obd *cli);
 int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max);

+__u16 obd_get_max_mod_rpcs_in_flight(struct client_obd *cli);
+int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, __u16 max);

 
 struct llog_handle;
 struct llog_rec_hdr;
 
 struct llog_handle;
 struct llog_rec_hdr;

diff --git a/lustre/mdc/lproc_mdc.c b/lustre/mdc/lproc_mdc.c
index 10eaf93..c5cb9d8 100644 (file)
--- a/lustre/mdc/lproc_mdc.c
+++ b/lustre/mdc/lproc_mdc.c
@@ -76,6 +76,44 @@ static ssize_t mdc_max_rpcs_in_flight_seq_write(struct file *file,
 }
 LPROC_SEQ_FOPS(mdc_max_rpcs_in_flight);
 
 }
 LPROC_SEQ_FOPS(mdc_max_rpcs_in_flight);
 

+
+static int mdc_max_mod_rpcs_in_flight_seq_show(struct seq_file *m, void *v)
+{
+       struct obd_device *dev = m->private;
+       __u16 max;
+       int rc;
+
+       max = obd_get_max_mod_rpcs_in_flight(&dev->u.cli);
+       rc = seq_printf(m, "%hu\n", max);
+
+       return rc;
+}
+
+static ssize_t mdc_max_mod_rpcs_in_flight_seq_write(struct file *file,
+                                                   const char *buffer,
+                                                   size_t count,
+                                                   loff_t *off)
+{
+       struct obd_device *dev =
+                       ((struct seq_file *)file->private_data)->private;
+       int val;
+       int rc;
+
+       rc = lprocfs_write_helper(buffer, count, &val);
+       if (rc != 0)
+               return rc;
+
+       if (val < 0 || val > USHRT_MAX)
+               return -ERANGE;
+
+       rc = obd_set_max_mod_rpcs_in_flight(&dev->u.cli, val);
+       if (rc != 0)
+               count = rc;
+
+       return count;
+}
+LPROC_SEQ_FOPS(mdc_max_mod_rpcs_in_flight);
+

 LPROC_SEQ_FOPS_WO_TYPE(mdc, ping);
 
 LPROC_SEQ_FOPS_RO_TYPE(mdc, uuid);
 LPROC_SEQ_FOPS_WO_TYPE(mdc, ping);
 
 LPROC_SEQ_FOPS_RO_TYPE(mdc, uuid);


@@ -134,6 +172,8 @@ struct lprocfs_vars lprocfs_mdc_obd_vars[] = {
          .fops =       &mdc_obd_max_pages_per_rpc_fops },
        { .name =       "max_rpcs_in_flight",
          .fops =       &mdc_max_rpcs_in_flight_fops    },
          .fops =       &mdc_obd_max_pages_per_rpc_fops },
        { .name =       "max_rpcs_in_flight",
          .fops =       &mdc_max_rpcs_in_flight_fops    },

+       { .name =       "max_mod_rpcs_in_flight",
+         .fops =       &mdc_max_mod_rpcs_in_flight_fops        },

        { .name =       "timeouts",
          .fops =       &mdc_timeouts_fops              },
        { .name =       "import",
        { .name =       "timeouts",
          .fops =       &mdc_timeouts_fops              },
        { .name =       "import",

diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c
index 82bc3c1..999e81a 100644 (file)
--- a/lustre/mdc/mdc_request.c
+++ b/lustre/mdc/mdc_request.c
@@ -2756,8 +2756,12 @@ static int mdc_setup(struct obd_device *obd, struct lustre_cfg *cfg)
         if (rc) {
                 mdc_cleanup(obd);
                 CERROR("failed to setup llogging subsystems\n");
         if (rc) {
                 mdc_cleanup(obd);
                 CERROR("failed to setup llogging subsystems\n");

+               RETURN(rc);

         }
 
         }
 

+       spin_lock_init(&cli->cl_mod_rpcs_lock);
+       cli->cl_max_mod_rpcs_in_flight = OBD_MAX_RIF_DEFAULT - 1;
+

         RETURN(rc);
 
 err_close_lock:
         RETURN(rc);
 
 err_close_lock:

diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c
index 784082a..4023a7d 100644 (file)
--- a/lustre/obdclass/genops.c
+++ b/lustre/obdclass/genops.c
@@ -1965,10 +1965,30 @@ int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max)
        __u32                           old;
        int                             diff;
        int                             i;
        __u32                           old;
        int                             diff;
        int                             i;

+       char                            *typ_name;
+       int                             rc;

 
        if (max > OBD_MAX_RIF_MAX || max < 1)
                return -ERANGE;
 
 
        if (max > OBD_MAX_RIF_MAX || max < 1)
                return -ERANGE;
 

+       typ_name = cli->cl_import->imp_obd->obd_type->typ_name;
+       if (strcmp(typ_name, LUSTRE_MDC_NAME) == 0) {
+               /* adjust max_mod_rpcs_in_flight to ensure it is always
+                * strictly lower that max_rpcs_in_flight */
+               if (max < 2) {
+                       CERROR("%s: cannot set max_rpcs_in_flight to 1 "
+                              "because it must be higher than "
+                              "max_mod_rpcs_in_flight value",
+                              cli->cl_import->imp_obd->obd_name);
+                       return -ERANGE;
+               }
+               if (max <= cli->cl_max_mod_rpcs_in_flight) {
+                       rc = obd_set_max_mod_rpcs_in_flight(cli, max - 1);
+                       if (rc != 0)
+                               return rc;
+               }
+       }
+

        spin_lock(&cli->cl_loi_list_lock);
        old = cli->cl_max_rpcs_in_flight;
        cli->cl_max_rpcs_in_flight = max;
        spin_lock(&cli->cl_loi_list_lock);
        old = cli->cl_max_rpcs_in_flight;
        cli->cl_max_rpcs_in_flight = max;


@@ -1990,3 +2010,50 @@ int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max)
        return 0;
 }
 EXPORT_SYMBOL(obd_set_max_rpcs_in_flight);
        return 0;
 }
 EXPORT_SYMBOL(obd_set_max_rpcs_in_flight);

+
+__u16 obd_get_max_mod_rpcs_in_flight(struct client_obd *cli)
+{
+       return cli->cl_max_mod_rpcs_in_flight;
+}
+EXPORT_SYMBOL(obd_get_max_mod_rpcs_in_flight);
+
+int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, __u16 max)
+{
+       struct obd_connect_data *ocd;
+       __u16 maxmodrpcs;
+
+       if (max > OBD_MAX_RIF_MAX || max < 1)
+               return -ERANGE;
+
+       /* cannot exceed or equal max_rpcs_in_flight */
+       if (max >= cli->cl_max_rpcs_in_flight) {
+               CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) "
+                      "higher or equal to max_rpcs_in_flight value (%u)\n",
+                      cli->cl_import->imp_obd->obd_name,
+                      max, cli->cl_max_rpcs_in_flight);
+               return -ERANGE;
+       }
+
+       /* cannot exceed max modify RPCs in flight supported by the server */
+       ocd = &cli->cl_import->imp_connect_data;
+       if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS)
+               maxmodrpcs = ocd->ocd_maxmodrpcs;
+       else
+               maxmodrpcs = 1;
+       if (max > maxmodrpcs) {
+               CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) "
+                      "higher than max_mod_rpcs_per_client value (%hu) "
+                      "returned by the server at connection\n",
+                      cli->cl_import->imp_obd->obd_name,
+                      max, maxmodrpcs);
+               return -ERANGE;
+       }
+
+       cli->cl_max_mod_rpcs_in_flight = max;
+
+       /* will have to wakeup waiters if max has been increased */
+
+       return 0;
+}
+EXPORT_SYMBOL(obd_set_max_mod_rpcs_in_flight);
+

diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c
index e4d1b9a..aa07573 100644 (file)
--- a/lustre/ptlrpc/import.c
+++ b/lustre/ptlrpc/import.c
@@ -897,6 +897,14 @@ static int ptlrpc_connect_set_flags(struct obd_import *imp,
 
        client_adjust_max_dirty(cli);
 
 
        client_adjust_max_dirty(cli);
 

+       /* Update client max modify RPCs in flight with value returned
+        * by the server */
+       if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS)
+               cli->cl_max_mod_rpcs_in_flight = min(
+                                       cli->cl_max_mod_rpcs_in_flight,
+                                       ocd->ocd_maxmodrpcs);
+       else
+               cli->cl_max_mod_rpcs_in_flight = 1;

 
        /* Reset ns_connect_flags only for initial connect. It might be
         * changed in while using FS and if we reset it in reconnect
 
        /* Reset ns_connect_flags only for initial connect. It might be
         * changed in while using FS and if we reset it in reconnect