LU-56 ptlrpc: common code to validate nthreads

Add a common function to validate ptlrpc service threads number,
instead of checking it before registration of each service.
This function (ptlrpc_server_nthreads_check) will be enhanced
soon for CPT affinity ptlrpc service.

Signed-off-by: Liang Zhen <liang@whamcloud.com>
Change-Id: I5db09396f4a95e36d8f5d38777c51eafb7dea5b1
Reviewed-on: http://review.whamcloud.com/2879
Tested-by: Hudson
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: wangdi <di.wang@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h
index 4d8341d..0ab359f 100644 (file)
--- a/lustre/include/lustre_net.h
+++ b/lustre/include/lustre_net.h
@@ -106,6 +106,8 @@
 # endif
 #endif /* __KERNEL__ */
 
+#define PTLRPC_NTHRS_MIN       2
+
 /**
  * The following constants determine how memory is used to buffer incoming
  * service requests.
@@ -130,8 +132,8 @@
 #define LDLM_MAXREPSIZE (1024)
 
 /** Absolute limits */
-#define MDT_MIN_THREADS 2UL
 #ifndef MDT_MAX_THREADS
+#define MDT_MIN_THREADS PTLRPC_NTHRS_MIN
 #define MDT_MAX_THREADS 512UL
 #endif
 #define MDS_NBUFS       (64 * cfs_num_online_cpus())
@@ -1583,6 +1585,9 @@ struct ptlrpc_service_thr_conf {
        unsigned int                    tc_nthrs_min;
        /* max number of service threads to start */
        unsigned int                    tc_nthrs_max;
+       /* user specified threads number, it will be validated due to
+        * other members of this structure. */
+       unsigned int                    tc_nthrs_user;
        /* set NUMA node affinity for service threads */
        unsigned int                    tc_cpu_affinity;
        /* Tags for lu_context associated with service thread */

diff --git a/lustre/ldlm/ldlm_lockd.c b/lustre/ldlm/ldlm_lockd.c
index 2c65a42..0c4a181 100644 (file)
--- a/lustre/ldlm/ldlm_lockd.c
+++ b/lustre/ldlm/ldlm_lockd.c
@@ -52,11 +52,9 @@
 #include <libcfs/list.h>
 #include "ldlm_internal.h"
 
-#ifdef __KERNEL__
 static int ldlm_num_threads;
 CFS_MODULE_PARM(ldlm_num_threads, "i", int, 0444,
                 "number of DLM service threads to start");
-#endif
 
 extern cfs_mem_cache_t *ldlm_resource_slab;
 extern cfs_mem_cache_t *ldlm_lock_slab;
@@ -2524,8 +2522,6 @@ static int ldlm_setup(void)
        static struct ptlrpc_service_conf       conf;
        struct ldlm_bl_pool                     *blp = NULL;
         int rc = 0;
-        int ldlm_min_threads = LDLM_THREADS_AUTO_MIN;
-        int ldlm_max_threads = LDLM_THREADS_AUTO_MAX;
 #ifdef __KERNEL__
         int i;
 #endif
@@ -2544,16 +2540,6 @@ static int ldlm_setup(void)
                GOTO(out, rc);
 #endif
 
-#ifdef __KERNEL__
-        if (ldlm_num_threads) {
-                /* If ldlm_num_threads is set, it is the min and the max. */
-                if (ldlm_num_threads > LDLM_THREADS_AUTO_MAX)
-                        ldlm_num_threads = LDLM_THREADS_AUTO_MAX;
-                if (ldlm_num_threads < LDLM_THREADS_AUTO_MIN)
-                        ldlm_num_threads = LDLM_THREADS_AUTO_MIN;
-                ldlm_min_threads = ldlm_max_threads = ldlm_num_threads;
-        }
-#endif
        memset(&conf, 0, sizeof(conf));
        conf = (typeof(conf)) {
                .psc_name               = "ldlm_cbd",
@@ -2568,8 +2554,9 @@ static int ldlm_setup(void)
                },
                .psc_thr                = {
                        .tc_thr_name            = "ldlm_cb",
-                       .tc_nthrs_min           = ldlm_min_threads,
-                       .tc_nthrs_max           = ldlm_max_threads,
+                       .tc_nthrs_min           = LDLM_THREADS_AUTO_MIN,
+                       .tc_nthrs_max           = LDLM_THREADS_AUTO_MAX,
+                       .tc_nthrs_user          = ldlm_num_threads,
                        .tc_ctx_tags            = LCT_MD_THREAD | \
                                                  LCT_DT_THREAD,
                },
@@ -2602,8 +2589,9 @@ static int ldlm_setup(void)
                },
                .psc_thr                = {
                        .tc_thr_name            = "ldlm_cn",
-                       .tc_nthrs_min           = ldlm_min_threads,
-                       .tc_nthrs_max           = ldlm_max_threads,
+                       .tc_nthrs_min           = LDLM_THREADS_AUTO_MIN,
+                       .tc_nthrs_max           = LDLM_THREADS_AUTO_MAX,
+                       .tc_nthrs_user          = ldlm_num_threads,
                        .tc_ctx_tags            = LCT_MD_THREAD | \
                                                  LCT_DT_THREAD | \
                                                  LCT_CL_THREAD,
@@ -2634,10 +2622,18 @@ static int ldlm_setup(void)
         cfs_waitq_init(&blp->blp_waitq);
         cfs_atomic_set(&blp->blp_num_threads, 0);
         cfs_atomic_set(&blp->blp_busy_threads, 0);
-        blp->blp_min_threads = ldlm_min_threads;
-        blp->blp_max_threads = ldlm_max_threads;
 
 #ifdef __KERNEL__
+       if (ldlm_num_threads == 0) {
+               blp->blp_min_threads = LDLM_THREADS_AUTO_MIN;
+               blp->blp_max_threads = LDLM_THREADS_AUTO_MAX;
+       } else {
+               blp->blp_min_threads = blp->blp_max_threads = \
+                       min_t(int, LDLM_THREADS_AUTO_MAX,
+                                  max_t(int, LDLM_THREADS_AUTO_MIN,
+                                             ldlm_num_threads));
+       }
+
         for (i = 0; i < blp->blp_min_threads; i++) {
                 rc = ldlm_bl_thread_start(blp);
                 if (rc < 0)

diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index f240536..d81d0a5 100644 (file)
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -97,8 +97,6 @@ ldlm_mode_t mdt_dlm_lock_modes[] = {
  * Initialized in mdt_mod_init().
  */
 static unsigned long mdt_num_threads;
-static unsigned long mdt_min_threads;
-static unsigned long mdt_max_threads;
 
 /* ptlrpc request handler for MDT. All handlers are
  * grouped into several slices - struct mdt_opc_slice,
@@ -3931,8 +3929,9 @@ static int mdt_start_ptlrpc_service(struct mdt_device *m)
                 */
                .psc_thr                = {
                        .tc_thr_name            = LUSTRE_MDT_NAME,
-                       .tc_nthrs_min           = mdt_min_threads,
-                       .tc_nthrs_max           = mdt_max_threads,
+                       .tc_nthrs_min           = MDT_MIN_THREADS,
+                       .tc_nthrs_max           = MDT_MAX_THREADS,
+                       .tc_nthrs_user          = mdt_num_threads,
                        .tc_ctx_tags            = LCT_MD_THREAD,
                },
                .psc_ops                = {
@@ -3967,8 +3966,9 @@ static int mdt_start_ptlrpc_service(struct mdt_device *m)
                },
                .psc_thr                = {
                        .tc_thr_name            = "mdt_rdpg",
-                       .tc_nthrs_min           = mdt_min_threads,
-                       .tc_nthrs_max           = mdt_max_threads,
+                       .tc_nthrs_min           = MDT_MIN_THREADS,
+                       .tc_nthrs_max           = MDT_MAX_THREADS,
+                       .tc_nthrs_user          = mdt_num_threads,
                        .tc_ctx_tags            = LCT_MD_THREAD,
                },
                .psc_ops                = {
@@ -4006,8 +4006,9 @@ static int mdt_start_ptlrpc_service(struct mdt_device *m)
                },
                .psc_thr                = {
                        .tc_thr_name            = "mdt_attr",
-                       .tc_nthrs_min           = mdt_min_threads,
-                       .tc_nthrs_max           = mdt_max_threads,
+                       .tc_nthrs_min           = MDT_MIN_THREADS,
+                       .tc_nthrs_max           = MDT_MAX_THREADS,
+                       .tc_nthrs_user          = mdt_num_threads,
                        .tc_ctx_tags            = LCT_MD_THREAD,
                },
                .psc_ops                = {
@@ -4041,8 +4042,9 @@ static int mdt_start_ptlrpc_service(struct mdt_device *m)
                },
                .psc_thr                = {
                        .tc_thr_name            = "mdt_mdsc",
-                       .tc_nthrs_min           = mdt_min_threads,
-                       .tc_nthrs_max           = mdt_max_threads,
+                       .tc_nthrs_min           = MDT_MIN_THREADS,
+                       .tc_nthrs_max           = MDT_MAX_THREADS,
+                       .tc_nthrs_user          = mdt_num_threads,
                        .tc_ctx_tags            = LCT_MD_THREAD,
                },
                .psc_ops                = {
@@ -4076,10 +4078,10 @@ static int mdt_start_ptlrpc_service(struct mdt_device *m)
                },
                .psc_thr                = {
                        .tc_thr_name            = "mdt_mdss",
-                       .tc_nthrs_min           = mdt_min_threads,
-                       .tc_nthrs_max           = mdt_max_threads,
-                       .tc_ctx_tags            = LCT_MD_THREAD | \
-                                                 LCT_DT_THREAD,
+                       .tc_nthrs_min           = MDT_MIN_THREADS,
+                       .tc_nthrs_max           = MDT_MAX_THREADS,
+                       .tc_nthrs_user          = mdt_num_threads,
+                       .tc_ctx_tags            = LCT_MD_THREAD | LCT_DT_THREAD
                },
                .psc_ops                = {
                        .so_req_handler         = mdt_mdss_handle,
@@ -4114,10 +4116,10 @@ static int mdt_start_ptlrpc_service(struct mdt_device *m)
                },
                .psc_thr                = {
                        .tc_thr_name            = "mdt_dtss",
-                       .tc_nthrs_min           = mdt_min_threads,
-                       .tc_nthrs_max           = mdt_max_threads,
-                       .tc_ctx_tags            = LCT_MD_THREAD | \
-                                                 LCT_DT_THREAD,
+                       .tc_nthrs_min           = MDT_MIN_THREADS,
+                       .tc_nthrs_max           = MDT_MAX_THREADS,
+                       .tc_nthrs_user          = mdt_num_threads,
+                       .tc_ctx_tags            = LCT_MD_THREAD | LCT_DT_THREAD
                },
                .psc_ops                = {
                        .so_req_handler         = mdt_dtss_handle,
@@ -4148,10 +4150,10 @@ static int mdt_start_ptlrpc_service(struct mdt_device *m)
                },
                .psc_thr                = {
                        .tc_thr_name            = "mdt_fld",
-                       .tc_nthrs_min           = mdt_min_threads,
-                       .tc_nthrs_max           = mdt_max_threads,
-                       .tc_ctx_tags            = LCT_DT_THREAD | \
-                                                 LCT_MD_THREAD,
+                       .tc_nthrs_min           = MDT_MIN_THREADS,
+                       .tc_nthrs_max           = MDT_MAX_THREADS,
+                       .tc_nthrs_user          = mdt_num_threads,
+                       .tc_ctx_tags            = LCT_DT_THREAD | LCT_MD_THREAD
                },
                .psc_ops                = {
                        .so_req_handler         = mdt_fld_handle,
@@ -4185,8 +4187,9 @@ static int mdt_start_ptlrpc_service(struct mdt_device *m)
                },
                .psc_thr                = {
                        .tc_thr_name            = "mdt_mds",
-                       .tc_nthrs_min           = mdt_min_threads,
-                       .tc_nthrs_max           = mdt_max_threads,
+                       .tc_nthrs_min           = MDT_MIN_THREADS,
+                       .tc_nthrs_max           = MDT_MAX_THREADS,
+                       .tc_nthrs_user          = mdt_num_threads,
                        .tc_ctx_tags            = LCT_MD_THREAD,
                },
                .psc_ops                = {
@@ -5857,17 +5860,6 @@ static int __init mdt_mod_init(void)
         struct lprocfs_static_vars lvars;
         int rc;
 
-        if (mdt_num_threads > 0) {
-                if (mdt_num_threads > MDT_MAX_THREADS)
-                        mdt_num_threads = MDT_MAX_THREADS;
-                if (mdt_num_threads < MDT_MIN_THREADS)
-                        mdt_num_threads = MDT_MIN_THREADS;
-                mdt_max_threads = mdt_min_threads = mdt_num_threads;
-        } else {
-                mdt_max_threads = MDT_MAX_THREADS;
-                mdt_min_threads = MDT_MIN_THREADS;
-        }
-
         lprocfs_mdt_init_vars(&lvars);
         rc = class_register_type(&mdt_obd_device_ops, NULL,
                                  lvars.module_vars, LUSTRE_MDT_NAME,

diff --git a/lustre/ost/ost_handler.c b/lustre/ost/ost_handler.c
index dddb4dd..bf50ab3 100644 (file)
--- a/lustre/ost/ost_handler.c
+++ b/lustre/ost/ost_handler.c
@@ -2401,14 +2401,12 @@ static int ost_thread_init(struct ptlrpc_thread *thread)
 static int ost_setup(struct obd_device *obd, struct lustre_cfg* lcfg)
 {
        static struct ptlrpc_service_conf       svc_conf;
-        struct ost_obd *ost = &obd->u.ost;
-        struct lprocfs_static_vars lvars;
-        int oss_min_threads;
-        int oss_max_threads;
-        int oss_min_create_threads;
-        int oss_max_create_threads;
-        int rc;
-        ENTRY;
+       struct ost_obd *ost = &obd->u.ost;
+       struct lprocfs_static_vars lvars;
+       int oss_min_threads = OSS_THREADS_MIN;
+       int oss_max_threads = OSS_THREADS_MAX;
+       int rc;
+       ENTRY;
 
         rc = cfs_cleanup_group_info();
         if (rc)
@@ -2419,14 +2417,7 @@ static int ost_setup(struct obd_device *obd, struct lustre_cfg* lcfg)
 
         cfs_mutex_init(&ost->ost_health_mutex);
 
-        if (oss_num_threads) {
-                /* If oss_num_threads is set, it is the min and the max. */
-                if (oss_num_threads > OSS_THREADS_MAX)
-                        oss_num_threads = OSS_THREADS_MAX;
-                if (oss_num_threads < OSS_THREADS_MIN)
-                        oss_num_threads = OSS_THREADS_MIN;
-                oss_max_threads = oss_min_threads = oss_num_threads;
-       } else {
+       if (oss_num_threads == 0) {
                /* Base min threads on memory and cpus */
                oss_min_threads =
                        cfs_num_online_cpus() * CFS_NUM_CACHEPAGES >>
@@ -2437,7 +2428,7 @@ static int ost_setup(struct obd_device *obd, struct lustre_cfg* lcfg)
                if (oss_min_threads > OSS_THREADS_MAX / 4)
                        oss_min_threads = OSS_THREADS_MAX / 4;
                oss_max_threads = min(OSS_THREADS_MAX, oss_min_threads * 4 + 1);
-       }
+        }
 
        svc_conf = (typeof(svc_conf)) {
                .psc_name               = LUSTRE_OSS_NAME,
@@ -2454,6 +2445,7 @@ static int ost_setup(struct obd_device *obd, struct lustre_cfg* lcfg)
                        .tc_thr_name            = "ll_ost",
                        .tc_nthrs_min           = oss_min_threads,
                        .tc_nthrs_max           = oss_max_threads,
+                       .tc_nthrs_user          = oss_num_threads,
                        .tc_ctx_tags            = LCT_DT_THREAD,
                },
                .psc_ops                = {
@@ -2469,18 +2461,6 @@ static int ost_setup(struct obd_device *obd, struct lustre_cfg* lcfg)
                GOTO(out_lprocfs, rc);
         }
 
-        if (oss_num_create_threads) {
-                if (oss_num_create_threads > OSS_MAX_CREATE_THREADS)
-                        oss_num_create_threads = OSS_MAX_CREATE_THREADS;
-                if (oss_num_create_threads < OSS_MIN_CREATE_THREADS)
-                        oss_num_create_threads = OSS_MIN_CREATE_THREADS;
-                oss_min_create_threads = oss_max_create_threads =
-                        oss_num_create_threads;
-        } else {
-                oss_min_create_threads = OSS_MIN_CREATE_THREADS;
-                oss_max_create_threads = OSS_MAX_CREATE_THREADS;
-        }
-
        memset(&svc_conf, 0, sizeof(svc_conf));
        svc_conf = (typeof(svc_conf)) {
                .psc_name               = "ost_create",
@@ -2495,8 +2475,9 @@ static int ost_setup(struct obd_device *obd, struct lustre_cfg* lcfg)
                },
                .psc_thr                = {
                        .tc_thr_name            = "ll_ost_create",
-                       .tc_nthrs_min           = oss_min_create_threads,
-                       .tc_nthrs_max           = oss_max_create_threads,
+                       .tc_nthrs_min           = OSS_CR_THREADS_MIN,
+                       .tc_nthrs_max           = OSS_CR_THREADS_MAX,
+                       .tc_nthrs_user          = oss_num_create_threads,
                        .tc_ctx_tags            = LCT_DT_THREAD,
                },
                .psc_ops                = {
@@ -2528,6 +2509,7 @@ static int ost_setup(struct obd_device *obd, struct lustre_cfg* lcfg)
                        .tc_thr_name            = "ll_ost_io",
                        .tc_nthrs_min           = oss_min_threads,
                        .tc_nthrs_max           = oss_max_threads,
+                       .tc_nthrs_user          = oss_num_threads,
                        .tc_cpu_affinity        = 1,
                        .tc_ctx_tags            = LCT_DT_THREAD,
                },

diff --git a/lustre/ost/ost_internal.h b/lustre/ost/ost_internal.h
index 5c18a13..a751b43 100644 (file)
--- a/lustre/ost/ost_internal.h
+++ b/lustre/ost/ost_internal.h
@@ -62,8 +62,9 @@ struct ost_thread_local_cache {
 
 struct ost_thread_local_cache *ost_tls(struct ptlrpc_request *r);
 
-#define OSS_MIN_CREATE_THREADS  2UL
-#define OSS_MAX_CREATE_THREADS 16UL
+/* threads for handling "create" request */
+#define OSS_CR_THREADS_MIN     2UL
+#define OSS_CR_THREADS_MAX     16UL
 
 /* Quota stuff */
 extern quota_interface_t *quota_interface;

diff --git a/lustre/ptlrpc/service.c b/lustre/ptlrpc/service.c
index ffe84a1..d807e49 100644 (file)
--- a/lustre/ptlrpc/service.c
+++ b/lustre/ptlrpc/service.c
@@ -445,6 +445,53 @@ static void ptlrpc_at_timer(unsigned long castmeharder)
         cfs_waitq_signal(&svc->srv_waitq);
 }
 
+static void
+ptlrpc_server_nthreads_check(struct ptlrpc_service_conf *conf,
+                            int *min_p, int *max_p)
+{
+#ifdef __KERNEL__
+       struct ptlrpc_service_thr_conf  *tc = &conf->psc_thr;
+       int                             nthrs_min;
+       int                             nthrs;
+
+       nthrs_min = PTLRPC_NTHRS_MIN + (conf->psc_ops.so_hpreq_handler != NULL);
+       nthrs_min = max_t(int, nthrs_min, tc->tc_nthrs_min);
+
+       nthrs = tc->tc_nthrs_user;
+       if (nthrs != 0) { /* validate it */
+               nthrs = min_t(int, nthrs, tc->tc_nthrs_max);
+               nthrs = max_t(int, nthrs, nthrs_min);
+               *min_p = *max_p = nthrs;
+               return;
+       }
+
+       /*
+        * NB: we will add some common at here for estimating, for example:
+        * add a new member ptlrpc_service_thr_conf::tc_factor, and estimate
+        * threads number based on:
+        *     (online_cpus * conf::tc_factor) + conf::tc_nthrs_base.
+        *
+        * So we can remove code block like estimation in ost_setup, also,
+        * we might estimate MDS threads number as well instead of using
+        * absolute number, and have more threads on fat servers to improve
+        * availability of service.
+        *
+        * Also, we will need to validate threads number at here for
+        * CPT affinity service (CPU ParTiion) in the future.
+        * A service can have percpt thread-pool instead of a global thread
+        * pool for each service, which means user might not always get the
+        * threads number they want even they set it in conf::tc_nthrs_user,
+        * because we need to adjust threads number for each CPT, instead of
+        * just use (conf::tc_nthrs_user / NCPTS), to make sure each pool
+        * will be healthy.
+        */
+       *max_p = tc->tc_nthrs_max;
+       *min_p = nthrs_min;
+#else /* __KERNEL__ */
+       *max_p = *min_p = 1; /* whatever */
+#endif
+}
+
 /**
  * Initialize service on a given portal.
  * This includes starting serving threads , allocating and posting rqbds and
@@ -489,8 +536,10 @@ ptlrpc_register_service(struct ptlrpc_service_conf *conf,
        service->srv_req_portal         = conf->psc_buf.bc_req_portal;
        service->srv_request_seq        = 1; /* valid seq #s start at 1 */
        service->srv_request_max_cull_seq = 0;
-       service->srv_threads_min        = conf->psc_thr.tc_nthrs_min;
-       service->srv_threads_max        = conf->psc_thr.tc_nthrs_max;
+
+       ptlrpc_server_nthreads_check(conf, &service->srv_threads_min,
+                                    &service->srv_threads_max);
+
        service->srv_thread_name        = conf->psc_thr.tc_thr_name;
        service->srv_ctx_tags           = conf->psc_thr.tc_ctx_tags;
        service->srv_cpu_affinity       = !!conf->psc_thr.tc_cpu_affinity;