X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Fmdt%2Fmdt_mds.c;h=86a6e4159cd9986d709a7e925ff9faee455939a4;hb=38e6ee6fb779c34c7ef3eb4a010332385b3b84c1;hp=b9c806f73bb3903e2eb3c7af3df64195468fa311;hpb=3cbe63e72c83ba50c3553d28e2fe6bc9d18c347a;p=fs%2Flustre-release.git diff --git a/lustre/mdt/mdt_mds.c b/lustre/mdt/mdt_mds.c index b9c806f..86a6e41 100644 --- a/lustre/mdt/mdt_mds.c +++ b/lustre/mdt/mdt_mds.c @@ -23,7 +23,7 @@ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2013, 2015, Intel Corporation. + * Copyright (c) 2013, 2017, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -75,9 +75,27 @@ static unsigned long mds_num_threads; module_param(mds_num_threads, ulong, 0444); MODULE_PARM_DESC(mds_num_threads, "number of MDS service threads to start"); +static unsigned int mds_cpu_bind = 1; +module_param(mds_cpu_bind, uint, 0444); +MODULE_PARM_DESC(mds_cpu_bind, + "bind MDS threads to particular CPU partitions"); + int mds_max_io_threads = 512; module_param(mds_max_io_threads, int, 0444); -MODULE_PARM_DESC(mds_max_io_threads, "maximum number of MDS IO service threads"); +MODULE_PARM_DESC(mds_max_io_threads, + "maximum number of MDS IO service threads"); + +static unsigned int mds_io_cpu_bind = 1; +module_param(mds_io_cpu_bind, uint, 0444); +MODULE_PARM_DESC(mds_io_cpu_bind, + "bind MDS IO threads to particular CPU partitions"); + +static char *mds_io_num_cpts; +module_param(mds_io_num_cpts, charp, 0444); +MODULE_PARM_DESC(mds_io_num_cpts, + "CPU partitions MDS IO threads should run on"); + +static struct cfs_cpt_table *mdt_io_cptable; static char *mds_num_cpts; module_param(mds_num_cpts, charp, 0444); @@ -88,6 +106,11 @@ module_param(mds_rdpg_num_threads, ulong, 0444); MODULE_PARM_DESC(mds_rdpg_num_threads, "number of MDS readpage service threads to start"); +static unsigned int mds_rdpg_cpu_bind = 1; +module_param(mds_rdpg_cpu_bind, uint, 0444); +MODULE_PARM_DESC(mds_rdpg_cpu_bind, + "bind MDS readpage threads to particular CPU partitions"); + static char *mds_rdpg_num_cpts; module_param(mds_rdpg_num_cpts, charp, 0444); MODULE_PARM_DESC(mds_rdpg_num_cpts, @@ -99,6 +122,11 @@ module_param(mds_attr_num_threads, ulong, 0444); MODULE_PARM_DESC(mds_attr_num_threads, "number of MDS setattr service threads to start"); +static unsigned int mds_attr_cpu_bind = 1; +module_param(mds_attr_cpu_bind, uint, 0444); +MODULE_PARM_DESC(mds_attr_cpu_bind, + "bind MDS setattr threads to particular CPU partitions"); + static char *mds_attr_num_cpts; module_param(mds_attr_num_cpts, charp, 0444); MODULE_PARM_DESC(mds_attr_num_cpts, @@ -144,6 +172,11 @@ static void mds_stop_ptlrpc_service(struct mds_device *m) } mutex_unlock(&m->mds_health_mutex); + if (mdt_io_cptable != NULL) { + cfs_cpt_table_free(mdt_io_cptable); + mdt_io_cptable = NULL; + } + EXIT; } @@ -151,12 +184,10 @@ static int mds_start_ptlrpc_service(struct mds_device *m) { static struct ptlrpc_service_conf conf; struct obd_device *obd = m->mds_md_dev.md_lu_dev.ld_obd; - struct proc_dir_entry *procfs_entry; + nodemask_t *mask; int rc = 0; - ENTRY; - procfs_entry = obd->obd_proc_entry; - LASSERT(procfs_entry != NULL); + ENTRY; conf = (typeof(conf)) { .psc_name = LUSTRE_MDT_NAME, @@ -180,11 +211,15 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .tc_nthrs_base = MDS_NTHRS_BASE, .tc_nthrs_max = MDS_NTHRS_MAX, .tc_nthrs_user = mds_num_threads, - .tc_cpu_affinity = 1, - .tc_ctx_tags = LCT_MD_THREAD, + .tc_cpu_bind = mds_cpu_bind, + /* LCT_DT_THREAD is required as MDT threads may scan + * all LDLM namespaces (including OFD-originated) to + * cancel LDLM locks */ + .tc_ctx_tags = LCT_MD_THREAD | LCT_DT_THREAD, }, .psc_cpt = { .cc_pattern = mds_num_cpts, + .cc_affinity = true, }, .psc_ops = { .so_req_handler = tgt_request_handle, @@ -193,7 +228,7 @@ static int mds_start_ptlrpc_service(struct mds_device *m) }, }; m->mds_regular_service = ptlrpc_register_service(&conf, &obd->obd_kset, - procfs_entry); + obd->obd_debugfs_entry); if (IS_ERR(m->mds_regular_service)) { rc = PTR_ERR(m->mds_regular_service); CERROR("failed to start regular mdt service: %d\n", rc); @@ -225,11 +260,12 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .tc_nthrs_base = MDS_RDPG_NTHRS_BASE, .tc_nthrs_max = MDS_RDPG_NTHRS_MAX, .tc_nthrs_user = mds_rdpg_num_threads, - .tc_cpu_affinity = 1, + .tc_cpu_bind = mds_rdpg_cpu_bind, .tc_ctx_tags = LCT_MD_THREAD, }, .psc_cpt = { .cc_pattern = mds_rdpg_num_cpts, + .cc_affinity = true, }, .psc_ops = { .so_req_handler = tgt_request_handle, @@ -237,7 +273,7 @@ static int mds_start_ptlrpc_service(struct mds_device *m) }, }; m->mds_readpage_service = ptlrpc_register_service(&conf, &obd->obd_kset, - procfs_entry); + obd->obd_debugfs_entry); if (IS_ERR(m->mds_readpage_service)) { rc = PTR_ERR(m->mds_readpage_service); CERROR("failed to start readpage service: %d\n", rc); @@ -272,11 +308,12 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .tc_nthrs_base = MDS_SETA_NTHRS_BASE, .tc_nthrs_max = MDS_SETA_NTHRS_MAX, .tc_nthrs_user = mds_attr_num_threads, - .tc_cpu_affinity = 1, + .tc_cpu_bind = mds_attr_cpu_bind, .tc_ctx_tags = LCT_MD_THREAD, }, .psc_cpt = { .cc_pattern = mds_attr_num_cpts, + .cc_affinity = true, }, .psc_ops = { .so_req_handler = tgt_request_handle, @@ -285,7 +322,7 @@ static int mds_start_ptlrpc_service(struct mds_device *m) }, }; m->mds_setattr_service = ptlrpc_register_service(&conf, &obd->obd_kset, - procfs_entry); + obd->obd_debugfs_entry); if (IS_ERR(m->mds_setattr_service)) { rc = PTR_ERR(m->mds_setattr_service); CERROR("failed to start setattr service: %d\n", rc); @@ -317,12 +354,13 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .tc_nthrs_base = MDS_NTHRS_BASE, .tc_nthrs_max = MDS_NTHRS_MAX, .tc_nthrs_user = mds_num_threads, - .tc_cpu_affinity = 1, + .tc_cpu_bind = mds_cpu_bind, .tc_ctx_tags = LCT_MD_THREAD | LCT_DT_THREAD, }, .psc_cpt = { .cc_pattern = mds_num_cpts, + .cc_affinity = true, }, .psc_ops = { .so_req_handler = tgt_request_handle, @@ -331,7 +369,7 @@ static int mds_start_ptlrpc_service(struct mds_device *m) }, }; m->mds_out_service = ptlrpc_register_service(&conf, &obd->obd_kset, - procfs_entry); + obd->obd_debugfs_entry); if (IS_ERR(m->mds_out_service)) { rc = PTR_ERR(m->mds_out_service); CERROR("failed to start out service: %d\n", rc); @@ -367,7 +405,7 @@ static int mds_start_ptlrpc_service(struct mds_device *m) }, }; m->mds_mdsc_service = ptlrpc_register_service(&conf, &obd->obd_kset, - procfs_entry); + obd->obd_debugfs_entry); if (IS_ERR(m->mds_mdsc_service)) { rc = PTR_ERR(m->mds_mdsc_service); CERROR("failed to start seq controller service: %d\n", rc); @@ -404,7 +442,7 @@ static int mds_start_ptlrpc_service(struct mds_device *m) }, }; m->mds_mdss_service = ptlrpc_register_service(&conf, &obd->obd_kset, - procfs_entry); + obd->obd_debugfs_entry); if (IS_ERR(m->mds_mdss_service)) { rc = PTR_ERR(m->mds_mdss_service); CERROR("failed to start metadata seq server service: %d\n", rc); @@ -439,7 +477,7 @@ static int mds_start_ptlrpc_service(struct mds_device *m) }, }; m->mds_fld_service = ptlrpc_register_service(&conf, &obd->obd_kset, - procfs_entry); + obd->obd_debugfs_entry); if (IS_ERR(m->mds_fld_service)) { rc = PTR_ERR(m->mds_fld_service); CERROR("failed to start fld service: %d\n", rc); @@ -448,6 +486,32 @@ static int mds_start_ptlrpc_service(struct mds_device *m) GOTO(err_mds_svc, rc); } + + mask = cfs_cpt_nodemask(cfs_cpt_table, CFS_CPT_ANY); + /* event CPT feature is disabled in libcfs level by set partition + * number to 1, we still want to set node affinity for io service */ + if (cfs_cpt_number(cfs_cpt_table) == 1 && nodes_weight(*mask) > 1) { + int cpt = 0; + int i; + + mdt_io_cptable = cfs_cpt_table_alloc(nodes_weight(*mask)); + for_each_node_mask(i, *mask) { + if (mdt_io_cptable == NULL) { + CWARN("MDS failed to create CPT table\n"); + break; + } + + rc = cfs_cpt_set_node(mdt_io_cptable, cpt++, i); + if (!rc) { + CWARN("MDS Failed to set node %d for IO CPT table\n", + i); + cfs_cpt_table_free(mdt_io_cptable); + mdt_io_cptable = NULL; + break; + } + } + } + memset(&conf, 0, sizeof(conf)); conf = (typeof(conf)) { .psc_name = LUSTRE_MDT_NAME "_io", @@ -461,23 +525,31 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .bc_rep_portal = MDC_REPLY_PORTAL, }, .psc_thr = { - .tc_thr_name = "ll_mdt_io", + .tc_thr_name = LUSTRE_MDT_NAME "_io", .tc_thr_factor = OSS_THR_FACTOR, .tc_nthrs_init = OSS_NTHRS_INIT, .tc_nthrs_base = OSS_NTHRS_BASE, .tc_nthrs_max = mds_max_io_threads, - .tc_cpu_affinity = 1, + .tc_nthrs_user = mds_num_threads, + .tc_cpu_bind = mds_io_cpu_bind, .tc_ctx_tags = LCT_DT_THREAD | LCT_MD_THREAD, }, + .psc_cpt = { + .cc_cptable = mdt_io_cptable, + .cc_pattern = mdt_io_cptable == NULL ? + mds_io_num_cpts : NULL, + .cc_affinity = true, + }, .psc_ops = { .so_thr_init = tgt_io_thread_init, .so_thr_done = tgt_io_thread_done, .so_req_handler = tgt_request_handle, .so_req_printer = target_print_req, + .so_hpreq_handler = tgt_hpreq_handler, }, }; m->mds_io_service = ptlrpc_register_service(&conf, &obd->obd_kset, - procfs_entry); + obd->obd_debugfs_entry); if (IS_ERR(m->mds_io_service)) { rc = PTR_ERR(m->mds_io_service); CERROR("failed to start MDT I/O service: %d\n", rc); @@ -605,14 +677,14 @@ static int mds_health_check(const struct lu_env *env, struct obd_device *obd) return rc != 0 ? 1 : 0; } -static struct obd_ops mds_obd_device_ops = { +static const struct obd_ops mds_obd_device_ops = { .o_owner = THIS_MODULE, .o_health_check = mds_health_check, }; int mds_mod_init(void) { - return class_register_type(&mds_obd_device_ops, NULL, true, NULL, + return class_register_type(&mds_obd_device_ops, NULL, false, NULL, LUSTRE_MDS_NAME, &mds_device_type); }