X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fmdt%2Fmdt_mds.c;h=79afe4f940c000ba2c00b098de7429d19a6c06bb;hp=d91d1d40a152d22a18bc02720968dcea5d29c8b4;hb=ccabce23bd9e366c345c852f565766a799f61238;hpb=f0c1b06f2418d2016e23eb6e8277be85ca13fc53 diff --git a/lustre/mdt/mdt_mds.c b/lustre/mdt/mdt_mds.c index d91d1d4..79afe4f 100644 --- a/lustre/mdt/mdt_mds.c +++ b/lustre/mdt/mdt_mds.c @@ -23,7 +23,7 @@ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2013, Intel Corporation. + * Copyright (c) 2013, 2017, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -52,57 +52,64 @@ #include "mdt_internal.h" #include #include -#include +#include struct mds_device { /* super-class */ - struct md_device mds_md_dev; - struct ptlrpc_service *mds_regular_service; - struct ptlrpc_service *mds_readpage_service; - struct ptlrpc_service *mds_out_service; - struct ptlrpc_service *mds_setattr_service; - struct ptlrpc_service *mds_mdsc_service; - struct ptlrpc_service *mds_mdss_service; - struct ptlrpc_service *mds_fld_service; + struct md_device mds_md_dev; + struct ptlrpc_service *mds_regular_service; + struct ptlrpc_service *mds_readpage_service; + struct ptlrpc_service *mds_out_service; + struct ptlrpc_service *mds_setattr_service; + struct ptlrpc_service *mds_mdsc_service; + struct ptlrpc_service *mds_mdss_service; + struct ptlrpc_service *mds_fld_service; + struct ptlrpc_service *mds_io_service; + struct mutex mds_health_mutex; }; /* - * * Initialized in mdt_mod_init(). + * * Initialized in mds_mod_init(). * */ -static unsigned long mdt_num_threads; -CFS_MODULE_PARM(mdt_num_threads, "ul", ulong, 0444, - "number of MDS service threads to start " - "(deprecated in favor of mds_num_threads)"); - static unsigned long mds_num_threads; -CFS_MODULE_PARM(mds_num_threads, "ul", ulong, 0444, - "number of MDS service threads to start"); +module_param(mds_num_threads, ulong, 0444); +MODULE_PARM_DESC(mds_num_threads, "number of MDS service threads to start"); + +int mds_max_io_threads = 512; +module_param(mds_max_io_threads, int, 0444); +MODULE_PARM_DESC(mds_max_io_threads, "maximum number of MDS IO service threads"); static char *mds_num_cpts; -CFS_MODULE_PARM(mds_num_cpts, "c", charp, 0444, - "CPU partitions MDS threads should run on"); +module_param(mds_num_cpts, charp, 0444); +MODULE_PARM_DESC(mds_num_cpts, "CPU partitions MDS threads should run on"); static unsigned long mds_rdpg_num_threads; -CFS_MODULE_PARM(mds_rdpg_num_threads, "ul", ulong, 0444, - "number of MDS readpage service threads to start"); +module_param(mds_rdpg_num_threads, ulong, 0444); +MODULE_PARM_DESC(mds_rdpg_num_threads, + "number of MDS readpage service threads to start"); static char *mds_rdpg_num_cpts; -CFS_MODULE_PARM(mds_rdpg_num_cpts, "c", charp, 0444, - "CPU partitions MDS readpage threads should run on"); +module_param(mds_rdpg_num_cpts, charp, 0444); +MODULE_PARM_DESC(mds_rdpg_num_cpts, + "CPU partitions MDS readpage threads should run on"); /* NB: these two should be removed along with setattr service in the future */ static unsigned long mds_attr_num_threads; -CFS_MODULE_PARM(mds_attr_num_threads, "ul", ulong, 0444, - "number of MDS setattr service threads to start"); +module_param(mds_attr_num_threads, ulong, 0444); +MODULE_PARM_DESC(mds_attr_num_threads, + "number of MDS setattr service threads to start"); static char *mds_attr_num_cpts; -CFS_MODULE_PARM(mds_attr_num_cpts, "c", charp, 0444, - "CPU partitions MDS setattr threads should run on"); +module_param(mds_attr_num_cpts, charp, 0444); +MODULE_PARM_DESC(mds_attr_num_cpts, + "CPU partitions MDS setattr threads should run on"); /* device init/fini methods */ static void mds_stop_ptlrpc_service(struct mds_device *m) { ENTRY; + + mutex_lock(&m->mds_health_mutex); if (m->mds_regular_service != NULL) { ptlrpc_unregister_service(m->mds_regular_service); m->mds_regular_service = NULL; @@ -131,6 +138,12 @@ static void mds_stop_ptlrpc_service(struct mds_device *m) ptlrpc_unregister_service(m->mds_fld_service); m->mds_fld_service = NULL; } + if (m->mds_io_service != NULL) { + ptlrpc_unregister_service(m->mds_io_service); + m->mds_io_service = NULL; + } + mutex_unlock(&m->mds_health_mutex); + EXIT; } @@ -138,7 +151,7 @@ static int mds_start_ptlrpc_service(struct mds_device *m) { static struct ptlrpc_service_conf conf; struct obd_device *obd = m->mds_md_dev.md_lu_dev.ld_obd; - cfs_proc_dir_entry_t *procfs_entry; + struct proc_dir_entry *procfs_entry; int rc = 0; ENTRY; @@ -179,7 +192,8 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .so_hpreq_handler = ptlrpc_hpreq_handler, }, }; - m->mds_regular_service = ptlrpc_register_service(&conf, procfs_entry); + m->mds_regular_service = ptlrpc_register_service(&conf, &obd->obd_kset, + procfs_entry); if (IS_ERR(m->mds_regular_service)) { rc = PTR_ERR(m->mds_regular_service); CERROR("failed to start regular mdt service: %d\n", rc); @@ -222,7 +236,8 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .so_req_printer = target_print_req, }, }; - m->mds_readpage_service = ptlrpc_register_service(&conf, procfs_entry); + m->mds_readpage_service = ptlrpc_register_service(&conf, &obd->obd_kset, + procfs_entry); if (IS_ERR(m->mds_readpage_service)) { rc = PTR_ERR(m->mds_readpage_service); CERROR("failed to start readpage service: %d\n", rc); @@ -269,7 +284,8 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .so_hpreq_handler = NULL, }, }; - m->mds_setattr_service = ptlrpc_register_service(&conf, procfs_entry); + m->mds_setattr_service = ptlrpc_register_service(&conf, &obd->obd_kset, + procfs_entry); if (IS_ERR(m->mds_setattr_service)) { rc = PTR_ERR(m->mds_setattr_service); CERROR("failed to start setattr service: %d\n", rc); @@ -288,7 +304,7 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .bc_req_max_size = OUT_MAXREQSIZE, .bc_rep_max_size = OUT_MAXREPSIZE, .bc_req_portal = OUT_PORTAL, - .bc_rep_portal = MDC_REPLY_PORTAL, + .bc_rep_portal = OSC_REPLY_PORTAL, }, /* * We'd like to have a mechanism to set this on a per-device @@ -302,7 +318,8 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .tc_nthrs_max = MDS_NTHRS_MAX, .tc_nthrs_user = mds_num_threads, .tc_cpu_affinity = 1, - .tc_ctx_tags = LCT_MD_THREAD, + .tc_ctx_tags = LCT_MD_THREAD | + LCT_DT_THREAD, }, .psc_cpt = { .cc_pattern = mds_num_cpts, @@ -313,7 +330,8 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .so_hpreq_handler = NULL, }, }; - m->mds_out_service = ptlrpc_register_service(&conf, procfs_entry); + m->mds_out_service = ptlrpc_register_service(&conf, &obd->obd_kset, + procfs_entry); if (IS_ERR(m->mds_out_service)) { rc = PTR_ERR(m->mds_out_service); CERROR("failed to start out service: %d\n", rc); @@ -348,7 +366,8 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .so_hpreq_handler = NULL, }, }; - m->mds_mdsc_service = ptlrpc_register_service(&conf, procfs_entry); + m->mds_mdsc_service = ptlrpc_register_service(&conf, &obd->obd_kset, + procfs_entry); if (IS_ERR(m->mds_mdsc_service)) { rc = PTR_ERR(m->mds_mdsc_service); CERROR("failed to start seq controller service: %d\n", rc); @@ -384,7 +403,8 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .so_hpreq_handler = NULL, }, }; - m->mds_mdss_service = ptlrpc_register_service(&conf, procfs_entry); + m->mds_mdss_service = ptlrpc_register_service(&conf, &obd->obd_kset, + procfs_entry); if (IS_ERR(m->mds_mdss_service)) { rc = PTR_ERR(m->mds_mdss_service); CERROR("failed to start metadata seq server service: %d\n", rc); @@ -418,7 +438,8 @@ static int mds_start_ptlrpc_service(struct mds_device *m) .so_hpreq_handler = NULL, }, }; - m->mds_fld_service = ptlrpc_register_service(&conf, procfs_entry); + m->mds_fld_service = ptlrpc_register_service(&conf, &obd->obd_kset, + procfs_entry); if (IS_ERR(m->mds_fld_service)) { rc = PTR_ERR(m->mds_fld_service); CERROR("failed to start fld service: %d\n", rc); @@ -427,6 +448,43 @@ static int mds_start_ptlrpc_service(struct mds_device *m) GOTO(err_mds_svc, rc); } + memset(&conf, 0, sizeof(conf)); + conf = (typeof(conf)) { + .psc_name = LUSTRE_MDT_NAME "_io", + .psc_watchdog_factor = MDT_SERVICE_WATCHDOG_FACTOR, + .psc_buf = { + .bc_nbufs = OST_NBUFS, + .bc_buf_size = OST_IO_BUFSIZE, + .bc_req_max_size = OST_IO_MAXREQSIZE, + .bc_rep_max_size = OST_IO_MAXREPSIZE, + .bc_req_portal = MDS_IO_PORTAL, + .bc_rep_portal = MDC_REPLY_PORTAL, + }, + .psc_thr = { + .tc_thr_name = "ll_mdt_io", + .tc_thr_factor = OSS_THR_FACTOR, + .tc_nthrs_init = OSS_NTHRS_INIT, + .tc_nthrs_base = OSS_NTHRS_BASE, + .tc_nthrs_max = mds_max_io_threads, + .tc_cpu_affinity = 1, + .tc_ctx_tags = LCT_DT_THREAD | LCT_MD_THREAD, + }, + .psc_ops = { + .so_thr_init = tgt_io_thread_init, + .so_thr_done = tgt_io_thread_done, + .so_req_handler = tgt_request_handle, + .so_req_printer = target_print_req, + }, + }; + m->mds_io_service = ptlrpc_register_service(&conf, &obd->obd_kset, + procfs_entry); + if (IS_ERR(m->mds_io_service)) { + rc = PTR_ERR(m->mds_io_service); + CERROR("failed to start MDT I/O service: %d\n", rc); + m->mds_io_service = NULL; + GOTO(err_mds_svc, rc); + } + EXIT; err_mds_svc: if (rc) @@ -486,16 +544,18 @@ static struct lu_device *mds_device_alloc(const struct lu_env *env, /* set this lu_device to obd, because error handling need it */ obd->obd_lu_dev = l; - rc = lprocfs_obd_setup(obd, lprocfs_mds_obd_vars); + rc = lprocfs_obd_setup(obd, true); if (rc != 0) { mds_device_free(env, l); l = ERR_PTR(rc); return l; } - rc = mds_start_ptlrpc_service(m); + mutex_init(&m->mds_health_mutex); + rc = mds_start_ptlrpc_service(m); if (rc != 0) { + lprocfs_obd_cleanup(obd); mds_device_free(env, l); l = ERR_PTR(rc); return l; @@ -525,25 +585,35 @@ static struct lu_device_type mds_device_type = { .ldt_ctx_tags = LCT_MD_THREAD }; +static int mds_health_check(const struct lu_env *env, struct obd_device *obd) +{ + struct mds_device *mds = mds_dev(obd->obd_lu_dev); + int rc = 0; + + + mutex_lock(&mds->mds_health_mutex); + rc |= ptlrpc_service_health_check(mds->mds_regular_service); + rc |= ptlrpc_service_health_check(mds->mds_readpage_service); + rc |= ptlrpc_service_health_check(mds->mds_out_service); + rc |= ptlrpc_service_health_check(mds->mds_setattr_service); + rc |= ptlrpc_service_health_check(mds->mds_mdsc_service); + rc |= ptlrpc_service_health_check(mds->mds_mdss_service); + rc |= ptlrpc_service_health_check(mds->mds_fld_service); + rc |= ptlrpc_service_health_check(mds->mds_io_service); + mutex_unlock(&mds->mds_health_mutex); + + return rc != 0 ? 1 : 0; +} + static struct obd_ops mds_obd_device_ops = { .o_owner = THIS_MODULE, + .o_health_check = mds_health_check, }; int mds_mod_init(void) { - int rc; - - if (mdt_num_threads != 0 && mds_num_threads == 0) { - LCONSOLE_INFO("mdt_num_threads module parameter is deprecated, " - "use mds_num_threads instead or unset both for " - "dynamic thread startup\n"); - mds_num_threads = mdt_num_threads; - } - - rc = class_register_type(&mds_obd_device_ops, NULL, - lprocfs_mds_module_vars, LUSTRE_MDS_NAME, - &mds_device_type); - return rc; + return class_register_type(&mds_obd_device_ops, NULL, true, NULL, + LUSTRE_MDS_NAME, &mds_device_type); } void mds_mod_exit(void)