X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Fosp%2Flwp_dev.c;h=b94ec537d736d0efea44eea8d7766610df234c8d;hb=ae22814bccc6dfa3b72b15cef45ed7c2731d4e3e;hp=cb22b2f844ff222d0729afe0202ffc77a9d40086;hpb=bafa12c06d87c05bb263eed37ad8af1a2df99894;p=fs%2Flustre-release.git diff --git a/lustre/osp/lwp_dev.c b/lustre/osp/lwp_dev.c index cb22b2f..b94ec53 100644 --- a/lustre/osp/lwp_dev.c +++ b/lustre/osp/lwp_dev.c @@ -21,7 +21,7 @@ * GPL HEADER END */ /* - * Copyright (c) 2013, Intel Corporation. + * Copyright (c) 2013, 2017, Intel Corporation. * Use is subject to license terms. * * lustre/osp/lwp_dev.c @@ -40,15 +40,17 @@ #define DEBUG_SUBSYSTEM S_OST #include -#include +#include #include -#include +#include + +#include "osp_internal.h" struct lwp_device { struct lu_device lpd_dev; struct obd_device *lpd_obd; /* corresponding OBD device */ - struct obd_uuid lpd_cluuid;/* UUID of LWP */ struct obd_export *lpd_exp; /* export of LWP */ + struct ptlrpc_thread lpd_notify_thread; /* notify thread */ int lpd_connects; /* use count, 0 or 1 */ }; @@ -80,12 +82,14 @@ static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp, char *lwp_name = lwp->lpd_obd->obd_name; char *server_uuid = NULL; char *ptr; - class_uuid_t uuid; struct obd_import *imp; int len = strlen(lwp_name) + 1; int rc; ENTRY; + thread_set_flags(&lwp->lpd_notify_thread, SVC_STOPPED); + init_waitqueue_head(&lwp->lpd_notify_thread.t_ctl_waitq); + OBD_ALLOC_PTR(bufs); if (bufs == NULL) RETURN(-ENOMEM); @@ -108,9 +112,10 @@ static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp, lustre_cfg_bufs_reset(bufs, lwp_name); lustre_cfg_bufs_set_string(bufs, 1, server_uuid); lustre_cfg_bufs_set_string(bufs, 2, nidstring); - lcfg = lustre_cfg_new(LCFG_SETUP, bufs); - if (lcfg == NULL) + OBD_ALLOC(lcfg, lustre_cfg_len(bufs->lcfg_bufcount, bufs->lcfg_buflen)); + if (!lcfg) GOTO(out, rc = -ENOMEM); + lustre_cfg_init(lcfg, LCFG_SETUP, bufs); rc = client_obd_setup(lwp->lpd_obd, lcfg); if (rc != 0) { @@ -121,18 +126,14 @@ static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp, imp = lwp->lpd_obd->u.cli.cl_import; rc = ptlrpc_init_import(imp); - if (rc) - GOTO(out, rc); - - ll_generate_random_uuid(uuid); - class_uuid_unparse(uuid, &lwp->lpd_cluuid); out: if (bufs != NULL) OBD_FREE_PTR(bufs); if (server_uuid != NULL) OBD_FREE(server_uuid, len); - if (lcfg != NULL) - lustre_cfg_free(lcfg); + if (lcfg) + OBD_FREE(lcfg, lustre_cfg_len(lcfg->lcfg_bufcount, + lcfg->lcfg_buflens)); if (rc) client_obd_cleanup(lwp->lpd_obd); @@ -220,7 +221,7 @@ static int lwp_process_config(const struct lu_env *env, RETURN(rc); } -const struct lu_device_operations lwp_lu_ops = { +static const struct lu_device_operations lwp_lu_ops = { .ldo_process_config = lwp_process_config, }; @@ -236,8 +237,8 @@ const struct lu_device_operations lwp_lu_ops = { * \retval -ENODEV if the device name cannot be found * \retval negative numbers on other errors */ -int lwp_init0(const struct lu_env *env, struct lwp_device *lwp, - struct lu_device_type *ldt, struct lustre_cfg *cfg) +static int lwp_init0(const struct lu_env *env, struct lwp_device *lwp, + struct lu_device_type *ldt, struct lustre_cfg *cfg) { int rc; ENTRY; @@ -259,16 +260,32 @@ int lwp_init0(const struct lu_env *env, struct lwp_device *lwp, RETURN(rc); } + rc = lprocfs_obd_setup(lwp->lpd_obd, true); + if (rc) { + CERROR("%s: lprocfs_obd_setup failed. %d\n", + lwp->lpd_obd->obd_name, rc); + ptlrpcd_decref(); + RETURN(rc); + } + rc = lwp_setup(env, lwp, lustre_cfg_string(cfg, 1)); if (rc) { CERROR("%s: setup lwp failed. %d\n", lwp->lpd_obd->obd_name, rc); + lprocfs_obd_cleanup(lwp->lpd_obd); ptlrpcd_decref(); RETURN(rc); } - if (lprocfs_obd_setup(lwp->lpd_obd) == 0) - ptlrpc_lprocfs_register_obd(lwp->lpd_obd); + rc = sptlrpc_lprocfs_cliobd_attach(lwp->lpd_obd); + if (rc) { + CERROR("%s: sptlrpc_lprocfs_cliobd_attached failed. %d\n", + lwp->lpd_obd->obd_name, rc); + ptlrpcd_decref(); + RETURN(rc); + } + + ptlrpc_lprocfs_register_obd(lwp->lpd_obd); RETURN(0); } @@ -350,28 +367,22 @@ static struct lu_device *lwp_device_alloc(const struct lu_env *env, static struct lu_device *lwp_device_fini(const struct lu_env *env, struct lu_device *ludev) { - struct lwp_device *m = lu2lwp_dev(ludev); - struct obd_import *imp; - int rc; + struct lwp_device *m = lu2lwp_dev(ludev); + struct ptlrpc_thread *thread = &m->lpd_notify_thread; + int rc; ENTRY; + wait_event_idle(thread->t_ctl_waitq, thread_is_stopped(thread)); + if (m->lpd_exp != NULL) class_disconnect(m->lpd_exp); - imp = m->lpd_obd->u.cli.cl_import; - - if (imp->imp_rq_pool) { - ptlrpc_free_rq_pool(imp->imp_rq_pool); - imp->imp_rq_pool = NULL; - } - LASSERT(m->lpd_obd); - ptlrpc_lprocfs_unregister_obd(m->lpd_obd); - lprocfs_obd_cleanup(m->lpd_obd); - rc = client_obd_cleanup(m->lpd_obd); LASSERTF(rc == 0, "error %d\n", rc); + ptlrpc_lprocfs_unregister_obd(m->lpd_obd); + ptlrpcd_decref(); RETURN(NULL); @@ -390,6 +401,66 @@ struct lu_device_type lwp_device_type = { .ldt_ctx_tags = LCT_MD_THREAD }; +static int lwp_notify_main(void *args) +{ + struct obd_export *exp = (struct obd_export *)args; + struct lwp_device *lwp; + struct ptlrpc_thread *thread; + + LASSERT(exp != NULL); + class_export_get(exp); + + lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev); + thread = &lwp->lpd_notify_thread; + + thread_set_flags(thread, SVC_RUNNING); + wake_up(&thread->t_ctl_waitq); + + lustre_notify_lwp_list(exp); + + class_export_put(exp); + thread_set_flags(thread, SVC_STOPPED); + wake_up(&thread->t_ctl_waitq); + return 0; +} + +/* + * Some notify callbacks may cause deadlock in failover + * scenario, so we have to start thread to run callbacks + * asynchronously. See LU-6273. + */ +static void lwp_notify_users(struct obd_export *exp) +{ + struct lwp_device *lwp; + struct ptlrpc_thread *thread; + struct task_struct *task; + char name[MTI_NAME_MAXLEN]; + + LASSERT(exp != NULL); + lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev); + thread = &lwp->lpd_notify_thread; + + snprintf(name, MTI_NAME_MAXLEN, "lwp_notify_%s", + exp->exp_obd->obd_name); + + /* Notify happens only on LWP setup, so there shouldn't + * be notify thread running */ + if (!thread_is_stopped(thread)) { + CERROR("LWP notify thread: %s wasn't stopped\n", name); + return; + } + + task = kthread_run(lwp_notify_main, exp, name); + if (IS_ERR(task)) { + thread_set_flags(thread, SVC_STOPPED); + CERROR("Failed to start LWP notify thread:%s. %lu\n", + name, PTR_ERR(task)); + } + + wait_event_idle(thread->t_ctl_waitq, + thread_is_running(thread) || thread_is_stopped(thread)); +} + /** * Implementation of OBD device operations obd_ops::o_connect. * @@ -445,6 +516,7 @@ static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp, ocd->ocd_version = LUSTRE_VERSION_CODE; imp->imp_connect_flags_orig = ocd->ocd_connect_flags; + imp->imp_connect_flags2_orig = ocd->ocd_connect_flags2; rc = ptlrpc_connect_import(imp); if (rc != 0) { @@ -466,6 +538,9 @@ out_dis: out_sem: up_write(&cli->cl_sem); + if (rc == 0) + lwp_notify_users(*exp); + return rc; } @@ -530,6 +605,23 @@ static int lwp_import_event(struct obd_device *obd, struct obd_import *imp, return 0; } +static int lwp_set_info_async(const struct lu_env *env, + struct obd_export *exp, + u32 keylen, void *key, + u32 vallen, void *val, + struct ptlrpc_request_set *set) +{ + ENTRY; + + if (KEY_IS(KEY_SPTLRPC_CONF)) { + sptlrpc_conf_client_adapt(exp->exp_obd); + RETURN(0); + } + + CERROR("Unknown key %s\n", (char *)key); + RETURN(-EINVAL); +} + struct obd_ops lwp_obd_device_ops = { .o_owner = THIS_MODULE, .o_add_conn = client_import_add_conn, @@ -537,4 +629,5 @@ struct obd_ops lwp_obd_device_ops = { .o_connect = lwp_obd_connect, .o_disconnect = lwp_obd_disconnect, .o_import_event = lwp_import_event, + .o_set_info_async = lwp_set_info_async, };