From b1848aa5b23fd332362e9ae3d5aab31d8dd9d920 Mon Sep 17 00:00:00 2001 From: Niu Yawei Date: Tue, 8 Sep 2015 09:41:58 -0400 Subject: [PATCH] LU-6273 lwp: notify LWP users in dedicated thread On OST/MDT mount, the client config log will be processed to setup the LWP as following: 1> Process the LCFG_ADD_UUID record to setup LWP device, then connect to the server target. (see lustre_lwp_setup()); 2> Process the LCFG_ADD_CONN record to add failover connection; We can see that if the mount process is blocked in step 1 it will never have a chance to add failover connection, and LWP won't be able to switch to failover node forever. Unfortunately, the callbacks for FLD user could block the step 1. See ofd/mdt_register_lwp_callback() calls fld_client_rpc() which will send FLD RPC in a deadloop if the connection isn't available. This patch solve the problem by using a per LWP dedicated thread to run the notify callbacks. Signed-off-by: Niu Yawei Signed-off-by: wang di Change-Id: Ic0d89f1524ea0c1a3e7fc3833e16ecbad2123454 Reviewed-on: http://review.whamcloud.com/16303 Reviewed-by: James Simmons Tested-by: Jenkins Reviewed-by: Mike Pershin Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin --- lustre/include/obd_class.h | 1 + lustre/obdclass/obd_mount_server.c | 4 +- lustre/osp/lwp_dev.c | 79 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index f408800..e0c0ca3 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -1699,6 +1699,7 @@ int lustre_register_lwp_item(const char *lwpname, struct obd_export **exp, register_lwp_cb cb_func, void *cb_data); void lustre_deregister_lwp_item(struct obd_export **exp); struct obd_export *lustre_find_lwp_by_index(const char *dev, __u32 idx); +void lustre_notify_lwp_list(struct obd_export *exp); int tgt_name2lwp_name(const char *tgt_name, char *lwp_name, int len, __u32 idx); #endif /* HAVE_SERVER_SUPPORT */ int lustre_register_fs(void); diff --git a/lustre/obdclass/obd_mount_server.c b/lustre/obdclass/obd_mount_server.c index ef453b0..f0a4695 100644 --- a/lustre/obdclass/obd_mount_server.c +++ b/lustre/obdclass/obd_mount_server.c @@ -492,7 +492,7 @@ err_lmi: } EXPORT_SYMBOL(lustre_find_lwp_by_index); -static void lustre_notify_lwp_list(struct obd_export *exp) +void lustre_notify_lwp_list(struct obd_export *exp) { struct lwp_register_item *lri, *tmp; LASSERT(exp != NULL); @@ -509,6 +509,7 @@ static void lustre_notify_lwp_list(struct obd_export *exp) } mutex_unlock(&lwp_register_list_lock); } +EXPORT_SYMBOL(lustre_notify_lwp_list); static int lustre_lwp_connect(struct obd_device *lwp) { @@ -560,7 +561,6 @@ static int lustre_lwp_connect(struct obd_device *lwp) if (unlikely(lwp->obd_lwp_export != NULL)) class_export_put(lwp->obd_lwp_export); lwp->obd_lwp_export = class_export_get(exp); - lustre_notify_lwp_list(exp); } GOTO(out, rc); diff --git a/lustre/osp/lwp_dev.c b/lustre/osp/lwp_dev.c index 73d5deb..574323e 100644 --- a/lustre/osp/lwp_dev.c +++ b/lustre/osp/lwp_dev.c @@ -42,6 +42,7 @@ #include #include #include +#include #include "osp_internal.h" @@ -50,6 +51,7 @@ struct lwp_device { struct obd_device *lpd_obd; /* corresponding OBD device */ struct obd_uuid lpd_cluuid;/* UUID of LWP */ struct obd_export *lpd_exp; /* export of LWP */ + struct ptlrpc_thread lpd_notify_thread; /* notify thread */ int lpd_connects; /* use count, 0 or 1 */ }; @@ -87,6 +89,9 @@ static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp, int rc; ENTRY; + thread_set_flags(&lwp->lpd_notify_thread, SVC_STOPPED); + init_waitqueue_head(&lwp->lpd_notify_thread.t_ctl_waitq); + OBD_ALLOC_PTR(bufs); if (bufs == NULL) RETURN(-ENOMEM); @@ -353,11 +358,17 @@ static struct lu_device *lwp_device_alloc(const struct lu_env *env, static struct lu_device *lwp_device_fini(const struct lu_env *env, struct lu_device *ludev) { - struct lwp_device *m = lu2lwp_dev(ludev); - struct obd_import *imp; - int rc; + struct lwp_device *m = lu2lwp_dev(ludev); + struct ptlrpc_thread *thread = &m->lpd_notify_thread; + struct l_wait_info lwi = { 0 }; + struct obd_import *imp; + int rc; ENTRY; + if (!thread_is_stopped(thread)) + l_wait_event(thread->t_ctl_waitq, thread_is_stopped(thread), + &lwi); + if (m->lpd_exp != NULL) class_disconnect(m->lpd_exp); @@ -388,6 +399,65 @@ struct lu_device_type lwp_device_type = { .ldt_ctx_tags = LCT_MD_THREAD }; +static int lwp_notify_main(void *args) +{ + struct obd_export *exp = (struct obd_export *)args; + struct lwp_device *lwp; + struct ptlrpc_thread *thread; + + LASSERT(exp != NULL); + lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev); + thread = &lwp->lpd_notify_thread; + + thread_set_flags(thread, SVC_RUNNING); + wake_up(&thread->t_ctl_waitq); + + lustre_notify_lwp_list(exp); + + thread_set_flags(thread, SVC_STOPPED); + wake_up(&thread->t_ctl_waitq); + return 0; +} + +/* + * Some notify callbacks may cause deadlock in failover + * scenario, so we have to start thread to run callbacks + * asynchronously. See LU-6273. + */ +static void lwp_notify_users(struct obd_export *exp) +{ + struct lwp_device *lwp; + struct ptlrpc_thread *thread; + struct task_struct *task; + struct l_wait_info lwi = { 0 }; + char name[MTI_NAME_MAXLEN]; + + LASSERT(exp != NULL); + lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev); + thread = &lwp->lpd_notify_thread; + + snprintf(name, MTI_NAME_MAXLEN, "lwp_notify_%s", + exp->exp_obd->obd_name); + + /* Notify happens only on LWP setup, so there shouldn't + * be notify thread running */ + if (!thread_is_stopped(thread)) { + CERROR("LWP notify thread: %s wasn't stopped\n", name); + return; + } + + task = kthread_run(lwp_notify_main, exp, name); + if (IS_ERR(task)) { + thread_set_flags(thread, SVC_STOPPED); + CERROR("Failed to start LWP notify thread:%s. %lu\n", + name, PTR_ERR(task)); + } + + l_wait_event(thread->t_ctl_waitq, + thread_is_running(thread) || thread_is_stopped(thread), + &lwi); +} + /** * Implementation of OBD device operations obd_ops::o_connect. * @@ -464,6 +534,9 @@ out_dis: out_sem: up_write(&cli->cl_sem); + if (rc == 0) + lwp_notify_users(*exp); + return rc; } -- 1.8.3.1