LU-6273 lwp: notify LWP users in dedicated thread

On OST/MDT mount, the client config log will be processed to setup
the LWP as following:

1> Process the LCFG_ADD_UUID record to setup LWP device, then connect
   to the server target. (see lustre_lwp_setup());
2> Process the LCFG_ADD_CONN record to add failover connection;

We can see that if the mount process is blocked in step 1 it will
never have a chance to add failover connection, and LWP won't be able
to switch to failover node forever.

Unfortunately, the callbacks for FLD user could block the step 1.
See ofd/mdt_register_lwp_callback() calls fld_client_rpc() which will
send FLD RPC in a deadloop if the connection isn't available.

This patch solve the problem by using a per LWP dedicated thread to
run the notify callbacks.

Signed-off-by: Niu Yawei <yawei.niu@intel.com>
Signed-off-by: wang di <di.wang@intel.com>
Change-Id: Ic0d89f1524ea0c1a3e7fc3833e16ecbad2123454
Reviewed-on: http://review.whamcloud.com/16303
Reviewed-by: James Simmons <uja.ornl@yahoo.com>
Tested-by: Jenkins
Reviewed-by: Mike Pershin <mike.pershin@intel.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h
index f408800..e0c0ca3 100644 (file)
--- a/lustre/include/obd_class.h
+++ b/lustre/include/obd_class.h
@@ -1699,6 +1699,7 @@ int lustre_register_lwp_item(const char *lwpname, struct obd_export **exp,
                             register_lwp_cb cb_func, void *cb_data);
 void lustre_deregister_lwp_item(struct obd_export **exp);
 struct obd_export *lustre_find_lwp_by_index(const char *dev, __u32 idx);
+void lustre_notify_lwp_list(struct obd_export *exp);
 int tgt_name2lwp_name(const char *tgt_name, char *lwp_name, int len, __u32 idx);
 #endif /* HAVE_SERVER_SUPPORT */
 int lustre_register_fs(void);

diff --git a/lustre/obdclass/obd_mount_server.c b/lustre/obdclass/obd_mount_server.c
index ef453b0..f0a4695 100644 (file)
--- a/lustre/obdclass/obd_mount_server.c
+++ b/lustre/obdclass/obd_mount_server.c
@@ -492,7 +492,7 @@ err_lmi:
 }
 EXPORT_SYMBOL(lustre_find_lwp_by_index);
 
-static void lustre_notify_lwp_list(struct obd_export *exp)
+void lustre_notify_lwp_list(struct obd_export *exp)
 {
        struct lwp_register_item *lri, *tmp;
        LASSERT(exp != NULL);
@@ -509,6 +509,7 @@ static void lustre_notify_lwp_list(struct obd_export *exp)
        }
        mutex_unlock(&lwp_register_list_lock);
 }
+EXPORT_SYMBOL(lustre_notify_lwp_list);
 
 static int lustre_lwp_connect(struct obd_device *lwp)
 {
@@ -560,7 +561,6 @@ static int lustre_lwp_connect(struct obd_device *lwp)
                if (unlikely(lwp->obd_lwp_export != NULL))
                        class_export_put(lwp->obd_lwp_export);
                lwp->obd_lwp_export = class_export_get(exp);
-               lustre_notify_lwp_list(exp);
        }
 
        GOTO(out, rc);

diff --git a/lustre/osp/lwp_dev.c b/lustre/osp/lwp_dev.c
index 73d5deb..574323e 100644 (file)
--- a/lustre/osp/lwp_dev.c
+++ b/lustre/osp/lwp_dev.c
@@ -42,6 +42,7 @@
 #include <obd_class.h>
 #include <lustre_param.h>
 #include <lustre_log.h>
+#include <linux/kthread.h>
 
 #include "osp_internal.h"
 
@@ -50,6 +51,7 @@ struct lwp_device {
        struct obd_device      *lpd_obd;   /* corresponding OBD device */
        struct obd_uuid         lpd_cluuid;/* UUID of LWP */
        struct obd_export      *lpd_exp;   /* export of LWP */
+       struct ptlrpc_thread    lpd_notify_thread; /* notify thread */
        int                     lpd_connects; /* use count, 0 or 1 */
 };
 
@@ -87,6 +89,9 @@ static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp,
        int                      rc;
        ENTRY;
 
+       thread_set_flags(&lwp->lpd_notify_thread, SVC_STOPPED);
+       init_waitqueue_head(&lwp->lpd_notify_thread.t_ctl_waitq);
+
        OBD_ALLOC_PTR(bufs);
        if (bufs == NULL)
                RETURN(-ENOMEM);
@@ -353,11 +358,17 @@ static struct lu_device *lwp_device_alloc(const struct lu_env *env,
 static struct lu_device *lwp_device_fini(const struct lu_env *env,
                                         struct lu_device *ludev)
 {
-       struct lwp_device *m = lu2lwp_dev(ludev);
-       struct obd_import *imp;
-       int                rc;
+       struct lwp_device       *m = lu2lwp_dev(ludev);
+       struct ptlrpc_thread    *thread = &m->lpd_notify_thread;
+       struct l_wait_info       lwi = { 0 };
+       struct obd_import       *imp;
+       int                      rc;
        ENTRY;
 
+       if (!thread_is_stopped(thread))
+               l_wait_event(thread->t_ctl_waitq, thread_is_stopped(thread),
+                            &lwi);
+
        if (m->lpd_exp != NULL)
                class_disconnect(m->lpd_exp);
 
@@ -388,6 +399,65 @@ struct lu_device_type lwp_device_type = {
        .ldt_ctx_tags = LCT_MD_THREAD
 };
 
+static int lwp_notify_main(void *args)
+{
+       struct obd_export       *exp = (struct obd_export *)args;
+       struct lwp_device       *lwp;
+       struct ptlrpc_thread    *thread;
+
+       LASSERT(exp != NULL);
+       lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev);
+       thread = &lwp->lpd_notify_thread;
+
+       thread_set_flags(thread, SVC_RUNNING);
+       wake_up(&thread->t_ctl_waitq);
+
+       lustre_notify_lwp_list(exp);
+
+       thread_set_flags(thread, SVC_STOPPED);
+       wake_up(&thread->t_ctl_waitq);
+       return 0;
+}
+
+/*
+ * Some notify callbacks may cause deadlock in failover
+ * scenario, so we have to start thread to run callbacks
+ * asynchronously. See LU-6273.
+ */
+static void lwp_notify_users(struct obd_export *exp)
+{
+       struct lwp_device       *lwp;
+       struct ptlrpc_thread    *thread;
+       struct task_struct      *task;
+       struct l_wait_info       lwi = { 0 };
+       char                     name[MTI_NAME_MAXLEN];
+
+       LASSERT(exp != NULL);
+       lwp = lu2lwp_dev(exp->exp_obd->obd_lu_dev);
+       thread = &lwp->lpd_notify_thread;
+
+       snprintf(name, MTI_NAME_MAXLEN, "lwp_notify_%s",
+                exp->exp_obd->obd_name);
+
+       /* Notify happens only on LWP setup, so there shouldn't
+        * be notify thread running */
+       if (!thread_is_stopped(thread)) {
+               CERROR("LWP notify thread: %s wasn't stopped\n", name);
+               return;
+       }
+
+       task = kthread_run(lwp_notify_main, exp, name);
+       if (IS_ERR(task)) {
+               thread_set_flags(thread, SVC_STOPPED);
+               CERROR("Failed to start LWP notify thread:%s. %lu\n",
+                      name, PTR_ERR(task));
+       }
+
+       l_wait_event(thread->t_ctl_waitq,
+                    thread_is_running(thread) || thread_is_stopped(thread),
+                    &lwi);
+}
+
 /**
  * Implementation of OBD device operations obd_ops::o_connect.
  *
@@ -464,6 +534,9 @@ out_dis:
 out_sem:
        up_write(&cli->cl_sem);
 
+       if (rc == 0)
+               lwp_notify_users(*exp);
+
        return rc;
 }