LU-1187 lod: move seq client init from MDT to LOD

Move seq client init from MDT to LOD, so to unify
config log for adding OSP and MD OSP.

Change-Id: I93f91c0dd6f3568196a99eecf422e0b409abda31
Signed-off-by: Wang Di <di.wang@whamcloud.com>
Reviewed-on: http://review.whamcloud.com/4337
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Alex Zhuravlev <alexey.zhuravlev@intel.com>
Reviewed-by: Mike Pershin <mike.pershin@intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>

diff --git a/lustre/lod/lod_dev.c b/lustre/lod/lod_dev.c
index 4d1d854..5ea6737 100644 (file)
--- a/lustre/lod/lod_dev.c
+++ b/lustre/lod/lod_dev.c
@@ -44,6 +44,7 @@
 #define DEBUG_SUBSYSTEM S_MDS
 
 #include <obd_class.h>
 #define DEBUG_SUBSYSTEM S_MDS
 
 #include <obd_class.h>

+#include <lustre_fid.h>

 #include <lustre_param.h>
 
 #include "lod_internal.h"
 #include <lustre_param.h>
 
 #include "lod_internal.h"


@@ -213,6 +214,120 @@ static int lodname2mdt_index(char *lodname, int *index)
        return 0;
 }
 
        return 0;
 }
 

+/*
+ * Init client sequence manager which is used by local MDS to talk to sequence
+ * controller on remote node.
+ */
+static int lod_seq_init_cli(const struct lu_env *env,
+                           struct lod_device *lod,
+                           char *tgtuuid, int index)
+{
+       struct seq_server_site  *ss;
+       struct obd_device       *osp;
+       int                     rc;
+       char                    *prefix;
+       struct obd_uuid         obd_uuid;
+       ENTRY;
+
+       ss = lu_site2seq(lod2lu_dev(lod)->ld_site);
+       LASSERT(ss != NULL);
+
+       /* check if this is adding the first MDC and controller is not yet
+        * initialized. */
+       if (index != 0 || ss->ss_client_seq)
+               RETURN(0);
+
+       obd_str2uuid(&obd_uuid, tgtuuid);
+       osp = class_find_client_obd(&obd_uuid, LUSTRE_OSP_NAME,
+                                  &lod->lod_dt_dev.dd_lu_dev.ld_obd->obd_uuid);
+       if (osp == NULL) {
+               CERROR("%s: can't find %s device\n",
+                       lod->lod_dt_dev.dd_lu_dev.ld_obd->obd_name,
+                       tgtuuid);
+               RETURN(-EINVAL);
+       }
+
+       if (!osp->obd_set_up) {
+               CERROR("target %s not set up\n", osp->obd_name);
+               rc = -EINVAL;
+       }
+
+       LASSERT(ss->ss_control_exp);
+       OBD_ALLOC_PTR(ss->ss_client_seq);
+       if (ss->ss_client_seq == NULL)
+               RETURN(-ENOMEM);
+
+       OBD_ALLOC(prefix, MAX_OBD_NAME + 5);
+       if (!prefix) {
+               OBD_FREE_PTR(ss->ss_client_seq);
+               ss->ss_client_seq = NULL;
+               RETURN(-ENOMEM);
+       }
+
+       snprintf(prefix, MAX_OBD_NAME + 5, "ctl-%s", osp->obd_name);
+       rc = seq_client_init(ss->ss_client_seq, ss->ss_control_exp,
+                            LUSTRE_SEQ_METADATA, prefix, NULL);
+       OBD_FREE(prefix, MAX_OBD_NAME + 5);
+       if (rc) {
+               OBD_FREE_PTR(ss->ss_client_seq);
+               ss->ss_client_seq = NULL;
+               RETURN(rc);
+       }
+
+       LASSERT(ss->ss_server_seq != NULL);
+       rc = seq_server_set_cli(ss->ss_server_seq, ss->ss_client_seq,
+                               env);
+
+       RETURN(rc);
+}
+
+static void lod_seq_fini_cli(struct lod_device *lod)
+{
+       struct seq_server_site *ss;
+
+       ENTRY;
+
+       ss = lu_site2seq(lod2lu_dev(lod)->ld_site);
+       if (ss == NULL) {
+               EXIT;
+               return;
+       }
+
+       if (ss->ss_server_seq)
+               seq_server_set_cli(ss->ss_server_seq,
+                          NULL, NULL);
+
+       if (ss->ss_control_exp) {
+               class_export_put(ss->ss_control_exp);
+               ss->ss_control_exp = NULL;
+       }
+
+       EXIT;
+       return;
+}
+
+/**
+ * Procss config log on LOD
+ * \param env environment info
+ * \param dev lod device
+ * \param lcfg config log
+ *
+ * Add osc config log,
+ * marker  20 (flags=0x01, v2.2.49.56) lustre-OST0001  'add osc'
+ * add_uuid  nid=192.168.122.162@tcp(0x20000c0a87aa2)  0:  1:nidxxx
+ * attach    0:lustre-OST0001-osc-MDT0001  1:osc  2:lustre-MDT0001-mdtlov_UUID
+ * setup     0:lustre-OST0001-osc-MDT0001  1:lustre-OST0001_UUID  2:nid
+ * lov_modify_tgts add 0:lustre-MDT0001-mdtlov  1:lustre-OST0001_UUID  2:1  3:1
+ * marker  20 (flags=0x02, v2.2.49.56) lustre-OST0001  'add osc'
+ *
+ * Add mdc config log
+ * marker  10 (flags=0x01, v2.2.49.56) lustre-MDT0000  'add osp'
+ * add_uuid  nid=192.168.122.162@tcp(0x20000c0a87aa2)  0:  1:nid
+ * attach 0:lustre-MDT0000-osp-MDT0001  1:osp  2:lustre-MDT0001-mdtlov_UUID
+ * setup     0:lustre-MDT0000-osp-MDT0001  1:lustre-MDT0000_UUID  2:nid
+ * modify_mdc_tgts add 0:lustre-MDT0001  1:lustre-MDT0000_UUID  2:0  3:1
+ * marker  10 (flags=0x02, v2.2.49.56) lustre-MDT0000_UUID  'add osp'
+ **/

 static int lod_process_config(const struct lu_env *env,
                              struct lu_device *dev,
                              struct lustre_cfg *lcfg)
 static int lod_process_config(const struct lu_env *env,
                              struct lu_device *dev,
                              struct lustre_cfg *lcfg)


@@ -260,6 +375,9 @@ static int lod_process_config(const struct lu_env *env,
                        mdt_index = index;
                        rc = lod_add_device(env, lod, arg1, index, gen,
                                            mdt_index, LUSTRE_MDC_NAME, 1);
                        mdt_index = index;
                        rc = lod_add_device(env, lod, arg1, index, gen,
                                            mdt_index, LUSTRE_MDC_NAME, 1);

+                       if (rc == 0)
+                               rc = lod_seq_init_cli(env, lod, arg1,
+                                                     mdt_index);

                } else if (lcfg->lcfg_command == LCFG_LOV_ADD_INA) {
                        /*FIXME: Add mdt_index for LCFG_LOV_ADD_INA*/
                        mdt_index = 0;
                } else if (lcfg->lcfg_command == LCFG_LOV_ADD_INA) {
                        /*FIXME: Add mdt_index for LCFG_LOV_ADD_INA*/
                        mdt_index = 0;


@@ -289,6 +407,8 @@ static int lod_process_config(const struct lu_env *env,
                lu_dev_del_linkage(dev->ld_site, dev);
                lod_cleanup_desc_tgts(env, lod, &lod->lod_mdt_descs, lcfg);
                lod_cleanup_desc_tgts(env, lod, &lod->lod_ost_descs, lcfg);
                lu_dev_del_linkage(dev->ld_site, dev);
                lod_cleanup_desc_tgts(env, lod, &lod->lod_mdt_descs, lcfg);
                lod_cleanup_desc_tgts(env, lod, &lod->lod_ost_descs, lcfg);

+
+               lod_seq_fini_cli(lod);

                /*
                 * do cleanup on underlying storage only when
                 * all OSPs are cleaned up, as they use that OSD as well
                /*
                 * do cleanup on underlying storage only when
                 * all OSPs are cleaned up, as they use that OSD as well

diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index 8132a6a..41cc425 100644 (file)
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -3914,100 +3914,6 @@ out_seq_fini:
 }
 
 /*
 }
 
 /*

- * Init client sequence manager which is used by local MDS to talk to sequence
- * controller on remote node.
- */
-static int mdt_seq_init_cli(const struct lu_env *env,
-                            struct mdt_device *m,
-                            struct lustre_cfg *cfg)
-{
-       struct seq_server_site  *ss = mdt_seq_site(m);
-       struct obd_device *mdc;
-       int                rc;
-       int                index;
-       struct mdt_thread_info *info;
-       char *p, *index_string = lustre_cfg_string(cfg, 2);
-       ENTRY;
-
-       info = lu_context_key_get(&env->le_ctx, &mdt_thread_key);
-
-       LASSERT(index_string);
-       index = simple_strtol(index_string, &p, 10);
-       if (*p) {
-               CERROR("%s: Invalid index in lustre_cgf, offset 2\n",
-                     mdt2obd_dev(m)->obd_name);
-               RETURN(-EINVAL);
-       }
-
-       /* check if this is adding the first MDC and controller is not yet
-        * initialized. */
-       if (index != 0 || ss->ss_client_seq)
-               RETURN(0);
-
-       mdc = class_name2obd(lustre_cfg_string(cfg, 1));
-       if (!mdc) {
-               CERROR("%s: can't find %s device\n",
-                      mdt2obd_dev(m)->obd_name, lustre_cfg_string(cfg, 1));
-               rc = -ENOENT;
-       } else if (!mdc->obd_set_up) {
-               CERROR("%s: target %s not set up\n",
-                      mdt2obd_dev(m)->obd_name, mdc->obd_name);
-               rc = -EINVAL;
-       } else {
-               LASSERT(ss->ss_control_exp);
-               OBD_ALLOC_PTR(ss->ss_client_seq);
-               if (ss->ss_client_seq != NULL) {
-                       char *prefix;
-
-                       OBD_ALLOC(prefix, MAX_OBD_NAME + 5);
-                       if (!prefix)
-                               RETURN(-ENOMEM);
-
-                       snprintf(prefix, MAX_OBD_NAME + 5, "ctl-%s",
-                                mdc->obd_name);
-
-                       rc = seq_client_init(ss->ss_client_seq,
-                                            ss->ss_control_exp,
-                                            LUSTRE_SEQ_METADATA,
-                                            prefix, NULL);
-                       OBD_FREE(prefix, MAX_OBD_NAME + 5);
-               } else
-                       rc = -ENOMEM;
-
-               if (rc)
-                       RETURN(rc);
-
-               LASSERT(ss->ss_server_seq != NULL);
-               rc = seq_server_set_cli(ss->ss_server_seq, ss->ss_client_seq,
-                                       env);
-       }
-
-       RETURN(rc);
-}
-
-static void mdt_seq_fini_cli(struct mdt_device *m)
-{
-       struct seq_server_site *ss;
-
-       ENTRY;
-
-       ss = mdt_seq_site(m);
-
-       if (ss == NULL)
-               RETURN_EXIT;
-
-       if (ss->ss_server_seq)
-               seq_server_set_cli(ss->ss_server_seq, NULL, NULL);
-
-       if (ss->ss_control_exp) {
-               class_export_put(ss->ss_control_exp);
-               ss->ss_control_exp = NULL;
-       }
-
-       EXIT;
-}
-
-/*

  * FLD wrappers
  */
 static int mdt_fld_fini(const struct lu_env *env,
  * FLD wrappers
  */
 static int mdt_fld_fini(const struct lu_env *env,


@@ -4461,7 +4367,6 @@ static void mdt_fini(const struct lu_env *env, struct mdt_device *m)
         }
 
         mdt_seq_fini(env, m);
         }
 
         mdt_seq_fini(env, m);

-        mdt_seq_fini_cli(m);

         mdt_fld_fini(env, m);
         sptlrpc_rule_set_free(&m->mdt_sptlrpc_rset);
 
         mdt_fld_fini(env, m);
         sptlrpc_rule_set_free(&m->mdt_sptlrpc_rset);
 


@@ -4620,6 +4525,14 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m,
         if (rc)
                 GOTO(err_fini_stack, rc);
 
         if (rc)
                 GOTO(err_fini_stack, rc);
 

+       rc = mdt_fld_init(env, obd->obd_name, m);
+       if (rc)
+               GOTO(err_lut, rc);
+
+       rc = mdt_seq_init(env, obd->obd_name, m);
+       if (rc)
+               GOTO(err_fini_fld, rc);
+

         snprintf(info->mti_u.ns_name, sizeof info->mti_u.ns_name,
                  LUSTRE_MDT_NAME"-%p", m);
         m->mdt_namespace = ldlm_namespace_new(obd, info->mti_u.ns_name,
         snprintf(info->mti_u.ns_name, sizeof info->mti_u.ns_name,
                  LUSTRE_MDT_NAME"-%p", m);
         m->mdt_namespace = ldlm_namespace_new(obd, info->mti_u.ns_name,


@@ -4723,9 +4636,11 @@ err_free_ns:
         ldlm_namespace_free(m->mdt_namespace, NULL, 0);
         obd->obd_namespace = m->mdt_namespace = NULL;
 err_fini_seq:
         ldlm_namespace_free(m->mdt_namespace, NULL, 0);
         obd->obd_namespace = m->mdt_namespace = NULL;
 err_fini_seq:

-        mdt_seq_fini(env, m);
-        mdt_fld_fini(env, m);
-        tgt_fini(env, &m->mdt_lut);
+       mdt_seq_fini(env, m);
+err_fini_fld:
+       mdt_fld_fini(env, m);
+err_lut:
+       tgt_fini(env, &m->mdt_lut);

 err_fini_stack:
         mdt_stack_fini(env, m, md2lu_dev(m->mdt_child));
 err_lmi:
 err_fini_stack:
         mdt_stack_fini(env, m, md2lu_dev(m->mdt_child));
 err_lmi:


@@ -4805,17 +4720,6 @@ static int mdt_process_config(const struct lu_env *env,
 
                break;
        }
 
                break;
        }

-        case LCFG_ADD_MDC:
-                /*
-                 * Add mdc hook to get first MDT uuid and connect it to
-                 * ls->controller to use for seq manager.
-                 */
-                rc = next->ld_ops->ldo_process_config(env, next, cfg);
-                if (rc)
-                        CERROR("Can't add mdc, rc %d\n", rc);
-                else
-                        rc = mdt_seq_init_cli(env, mdt_dev(d), cfg);
-                break;

         default:
                 /* others are passed further */
                 rc = next->ld_ops->ldo_process_config(env, next, cfg);
         default:
                 /* others are passed further */
                 rc = next->ld_ops->ldo_process_config(env, next, cfg);


@@ -4920,14 +4824,6 @@ static int mdt_prepare(const struct lu_env *env,
        if (rc)
                RETURN(rc);
 
        if (rc)
                RETURN(rc);
 

-       rc = mdt_fld_init(env, obd->obd_name, mdt);
-       if (rc)
-               RETURN(rc);
-
-       rc = mdt_seq_init(env, obd->obd_name, mdt);
-       if (rc)
-               RETURN(rc);
-

        rc = mdt->mdt_child->md_ops->mdo_root_get(env, mdt->mdt_child,
                                                  &mdt->mdt_md_root_fid);
        if (rc)
        rc = mdt->mdt_child->md_ops->mdo_root_get(env, mdt->mdt_child,
                                                  &mdt->mdt_md_root_fid);
        if (rc)