LU-1711 mount: lu_site is provided by OSD

previously it was created by MDT, which is started after MGS.
so we relocate lu_site in OSD and now it's created before
any service like MDS/MGS.

Signed-off-by: Alex Zhuravlev <bzzz@whamcloud.com>
Change-Id: I43312218c293df509064cc6a854074e652a6e691
Reviewed-on: http://review.whamcloud.com/3651
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Jinshan Xiong <jinshan.xiong@whamcloud.com>

diff --git a/lustre/fid/fid_handler.c b/lustre/fid/fid_handler.c
index b831b3a..809e0f5 100644 (file)
--- a/lustre/fid/fid_handler.c
+++ b/lustre/fid/fid_handler.c
@@ -203,7 +203,7 @@ static int range_alloc_set(const struct lu_env *env,
 
         if (range_is_exhausted(loset)) {
                 /* reached high water mark. */
-                struct lu_device *dev = seq->lss_site->ms_lu.ls_top_dev;
+               struct lu_device *dev = seq->lss_site->ms_lu->ls_top_dev;
                 int obd_num_clients = dev->ld_obd->obd_num_exports;
                 __u64 set_sz;
 

diff --git a/lustre/include/lu_object.h b/lustre/include/lu_object.h
index 2ae02b3..fb7d0b3 100644 (file)
--- a/lustre/include/lu_object.h
+++ b/lustre/include/lu_object.h
@@ -644,6 +644,10 @@ struct lu_site {
          */
         struct lprocfs_stats     *ls_stats;
         struct lprocfs_stats     *ls_time_stats;
+       /**
+        * XXX: a hack! fld has to find md_site via site, remove when possible
+        */
+       struct md_site           *ld_md_site;
 };
 
 static inline struct lu_site_bkt_data *

diff --git a/lustre/include/md_object.h b/lustre/include/md_object.h
index 973c050..7cf4c68 100644 (file)
--- a/lustre/include/md_object.h
+++ b/lustre/include/md_object.h
@@ -527,7 +527,7 @@ struct md_object {
  * md-server site.
  */
 struct md_site {
-        struct lu_site ms_lu;
+       struct lu_site       *ms_lu;
         /**
          * mds number of this site.
          */
@@ -590,7 +590,7 @@ static inline struct md_device *md_obj2dev(const struct md_object *o)
 
 static inline struct md_site *lu_site2md(const struct lu_site *s)
 {
-        return container_of0(s, struct md_site, ms_lu);
+       return s->ld_md_site;
 }
 
 static inline int md_device_init(struct md_device *md, struct lu_device_type *t)

diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index c7e1b68..ef39c28 100644 (file)
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -4487,7 +4487,13 @@ static int mdt_stack_init(struct lu_env *env,
        tmp = m->mdt_bottom_exp->exp_obd->obd_lu_dev;
        LASSERT(tmp);
        m->mdt_bottom = lu2dt_dev(tmp);
-       tmp->ld_site = d->ld_site;
+
+       /* initialize site's pointers: md_site, top device */
+       d->ld_site = tmp->ld_site;
+       d->ld_site->ls_top_dev = d;
+       m->mdt_mite.ms_lu = tmp->ld_site;
+       tmp->ld_site->ld_md_site = &m->mdt_mite;
+       LASSERT(d->ld_site);
        d = tmp;
 
         tmp = mdt_layer_setup(env, LUSTRE_MDD_NAME, d, cfg);
@@ -4585,7 +4591,6 @@ static void mdt_fini(const struct lu_env *env, struct mdt_device *m)
 {
         struct md_device  *next = m->mdt_child;
         struct lu_device  *d    = &m->mdt_md_dev.md_lu_dev;
-        struct lu_site    *ls   = d->ld_site;
         struct obd_device *obd = mdt2obd_dev(m);
         ENTRY;
 
@@ -4636,14 +4641,6 @@ static void mdt_fini(const struct lu_env *env, struct mdt_device *m)
          */
         mdt_stack_fini(env, m, md2lu_dev(m->mdt_child));
 
-        if (ls) {
-                struct md_site *mite;
-
-                lu_site_fini(ls);
-                mite = lu_site2md(ls);
-                OBD_FREE_PTR(mite);
-                d->ld_site = NULL;
-        }
         LASSERT(cfs_atomic_read(&d->ld_ref) == 0);
 
        server_put_mount(mdt2obd_dev(m)->obd_name, NULL);
@@ -4754,25 +4751,23 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m,
         m->mdt_nosquash_strlen = 0;
         cfs_init_rwsem(&m->mdt_squash_sem);
 
-        OBD_ALLOC_PTR(mite);
-        if (mite == NULL)
-                GOTO(err_lmi, rc = -ENOMEM);
-
-        s = &mite->ms_lu;
-
         m->mdt_md_dev.md_lu_dev.ld_ops = &mdt_lu_ops;
         m->mdt_md_dev.md_lu_dev.ld_obd = obd;
         /* set this lu_device to obd, because error handling need it */
         obd->obd_lu_dev = &m->mdt_md_dev.md_lu_dev;
 
-        rc = lu_site_init(s, &m->mdt_md_dev.md_lu_dev);
-        if (rc) {
-                CERROR("Can't init lu_site, rc %d\n", rc);
-                GOTO(err_free_site, rc);
-        }
+       /* init the stack */
+       rc = mdt_stack_init((struct lu_env *)env, m, cfg, lmi);
+       if (rc) {
+               CERROR("Can't init device stack, rc %d\n", rc);
+               RETURN(rc);
+       }
+
+       s = m->mdt_md_dev.md_lu_dev.ld_site;
+       mite = &m->mdt_mite;
 
         /* set server index */
-        lu_site2md(s)->ms_node_id = node_id;
+       mite->ms_node_id = node_id;
 
         /* failover is the default
          * FIXME: we do not failout mds0/mgs, which may cause some problems.
@@ -4790,13 +4785,6 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m,
                 }
         }
 
-        /* init the stack */
-        rc = mdt_stack_init((struct lu_env *)env, m, cfg, lmi);
-        if (rc) {
-                CERROR("Can't init device stack, rc %d\n", rc);
-                GOTO(err_lu_site, rc);
-        }
-
         rc = lut_init(env, &m->mdt_lut, obd, m->mdt_bottom);
         if (rc)
                 GOTO(err_fini_stack, rc);
@@ -4891,10 +4879,6 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m,
 
         ping_evictor_start();
 
-        rc = lu_site_init_finish(s);
-        if (rc)
-                GOTO(err_stop_service, rc);
-
         if (obd->obd_recovering == 0)
                 mdt_postrecov(env, m);
 
@@ -4908,7 +4892,6 @@ static int mdt_init0(const struct lu_env *env, struct mdt_device *m,
 
         RETURN(0);
 
-err_stop_service:
         ping_evictor_stop();
         mdt_stop_ptlrpc_service(m);
 err_procfs:
@@ -4940,10 +4923,6 @@ err_lut:
         lut_fini(env, &m->mdt_lut);
 err_fini_stack:
         mdt_stack_fini(env, m, md2lu_dev(m->mdt_child));
-err_lu_site:
-        lu_site_fini(s);
-err_free_site:
-        OBD_FREE_PTR(mite);
 err_lmi:
        if (lmi)
                server_put_mount(dev, lmi->lmi_mnt);

diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h
index e15524d..4c33a37 100644 (file)
--- a/lustre/mdt/mdt_internal.h
+++ b/lustre/mdt/mdt_internal.h
@@ -98,6 +98,7 @@ struct mdt_file_data {
 struct mdt_device {
         /* super-class */
         struct md_device           mdt_md_dev;
+       struct md_site             mdt_mite;
         struct ptlrpc_service     *mdt_regular_service;
         struct ptlrpc_service     *mdt_readpage_service;
         struct ptlrpc_service     *mdt_xmds_service;
@@ -467,9 +468,9 @@ static inline struct lu_site *mdt_lu_site(const struct mdt_device *mdt)
         return mdt->mdt_md_dev.md_lu_dev.ld_site;
 }
 
-static inline struct md_site *mdt_md_site(const struct mdt_device *mdt)
+static inline struct md_site *mdt_md_site(struct mdt_device *mdt)
 {
-        return lu_site2md(mdt_lu_site(mdt));
+       return &mdt->mdt_mite;
 }
 
 static inline void mdt_export_evict(struct obd_export *exp)

diff --git a/lustre/obdclass/lu_object.c b/lustre/obdclass/lu_object.c
index caa2c01..a91b4b6 100644 (file)
--- a/lustre/obdclass/lu_object.c
+++ b/lustre/obdclass/lu_object.c
@@ -1213,15 +1213,6 @@ void lu_stack_fini(const struct lu_env *env, struct lu_device *top)
         /* purge again. */
         lu_site_purge(env, site, ~0);
 
-        if (!cfs_hash_is_empty(site->ls_obj_hash)) {
-                /*
-                 * Uh-oh, objects still exist.
-                 */
-                LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
-
-                lu_site_print(env, site, &msgdata, lu_cdebug_printer);
-        }
-
         for (scan = top; scan != NULL; scan = next) {
                 const struct lu_device_type *ldt = scan->ld_type;
                 struct obd_type             *type;

diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c
index b06c04b..7b2710e 100644 (file)
--- a/lustre/ofd/ofd_dev.c
+++ b/lustre/ofd/ofd_dev.c
@@ -90,6 +90,12 @@ static int ofd_connect_to_next(const struct lu_env *env, struct ofd_device *m,
                GOTO(out, rc);
        }
 
+       m->ofd_dt_dev.dd_lu_dev.ld_site =
+               m->ofd_osd_exp->exp_obd->obd_lu_dev->ld_site;
+       LASSERT(m->ofd_dt_dev.dd_lu_dev.ld_site);
+       m->ofd_osd = lu2dt_dev(m->ofd_osd_exp->exp_obd->obd_lu_dev);
+       m->ofd_dt_dev.dd_lu_dev.ld_site->ls_top_dev = &m->ofd_dt_dev.dd_lu_dev;
+
 out:
        if (data)
                OBD_FREE_PTR(data);
@@ -99,7 +105,6 @@ out:
 static int ofd_stack_init(const struct lu_env *env,
                          struct ofd_device *m, struct lustre_cfg *cfg)
 {
-       struct lu_device        *ofd_lu = &m->ofd_dt_dev.dd_lu_dev;
        const char              *dev = lustre_cfg_string(cfg, 0);
        struct lu_device        *d;
        struct ofd_thread_info  *info = ofd_info(env);
@@ -130,9 +135,6 @@ static int ofd_stack_init(const struct lu_env *env,
        LASSERT(d);
        m->ofd_osd = lu2dt_dev(d);
 
-       LASSERT(ofd_lu->ld_site);
-       d->ld_site = ofd_lu->ld_site;
-
        snprintf(info->fti_u.name, sizeof(info->fti_u.name),
                 "%s-osd", lustre_cfg_string(cfg, 0));
 
@@ -491,15 +493,10 @@ static int ofd_init0(const struct lu_env *env, struct ofd_device *m,
        if (info == NULL)
                RETURN(-EFAULT);
 
-       rc = lu_site_init(&m->ofd_site, &m->ofd_dt_dev.dd_lu_dev);
-       if (rc)
-               GOTO(err_fini_proc, rc);
-       m->ofd_site.ls_top_dev = &m->ofd_dt_dev.dd_lu_dev;
-
        rc = ofd_stack_init(env, m, cfg);
        if (rc) {
                CERROR("Can't init device stack, rc %d\n", rc);
-               GOTO(err_lu_site, rc);
+               GOTO(err_fini_proc, rc);
        }
 
        /* populate cached statfs data */
@@ -556,14 +553,7 @@ static int ofd_init0(const struct lu_env *env, struct ofd_device *m,
 
        target_recovery_init(&m->ofd_lut, ost_handle);
 
-       rc = lu_site_init_finish(&m->ofd_site);
-       if (rc)
-               GOTO(err_fs_cleanup, rc);
-
        RETURN(0);
-err_fs_cleanup:
-       target_recovery_fini(obd);
-       ofd_fs_cleanup(env, m);
 err_fini_lut:
        lut_fini(env, &m->ofd_lut);
 err_free_ns:
@@ -571,8 +561,6 @@ err_free_ns:
        obd->obd_namespace = m->ofd_namespace = NULL;
 err_fini_stack:
        ofd_stack_fini(env, m, &m->ofd_osd->dd_lu_dev);
-err_lu_site:
-       lu_site_fini(&m->ofd_site);
 err_fini_proc:
        ofd_procfs_fini(m);
        return rc;
@@ -600,7 +588,6 @@ static void ofd_fini(const struct lu_env *env, struct ofd_device *m)
        }
 
        ofd_stack_fini(env, m, &m->ofd_dt_dev.dd_lu_dev);
-       lu_site_fini(&m->ofd_site);
        ofd_procfs_fini(m);
        LASSERT(cfs_atomic_read(&d->ld_ref) == 0);
        server_put_mount(obd->obd_name, NULL);

diff --git a/lustre/ofd/ofd_internal.h b/lustre/ofd/ofd_internal.h
index c94a7a2..2aca2d2 100644 (file)
--- a/lustre/ofd/ofd_internal.h
+++ b/lustre/ofd/ofd_internal.h
@@ -143,8 +143,6 @@ struct ofd_device {
                                 /* shall we grant space to clients not
                                  * supporting OBD_CONNECT_GRANT_PARAM? */
                                 ofd_grant_compat_disable:1;
-
-       struct lu_site           ofd_site;
 };
 
 static inline struct ofd_device *ofd_dev(struct lu_device *d)

diff --git a/lustre/osd-ldiskfs/osd_handler.c b/lustre/osd-ldiskfs/osd_handler.c
index f1c0506..1c37bd9 100644 (file)
--- a/lustre/osd-ldiskfs/osd_handler.c
+++ b/lustre/osd-ldiskfs/osd_handler.c
@@ -4497,6 +4497,14 @@ static int osd_device_init0(const struct lu_env *env,
                        GOTO(out_mnt, rc);
        }
 
+       rc = lu_site_init(&o->od_site, l);
+       if (rc)
+               GOTO(out_compat, rc);
+
+       rc = lu_site_init_finish(&o->od_site);
+       if (rc)
+               GOTO(out_compat, rc);
+
        rc = osd_procfs_init(o, o->od_svname);
        if (rc != 0) {
                CERROR("%s: can't initialize procfs: rc = %d\n",
@@ -4504,6 +4512,8 @@ static int osd_device_init0(const struct lu_env *env,
                GOTO(out_compat, rc);
        }
 
+       LASSERT(l->ld_site->ls_linkage.next && l->ld_site->ls_linkage.prev);
+
        RETURN(0);
 out_compat:
        osd_compat_fini(o);
@@ -4549,6 +4559,14 @@ static struct lu_device *osd_device_free(const struct lu_env *env,
         ENTRY;
 
         cleanup_capa_hash(o->od_capa_hash);
+       /* XXX: make osd top device in order to release reference */
+       d->ld_site->ls_top_dev = d;
+       lu_site_purge(env, d->ld_site, -1);
+       if (!cfs_hash_is_empty(d->ld_site->ls_obj_hash)) {
+               LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
+               lu_site_print(env, d->ld_site, &msgdata, lu_cdebug_printer);
+       }
+       lu_site_fini(&o->od_site);
         dt_device_fini(&o->od_dt_dev);
         OBD_FREE_PTR(o);
         RETURN(NULL);

diff --git a/lustre/osd-ldiskfs/osd_internal.h b/lustre/osd-ldiskfs/osd_internal.h
index 939d539..add8dd4 100644 (file)
--- a/lustre/osd-ldiskfs/osd_internal.h
+++ b/lustre/osd-ldiskfs/osd_internal.h
@@ -287,6 +287,7 @@ struct osd_device {
 
         struct fsfilt_operations *od_fsops;
        int                       od_connects;
+       struct lu_site            od_site;
 
         /*
          * mapping for legacy OST objids

diff --git a/lustre/osd-zfs/osd_handler.c b/lustre/osd-zfs/osd_handler.c
index fedf1e9..c34ab99 100644 (file)
--- a/lustre/osd-zfs/osd_handler.c
+++ b/lustre/osd-zfs/osd_handler.c
@@ -522,6 +522,14 @@ static int osd_mount(const struct lu_env *env,
        if (rc)
                GOTO(err, rc);
 
+       rc = lu_site_init(&o->od_site, osd2lu_dev(o));
+       if (rc)
+               GOTO(err, rc);
+
+       rc = lu_site_init_finish(&o->od_site);
+       if (rc)
+               GOTO(err, rc);
+
        /* Use our own ZAP for inode accounting by default, this can be changed
         * via procfs to estimate the inode usage from the block usage */
        o->od_quota_iused_est = 0;
@@ -619,9 +627,13 @@ static struct lu_device *osd_device_free(const struct lu_env *env,
 
        cleanup_capa_hash(o->od_capa_hash);
        /* XXX: make osd top device in order to release reference */
-       /*d->ld_site->ls_top_dev = d;
+       d->ld_site->ls_top_dev = d;
        lu_site_purge(env, d->ld_site, -1);
-       lu_site_fini(&o->od_site);*/
+       if (!cfs_hash_is_empty(d->ld_site->ls_obj_hash)) {
+               LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
+               lu_site_print(env, d->ld_site, &msgdata, lu_cdebug_printer);
+       }
+       lu_site_fini(&o->od_site);
        dt_device_fini(&o->od_dt_dev);
        OBD_FREE_PTR(o);