LU-8508 nodemap: improve object handling in cache saving

Saving cache files requires that the old cache be removed. This means
that the config object reference needs to change to point to the new
file. Previously this was done in a number of different places and
was more opaque. This patch hopefully makes it more transparent.

This patch also fixes a problem on MDTs/OSTs when creating a new
config. Previously all initial config creates would assume an MGS
context. A side-effect of the improved object handling is that a
target context is handled correctly.

Signed-off-by: Kit Westneat <kit.westneat@gmail.com>
Change-Id: Iee33a423b76f30eba849288c746e6528ecefa7c6
Reviewed-on: http://review.whamcloud.com/22004
Reviewed-by: John L. Hammond <john.hammond@intel.com>
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/include/lustre_nodemap.h b/lustre/include/lustre_nodemap.h
index 44527a0..c8bf411 100644 (file)
--- a/lustre/include/lustre_nodemap.h
+++ b/lustre/include/lustre_nodemap.h
@@ -143,17 +143,17 @@ void nodemap_test_nid(lnet_nid_t nid, char *name_buf, size_t name_len);
 int nodemap_test_id(lnet_nid_t nid, enum nodemap_id_type idtype,
                    __u32 client_id, __u32 *fs_id);
 
 int nodemap_test_id(lnet_nid_t nid, enum nodemap_id_type idtype,
                    __u32 client_id, __u32 *fs_id);
 

-enum nm_config_file_type {
-       NCFT_MGS,
-       NCFT_TGT,
-};
-struct nm_config_file *nm_config_file_register(const struct lu_env *env,
-                                              struct dt_object *obj,
-                                              struct local_oid_storage *los,
-                                              enum nm_config_file_type ncf_type);
-void nm_config_file_deregister(const struct lu_env *env,
-                              struct nm_config_file *ncf,
-                              enum nm_config_file_type ncf_type);
+struct nm_config_file *nm_config_file_register_mgs(const struct lu_env *env,
+                                                  struct dt_object *obj,
+                                                  struct local_oid_storage *los);
+struct dt_device;
+struct nm_config_file *nm_config_file_register_tgt(const struct lu_env *env,
+                                                  struct dt_device *dev,
+                                                  struct local_oid_storage *los);
+void nm_config_file_deregister_mgs(const struct lu_env *env,
+                                  struct nm_config_file *ncf);
+void nm_config_file_deregister_tgt(const struct lu_env *env,
+                                  struct nm_config_file *ncf);

 struct lu_nodemap *nodemap_get_from_exp(struct obd_export *exp);
 void nodemap_putref(struct lu_nodemap *nodemap);
 
 struct lu_nodemap *nodemap_get_from_exp(struct obd_export *exp);
 void nodemap_putref(struct lu_nodemap *nodemap);
 


@@ -193,10 +193,6 @@ void nodemap_config_set_active_mgc(struct nodemap_config *config);
 int nodemap_process_idx_pages(struct nodemap_config *config, union lu_page *lip,
                              struct lu_nodemap **recent_nodemap);
 
 int nodemap_process_idx_pages(struct nodemap_config *config, union lu_page *lip,
                              struct lu_nodemap **recent_nodemap);
 

-struct dt_device;
-int nodemap_fs_init(const struct lu_env *env, struct dt_device *dev,
-                   struct obd_device *obd, struct local_oid_storage *los);
-void nodemap_fs_fini(const struct lu_env *env, struct obd_device *obd);

 #else /* disable nodemap processing in MGC of non-servers */
 static inline int nodemap_process_idx_pages(void *config,
                                            union lu_page *lip,
 #else /* disable nodemap processing in MGC of non-servers */
 static inline int nodemap_process_idx_pages(void *config,
                                            union lu_page *lip,

diff --git a/lustre/mdd/mdd_device.c b/lustre/mdd/mdd_device.c
index 2728b20..4a7b872 100644 (file)
--- a/lustre/mdd/mdd_device.c
+++ b/lustre/mdd/mdd_device.c
@@ -884,7 +884,7 @@ static void mdd_device_shutdown(const struct lu_env *env, struct mdd_device *m,
        mdd_changelog_fini(env, m);
        orph_index_fini(env, m);
        mdd_dot_lustre_cleanup(env, m);
        mdd_changelog_fini(env, m);
        orph_index_fini(env, m);
        mdd_dot_lustre_cleanup(env, m);

-       nodemap_fs_fini(env, mdd2obd_dev(m));
+       nm_config_file_deregister_tgt(env, mdd2obd_dev(m)->u.obt.obt_nodemap_config_file);

        if (m->mdd_los != NULL) {
                local_oid_storage_fini(env, m->mdd_los);
                m->mdd_los = NULL;
        if (m->mdd_los != NULL) {
                local_oid_storage_fini(env, m->mdd_los);
                m->mdd_los = NULL;


@@ -995,10 +995,11 @@ static int mdd_prepare(const struct lu_env *env,
                        struct lu_device *pdev,
                        struct lu_device *cdev)
 {
                        struct lu_device *pdev,
                        struct lu_device *cdev)
 {

-       struct mdd_device       *mdd = lu2mdd_dev(cdev);
-       struct lu_device        *next = &mdd->mdd_child->dd_lu_dev;
-       struct lu_fid            fid;
-       int                      rc;
+       struct mdd_device *mdd = lu2mdd_dev(cdev);
+       struct lu_device *next = &mdd->mdd_child->dd_lu_dev;
+       struct nm_config_file *nodemap_config;
+       struct lu_fid fid;
+       int rc;

 
        ENTRY;
 
 
        ENTRY;
 


@@ -1058,10 +1059,12 @@ static int mdd_prepare(const struct lu_env *env,
        if (rc != 0)
                GOTO(out_changelog, rc);
 
        if (rc != 0)
                GOTO(out_changelog, rc);
 

-       rc = nodemap_fs_init(env, mdd->mdd_bottom, mdd2obd_dev(mdd),
-                            mdd->mdd_los);
-       if (rc != 0)
-               GOTO(out_hsm, rc);
+       nodemap_config = nm_config_file_register_tgt(env, mdd->mdd_bottom,
+                                                    mdd->mdd_los);
+       if (IS_ERR(nodemap_config))
+               GOTO(out_hsm, rc = PTR_ERR(nodemap_config));
+
+       mdd2obd_dev(mdd)->u.obt.obt_nodemap_config_file = nodemap_config;

 
        rc = lfsck_register(env, mdd->mdd_bottom, mdd->mdd_child,
                            mdd2obd_dev(mdd), mdd_lfsck_out_notify,
 
        rc = lfsck_register(env, mdd->mdd_bottom, mdd->mdd_child,
                            mdd2obd_dev(mdd), mdd_lfsck_out_notify,


@@ -1075,7 +1078,8 @@ static int mdd_prepare(const struct lu_env *env,
        RETURN(0);
 
 out_nodemap:
        RETURN(0);
 
 out_nodemap:

-       nodemap_fs_fini(env, mdd2obd_dev(mdd));
+       nm_config_file_deregister_tgt(env, mdd2obd_dev(mdd)->u.obt.obt_nodemap_config_file);
+       mdd2obd_dev(mdd)->u.obt.obt_nodemap_config_file = NULL;

 out_hsm:
        mdd_hsm_actions_llog_fini(env, mdd);
 out_changelog:
 out_hsm:
        mdd_hsm_actions_llog_fini(env, mdd);
 out_changelog:

diff --git a/lustre/mgs/mgs_fs.c b/lustre/mgs/mgs_fs.c
index 338858e..00de194 100644 (file)
--- a/lustre/mgs/mgs_fs.c
+++ b/lustre/mgs/mgs_fs.c
@@ -163,7 +163,7 @@ int mgs_fs_setup(const struct lu_env *env, struct mgs_device *mgs)
 
        mgs->mgs_configs_dir = o;
 
 
        mgs->mgs_configs_dir = o;
 

-       /* colocated MDT will cache config in target / dir */
+       /* colocated MDT will cache config in target root dir */

        nm_config_file_obj = local_index_find_or_create(env, mgs->mgs_los,
                                                        mgs->mgs_configs_dir,
                                                        LUSTRE_NODEMAP_NAME,
        nm_config_file_obj = local_index_find_or_create(env, mgs->mgs_los,
                                                        mgs->mgs_configs_dir,
                                                        LUSTRE_NODEMAP_NAME,


@@ -182,10 +182,10 @@ int mgs_fs_setup(const struct lu_env *env, struct mgs_device *mgs)
                        GOTO(out_configs, rc);
                }
        }
                        GOTO(out_configs, rc);
                }
        }

-       nm_config_file = nm_config_file_register(env, nm_config_file_obj,
-                                                mgs->mgs_los, NCFT_MGS);
+       nm_config_file = nm_config_file_register_mgs(env, nm_config_file_obj,
+                                                    mgs->mgs_los);
+       lu_object_put(env, &nm_config_file_obj->do_lu);

        if (IS_ERR(nm_config_file)) {
        if (IS_ERR(nm_config_file)) {

-               lu_object_put(env, &nm_config_file_obj->do_lu);

                CERROR("%s: error loading nodemap config file, file must be "
                       "removed via ldiskfs: rc = %ld\n",
                       mgs->mgs_obd->obd_name, PTR_ERR(nm_config_file));
                CERROR("%s: error loading nodemap config file, file must be "
                       "removed via ldiskfs: rc = %ld\n",
                       mgs->mgs_obd->obd_name, PTR_ERR(nm_config_file));


@@ -203,7 +203,7 @@ int mgs_fs_setup(const struct lu_env *env, struct mgs_device *mgs)
 
 out_nm:
        if (rc < 0) {
 
 out_nm:
        if (rc < 0) {

-               nm_config_file_deregister(env, nm_config_file, NCFT_MGS);
+               nm_config_file_deregister_mgs(env, nm_config_file);

                mgs->mgs_obd->u.obt.obt_nodemap_config_file = NULL;
        }
 out_configs:
                mgs->mgs_obd->u.obt.obt_nodemap_config_file = NULL;
        }
 out_configs:


@@ -235,9 +235,9 @@ int mgs_fs_cleanup(const struct lu_env *env, struct mgs_device *mgs)
                mgs->mgs_nidtbl_dir = NULL;
        }
        if (mgs->mgs_obd->u.obt.obt_nodemap_config_file != NULL) {
                mgs->mgs_nidtbl_dir = NULL;
        }
        if (mgs->mgs_obd->u.obt.obt_nodemap_config_file != NULL) {

-               nm_config_file_deregister(env,
-                               mgs->mgs_obd->u.obt.obt_nodemap_config_file,
-                               NCFT_MGS);
+               struct nm_config_file *ncf = mgs->mgs_obd->u.obt.obt_nodemap_config_file;
+
+               nm_config_file_deregister_mgs(env, ncf);

                mgs->mgs_obd->u.obt.obt_nodemap_config_file = NULL;
        }
 
                mgs->mgs_obd->u.obt.obt_nodemap_config_file = NULL;
        }
 

diff --git a/lustre/ofd/ofd_dev.c b/lustre/ofd/ofd_dev.c
index 40cbf71..9e810a2 100644 (file)
--- a/lustre/ofd/ofd_dev.c
+++ b/lustre/ofd/ofd_dev.c
@@ -2884,12 +2884,13 @@ struct lu_context_key ofd_thread_key = {
 static int ofd_init0(const struct lu_env *env, struct ofd_device *m,
                     struct lu_device_type *ldt, struct lustre_cfg *cfg)
 {
 static int ofd_init0(const struct lu_env *env, struct ofd_device *m,
                     struct lu_device_type *ldt, struct lustre_cfg *cfg)
 {

-       const char              *dev = lustre_cfg_string(cfg, 0);
-       struct ofd_thread_info  *info = NULL;
-       struct obd_device       *obd;
-       struct obd_statfs       *osfs;
-       struct lu_fid            fid;
-       int                      rc;
+       const char *dev = lustre_cfg_string(cfg, 0);
+       struct ofd_thread_info *info = NULL;
+       struct obd_device *obd;
+       struct obd_statfs *osfs;
+       struct lu_fid fid;
+       struct nm_config_file *nodemap_config;
+       int rc;

 
        ENTRY;
 
 
        ENTRY;
 


@@ -3032,9 +3033,12 @@ static int ofd_init0(const struct lu_env *env, struct ofd_device *m,
        if (rc != 0)
                GOTO(err_fini_fs, rc);
 
        if (rc != 0)
                GOTO(err_fini_fs, rc);
 

-       rc = nodemap_fs_init(env, m->ofd_osd, obd, m->ofd_los);
-       if (rc != 0)
-               GOTO(err_fini_los, rc);
+       nodemap_config = nm_config_file_register_tgt(env, m->ofd_osd,
+                                                    m->ofd_los);
+       if (IS_ERR(nodemap_config))
+               GOTO(err_fini_los, rc = PTR_ERR(nodemap_config));
+
+       obd->u.obt.obt_nodemap_config_file = nodemap_config;

 
        rc = ofd_start_inconsistency_verification_thread(m);
        if (rc != 0)
 
        rc = ofd_start_inconsistency_verification_thread(m);
        if (rc != 0)


@@ -3045,7 +3049,8 @@ static int ofd_init0(const struct lu_env *env, struct ofd_device *m,
        RETURN(0);
 
 err_fini_nm:
        RETURN(0);
 
 err_fini_nm:

-       nodemap_fs_fini(env, obd);
+       nm_config_file_deregister_tgt(env, obd->u.obt.obt_nodemap_config_file);
+       obd->u.obt.obt_nodemap_config_file = NULL;

 err_fini_los:
        local_oid_storage_fini(env, m->ofd_los);
        m->ofd_los = NULL;
 err_fini_los:
        local_oid_storage_fini(env, m->ofd_los);
        m->ofd_los = NULL;


@@ -3093,7 +3098,8 @@ static void ofd_fini(const struct lu_env *env, struct ofd_device *m)
        ofd_stop_inconsistency_verification_thread(m);
        lfsck_degister(env, m->ofd_osd);
        ofd_fs_cleanup(env, m);
        ofd_stop_inconsistency_verification_thread(m);
        lfsck_degister(env, m->ofd_osd);
        ofd_fs_cleanup(env, m);

-       nodemap_fs_fini(env, obd);
+       nm_config_file_deregister_tgt(env, obd->u.obt.obt_nodemap_config_file);
+       obd->u.obt.obt_nodemap_config_file = NULL;

 
        if (m->ofd_los != NULL) {
                local_oid_storage_fini(env, m->ofd_los);
 
        if (m->ofd_los != NULL) {
                local_oid_storage_fini(env, m->ofd_los);

diff --git a/lustre/ptlrpc/nodemap_storage.c b/lustre/ptlrpc/nodemap_storage.c
index f4abb34..46a467c 100644 (file)
--- a/lustre/ptlrpc/nodemap_storage.c
+++ b/lustre/ptlrpc/nodemap_storage.c
@@ -153,10 +153,14 @@ static void nodemap_inc_version(const struct lu_env *env,
        dt_version_set(env, nodemap_idx, ver + 1, th);
 }
 
        dt_version_set(env, nodemap_idx, ver + 1, th);
 }
 

+enum ncfc_find_create {
+       NCFC_CREATE_NEW = 1,
+};
+

 static struct dt_object *nodemap_cache_find_create(const struct lu_env *env,
                                                   struct dt_device *dev,
                                                   struct local_oid_storage *los,
 static struct dt_object *nodemap_cache_find_create(const struct lu_env *env,
                                                   struct dt_device *dev,
                                                   struct local_oid_storage *los,

-                                                  bool force_create)
+                                                  enum ncfc_find_create create_new)

 {
        struct lu_fid root_fid;
        struct dt_object *root_obj;
 {
        struct lu_fid root_fid;
        struct dt_object *root_obj;


@@ -172,8 +176,8 @@ static struct dt_object *nodemap_cache_find_create(const struct lu_env *env,
                GOTO(out, nm_obj = root_obj);
 
 again:
                GOTO(out, nm_obj = root_obj);
 
 again:

-       /* if loading index fails the first time, try again with force_create */
-       if (force_create) {
+       /* if loading index fails the first time, create new index */
+       if (create_new == NCFC_CREATE_NEW) {

                CDEBUG(D_INFO, "removing old index, creating new one\n");
                rc = local_object_unlink(env, dev, root_obj,
                                         LUSTRE_NODEMAP_NAME);
                CDEBUG(D_INFO, "removing old index, creating new one\n");
                rc = local_object_unlink(env, dev, root_obj,
                                         LUSTRE_NODEMAP_NAME);


@@ -198,18 +202,19 @@ again:
                /* even if loading from tgt fails, connecting to MGS will
                 * rewrite the config
                 */
                /* even if loading from tgt fails, connecting to MGS will
                 * rewrite the config
                 */

-               if (rc < 0 && !force_create) {
+               if (rc < 0) {
+                       lu_object_put(env, &nm_obj->do_lu);
+
+                       if (create_new == NCFC_CREATE_NEW)
+                               GOTO(out_root, nm_obj = ERR_PTR(rc));
+

                        CERROR("cannot load nodemap index from disk, creating "
                               "new index: rc = %d\n", rc);
                        CERROR("cannot load nodemap index from disk, creating "
                               "new index: rc = %d\n", rc);

-                       lu_object_put(env, &nm_obj->do_lu);
-                       force_create = true;
+                       create_new = NCFC_CREATE_NEW;

                        goto again;
                }
        }
 
                        goto again;
                }
        }
 

-       if (rc < 0)
-               nm_obj = ERR_PTR(rc);
-

 out_root:
        lu_object_put(env, &root_obj->do_lu);
 out:
 out_root:
        lu_object_put(env, &root_obj->do_lu);
 out:


@@ -355,6 +360,7 @@ enum nm_add_update {
 };
 
 static int nodemap_idx_nodemap_add_update(const struct lu_nodemap *nodemap,
 };
 
 static int nodemap_idx_nodemap_add_update(const struct lu_nodemap *nodemap,

+                                         struct dt_object *idx,

                                          enum nm_add_update update)
 {
        struct nodemap_key nk;
                                          enum nm_add_update update)
 {
        struct nodemap_key nk;


@@ -364,11 +370,6 @@ static int nodemap_idx_nodemap_add_update(const struct lu_nodemap *nodemap,
 
        ENTRY;
 
 
        ENTRY;
 

-       if (nodemap_mgs_ncf == NULL) {
-               CERROR("cannot add nodemap config to non-existing MGS.\n");
-               return -EINVAL;
-       }
-

        rc = lu_env_init(&env, LCT_LOCAL);
        if (rc)
                RETURN(rc);
        rc = lu_env_init(&env, LCT_LOCAL);
        if (rc)
                RETURN(rc);


@@ -377,11 +378,9 @@ static int nodemap_idx_nodemap_add_update(const struct lu_nodemap *nodemap,
        nodemap_cluster_rec_init(&nr, nodemap);
 
        if (update == NM_UPDATE)
        nodemap_cluster_rec_init(&nr, nodemap);
 
        if (update == NM_UPDATE)

-               rc = nodemap_idx_update(&env, nodemap_mgs_ncf->ncf_obj,
-                                       &nk, &nr);
+               rc = nodemap_idx_update(&env, idx, &nk, &nr);

        else
        else

-               rc = nodemap_idx_insert(&env, nodemap_mgs_ncf->ncf_obj,
-                                       &nk, &nr);
+               rc = nodemap_idx_insert(&env, idx, &nk, &nr);

 
        lu_env_fini(&env);
 
 
        lu_env_fini(&env);
 


@@ -390,12 +389,24 @@ static int nodemap_idx_nodemap_add_update(const struct lu_nodemap *nodemap,
 
 int nodemap_idx_nodemap_add(const struct lu_nodemap *nodemap)
 {
 
 int nodemap_idx_nodemap_add(const struct lu_nodemap *nodemap)
 {

-       return nodemap_idx_nodemap_add_update(nodemap, NM_ADD);
+       if (nodemap_mgs_ncf == NULL) {
+               CERROR("cannot add nodemap config to non-existing MGS.\n");
+               return -EINVAL;
+       }
+
+       return nodemap_idx_nodemap_add_update(nodemap, nodemap_mgs_ncf->ncf_obj,
+                                             NM_ADD);

 }
 
 int nodemap_idx_nodemap_update(const struct lu_nodemap *nodemap)
 {
 }
 
 int nodemap_idx_nodemap_update(const struct lu_nodemap *nodemap)
 {

-       return nodemap_idx_nodemap_add_update(nodemap, NM_UPDATE);
+       if (nodemap_mgs_ncf == NULL) {
+               CERROR("cannot add nodemap config to non-existing MGS.\n");
+               return -EINVAL;
+       }
+
+       return nodemap_idx_nodemap_add_update(nodemap, nodemap_mgs_ncf->ncf_obj,
+                                             NM_UPDATE);

 }
 
 int nodemap_idx_nodemap_del(const struct lu_nodemap *nodemap)
 }
 
 int nodemap_idx_nodemap_del(const struct lu_nodemap *nodemap)


@@ -856,6 +867,7 @@ out:
                } else {
                        rc = nodemap_idx_nodemap_add_update(
                                        new_config->nmc_default_nodemap,
                } else {
                        rc = nodemap_idx_nodemap_add_update(
                                        new_config->nmc_default_nodemap,

+                                       nodemap_idx,

                                        NM_ADD);
                        nodemap_putref(new_config->nmc_default_nodemap);
                }
                                        NM_ADD);
                        nodemap_putref(new_config->nmc_default_nodemap);
                }


@@ -882,10 +894,10 @@ out:
 /**
  * Step through active config and write to disk.
  */
 /**
  * Step through active config and write to disk.
  */

-int nodemap_save_config_cache(const struct lu_env *env,
-                             struct nm_config_file *ncf)
+struct dt_object *nodemap_save_config_cache(const struct lu_env *env,
+                                           struct dt_device *dev,
+                                           struct local_oid_storage *los)

 {
 {

-       struct dt_device *dev;

        struct dt_object *o;
        struct lu_nodemap *nodemap;
        struct lu_nodemap *nm_tmp;
        struct dt_object *o;
        struct lu_nodemap *nodemap;
        struct lu_nodemap *nm_tmp;


@@ -901,21 +913,10 @@ int nodemap_save_config_cache(const struct lu_env *env,
 
        ENTRY;
 
 
        ENTRY;
 

-       if (ncf->ncf_los == NULL || ncf->ncf_obj == NULL)
-               RETURN(-EIO);
-
-       dev = lu2dt_dev(ncf->ncf_obj->do_lu.lo_dev);
-
-       /* nodemap_cache_find_create will delete old conf file, so put here */
-       lu_object_put_nocache(env, &ncf->ncf_obj->do_lu);
-       ncf->ncf_obj = NULL;
-
-       /* force create a new index file to fill with active config */
-       o = nodemap_cache_find_create(env, dev, ncf->ncf_los, true);
+       /* create a new index file to fill with active config */
+       o = nodemap_cache_find_create(env, dev, los, NCFC_CREATE_NEW);

        if (IS_ERR(o))
        if (IS_ERR(o))

-               GOTO(out, rc = PTR_ERR(o));
-
-       ncf->ncf_obj = o;
+               GOTO(out, o);

 
        mutex_lock(&active_config_lock);
 
 
        mutex_lock(&active_config_lock);
 


@@ -982,7 +983,13 @@ int nodemap_save_config_cache(const struct lu_env *env,
 
 out:
        mutex_unlock(&active_config_lock);
 
 out:
        mutex_unlock(&active_config_lock);

-       RETURN(rc);
+
+       if (rc < 0) {
+               lu_object_put(env, &o->do_lu);
+               o = ERR_PTR(rc);
+       }
+
+       RETURN(o);

 }
 
 static void nodemap_save_all_caches(void)
 }
 
 static void nodemap_save_all_caches(void)


@@ -1000,10 +1007,20 @@ static void nodemap_save_all_caches(void)
 
        mutex_lock(&ncf_list_lock);
        list_for_each_entry(ncf, &ncf_list_head, ncf_list) {
 
        mutex_lock(&ncf_list_lock);
        list_for_each_entry(ncf, &ncf_list_head, ncf_list) {

-               rc = nodemap_save_config_cache(&env, ncf);
-               if (rc < 0 && ncf->ncf_obj != NULL)
+               struct dt_device *dev = lu2dt_dev(ncf->ncf_obj->do_lu.lo_dev);
+               struct obd_device *obd = ncf->ncf_obj->do_lu.lo_dev->ld_obd;
+               struct dt_object *o;
+
+               /* put current config file so save conf can rewrite it */
+               lu_object_put_nocache(&env, &ncf->ncf_obj->do_lu);
+               ncf->ncf_obj = NULL;
+
+               o = nodemap_save_config_cache(&env, dev, ncf->ncf_los);
+               if (IS_ERR(o))

                        CWARN("%s: error writing to nodemap config: rc = %d\n",
                        CWARN("%s: error writing to nodemap config: rc = %d\n",

-                             ncf->ncf_obj->do_lu.lo_dev->ld_obd->obd_name, rc);
+                             obd->obd_name, rc);
+               else
+                       ncf->ncf_obj = o;

        }
        mutex_unlock(&ncf_list_lock);
 
        }
        mutex_unlock(&ncf_list_lock);
 


@@ -1042,90 +1059,142 @@ EXPORT_SYMBOL(nodemap_config_set_active_mgc);
  * \retval     -ENOMEM         memory allocation failure
  * \retval     -ENOENT         error loading nodemap config
  * \retval     -EINVAL         error loading nodemap config
  * \retval     -ENOMEM         memory allocation failure
  * \retval     -ENOENT         error loading nodemap config
  * \retval     -EINVAL         error loading nodemap config

+ * \retval     -EEXIST         nodemap config already registered for MGS

  */
  */

-struct nm_config_file *nm_config_file_register(const struct lu_env *env,
-                                              struct dt_object *obj,
-                                              struct local_oid_storage *los,
-                                              enum nm_config_file_type ncf_type)
+struct nm_config_file *nm_config_file_register_mgs(const struct lu_env *env,
+                                                  struct dt_object *obj,
+                                                  struct local_oid_storage *los)

 {
        struct nm_config_file *ncf;
 {
        struct nm_config_file *ncf;

-       bool save_config = false;

        int rc = 0;
        ENTRY;
 
        int rc = 0;
        ENTRY;
 

+       if (nodemap_mgs_ncf != NULL)
+               GOTO(out, ncf = ERR_PTR(-EEXIST));
+

        OBD_ALLOC_PTR(ncf);
        if (ncf == NULL)
        OBD_ALLOC_PTR(ncf);
        if (ncf == NULL)

-               RETURN(ERR_PTR(-ENOMEM));
+               GOTO(out, ncf = ERR_PTR(-ENOMEM));
+
+       /* if loading from cache, prevent activation of MGS config until cache
+        * loading is done, so disk config is overwritten by MGS config.
+        */
+       mutex_lock(&nodemap_config_loaded_lock);
+       rc = nodemap_load_entries(env, obj);
+       if (!rc)
+               nodemap_config_loaded = true;
+       mutex_unlock(&nodemap_config_loaded_lock);
+
+       if (rc) {
+               OBD_FREE_PTR(ncf);
+               GOTO(out, ncf = ERR_PTR(rc));
+       }
+
+       lu_object_get(&obj->do_lu);

 
        ncf->ncf_obj = obj;
        ncf->ncf_los = los;
 
 
        ncf->ncf_obj = obj;
        ncf->ncf_los = los;
 

-       if (ncf_type == NCFT_MGS) {
-               nodemap_mgs_ncf = ncf;
-       } else {
-               mutex_lock(&ncf_list_lock);
-               list_add(&ncf->ncf_list, &ncf_list_head);
-               mutex_unlock(&ncf_list_lock);
-       }
+       nodemap_mgs_ncf = ncf;

 
 

-       /* prevent activation of config loaded from MGS until disk is loaded
-        * so disk config is overwritten by MGS config.
-        */
+out:
+       return ncf;
+}
+EXPORT_SYMBOL(nm_config_file_register_mgs);
+
+struct nm_config_file *nm_config_file_register_tgt(const struct lu_env *env,
+                                                  struct dt_device *dev,
+                                                  struct local_oid_storage *los)
+{
+       struct nm_config_file *ncf;
+       struct dt_object *config_obj = NULL;
+       int rc = 0;
+
+       OBD_ALLOC_PTR(ncf);
+       if (ncf == NULL)
+               RETURN(ERR_PTR(-ENOMEM));
+
+       /* don't load from cache if config already loaded */

        mutex_lock(&nodemap_config_loaded_lock);
        mutex_lock(&nodemap_config_loaded_lock);

-       if (ncf_type == NCFT_MGS || !nodemap_config_loaded)
-               rc = nodemap_load_entries(env, obj);
-       else
-               save_config = true;
-       nodemap_config_loaded = true;
+       if (!nodemap_config_loaded) {
+               config_obj = nodemap_cache_find_create(env, dev, los, 0);
+               if (IS_ERR(config_obj))
+                       rc = PTR_ERR(config_obj);
+               else
+                       rc = nodemap_load_entries(env, config_obj);
+
+               if (!rc)
+                       nodemap_config_loaded = true;
+       }

        mutex_unlock(&nodemap_config_loaded_lock);
        mutex_unlock(&nodemap_config_loaded_lock);

+       if (rc)
+               GOTO(out_ncf, rc);

 
 

-       /* sync on disk caches with loaded config in memory */
-       if (save_config)
-               rc = nodemap_save_config_cache(env, ncf);
+       /* sync on disk caches w/ loaded config in memory, ncf_obj may change */
+       if (!config_obj) {
+               config_obj = nodemap_save_config_cache(env, dev, los);
+               if (IS_ERR(config_obj))
+                       GOTO(out_ncf, rc = PTR_ERR(config_obj));
+       }

 
 

-       if (rc < 0) {
-               if (ncf_type == NCFT_MGS) {
-                       nodemap_mgs_ncf = NULL;
-               } else {
-                       mutex_lock(&ncf_list_lock);
-                       list_del(&ncf->ncf_list);
-                       mutex_unlock(&ncf_list_lock);
-               }
+       ncf->ncf_obj = config_obj;
+       ncf->ncf_los = los;

 
 

+       mutex_lock(&ncf_list_lock);
+       list_add(&ncf->ncf_list, &ncf_list_head);
+       mutex_unlock(&ncf_list_lock);
+
+out_ncf:
+       if (rc) {

                OBD_FREE_PTR(ncf);
                RETURN(ERR_PTR(rc));
        }
 
        RETURN(ncf);
 }
                OBD_FREE_PTR(ncf);
                RETURN(ERR_PTR(rc));
        }
 
        RETURN(ncf);
 }

-EXPORT_SYMBOL(nm_config_file_register);
+EXPORT_SYMBOL(nm_config_file_register_tgt);

 
 /**
  * Deregister a nm_config_file. Should be called by targets during cleanup.
  *
  * \param ncf  config file to deregister
  */
 
 /**
  * Deregister a nm_config_file. Should be called by targets during cleanup.
  *
  * \param ncf  config file to deregister
  */

-void nm_config_file_deregister(const struct lu_env *env,
-                              struct nm_config_file *ncf,
-                              enum nm_config_file_type ncf_type)
+void nm_config_file_deregister_mgs(const struct lu_env *env,
+                                  struct nm_config_file *ncf)

 {
        ENTRY;
 {
        ENTRY;

+       LASSERT(nodemap_mgs_ncf == ncf);
+
+       nodemap_mgs_ncf = NULL;
+       if (ncf->ncf_obj)
+               lu_object_put(env, &ncf->ncf_obj->do_lu);
+
+       OBD_FREE_PTR(ncf);
+
+       EXIT;
+}
+EXPORT_SYMBOL(nm_config_file_deregister_mgs);
+
+void nm_config_file_deregister_tgt(const struct lu_env *env,
+                                  struct nm_config_file *ncf)
+{
+       ENTRY;
+
+       if (ncf == NULL)
+               return;
+
+       mutex_lock(&ncf_list_lock);
+       list_del(&ncf->ncf_list);
+       mutex_unlock(&ncf_list_lock);

 
        if (ncf->ncf_obj)
                lu_object_put(env, &ncf->ncf_obj->do_lu);
 
 
        if (ncf->ncf_obj)
                lu_object_put(env, &ncf->ncf_obj->do_lu);
 

-       if (ncf_type == NCFT_TGT) {
-               mutex_lock(&ncf_list_lock);
-               list_del(&ncf->ncf_list);
-               mutex_unlock(&ncf_list_lock);
-       } else {
-               nodemap_mgs_ncf = NULL;
-       }

        OBD_FREE_PTR(ncf);
 
        EXIT;
 }
        OBD_FREE_PTR(ncf);
 
        EXIT;
 }

-EXPORT_SYMBOL(nm_config_file_deregister);
+EXPORT_SYMBOL(nm_config_file_deregister_tgt);

 
 int nodemap_process_idx_pages(struct nodemap_config *config, union lu_page *lip,
                              struct lu_nodemap **recent_nodemap)
 
 int nodemap_process_idx_pages(struct nodemap_config *config, union lu_page *lip,
                              struct lu_nodemap **recent_nodemap)


@@ -1299,51 +1368,3 @@ out:
        return rc;
 }
 EXPORT_SYMBOL(nodemap_get_config_req);
        return rc;
 }
 EXPORT_SYMBOL(nodemap_get_config_req);

-
-int nodemap_fs_init(const struct lu_env *env, struct dt_device *dev,
-                   struct obd_device *obd, struct local_oid_storage *los)
-{
-       struct dt_object        *config_obj;
-       struct nm_config_file   *nm_config_file;
-       int                      rc = 0;
-       ENTRY;
-
-       CDEBUG(D_INFO, "%s: finding nodemap index\n", obd->obd_name);
-       /* load or create the index file from disk (don't force create) */
-       config_obj = nodemap_cache_find_create(env, dev, los, false);
-       if (IS_ERR(config_obj))
-               GOTO(out, rc = PTR_ERR(config_obj));
-
-       CDEBUG(D_INFO, "%s: registering nodemap index\n", obd->obd_name);
-
-       nm_config_file = nm_config_file_register(env, config_obj, los,
-                                                NCFT_TGT);
-       if (IS_ERR(nm_config_file)) {
-               CERROR("%s: error loading nodemap config file, file must be "
-                      "removed via ldiskfs: rc = %ld\n",
-                      obd->obd_name, PTR_ERR(nm_config_file));
-               GOTO(out, rc = PTR_ERR(nm_config_file));
-       }
-
-       obd->u.obt.obt_nodemap_config_file = nm_config_file;
-
-       /* save los in case object needs to be re-created */
-       nm_config_file->ncf_los = los;
-
-       EXIT;
-
-out:
-       return rc;
-}
-EXPORT_SYMBOL(nodemap_fs_init);
-
-void nodemap_fs_fini(const struct lu_env *env, struct obd_device *obd)
-{
-       if (obd->u.obt.obt_nodemap_config_file == NULL)
-               return;
-
-       nm_config_file_deregister(env, obd->u.obt.obt_nodemap_config_file,
-                                 NCFT_TGT);
-       obd->u.obt.obt_nodemap_config_file = NULL;
-}
-EXPORT_SYMBOL(nodemap_fs_fini);