From: Kit Westneat Date: Mon, 22 Feb 2016 19:27:29 +0000 (-0500) Subject: LU-5092 nodemap: remove nodemap_idx_action, only act on MGS X-Git-Tag: 2.8.56~126 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=ffe234a077576f0f66f6fe9c165be180133be89a LU-5092 nodemap: remove nodemap_idx_action, only act on MGS Only the MGS config needs to be modified on config changes. The target configs will be loaded from the MGS and written whole to disk. Signed-off-by: Kit Westneat Change-Id: I3114258d6e18d293cdf29ae47a28465f245387de Reviewed-on: http://review.whamcloud.com/18554 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin --- diff --git a/lustre/include/lustre_nodemap.h b/lustre/include/lustre_nodemap.h index 3f15a3c..db10bda 100644 --- a/lustre/include/lustre_nodemap.h +++ b/lustre/include/lustre_nodemap.h @@ -103,8 +103,9 @@ struct lu_nodemap { * not be needed. */ struct nm_config_file { - struct dt_object *ncf_obj; - struct list_head ncf_list; + struct local_oid_storage *ncf_los; + struct dt_object *ncf_obj; + struct list_head ncf_list; }; void nodemap_activate(const bool value); @@ -139,10 +140,18 @@ void nodemap_test_nid(lnet_nid_t nid, char *name_buf, size_t name_len); #endif __u32 nodemap_test_id(lnet_nid_t nid, enum nodemap_id_type idtype, __u32 client_id); + +enum nm_config_file_type { + NCFT_MGS, + NCFT_TGT, +}; struct nm_config_file *nm_config_file_register(const struct lu_env *env, - struct dt_object *obj); + struct dt_object *obj, + struct local_oid_storage *los, + enum nm_config_file_type ncf_type); void nm_config_file_deregister(const struct lu_env *env, - struct nm_config_file *ncf); + struct nm_config_file *ncf, + enum nm_config_file_type ncf_type); struct lu_nodemap *nodemap_get_from_exp(struct obd_export *exp); void nodemap_putref(struct lu_nodemap *nodemap); @@ -177,7 +186,7 @@ struct nodemap_config { struct nodemap_config *nodemap_config_alloc(void); void nodemap_config_dealloc(struct nodemap_config *config); -void nodemap_config_set_active(struct nodemap_config *config); +void nodemap_config_set_active_mgc(struct nodemap_config *config); int nodemap_process_idx_pages(struct nodemap_config *config, union lu_page *lip, struct lu_nodemap **recent_nodemap); diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c index 7b4db0c..0d568d3 100644 --- a/lustre/mgc/mgc_request.c +++ b/lustre/mgc/mgc_request.c @@ -1770,7 +1770,7 @@ out: if (new_config != NULL) { /* recent_nodemap cannot be used after set_active/dealloc */ if (rc == 0) - nodemap_config_set_active(new_config); + nodemap_config_set_active_mgc(new_config); else nodemap_config_dealloc(new_config); } diff --git a/lustre/mgs/mgs_fs.c b/lustre/mgs/mgs_fs.c index a3ce58fc..338858e 100644 --- a/lustre/mgs/mgs_fs.c +++ b/lustre/mgs/mgs_fs.c @@ -163,6 +163,7 @@ int mgs_fs_setup(const struct lu_env *env, struct mgs_device *mgs) mgs->mgs_configs_dir = o; + /* colocated MDT will cache config in target / dir */ nm_config_file_obj = local_index_find_or_create(env, mgs->mgs_los, mgs->mgs_configs_dir, LUSTRE_NODEMAP_NAME, @@ -181,7 +182,8 @@ int mgs_fs_setup(const struct lu_env *env, struct mgs_device *mgs) GOTO(out_configs, rc); } } - nm_config_file = nm_config_file_register(env, nm_config_file_obj); + nm_config_file = nm_config_file_register(env, nm_config_file_obj, + mgs->mgs_los, NCFT_MGS); if (IS_ERR(nm_config_file)) { lu_object_put(env, &nm_config_file_obj->do_lu); CERROR("%s: error loading nodemap config file, file must be " @@ -201,7 +203,7 @@ int mgs_fs_setup(const struct lu_env *env, struct mgs_device *mgs) out_nm: if (rc < 0) { - nm_config_file_deregister(env, nm_config_file); + nm_config_file_deregister(env, nm_config_file, NCFT_MGS); mgs->mgs_obd->u.obt.obt_nodemap_config_file = NULL; } out_configs: @@ -234,7 +236,8 @@ int mgs_fs_cleanup(const struct lu_env *env, struct mgs_device *mgs) } if (mgs->mgs_obd->u.obt.obt_nodemap_config_file != NULL) { nm_config_file_deregister(env, - mgs->mgs_obd->u.obt.obt_nodemap_config_file); + mgs->mgs_obd->u.obt.obt_nodemap_config_file, + NCFT_MGS); mgs->mgs_obd->u.obt.obt_nodemap_config_file = NULL; } diff --git a/lustre/ptlrpc/nodemap_handler.c b/lustre/ptlrpc/nodemap_handler.c index 10bbc49..1c99301 100644 --- a/lustre/ptlrpc/nodemap_handler.c +++ b/lustre/ptlrpc/nodemap_handler.c @@ -70,15 +70,16 @@ static void nodemap_destroy(struct lu_nodemap *nodemap) down_read(&active_config->nmc_range_tree_lock); nm_member_reclassify_nodemap(nodemap); up_read(&active_config->nmc_range_tree_lock); - mutex_unlock(&active_config_lock); - - if (!list_empty(&nodemap->nm_member_list)) - CWARN("nodemap_destroy failed to reclassify all members\n"); write_lock(&nodemap->nm_idmap_lock); idmap_delete_tree(nodemap); write_unlock(&nodemap->nm_idmap_lock); + mutex_unlock(&active_config_lock); + + if (!list_empty(&nodemap->nm_member_list)) + CWARN("nodemap_destroy failed to reclassify all members\n"); + nm_member_delete_list(nodemap); OBD_FREE_PTR(nodemap); @@ -264,6 +265,7 @@ struct lu_nodemap *nodemap_classify_nid(lnet_nid_t nid) else nodemap = active_config->nmc_default_nodemap; + LASSERT(nodemap != NULL); nodemap_getref(nodemap); return nodemap; @@ -361,6 +363,7 @@ int nodemap_add_member(lnet_nid_t nid, struct obd_export *exp) { struct lu_nodemap *nodemap; int rc; + ENTRY; mutex_lock(&active_config_lock); down_read(&active_config->nmc_range_tree_lock); @@ -373,7 +376,7 @@ int nodemap_add_member(lnet_nid_t nid, struct obd_export *exp) nodemap_putref(nodemap); - return rc; + RETURN(rc); } EXPORT_SYMBOL(nodemap_add_member); @@ -1352,6 +1355,7 @@ void nodemap_config_set_active(struct nodemap_config *config) ENTRY; LASSERT(active_config != config); + LASSERT(config->nmc_default_nodemap); mutex_lock(&active_config_lock); @@ -1392,7 +1396,6 @@ void nodemap_config_set_active(struct nodemap_config *config) EXIT; } -EXPORT_SYMBOL(nodemap_config_set_active); /** * Cleanup nodemap module on exit diff --git a/lustre/ptlrpc/nodemap_internal.h b/lustre/ptlrpc/nodemap_internal.h index 0676aa7..f447eef 100644 --- a/lustre/ptlrpc/nodemap_internal.h +++ b/lustre/ptlrpc/nodemap_internal.h @@ -110,6 +110,7 @@ static inline __u32 nm_idx_set_type(unsigned int id, enum nodemap_idx_type t) return (id & NM_TYPE_MASK) | (t << NM_TYPE_SHIFT); } +void nodemap_config_set_active(struct nodemap_config *config); struct lu_nodemap *nodemap_create(const char *name, struct nodemap_config *config, bool is_default); diff --git a/lustre/ptlrpc/nodemap_storage.c b/lustre/ptlrpc/nodemap_storage.c index ef81a47..b117120 100644 --- a/lustre/ptlrpc/nodemap_storage.c +++ b/lustre/ptlrpc/nodemap_storage.c @@ -60,10 +60,13 @@ #include #include "nodemap_internal.h" -/* list of registered nodemap index files */ +/* list of registered nodemap index files, except MGS */ static LIST_HEAD(ncf_list_head); static DEFINE_MUTEX(ncf_list_lock); +/* MGS index is different than others, others are listeners to MGS idx */ +static struct nm_config_file *nodemap_mgs_ncf; + /* lu_nodemap flags */ enum nm_flag_shifts { NM_FL_ALLOW_ROOT_ACCESS = 0x1, @@ -145,7 +148,7 @@ static void nodemap_inc_version(const struct lu_env *env, dt_version_set(env, nodemap_idx, ver + 1, th); } -static int nodemap_idx_insert(struct lu_env *env, +static int nodemap_idx_insert(const struct lu_env *env, struct dt_object *idx, const struct nodemap_key *nk, const union nodemap_rec *nr) @@ -188,7 +191,7 @@ out: return rc; } -static int nodemap_idx_update(struct lu_env *env, +static int nodemap_idx_update(const struct lu_env *env, struct dt_object *idx, const struct nodemap_key *nk, const union nodemap_rec *nr) @@ -239,7 +242,7 @@ out: return rc; } -static int nodemap_idx_delete(struct lu_env *env, +static int nodemap_idx_delete(const struct lu_env *env, struct dt_object *idx, const struct nodemap_key *nk, const union nodemap_rec *unused) @@ -278,46 +281,6 @@ out: return rc; } -typedef int (*nm_idx_cb_t)(struct lu_env *env, - struct dt_object *idx, - const struct nodemap_key *nk, - const union nodemap_rec *nr); - -/** - * Iterates through all the registered nodemap_config_files and calls the - * given callback with the ncf as a parameter, as well as the given key and rec. - * - * \param cb_f callback function to call - * \param nk key of the record to act upon - * \param nr record to act upon, NULL for the delete action - */ -static int nodemap_idx_action(nm_idx_cb_t cb_f, struct nodemap_key *nk, - union nodemap_rec *nr) -{ - struct nm_config_file *ncf; - struct lu_env env; - int rc = 0; - int rc2 = 0; - - rc = lu_env_init(&env, LCT_LOCAL); - if (rc != 0) - return rc; - - mutex_lock(&ncf_list_lock); - list_for_each_entry(ncf, &ncf_list_head, ncf_list) { - rc2 = cb_f(&env, ncf->ncf_obj, nk, nr); - if (rc2 < 0) { - CWARN("%s: error writing to nodemap config: rc = %d\n", - ncf->ncf_obj->do_lu.lo_dev->ld_obd->obd_name, rc); - rc = rc2; - } - } - mutex_unlock(&ncf_list_lock); - lu_env_fini(&env); - - return 0; -} - enum nm_add_update { NM_ADD = 0, NM_UPDATE = 1, @@ -326,19 +289,33 @@ enum nm_add_update { static int nodemap_idx_nodemap_add_update(const struct lu_nodemap *nodemap, enum nm_add_update update) { - struct nodemap_key nk; - union nodemap_rec nr; + struct nodemap_key nk; + union nodemap_rec nr; + struct lu_env env; int rc = 0; ENTRY; + if (nodemap_mgs_ncf == NULL) { + CERROR("cannot add nodemap config to non-existing MGS.\n"); + return -EINVAL; + } + + rc = lu_env_init(&env, LCT_LOCAL); + if (rc) + RETURN(rc); + nodemap_cluster_key_init(&nk, nodemap->nm_id); nodemap_cluster_rec_init(&nr, nodemap); if (update == NM_UPDATE) - rc = nodemap_idx_action(nodemap_idx_update, &nk, &nr); + rc = nodemap_idx_update(&env, nodemap_mgs_ncf->ncf_obj, + &nk, &nr); else - rc = nodemap_idx_action(nodemap_idx_insert, &nk, &nr); + rc = nodemap_idx_insert(&env, nodemap_mgs_ncf->ncf_obj, + &nk, &nr); + + lu_env_fini(&env); RETURN(rc); } @@ -361,17 +338,28 @@ int nodemap_idx_nodemap_del(const struct lu_nodemap *nodemap) struct lu_nid_range *range; struct lu_nid_range *range_temp; struct nodemap_key nk; + struct lu_env env; int rc = 0; int rc2 = 0; ENTRY; + if (nodemap_mgs_ncf == NULL) { + CERROR("cannot add nodemap config to non-existing MGS.\n"); + return -EINVAL; + } + + rc = lu_env_init(&env, LCT_LOCAL); + if (rc != 0) + RETURN(rc); + root = nodemap->nm_fs_to_client_uidmap; nm_rbtree_postorder_for_each_entry_safe(idmap, temp, &root, id_fs_to_client) { nodemap_idmap_key_init(&nk, nodemap->nm_id, NODEMAP_UID, idmap->id_client); - rc2 = nodemap_idx_action(nodemap_idx_delete, &nk, NULL); + rc2 = nodemap_idx_delete(&env, nodemap_mgs_ncf->ncf_obj, + &nk, NULL); if (rc2 < 0) rc = rc2; } @@ -381,7 +369,8 @@ int nodemap_idx_nodemap_del(const struct lu_nodemap *nodemap) id_client_to_fs) { nodemap_idmap_key_init(&nk, nodemap->nm_id, NODEMAP_GID, idmap->id_client); - rc2 = nodemap_idx_action(nodemap_idx_delete, &nk, NULL); + rc2 = nodemap_idx_delete(&env, nodemap_mgs_ncf->ncf_obj, + &nk, NULL); if (rc2 < 0) rc = rc2; } @@ -389,16 +378,19 @@ int nodemap_idx_nodemap_del(const struct lu_nodemap *nodemap) list_for_each_entry_safe(range, range_temp, &nodemap->nm_ranges, rn_list) { nodemap_range_key_init(&nk, nodemap->nm_id, range->rn_id); - rc2 = nodemap_idx_action(nodemap_idx_delete, &nk, NULL); + rc2 = nodemap_idx_delete(&env, nodemap_mgs_ncf->ncf_obj, + &nk, NULL); if (rc2 < 0) rc = rc2; } nodemap_cluster_key_init(&nk, nodemap->nm_id); - rc2 = nodemap_idx_action(nodemap_idx_delete, &nk, NULL); + rc2 = nodemap_idx_delete(&env, nodemap_mgs_ncf->ncf_obj, &nk, NULL); if (rc2 < 0) rc = rc2; + lu_env_fini(&env); + RETURN(rc); } @@ -407,22 +399,50 @@ int nodemap_idx_range_add(const struct lu_nid_range *range, { struct nodemap_key nk; union nodemap_rec nr; + struct lu_env env; + int rc = 0; ENTRY; + if (nodemap_mgs_ncf == NULL) { + CERROR("cannot add nodemap config to non-existing MGS.\n"); + return -EINVAL; + } + + rc = lu_env_init(&env, LCT_LOCAL); + if (rc != 0) + RETURN(rc); + nodemap_range_key_init(&nk, range->rn_nodemap->nm_id, range->rn_id); nodemap_range_rec_init(&nr, nid); - RETURN(nodemap_idx_action(nodemap_idx_insert, &nk, &nr)); + rc = nodemap_idx_insert(&env, nodemap_mgs_ncf->ncf_obj, &nk, &nr); + lu_env_fini(&env); + + RETURN(rc); } int nodemap_idx_range_del(const struct lu_nid_range *range) { struct nodemap_key nk; + struct lu_env env; + int rc = 0; ENTRY; + if (nodemap_mgs_ncf == NULL) { + CERROR("cannot add nodemap config to non-existing MGS.\n"); + return -EINVAL; + } + + rc = lu_env_init(&env, LCT_LOCAL); + if (rc != 0) + RETURN(rc); + nodemap_range_key_init(&nk, range->rn_nodemap->nm_id, range->rn_id); - RETURN(nodemap_idx_action(nodemap_idx_delete, &nk, NULL)); + rc = nodemap_idx_delete(&env, nodemap_mgs_ncf->ncf_obj, &nk, NULL); + lu_env_fini(&env); + + RETURN(rc); } int nodemap_idx_idmap_add(const struct lu_nodemap *nodemap, @@ -431,12 +451,26 @@ int nodemap_idx_idmap_add(const struct lu_nodemap *nodemap, { struct nodemap_key nk; union nodemap_rec nr; + struct lu_env env; + int rc = 0; ENTRY; + if (nodemap_mgs_ncf == NULL) { + CERROR("cannot add nodemap config to non-existing MGS.\n"); + return -EINVAL; + } + + rc = lu_env_init(&env, LCT_LOCAL); + if (rc != 0) + RETURN(rc); + nodemap_idmap_key_init(&nk, nodemap->nm_id, id_type, map[0]); nodemap_idmap_rec_init(&nr, map[1]); - RETURN(nodemap_idx_action(nodemap_idx_insert, &nk, &nr)); + rc = nodemap_idx_insert(&env, nodemap_mgs_ncf->ncf_obj, &nk, &nr); + lu_env_fini(&env); + + RETURN(rc); } int nodemap_idx_idmap_del(const struct lu_nodemap *nodemap, @@ -444,26 +478,57 @@ int nodemap_idx_idmap_del(const struct lu_nodemap *nodemap, const u32 map[2]) { struct nodemap_key nk; + struct lu_env env; + int rc = 0; ENTRY; + if (nodemap_mgs_ncf == NULL) { + CERROR("cannot add nodemap config to non-existing MGS.\n"); + return -EINVAL; + } + + rc = lu_env_init(&env, LCT_LOCAL); + if (rc != 0) + RETURN(rc); + nodemap_idmap_key_init(&nk, nodemap->nm_id, id_type, map[0]); - RETURN(nodemap_idx_action(nodemap_idx_delete, &nk, NULL)); + rc = nodemap_idx_delete(&env, nodemap_mgs_ncf->ncf_obj, &nk, NULL); + lu_env_fini(&env); + + RETURN(rc); } static int nodemap_idx_global_add_update(bool value, enum nm_add_update update) { struct nodemap_key nk; union nodemap_rec nr; + struct lu_env env; + int rc = 0; ENTRY; + if (nodemap_mgs_ncf == NULL) { + CERROR("cannot add nodemap config to non-existing MGS.\n"); + return -EINVAL; + } + + rc = lu_env_init(&env, LCT_LOCAL); + if (rc != 0) + RETURN(rc); + nodemap_global_key_init(&nk); nodemap_global_rec_init(&nr, value); if (update == NM_UPDATE) - RETURN(nodemap_idx_action(nodemap_idx_update, &nk, &nr)); + rc = nodemap_idx_update(&env, nodemap_mgs_ncf->ncf_obj, + &nk, &nr); else - RETURN(nodemap_idx_action(nodemap_idx_insert, &nk, &nr)); + rc = nodemap_idx_insert(&env, nodemap_mgs_ncf->ncf_obj, + &nk, &nr); + + lu_env_fini(&env); + + RETURN(rc); } int nodemap_idx_nodemap_activate(bool value) @@ -615,6 +680,7 @@ static int nodemap_process_keyrec(struct nodemap_config *config, CERROR("got keyrec pair for unknown type %d\n", type); break; } + rc = type; out: @@ -649,7 +715,6 @@ static int nodemap_load_entries(const struct lu_env *env, GOTO(out_iops, rc = 0); } - /* acquires active config lock */ new_config = nodemap_config_alloc(); if (IS_ERR(new_config)) { rc = PTR_ERR(new_config); @@ -716,8 +781,14 @@ out: } /* new nodemap config won't have an active/inactive record */ - if (rc == 0 && loaded_global_idx == false) - rc = nodemap_idx_global_add_update(false, NM_ADD); + if (rc == 0 && loaded_global_idx == false) { + struct nodemap_key nk; + union nodemap_rec nr; + + nodemap_global_key_init(&nk); + nodemap_global_rec_init(&nr, false); + rc = nodemap_idx_insert(env, nodemap_idx, &nk, &nr); + } if (rc == 0) nodemap_config_set_active(new_config); @@ -727,6 +798,25 @@ out: RETURN(rc); } +/* tracks if config still needs to be loaded, either from disk or network */ +static bool nodemap_config_loaded; +static DEFINE_MUTEX(nodemap_config_loaded_lock); + +/** + * Ensures that configs loaded over the wire are prioritized over those loaded + * from disk. + * + * \param config config to set as the active config + */ +void nodemap_config_set_active_mgc(struct nodemap_config *config) +{ + mutex_lock(&nodemap_config_loaded_lock); + nodemap_config_set_active(config); + nodemap_config_loaded = true; + mutex_unlock(&nodemap_config_loaded_lock); +} +EXPORT_SYMBOL(nodemap_config_set_active_mgc); + /** * Register a dt_object representing the config index file. This should be * called by targets in order to load the nodemap configuration from disk. The @@ -741,11 +831,12 @@ out: * \retval -EINVAL error loading nodemap config */ struct nm_config_file *nm_config_file_register(const struct lu_env *env, - struct dt_object *obj) + struct dt_object *obj, + struct local_oid_storage *los, + enum nm_config_file_type ncf_type) { struct nm_config_file *ncf; - bool load_entries = false; - int rc; + int rc = 0; ENTRY; OBD_ALLOC_PTR(ncf); @@ -753,24 +844,36 @@ struct nm_config_file *nm_config_file_register(const struct lu_env *env, RETURN(ERR_PTR(-ENOMEM)); ncf->ncf_obj = obj; - mutex_lock(&ncf_list_lock); - - /* if this is first config file, we load it from disk */ - if (list_empty(&ncf_list_head)) - load_entries = true; + ncf->ncf_los = los; - list_add(&ncf->ncf_list, &ncf_list_head); - mutex_unlock(&ncf_list_lock); + if (ncf_type == NCFT_MGS) { + nodemap_mgs_ncf = ncf; + } else { + mutex_lock(&ncf_list_lock); + list_add(&ncf->ncf_list, &ncf_list_head); + mutex_unlock(&ncf_list_lock); + } - if (load_entries) { + /* prevent activation of config loaded from MGS until disk is loaded + * so disk config is overwritten by MGS config. + */ + mutex_lock(&nodemap_config_loaded_lock); + if (ncf_type == NCFT_MGS || !nodemap_config_loaded) rc = nodemap_load_entries(env, obj); - if (rc < 0) { + nodemap_config_loaded = true; + mutex_unlock(&nodemap_config_loaded_lock); + + if (rc < 0) { + if (ncf_type == NCFT_MGS) { + nodemap_mgs_ncf = NULL; + } else { mutex_lock(&ncf_list_lock); list_del(&ncf->ncf_list); mutex_unlock(&ncf_list_lock); - OBD_FREE_PTR(ncf); - RETURN(ERR_PTR(rc)); } + + OBD_FREE_PTR(ncf); + RETURN(ERR_PTR(rc)); } RETURN(ncf); @@ -783,15 +886,21 @@ EXPORT_SYMBOL(nm_config_file_register); * \param ncf config file to deregister */ void nm_config_file_deregister(const struct lu_env *env, - struct nm_config_file *ncf) + struct nm_config_file *ncf, + enum nm_config_file_type ncf_type) { ENTRY; - lu_object_put(env, &ncf->ncf_obj->do_lu); + if (ncf->ncf_obj) + lu_object_put(env, &ncf->ncf_obj->do_lu); - mutex_lock(&ncf_list_lock); - list_del(&ncf->ncf_list); - mutex_unlock(&ncf_list_lock); + if (ncf_type == NCFT_TGT) { + mutex_lock(&ncf_list_lock); + list_del(&ncf->ncf_list); + mutex_unlock(&ncf_list_lock); + } else { + nodemap_mgs_ncf = NULL; + } OBD_FREE_PTR(ncf); EXIT; @@ -809,6 +918,7 @@ int nodemap_process_idx_pages(struct nodemap_config *config, union lu_page *lip, int rc = 0; int size = dt_nodemap_features.dif_keysize_max + dt_nodemap_features.dif_recsize_max; + ENTRY; for (j = 0; j < LU_PAGE_COUNT; j++) { if (lip->lp_idx.lip_magic != LIP_MAGIC) @@ -829,6 +939,8 @@ int nodemap_process_idx_pages(struct nodemap_config *config, union lu_page *lip, } lip++; } + + EXIT; return 0; } EXPORT_SYMBOL(nodemap_process_idx_pages);