From: Kit Westneat Date: Thu, 11 Jun 2015 23:03:25 +0000 (-0400) Subject: LU-5092 nodemap: add structure to hold nodemap config X-Git-Tag: 2.7.58~47 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=2e0d4e010eb4c2e28507f55c1b649d70bccd06a0 LU-5092 nodemap: add structure to hold nodemap config This patch moves global state variables into a configuration structure so that new configurations can be more easily loaded and swapped into the active role. Signed-off-by: Kit Westneat Change-Id: Ib0d51d56154d5e831b13f2935feab9bd73944bcc Reviewed-on: http://review.whamcloud.com/14254 Tested-by: Jenkins Reviewed-by: John L. Hammond Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- diff --git a/lustre/include/lustre_nodemap.h b/lustre/include/lustre_nodemap.h index 221f8d7..375154d 100644 --- a/lustre/include/lustre_nodemap.h +++ b/lustre/include/lustre_nodemap.h @@ -47,6 +47,12 @@ enum nodemap_tree_type { NODEMAP_CLIENT_TO_FS, }; +struct nodemap_pde { + char npe_name[LUSTRE_NODEMAP_NAME_LENGTH + 1]; + struct proc_dir_entry *npe_proc_entry; + struct list_head npe_list_member; +}; + /** The nodemap id 0 will be the default nodemap. It will have a configuration * set by the MGS, but no ranges will be allowed as all NIDs that do not map * will be added to the default nodemap @@ -81,16 +87,15 @@ struct lu_nodemap { struct rb_root nm_fs_to_client_gidmap; /* GID map keyed by remote UID */ struct rb_root nm_client_to_fs_gidmap; - /* proc directory entry */ - struct proc_dir_entry *nm_proc_entry; /* attached client members of this nodemap */ struct mutex nm_member_list_lock; struct list_head nm_member_list; /* access by nodemap name */ struct hlist_node nm_hash; + struct nodemap_pde *nm_pde_data; /* used when unloading nodemaps */ - struct list_head nm_list; + struct list_head nm_list; }; void nodemap_activate(const bool value); diff --git a/lustre/ptlrpc/nodemap_handler.c b/lustre/ptlrpc/nodemap_handler.c index 984e45d..5bf4932 100644 --- a/lustre/ptlrpc/nodemap_handler.c +++ b/lustre/ptlrpc/nodemap_handler.c @@ -44,29 +44,15 @@ /* nodemap proc root proc directory under fs/lustre */ struct proc_dir_entry *proc_lustre_nodemap_root; -/* Highest numerical lu_nodemap.nm_id defined */ -static atomic_t nodemap_highest_id; - -/* Simple flag to determine if nodemaps are active */ +/* Copy of config active flag to avoid locking in mapping functions */ bool nodemap_active; -/** - * pointer to default nodemap kept to keep from - * lookup it up in the hash since it is needed - * more often - */ -static struct lu_nodemap *default_nodemap; - -/** - * Lock required to access the range tree. +/* Lock protecting the active config, useful primarily when proc and + * nodemap_hash might be replaced when loading a new config + * Any time the active config is referenced, the lock should be held. */ -rwlock_t nm_range_tree_lock; - -/** - * Hash keyed on nodemap name containing all - * nodemaps - */ -static struct cfs_hash *nodemap_hash; +DEFINE_MUTEX(active_config_lock); +struct nodemap_config *active_config; /** * Nodemap destructor @@ -75,22 +61,21 @@ static struct cfs_hash *nodemap_hash; */ static void nodemap_destroy(struct lu_nodemap *nodemap) { - struct lu_nid_range *range; - struct lu_nid_range *range_temp; + if (nodemap->nm_pde_data != NULL) + lprocfs_nodemap_remove(nodemap->nm_pde_data); - write_lock(&nm_range_tree_lock); - list_for_each_entry_safe(range, range_temp, &nodemap->nm_ranges, - rn_list) { - range_delete(range); - } - write_unlock(&nm_range_tree_lock); + mutex_lock(&active_config_lock); + down_read(&active_config->nmc_range_tree_lock); + nm_member_reclassify_nodemap(nodemap); + up_read(&active_config->nmc_range_tree_lock); + mutex_unlock(&active_config_lock); + + if (!list_empty(&nodemap->nm_member_list)) + CWARN("nodemap_destroy failed to reclassify all members\n"); write_lock(&nodemap->nm_idmap_lock); idmap_delete_tree(nodemap); write_unlock(&nodemap->nm_idmap_lock); - nm_member_reclassify_nodemap(nodemap); - if (!list_empty(&nodemap->nm_member_list)) - CWARN("nodemap_destroy failed to reclassify all members\n"); nm_member_delete_list(nodemap); @@ -105,6 +90,10 @@ static void nodemap_getref(struct lu_nodemap *nodemap) atomic_inc(&nodemap->nm_refcount); } +/** + * Destroy nodemap if last reference is put. Should be called outside + * active_config_lock + */ void nodemap_putref(struct lu_nodemap *nodemap) { LASSERT(nodemap != NULL); @@ -173,64 +162,23 @@ static struct cfs_hash_ops nodemap_hash_operations = { /* end of cfs_hash functions */ /** - * Helper iterator to convert nodemap hash to list. - * - * \param hs hash structure - * \param bd bucket descriptor - * \param hnode hash node - * \param nodemap_list_head list head for list of nodemaps in hash - */ -static int nodemap_cleanup_iter_cb(struct cfs_hash *hs, struct cfs_hash_bd *bd, - struct hlist_node *hnode, - void *nodemap_list_head) -{ - struct lu_nodemap *nodemap; - - nodemap = hlist_entry(hnode, struct lu_nodemap, nm_hash); - list_add(&nodemap->nm_list, (struct list_head *)nodemap_list_head); - - return 0; -} - -/** - * Walk the nodemap_hash and remove all nodemaps. - */ -void nodemap_cleanup_all(void) -{ - struct lu_nodemap *nodemap = NULL; - struct list_head *pos, *next; - struct list_head nodemap_list_head = LIST_HEAD_INIT(nodemap_list_head); - - cfs_hash_for_each_safe(nodemap_hash, nodemap_cleanup_iter_cb, - &nodemap_list_head); - cfs_hash_putref(nodemap_hash); - - /* Because nodemap_destroy might sleep, we can't destroy them - * in cfs_hash_for_each. Instead we build a list and destroy here - */ - list_for_each_safe(pos, next, &nodemap_list_head) { - nodemap = list_entry(pos, struct lu_nodemap, nm_list); - nodemap_putref(nodemap); - } -} - -/** * Initialize nodemap_hash * * \retval 0 success * \retval -ENOMEM cannot create hash */ -static int nodemap_init_hash(void) +static int nodemap_init_hash(struct nodemap_config *nmc) { - nodemap_hash = cfs_hash_create("NODEMAP", HASH_NODEMAP_CUR_BITS, - HASH_NODEMAP_MAX_BITS, - HASH_NODEMAP_BKT_BITS, 0, - CFS_HASH_MIN_THETA, - CFS_HASH_MAX_THETA, - &nodemap_hash_operations, - CFS_HASH_DEFAULT); - - if (nodemap_hash == NULL) { + nmc->nmc_nodemap_hash = cfs_hash_create("NODEMAP", + HASH_NODEMAP_CUR_BITS, + HASH_NODEMAP_MAX_BITS, + HASH_NODEMAP_BKT_BITS, 0, + CFS_HASH_MIN_THETA, + CFS_HASH_MAX_THETA, + &nodemap_hash_operations, + CFS_HASH_DEFAULT); + + if (nmc->nmc_nodemap_hash == NULL) { CERROR("cannot create nodemap_hash table\n"); return -ENOMEM; } @@ -262,32 +210,30 @@ static bool nodemap_name_is_valid(const char *name) /** * Nodemap lookup * - * Look nodemap up in the nodemap hash + * Look nodemap up in the active_config nodemap hash. Caller should hold the + * active_config_lock. * * \param name name of nodemap - * \param nodemap found nodemap or NULL - * \retval lu_nodemap named nodemap - * \retval NULL nodemap doesn't exist + * \retval nodemap pointer set to found nodemap + * \retval -EINVAL name is not valid + * \retval -ENOENT nodemap not found */ -static int nodemap_lookup(const char *name, struct lu_nodemap **nodemap) +struct lu_nodemap *nodemap_lookup(const char *name) { - int rc = 0; - - *nodemap = NULL; + struct lu_nodemap *nodemap = NULL; if (!nodemap_name_is_valid(name)) - GOTO(out, rc = -EINVAL); + return ERR_PTR(-EINVAL); - *nodemap = cfs_hash_lookup(nodemap_hash, name); - if (*nodemap == NULL) - rc = -ENOENT; + nodemap = cfs_hash_lookup(active_config->nmc_nodemap_hash, name); + if (nodemap == NULL) + return ERR_PTR(-ENOENT); -out: - return rc; + return nodemap; } /** - * Classify the nid into the proper nodemap. Caller must hold + * Classify the nid into the proper nodemap. Caller must hold active config and * nm_range_tree_lock, and call nodemap_putref when done with nodemap. * * \param nid nid to classify @@ -299,11 +245,12 @@ struct lu_nodemap *nodemap_classify_nid(lnet_nid_t nid) struct lu_nid_range *range; struct lu_nodemap *nodemap; - range = range_search(nid); + range = range_search(&active_config->nmc_range_tree, nid); if (range != NULL) nodemap = range->rn_nodemap; else - nodemap = default_nodemap; + nodemap = active_config->nmc_default_nodemap; + nodemap_getref(nodemap); return nodemap; @@ -402,11 +349,17 @@ int nodemap_add_member(lnet_nid_t nid, struct obd_export *exp) struct lu_nodemap *nodemap; int rc; - read_lock(&nm_range_tree_lock); + mutex_lock(&active_config_lock); + down_read(&active_config->nmc_range_tree_lock); + nodemap = nodemap_classify_nid(nid); rc = nm_member_add(nodemap, exp); - read_unlock(&nm_range_tree_lock); + + up_read(&active_config->nmc_range_tree_lock); + mutex_unlock(&active_config_lock); + nodemap_putref(nodemap); + return rc; } EXPORT_SYMBOL(nodemap_add_member); @@ -436,28 +389,47 @@ EXPORT_SYMBOL(nodemap_del_member); * * \retval 0 on success */ -int nodemap_add_idmap(const char *name, enum nodemap_id_type id_type, - const __u32 map[2]) +static int nodemap_add_idmap_helper(struct lu_nodemap *nodemap, + enum nodemap_id_type id_type, + const __u32 map[2]) { - struct lu_nodemap *nodemap = NULL; struct lu_idmap *idmap; int rc = 0; - rc = nodemap_lookup(name, &nodemap); - if (nodemap == NULL || is_default_nodemap(nodemap)) - GOTO(out, rc = -EINVAL); - idmap = idmap_create(map[0], map[1]); if (idmap == NULL) - GOTO(out_putref, rc = -ENOMEM); + GOTO(out, rc = -ENOMEM); write_lock(&nodemap->nm_idmap_lock); idmap_insert(id_type, idmap, nodemap); write_unlock(&nodemap->nm_idmap_lock); nm_member_revoke_locks(nodemap); -out_putref: +out: + return rc; +} +int nodemap_add_idmap(const char *name, enum nodemap_id_type id_type, + const __u32 map[2]) +{ + struct lu_nodemap *nodemap = NULL; + int rc; + + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(name); + if (IS_ERR(nodemap)) { + mutex_unlock(&active_config_lock); + GOTO(out, rc = PTR_ERR(nodemap)); + } + + if (is_default_nodemap(nodemap)) + rc = -EINVAL; + else + rc = nodemap_add_idmap_helper(nodemap, id_type, map); + + mutex_unlock(&active_config_lock); + nodemap_putref(nodemap); + out: return rc; } @@ -481,24 +453,31 @@ int nodemap_del_idmap(const char *name, enum nodemap_id_type id_type, struct lu_idmap *idmap = NULL; int rc = 0; - rc = nodemap_lookup(name, &nodemap); - if (nodemap == NULL || is_default_nodemap(nodemap)) - GOTO(out, rc = -EINVAL); + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(name); + if (IS_ERR(nodemap)) { + mutex_unlock(&active_config_lock); + GOTO(out, rc = PTR_ERR(nodemap)); + } + + if (is_default_nodemap(nodemap)) + GOTO(out_putref, rc = -EINVAL); write_lock(&nodemap->nm_idmap_lock); idmap = idmap_search(nodemap, NODEMAP_CLIENT_TO_FS, id_type, map[0]); - if (idmap == NULL) { - write_unlock(&nodemap->nm_idmap_lock); - GOTO(out_putref, rc = -EINVAL); - } - - idmap_delete(id_type, idmap, nodemap); + if (idmap == NULL) + rc = -EINVAL; + else + idmap_delete(id_type, idmap, nodemap); write_unlock(&nodemap->nm_idmap_lock); - nm_member_revoke_locks(nodemap); out_putref: + mutex_unlock(&active_config_lock); + if (rc == 0) + nm_member_revoke_locks(nodemap); nodemap_putref(nodemap); + out: return rc; } @@ -645,46 +624,65 @@ EXPORT_SYMBOL(nodemap_map_acl); /* * add nid range to nodemap - * \param name nodemap name + * \param nodemap nodemap to add range to * \param range_st string containing nid range * \retval 0 on success * * add an range to the global range tree and attached the * range to the named nodemap. */ -int nodemap_add_range(const char *name, const lnet_nid_t nid[2]) +static int nodemap_add_range_helper(struct nodemap_config *config, + struct lu_nodemap *nodemap, + const lnet_nid_t nid[2]) { - struct lu_nodemap *nodemap = NULL; struct lu_nid_range *range; int rc; - rc = nodemap_lookup(name, &nodemap); - if (nodemap == NULL || is_default_nodemap(nodemap)) - GOTO(out, rc = -EINVAL); - - range = range_create(nid[0], nid[1], nodemap); - if (range == NULL) - GOTO(out_putref, rc = -ENOMEM); + down_write(&config->nmc_range_tree_lock); + range = range_create(&config->nmc_range_tree, nid[0], nid[1], nodemap); + if (range == NULL) { + up_write(&config->nmc_range_tree_lock); + GOTO(out, rc = -ENOMEM); + } - write_lock(&nm_range_tree_lock); - rc = range_insert(range); + rc = range_insert(&config->nmc_range_tree, range); if (rc != 0) { CERROR("cannot insert nodemap range into '%s': rc = %d\n", nodemap->nm_name, rc); - write_unlock(&nm_range_tree_lock); + up_write(&config->nmc_range_tree_lock); list_del(&range->rn_list); range_destroy(range); - GOTO(out_putref, rc = -ENOMEM); + GOTO(out, rc = -ENOMEM); } list_add(&range->rn_list, &nodemap->nm_ranges); - write_unlock(&nm_range_tree_lock); + nm_member_reclassify_nodemap(config->nmc_default_nodemap); + up_write(&config->nmc_range_tree_lock); - nm_member_reclassify_nodemap(default_nodemap); - nm_member_revoke_locks(default_nodemap); + nm_member_revoke_locks(config->nmc_default_nodemap); nm_member_revoke_locks(nodemap); -out_putref: +out: + return rc; +} +int nodemap_add_range(const char *name, const lnet_nid_t nid[2]) +{ + struct lu_nodemap *nodemap = NULL; + int rc; + + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(name); + if (IS_ERR(nodemap)) { + mutex_unlock(&active_config_lock); + GOTO(out, rc = PTR_ERR(nodemap)); + } + + if (is_default_nodemap(nodemap)) + rc = -EINVAL; + else + rc = nodemap_add_range_helper(active_config, nodemap, nid); + mutex_unlock(&active_config_lock); + nodemap_putref(nodemap); out: return rc; @@ -706,24 +704,31 @@ int nodemap_del_range(const char *name, const lnet_nid_t nid[2]) struct lu_nid_range *range; int rc = 0; - rc = nodemap_lookup(name, &nodemap); - if (nodemap == NULL || is_default_nodemap(nodemap)) - GOTO(out, rc = -EINVAL); + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(name); + if (IS_ERR(nodemap)) { + mutex_unlock(&active_config_lock); + GOTO(out, rc = PTR_ERR(nodemap)); + } - write_lock(&nm_range_tree_lock); - range = range_find(nid[0], nid[1]); + if (is_default_nodemap(nodemap)) + GOTO(out_putref, rc = -EINVAL); + + down_write(&active_config->nmc_range_tree_lock); + range = range_find(&active_config->nmc_range_tree, nid[0], nid[1]); if (range == NULL) { - write_unlock(&nm_range_tree_lock); + up_write(&active_config->nmc_range_tree_lock); GOTO(out_putref, rc = -EINVAL); } - - range_delete(range); - write_unlock(&nm_range_tree_lock); + range_delete(&active_config->nmc_range_tree, range); nm_member_reclassify_nodemap(nodemap); - nm_member_revoke_locks(default_nodemap); + up_write(&active_config->nmc_range_tree_lock); + + nm_member_revoke_locks(active_config->nmc_default_nodemap); nm_member_revoke_locks(nodemap); out_putref: + mutex_unlock(&active_config_lock); nodemap_putref(nodemap); out: return rc; @@ -739,21 +744,31 @@ EXPORT_SYMBOL(nodemap_del_range); * the default flags should be inherited from the default nodemap. * The adds nodemap to nodemap_hash. * + * Requires that the caller take the active_config_lock + * * \param name name of nodemap * \param is_default true if default nodemap - * \retval 0 success + * \retval nodemap success * \retval -EINVAL invalid nodemap name * \retval -EEXIST nodemap already exists * \retval -ENOMEM cannot allocate memory for nodemap */ -static int nodemap_create(const char *name, bool is_default) +struct lu_nodemap *nodemap_create(const char *name, + struct nodemap_config *config, + bool is_default) { struct lu_nodemap *nodemap = NULL; - int rc = 0; + struct cfs_hash *hash = config->nmc_nodemap_hash; + int rc = 0; if (!nodemap_name_is_valid(name)) GOTO(out, rc = -EINVAL); + if (hash == NULL) { + CERROR("Config nodemap hash is NULL, unable to add %s\n", name); + GOTO(out, rc = -EINVAL); + } + OBD_ALLOC_PTR(nodemap); if (nodemap == NULL) { CERROR("cannot allocate memory (%zu bytes)" @@ -764,11 +779,11 @@ static int nodemap_create(const char *name, bool is_default) /* * take an extra reference to prevent nodemap from being destroyed - * while its being created. + * while it's being created. */ atomic_set(&nodemap->nm_refcount, 2); snprintf(nodemap->nm_name, sizeof(nodemap->nm_name), "%s", name); - rc = cfs_hash_add_unique(nodemap_hash, name, &nodemap->nm_hash); + rc = cfs_hash_add_unique(hash, name, &nodemap->nm_hash); if (rc != 0) { OBD_FREE_PTR(nodemap); GOTO(out, rc = -EEXIST); @@ -794,11 +809,13 @@ static int nodemap_create(const char *name, bool is_default) nodemap->nm_squash_uid = NODEMAP_NOBODY_UID; nodemap->nm_squash_gid = NODEMAP_NOBODY_GID; - lprocfs_nodemap_register(name, is_default, nodemap); - - default_nodemap = nodemap; + config->nmc_default_nodemap = nodemap; } else { - nodemap->nm_id = atomic_inc_return(&nodemap_highest_id); + struct lu_nodemap *default_nodemap = + config->nmc_default_nodemap; + + config->nmc_nodemap_highest_id++; + nodemap->nm_id = config->nmc_nodemap_highest_id; nodemap->nmf_trust_client_ids = default_nodemap->nmf_trust_client_ids; nodemap->nmf_allow_root_access = @@ -808,20 +825,13 @@ static int nodemap_create(const char *name, bool is_default) nodemap->nm_squash_uid = default_nodemap->nm_squash_uid; nodemap->nm_squash_gid = default_nodemap->nm_squash_gid; - - lprocfs_nodemap_register(name, is_default, nodemap); } - if (rc == 0) { - nodemap_putref(nodemap); - goto out; - } - - CERROR("cannot add nodemap: '%s': rc = %d\n", name, rc); - nodemap_destroy(nodemap); + return nodemap; out: - return rc; + CERROR("cannot add nodemap: '%s': rc = %d\n", name, rc); + return ERR_PTR(rc); } /** @@ -837,11 +847,14 @@ int nodemap_set_allow_root(const char *name, bool allow_root) struct lu_nodemap *nodemap = NULL; int rc = 0; - rc = nodemap_lookup(name, &nodemap); - if (nodemap == NULL) - GOTO(out, rc = -ENOENT); + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(name); + mutex_unlock(&active_config_lock); + if (IS_ERR(nodemap)) + GOTO(out, rc = PTR_ERR(nodemap)); nodemap->nmf_allow_root_access = allow_root; + nm_member_revoke_locks(nodemap); nodemap_putref(nodemap); out: @@ -863,11 +876,14 @@ int nodemap_set_trust_client_ids(const char *name, bool trust_client_ids) struct lu_nodemap *nodemap = NULL; int rc = 0; - rc = nodemap_lookup(name, &nodemap); - if (nodemap == NULL) - GOTO(out, rc = -ENOENT); + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(name); + mutex_unlock(&active_config_lock); + if (IS_ERR(nodemap)) + GOTO(out, rc = PTR_ERR(nodemap)); nodemap->nmf_trust_client_ids = trust_client_ids; + nm_member_revoke_locks(nodemap); nodemap_putref(nodemap); out: @@ -892,11 +908,14 @@ int nodemap_set_squash_uid(const char *name, uid_t uid) struct lu_nodemap *nodemap = NULL; int rc = 0; - rc = nodemap_lookup(name, &nodemap); - if (nodemap == NULL) - GOTO(out, rc = -ENOENT); + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(name); + mutex_unlock(&active_config_lock); + if (IS_ERR(nodemap)) + GOTO(out, rc = PTR_ERR(nodemap)); nodemap->nm_squash_uid = uid; + nm_member_revoke_locks(nodemap); nodemap_putref(nodemap); out: @@ -921,11 +940,14 @@ int nodemap_set_squash_gid(const char *name, gid_t gid) struct lu_nodemap *nodemap = NULL; int rc = 0; - rc = nodemap_lookup(name, &nodemap); - if (nodemap == NULL) - GOTO(out, rc = -ENOENT); + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(name); + mutex_unlock(&active_config_lock); + if (IS_ERR(nodemap)) + GOTO(out, rc = PTR_ERR(nodemap)); nodemap->nm_squash_gid = gid; + nm_member_revoke_locks(nodemap); nodemap_putref(nodemap); out: @@ -956,7 +978,21 @@ EXPORT_SYMBOL(nodemap_can_setquota); */ int nodemap_add(const char *nodemap_name) { - return nodemap_create(nodemap_name, 0); + struct lu_nodemap *nodemap; + int rc; + + mutex_lock(&active_config_lock); + nodemap = nodemap_create(nodemap_name, active_config, 0); + if (IS_ERR(nodemap)) { + mutex_unlock(&active_config_lock); + return PTR_ERR(nodemap); + } + + rc = lprocfs_nodemap_register(nodemap, 0); + mutex_unlock(&active_config_lock); + nodemap_putref(nodemap); + + return rc; } EXPORT_SYMBOL(nodemap_add); @@ -970,22 +1006,39 @@ EXPORT_SYMBOL(nodemap_add); */ int nodemap_del(const char *nodemap_name) { - struct lu_nodemap *nodemap; - int rc = 0; + struct lu_nodemap *nodemap; + struct lu_nid_range *range; + struct lu_nid_range *range_temp; + int rc = 0; if (strcmp(nodemap_name, DEFAULT_NODEMAP) == 0) - GOTO(out, rc = -EINVAL); + RETURN(-EINVAL); - nodemap = cfs_hash_del_key(nodemap_hash, nodemap_name); - if (nodemap == NULL) + mutex_lock(&active_config_lock); + nodemap = cfs_hash_del_key(active_config->nmc_nodemap_hash, + nodemap_name); + if (nodemap == NULL) { + mutex_unlock(&active_config_lock); GOTO(out, rc = -ENOENT); + } + + /* erase nodemap from active ranges to prevent client assignment */ + down_write(&active_config->nmc_range_tree_lock); + list_for_each_entry_safe(range, range_temp, &nodemap->nm_ranges, + rn_list) + range_delete(&active_config->nmc_range_tree, range); + up_write(&active_config->nmc_range_tree_lock); /* * remove procfs here in case nodemap_create called with same name - * before nodemap_destory is run. + * before nodemap_destroy is run. */ - lprocfs_remove(&nodemap->nm_proc_entry); + lprocfs_nodemap_remove(nodemap->nm_pde_data); + nodemap->nm_pde_data = NULL; + mutex_unlock(&active_config_lock); + nodemap_putref(nodemap); + out: return rc; } @@ -998,58 +1051,215 @@ EXPORT_SYMBOL(nodemap_del); */ void nodemap_activate(const bool value) { + mutex_lock(&active_config_lock); + active_config->nmc_nodemap_is_active = value; + + /* copy active value to global to avoid locking in map functions */ nodemap_active = value; + mutex_unlock(&active_config_lock); nm_member_revoke_all(); } EXPORT_SYMBOL(nodemap_activate); /** - * Cleanup nodemap module on exit + * Helper iterator to convert nodemap hash to list. + * + * \param hs hash structure + * \param bd bucket descriptor + * \param hnode hash node + * \param nodemap_list_head list head for list of nodemaps in hash */ -void nodemap_mod_exit(void) +static int nodemap_cleanup_iter_cb(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode, + void *nodemap_list_head) { - nodemap_cleanup_all(); - lprocfs_remove(&proc_lustre_nodemap_root); + struct lu_nodemap *nodemap; + + nodemap = hlist_entry(hnode, struct lu_nodemap, nm_hash); + list_add(&nodemap->nm_list, nodemap_list_head); + + cfs_hash_bd_del_locked(hs, bd, hnode); + + return 0; } /** - * Initialize the nodemap module + * Walk the nodemap_hash and remove all nodemaps. */ -int nodemap_mod_init(void) +void nodemap_config_cleanup(struct nodemap_config *config) { + struct lu_nodemap *nodemap = NULL; + struct lu_nodemap *nodemap_temp; + struct lu_nid_range *range; + struct lu_nid_range *range_temp; + LIST_HEAD(nodemap_list_head); + + cfs_hash_for_each_safe(config->nmc_nodemap_hash, + nodemap_cleanup_iter_cb, &nodemap_list_head); + cfs_hash_putref(config->nmc_nodemap_hash); + + /* Because nodemap_destroy might sleep, we can't destroy them + * in cfs_hash_for_each, so we build a list there and destroy here + */ + list_for_each_entry_safe(nodemap, nodemap_temp, &nodemap_list_head, + nm_list) { + down_write(&config->nmc_range_tree_lock); + list_for_each_entry_safe(range, range_temp, &nodemap->nm_ranges, + rn_list) + range_delete(&config->nmc_range_tree, range); + up_write(&config->nmc_range_tree_lock); + + nodemap_putref(nodemap); + } +} + +struct nodemap_config *nodemap_config_alloc(void) +{ + struct nodemap_config *config; int rc = 0; - rc = nodemap_init_hash(); - if (rc != 0) - goto cleanup; + OBD_ALLOC_PTR(config); + if (config == NULL) + return ERR_PTR(-ENOMEM); - rwlock_init(&nm_range_tree_lock); - nodemap_procfs_init(); - rc = nodemap_create(DEFAULT_NODEMAP, 1); + rc = nodemap_init_hash(config); + if (rc != 0) { + OBD_FREE_PTR(config); + return ERR_PTR(rc); + } -cleanup: - if (rc != 0) - nodemap_mod_exit(); + init_rwsem(&config->nmc_range_tree_lock); - return rc; + return config; } -static int nm_member_revoke_all_cb(struct cfs_hash *hs, struct cfs_hash_bd *bd, - struct hlist_node *hnode, void *data) +void nodemap_config_dealloc(struct nodemap_config *config) +{ + nodemap_config_cleanup(config); + OBD_FREE_PTR(config); +} + +static int nm_hash_list_cb(struct cfs_hash *hs, struct cfs_hash_bd *bd, + struct hlist_node *hnode, + void *nodemap_list_head) { struct lu_nodemap *nodemap; nodemap = hlist_entry(hnode, struct lu_nodemap, nm_hash); - nm_member_revoke_locks(nodemap); + list_add(&nodemap->nm_list, nodemap_list_head); return 0; } +void nodemap_config_set_active(struct nodemap_config *config) +{ + struct nodemap_config *old_config = active_config; + struct lu_nodemap *nodemap; + struct lu_nodemap *tmp; + LIST_HEAD(nodemap_list_head); + + ENTRY; + + LASSERT(active_config != config); + + mutex_lock(&active_config_lock); + + /* move proc entries from already existing nms, create for new nms */ + cfs_hash_for_each_safe(config->nmc_nodemap_hash, + nm_hash_list_cb, &nodemap_list_head); + list_for_each_entry_safe(nodemap, tmp, &nodemap_list_head, nm_list) { + struct lu_nodemap *old_nm = NULL; + + if (active_config != NULL) + old_nm = cfs_hash_lookup( + active_config->nmc_nodemap_hash, + nodemap->nm_name); + if (old_nm != NULL) { + nodemap->nm_pde_data = old_nm->nm_pde_data; + old_nm->nm_pde_data = NULL; + nodemap_putref(old_nm); + } else { + bool is_def = (nodemap == config->nmc_default_nodemap); + + lprocfs_nodemap_register(nodemap, is_def); + } + } + + /* if new config is inactive, deactivate live config before switching */ + if (!config->nmc_nodemap_is_active) + nodemap_active = false; + active_config = config; + if (config->nmc_nodemap_is_active) + nodemap_active = true; + + mutex_unlock(&active_config_lock); + + if (old_config != NULL) + nodemap_config_dealloc(old_config); + + nm_member_revoke_all(); + + EXIT; +} + +/** + * Cleanup nodemap module on exit + */ +void nodemap_mod_exit(void) +{ + nodemap_config_dealloc(active_config); + nodemap_procfs_exit(); +} + +/** + * Initialize the nodemap module + */ +int nodemap_mod_init(void) +{ + struct nodemap_config *new_config; + struct lu_nodemap *nodemap; + int rc = 0; + + rc = nodemap_procfs_init(); + if (rc != 0) + return rc; + + new_config = nodemap_config_alloc(); + if (IS_ERR(new_config)) { + nodemap_procfs_exit(); + GOTO(out, rc = PTR_ERR(new_config)); + } + + nodemap = nodemap_create(DEFAULT_NODEMAP, new_config, 1); + if (IS_ERR(nodemap)) { + nodemap_config_dealloc(new_config); + nodemap_procfs_exit(); + GOTO(out, rc = PTR_ERR(nodemap)); + } + + nodemap_config_set_active(new_config); + nodemap_putref(nodemap); + +out: + return rc; +} + /** * Revoke locks for all nodemaps. */ -void nm_member_revoke_all() +void nm_member_revoke_all(void) { - cfs_hash_for_each_safe(nodemap_hash, nm_member_revoke_all_cb, NULL); + struct lu_nodemap *nodemap; + struct lu_nodemap *tmp; + LIST_HEAD(nodemap_list_head); + + mutex_lock(&active_config_lock); + cfs_hash_for_each_safe(active_config->nmc_nodemap_hash, + nm_hash_list_cb, &nodemap_list_head); + + /* revoke_locks sleeps, so can't call in cfs hash cb */ + list_for_each_entry_safe(nodemap, tmp, &nodemap_list_head, nm_list) + nm_member_revoke_locks(nodemap); + mutex_unlock(&active_config_lock); } /** @@ -1065,9 +1275,11 @@ void nodemap_test_nid(lnet_nid_t nid, char *name_buf, size_t name_len) { struct lu_nodemap *nodemap; - read_lock(&nm_range_tree_lock); + mutex_lock(&active_config_lock); + down_read(&active_config->nmc_range_tree_lock); nodemap = nodemap_classify_nid(nid); - read_unlock(&nm_range_tree_lock); + up_read(&active_config->nmc_range_tree_lock); + mutex_unlock(&active_config_lock); strncpy(name_buf, nodemap->nm_name, name_len); if (name_len > 0) @@ -1093,9 +1305,11 @@ __u32 nodemap_test_id(lnet_nid_t nid, enum nodemap_id_type idtype, struct lu_nodemap *nodemap; __u32 fs_id; - read_lock(&nm_range_tree_lock); + mutex_lock(&active_config_lock); + down_read(&active_config->nmc_range_tree_lock); nodemap = nodemap_classify_nid(nid); - read_unlock(&nm_range_tree_lock); + up_read(&active_config->nmc_range_tree_lock); + mutex_unlock(&active_config_lock); fs_id = nodemap_map_id(nodemap, idtype, NODEMAP_CLIENT_TO_FS, client_id); diff --git a/lustre/ptlrpc/nodemap_internal.h b/lustre/ptlrpc/nodemap_internal.h index 34c4c64..86c55b0 100644 --- a/lustre/ptlrpc/nodemap_internal.h +++ b/lustre/ptlrpc/nodemap_internal.h @@ -46,11 +46,12 @@ struct lprocfs_static_vars; extern struct proc_dir_entry *proc_lustre_nodemap_root; /* flag if nodemap is active */ extern bool nodemap_active; -/* lock for range interval tree, used in nodemap_lproc.c */ -extern rwlock_t nm_range_tree_lock; + +extern struct mutex active_config_lock; +extern struct nodemap_config *active_config; struct lu_nid_range { - /* unique id set my mgs */ + /* unique id set by mgs */ unsigned int rn_id; /* lu_nodemap containing this range */ struct lu_nodemap *rn_nodemap; @@ -71,16 +72,62 @@ struct lu_idmap { struct rb_node id_fs_to_client; }; +struct nodemap_range_tree { + struct interval_node *nmrt_range_interval_root; + unsigned int nmrt_range_highest_id; +}; + +struct nodemap_config { + /* Highest numerical lu_nodemap.nm_id defined */ + unsigned int nmc_nodemap_highest_id; + + /* Simple flag to determine if nodemaps are active */ + bool nmc_nodemap_is_active; + + /* Pointer to default nodemap as it is needed more often */ + struct lu_nodemap *nmc_default_nodemap; + + /** + * Lock required to access the range tree. + */ + struct rw_semaphore nmc_range_tree_lock; + struct nodemap_range_tree nmc_range_tree; + + /** + * Hash keyed on nodemap name containing all + * nodemaps + */ + struct cfs_hash *nmc_nodemap_hash; +}; + +struct nodemap_config *nodemap_config_alloc(void); +void nodemap_config_dealloc(struct nodemap_config *config); +void nodemap_config_set_active(struct nodemap_config *config); +struct lu_nodemap *nodemap_create(const char *name, + struct nodemap_config *config, + bool is_default); +void nodemap_putref(struct lu_nodemap *nodemap); +struct lu_nodemap *nodemap_lookup(const char *name); + int nodemap_procfs_init(void); -int lprocfs_nodemap_register(const char *name, bool is_default_nodemap, - struct lu_nodemap *nodemap); -struct lu_nid_range *range_create(lnet_nid_t min, lnet_nid_t max, +void nodemap_procfs_exit(void); +int lprocfs_nodemap_register(struct lu_nodemap *nodemap, + bool is_default_nodemap); +void lprocfs_nodemap_remove(struct nodemap_pde *nodemap_pde); +struct lu_nid_range *nodemap_range_find(lnet_nid_t start_nid, + lnet_nid_t end_nid); +struct lu_nid_range *range_create(struct nodemap_range_tree *nm_range_tree, + lnet_nid_t start_nid, lnet_nid_t end_nid, struct lu_nodemap *nodemap); void range_destroy(struct lu_nid_range *range); -int range_insert(struct lu_nid_range *data); -void range_delete(struct lu_nid_range *data); -struct lu_nid_range *range_search(lnet_nid_t nid); -struct lu_nid_range *range_find(lnet_nid_t start_nid, lnet_nid_t end_nid); +int range_insert(struct nodemap_range_tree *nm_range_tree, + struct lu_nid_range *data); +void range_delete(struct nodemap_range_tree *nm_range_tree, + struct lu_nid_range *data); +struct lu_nid_range *range_search(struct nodemap_range_tree *nm_range_tree, + lnet_nid_t nid); +struct lu_nid_range *range_find(struct nodemap_range_tree *nm_range_tree, + lnet_nid_t start_nid, lnet_nid_t end_nid); int range_parse_nidstring(char *range_string, lnet_nid_t *start_nid, lnet_nid_t *end_nid); void range_init_tree(void); @@ -94,7 +141,6 @@ struct lu_idmap *idmap_search(struct lu_nodemap *nodemap, enum nodemap_tree_type, enum nodemap_id_type id_type, __u32 id); -int nodemap_cleanup_nodemaps(void); int nm_member_add(struct lu_nodemap *nodemap, struct obd_export *exp); void nm_member_del(struct lu_nodemap *nodemap, struct obd_export *exp); void nm_member_delete_list(struct lu_nodemap *nodemap); diff --git a/lustre/ptlrpc/nodemap_lproc.c b/lustre/ptlrpc/nodemap_lproc.c index 123e7a7..0c8db40 100644 --- a/lustre/ptlrpc/nodemap_lproc.c +++ b/lustre/ptlrpc/nodemap_lproc.c @@ -43,6 +43,8 @@ * yet */ #define NODEMAP_PROC_DEBUG 1 +static LIST_HEAD(nodemap_pde_list); + /** * Reads and prints the idmap for the given nodemap. * @@ -52,10 +54,21 @@ */ static int nodemap_idmap_show(struct seq_file *m, void *data) { - struct lu_nodemap *nodemap = m->private; + struct lu_nodemap *nodemap; struct lu_idmap *idmap; struct rb_node *node; bool cont = 0; + int rc; + + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(m->private); + mutex_unlock(&active_config_lock); + if (IS_ERR(nodemap)) { + rc = PTR_ERR(nodemap); + CERROR("cannot find nodemap '%s': rc = %d\n", + (char *)m->private, rc); + return rc; + } seq_printf(m, "[\n"); read_lock(&nodemap->nm_idmap_lock); @@ -84,6 +97,7 @@ static int nodemap_idmap_show(struct seq_file *m, void *data) seq_printf(m, "\n"); seq_printf(m, "]\n"); + nodemap_putref(nodemap); return 0; } @@ -96,9 +110,7 @@ static int nodemap_idmap_show(struct seq_file *m, void *data) */ static int nodemap_idmap_open(struct inode *inode, struct file *file) { - struct lu_nodemap *nodemap = PDE_DATA(inode); - - return single_open(file, nodemap_idmap_show, nodemap); + return single_open(file, nodemap_idmap_show, PDE_DATA(inode)); } /** @@ -110,15 +122,26 @@ static int nodemap_idmap_open(struct inode *inode, struct file *file) */ static int nodemap_ranges_show(struct seq_file *m, void *data) { - struct lu_nodemap *nodemap = m->private; + struct lu_nodemap *nodemap; struct lu_nid_range *range; struct interval_node_extent ext; char start_nidstr[LNET_NIDSTR_SIZE]; char end_nidstr[LNET_NIDSTR_SIZE]; bool cont = false; + int rc; + + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(m->private); + if (IS_ERR(nodemap)) { + mutex_unlock(&active_config_lock); + rc = PTR_ERR(nodemap); + CERROR("cannot find nodemap '%s': rc = %d\n", + (char *)m->private, rc); + return rc; + } seq_printf(m, "[\n"); - read_lock(&nm_range_tree_lock); + down_read(&active_config->nmc_range_tree_lock); list_for_each_entry(range, &nodemap->nm_ranges, rn_list) { if (cont) seq_printf(m, ",\n"); @@ -129,10 +152,12 @@ static int nodemap_ranges_show(struct seq_file *m, void *data) seq_printf(m, " { id: %u, start_nid: %s, end_nid: %s }", range->rn_id, start_nidstr, end_nidstr); } - read_unlock(&nm_range_tree_lock); + up_read(&active_config->nmc_range_tree_lock); + mutex_unlock(&active_config_lock); seq_printf(m, "\n"); seq_printf(m, "]\n"); + nodemap_putref(nodemap); return 0; } @@ -145,9 +170,7 @@ static int nodemap_ranges_show(struct seq_file *m, void *data) */ static int nodemap_ranges_open(struct inode *inode, struct file *file) { - struct lu_nodemap *nodemap = PDE_DATA(inode); - - return single_open(file, nodemap_ranges_show, nodemap); + return single_open(file, nodemap_ranges_show, PDE_DATA(inode)); } /** @@ -159,9 +182,20 @@ static int nodemap_ranges_open(struct inode *inode, struct file *file) */ static int nodemap_exports_show(struct seq_file *m, void *data) { - struct lu_nodemap *nodemap = m->private; + struct lu_nodemap *nodemap; struct obd_export *exp; char nidstr[LNET_NIDSTR_SIZE] = ""; + int rc; + + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(m->private); + mutex_unlock(&active_config_lock); + if (IS_ERR(nodemap)) { + rc = PTR_ERR(nodemap); + CERROR("cannot find nodemap '%s': rc = %d\n", + (char *)m->private, rc); + return rc; + } seq_printf(m, "[\n"); @@ -180,6 +214,7 @@ static int nodemap_exports_show(struct seq_file *m, void *data) seq_printf(m, "\n"); seq_printf(m, "]\n"); + nodemap_putref(nodemap); return 0; } @@ -192,9 +227,7 @@ static int nodemap_exports_show(struct seq_file *m, void *data) */ static int nodemap_exports_open(struct inode *inode, struct file *file) { - struct lu_nodemap *nodemap = PDE_DATA(inode); - - return single_open(file, nodemap_exports_show, nodemap); + return single_open(file, nodemap_exports_show, PDE_DATA(inode)); } /** @@ -241,7 +274,7 @@ nodemap_active_seq_write(struct file *file, const char __user *buffer, if (rc != 0) return -EINVAL; - nodemap_active = active; + nodemap_activate(active); return count; } @@ -256,9 +289,22 @@ LPROC_SEQ_FOPS(nodemap_active); */ static int nodemap_id_seq_show(struct seq_file *m, void *data) { - struct lu_nodemap *nodemap = m->private; + struct lu_nodemap *nodemap; + int rc; + + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(m->private); + mutex_unlock(&active_config_lock); + if (IS_ERR(nodemap)) { + rc = PTR_ERR(nodemap); + CERROR("cannot find nodemap '%s': rc = %d\n", + (char *)m->private, rc); + return rc; + } - return seq_printf(m, "%u\n", nodemap->nm_id); + rc = seq_printf(m, "%u\n", nodemap->nm_id); + nodemap_putref(nodemap); + return rc; } LPROC_SEQ_FOPS_RO(nodemap_id); @@ -271,9 +317,22 @@ LPROC_SEQ_FOPS_RO(nodemap_id); */ static int nodemap_squash_uid_seq_show(struct seq_file *m, void *data) { - struct lu_nodemap *nodemap = m->private; + struct lu_nodemap *nodemap; + int rc; + + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(m->private); + mutex_unlock(&active_config_lock); + if (IS_ERR(nodemap)) { + rc = PTR_ERR(nodemap); + CERROR("cannot find nodemap '%s': rc = %d\n", + (char *)m->private, rc); + return rc; + } - return seq_printf(m, "%u\n", nodemap->nm_squash_uid); + rc = seq_printf(m, "%u\n", nodemap->nm_squash_uid); + nodemap_putref(nodemap); + return rc; } /** @@ -285,9 +344,22 @@ static int nodemap_squash_uid_seq_show(struct seq_file *m, void *data) */ static int nodemap_squash_gid_seq_show(struct seq_file *m, void *data) { - struct lu_nodemap *nodemap = m->private; + struct lu_nodemap *nodemap; + int rc; + + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(m->private); + mutex_unlock(&active_config_lock); + if (IS_ERR(nodemap)) { + rc = PTR_ERR(nodemap); + CERROR("cannot find nodemap '%s': rc = %d\n", + (char *)m->private, rc); + return rc; + } - return seq_printf(m, "%u\n", nodemap->nm_squash_gid); + rc = seq_printf(m, "%u\n", nodemap->nm_squash_gid); + nodemap_putref(nodemap); + return rc; } /** @@ -299,9 +371,22 @@ static int nodemap_squash_gid_seq_show(struct seq_file *m, void *data) */ static int nodemap_trusted_seq_show(struct seq_file *m, void *data) { - struct lu_nodemap *nodemap = m->private; + struct lu_nodemap *nodemap; + int rc; + + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(m->private); + mutex_unlock(&active_config_lock); + if (IS_ERR(nodemap)) { + rc = PTR_ERR(nodemap); + CERROR("cannot find nodemap '%s': rc = %d\n", + (char *)m->private, rc); + return rc; + } - return seq_printf(m, "%d\n", (int)nodemap->nmf_trust_client_ids); + rc = seq_printf(m, "%d\n", (int)nodemap->nmf_trust_client_ids); + nodemap_putref(nodemap); + return rc; } /** @@ -313,9 +398,22 @@ static int nodemap_trusted_seq_show(struct seq_file *m, void *data) */ static int nodemap_admin_seq_show(struct seq_file *m, void *data) { - struct lu_nodemap *nodemap = m->private; + struct lu_nodemap *nodemap; + int rc; + + mutex_lock(&active_config_lock); + nodemap = nodemap_lookup(m->private); + mutex_unlock(&active_config_lock); + if (IS_ERR(nodemap)) { + rc = PTR_ERR(nodemap); + CERROR("cannot find nodemap '%s': rc = %d\n", + (char *)m->private, rc); + return rc; + } - return seq_printf(m, "%d\n", (int)nodemap->nmf_allow_root_access); + rc = seq_printf(m, "%d\n", (int)nodemap->nmf_allow_root_access); + nodemap_putref(nodemap); + return rc; } #ifdef NODEMAP_PROC_DEBUG @@ -370,7 +468,6 @@ nodemap_squash_uid_seq_write(struct file *file, const char __user *buffer, { char squash[NODEMAP_LPROC_ID_LEN + 1]; struct seq_file *m = file->private_data; - struct lu_nodemap *nodemap = m->private; long unsigned int squash_uid; int rc; @@ -388,7 +485,9 @@ nodemap_squash_uid_seq_write(struct file *file, const char __user *buffer, if (rc != 0) return -EINVAL; - nodemap->nm_squash_uid = squash_uid; + rc = nodemap_set_squash_uid(m->private, squash_uid); + if (rc != 0) + return rc; return count; } @@ -409,7 +508,6 @@ nodemap_squash_gid_seq_write(struct file *file, const char __user *buffer, { char squash[NODEMAP_LPROC_ID_LEN + 1]; struct seq_file *m = file->private_data; - struct lu_nodemap *nodemap = m->private; long unsigned int squash_gid; int rc; @@ -427,7 +525,9 @@ nodemap_squash_gid_seq_write(struct file *file, const char __user *buffer, if (rc != 0) return -EINVAL; - nodemap->nm_squash_gid = squash_gid; + rc = nodemap_set_squash_gid(m->private, squash_gid); + if (rc != 0) + return rc; return count; } @@ -447,17 +547,18 @@ nodemap_trusted_seq_write(struct file *file, const char __user *buffer, size_t count, loff_t *off) { struct seq_file *m = file->private_data; - struct lu_nodemap *nodemap = m->private; int flags; int rc; rc = nodemap_proc_read_flag(buffer, count, &flags); - if (rc >= 0) { - nodemap->nmf_trust_client_ids = !!flags; - nm_member_revoke_locks(nodemap); - } + if (rc < 0) + return rc; - return rc; + rc = nodemap_set_trust_client_ids(m->private, flags); + if (rc != 0) + return rc; + + return count; } /** @@ -475,17 +576,18 @@ nodemap_admin_seq_write(struct file *file, const char __user *buffer, size_t count, loff_t *off) { struct seq_file *m = file->private_data; - struct lu_nodemap *nodemap = m->private; int flags; int rc; rc = nodemap_proc_read_flag(buffer, count, &flags); - if (rc >= 0) { - nodemap->nmf_allow_root_access = !!flags; - nm_member_revoke_locks(nodemap); - } + if (rc < 0) + return rc; - return rc; + rc = nodemap_set_allow_root(m->private, flags); + if (rc != 0) + return rc; + + return count; } /** @@ -1030,37 +1132,79 @@ int nodemap_procfs_init(void) } /** + * Cleanup nodemap proc entry data structures. + */ +void nodemap_procfs_exit(void) +{ + struct nodemap_pde *nm_pde; + struct nodemap_pde *tmp; + + lprocfs_remove(&proc_lustre_nodemap_root); + list_for_each_entry_safe(nm_pde, tmp, &nodemap_pde_list, + npe_list_member) { + list_del(&nm_pde->npe_list_member); + OBD_FREE_PTR(nm_pde); + } +} + +/** + * Remove a nodemap's procfs entry and related data. + */ +void lprocfs_nodemap_remove(struct nodemap_pde *nm_pde) +{ + lprocfs_remove(&nm_pde->npe_proc_entry); + list_del(&nm_pde->npe_list_member); + OBD_FREE_PTR(nm_pde); +} + +/** * Register the proc directory for a nodemap * - * \param name name of nodemap + * \param nodemap nodemap to make the proc dir for * \param is_default: 1 if default nodemap * \retval 0 success */ -int lprocfs_nodemap_register(const char *name, - bool is_default, - struct lu_nodemap *nodemap) +int lprocfs_nodemap_register(struct lu_nodemap *nodemap, bool is_default) { - struct proc_dir_entry *nodemap_proc_entry; - int rc = 0; - - if (is_default) - nodemap_proc_entry = - lprocfs_register(name, proc_lustre_nodemap_root, - lprocfs_default_nodemap_vars, - nodemap); + struct nodemap_pde *nm_entry; + int rc = 0; + + OBD_ALLOC_PTR(nm_entry); + if (nm_entry == NULL) + GOTO(out, rc = -ENOMEM); + + nm_entry->npe_proc_entry = proc_mkdir(nodemap->nm_name, + proc_lustre_nodemap_root); + if (IS_ERR(nm_entry->npe_proc_entry)) + GOTO(out, rc = PTR_ERR(nm_entry->npe_proc_entry)); + + snprintf(nm_entry->npe_name, sizeof(nm_entry->npe_name), "%s", + nodemap->nm_name); + + /* Use the nodemap name as stored on the PDE as the private data. This + * is so a nodemap struct can be replaced without updating the proc + * entries. + */ + rc = lprocfs_add_vars(nm_entry->npe_proc_entry, + (is_default ? lprocfs_default_nodemap_vars : + lprocfs_nodemap_vars), + nm_entry->npe_name); + if (rc != 0) + lprocfs_remove(&nm_entry->npe_proc_entry); else - nodemap_proc_entry = - lprocfs_register(name, proc_lustre_nodemap_root, - lprocfs_nodemap_vars, - nodemap); - - if (IS_ERR(nodemap_proc_entry)) { - rc = PTR_ERR(nodemap_proc_entry); - CERROR("cannot create 'nodemap/%s': rc = %d\n", name, rc); - nodemap_proc_entry = NULL; + list_add(&nm_entry->npe_list_member, &nodemap_pde_list); + +out: + if (rc != 0) { + CERROR("cannot create 'nodemap/%s': rc = %d\n", + nodemap->nm_name, rc); + if (nm_entry != NULL) { + OBD_FREE_PTR(nm_entry); + nm_entry = NULL; + } } - nodemap->nm_proc_entry = nodemap_proc_entry; + nodemap->nm_pde_data = nm_entry; return rc; } diff --git a/lustre/ptlrpc/nodemap_member.c b/lustre/ptlrpc/nodemap_member.c index 81a0980..d4ffc23 100644 --- a/lustre/ptlrpc/nodemap_member.c +++ b/lustre/ptlrpc/nodemap_member.c @@ -126,9 +126,6 @@ static void nm_member_exp_revoke(struct obd_export *exp) ldlm_revoke_export_locks(exp); } -/* Mutex used to serialize calls to reclassify_nodemap_lock */ -DEFINE_MUTEX(reclassify_nodemap_lock); - /** * Reclassify the members of a nodemap after range changes or activation. * This function reclassifies the members of a nodemap based on the member @@ -136,15 +133,8 @@ DEFINE_MUTEX(reclassify_nodemap_lock); * classified as being part of this nodemap are moved to the nodemap whose * NID ranges contain the export's NID, and their locks are revoked. * - * Calls to this function are serialized due to a potential deadlock: Say there - * is a nodemap A and a nodemap B that both need to reclassify their members. - * If there is a member in nodemap A that should be in nodemap B, reclassify - * will attempt to add the member to nodemap B. If nodemap B is also - * reclassifying its members, then its hash is locked and nodemap A's attempt - * to add will block and wait for nodemap B's reclassify to finish. If - * nodemap B's reclassify then attempts to reclassify a member that should be - * in nodemap A, it will also try add the member to nodemap A's locked hash, - * causing a deadlock. + * Callers should hold the active_config_lock and active_config + * nmc_range_tree_lock. * * \param nodemap nodemap with members to reclassify */ @@ -154,17 +144,13 @@ void nm_member_reclassify_nodemap(struct lu_nodemap *nodemap) struct obd_export *tmp; struct lu_nodemap *new_nodemap; - /* reclassify only one nodemap at a time to avoid deadlock */ - mutex_lock(&reclassify_nodemap_lock); mutex_lock(&nodemap->nm_member_list_lock); list_for_each_entry_safe(exp, tmp, &nodemap->nm_member_list, exp_target_data.ted_nodemap_member) { lnet_nid_t nid = exp->exp_connection->c_peer.nid; - /* nodemap_classify_nid requires range tree lock */ - read_lock(&nm_range_tree_lock); + /* nodemap_classify_nid requires nmc_range_tree_lock */ new_nodemap = nodemap_classify_nid(nid); - read_unlock(&nm_range_tree_lock); if (new_nodemap != nodemap) { /* don't use member_del because ted_nodemap * should never be null @@ -179,10 +165,13 @@ void nm_member_reclassify_nodemap(struct lu_nodemap *nodemap) mutex_unlock(&new_nodemap->nm_member_list_lock); nm_member_exp_revoke(exp); } + + /* This put won't destroy new_nodemap because any nodemap_del + * call done on new_nodemap blocks on our active_config_lock + */ nodemap_putref(new_nodemap); } mutex_unlock(&nodemap->nm_member_list_lock); - mutex_unlock(&reclassify_nodemap_lock); } /** diff --git a/lustre/ptlrpc/nodemap_range.c b/lustre/ptlrpc/nodemap_range.c index 3a43b2f..e8b5a24 100644 --- a/lustre/ptlrpc/nodemap_range.c +++ b/lustre/ptlrpc/nodemap_range.c @@ -40,14 +40,6 @@ * controlled to prevent read access during update operations. */ -static struct interval_node *range_interval_root; -static atomic_t range_highest_id; - -void range_init_tree(void) -{ - range_interval_root = NULL; -} - /* * callback for iterating over the interval tree * @@ -77,7 +69,8 @@ static enum interval_iter range_cb(struct interval_node *n, void *data) * \param nodemap nodemap that contains this range * \retval lu_nid_range on success, NULL on failure */ -struct lu_nid_range *range_create(lnet_nid_t start_nid, lnet_nid_t end_nid, +struct lu_nid_range *range_create(struct nodemap_range_tree *nm_range_tree, + lnet_nid_t start_nid, lnet_nid_t end_nid, struct lu_nodemap *nodemap) { struct lu_nid_range *range; @@ -93,7 +86,8 @@ struct lu_nid_range *range_create(lnet_nid_t start_nid, lnet_nid_t end_nid, return NULL; } - range->rn_id = atomic_inc_return(&range_highest_id); + nm_range_tree->nmrt_range_highest_id++; + range->rn_id = nm_range_tree->nmrt_range_highest_id; range->rn_nodemap = nodemap; interval_set(&range->rn_node, start_nid, end_nid); INIT_LIST_HEAD(&range->rn_list); @@ -108,7 +102,8 @@ struct lu_nid_range *range_create(lnet_nid_t start_nid, lnet_nid_t end_nid, * \param end_nid ending nid * \retval matching range or NULL */ -struct lu_nid_range *range_find(lnet_nid_t start_nid, lnet_nid_t end_nid) +struct lu_nid_range *range_find(struct nodemap_range_tree *nm_range_tree, + lnet_nid_t start_nid, lnet_nid_t end_nid) { struct lu_nid_range *range = NULL; struct interval_node *interval = NULL; @@ -117,7 +112,7 @@ struct lu_nid_range *range_find(lnet_nid_t start_nid, lnet_nid_t end_nid) .end = end_nid }; - interval = interval_find(range_interval_root, &ext); + interval = interval_find(nm_range_tree->nmrt_range_interval_root, &ext); if (interval != NULL) range = container_of(interval, struct lu_nid_range, @@ -147,15 +142,18 @@ void range_destroy(struct lu_nid_range *range) * does not overlap so that each nid can belong * to exactly one range */ -int range_insert(struct lu_nid_range *range) +int range_insert(struct nodemap_range_tree *nm_range_tree, + struct lu_nid_range *range) { struct interval_node_extent ext = range->rn_node.in_extent; - if (interval_is_overlapped(range_interval_root, &ext) != 0) + if (interval_is_overlapped(nm_range_tree->nmrt_range_interval_root, + &ext) != 0) return -EEXIST; - interval_insert(&range->rn_node, &range_interval_root); + interval_insert(&range->rn_node, + &nm_range_tree->nmrt_range_interval_root); return 0; } @@ -166,12 +164,14 @@ int range_insert(struct lu_nid_range *range) * * \param range range to remove */ -void range_delete(struct lu_nid_range *range) +void range_delete(struct nodemap_range_tree *nm_range_tree, + struct lu_nid_range *range) { if (range == NULL || interval_is_intree(&range->rn_node) == 0) return; list_del(&range->rn_list); - interval_erase(&range->rn_node, &range_interval_root); + interval_erase(&range->rn_node, + &nm_range_tree->nmrt_range_interval_root); range_destroy(range); } @@ -180,7 +180,8 @@ void range_delete(struct lu_nid_range *range) * * \param nid nid to search for */ -struct lu_nid_range *range_search(lnet_nid_t nid) +struct lu_nid_range *range_search(struct nodemap_range_tree *nm_range_tree, + lnet_nid_t nid) { struct lu_nid_range *ret = NULL; struct interval_node_extent ext = { @@ -188,7 +189,8 @@ struct lu_nid_range *range_search(lnet_nid_t nid) .end = nid }; - interval_search(range_interval_root, &ext, range_cb, &ret); + interval_search(nm_range_tree->nmrt_range_interval_root, &ext, + range_cb, &ret); return ret; } diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh index 03e3712..e5cad39 100755 --- a/lustre/tests/sanity-sec.sh +++ b/lustre/tests/sanity-sec.sh @@ -1525,6 +1525,23 @@ test_23() { } run_test 23 "test mapped ACLs" +test_24() { + nodemap_test_setup + + trap nodemap_test_cleanup EXIT + for node in $(all_server_nodes); do + local node_ip=$(host_nids_address $node $NETTYPE) + do_node $node_ip 'find /proc/fs/lustre/nodemap -exec \ + cat {} \;' &> /dev/null + do_node $node_ip 'find /proc/fs/lustre/nodemap \ + -type f -perm /444 | xargs cat' &> /dev/null || + error "proc readable file read failed" + done + + nodemap_test_cleanup +} +run_test 24 "check nodemap proc files for LBUGs and Oopses" + log "cleanup: ======================================================" sec_unsetup() {