X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Fptlrpc%2Fnodemap_member.c;h=275aaaedcd31dc939323e89708927b022c6db069;hp=d4ffc23b43933661eb3b09697467a2d54a934dca;hb=5315db3f1066619d6effe4f778d2df3ad1ba738f;hpb=2e0d4e010eb4c2e28507f55c1b649d70bccd06a0 diff --git a/lustre/ptlrpc/nodemap_member.c b/lustre/ptlrpc/nodemap_member.c index d4ffc23..275aaae 100644 --- a/lustre/ptlrpc/nodemap_member.c +++ b/lustre/ptlrpc/nodemap_member.c @@ -32,25 +32,44 @@ #define HASH_NODEMAP_MEMBER_CUR_BITS 3 #define HASH_NODEMAP_MEMBER_MAX_BITS 7 + /** - * Delete a member from a member list + * Delete an export from a nodemap's member list. Called after client + * disconnects, or during system shutdown. + * + * Requires active_config_lock and nodemap's nm_member_list_lock. * * \param nodemap nodemap containing list * \param exp export member to delete */ void nm_member_del(struct lu_nodemap *nodemap, struct obd_export *exp) { - mutex_lock(&nodemap->nm_member_list_lock); + ENTRY; + + /* because all changes to ted_nodemap are with active_config_lock */ + LASSERT(exp->exp_target_data.ted_nodemap == nodemap); + + /* protected by nm_member_list_lock */ list_del_init(&exp->exp_target_data.ted_nodemap_member); - mutex_unlock(&nodemap->nm_member_list_lock); + spin_lock(&exp->exp_target_data.ted_nodemap_lock); exp->exp_target_data.ted_nodemap = NULL; + spin_unlock(&exp->exp_target_data.ted_nodemap_lock); + + /* ref formerly held by ted_nodemap */ + nodemap_putref(nodemap); + + /* ref formerly held by ted_nodemap_member */ class_export_put(exp); + + EXIT; } /** * Delete a member list from a nodemap * + * Requires active config lock. + * * \param nodemap nodemap to remove the list from */ void nm_member_delete_list(struct lu_nodemap *nodemap) @@ -60,17 +79,16 @@ void nm_member_delete_list(struct lu_nodemap *nodemap) mutex_lock(&nodemap->nm_member_list_lock); list_for_each_entry_safe(exp, tmp, &nodemap->nm_member_list, - exp_target_data.ted_nodemap_member) { - exp->exp_target_data.ted_nodemap = NULL; - list_del_init(&exp->exp_target_data.ted_nodemap_member); - class_export_put(exp); - } + exp_target_data.ted_nodemap_member) + nm_member_del(nodemap, exp); mutex_unlock(&nodemap->nm_member_list_lock); } /** * Add a member export to a nodemap * + * Must be called under active_config_lock. + * * \param nodemap nodemap to add to * \param exp obd_export to add * \retval -EEXIST export is already part of a different nodemap @@ -78,17 +96,22 @@ void nm_member_delete_list(struct lu_nodemap *nodemap) */ int nm_member_add(struct lu_nodemap *nodemap, struct obd_export *exp) { + ENTRY; + if (exp == NULL) { CWARN("attempted to add null export to nodemap %s\n", nodemap->nm_name); - return -EINVAL; + RETURN(-EINVAL); } + mutex_lock(&nodemap->nm_member_list_lock); if (exp->exp_target_data.ted_nodemap != NULL && !list_empty(&exp->exp_target_data.ted_nodemap_member)) { + mutex_unlock(&nodemap->nm_member_list_lock); + /* export is already member of nodemap */ if (exp->exp_target_data.ted_nodemap == nodemap) - return 0; + RETURN(0); /* possibly reconnecting while about to be reclassified */ CWARN("export %p %s already hashed, failed to add to " @@ -97,17 +120,20 @@ int nm_member_add(struct lu_nodemap *nodemap, struct obd_export *exp) nodemap->nm_name, (exp->exp_target_data.ted_nodemap == NULL) ? "unknown" : exp->exp_target_data.ted_nodemap->nm_name); - return -EEXIST; + RETURN(-EEXIST); } class_export_get(exp); + nodemap_getref(nodemap); + /* ted_nodemap changes also require ac lock, member_list_lock */ + spin_lock(&exp->exp_target_data.ted_nodemap_lock); exp->exp_target_data.ted_nodemap = nodemap; - mutex_lock(&nodemap->nm_member_list_lock); + spin_unlock(&exp->exp_target_data.ted_nodemap_lock); list_add(&exp->exp_target_data.ted_nodemap_member, &nodemap->nm_member_list); mutex_unlock(&nodemap->nm_member_list_lock); - return 0; + RETURN(0); } /** @@ -144,46 +170,80 @@ void nm_member_reclassify_nodemap(struct lu_nodemap *nodemap) struct obd_export *tmp; struct lu_nodemap *new_nodemap; + ENTRY; + mutex_lock(&nodemap->nm_member_list_lock); + list_for_each_entry_safe(exp, tmp, &nodemap->nm_member_list, exp_target_data.ted_nodemap_member) { - lnet_nid_t nid = exp->exp_connection->c_peer.nid; + lnet_nid_t nid; + + /* if no conn assigned to this exp, reconnect will reclassify */ + spin_lock(&exp->exp_lock); + if (exp->exp_connection) { + nid = exp->exp_connection->c_peer.nid; + } else { + spin_unlock(&exp->exp_lock); + continue; + } + spin_unlock(&exp->exp_lock); /* nodemap_classify_nid requires nmc_range_tree_lock */ new_nodemap = nodemap_classify_nid(nid); + if (IS_ERR(new_nodemap)) + continue; + if (new_nodemap != nodemap) { + /* could deadlock if new_nodemap also reclassifying, + * active_config_lock serializes reclassifies + */ + mutex_lock(&new_nodemap->nm_member_list_lock); + /* don't use member_del because ted_nodemap - * should never be null + * should never be NULL with a live export */ list_del_init(&exp->exp_target_data.ted_nodemap_member); + + /* keep the new_nodemap ref from classify */ + spin_lock(&exp->exp_target_data.ted_nodemap_lock); exp->exp_target_data.ted_nodemap = new_nodemap; + spin_unlock(&exp->exp_target_data.ted_nodemap_lock); + nodemap_putref(nodemap); - /* could deadlock if new_nodemap also reclassifying */ - mutex_lock(&new_nodemap->nm_member_list_lock); list_add(&exp->exp_target_data.ted_nodemap_member, &new_nodemap->nm_member_list); mutex_unlock(&new_nodemap->nm_member_list_lock); - nm_member_exp_revoke(exp); - } - /* This put won't destroy new_nodemap because any nodemap_del - * call done on new_nodemap blocks on our active_config_lock - */ - nodemap_putref(new_nodemap); + if (nodemap_active) + nm_member_exp_revoke(exp); + } else { + nodemap_putref(new_nodemap); + } } mutex_unlock(&nodemap->nm_member_list_lock); + + EXIT; } /** - * Revoke the locks for member exports. Changing the idmap is - * akin to deleting the security context. If the locks are not - * canceled, the client could cache permissions that are no - * longer correct with the map. + * Revoke the locks for member exports if nodemap system is active. + * + * Changing the idmap is akin to deleting the security context. If the locks + * are not canceled, the client could cache permissions that are no longer + * correct with the map. * * \param nodemap nodemap that has been altered */ void nm_member_revoke_locks(struct lu_nodemap *nodemap) { + if (!nodemap_active) + return; + + nm_member_revoke_locks_always(nodemap); +} + +void nm_member_revoke_locks_always(struct lu_nodemap *nodemap) +{ struct obd_export *exp; struct obd_export *tmp;