lustre/ptlrpc/nodemap_member.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (C) 2013, Trustees of Indiana University
  24  * Author: Joshua Walgenbach <jjw@iu.edu>
  25  */
  26 #include <linux/module.h>
  27 #include <lustre_net.h>
  28 #include <obd_class.h>
  29 #include "nodemap_internal.h"
  30
  31 #define HASH_NODEMAP_MEMBER_BKT_BITS 3
  32 #define HASH_NODEMAP_MEMBER_CUR_BITS 3
  33 #define HASH_NODEMAP_MEMBER_MAX_BITS 7
  34
  35
  36 /**
  37  * Delete an export from a nodemap's member list. Called after client
  38  * disconnects, or during system shutdown.
  39  *
  40  * Requires active_config_lock and nodemap's nm_member_list_lock.
  41  *
  42  * \param       nodemap         nodemap containing list
  43  * \param       exp             export member to delete
  44  */
  45 void nm_member_del(struct lu_nodemap *nodemap, struct obd_export *exp)
  46 {
  47         ENTRY;
  48
  49         /* because all changes to ted_nodemap are with active_config_lock */
  50         LASSERT(exp->exp_target_data.ted_nodemap == nodemap);
  51
  52         /* protected by nm_member_list_lock */
  53         list_del_init(&exp->exp_target_data.ted_nodemap_member);
  54
  55         spin_lock(&exp->exp_target_data.ted_nodemap_lock);
  56         exp->exp_target_data.ted_nodemap = NULL;
  57         spin_unlock(&exp->exp_target_data.ted_nodemap_lock);
  58
  59         /* ref formerly held by ted_nodemap */
  60         nodemap_putref(nodemap);
  61
  62         /* ref formerly held by ted_nodemap_member */
  63         class_export_put(exp);
  64
  65         EXIT;
  66 }
  67
  68 /**
  69  * Delete a member list from a nodemap
  70  *
  71  * Requires active config lock.
  72  *
  73  * \param       nodemap         nodemap to remove the list from
  74  */
  75 void nm_member_delete_list(struct lu_nodemap *nodemap)
  76 {
  77         struct obd_export *exp;
  78         struct obd_export *tmp;
  79
  80         mutex_lock(&nodemap->nm_member_list_lock);
  81         list_for_each_entry_safe(exp, tmp, &nodemap->nm_member_list,
  82                                  exp_target_data.ted_nodemap_member)
  83                 nm_member_del(nodemap, exp);
  84         mutex_unlock(&nodemap->nm_member_list_lock);
  85 }
  86
  87 /**
  88  * Add a member export to a nodemap
  89  *
  90  * Must be called under active_config_lock.
  91  *
  92  * \param       nodemap         nodemap to add to
  93  * \param       exp             obd_export to add
  94  * \retval      -EEXIST         export is already part of a different nodemap
  95  * \retval      -EINVAL         export is NULL
  96  */
  97 int nm_member_add(struct lu_nodemap *nodemap, struct obd_export *exp)
  98 {
  99         ENTRY;
 100
 101         if (exp == NULL) {
 102                 CWARN("attempted to add null export to nodemap %s\n",
 103                       nodemap->nm_name);
 104                 RETURN(-EINVAL);
 105         }
 106
 107         mutex_lock(&nodemap->nm_member_list_lock);
 108         if (exp->exp_target_data.ted_nodemap != NULL &&
 109             !list_empty(&exp->exp_target_data.ted_nodemap_member)) {
 110                 mutex_unlock(&nodemap->nm_member_list_lock);
 111
 112                 /* export is already member of nodemap */
 113                 if (exp->exp_target_data.ted_nodemap == nodemap)
 114                         RETURN(0);
 115
 116                 /* possibly reconnecting while about to be reclassified */
 117                 CWARN("export %p %s already hashed, failed to add to "
 118                       "nodemap %s already member of %s\n", exp,
 119                       exp->exp_client_uuid.uuid,
 120                       nodemap->nm_name,
 121                       (exp->exp_target_data.ted_nodemap == NULL) ? "unknown" :
 122                                 exp->exp_target_data.ted_nodemap->nm_name);
 123                 RETURN(-EEXIST);
 124         }
 125
 126         class_export_get(exp);
 127         nodemap_getref(nodemap);
 128         /* ted_nodemap changes also require ac lock, member_list_lock */
 129         spin_lock(&exp->exp_target_data.ted_nodemap_lock);
 130         exp->exp_target_data.ted_nodemap = nodemap;
 131         spin_unlock(&exp->exp_target_data.ted_nodemap_lock);
 132         list_add(&exp->exp_target_data.ted_nodemap_member,
 133                  &nodemap->nm_member_list);
 134         mutex_unlock(&nodemap->nm_member_list_lock);
 135
 136         RETURN(0);
 137 }
 138
 139 /**
 140  * Revokes the locks on an export if it is attached to an MDT and not in
 141  * recovery. As a performance enhancement, the lock revoking process could
 142  * revoke only the locks that cover files affected by the nodemap change.
 143  */
 144 static void nm_member_exp_revoke(struct obd_export *exp)
 145 {
 146         struct obd_type *type = exp->exp_obd->obd_type;
 147         if (strcmp(type->typ_name, LUSTRE_MDT_NAME) != 0)
 148                 return;
 149         if (exp->exp_obd->obd_recovering)
 150                 return;
 151
 152         ldlm_revoke_export_locks(exp);
 153 }
 154
 155 /**
 156  * Reclassify the members of a nodemap after range changes or activation.
 157  * This function reclassifies the members of a nodemap based on the member
 158  * export's NID and the nodemap's new NID ranges. Exports that are no longer
 159  * classified as being part of this nodemap are moved to the nodemap whose
 160  * NID ranges contain the export's NID, and their locks are revoked.
 161  *
 162  * Callers should hold the active_config_lock and active_config
 163  * nmc_range_tree_lock.
 164  *
 165  * \param       nodemap         nodemap with members to reclassify
 166  */
 167 void nm_member_reclassify_nodemap(struct lu_nodemap *nodemap)
 168 {
 169         struct obd_export *exp;
 170         struct obd_export *tmp;
 171         struct lu_nodemap *new_nodemap;
 172
 173         ENTRY;
 174
 175         mutex_lock(&nodemap->nm_member_list_lock);
 176
 177         list_for_each_entry_safe(exp, tmp, &nodemap->nm_member_list,
 178                                  exp_target_data.ted_nodemap_member) {
 179                 lnet_nid_t nid;
 180
 181                 /* if no conn assigned to this exp, reconnect will reclassify */
 182                 spin_lock(&exp->exp_lock);
 183                 if (exp->exp_connection) {
 184                         nid = exp->exp_connection->c_peer.nid;
 185                 } else {
 186                         spin_unlock(&exp->exp_lock);
 187                         continue;
 188                 }
 189                 spin_unlock(&exp->exp_lock);
 190
 191                 /* nodemap_classify_nid requires nmc_range_tree_lock */
 192                 new_nodemap = nodemap_classify_nid(nid);
 193                 if (IS_ERR(new_nodemap))
 194                         continue;
 195
 196                 if (new_nodemap != nodemap) {
 197                         /* could deadlock if new_nodemap also reclassifying,
 198                          * active_config_lock serializes reclassifies
 199                          */
 200                         mutex_lock(&new_nodemap->nm_member_list_lock);
 201
 202                         /* don't use member_del because ted_nodemap
 203                          * should never be NULL with a live export
 204                          */
 205                         list_del_init(&exp->exp_target_data.ted_nodemap_member);
 206
 207                         /* keep the new_nodemap ref from classify */
 208                         spin_lock(&exp->exp_target_data.ted_nodemap_lock);
 209                         exp->exp_target_data.ted_nodemap = new_nodemap;
 210                         spin_unlock(&exp->exp_target_data.ted_nodemap_lock);
 211                         nodemap_putref(nodemap);
 212
 213                         list_add(&exp->exp_target_data.ted_nodemap_member,
 214                                  &new_nodemap->nm_member_list);
 215                         mutex_unlock(&new_nodemap->nm_member_list_lock);
 216
 217                         if (nodemap_active)
 218                                 nm_member_exp_revoke(exp);
 219                 } else {
 220                         nodemap_putref(new_nodemap);
 221                 }
 222         }
 223         mutex_unlock(&nodemap->nm_member_list_lock);
 224
 225         EXIT;
 226 }
 227
 228 /**
 229  * Revoke the locks for member exports if nodemap system is active.
 230  *
 231  * Changing the idmap is akin to deleting the security context. If the locks
 232  * are not canceled, the client could cache permissions that are no longer
 233  * correct with the map.
 234  *
 235  * \param       nodemap         nodemap that has been altered
 236  */
 237 void nm_member_revoke_locks(struct lu_nodemap *nodemap)
 238 {
 239         if (!nodemap_active)
 240                 return;
 241
 242         nm_member_revoke_locks_always(nodemap);
 243 }
 244
 245 void nm_member_revoke_locks_always(struct lu_nodemap *nodemap)
 246 {
 247         struct obd_export *exp;
 248         struct obd_export *tmp;
 249
 250         mutex_lock(&nodemap->nm_member_list_lock);
 251         list_for_each_entry_safe(exp, tmp, &nodemap->nm_member_list,
 252                             exp_target_data.ted_nodemap_member)
 253                 nm_member_exp_revoke(exp);
 254         mutex_unlock(&nodemap->nm_member_list_lock);
 255 }