Whamcloud - gitweb
LU-16524 nodemap: add rbac property to nodemap
[fs/lustre-release.git] / lustre / ptlrpc / nodemap_storage.c
index 65a8884..73a6f31 100644 (file)
@@ -67,44 +67,52 @@ static DEFINE_MUTEX(ncf_list_lock);
 /* MGS index is different than others, others are listeners to MGS idx */
 static struct nm_config_file *nodemap_mgs_ncf;
 
-/* lu_nodemap flags */
-enum nm_flag_shifts {
-       NM_FL_ALLOW_ROOT_ACCESS = 0x1,
-       NM_FL_TRUST_CLIENT_IDS = 0x2,
-       NM_FL_DENY_UNKNOWN = 0x4,
-       NM_FL_MAP_UID_ONLY = 0x8,
-       NM_FL_MAP_GID_ONLY = 0x10,
-       NM_FL_ENABLE_AUDIT = 0x20,
-};
-
-static void nodemap_cluster_key_init(struct nodemap_key *nk, unsigned int nm_id)
+static void nodemap_cluster_key_init(struct nodemap_key *nk, unsigned int nm_id,
+                                    enum nodemap_cluster_rec_subid subid)
 {
        nk->nk_nodemap_id = cpu_to_le32(nm_idx_set_type(nm_id,
                                                        NODEMAP_CLUSTER_IDX));
-       nk->nk_unused = 0;
+       nk->nk_cluster_subid = subid;
 }
 
 static void nodemap_cluster_rec_init(union nodemap_rec *nr,
                                     const struct lu_nodemap *nodemap)
 {
-       CLASSERT(sizeof(nr->ncr.ncr_name) == sizeof(nodemap->nm_name));
+       BUILD_BUG_ON(sizeof(nr->ncr.ncr_name) != sizeof(nodemap->nm_name));
 
-       strncpy(nr->ncr.ncr_name, nodemap->nm_name, sizeof(nodemap->nm_name));
+       strncpy(nr->ncr.ncr_name, nodemap->nm_name, sizeof(nr->ncr.ncr_name));
        nr->ncr.ncr_squash_uid = cpu_to_le32(nodemap->nm_squash_uid);
        nr->ncr.ncr_squash_gid = cpu_to_le32(nodemap->nm_squash_gid);
-       nr->ncr.ncr_flags = cpu_to_le32(
+       nr->ncr.ncr_squash_projid = cpu_to_le32(nodemap->nm_squash_projid);
+       nr->ncr.ncr_flags =
                (nodemap->nmf_trust_client_ids ?
                        NM_FL_TRUST_CLIENT_IDS : 0) |
                (nodemap->nmf_allow_root_access ?
                        NM_FL_ALLOW_ROOT_ACCESS : 0) |
                (nodemap->nmf_deny_unknown ?
                        NM_FL_DENY_UNKNOWN : 0) |
-               (nodemap->nmf_map_uid_only ?
-                       NM_FL_MAP_UID_ONLY : 0) |
-               (nodemap->nmf_map_gid_only ?
-                       NM_FL_MAP_GID_ONLY : 0) |
+               (nodemap->nmf_map_mode & NODEMAP_MAP_UID ?
+                       NM_FL_MAP_UID : 0) |
+               (nodemap->nmf_map_mode & NODEMAP_MAP_GID ?
+                       NM_FL_MAP_GID : 0) |
+               (nodemap->nmf_map_mode & NODEMAP_MAP_PROJID ?
+                       NM_FL_MAP_PROJID : 0) |
                (nodemap->nmf_enable_audit ?
-                       NM_FL_ENABLE_AUDIT : 0));
+                       NM_FL_ENABLE_AUDIT : 0) |
+               (nodemap->nmf_forbid_encryption ?
+                       NM_FL_FORBID_ENCRYPT : 0);
+       nr->ncr.ncr_flags2 =
+               (nodemap->nmf_readonly_mount ?
+                       NM_FL2_READONLY_MOUNT : 0);
+}
+
+static void nodemap_cluster_roles_rec_init(union nodemap_rec *nr,
+                                          const struct lu_nodemap *nodemap)
+{
+       struct nodemap_cluster_roles_rec *ncrr = &nr->ncrr;
+
+       memset(ncrr, 0, sizeof(struct nodemap_cluster_roles_rec));
+       ncrr->ncrr_roles = cpu_to_le64(nodemap->nmf_rbac);
 }
 
 static void nodemap_idmap_key_init(struct nodemap_key *nk, unsigned int nm_id,
@@ -115,8 +123,12 @@ static void nodemap_idmap_key_init(struct nodemap_key *nk, unsigned int nm_id,
 
        if (id_type == NODEMAP_UID)
                idx_type = NODEMAP_UIDMAP_IDX;
-       else
+       else if (id_type == NODEMAP_GID)
                idx_type = NODEMAP_GIDMAP_IDX;
+       else if (id_type == NODEMAP_PROJID)
+               idx_type = NODEMAP_PROJIDMAP_IDX;
+       else
+               idx_type = NODEMAP_EMPTY_IDX;
 
        nk->nk_nodemap_id = cpu_to_le32(nm_idx_set_type(nm_id, idx_type));
        nk->nk_id_client = cpu_to_le32(id_client);
@@ -249,7 +261,7 @@ static int nodemap_idx_insert(const struct lu_env *env,
        struct dt_device *dev = lu2dt_dev(idx->do_lu.lo_dev);
        int rc;
 
-       CLASSERT(sizeof(union nodemap_rec) == 32);
+       BUILD_BUG_ON(sizeof(union nodemap_rec) != 32);
 
        th = dt_trans_create(env, dev);
 
@@ -273,7 +285,7 @@ static int nodemap_idx_insert(const struct lu_env *env,
        dt_write_lock(env, idx, 0);
 
        rc = dt_insert(env, idx, (const struct dt_rec *)nr,
-                      (const struct dt_key *)nk, th, 1);
+                      (const struct dt_key *)nk, th);
 
        nodemap_inc_version(env, idx, th);
        dt_write_unlock(env, idx);
@@ -321,7 +333,7 @@ static int nodemap_idx_update(const struct lu_env *env,
                GOTO(out_lock, rc);
 
        rc = dt_insert(env, idx, (const struct dt_rec *)nr,
-                      (const struct dt_key *)nk, th, 1);
+                      (const struct dt_key *)nk, th);
        if (rc != 0)
                GOTO(out_lock, rc);
 
@@ -378,9 +390,10 @@ enum nm_add_update {
        NM_UPDATE = 1,
 };
 
-static int nodemap_idx_nodemap_add_update(const struct lu_nodemap *nodemap,
+static int nodemap_idx_cluster_add_update(const struct lu_nodemap *nodemap,
                                          struct dt_object *idx,
-                                         enum nm_add_update update)
+                                         enum nm_add_update update,
+                                         enum nodemap_cluster_rec_subid subid)
 {
        struct nodemap_key nk;
        union nodemap_rec nr;
@@ -389,43 +402,51 @@ static int nodemap_idx_nodemap_add_update(const struct lu_nodemap *nodemap,
 
        ENTRY;
 
+       if (idx == NULL) {
+               if (nodemap_mgs_ncf == NULL) {
+                       CERROR("cannot add nodemap config to non-existing MGS.\n");
+                       return -EINVAL;
+               }
+               idx = nodemap_mgs_ncf->ncf_obj;
+       }
+
        rc = lu_env_init(&env, LCT_LOCAL);
        if (rc)
                RETURN(rc);
 
-       nodemap_cluster_key_init(&nk, nodemap->nm_id);
-       nodemap_cluster_rec_init(&nr, nodemap);
+       nodemap_cluster_key_init(&nk, nodemap->nm_id, subid);
+       switch (subid) {
+       case NODEMAP_CLUSTER_REC:
+               nodemap_cluster_rec_init(&nr, nodemap);
+               break;
+       case NODEMAP_CLUSTER_ROLES:
+               nodemap_cluster_roles_rec_init(&nr, nodemap);
+               break;
+       default:
+               CWARN("%s: unknown subtype %u\n", nodemap->nm_name, subid);
+               GOTO(fini, rc = -EINVAL);
+       }
 
        if (update == NM_UPDATE)
                rc = nodemap_idx_update(&env, idx, &nk, &nr);
        else
                rc = nodemap_idx_insert(&env, idx, &nk, &nr);
 
+fini:
        lu_env_fini(&env);
-
        RETURN(rc);
 }
 
 int nodemap_idx_nodemap_add(const struct lu_nodemap *nodemap)
 {
-       if (nodemap_mgs_ncf == NULL) {
-               CERROR("cannot add nodemap config to non-existing MGS.\n");
-               return -EINVAL;
-       }
-
-       return nodemap_idx_nodemap_add_update(nodemap, nodemap_mgs_ncf->ncf_obj,
-                                             NM_ADD);
+       return nodemap_idx_cluster_add_update(nodemap, NULL,
+                                             NM_ADD, NODEMAP_CLUSTER_REC);
 }
 
 int nodemap_idx_nodemap_update(const struct lu_nodemap *nodemap)
 {
-       if (nodemap_mgs_ncf == NULL) {
-               CERROR("cannot add nodemap config to non-existing MGS.\n");
-               return -EINVAL;
-       }
-
-       return nodemap_idx_nodemap_add_update(nodemap, nodemap_mgs_ncf->ncf_obj,
-                                             NM_UPDATE);
+       return nodemap_idx_cluster_add_update(nodemap, NULL,
+                                             NM_UPDATE, NODEMAP_CLUSTER_REC);
 }
 
 int nodemap_idx_nodemap_del(const struct lu_nodemap *nodemap)
@@ -451,6 +472,11 @@ int nodemap_idx_nodemap_del(const struct lu_nodemap *nodemap)
        if (rc != 0)
                RETURN(rc);
 
+       nodemap_cluster_key_init(&nk, nodemap->nm_id, NODEMAP_CLUSTER_ROLES);
+       rc2 = nodemap_idx_delete(&env, nodemap_mgs_ncf->ncf_obj, &nk, NULL);
+       if (rc2 < 0 && rc2 != -ENOENT)
+               rc = rc2;
+
        root = nodemap->nm_fs_to_client_uidmap;
        nm_rbtree_postorder_for_each_entry_safe(idmap, temp, &root,
                                                id_fs_to_client) {
@@ -473,6 +499,17 @@ int nodemap_idx_nodemap_del(const struct lu_nodemap *nodemap)
                        rc = rc2;
        }
 
+       root = nodemap->nm_client_to_fs_projidmap;
+       nm_rbtree_postorder_for_each_entry_safe(idmap, temp, &root,
+                                               id_client_to_fs) {
+               nodemap_idmap_key_init(&nk, nodemap->nm_id, NODEMAP_PROJID,
+                                      idmap->id_client);
+               rc2 = nodemap_idx_delete(&env, nodemap_mgs_ncf->ncf_obj,
+                                        &nk, NULL);
+               if (rc2 < 0)
+                       rc = rc2;
+       }
+
        list_for_each_entry_safe(range, range_temp, &nodemap->nm_ranges,
                                 rn_list) {
                nodemap_range_key_init(&nk, nodemap->nm_id, range->rn_id);
@@ -482,7 +519,7 @@ int nodemap_idx_nodemap_del(const struct lu_nodemap *nodemap)
                        rc = rc2;
        }
 
-       nodemap_cluster_key_init(&nk, nodemap->nm_id);
+       nodemap_cluster_key_init(&nk, nodemap->nm_id, NODEMAP_CLUSTER_REC);
        rc2 = nodemap_idx_delete(&env, nodemap_mgs_ncf->ncf_obj, &nk, NULL);
        if (rc2 < 0)
                rc = rc2;
@@ -492,6 +529,42 @@ int nodemap_idx_nodemap_del(const struct lu_nodemap *nodemap)
        RETURN(rc);
 }
 
+int nodemap_idx_cluster_roles_add(const struct lu_nodemap *nodemap)
+{
+       return nodemap_idx_cluster_add_update(nodemap, NULL, NM_ADD,
+                                             NODEMAP_CLUSTER_ROLES);
+}
+
+int nodemap_idx_cluster_roles_update(const struct lu_nodemap *nodemap)
+{
+       return nodemap_idx_cluster_add_update(nodemap, NULL, NM_UPDATE,
+                                             NODEMAP_CLUSTER_ROLES);
+}
+
+int nodemap_idx_cluster_roles_del(const struct lu_nodemap *nodemap)
+{
+       struct nodemap_key nk;
+       struct lu_env env;
+       int rc = 0;
+
+       ENTRY;
+
+       if (nodemap_mgs_ncf == NULL) {
+               CERROR("cannot add nodemap config to non-existing MGS.\n");
+               return -EINVAL;
+       }
+
+       rc = lu_env_init(&env, LCT_LOCAL);
+       if (rc != 0)
+               RETURN(rc);
+
+       nodemap_cluster_key_init(&nk, nodemap->nm_id, NODEMAP_CLUSTER_ROLES);
+       rc = nodemap_idx_delete(&env, nodemap_mgs_ncf->ncf_obj, &nk, NULL);
+
+       lu_env_fini(&env);
+       RETURN(rc);
+}
+
 int nodemap_idx_range_add(const struct lu_nid_range *range,
                          const lnet_nid_t nid[2])
 {
@@ -642,6 +715,93 @@ static enum nodemap_idx_type nodemap_get_key_type(const struct nodemap_key *key)
        return nm_idx_get_type(nodemap_id);
 }
 
+static int nodemap_get_key_subtype(const struct nodemap_key *key)
+{
+       enum nodemap_idx_type type = nodemap_get_key_type(key);
+
+       return type == NODEMAP_CLUSTER_IDX ? key->nk_cluster_subid : -1;
+}
+
+static int nodemap_cluster_rec_helper(struct nodemap_config *config,
+                                     u32 nodemap_id,
+                                     const union nodemap_rec *rec,
+                                     struct lu_nodemap **recent_nodemap)
+{
+       struct lu_nodemap *nodemap, *old_nm;
+       enum nm_flag_bits flags;
+       enum nm_flag2_bits flags2;
+
+       nodemap = cfs_hash_lookup(config->nmc_nodemap_hash, rec->ncr.ncr_name);
+       if (nodemap == NULL) {
+               if (nodemap_id == LUSTRE_NODEMAP_DEFAULT_ID)
+                       nodemap = nodemap_create(rec->ncr.ncr_name, config, 1);
+               else
+                       nodemap = nodemap_create(rec->ncr.ncr_name, config, 0);
+               if (IS_ERR(nodemap))
+                       return PTR_ERR(nodemap);
+
+               /* we need to override the local ID with the saved ID */
+               nodemap->nm_id = nodemap_id;
+               if (nodemap_id > config->nmc_nodemap_highest_id)
+                       config->nmc_nodemap_highest_id = nodemap_id;
+
+       } else if (nodemap->nm_id != nodemap_id) {
+               nodemap_putref(nodemap);
+               return -EINVAL;
+       }
+
+       nodemap->nm_squash_uid = le32_to_cpu(rec->ncr.ncr_squash_uid);
+       nodemap->nm_squash_gid = le32_to_cpu(rec->ncr.ncr_squash_gid);
+       nodemap->nm_squash_projid = le32_to_cpu(rec->ncr.ncr_squash_projid);
+
+       flags = rec->ncr.ncr_flags;
+       nodemap->nmf_allow_root_access = flags & NM_FL_ALLOW_ROOT_ACCESS;
+       nodemap->nmf_trust_client_ids = flags & NM_FL_TRUST_CLIENT_IDS;
+       nodemap->nmf_deny_unknown = flags & NM_FL_DENY_UNKNOWN;
+       nodemap->nmf_map_mode =
+               (flags & NM_FL_MAP_UID ? NODEMAP_MAP_UID : 0) |
+               (flags & NM_FL_MAP_GID ? NODEMAP_MAP_GID : 0) |
+               (flags & NM_FL_MAP_PROJID ? NODEMAP_MAP_PROJID : 0);
+       if (nodemap->nmf_map_mode == NODEMAP_MAP_BOTH_LEGACY)
+               nodemap->nmf_map_mode = NODEMAP_MAP_BOTH;
+       nodemap->nmf_enable_audit = flags & NM_FL_ENABLE_AUDIT;
+       nodemap->nmf_forbid_encryption = flags & NM_FL_FORBID_ENCRYPT;
+       flags2 = rec->ncr.ncr_flags2;
+       nodemap->nmf_readonly_mount = flags2 & NM_FL2_READONLY_MOUNT;
+       /* by default, and in the absence of cluster_roles, grant all roles */
+       nodemap->nmf_rbac = NODEMAP_RBAC_ALL;
+
+       /* The fileset should be saved otherwise it will be empty
+        * every time in case of "NODEMAP_CLUSTER_IDX".
+        */
+       mutex_lock(&active_config_lock);
+       old_nm = nodemap_lookup(rec->ncr.ncr_name);
+       if (!IS_ERR(old_nm) && old_nm->nm_fileset[0] != '\0')
+               strlcpy(nodemap->nm_fileset, old_nm->nm_fileset,
+                       sizeof(nodemap->nm_fileset));
+       mutex_unlock(&active_config_lock);
+       if (!IS_ERR(old_nm))
+               nodemap_putref(old_nm);
+
+       if (*recent_nodemap == NULL) {
+               *recent_nodemap = nodemap;
+               INIT_LIST_HEAD(&nodemap->nm_list);
+       } else {
+               list_add(&nodemap->nm_list, &(*recent_nodemap)->nm_list);
+       }
+       nodemap_putref(nodemap);
+
+       return 0;
+}
+
+static int nodemap_cluster_roles_helper(struct lu_nodemap *nodemap,
+                                       const union nodemap_rec *rec)
+{
+       nodemap->nmf_rbac = le64_to_cpu(rec->ncrr.ncrr_roles);
+
+       return 0;
+}
+
 /**
  * Process a key/rec pair and modify the new configuration.
  *
@@ -663,7 +823,7 @@ static int nodemap_process_keyrec(struct nodemap_config *config,
        struct lu_nodemap *nodemap = NULL;
        enum nodemap_idx_type type;
        enum nodemap_id_type id_type;
-       u8 flags;
+       int subtype;
        u32 nodemap_id;
        lnet_nid_t nid[2];
        u32 map[2];
@@ -671,18 +831,20 @@ static int nodemap_process_keyrec(struct nodemap_config *config,
 
        ENTRY;
 
-       CLASSERT(sizeof(union nodemap_rec) == 32);
+       BUILD_BUG_ON(sizeof(union nodemap_rec) != 32);
 
        nodemap_id = le32_to_cpu(key->nk_nodemap_id);
        type = nodemap_get_key_type(key);
+       subtype = nodemap_get_key_subtype(key);
        nodemap_id = nm_idx_set_type(nodemap_id, 0);
 
-       CDEBUG(D_INFO, "found config entry, nm_id %d type %d\n",
-              nodemap_id, type);
+       CDEBUG(D_INFO, "found config entry, nm_id %d type %d subtype %d\n",
+              nodemap_id, type, subtype);
 
        /* find the correct nodemap in the load list */
        if (type == NODEMAP_RANGE_IDX || type == NODEMAP_UIDMAP_IDX ||
-           type == NODEMAP_GIDMAP_IDX) {
+           type == NODEMAP_GIDMAP_IDX || type == NODEMAP_PROJIDMAP_IDX ||
+           (type == NODEMAP_CLUSTER_IDX && subtype != NODEMAP_CLUSTER_REC)) {
                struct lu_nodemap *tmp = NULL;
 
                nodemap = *recent_nodemap;
@@ -714,57 +876,24 @@ static int nodemap_process_keyrec(struct nodemap_config *config,
                              nodemap_id);
                break;
        case NODEMAP_CLUSTER_IDX:
-               nodemap = cfs_hash_lookup(config->nmc_nodemap_hash,
-                                         rec->ncr.ncr_name);
-               if (nodemap == NULL) {
-                       if (nodemap_id == LUSTRE_NODEMAP_DEFAULT_ID) {
-                               nodemap = nodemap_create(rec->ncr.ncr_name,
-                                                        config, 1);
-                               config->nmc_default_nodemap = nodemap;
-                       } else {
-                               nodemap = nodemap_create(rec->ncr.ncr_name,
-                                                        config, 0);
-                       }
-                       if (IS_ERR(nodemap))
-                               GOTO(out, rc = PTR_ERR(nodemap));
-
-                       /* we need to override the local ID with the saved ID */
-                       nodemap->nm_id = nodemap_id;
-                       if (nodemap_id > config->nmc_nodemap_highest_id)
-                               config->nmc_nodemap_highest_id = nodemap_id;
-
-               } else if (nodemap->nm_id != nodemap_id) {
-                       nodemap_putref(nodemap);
-                       GOTO(out, rc = -EINVAL);
-               }
-
-               nodemap->nm_squash_uid =
-                               le32_to_cpu(rec->ncr.ncr_squash_uid);
-               nodemap->nm_squash_gid =
-                               le32_to_cpu(rec->ncr.ncr_squash_gid);
-
-               flags = le32_to_cpu(rec->ncr.ncr_flags);
-               nodemap->nmf_allow_root_access =
-                                       flags & NM_FL_ALLOW_ROOT_ACCESS;
-               nodemap->nmf_trust_client_ids =
-                                       flags & NM_FL_TRUST_CLIENT_IDS;
-               nodemap->nmf_deny_unknown =
-                                       flags & NM_FL_DENY_UNKNOWN;
-               nodemap->nmf_map_uid_only =
-                                       flags & NM_FL_MAP_UID_ONLY;
-               nodemap->nmf_map_gid_only =
-                                       flags & NM_FL_MAP_GID_ONLY;
-               nodemap->nmf_enable_audit =
-                                       flags & NM_FL_ENABLE_AUDIT;
-
-               if (*recent_nodemap == NULL) {
-                       *recent_nodemap = nodemap;
-                       INIT_LIST_HEAD(&nodemap->nm_list);
-               } else {
-                       list_add(&nodemap->nm_list,
-                                &(*recent_nodemap)->nm_list);
+               switch (nodemap_get_key_subtype(key)) {
+               case NODEMAP_CLUSTER_REC:
+                       rc = nodemap_cluster_rec_helper(config, nodemap_id, rec,
+                                                       recent_nodemap);
+                       if (rc != 0)
+                               GOTO(out, rc);
+                       break;
+               case NODEMAP_CLUSTER_ROLES:
+                       rc = nodemap_cluster_roles_helper(nodemap, rec);
+                       if (rc != 0)
+                               GOTO(out, rc);
+                       break;
+               default:
+                       CWARN("%s: ignoring keyrec of type %d with subtype %u\n",
+                             nodemap->nm_name, NODEMAP_CLUSTER_IDX,
+                             nodemap_get_key_subtype(key));
+                       break;
                }
-               nodemap_putref(nodemap);
                break;
        case NODEMAP_RANGE_IDX:
                nid[0] = le64_to_cpu(rec->nrr.nrr_start_nid);
@@ -777,23 +906,40 @@ static int nodemap_process_keyrec(struct nodemap_config *config,
                break;
        case NODEMAP_UIDMAP_IDX:
        case NODEMAP_GIDMAP_IDX:
+       case NODEMAP_PROJIDMAP_IDX:
                map[0] = le32_to_cpu(key->nk_id_client);
                map[1] = le32_to_cpu(rec->nir.nir_id_fs);
 
                if (type == NODEMAP_UIDMAP_IDX)
                        id_type = NODEMAP_UID;
-               else
+               else if (type == NODEMAP_GIDMAP_IDX)
                        id_type = NODEMAP_GID;
+               else if (type == NODEMAP_PROJIDMAP_IDX)
+                       id_type = NODEMAP_PROJID;
+               else
+                       GOTO(out, rc = -EINVAL);
 
                rc = nodemap_add_idmap_helper(nodemap, id_type, map);
                if (rc != 0)
                        GOTO(out, rc);
                break;
        case NODEMAP_GLOBAL_IDX:
-               config->nmc_nodemap_is_active = rec->ngr.ngr_is_active;
+               switch (key->nk_unused) {
+               case 0:
+                       config->nmc_nodemap_is_active = rec->ngr.ngr_is_active;
+                       break;
+               default:
+                       CWARN("%s: ignoring keyrec of type %d with subtype %u\n",
+                             recent_nodemap ?
+                              (*recent_nodemap)->nm_name : "nodemap",
+                             NODEMAP_GLOBAL_IDX, key->nk_unused);
+                       break;
+               }
                break;
        default:
-               CERROR("got keyrec pair for unknown type %d\n", type);
+               CWARN("%s: ignoring key %u:%u for unknown type %u\n",
+                     recent_nodemap ? (*recent_nodemap)->nm_name : "nodemap",
+                     key->nk_nodemap_id & 0x0FFFFFFF, key->nk_unused, type);
                break;
        }
 
@@ -860,14 +1006,18 @@ static int nodemap_load_entries(const struct lu_env *env,
                struct nodemap_key *key;
                union nodemap_rec rec;
                enum nodemap_idx_type key_type;
+               int sub_type;
 
                key = (struct nodemap_key *)iops->key(env, it);
                key_type = nodemap_get_key_type((struct nodemap_key *)key);
+               sub_type = nodemap_get_key_subtype((struct nodemap_key *)key);
                if ((cur_pass == NM_READ_CLUSTERS &&
-                               key_type == NODEMAP_CLUSTER_IDX) ||
+                    key_type == NODEMAP_CLUSTER_IDX &&
+                    sub_type == NODEMAP_CLUSTER_REC) ||
                    (cur_pass == NM_READ_ATTRIBUTES &&
-                               key_type != NODEMAP_CLUSTER_IDX &&
-                               key_type != NODEMAP_EMPTY_IDX)) {
+                    (key_type != NODEMAP_CLUSTER_IDX ||
+                     sub_type != NODEMAP_CLUSTER_REC) &&
+                    key_type != NODEMAP_EMPTY_IDX)) {
                        rc = iops->rec(env, it, (struct dt_rec *)&rec, 0);
                        if (rc != -ESTALE) {
                                if (rc != 0)
@@ -923,15 +1073,15 @@ out:
 
        if (new_config->nmc_default_nodemap == NULL) {
                /* new MGS won't have a default nm on disk, so create it here */
-               new_config->nmc_default_nodemap =
+               struct lu_nodemap *nodemap =
                        nodemap_create(DEFAULT_NODEMAP, new_config, 1);
-               if (IS_ERR(new_config->nmc_default_nodemap)) {
-                       rc = PTR_ERR(new_config->nmc_default_nodemap);
+               if (IS_ERR(nodemap)) {
+                       rc = PTR_ERR(nodemap);
                } else {
-                       rc = nodemap_idx_nodemap_add_update(
+                       rc = nodemap_idx_cluster_add_update(
                                        new_config->nmc_default_nodemap,
                                        nodemap_idx,
-                                       NM_ADD);
+                                       NM_ADD, NODEMAP_CLUSTER_REC);
                        nodemap_putref(new_config->nmc_default_nodemap);
                }
        }
@@ -988,7 +1138,8 @@ struct dt_object *nodemap_save_config_cache(const struct lu_env *env,
                               nm_hash_list_cb, &nodemap_list_head);
 
        list_for_each_entry_safe(nodemap, nm_tmp, &nodemap_list_head, nm_list) {
-               nodemap_cluster_key_init(&nk, nodemap->nm_id);
+               nodemap_cluster_key_init(&nk, nodemap->nm_id,
+                                        NODEMAP_CLUSTER_REC);
                nodemap_cluster_rec_init(&nr, nodemap);
 
                rc2 = nodemap_idx_insert(env, o, &nk, &nr);
@@ -997,12 +1148,24 @@ struct dt_object *nodemap_save_config_cache(const struct lu_env *env,
                        continue;
                }
 
+               /* only insert NODEMAP_CLUSTER_ROLES idx in saved config cache
+                * if nmf_rbac is not default value NODEMAP_RBAC_ALL
+                */
+               if (nodemap->nmf_rbac != NODEMAP_RBAC_ALL) {
+                       nodemap_cluster_key_init(&nk, nodemap->nm_id,
+                                                NODEMAP_CLUSTER_ROLES);
+                       nodemap_cluster_roles_rec_init(&nr, nodemap);
+                       rc2 = nodemap_idx_insert(env, o, &nk, &nr);
+                       if (rc2 < 0)
+                               rc = rc2;
+               }
+
                down_read(&active_config->nmc_range_tree_lock);
                list_for_each_entry_safe(range, range_temp, &nodemap->nm_ranges,
                                         rn_list) {
                        lnet_nid_t nid[2] = {
-                               range->rn_node.in_extent.start,
-                               range->rn_node.in_extent.end
+                               range->rn_start,
+                               range->rn_end
                        };
                        nodemap_range_key_init(&nk, nodemap->nm_id,
                                               range->rn_id);
@@ -1037,6 +1200,18 @@ struct dt_object *nodemap_save_config_cache(const struct lu_env *env,
                        if (rc2 < 0)
                                rc = rc2;
                }
+
+               root = nodemap->nm_client_to_fs_projidmap;
+               nm_rbtree_postorder_for_each_entry_safe(idmap, id_tmp, &root,
+                                                       id_client_to_fs) {
+                       nodemap_idmap_key_init(&nk, nodemap->nm_id,
+                                              NODEMAP_PROJID,
+                                              idmap->id_client);
+                       nodemap_idmap_rec_init(&nr, idmap->id_fs);
+                       rc2 = nodemap_idx_insert(env, o, &nk, &nr);
+                       if (rc2 < 0)
+                               rc = rc2;
+               }
        }
        nodemap_global_key_init(&nk);
        nodemap_global_rec_init(&nr, active_config->nmc_nodemap_is_active);
@@ -1296,8 +1471,9 @@ int nodemap_process_idx_pages(struct nodemap_config *config, union lu_page *lip,
 }
 EXPORT_SYMBOL(nodemap_process_idx_pages);
 
-static int nodemap_page_build(const struct lu_env *env, union lu_page *lp,
-                             size_t nob, const struct dt_it_ops *iops,
+static int nodemap_page_build(const struct lu_env *env, struct dt_object *obj,
+                             union lu_page *lp, size_t bytes,
+                             const struct dt_it_ops *iops,
                              struct dt_it *it, __u32 attr, void *arg)
 {
        struct idx_info *ii = (struct idx_info *)arg;
@@ -1307,20 +1483,21 @@ static int nodemap_page_build(const struct lu_env *env, union lu_page *lp,
        int rc;
        ENTRY;
 
-       if (nob < LIP_HDR_SIZE)
+       if (bytes < LIP_HDR_SIZE)
                return -EINVAL;
 
        /* initialize the header of the new container */
        memset(lip, 0, LIP_HDR_SIZE);
        lip->lip_magic = LIP_MAGIC;
-       nob           -= LIP_HDR_SIZE;
+       bytes -= LIP_HDR_SIZE;
 
        entry = lip->lip_entries;
        do {
-               char            *tmp_entry = entry;
-               struct dt_key   *key;
-               __u64           hash;
+               char *tmp_entry = entry;
+               struct dt_key *key;
+               __u64 hash;
                enum nodemap_idx_type key_type;
+               int sub_type;
 
                /* fetch 64-bit hash value */
                hash = iops->store(env, it);
@@ -1331,7 +1508,7 @@ static int nodemap_page_build(const struct lu_env *env, union lu_page *lp,
                                GOTO(out, rc = 0);
                }
 
-               if (nob < size) {
+               if (bytes < size) {
                        if (lip->lip_nr == 0)
                                GOTO(out, rc = -EINVAL);
                        GOTO(out, rc = 0);
@@ -1339,14 +1516,17 @@ static int nodemap_page_build(const struct lu_env *env, union lu_page *lp,
 
                key = iops->key(env, it);
                key_type = nodemap_get_key_type((struct nodemap_key *)key);
+               sub_type = nodemap_get_key_subtype((struct nodemap_key *)key);
 
                /* on the first pass, get only the cluster types. On second
                 * pass, get all the rest */
                if ((ii->ii_attrs == NM_READ_CLUSTERS &&
-                               key_type == NODEMAP_CLUSTER_IDX) ||
+                    key_type == NODEMAP_CLUSTER_IDX &&
+                    sub_type == NODEMAP_CLUSTER_REC) ||
                    (ii->ii_attrs == NM_READ_ATTRIBUTES &&
-                               key_type != NODEMAP_CLUSTER_IDX &&
-                               key_type != NODEMAP_EMPTY_IDX)) {
+                    (key_type != NODEMAP_CLUSTER_IDX ||
+                     sub_type != NODEMAP_CLUSTER_REC) &&
+                    key_type != NODEMAP_EMPTY_IDX)) {
                        memcpy(tmp_entry, key, ii->ii_keysize);
                        tmp_entry += ii->ii_keysize;
 
@@ -1364,7 +1544,7 @@ static int nodemap_page_build(const struct lu_env *env, union lu_page *lp,
                                        ii->ii_hash_start = hash;
 
                                entry = tmp_entry + ii->ii_recsize;
-                               nob -= size;
+                               bytes -= size;
                        }
                }
 
@@ -1446,12 +1626,12 @@ EXPORT_SYMBOL(nodemap_index_read);
 int nodemap_get_config_req(struct obd_device *mgs_obd,
                           struct ptlrpc_request *req)
 {
+       const struct ptlrpc_bulk_frag_ops *frag_ops = &ptlrpc_bulk_kiov_pin_ops;
        struct mgs_config_body *body;
        struct mgs_config_res *res;
        struct lu_rdpg rdpg;
        struct idx_info nodemap_ii;
        struct ptlrpc_bulk_desc *desc;
-       struct l_wait_info lwi;
        struct tg_export_data *rqexp_ted = &req->rq_export->exp_target_data;
        int i;
        int page_count;
@@ -1462,7 +1642,7 @@ int nodemap_get_config_req(struct obd_device *mgs_obd,
        if (!body)
                RETURN(-EINVAL);
 
-       if (body->mcb_type != CONFIG_T_NODEMAP)
+       if (body->mcb_type != MGS_CFG_T_NODEMAP)
                RETURN(-EINVAL);
 
        rdpg.rp_count = (body->mcb_units << body->mcb_bits);
@@ -1475,7 +1655,7 @@ int nodemap_get_config_req(struct obd_device *mgs_obd,
               body->mcb_name, rdpg.rp_count);
 
        /* allocate pages to store the containers */
-       OBD_ALLOC(rdpg.rp_pages, sizeof(*rdpg.rp_pages) * rdpg.rp_npages);
+       OBD_ALLOC_PTR_ARRAY(rdpg.rp_pages, rdpg.rp_npages);
        if (rdpg.rp_pages == NULL)
                RETURN(-ENOMEM);
        for (i = 0; i < rdpg.rp_npages; i++) {
@@ -1491,7 +1671,7 @@ int nodemap_get_config_req(struct obd_device *mgs_obd,
        nodemap_ii.ii_attrs = body->mcb_nm_cur_pass;
 
        bytes = nodemap_index_read(req->rq_svc_thread->t_env,
-                                  mgs_obd->u.obt.obt_nodemap_config_file,
+                                  obd2obt(mgs_obd)->obt_nodemap_config_file,
                                   &nodemap_ii, &rdpg);
        if (bytes < 0)
                GOTO(out, rc = bytes);
@@ -1507,20 +1687,18 @@ int nodemap_get_config_req(struct obd_device *mgs_obd,
        page_count = (bytes + PAGE_SIZE - 1) >> PAGE_SHIFT;
        LASSERT(page_count <= rdpg.rp_count);
        desc = ptlrpc_prep_bulk_exp(req, page_count, 1,
-                                   PTLRPC_BULK_PUT_SOURCE |
-                                       PTLRPC_BULK_BUF_KIOV,
-                                   MGS_BULK_PORTAL,
-                                   &ptlrpc_bulk_kiov_pin_ops);
+                                   PTLRPC_BULK_PUT_SOURCE,
+                                   MGS_BULK_PORTAL, frag_ops);
        if (desc == NULL)
                GOTO(out, rc = -ENOMEM);
 
        for (i = 0; i < page_count && bytes > 0; i++) {
-               ptlrpc_prep_bulk_page_pin(desc, rdpg.rp_pages[i], 0,
-                                         min_t(int, bytes, PAGE_SIZE));
+               frag_ops->add_kiov_frag(desc, rdpg.rp_pages[i], 0,
+                                       min_t(int, bytes, PAGE_SIZE));
                bytes -= PAGE_SIZE;
        }
 
-       rc = target_bulk_io(req->rq_export, desc, &lwi);
+       rc = target_bulk_io(req->rq_export, desc);
        ptlrpc_free_bulk(desc);
 
 out:
@@ -1528,8 +1706,7 @@ out:
                for (i = 0; i < rdpg.rp_npages; i++)
                        if (rdpg.rp_pages[i] != NULL)
                                __free_page(rdpg.rp_pages[i]);
-               OBD_FREE(rdpg.rp_pages,
-                        rdpg.rp_npages * sizeof(rdpg.rp_pages[0]));
+               OBD_FREE_PTR_ARRAY(rdpg.rp_pages, rdpg.rp_npages);
        }
        return rc;
 }