From 971e025f5fb77f4eaaa1e9070598dfa6292a9678 Mon Sep 17 00:00:00 2001
From: Sebastien Buisson <sbuisson@ddn.com>
Date: Fri, 3 Feb 2023 14:11:51 +0100
Subject: [PATCH] LU-16524 sec: enforce rbac roles

There are 5 different rbac roles defined via nodemap:
- byfid_ops, to allow operations by FID (e.g. 'lfs rmfid').
- chlg_ops, to allow access to Lustre Changelogs.
- dne_ops, to allow operations related to DNE (e.g. 'lfs mkdir').
- file_perms, to allow modifications of file permissions and owners.
- quota_ops, to allow quota modifications.
Enforce these roles by checking the value of the 'rbac' nodemap
property on server side and returning -EPERM if operation is
forbidden.

Add sanity-sec test_64* to exercise these capabilities.

Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Change-Id: I37057f0ab50c02fa99db03cb04149a437e35ee0a
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49907
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Patrick Farrell <pfarrell@whamcloud.com>
---
 lustre/include/md_object.h      |   5 +
 lustre/mdd/mdd_object.c         |  44 ++++--
 lustre/mdt/mdt_coordinator.c    |   6 +
 lustre/mdt/mdt_handler.c        |  15 +-
 lustre/mdt/mdt_internal.h       |   5 +-
 lustre/mdt/mdt_lib.c            |  23 +++
 lustre/mdt/mdt_open.c           |   8 +
 lustre/mdt/mdt_reint.c          |  37 +++--
 lustre/mdt/mdt_restripe.c       |   6 +
 lustre/mdt/mdt_xattr.c          |  10 +-
 lustre/obdecho/echo_client.c    |   6 +
 lustre/ptlrpc/nodemap_handler.c |   3 +-
 lustre/tests/sanity-sec.sh      | 323 ++++++++++++++++++++++++++++++++++++++++
 13 files changed, 459 insertions(+), 32 deletions(-)

diff --git a/lustre/include/md_object.h b/lustre/include/md_object.h
index daa62d8..f849fee 100644
--- a/lustre/include/md_object.h
+++ b/lustre/include/md_object.h
@@ -717,6 +717,11 @@ struct lu_ucred {
 	char			 uc_jobid[LUSTRE_JOBID_SIZE];
 	lnet_nid_t		 uc_nid;
 	bool			 uc_enable_audit;
+	int			 uc_rbac_file_perms:1;
+	int			 uc_rbac_dne_ops:1;
+	int			 uc_rbac_quota_ops:1;
+	int			 uc_rbac_byfid_ops:1;
+	int			 uc_rbac_chlg_ops:1;
 };
 
 struct lu_ucred *lu_ucred(const struct lu_env *env);
diff --git a/lustre/mdd/mdd_object.c b/lustre/mdd/mdd_object.c
index ed8120a..adfc1eb 100644
--- a/lustre/mdd/mdd_object.c
+++ b/lustre/mdd/mdd_object.c
@@ -691,10 +691,14 @@ static int mdd_fix_attr(const struct lu_env *env, struct mdd_object *obj,
 		RETURN(0);
 
 	if (is_project_state_change(oattr, la)) {
-		if (!cap_raised(uc->uc_cap, CAP_SYS_RESOURCE) &&
-		    !lustre_in_group_p(uc, ma->ma_enable_chprojid_gid) &&
-		    !(ma->ma_enable_chprojid_gid == -1 &&
-		      mdd_permission_internal(env, obj, oattr, MAY_WRITE)))
+		/* we want rbac roles to have precedence over any other
+		 * permission or capability checks
+		 */
+		if (!uc->uc_rbac_file_perms ||
+		    (!cap_raised(uc->uc_cap, CAP_SYS_RESOURCE) &&
+		     !lustre_in_group_p(uc, ma->ma_enable_chprojid_gid) &&
+		     !(ma->ma_enable_chprojid_gid == -1 &&
+		       mdd_permission_internal(env, obj, oattr, MAY_WRITE))))
 			RETURN(-EPERM);
 	}
 
@@ -795,9 +799,13 @@ static int mdd_fix_attr(const struct lu_env *env, struct mdd_object *obj,
 
 	/* Make sure a caller can chmod. */
 	if (la->la_valid & LA_MODE) {
-		if (!(flags & MDS_PERM_BYPASS) &&
-		    (uc->uc_fsuid != oattr->la_uid) &&
-		    !cap_raised(uc->uc_cap, CAP_FOWNER))
+		/* we want rbac roles to have precedence over any other
+		 * permission or capability checks
+		 */
+		if (!uc->uc_rbac_file_perms ||
+		    (!(flags & MDS_PERM_BYPASS) &&
+		     (uc->uc_fsuid != oattr->la_uid) &&
+		     !cap_raised(uc->uc_cap, CAP_FOWNER)))
 			RETURN(-EPERM);
 
 		if (la->la_mode == (umode_t) -1)
@@ -819,9 +827,13 @@ static int mdd_fix_attr(const struct lu_env *env, struct mdd_object *obj,
 	if (la->la_valid & LA_UID) {
 		if (la->la_uid == (uid_t) -1)
 			la->la_uid = oattr->la_uid;
-		if (((uc->uc_fsuid != oattr->la_uid) ||
-		     (la->la_uid != oattr->la_uid)) &&
-		    !cap_raised(uc->uc_cap, CAP_CHOWN))
+		/* we want rbac roles to have precedence over any other
+		 * permission or capability checks
+		 */
+		if (!uc->uc_rbac_file_perms ||
+		    ((uc->uc_fsuid != oattr->la_uid ||
+		      la->la_uid != oattr->la_uid) &&
+		     !cap_raised(uc->uc_cap, CAP_CHOWN)))
 			RETURN(-EPERM);
 
 		/* If the user or group of a non-directory has been
@@ -844,10 +856,14 @@ static int mdd_fix_attr(const struct lu_env *env, struct mdd_object *obj,
 	if (la->la_valid & LA_GID) {
 		if (la->la_gid == (gid_t) -1)
 			la->la_gid = oattr->la_gid;
-		if (((uc->uc_fsuid != oattr->la_uid) ||
-		     ((la->la_gid != oattr->la_gid) &&
-		      !lustre_in_group_p(uc, la->la_gid))) &&
-		    !cap_raised(uc->uc_cap, CAP_CHOWN))
+		/* we want rbac roles to have precedence over any other
+		 * permission or capability checks
+		 */
+		if (!uc->uc_rbac_file_perms ||
+		    ((uc->uc_fsuid != oattr->la_uid ||
+		      (la->la_gid != oattr->la_gid &&
+		       !lustre_in_group_p(uc, la->la_gid))) &&
+		     !cap_raised(uc->uc_cap, CAP_CHOWN)))
 			RETURN(-EPERM);
 
 		/* Likewise, if the user or group of a non-directory
diff --git a/lustre/mdt/mdt_coordinator.c b/lustre/mdt/mdt_coordinator.c
index d64a22e..f212c25 100644
--- a/lustre/mdt/mdt_coordinator.c
+++ b/lustre/mdt/mdt_coordinator.c
@@ -1030,6 +1030,12 @@ int hsm_init_ucred(struct lu_ucred *uc)
 	uc->uc_identity = NULL;
 	/* always record internal HSM activity if also enabled globally */
 	uc->uc_enable_audit = 1;
+	/* do not let rbac interfere with HSM internal processing */
+	uc->uc_rbac_file_perms = 1;
+	uc->uc_rbac_dne_ops = 1;
+	uc->uc_rbac_quota_ops = 1;
+	uc->uc_rbac_byfid_ops = 1;
+	uc->uc_rbac_chlg_ops = 1;
 
 	RETURN(0);
 }
diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index 01a3610..9af1a5d 100644
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -2548,6 +2548,11 @@ static int mdt_rmfid_check_permission(struct mdt_thread_info *info,
 	if (la->la_flags & LUSTRE_IMMUTABLE_FL)
 			rc = -EACCES;
 
+	/* we want rbac roles to have precedence over any other
+	 * permission or capability checks
+	 */
+	if (!uc->uc_rbac_byfid_ops)
+		RETURN(-EPERM);
 	if (cap_raised(uc->uc_cap, CAP_DAC_OVERRIDE))
 		RETURN(0);
 	if (uc->uc_fsuid == la->la_uid) {
@@ -2779,7 +2784,7 @@ static int mdt_set_info(struct tgt_session_info *tsi)
 			__swab32s(&cs->cs_id);
 		}
 
-		if (!mdt_is_rootadmin(tsi2mdt_info(tsi)))
+		if (!mdt_changelog_allow(tsi2mdt_info(tsi)))
 			RETURN(-EACCES);
 		rc = mdt_iocontrol(OBD_IOC_CHANGELOG_CLEAR, req->rq_export,
 				   vallen, val, NULL);
@@ -5719,7 +5724,7 @@ static int mdt_llog_open(struct tgt_session_info *tsi)
 {
 	ENTRY;
 
-	if (!mdt_is_rootadmin(tsi2mdt_info(tsi)))
+	if (!mdt_changelog_allow(tsi2mdt_info(tsi)))
 		RETURN(err_serious(-EACCES));
 
 	RETURN(tgt_llog_open(tsi));
@@ -6714,6 +6719,12 @@ static int mdt_ctxt_add_dirty_flag(struct lu_env *env,
 	lu_context_enter(&ses);
 
 	mdt_ucred(info)->uc_valid = UCRED_OLD;
+	/* do not let rbac interfere with dirty flag internal system event */
+	mdt_ucred(info)->uc_rbac_file_perms = 1;
+	mdt_ucred(info)->uc_rbac_dne_ops = 1;
+	mdt_ucred(info)->uc_rbac_quota_ops = 1;
+	mdt_ucred(info)->uc_rbac_byfid_ops = 1;
+	mdt_ucred(info)->uc_rbac_chlg_ops = 1;
 	rc = mdt_add_dirty_flag(info, mfd->mfd_object, &info->mti_attr);
 
 	lu_context_exit(&ses);
diff --git a/lustre/mdt/mdt_internal.h b/lustre/mdt/mdt_internal.h
index 60d6d30..8b95ff9 100644
--- a/lustre/mdt/mdt_internal.h
+++ b/lustre/mdt/mdt_internal.h
@@ -1432,7 +1432,7 @@ long mdt_grant_connect(const struct lu_env *env, struct obd_export *exp,
 		       u64 want, bool conservative);
 extern struct kmem_cache *ldlm_glimpse_work_kmem;
 
-static inline bool mdt_is_rootadmin(struct mdt_thread_info *info)
+static inline bool mdt_changelog_allow(struct mdt_thread_info *info)
 {
 	struct lu_ucred *uc = NULL;
 	bool is_admin;
@@ -1452,7 +1452,8 @@ static inline bool mdt_is_rootadmin(struct mdt_thread_info *info)
 
 	uc = mdt_ucred(info);
 	is_admin = (uc->uc_uid == 0 && uc->uc_gid == 0 &&
-		    cap_raised(uc->uc_cap, CAP_SYS_ADMIN));
+		    cap_raised(uc->uc_cap, CAP_SYS_ADMIN) &&
+		    uc->uc_rbac_chlg_ops);
 
 	mdt_exit_ucred(info);
 
diff --git a/lustre/mdt/mdt_lib.c b/lustre/mdt/mdt_lib.c
index 8099fa7..33eddf3 100644
--- a/lustre/mdt/mdt_lib.c
+++ b/lustre/mdt/mdt_lib.c
@@ -165,6 +165,27 @@ static void ucred_set_audit_enabled(struct mdt_thread_info *info,
 	uc->uc_enable_audit = audit;
 }
 
+static void ucred_set_rbac_roles(struct mdt_thread_info *info,
+				 struct lu_ucred *uc)
+{
+	struct lu_nodemap *nodemap = NULL;
+	enum nodemap_rbac_roles rbac = NODEMAP_RBAC_ALL;
+
+	if (info && info->mti_exp) {
+		nodemap = nodemap_get_from_exp(info->mti_exp);
+		if (!IS_ERR_OR_NULL(nodemap)) {
+			rbac = nodemap->nmf_rbac;
+			nodemap_putref(nodemap);
+		}
+	}
+
+	uc->uc_rbac_file_perms = !!(rbac & NODEMAP_RBAC_FILE_PERMS);
+	uc->uc_rbac_dne_ops = !!(rbac & NODEMAP_RBAC_DNE_OPS);
+	uc->uc_rbac_quota_ops = !!(rbac & NODEMAP_RBAC_QUOTA_OPS);
+	uc->uc_rbac_byfid_ops = !!(rbac & NODEMAP_RBAC_BYFID_OPS);
+	uc->uc_rbac_chlg_ops = !!(rbac & NODEMAP_RBAC_CHLG_OPS);
+}
+
 static int new_init_ucred(struct mdt_thread_info *info, ucred_init_type_t type,
 			  void *buf)
 {
@@ -330,6 +351,7 @@ static int new_init_ucred(struct mdt_thread_info *info, ucred_init_type_t type,
 	ucred_set_jobid(info, ucred);
 	ucred_set_nid(info, ucred);
 	ucred_set_audit_enabled(info, ucred);
+	ucred_set_rbac_roles(info, ucred);
 
 	EXIT;
 
@@ -501,6 +523,7 @@ static int old_init_ucred_common(struct mdt_thread_info *info,
 	ucred_set_jobid(info, uc);
 	ucred_set_nid(info, uc);
 	ucred_set_audit_enabled(info, uc);
+	ucred_set_rbac_roles(info, uc);
 
 	EXIT;
 
diff --git a/lustre/mdt/mdt_open.c b/lustre/mdt/mdt_open.c
index 7cbe4c9..c698cc8 100644
--- a/lustre/mdt/mdt_open.c
+++ b/lustre/mdt/mdt_open.c
@@ -1328,6 +1328,7 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
 	struct ldlm_reply *ldlm_rep;
 	struct mdt_body *repbody;
 	struct lu_fid *child_fid = &info->mti_tmp_fid1;
+	struct lu_ucred *uc = mdt_ucred(info);
 	struct md_attr *ma = &info->mti_attr;
 	u64 open_flags = info->mti_spec.sp_cr_flags;
 	u64 ibits = 0;
@@ -1370,6 +1371,13 @@ int mdt_reint_open(struct mdt_thread_info *info, struct mdt_lock_handle *lhc)
 	       PFID(rr->rr_fid1), PNAME(&rr->rr_name), PFID(rr->rr_fid2),
 	       open_flags, ma->ma_attr.la_mode, msg_flags);
 
+	/* Prevent by-fid operation if parent fid is .lustre/fid.
+	 * Also, we want rbac roles to have precedence over any other
+	 * permission or capability checks
+	 */
+	if (lu_fid_eq(rr->rr_fid1, &LU_OBF_FID) && !uc->uc_rbac_byfid_ops)
+		GOTO(out, result = -EPERM);
+
 	if (info->mti_cross_ref) {
 		/* This is cross-ref open */
 		mdt_set_disposition(info, ldlm_rep,
diff --git a/lustre/mdt/mdt_reint.c b/lustre/mdt/mdt_reint.c
index 41ad354..b16b197 100644
--- a/lustre/mdt/mdt_reint.c
+++ b/lustre/mdt/mdt_reint.c
@@ -355,6 +355,7 @@ static int mdt_restripe(struct mdt_thread_info *info,
 	struct lu_fid *fid = &info->mti_tmp_fid2;
 	struct ldlm_enqueue_info *einfo = &info->mti_einfo[0];
 	struct lmv_user_md *lum = spec->u.sp_ea.eadata;
+	struct lu_ucred *uc = mdt_ucred(info);
 	struct lmv_mds_md_v1 *lmv;
 	struct mdt_object *child;
 	struct mdt_lock_handle *lhp;
@@ -363,7 +364,11 @@ static int mdt_restripe(struct mdt_thread_info *info,
 	int rc;
 
 	ENTRY;
-	if (!mdt->mdt_enable_dir_restripe)
+
+	/* we want rbac roles to have precedence over any other
+	 * permission or capability checks
+	 */
+	if (!mdt->mdt_enable_dir_restripe && !uc->uc_rbac_dne_ops)
 		RETURN(-EPERM);
 
 	LASSERT(lum);
@@ -543,9 +548,13 @@ static int mdt_create(struct mdt_thread_info *info)
 		    LMV_HASH_TYPE_CRUSH)
 			RETURN(-EPROTO);
 
-		if (!cap_raised(uc->uc_cap, CAP_SYS_ADMIN) &&
-		    uc->uc_gid != mdt->mdt_enable_remote_dir_gid &&
-		    mdt->mdt_enable_remote_dir_gid != -1)
+		/* we want rbac roles to have precedence over any other
+		 * permission or capability checks
+		 */
+		if (!uc->uc_rbac_dne_ops ||
+		    (!cap_raised(uc->uc_cap, CAP_SYS_ADMIN) &&
+		     uc->uc_gid != mdt->mdt_enable_remote_dir_gid &&
+		     mdt->mdt_enable_remote_dir_gid != -1))
 			RETURN(-EPERM);
 
 		/* restripe if later found dir exists, MDS_OPEN_CREAT means
@@ -935,9 +944,13 @@ static int mdt_reint_setattr(struct mdt_thread_info *info,
 			    !mdt->mdt_enable_striped_dir)
 				GOTO(out_put, rc = -EPERM);
 
-			if (!cap_raised(uc->uc_cap, CAP_SYS_ADMIN) &&
-			    uc->uc_gid != mdt->mdt_enable_remote_dir_gid &&
-			    mdt->mdt_enable_remote_dir_gid != -1)
+			/* we want rbac roles to have precedence over any other
+			 * permission or capability checks
+			 */
+			if (!uc->uc_rbac_dne_ops ||
+			    (!cap_raised(uc->uc_cap, CAP_SYS_ADMIN) &&
+			     uc->uc_gid != mdt->mdt_enable_remote_dir_gid &&
+			     mdt->mdt_enable_remote_dir_gid != -1))
 				GOTO(out_put, rc = -EPERM);
 		}
 
@@ -2262,9 +2275,13 @@ int mdt_reint_migrate(struct mdt_thread_info *info,
 	if (!mdt->mdt_enable_remote_dir || !mdt->mdt_enable_dir_migration)
 		RETURN(-EPERM);
 
-	if (uc && !cap_raised(uc->uc_cap, CAP_SYS_ADMIN) &&
-	    uc->uc_gid != mdt->mdt_enable_remote_dir_gid &&
-	    mdt->mdt_enable_remote_dir_gid != -1)
+	/* we want rbac roles to have precedence over any other
+	 * permission or capability checks
+	 */
+	if (uc && (!uc->uc_rbac_dne_ops ||
+		   (!cap_raised(uc->uc_cap, CAP_SYS_ADMIN) &&
+		    uc->uc_gid != mdt->mdt_enable_remote_dir_gid &&
+		    mdt->mdt_enable_remote_dir_gid != -1)))
 		RETURN(-EPERM);
 
 	/*
diff --git a/lustre/mdt/mdt_restripe.c b/lustre/mdt/mdt_restripe.c
index 07fe836..e6c2e0e 100644
--- a/lustre/mdt/mdt_restripe.c
+++ b/lustre/mdt/mdt_restripe.c
@@ -937,6 +937,12 @@ int mdt_restriper_start(struct mdt_device *mdt)
 	uc->uc_umask = 0644;
 	uc->uc_ginfo = NULL;
 	uc->uc_identity = NULL;
+	/* do not let rbac interfere with restriper internal processing */
+	uc->uc_rbac_file_perms = 1;
+	uc->uc_rbac_dne_ops = 1;
+	uc->uc_rbac_quota_ops = 1;
+	uc->uc_rbac_byfid_ops = 1;
+	uc->uc_rbac_chlg_ops = 1;
 
 	task = kthread_create(mdt_restriper_main, info, "mdt_restriper_%03d",
 			      mdt_seq_site(mdt)->ss_node_id);
diff --git a/lustre/mdt/mdt_xattr.c b/lustre/mdt/mdt_xattr.c
index 819086b..b51f9b2 100644
--- a/lustre/mdt/mdt_xattr.c
+++ b/lustre/mdt/mdt_xattr.c
@@ -346,9 +346,13 @@ int mdt_dir_layout_update(struct mdt_thread_info *info)
 	if (!mdt->mdt_enable_dir_migration)
 		RETURN(-EPERM);
 
-	if (!cap_raised(uc->uc_cap, CAP_SYS_ADMIN) &&
-	    uc->uc_gid != mdt->mdt_enable_remote_dir_gid &&
-	    mdt->mdt_enable_remote_dir_gid != -1)
+	/* we want rbac roles to have precedence over any other
+	 * permission or capability checks
+	 */
+	if (!uc->uc_rbac_dne_ops ||
+	    (!cap_raised(uc->uc_cap, CAP_SYS_ADMIN) &&
+	     uc->uc_gid != mdt->mdt_enable_remote_dir_gid &&
+	     mdt->mdt_enable_remote_dir_gid != -1))
 		RETURN(-EPERM);
 
 	obj = mdt_object_find(env, mdt, rr->rr_fid1);
diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c
index f9af2e2..6084317 100644
--- a/lustre/obdecho/echo_client.c
+++ b/lustre/obdecho/echo_client.c
@@ -1831,6 +1831,12 @@ static void echo_ucred_init(struct lu_env *env)
 	}
 	ucred->uc_cap = kcap;
 	ucred->uc_valid = UCRED_NEW;
+	/* do not let rbac interfere with obdecho */
+	ucred->uc_rbac_file_perms = 1;
+	ucred->uc_rbac_dne_ops = 1;
+	ucred->uc_rbac_quota_ops = 1;
+	ucred->uc_rbac_byfid_ops = 1;
+	ucred->uc_rbac_chlg_ops = 1;
 }
 
 static void echo_ucred_fini(struct lu_env *env)
diff --git a/lustre/ptlrpc/nodemap_handler.c b/lustre/ptlrpc/nodemap_handler.c
index 673b613..159ea99 100644
--- a/lustre/ptlrpc/nodemap_handler.c
+++ b/lustre/ptlrpc/nodemap_handler.c
@@ -1489,7 +1489,8 @@ bool nodemap_can_setquota(struct lu_nodemap *nodemap, __u32 qc_type, __u32 id)
 	if (!nodemap_active)
 		return true;
 
-	if (!nodemap || !nodemap->nmf_allow_root_access)
+	if (!nodemap || !nodemap->nmf_allow_root_access ||
+	    !(nodemap->nmf_rbac & NODEMAP_RBAC_QUOTA_OPS))
 		return false;
 
 	if (qc_type == PRJQUOTA) {
diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh
index 072c33c..f60112d 100755
--- a/lustre/tests/sanity-sec.sh
+++ b/lustre/tests/sanity-sec.sh
@@ -5347,6 +5347,329 @@ test_63() {
 }
 run_test 63 "fid2path with encrypted files"
 
+setup_64() {
+	do_facet mgs $LCTL nodemap_activate 1
+	wait_nm_sync active
+
+	do_facet mgs $LCTL nodemap_del c0 || true
+	wait_nm_sync c0 id ''
+
+	do_facet mgs $LCTL nodemap_modify --name default \
+		--property admin --value 1
+	do_facet mgs $LCTL nodemap_modify --name default \
+		--property trusted --value 1
+	wait_nm_sync default admin_nodemap
+	wait_nm_sync default trusted_nodemap
+
+	client_ip=$(host_nids_address $HOSTNAME $NETTYPE)
+	client_nid=$(h2nettype $client_ip)
+	do_facet mgs $LCTL nodemap_add c0
+	do_facet mgs $LCTL nodemap_add_range \
+		 --name c0 --range $client_nid
+	do_facet mgs $LCTL nodemap_modify --name c0 \
+		 --property admin --value 1
+	do_facet mgs $LCTL nodemap_modify --name c0 \
+		 --property trusted --value 1
+	wait_nm_sync c0 admin_nodemap
+	wait_nm_sync c0 trusted_nodemap
+}
+
+cleanup_64() {
+	do_facet mgs $LCTL nodemap_del c0
+	do_facet mgs $LCTL nodemap_modify --name default \
+		 --property admin --value 0
+	do_facet mgs $LCTL nodemap_modify --name default \
+		 --property trusted --value 0
+	wait_nm_sync default admin_nodemap
+	wait_nm_sync default trusted_nodemap
+
+	do_facet mgs $LCTL nodemap_activate 0
+	wait_nm_sync active 0
+}
+
+test_64a() {
+	local testfile=$DIR/$tdir/$tfile
+	local rbac
+
+	(( MDS1_VERSION >= $(version_code 2.15.54) )) ||
+		skip "Need MDS >= 2.15.54 for role-based controls"
+
+	stack_trap cleanup_64 EXIT
+	mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+	setup_64
+
+	# check default value for rbac is all
+	rbac=$(do_facet mds $LCTL get_param -n nodemap.c0.rbac)
+	for role in file_perms \
+		    dne_ops \
+		    quota_ops \
+		    byfid_ops \
+		    chlg_ops \
+		    ;
+	do
+		[[ "$rbac" =~ "$role" ]] ||
+			error "role '$role' not in default '$rbac'"
+	done
+
+	do_facet mgs $LCTL nodemap_modify --name c0 \
+		 --property rbac --value file_perms
+	wait_nm_sync c0 rbac
+	touch $testfile
+	stack_trap "set +vx"
+	set -vx
+	chmod 777 $testfile || error "chmod failed"
+	chown $TSTUSR:$TSTUSR $testfile || error "chown failed"
+	chgrp $TSTUSR $testfile || error "chgrp failed"
+	$LFS project -p 1000 $testfile || error "setting project failed"
+	set +vx
+	rm -f $testfile
+	do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
+	wait_nm_sync c0 rbac
+	touch $testfile
+	set -vx
+	chmod 777 $testfile && error "chmod should fail"
+	chown $TSTUSR:$TSTUSR $testfile && error "chown should fail"
+	chgrp $TSTUSR $testfile && error "chgrp should fail"
+	$LFS project -p 1000 $testfile && error "setting project should fail"
+	set +vx
+}
+run_test 64a "Nodemap enforces file_perms RBAC roles"
+
+test_64b() {
+	local testdir=$DIR/$tdir/${tfile}.d
+	local dir_restripe
+
+	(( MDS1_VERSION >= $(version_code 2.15.54) )) ||
+		skip "Need MDS >= 2.15.54 for role-based controls"
+
+	(( MDSCOUNT >= 2 )) || skip "mdt count $MDSCOUNT, skipping dne_ops role"
+
+	stack_trap cleanup_64 EXIT
+	mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+	setup_64
+
+        dir_restripe=$(do_node $mds1_HOST \
+		"$LCTL get_param -n mdt.*MDT0000.enable_dir_restripe")
+	[ -n "$dir_restripe" ] || dir_restripe=0
+	do_nodes $(comma_list $(all_mdts_nodes)) \
+		$LCTL set_param mdt.*.enable_dir_restripe=1 ||
+			error "enabling dir_restripe failed"
+	stack_trap "do_nodes $(comma_list $(all_mdts_nodes)) \
+	      $LCTL set_param mdt.*.enable_dir_restripe=$dir_restripe" EXIT
+	do_facet mgs $LCTL nodemap_modify --name c0 --property rbac \
+		 --value dne_ops
+	wait_nm_sync c0 rbac
+	$LFS mkdir -i 0 ${testdir}_for_migr ||
+		error "$LFS mkdir ${testdir}_for_migr failed (1)"
+	touch ${testdir}_for_migr/file001 ||
+		error "touch ${testdir}_for_migr/file001 failed (1)"
+	$LFS mkdir -i 0 ${testdir}_mdt0 ||
+		error "$LFS mkdir ${testdir}_mdt0 failed (1)"
+	$LFS mkdir -i 1 ${testdir}_mdt1 ||
+		error "$LFS mkdir ${testdir}_mdt1 failed (1)"
+	set -vx
+	$LFS mkdir -i 1 $testdir || error "$LFS mkdir failed (1)"
+	rmdir $testdir
+	$LFS mkdir -c 2 $testdir || error "$LFS mkdir failed (2)"
+	rmdir $testdir
+	mkdir $testdir
+	$LFS setdirstripe -c 2 $testdir || error "$LFS setdirstripe failed"
+	rmdir $testdir
+	$LFS migrate -m 1 ${testdir}_for_migr || error "$LFS migrate failed"
+	touch ${testdir}_mdt0/fileA || error "touch fileA failed (1)"
+	mv ${testdir}_mdt0/fileA ${testdir}_mdt1/ || error "mv failed (1)"
+	set +vx
+	rm -rf ${testdir}*
+	$LFS mkdir -i 0 ${testdir}_for_migr ||
+		error "$LFS mkdir ${testdir}_for_migr failed (2)"
+	touch ${testdir}_for_migr/file001 ||
+		error "touch ${testdir}_for_migr/file001 failed (2)"
+	$LFS mkdir -i 0 ${testdir}_mdt0 ||
+		error "$LFS mkdir ${testdir}_mdt0 failed (2)"
+	$LFS mkdir -i 1 ${testdir}_mdt1 ||
+		error "$LFS mkdir ${testdir}_mdt1 failed (2)"
+
+	do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
+	wait_nm_sync c0 rbac
+	set -vx
+	$LFS mkdir -i 1 $testdir && error "$LFS mkdir should fail (1)"
+	$LFS mkdir -c 2 $testdir && error "$LFS mkdir should fail (2)"
+	mkdir $testdir
+	$LFS setdirstripe -c 2 $testdir && error "$LFS setdirstripe should fail"
+	rmdir $testdir
+	$LFS migrate -m 1 ${testdir}_for_migr &&
+		error "$LFS migrate should fail"
+	touch ${testdir}_mdt0/fileA || error "touch fileA failed (2)"
+	mv ${testdir}_mdt0/fileA ${testdir}_mdt1/ || error "mv failed (2)"
+	set +vx
+}
+run_test 64b "Nodemap enforces dne_ops RBAC roles"
+
+test_64c() {
+	(( MDS1_VERSION >= $(version_code 2.15.54) )) ||
+		skip "Need MDS >= 2.15.54 for role-based controls"
+
+	stack_trap cleanup_64 EXIT
+	mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+	setup_64
+
+	do_facet mgs $LCTL nodemap_modify --name c0 \
+		 --property rbac --value quota_ops
+	wait_nm_sync c0 rbac
+	set -vx
+	$LFS setquota -u $USER0 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT ||
+		error "lfs setquota -u failed"
+	$LFS setquota -u $USER0 --delete $MOUNT
+	$LFS setquota -g $USER0 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT ||
+		error "lfs setquota -g failed"
+	$LFS setquota -g $USER0 --delete $MOUNT
+	$LFS setquota -p 1000 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT ||
+		error "lfs setquota -p failed"
+	$LFS setquota -p 1000 --delete $MOUNT
+
+	$LFS setquota -U -b 10G -B 11G -i 100K -I 105K $MOUNT ||
+		error "lfs setquota -U failed"
+	$LFS setquota -U -b 0 -B 0 -i 0 -I 0 $MOUNT
+	$LFS setquota -G -b 10G -B 11G -i 100K -I 105K $MOUNT ||
+		error "lfs setquota -G failed"
+	$LFS setquota -G -b 0 -B 0 -i 0 -I 0 $MOUNT
+	$LFS setquota -P -b 10G -B 11G -i 100K -I 105K $MOUNT ||
+		error "lfs setquota -P failed"
+	$LFS setquota -P -b 0 -B 0 -i 0 -I 0 $MOUNT
+	$LFS setquota -u $USER0 -D $MOUNT ||
+		error "lfs setquota -u -D failed"
+	$LFS setquota -u $USER0 --delete $MOUNT
+	$LFS setquota -g $USER0 -D $MOUNT ||
+		error "lfs setquota -g -D failed"
+	$LFS setquota -g $USER0 --delete $MOUNT
+	$LFS setquota -p 1000 -D $MOUNT ||
+		error "lfs setquota -p -D failed"
+	$LFS setquota -p 1000 --delete $MOUNT
+	set +vx
+
+	do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
+	wait_nm_sync c0 rbac
+
+	set -vx
+	$LFS setquota -u $USER0 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT &&
+		error "lfs setquota -u should fail"
+	$LFS setquota -u $USER0 --delete $MOUNT
+	$LFS setquota -g $USER0 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT &&
+		error "lfs setquota -g should fail"
+	$LFS setquota -g $USER0 --delete $MOUNT
+	$LFS setquota -p 1000 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT &&
+		error "lfs setquota -p should fail"
+	$LFS setquota -p 1000 --delete $MOUNT
+
+	$LFS setquota -U -b 10G -B 11G -i 100K -I 105K $MOUNT &&
+		error "lfs setquota -U should fail"
+	$LFS setquota -G -b 10G -B 11G -i 100K -I 105K $MOUNT &&
+		error "lfs setquota -G should fail"
+	$LFS setquota -P -b 10G -B 11G -i 100K -I 105K $MOUNT &&
+		error "lfs setquota -P should fail"
+	$LFS setquota -u $USER0 -D $MOUNT &&
+		error "lfs setquota -u -D should fail"
+	$LFS setquota -u $USER0 --delete $MOUNT
+	$LFS setquota -g $USER0 -D $MOUNT &&
+		error "lfs setquota -g -D should fail"
+	$LFS setquota -g $USER0 --delete $MOUNT
+	$LFS setquota -p 1000 -D $MOUNT &&
+		error "lfs setquota -p -D should fail"
+	$LFS setquota -p 1000 --delete $MOUNT
+	set +vx
+}
+run_test 64c "Nodemap enforces quota_ops RBAC roles"
+
+test_64d() {
+	local testfile=$DIR/$tdir/$tfile
+	local fid
+
+	(( MDS1_VERSION >= $(version_code 2.15.54) )) ||
+		skip "Need MDS >= 2.15.54 for role-based controls"
+
+	stack_trap cleanup_64 EXIT
+	mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+	setup_64
+
+	do_facet mgs $LCTL nodemap_modify --name c0 \
+		 --property rbac --value byfid_ops
+	wait_nm_sync c0 rbac
+
+	touch $testfile
+	fid=$(lfs path2fid $testfile)
+	set -vx
+	$LFS fid2path $MOUNT $fid || error "fid2path $fid failed (1)"
+	cat $MOUNT/.lustre/fid/$fid || error "cat by fid failed"
+	lfs rmfid $MOUNT $fid || error "lfs rmfid failed"
+	set +vx
+
+	do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
+	wait_nm_sync c0 rbac
+
+	touch $testfile
+	fid=$(lfs path2fid $testfile)
+	set -vx
+	$LFS fid2path $MOUNT $fid || error "fid2path $fid failed (2)"
+	cat $MOUNT/.lustre/fid/$fid && error "cat by fid should fail"
+	lfs rmfid $MOUNT $fid && error "lfs rmfid should fail"
+	set +vx
+	rm -f $testfile
+}
+run_test 64d "Nodemap enforces byfid_ops RBAC roles"
+
+test_64e() {
+	local testfile=$DIR/$tdir/$tfile
+	local testdir=$DIR/$tdir/${tfile}.d
+
+	(( MDS1_VERSION >= $(version_code 2.15.54) )) ||
+		skip "Need MDS >= 2.15.54 for role-based controls"
+
+	stack_trap cleanup_64 EXIT
+	mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
+	setup_64
+
+	# activate changelogs
+	changelog_register || error "changelog_register failed"
+	local cl_user="${CL_USERS[$SINGLEMDS]%% *}"
+	changelog_users $SINGLEMDS | grep -q $cl_user ||
+		error "User $cl_user not found in changelog_users"
+	changelog_chmask ALL
+
+	# do some IOs
+	mkdir $testdir || error "failed to mkdir $testdir"
+	touch $testfile || error "failed to touch $testfile"
+
+	do_facet mgs $LCTL nodemap_modify --name c0 \
+		 --property rbac --value chlg_ops
+	wait_nm_sync c0 rbac
+
+	# access changelogs
+	echo "changelogs dump"
+	changelog_dump || error "failed to dump changelogs"
+	echo "changelogs clear"
+	changelog_clear 0 || error "failed to clear changelogs"
+
+	rm -rf $testdir $testfile || error "rm -rf $testdir $testfile failed"
+
+	do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
+	wait_nm_sync c0 rbac
+
+	# do some IOs
+	mkdir $testdir || error "failed to mkdir $testdir"
+	touch $testfile || error "failed to touch $testfile"
+
+	# access changelogs
+	echo "changelogs dump"
+	changelog_dump && error "dump changelogs should fail"
+	echo "changelogs clear"
+	changelog_clear 0 && error "clear changelogs should fail"
+	rm -rf $testdir $testfile
+
+	do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value all
+	wait_nm_sync c0 rbac
+}
+run_test 64e "Nodemap enforces chlg_ops RBAC roles"
+
 log "cleanup: ======================================================"
 
 sec_unsetup() {
-- 
1.8.3.1