From: Kit Westneat <kit.westneat@gmail.com>
Date: Mon, 12 Jan 2015 06:51:03 +0000 (-0500)
Subject: LU-4647 tests: modify sanity-sec to test multiple MDSes
X-Git-Tag: 2.7.52~32
X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=e62ce6e1252b33b37130556f31ceb22e8410dfcb

LU-4647 tests: modify sanity-sec to test multiple MDSes

This patch modifies santiy-sec to distribute the nodemap to all the
MDSes in the system. It also uses test_mkdir to create directories on
multiple MDTs, allowing DNE and nodemapping to be tested.

Test-Parameters: alwaysuploadlogs envdefinitions=SLOW=yes \
mdtfilesystemtype=ldiskfs mdsfilesystemtype=ldiskfs ostfilesystemtype=ldiskfs \
mdtcount=4 testlist=sanity-sec

Signed-off-by: Kit Westneat <kit.westneat@gmail.com>
Change-Id: I48f47b72e404614167c715a682d99351295d8189
Reviewed-on: http://review.whamcloud.com/13343
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Jian Yu <jian.yu@intel.com>
Reviewed-by: Nathaniel Clark <nathaniel.l.clark@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
---

diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh
index 30f81da..e071394 100644
--- a/lustre/tests/sanity-sec.sh
+++ b/lustre/tests/sanity-sec.sh
@@ -80,9 +80,6 @@ DIR=${DIR:-$MOUNT}
 [ `echo $MOUNT | wc -w` -gt 1 ] && \
 	echo "NAME=$MOUNT mounted more than once" && sec_cleanup && exit 0
 
-[ $MDSCOUNT -gt 1 ] && \
-	echo "skip multi-MDS test" && sec_cleanup && exit 0
-
 # for GSS_SUP
 GSS_REF=$(lsmod | grep ^ptlrpc_gss | awk '{print $3}')
 if [ ! -z "$GSS_REF" -a "$GSS_REF" != "0" ]; then
@@ -1216,6 +1213,18 @@ test_15() {
 }
 run_test 15 "test id mapping"
 
+# Until nodemaps are distributed by MGS, they need to be distributed manually
+# This function and all calls to it should be removed once the MGS distributes
+# nodemaps to the MDS and OSS nodes directly.
+do_servers_not_mgs() {
+	local mgs_ip=$(host_nids_address $mgs_HOST $NETTYPE)
+	for node in $(all_server_nodes); do
+		local node_ip=$(host_nids_address $node $NETTYPE)
+		[ $node_ip == $mgs_ip ] && continue
+		do_node $node_ip $*
+	done
+}
+
 create_fops_nodemaps() {
 	local i=0
 	local client
@@ -1225,19 +1234,20 @@ create_fops_nodemaps() {
 		do_facet mgs $LCTL nodemap_add c${i} || return 1
 		do_facet mgs $LCTL nodemap_add_range 	\
 			--name c${i} --range $client_nid || return 1
-		do_facet ost0 $LCTL set_param nodemap.add_nodemap=c${i} ||
+		do_servers_not_mgs $LCTL set_param nodemap.add_nodemap=c${i} ||
+			return 1
+		do_servers_not_mgs "$LCTL set_param \
+			nodemap.add_nodemap_range='c${i} $client_nid'" ||
 			return 1
-		do_facet ost0 "$LCTL set_param nodemap.add_nodemap_range='c$i \
-			$client_nid'" || return 1
 		for map in ${FOPS_IDMAPS[i]}; do
 			do_facet mgs $LCTL nodemap_add_idmap --name c${i} \
 				--idtype uid --idmap ${map} || return 1
-			do_facet ost0 "$LCTL set_param \
+			do_servers_not_mgs "$LCTL set_param \
 				nodemap.add_nodemap_idmap='c$i uid ${map}'" ||
 				return 1
 			do_facet mgs $LCTL nodemap_add_idmap --name c${i} \
 				--idtype gid --idmap ${map} || return 1
-			do_facet ost0 "$LCTL set_param \
+			do_servers_not_mgs "$LCTL set_param \
 				nodemap.add_nodemap_idmap='c$i gid ${map}'" ||
 				return 1
 		done
@@ -1254,13 +1264,26 @@ delete_fops_nodemaps() {
 	local client
 	for client in $clients; do
 		do_facet mgs $LCTL nodemap_del c${i} || return 1
-		do_facet ost0 $LCTL set_param nodemap.remove_nodemap=c${i} ||
+		do_servers_not_mgs $LCTL set_param nodemap.remove_nodemap=c$i ||
 			return 1
 		i=$((i + 1))
 	done
 	return 0
 }
 
+fops_mds_index=0
+nm_test_mkdir() {
+	if [ $MDSCOUNT -le 1 ]; then
+		do_node ${clients_arr[0]} mkdir -p $DIR/$tdir
+	else
+		# round-robin MDTs to test DNE nodemap support
+		[ ! -d $DIR ] && do_node ${clients_arr[0]} mkdir -p $DIR
+		do_node ${clients_arr[0]} $LFS setdirstripe -c 1 -i \
+			$((fops_mds_index % MDSCOUNT)) $DIR/$tdir
+		((fops_mds_index++))
+	fi
+}
+
 # acl test directory needs to be initialized on a privileged client
 fops_test_setup() {
 	local admin=$(do_facet mgs $LCTL get_param -n nodemap.c0.admin_nodemap)
@@ -1269,19 +1292,19 @@ fops_test_setup() {
 
 	do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 1
 	do_facet mgs $LCTL nodemap_modify --name c0 --property trusted --value 1
-	do_facet ost0 $LCTL set_param nodemap.c0.admin_nodemap=1
-	do_facet ost0 $LCTL set_param nodemap.c0.trusted_nodemap=1
+	do_servers_not_mgs $LCTL set_param nodemap.c0.admin_nodemap=1
+	do_servers_not_mgs $LCTL set_param nodemap.c0.trusted_nodemap=1
 
 	do_node ${clients_arr[0]} rm -rf $DIR/$tdir
-	do_node ${clients_arr[0]} mkdir -p $DIR/$tdir
+	nm_test_mkdir
 	do_node ${clients_arr[0]} chown $user $DIR/$tdir
 
 	do_facet mgs $LCTL nodemap_modify --name c0 \
 		--property admin --value $admin
 	do_facet mgs $LCTL nodemap_modify --name c0 \
 		--property trusted --value $trust
-	do_facet ost0 $LCTL set_param nodemap.c0.admin_nodemap=$admin
-	do_facet ost0 $LCTL set_param nodemap.c0.trusted_nodemap=$trust
+	do_servers_not_mgs $LCTL set_param nodemap.c0.admin_nodemap=$admin
+	do_servers_not_mgs $LCTL set_param nodemap.c0.trusted_nodemap=$trust
 
 	# flush MDT locks to make sure they are reacquired before test
 	do_node ${clients_arr[0]} lctl set_param \
@@ -1443,18 +1466,23 @@ test_fops() {
 					     $RUNAS_CMD -u$u -g$u -G$u"
 				for perm_bits in $perm_bit_list; do
 					local mode=$(printf %03o $perm_bits)
+					local key
+					key="$mapmode:$user:c$cli_i:$u:$mode"
 					do_facet mgs $LCTL nodemap_modify \
 						--name c$cli_i \
 						--property admin \
 						--value 1
-					do_node $client chmod $mode $DIR/$tdir
+					do_servers_not_mgs $LCTL set_param \
+						nodemap.c$cli_i.admin_nodemap=1
+					do_node $client chmod $mode $DIR/$tdir \
+						|| error unable to chmod $key
 					do_facet mgs $LCTL nodemap_modify \
 						--name c$cli_i \
 						--property admin \
 						--value $admin
+					do_servers_not_mgs $LCTL set_param \
+					    nodemap.c$cli_i.admin_nodemap=$admin
 
-					local key
-					key="$mapmode:$user:c$cli_i:$u:$mode"
 					do_create_delete "$run_u" "$key"
 				done
 
@@ -1474,13 +1502,14 @@ nodemap_test_setup() {
 	local rc
 	local active_nodemap=$1
 
-	do_facet mgs $LCTL set_param $IDENTITY_UPCALL=NONE
-
 	remote_mgs_nodsh && skip "remote MGS with nodsh" && return
 	[ $(lustre_version_code $SINGLEMGS) -lt $(version_code 2.6.90) ] &&
 		skip "Skip test on $(get_lustre_version) MGS, need 2.6.90+" &&
 		return
 
+	do_nodes $(comma_list $(all_mdts_nodes)) $LCTL set_param \
+		mdt.*.identity_upcall=NONE
+
 	rc=0
 	create_fops_nodemaps
 	rc=$?
@@ -1488,18 +1517,18 @@ nodemap_test_setup() {
 
 	if [ "$active_nodemap" == "0" ]; then
 		do_facet mgs $LCTL set_param nodemap.active=0
-		do_facet ost0 $LCTL set_param nodemap.active=0
+		do_servers_not_mgs $LCTL set_param nodemap.active=0
 		return
 	fi
 
 	do_facet mgs $LCTL nodemap_activate 1
-	do_facet ost0 $LCTL set_param nodemap.active=1
+	do_servers_not_mgs $LCTL set_param nodemap.active=1
 	do_facet mgs $LCTL nodemap_modify --name default \
 		--property admin --value 1
 	do_facet mgs $LCTL nodemap_modify --name default \
 		--property trusted --value 1
-	do_facet ost0 $LCTL set_param nodemap.default.admin_nodemap=1
-	do_facet ost0 $LCTL set_param nodemap.default.trusted_nodemap=1
+	do_servers_not_mgs $LCTL set_param nodemap.default.admin_nodemap=1
+	do_servers_not_mgs $LCTL set_param nodemap.default.trusted_nodemap=1
 }
 
 nodemap_test_cleanup() {
@@ -1517,10 +1546,12 @@ nodemap_clients_admin_trusted() {
 	for client in $clients; do
 		do_facet mgs $LCTL nodemap_modify --name c0 \
 			--property admin --value $admin
-		do_facet ost0 $LCTL set_param nodemap.c${i}.admin_nodemap=$admin
+		do_servers_not_mgs $LCTL set_param \
+			nodemap.c${i}.admin_nodemap=$admin
 		do_facet mgs $LCTL nodemap_modify --name c0 \
 			--property trusted --value $tr
-		do_facet ost0 $LCTL set_param nodemap.c${i}.trusted_nodemap=$tr
+		do_servers_not_mgs $LCTL set_param \
+			nodemap.c${i}.trusted_nodemap=$tr
 		i=$((i + 1))
 	done
 }
@@ -1575,8 +1606,10 @@ test_21() {
 			--property admin --value 0
 		do_facet mgs $LCTL nodemap_modify --name c${i} \
 			--property trusted --value $x
-		do_facet ost0 $LCTL set_param nodemap.c${i}.admin_nodemap=0
-		do_facet ost0 $LCTL set_param nodemap.c${i}.trusted_nodemap=$x
+		do_servers_not_mgs $LCTL set_param \
+			nodemap.c${i}.admin_nodemap=0
+		do_servers_not_mgs $LCTL set_param \
+			nodemap.c${i}.trusted_nodemap=$x
 		x=0
 		i=$((i + 1))
 	done
@@ -1594,8 +1627,10 @@ test_22() {
 			--property admin --value 1
 		do_facet mgs $LCTL nodemap_modify --name c${i} \
 			--property trusted --value $x
-		do_facet ost0 $LCTL set_param nodemap.c${i}.admin_nodemap=1
-		do_facet ost0 $LCTL set_param nodemap.c${i}.trusted_nodemap=$x
+		do_servers_not_mgs $LCTL set_param \
+			nodemap.c${i}.admin_nodemap=1
+		do_servers_not_mgs $LCTL set_param \
+			nodemap.c${i}.trusted_nodemap=$x
 		x=0
 		i=$((i + 1))
 	done
@@ -1612,19 +1647,20 @@ nodemap_acl_test_setup() {
 
 	do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 1
 	do_facet mgs $LCTL nodemap_modify --name c0 --property trusted --value 1
-	do_facet ost0 $LCTL set_param nodemap.c0.admin_nodemap=1
-	do_facet ost0 $LCTL set_param nodemap.c0.trusted_nodemap=1
+	do_servers_not_mgs $LCTL set_param nodemap.c0.admin_nodemap=1
+	do_servers_not_mgs $LCTL set_param nodemap.c0.trusted_nodemap=1
 
 	do_node ${clients_arr[0]} rm -rf $DIR/$tdir
-	do_node ${clients_arr[0]} mkdir -p $DIR/$tdir
-	do_node ${clients_arr[0]} chmod a+rwx $DIR/$tdir
+	nm_test_mkdir
+	do_node ${clients_arr[0]} chmod a+rwx $DIR/$tdir ||
+		error unable to chmod a+rwx test dir $DIR/$tdir
 
 	do_facet mgs $LCTL nodemap_modify --name c0 \
 		--property admin --value $admin
 	do_facet mgs $LCTL nodemap_modify --name c0 \
 		--property trusted --value $trust
-	do_facet ost0 $LCTL set_param nodemap.c0.admin_nodemap=$admin
-	do_facet ost0 $LCTL set_param nodemap.c0.trusted_nodemap=$trust
+	do_servers_not_mgs $LCTL set_param nodemap.c0.admin_nodemap=$admin
+	do_servers_not_mgs $LCTL set_param nodemap.c0.trusted_nodemap=$trust
 
 }
 
@@ -1677,13 +1713,13 @@ test_23() {
 
 	do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 1
 	do_facet mgs $LCTL nodemap_modify --name c0 --property trusted --value 1
-	do_facet ost0 $LCTL set_param nodemap.c0.admin_nodemap=1
-	do_facet ost0 $LCTL set_param nodemap.c0.trusted_nodemap=1
+	do_servers_not_mgs $LCTL set_param nodemap.c0.admin_nodemap=1
+	do_servers_not_mgs $LCTL set_param nodemap.c0.trusted_nodemap=1
 
 	do_facet mgs $LCTL nodemap_modify --name c1 --property admin --value 0
 	do_facet mgs $LCTL nodemap_modify --name c1 --property trusted --value 0
-	do_facet ost0 $LCTL set_param nodemap.c1.admin_nodemap=0
-	do_facet ost0 $LCTL set_param nodemap.c1.trusted_nodemap=0
+	do_servers_not_mgs $LCTL set_param nodemap.c1.admin_nodemap=0
+	do_servers_not_mgs $LCTL set_param nodemap.c1.trusted_nodemap=0
 
 	# setfacl on trusted cluster to unmapped user, verify it's not seen
 	nodemap_acl_test $unmapped_fs ${clients_arr[0]} ${clients_arr[1]} ||
@@ -1704,8 +1740,8 @@ test_23() {
 	# 2 mapped clusters
 	do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 0
 	do_facet mgs $LCTL nodemap_modify --name c0 --property trusted --value 0
-	do_facet ost0 $LCTL set_param nodemap.c0.admin_nodemap=0
-	do_facet ost0 $LCTL set_param nodemap.c0.trusted_nodemap=0
+	do_servers_not_mgs $LCTL set_param nodemap.c0.admin_nodemap=0
+	do_servers_not_mgs $LCTL set_param nodemap.c0.trusted_nodemap=0
 
 	# setfacl to mapped user on c1, also mapped to c0, verify it's seen
 	nodemap_acl_test $mapped_c1 ${clients_arr[1]} ${clients_arr[0]} &&