3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
11 LUSTRE=${LUSTRE:-$(dirname $0)/..}
12 . $LUSTRE/tests/test-framework.sh
17 ALWAYS_EXCEPT="$SANITY_SEC_EXCEPT "
18 if [[ "$MDS1_OS_ID" == "rhel" || "$MDS1_OS_ID_LIKE" =~ "rhel" ]] &&
19 (( $MDS1_OS_VERSION_CODE == $(version_code 9.3) )); then
20 always_except LU-16915 51
23 [ "$SLOW" = "no" ] && EXCEPT_SLOW="26"
25 NODEMAP_TESTS=$(seq 7 26)
27 if ! check_versions; then
28 echo "It is NOT necessary to test nodemap under interoperation mode"
29 EXCEPT="$EXCEPT $NODEMAP_TESTS"
34 RUNAS_CMD=${RUNAS_CMD:-runas}
36 WTL=${WTL:-"$LUSTRE/tests/write_time_limit"}
39 PERM_CONF=$CONFDIR/perm.conf
41 HOSTNAME_CHECKSUM=$(hostname | sum | awk '{ print $1 }')
42 SUBNET_CHECKSUM=$(expr $HOSTNAME_CHECKSUM % 250 + 1)
44 require_dsh_mds || exit 0
45 require_dsh_ost || exit 0
47 clients=${CLIENTS//,/ }
48 num_clients=$(get_node_count ${clients})
49 clients_arr=($clients)
51 echo "was USER0=$(getent passwd | grep :${ID0:-500}:)"
52 echo "was USER1=$(getent passwd | grep :${ID1:-501}:)"
57 echo "now USER0=$USER0=$ID0:$(id -g $USER0), USER1=$USER1=$ID1:$(id -g $USER1)"
59 if [ "$SLOW" == "yes" ]; then
62 NODEMAP_IPADDR_LIST="1 10 64 128 200 250"
67 NODEMAP_IPADDR_LIST="1 250"
70 NODEMAP_MAX_ID=$((ID0 + NODEMAP_ID_COUNT))
73 skip "need to add user0 ($ID0:$ID0)" && exit 0
76 skip "need to add user1 ($ID1:$ID1)" && exit 0
78 IDBASE=${IDBASE:-60000}
80 # changes to mappings must be reflected in test 23
82 [0]="$((IDBASE+3)):$((IDBASE+0)) $((IDBASE+4)):$((IDBASE+2))"
83 [1]="$((IDBASE+5)):$((IDBASE+1)) $((IDBASE+6)):$((IDBASE+2))"
86 check_and_setup_lustre
91 GSS_REF=$(lsmod | grep ^ptlrpc_gss | awk '{print $3}')
92 if [ ! -z "$GSS_REF" -a "$GSS_REF" != "0" ]; then
94 echo "with GSS support"
97 echo "without GSS support"
100 MDT=$(mdtname_from_index 0 $MOUNT)
101 [[ -z "$MDT" ]] && error "fail to get MDT0000 device name" && exit 1
102 do_facet $SINGLEMDS "mkdir -p $CONFDIR"
103 IDENTITY_FLUSH=mdt.$MDT.identity_flush
112 if ! $RUNAS_CMD -u $user krb5_login.sh; then
113 error "$user login kerberos failed."
117 if ! $RUNAS_CMD -u $user -g $group ls $DIR > /dev/null 2>&1; then
118 $RUNAS_CMD -u $user lfs flushctx -k
119 $RUNAS_CMD -u $user krb5_login.sh
120 if ! $RUNAS_CMD -u$user -g$group ls $DIR > /dev/null 2>&1; then
121 error "init $user $group failed."
127 declare -a identity_old
130 # wait for SSK flavor to be applied if necessary
133 wait_flavor all2all $SK_FLAVOR
135 wait_flavor cli2mdt $SK_FLAVOR
136 wait_flavor cli2ost $SK_FLAVOR
142 for ((num = 1; num <= $MDSCOUNT; num++)); do
143 switch_identity $num true || identity_old[$num]=$?
146 if ! $RUNAS_CMD -u $ID0 ls $DIR > /dev/null 2>&1; then
147 sec_login $USER0 $USER0
150 if ! $RUNAS_CMD -u $ID1 ls $DIR > /dev/null 2>&1; then
151 sec_login $USER1 $USER1
157 # run as different user
161 chmod 0755 $DIR || error "chmod (1) Failed"
162 rm -rf $DIR/$tdir || error "rm (1) Failed"
163 mkdir -p $DIR/$tdir || error "mkdir (1) Failed"
165 # $DIR/$tdir owner changed to USER0(sanityusr)
166 chown $USER0 $DIR/$tdir || error "chown (2) Failed"
167 chmod 0755 $DIR/$tdir || error "chmod (2) Failed"
169 # Run as ID0 cmd must pass
170 $RUNAS_CMD -u $ID0 ls -ali $DIR || error "ls (1) Failed"
171 # Remove non-existing file f0
172 rm -f $DIR/f0 || error "rm (2) Failed"
174 # It is expected that this cmd should fail
175 # $DIR has only r-x rights for group and other
176 $RUNAS_CMD -u $ID0 touch $DIR/f0
177 (( $? == 0 )) && error "touch (1) should not pass"
179 # This must pass. $DIR/$tdir/ is owned by ID0/USER0
180 $RUNAS_CMD -u $ID0 touch $DIR/$tdir/f1 || error "touch (2) Failed"
182 # It is expected that this cmd should fail
183 # $tdir has rwxr-xr-x rights for $ID0
184 $RUNAS_CMD -u $ID1 touch $DIR/$tdir/f2
185 (( $? == 0 )) && error "touch (3) should not pass"
187 touch $DIR/$tdir/f3 || error "touch (4) Failed"
188 chown root $DIR/$tdir || error "chown (3) Failed"
189 chgrp $USER0 $DIR/$tdir || error "chgrp (1) Failed"
190 chmod 0775 $DIR/$tdir || error "chmod (3) Failed"
192 # Owner is root and group is USER0
193 $RUNAS_CMD -u $USER0 -g $USER0 touch $DIR/$tdir/f4 ||
194 error "touch (5) Failed"
196 # It is expected that this cmd should fail
197 # $tdir has rwxrwxr-x rights for group sanityusr/ID0, ID1 will fail
198 $RUNAS_CMD -u $ID1 -g $ID1 touch $DIR/$tdir/f5
199 (( $? == 0 )) && error "touch (6) should not pass"
201 touch $DIR/$tdir/f6 || error "touch (7) Failed"
202 rm -rf $DIR/$tdir || error "rm (3) Failed"
204 run_test 0 "uid permission ============================="
208 [ $GSS_SUP = 0 ] && skip "without GSS support." && return
211 mkdir_on_mdt0 $DIR/$tdir
213 chown $USER0 $DIR/$tdir || error "chown (1)"
214 $RUNAS_CMD -u $ID1 -v $ID0 touch $DIR/$tdir/f0 && error "touch (2)"
215 echo "enable uid $ID1 setuid"
216 do_facet $SINGLEMDS "echo '* $ID1 setuid' >> $PERM_CONF"
217 do_facet $SINGLEMDS "lctl set_param -n $IDENTITY_FLUSH=-1"
218 $RUNAS_CMD -u $ID1 -v $ID0 touch $DIR/$tdir/f1 || error "touch (3)"
220 chown root $DIR/$tdir || error "chown (4)"
221 chgrp $USER0 $DIR/$tdir || error "chgrp (5)"
222 chmod 0770 $DIR/$tdir || error "chmod (6)"
223 $RUNAS_CMD -u $ID1 -g $ID1 touch $DIR/$tdir/f2 && error "touch (7)"
224 $RUNAS_CMD -u$ID1 -g$ID1 -j$ID0 touch $DIR/$tdir/f3 && error "touch (8)"
225 echo "enable uid $ID1 setuid,setgid"
226 do_facet $SINGLEMDS "echo '* $ID1 setuid,setgid' > $PERM_CONF"
227 do_facet $SINGLEMDS "lctl set_param -n $IDENTITY_FLUSH=-1"
228 $RUNAS_CMD -u $ID1 -g $ID1 -j $ID0 touch $DIR/$tdir/f4 ||
230 $RUNAS_CMD -u $ID1 -v $ID0 -g $ID1 -j $ID0 touch $DIR/$tdir/f5 ||
235 do_facet $SINGLEMDS "rm -f $PERM_CONF"
236 do_facet $SINGLEMDS "lctl set_param -n $IDENTITY_FLUSH=-1"
238 run_test 1 "setuid/gid ============================="
240 # bug 3285 - supplementary group should always succeed.
241 # NB: the supplementary groups are set for local client only,
242 # as for remote client, the groups of the specified uid on MDT
243 # will be obtained by upcall /usr/sbin/l_getidentity and used.
245 [[ "$MDS1_VERSION" -ge $(version_code 2.6.93) ]] ||
246 [[ "$MDS1_VERSION" -ge $(version_code 2.5.35) &&
247 "$MDS1_VERSION" -lt $(version_code 2.5.50) ]] ||
248 skip "Need MDS version at least 2.6.93 or 2.5.35"
251 mkdir_on_mdt0 -p $DIR/$tdir
252 chmod 0771 $DIR/$tdir
253 chgrp $ID0 $DIR/$tdir
254 $RUNAS_CMD -u $ID0 ls $DIR/$tdir || error "setgroups (1)"
255 do_facet $SINGLEMDS "echo '* $ID1 setgrp' > $PERM_CONF"
256 do_facet $SINGLEMDS "lctl set_param -n $IDENTITY_FLUSH=-1"
257 $RUNAS_CMD -u $ID1 -G1,2,$ID0 ls $DIR/$tdir ||
258 error "setgroups (2)"
259 $RUNAS_CMD -u $ID1 -G1,2 ls $DIR/$tdir && error "setgroups (3)"
262 do_facet $SINGLEMDS "rm -f $PERM_CONF"
263 do_facet $SINGLEMDS "lctl set_param -n $IDENTITY_FLUSH=-1"
265 run_test 4 "set supplementary group ==============="
271 squash_id default ${NOBODY_UID:-65534} 0
272 wait_nm_sync default squash_uid '' inactive
273 squash_id default ${NOBODY_UID:-65534} 1
274 wait_nm_sync default squash_gid '' inactive
275 for (( i = 0; i < NODEMAP_COUNT; i++ )); do
276 local csum=${HOSTNAME_CHECKSUM}_${i}
278 do_facet mgs $LCTL nodemap_add $csum
280 if [ $rc -ne 0 ]; then
281 echo "nodemap_add $csum failed with $rc"
285 wait_update_facet --verbose mgs \
286 "$LCTL get_param nodemap.$csum.id 2>/dev/null | \
287 grep -c $csum || true" 1 30 ||
290 for (( i = 0; i < NODEMAP_COUNT; i++ )); do
291 local csum=${HOSTNAME_CHECKSUM}_${i}
293 wait_nm_sync $csum id '' inactive
301 for ((i = 0; i < NODEMAP_COUNT; i++)); do
302 local csum=${HOSTNAME_CHECKSUM}_${i}
304 if ! do_facet mgs $LCTL nodemap_del $csum; then
305 error "nodemap_del $csum failed with $?"
309 wait_update_facet --verbose mgs \
310 "$LCTL get_param nodemap.$csum.id 2>/dev/null | \
311 grep -c $csum || true" 0 30 ||
314 for (( i = 0; i < NODEMAP_COUNT; i++ )); do
315 local csum=${HOSTNAME_CHECKSUM}_${i}
317 wait_nm_sync $csum id '' inactive
324 local cmd="$LCTL nodemap_add_range"
328 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
329 range="$SUBNET_CHECKSUM.${2}.${j}.[1-253]@tcp"
330 if ! do_facet mgs $cmd --name $1 --range $range; then
339 local cmd="$LCTL nodemap_del_range"
343 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
344 range="$SUBNET_CHECKSUM.${2}.${j}.[1-253]@tcp"
345 if ! do_facet mgs $cmd --name $1 --range $range; then
355 local cmd="$LCTL nodemap_add_idmap"
359 (( $MDS1_VERSION >= $(version_code 2.14.52) )) || do_proj=false
361 echo "Start to add idmaps ..."
362 for ((i = 0; i < NODEMAP_COUNT; i++)); do
365 for ((j = $ID0; j < NODEMAP_MAX_ID; j++)); do
366 local csum=${HOSTNAME_CHECKSUM}_${i}
368 local fs_id=$((j + 1))
370 if ! do_facet mgs $cmd --name $csum --idtype uid \
371 --idmap $client_id:$fs_id; then
374 if ! do_facet mgs $cmd --name $csum --idtype gid \
375 --idmap $client_id:$fs_id; then
379 if ! do_facet mgs $cmd --name $csum \
380 --idtype projid --idmap \
381 $client_id:$fs_id; then
393 local cmd="$LCTL nodemap_add_idmap"
396 echo "Start to add root idmaps ..."
397 for ((i = 0; i < NODEMAP_COUNT; i++)); do
398 local csum=${HOSTNAME_CHECKSUM}_${i}
400 if ! do_facet mgs $cmd --name $csum --idtype uid \
404 if ! do_facet mgs $cmd --name $csum --idtype gid \
413 update_idmaps() { #LU-10040
414 [ "$MGS_VERSION" -lt $(version_code 2.10.55) ] &&
415 skip "Need MGS >= 2.10.55"
417 local csum=${HOSTNAME_CHECKSUM}_0
418 local old_id_client=$ID0
419 local old_id_fs=$((ID0 + 1))
420 local new_id=$((ID0 + 100))
425 echo "Start to update idmaps ..."
427 #Inserting an existed idmap should return error
428 cmd="$LCTL nodemap_add_idmap --name $csum --idtype uid"
430 $cmd --idmap $old_id_client:$old_id_fs 2>/dev/null; then
431 error "insert idmap {$old_id_client:$old_id_fs} " \
432 "should return error"
437 #Update id_fs and check it
438 if ! do_facet mgs $cmd --idmap $old_id_client:$new_id; then
439 error "$cmd --idmap $old_id_client:$new_id failed"
443 tmp_id=$(do_facet mgs $LCTL get_param -n nodemap.$csum.idmap |
444 awk '{ print $7 }' | sed -n '2p')
445 [ $tmp_id != $new_id ] && { error "new id_fs $tmp_id != $new_id"; \
446 rc=$((rc + 1)); return $rc; }
448 #Update id_client and check it
449 if ! do_facet mgs $cmd --idmap $new_id:$new_id; then
450 error "$cmd --idmap $new_id:$new_id failed"
454 tmp_id=$(do_facet mgs $LCTL get_param -n nodemap.$csum.idmap |
455 awk '{ print $5 }' | sed -n "$((NODEMAP_ID_COUNT + 1)) p")
456 tmp_id=$(echo ${tmp_id%,*}) #e.g. "501,"->"501"
457 [ $tmp_id != $new_id ] && { error "new id_client $tmp_id != $new_id"; \
458 rc=$((rc + 1)); return $rc; }
460 #Delete above updated idmap
461 cmd="$LCTL nodemap_del_idmap --name $csum --idtype uid"
462 if ! do_facet mgs $cmd --idmap $new_id:$new_id; then
463 error "$cmd --idmap $new_id:$new_id failed"
468 #restore the idmaps to make delete_idmaps work well
469 cmd="$LCTL nodemap_add_idmap --name $csum --idtype uid"
470 if ! do_facet mgs $cmd --idmap $old_id_client:$old_id_fs; then
471 error "$cmd --idmap $old_id_client:$old_id_fs failed"
481 local cmd="$LCTL nodemap_del_idmap"
485 (( $MDS1_VERSION >= $(version_code 2.14.52) )) || do_proj=false
487 echo "Start to delete idmaps ..."
488 for ((i = 0; i < NODEMAP_COUNT; i++)); do
491 for ((j = $ID0; j < NODEMAP_MAX_ID; j++)); do
492 local csum=${HOSTNAME_CHECKSUM}_${i}
494 local fs_id=$((j + 1))
496 if ! do_facet mgs $cmd --name $csum --idtype uid \
497 --idmap $client_id:$fs_id; then
500 if ! do_facet mgs $cmd --name $csum --idtype gid \
501 --idmap $client_id:$fs_id; then
505 if ! do_facet mgs $cmd --name $csum \
506 --idtype projid --idmap \
507 $client_id:$fs_id; then
517 delete_root_idmaps() {
519 local cmd="$LCTL nodemap_del_idmap"
522 echo "Start to delete root idmaps ..."
523 for ((i = 0; i < NODEMAP_COUNT; i++)); do
524 local csum=${HOSTNAME_CHECKSUM}_${i}
526 if ! do_facet mgs $cmd --name $csum --idtype uid \
530 if ! do_facet mgs $cmd --name $csum --idtype gid \
543 local cmd="$LCTL nodemap_modify"
546 proc[0]="admin_nodemap"
547 proc[1]="trusted_nodemap"
551 for ((idx = 0; idx < 2; idx++)); do
552 if ! do_facet mgs $cmd --name $1 --property ${option[$idx]} \
557 if ! do_facet mgs $cmd --name $1 --property ${option[$idx]} \
567 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
568 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
572 cmd[0]="$LCTL nodemap_modify --property squash_uid"
573 cmd[1]="$LCTL nodemap_modify --property squash_gid"
574 cmd[2]="$LCTL nodemap_modify --property squash_projid"
576 if ! do_facet mgs ${cmd[$3]} --name $1 --value $2; then
581 # ensure that the squash defaults are the expected defaults
582 squash_id default ${NOBODY_UID:-65534} 0
583 wait_nm_sync default squash_uid '' inactive
584 squash_id default ${NOBODY_UID:-65534} 1
585 wait_nm_sync default squash_gid '' inactive
586 if [ "$MDS1_VERSION" -ge $(version_code 2.14.50) ]; then
587 squash_id default ${NOBODY_UID:-65534} 2
588 wait_nm_sync default squash_projid '' inactive
594 cmd="$LCTL nodemap_test_nid"
596 nid=$(do_facet mgs $cmd $1)
598 if [ $nid == $2 ]; then
606 # restore activation state
607 do_facet mgs $LCTL nodemap_activate 0
613 local cmd="$LCTL nodemap_test_id"
614 local do_root_idmap=true
617 (( $MDS1_VERSION >= $(version_code 2.15.60) )) || do_root_idmap=false
619 echo "Start to test idmaps ..."
620 ## nodemap deactivated
621 if ! do_facet mgs $LCTL nodemap_activate 0; then
624 for ((id = $ID0; id < NODEMAP_MAX_ID; id++)); do
627 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
628 local nid="$SUBNET_CHECKSUM.0.${j}.100@tcp"
629 local fs_id=$(do_facet mgs $cmd --nid $nid \
630 --idtype uid --id $id)
631 if [ $fs_id != $id ]; then
632 echo "expected $id, got $fs_id"
639 if ! do_facet mgs $LCTL nodemap_activate 1; then
643 for ((id = $ID0; id < NODEMAP_MAX_ID; id++)); do
644 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
645 nid="$SUBNET_CHECKSUM.0.${j}.100@tcp"
646 fs_id=$(do_facet mgs $cmd --nid $nid \
647 --idtype uid --id $id)
648 expected_id=$((id + 1))
649 if [ $fs_id != $expected_id ]; then
650 echo "expected $expected_id, got $fs_id"
657 for ((i = 0; i < NODEMAP_COUNT; i++)); do
658 local csum=${HOSTNAME_CHECKSUM}_${i}
660 if ! do_facet mgs $LCTL nodemap_modify --name $csum \
661 --property trusted --value 1; then
662 error "nodemap_modify $csum failed with $?"
667 for ((id = $ID0; id < NODEMAP_MAX_ID; id++)); do
668 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
669 nid="$SUBNET_CHECKSUM.0.${j}.100@tcp"
670 fs_id=$(do_facet mgs $cmd --nid $nid \
671 --idtype uid --id $id)
672 if [ $fs_id != $id ]; then
673 echo "expected $id, got $fs_id"
679 ## ensure allow_root_access is enabled
680 for ((i = 0; i < NODEMAP_COUNT; i++)); do
681 local csum=${HOSTNAME_CHECKSUM}_${i}
683 if ! do_facet mgs $LCTL nodemap_modify --name $csum \
684 --property admin --value 1; then
685 error "nodemap_modify $csum failed with $?"
690 ## check that root allowed
691 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
692 nid="$SUBNET_CHECKSUM.0.${j}.100@tcp"
693 fs_id=$(do_facet mgs $cmd --nid $nid --idtype uid --id 0)
694 if [ $fs_id != 0 ]; then
695 echo "root allowed expected 0, got $fs_id"
700 if $do_root_idmap; then
701 ## add mapping for root
704 ## check that root allowed
705 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
706 nid="$SUBNET_CHECKSUM.0.${j}.100@tcp"
707 fs_id=$(do_facet mgs $cmd --nid $nid \
709 if [ $fs_id != 0 ]; then
710 echo "root allowed expected 0, got $fs_id"
715 ## delete mapping for root
719 ## ensure allow_root_access is disabled
720 for ((i = 0; i < NODEMAP_COUNT; i++)); do
721 local csum=${HOSTNAME_CHECKSUM}_${i}
723 if ! do_facet mgs $LCTL nodemap_modify --name $csum \
724 --property admin --value 0; then
725 error "nodemap_modify ${HOSTNAME_CHECKSUM}_${i} "
731 ## check that root is mapped to NOBODY_UID
732 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
733 nid="$SUBNET_CHECKSUM.0.${j}.100@tcp"
734 fs_id=$(do_facet mgs $cmd --nid $nid --idtype uid --id 0)
735 if [ $fs_id != ${NOBODY_UID:-65534} ]; then
736 error "root squash expect ${NOBODY_UID:-65534} got $fs_id"
741 if $do_root_idmap; then
742 ## add mapping for root
745 ## check root is mapped
746 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
747 nid="$SUBNET_CHECKSUM.0.${j}.100@tcp"
748 fs_id=$(do_facet mgs $cmd --nid $nid \
751 if [ $fs_id != $expected_id ]; then
752 echo "expected $expected_id, got $fs_id"
757 ## delete mapping for root
761 ## reset client trust to 0
762 for ((i = 0; i < NODEMAP_COUNT; i++)); do
763 if ! do_facet mgs $LCTL nodemap_modify \
764 --name ${HOSTNAME_CHECKSUM}_${i} \
765 --property trusted --value 0; then
766 error "nodemap_modify ${HOSTNAME_CHECKSUM}_${i} "
778 remote_mgs_nodsh && skip "remote MGS with nodsh"
779 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
780 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
784 [[ $rc != 0 ]] && error "nodemap_add failed with $rc"
788 [[ $rc != 0 ]] && error "nodemap_del failed with $rc"
792 run_test 7 "nodemap create and delete"
797 remote_mgs_nodsh && skip "remote MGS with nodsh"
798 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
799 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
805 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
811 [[ $rc == 0 ]] && error "duplicate nodemap_add allowed with $rc" &&
817 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 3
821 run_test 8 "nodemap reject duplicates"
827 remote_mgs_nodsh && skip "remote MGS with nodsh"
828 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
829 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
834 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
837 for ((i = 0; i < NODEMAP_COUNT; i++)); do
838 if ! add_range ${HOSTNAME_CHECKSUM}_${i} $i; then
842 [[ $rc != 0 ]] && error "nodemap_add_range failed with $rc" && return 2
845 for ((i = 0; i < NODEMAP_COUNT; i++)); do
846 if ! delete_range ${HOSTNAME_CHECKSUM}_${i} $i; then
850 [[ $rc != 0 ]] && error "nodemap_del_range failed with $rc" && return 4
855 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 4
859 run_test 9 "nodemap range add"
864 remote_mgs_nodsh && skip "remote MGS with nodsh"
865 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
866 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
871 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
874 for ((i = 0; i < NODEMAP_COUNT; i++)); do
875 if ! add_range ${HOSTNAME_CHECKSUM}_${i} $i; then
879 [[ $rc != 0 ]] && error "nodemap_add_range failed with $rc" && return 2
882 for ((i = 0; i < NODEMAP_COUNT; i++)); do
883 if ! add_range ${HOSTNAME_CHECKSUM}_${i} $i; then
887 [[ $rc == 0 ]] && error "nodemap_add_range duplicate add with $rc" &&
892 for ((i = 0; i < NODEMAP_COUNT; i++)); do
893 if ! delete_range ${HOSTNAME_CHECKSUM}_${i} $i; then
897 [[ $rc != 0 ]] && error "nodemap_del_range failed with $rc" && return 4
901 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 5
905 run_test 10a "nodemap reject duplicate ranges"
908 [ "$MGS_VERSION" -lt $(version_code 2.10.53) ] &&
909 skip "Need MGS >= 2.10.53"
913 local nids="192.168.19.[0-255]@o2ib20"
915 do_facet mgs $LCTL nodemap_del $nm1 2>/dev/null
916 do_facet mgs $LCTL nodemap_del $nm2 2>/dev/null
918 do_facet mgs $LCTL nodemap_add $nm1 || error "Add $nm1 failed"
919 do_facet mgs $LCTL nodemap_add $nm2 || error "Add $nm2 failed"
920 do_facet mgs $LCTL nodemap_add_range --name $nm1 --range $nids ||
921 error "Add range $nids to $nm1 failed"
922 [ -n "$(do_facet mgs $LCTL get_param nodemap.$nm1.* |
923 grep start_nid)" ] || error "No range was found"
924 do_facet mgs $LCTL nodemap_del_range --name $nm2 --range $nids &&
925 error "Deleting range $nids from $nm2 should fail"
926 [ -n "$(do_facet mgs $LCTL get_param nodemap.$nm1.* |
927 grep start_nid)" ] || error "Range $nids should be there"
929 do_facet mgs $LCTL nodemap_del $nm1 || error "Delete $nm1 failed"
930 do_facet mgs $LCTL nodemap_del $nm2 || error "Delete $nm2 failed"
933 run_test 10b "delete range from the correct nodemap"
935 test_10c() { #LU-8912
936 [ "$MGS_VERSION" -lt $(version_code 2.10.57) ] &&
937 skip "Need MGS >= 2.10.57"
939 local nm="nodemap_lu8912"
940 local nid_range="10.210.[32-47].[0-255]@o2ib3"
941 local start_nid="10.210.32.0@o2ib3"
942 local end_nid="10.210.47.255@o2ib3"
943 local start_nid_found
946 do_facet mgs $LCTL nodemap_del $nm 2>/dev/null
947 do_facet mgs $LCTL nodemap_add $nm || error "Add $nm failed"
948 do_facet mgs $LCTL nodemap_add_range --name $nm --range $nid_range ||
949 error "Add range $nid_range to $nm failed"
951 start_nid_found=$(do_facet mgs $LCTL get_param nodemap.$nm.* |
952 awk -F '[,: ]' /start_nid/'{ print $9 }')
953 [ "$start_nid" == "$start_nid_found" ] ||
954 error "start_nid: $start_nid_found != $start_nid"
955 end_nid_found=$(do_facet mgs $LCTL get_param nodemap.$nm.* |
956 awk -F '[,: ]' /end_nid/'{ print $13 }')
957 [ "$end_nid" == "$end_nid_found" ] ||
958 error "end_nid: $end_nid_found != $end_nid"
960 do_facet mgs $LCTL nodemap_del $nm || error "Delete $nm failed"
963 run_test 10c "verfify contiguous range support"
965 test_10d() { #LU-8913
966 [ "$MGS_VERSION" -lt $(version_code 2.10.59) ] &&
967 skip "Need MGS >= 2.10.59"
969 local nm="nodemap_lu8913"
970 local nid_range="*@o2ib3"
971 local start_nid="0.0.0.0@o2ib3"
972 local end_nid="255.255.255.255@o2ib3"
973 local start_nid_found
976 do_facet mgs $LCTL nodemap_del $nm 2>/dev/null
977 do_facet mgs $LCTL nodemap_add $nm || error "Add $nm failed"
978 do_facet mgs $LCTL nodemap_add_range --name $nm --range $nid_range ||
979 error "Add range $nid_range to $nm failed"
981 start_nid_found=$(do_facet mgs $LCTL get_param nodemap.$nm.* |
982 awk -F '[,: ]' /start_nid/'{ print $9 }')
983 [ "$start_nid" == "$start_nid_found" ] ||
984 error "start_nid: $start_nid_found != $start_nid"
985 end_nid_found=$(do_facet mgs $LCTL get_param nodemap.$nm.* |
986 awk -F '[,: ]' /end_nid/'{ print $13 }')
987 [ "$end_nid" == "$end_nid_found" ] ||
988 error "end_nid: $end_nid_found != $end_nid"
990 do_facet mgs $LCTL nodemap_del $nm || error "Delete $nm failed"
993 run_test 10d "verfify nodemap range format '*@<net>' support"
998 remote_mgs_nodsh && skip "remote MGS with nodsh"
999 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
1000 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
1005 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
1008 for ((i = 0; i < NODEMAP_COUNT; i++)); do
1009 if ! modify_flags ${HOSTNAME_CHECKSUM}_${i}; then
1013 [[ $rc != 0 ]] && error "nodemap_modify with $rc" && return 2
1018 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 3
1022 run_test 11 "nodemap modify"
1027 remote_mgs_nodsh && skip "remote MGS with nodsh"
1028 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
1029 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
1034 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
1037 for ((i = 0; i < NODEMAP_COUNT; i++)); do
1038 if ! squash_id ${HOSTNAME_CHECKSUM}_${i} 88 0; then
1042 [[ $rc != 0 ]] && error "nodemap squash_uid with $rc" && return 2
1045 for ((i = 0; i < NODEMAP_COUNT; i++)); do
1046 if ! squash_id ${HOSTNAME_CHECKSUM}_${i} 88 1; then
1050 [[ $rc != 0 ]] && error "nodemap squash_gid with $rc" && return 3
1053 if (( $MDS1_VERSION >= $(version_code 2.14.52) )); then
1054 for ((i = 0; i < NODEMAP_COUNT; i++)); do
1055 if ! squash_id ${HOSTNAME_CHECKSUM}_${i} 88 2; then
1060 [[ $rc != 0 ]] && error "nodemap squash_projid with $rc" && return 5
1065 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 4
1069 run_test 12 "nodemap set squash ids"
1074 remote_mgs_nodsh && skip "remote MGS with nodsh"
1075 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
1076 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
1081 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
1084 for ((i = 0; i < NODEMAP_COUNT; i++)); do
1085 if ! add_range ${HOSTNAME_CHECKSUM}_${i} $i; then
1089 [[ $rc != 0 ]] && error "nodemap_add_range failed with $rc" && return 2
1092 for ((i = 0; i < NODEMAP_COUNT; i++)); do
1093 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
1094 for k in $NODEMAP_IPADDR_LIST; do
1095 if ! test_nid $SUBNET_CHECKSUM.$i.$j.$k \
1096 ${HOSTNAME_CHECKSUM}_${i}; then
1102 [[ $rc != 0 ]] && error "nodemap_test_nid failed with $rc" && return 3
1107 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 4
1111 run_test 13 "test nids"
1116 remote_mgs_nodsh && skip "remote MGS with nodsh"
1117 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
1118 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
1123 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
1126 for ((i = 0; i < NODEMAP_COUNT; i++)); do
1127 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
1128 for k in $NODEMAP_IPADDR_LIST; do
1129 if ! test_nid $SUBNET_CHECKSUM.$i.$j.$k \
1136 [[ $rc != 0 ]] && error "nodemap_test_nid failed with $rc" && return 3
1141 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 4
1145 run_test 14 "test default nodemap nid lookup"
1150 remote_mgs_nodsh && skip "remote MGS with nodsh"
1151 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
1152 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
1157 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
1159 for (( i = 0; i < NODEMAP_COUNT; i++ )); do
1160 local csum=${HOSTNAME_CHECKSUM}_${i}
1162 if ! do_facet mgs $LCTL nodemap_modify --name $csum \
1163 --property admin --value 0; then
1166 if ! do_facet mgs $LCTL nodemap_modify --name $csum \
1167 --property trusted --value 0; then
1171 [[ $rc != 0 ]] && error "nodemap_modify failed with $rc" && return 1
1174 for ((i = 0; i < NODEMAP_COUNT; i++)); do
1175 if ! add_range ${HOSTNAME_CHECKSUM}_${i} $i; then
1179 [[ $rc != 0 ]] && error "nodemap_add_range failed with $rc" && return 2
1184 [[ $rc != 0 ]] && error "nodemap_add_idmap failed with $rc" && return 3
1186 activedefault=$(do_facet mgs $LCTL get_param -n nodemap.active)
1187 if [[ "$activedefault" != "1" ]]; then
1188 stack_trap cleanup_active EXIT
1194 [[ $rc != 0 ]] && error "nodemap_test_id failed with $rc" && return 4
1199 [[ $rc != 0 ]] && error "update_idmaps failed with $rc" && return 5
1204 [[ $rc != 0 ]] && error "nodemap_del_idmap failed with $rc" && return 6
1209 [[ $rc != 0 ]] && error "nodemap_delete failed with $rc" && return 7
1213 run_test 15 "test id mapping"
1215 create_fops_nodemaps() {
1218 for client in $clients; do
1219 local client_ip=$(host_nids_address $client $NETTYPE)
1220 local client_nid=$(h2nettype $client_ip)
1221 [[ "$client_nid" =~ ":" ]] && client_nid+="/128"
1222 do_facet mgs $LCTL nodemap_add c${i} || return 1
1223 do_facet mgs $LCTL nodemap_add_range \
1224 --name c${i} --range $client_nid || {
1225 do_facet mgs $LCTL nodemap_del c${i}
1228 for map in ${FOPS_IDMAPS[i]}; do
1229 do_facet mgs $LCTL nodemap_add_idmap --name c${i} \
1230 --idtype uid --idmap ${map} || return 1
1231 do_facet mgs $LCTL nodemap_add_idmap --name c${i} \
1232 --idtype gid --idmap ${map} || return 1
1235 wait_nm_sync c$i idmap
1242 delete_fops_nodemaps() {
1245 for client in $clients; do
1246 do_facet mgs $LCTL nodemap_del c${i} || return 1
1254 if [ $MDSCOUNT -le 1 ]; then
1255 do_node ${clients_arr[0]} mkdir -p $DIR/$tdir
1257 # round-robin MDTs to test DNE nodemap support
1258 [ ! -d $DIR ] && do_node ${clients_arr[0]} mkdir -p $DIR
1259 do_node ${clients_arr[0]} $LFS setdirstripe -c 1 -i \
1260 $((fops_mds_index % MDSCOUNT)) $DIR/$tdir
1261 ((fops_mds_index++))
1265 # acl test directory needs to be initialized on a privileged client
1267 local admin=$(do_facet mgs $LCTL get_param -n nodemap.c0.admin_nodemap)
1268 local trust=$(do_facet mgs $LCTL get_param -n \
1269 nodemap.c0.trusted_nodemap)
1271 do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 1
1272 do_facet mgs $LCTL nodemap_modify --name c0 --property trusted --value 1
1274 wait_nm_sync c0 admin_nodemap
1275 wait_nm_sync c0 trusted_nodemap
1277 do_node ${clients_arr[0]} rm -rf $DIR/$tdir
1279 do_node ${clients_arr[0]} chown $user $DIR/$tdir
1281 do_facet mgs $LCTL nodemap_modify --name c0 \
1282 --property admin --value $admin
1283 do_facet mgs $LCTL nodemap_modify --name c0 \
1284 --property trusted --value $trust
1286 # flush MDT locks to make sure they are reacquired before test
1287 do_node ${clients_arr[0]} $LCTL set_param \
1288 ldlm.namespaces.$FSNAME-MDT*.lru_size=clear
1290 wait_nm_sync c0 admin_nodemap
1291 wait_nm_sync c0 trusted_nodemap
1294 # fileset test directory needs to be initialized on a privileged client
1295 fileset_test_setup() {
1298 if [ -n "$FILESET" -a -z "$SKIP_FILESET" ]; then
1299 cleanup_mount $MOUNT
1300 FILESET="" zconf_mount_clients $CLIENTS $MOUNT
1303 local admin=$(do_facet mgs $LCTL get_param -n \
1304 nodemap.${nm}.admin_nodemap)
1305 local trust=$(do_facet mgs $LCTL get_param -n \
1306 nodemap.${nm}.trusted_nodemap)
1308 do_facet mgs $LCTL nodemap_modify --name $nm --property admin --value 1
1309 do_facet mgs $LCTL nodemap_modify --name $nm --property trusted \
1312 wait_nm_sync $nm admin_nodemap
1313 wait_nm_sync $nm trusted_nodemap
1315 # create directory and populate it for subdir mount
1316 do_node ${clients_arr[0]} mkdir $MOUNT/$subdir ||
1317 error "unable to create dir $MOUNT/$subdir"
1318 do_node ${clients_arr[0]} touch $MOUNT/$subdir/this_is_$subdir ||
1319 error "unable to create file $MOUNT/$subdir/this_is_$subdir"
1320 do_node ${clients_arr[0]} mkdir $MOUNT/$subdir/$subsubdir ||
1321 error "unable to create dir $MOUNT/$subdir/$subsubdir"
1322 do_node ${clients_arr[0]} touch \
1323 $MOUNT/$subdir/$subsubdir/this_is_$subsubdir ||
1324 error "unable to create file \
1325 $MOUNT/$subdir/$subsubdir/this_is_$subsubdir"
1327 do_facet mgs $LCTL nodemap_modify --name $nm \
1328 --property admin --value $admin
1329 do_facet mgs $LCTL nodemap_modify --name $nm \
1330 --property trusted --value $trust
1332 # flush MDT locks to make sure they are reacquired before test
1333 do_node ${clients_arr[0]} $LCTL set_param \
1334 ldlm.namespaces.$FSNAME-MDT*.lru_size=clear
1336 wait_nm_sync $nm admin_nodemap
1337 wait_nm_sync $nm trusted_nodemap
1340 # fileset test directory needs to be initialized on a privileged client
1341 fileset_test_cleanup() {
1343 local admin=$(do_facet mgs $LCTL get_param -n \
1344 nodemap.${nm}.admin_nodemap)
1345 local trust=$(do_facet mgs $LCTL get_param -n \
1346 nodemap.${nm}.trusted_nodemap)
1348 do_facet mgs $LCTL nodemap_modify --name $nm --property admin --value 1
1349 do_facet mgs $LCTL nodemap_modify --name $nm --property trusted \
1352 wait_nm_sync $nm admin_nodemap
1353 wait_nm_sync $nm trusted_nodemap
1355 # cleanup directory created for subdir mount
1356 do_node ${clients_arr[0]} rm -rf $MOUNT/$subdir ||
1357 error "unable to remove dir $MOUNT/$subdir"
1359 do_facet mgs $LCTL nodemap_modify --name $nm \
1360 --property admin --value $admin
1361 do_facet mgs $LCTL nodemap_modify --name $nm \
1362 --property trusted --value $trust
1364 # flush MDT locks to make sure they are reacquired before test
1365 do_node ${clients_arr[0]} $LCTL set_param \
1366 ldlm.namespaces.$FSNAME-MDT*.lru_size=clear
1368 wait_nm_sync $nm admin_nodemap
1369 wait_nm_sync $nm trusted_nodemap
1370 if [ -n "$FILESET" -a -z "$SKIP_FILESET" ]; then
1371 cleanup_mount $MOUNT
1372 zconf_mount_clients $CLIENTS $MOUNT
1376 do_create_delete() {
1379 local testfile=$DIR/$tdir/$tfile
1383 if $run_u touch $testfile >& /dev/null; then
1385 $run_u rm $testfile && d=1
1389 local expected=$(get_cr_del_expected $key)
1390 [ "$res" != "$expected" ] &&
1391 error "test $key, wanted $expected, got $res" && rc=$((rc + 1))
1395 nodemap_check_quota() {
1397 $run_u lfs quota -q $DIR | awk '{ print $2; exit; }'
1400 do_fops_quota_test() {
1402 # fuzz quota used to account for possible indirect blocks, etc
1403 local quota_fuzz=$(fs_log_size)
1404 local qused_orig=$(nodemap_check_quota "$run_u")
1405 local qused_high=$((qused_orig + quota_fuzz))
1406 local qused_low=$((qused_orig - quota_fuzz))
1407 local testfile=$DIR/$tdir/$tfile
1408 $run_u dd if=/dev/zero of=$testfile oflag=sync bs=1M count=1 \
1409 >& /dev/null || error "unable to write quota test file"
1410 sync; sync_all_data || true
1412 local qused_new=$(nodemap_check_quota "$run_u")
1413 [ $((qused_new)) -lt $((qused_low + 1024)) -o \
1414 $((qused_new)) -gt $((qused_high + 1024)) ] &&
1415 error "$qused_new != $qused_orig + 1M after write, " \
1416 "fuzz is $quota_fuzz"
1417 $run_u rm $testfile || error "unable to remove quota test file"
1418 wait_delete_completed_mds
1420 qused_new=$(nodemap_check_quota "$run_u")
1421 [ $((qused_new)) -lt $((qused_low)) \
1422 -o $((qused_new)) -gt $((qused_high)) ] &&
1423 error "quota not reclaimed, expect $qused_orig, " \
1424 "got $qused_new, fuzz $quota_fuzz"
1427 get_fops_mapped_user() {
1430 for ((i=0; i < ${#FOPS_IDMAPS[@]}; i++)); do
1431 for map in ${FOPS_IDMAPS[i]}; do
1432 if [ $(cut -d: -f1 <<< "$map") == $cli_user ]; then
1433 cut -d: -f2 <<< "$map"
1441 get_cr_del_expected() {
1443 IFS=":" read -a key <<< "$1"
1444 local mapmode="${key[0]}"
1445 local mds_user="${key[1]}"
1446 local cluster="${key[2]}"
1447 local cli_user="${key[3]}"
1448 local mode="0${key[4]}"
1455 [[ $mapmode == *mapped* ]] && mapped=1
1456 # only c1 is mapped in these test cases
1457 [[ $mapmode == mapped_trusted* ]] && [ "$cluster" == "c0" ] && mapped=0
1458 [[ $mapmode == *noadmin* ]] && noadmin=1
1460 # o+wx works as long as the user isn't mapped
1461 if [ $((mode & 3)) -eq 3 ]; then
1465 # if client user is root, check if root is squashed
1466 if [ "$cli_user" == "0" ]; then
1467 # squash root succeed, if other bit is on
1470 1) [ "$other" == "1" ] && echo $SUCCESS
1471 [ "$other" == "0" ] && echo $FAILURE;;
1475 if [ "$mapped" == "0" ]; then
1476 [ "$other" == "1" ] && echo $SUCCESS
1477 [ "$other" == "0" ] && echo $FAILURE
1481 # if mapped user is mds user, check for u+wx
1482 mapped_user=$(get_fops_mapped_user $cli_user)
1483 [ "$mapped_user" == "-1" ] &&
1484 error "unable to find mapping for client user $cli_user"
1486 if [ "$mapped_user" == "$mds_user" -a \
1487 $(((mode & 0300) == 0300)) -eq 1 ]; then
1491 if [ "$mapped_user" != "$mds_user" -a "$other" == "1" ]; then
1498 test_fops_admin_cli_i=""
1499 test_fops_chmod_dir() {
1500 local current_cli_i=$1
1502 local dir_to_chmod=$3
1503 local new_admin_cli_i=""
1505 # do we need to set up a new admin client?
1506 [ "$current_cli_i" == "0" ] && [ "$test_fops_admin_cli_i" != "1" ] &&
1508 [ "$current_cli_i" != "0" ] && [ "$test_fops_admin_cli_i" != "0" ] &&
1511 # if only one client, and non-admin, need to flip admin everytime
1512 if [ "$num_clients" == "1" ]; then
1513 test_fops_admin_client=$clients
1514 test_fops_admin_val=$(do_facet mgs $LCTL get_param -n \
1515 nodemap.c0.admin_nodemap)
1516 if [ "$test_fops_admin_val" != "1" ]; then
1517 do_facet mgs $LCTL nodemap_modify \
1521 wait_nm_sync c0 admin_nodemap
1523 elif [ "$new_admin_cli_i" != "" ]; then
1524 # restore admin val to old admin client
1525 if [ "$test_fops_admin_cli_i" != "" ] &&
1526 [ "$test_fops_admin_val" != "1" ]; then
1527 do_facet mgs $LCTL nodemap_modify \
1528 --name c${test_fops_admin_cli_i} \
1530 --value $test_fops_admin_val
1531 wait_nm_sync c${test_fops_admin_cli_i} admin_nodemap
1534 test_fops_admin_cli_i=$new_admin_cli_i
1535 test_fops_admin_client=${clients_arr[$new_admin_cli_i]}
1536 test_fops_admin_val=$(do_facet mgs $LCTL get_param -n \
1537 nodemap.c${new_admin_cli_i}.admin_nodemap)
1539 if [ "$test_fops_admin_val" != "1" ]; then
1540 do_facet mgs $LCTL nodemap_modify \
1541 --name c${new_admin_cli_i} \
1544 wait_nm_sync c${new_admin_cli_i} admin_nodemap
1548 do_node $test_fops_admin_client chmod $perm_bits $DIR/$tdir || return 1
1550 # remove admin for single client if originally non-admin
1551 if [ "$num_clients" == "1" ] && [ "$test_fops_admin_val" != "1" ]; then
1552 do_facet mgs $LCTL nodemap_modify --name c0 --property admin \
1554 wait_nm_sync c0 admin_nodemap
1562 local single_client="$2"
1563 local client_user_list=([0]="0 $((IDBASE+3))"
1564 [1]="0 $((IDBASE+5))")
1565 local mds_users="-1 0"
1568 local perm_bit_list="3 $((0300))"
1569 # SLOW tests 000-007, 010-070, 100-700 (octal modes)
1570 if [ "$SLOW" == "yes" ]; then
1571 perm_bit_list="0 $(seq 1 7) $(seq 8 8 63) $(seq 64 64 511) \
1573 client_user_list=([0]="0 $((IDBASE+3)) $((IDBASE+4))"
1574 [1]="0 $((IDBASE+5)) $((IDBASE+6))")
1575 mds_users="-1 0 1 2"
1578 # force single_client to speed up test
1579 [ "$SLOW" == "yes" ] ||
1581 # step through mds users. -1 means root
1582 for mds_i in $mds_users; do
1583 local user=$((mds_i + IDBASE))
1587 [ "$mds_i" == "-1" ] && user=0
1589 echo mkdir -p $DIR/$tdir
1592 for client in $clients; do
1594 for u in ${client_user_list[$cli_i]}; do
1595 local run_u="do_node $client \
1596 $RUNAS_CMD -u$u -g$u -G$u"
1597 for perm_bits in $perm_bit_list; do
1598 local mode=$(printf %03o $perm_bits)
1600 key="$mapmode:$user:c$cli_i:$u:$mode"
1601 test_fops_chmod_dir $cli_i $mode \
1603 error cannot chmod $key
1604 do_create_delete "$run_u" "$key"
1608 test_fops_chmod_dir $cli_i 777 $DIR/$tdir ||
1609 error cannot chmod $key
1610 do_fops_quota_test "$run_u"
1613 cli_i=$((cli_i + 1))
1614 [ "$single_client" == "1" ] && break
1621 nodemap_version_check () {
1622 remote_mgs_nodsh && skip "remote MGS with nodsh" && return 1
1623 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
1624 skip "No nodemap on $MGS_VERSION MGS < 2.5.53" &&
1629 nodemap_test_setup() {
1631 local active_nodemap=1
1633 [ "$1" == "0" ] && active_nodemap=0
1635 do_nodes $(comma_list $(all_mdts_nodes)) \
1636 $LCTL set_param mdt.*.identity_upcall=NONE
1639 create_fops_nodemaps
1641 [[ $rc != 0 ]] && error "adding fops nodemaps failed $rc"
1643 do_facet mgs $LCTL nodemap_activate $active_nodemap
1646 do_facet mgs $LCTL nodemap_modify --name default \
1647 --property admin --value 1
1648 wait_nm_sync default admin_nodemap
1649 do_facet mgs $LCTL nodemap_modify --name default \
1650 --property trusted --value 1
1651 wait_nm_sync default trusted_nodemap
1654 nodemap_test_cleanup() {
1656 delete_fops_nodemaps
1658 [[ $rc != 0 ]] && error "removing fops nodemaps failed $rc"
1660 do_facet mgs $LCTL nodemap_modify --name default \
1661 --property admin --value 0
1662 wait_nm_sync default admin_nodemap
1663 do_facet mgs $LCTL nodemap_modify --name default \
1664 --property trusted --value 0
1665 wait_nm_sync default trusted_nodemap
1667 do_facet mgs $LCTL nodemap_activate 0
1668 wait_nm_sync active 0
1670 export SK_UNIQUE_NM=false
1674 nodemap_clients_admin_trusted() {
1678 for client in $clients; do
1679 do_facet mgs $LCTL nodemap_modify --name c0 \
1680 --property admin --value $admin
1681 do_facet mgs $LCTL nodemap_modify --name c0 \
1682 --property trusted --value $tr
1685 wait_nm_sync c$((i - 1)) admin_nodemap
1686 wait_nm_sync c$((i - 1)) trusted_nodemap
1690 nodemap_version_check || return 0
1691 nodemap_test_setup 0
1693 trap nodemap_test_cleanup EXIT
1695 nodemap_test_cleanup
1697 run_test 16 "test nodemap all_off fileops"
1701 [ "$MDS1_VERSION" -lt $(version_code 2.11.55) ]; then
1702 skip "Need MDS >= 2.11.55"
1704 local check_proj=true
1706 (( $MDS1_VERSION >= $(version_code 2.14.52) )) || check_proj=false
1708 nodemap_version_check || return 0
1711 trap nodemap_test_cleanup EXIT
1712 nodemap_clients_admin_trusted 0 1
1713 test_fops trusted_noadmin 1
1714 if $check_proj; then
1715 do_facet mgs $LCTL nodemap_modify --name c0 \
1716 --property map_mode --value projid
1717 wait_nm_sync c0 map_mode
1719 test_fops trusted_noadmin 1
1720 nodemap_test_cleanup
1722 run_test 17 "test nodemap trusted_noadmin fileops"
1726 [ "$MDS1_VERSION" -lt $(version_code 2.11.55) ]; then
1727 skip "Need MDS >= 2.11.55"
1730 nodemap_version_check || return 0
1733 trap nodemap_test_cleanup EXIT
1734 nodemap_clients_admin_trusted 0 0
1735 test_fops mapped_noadmin 1
1736 nodemap_test_cleanup
1738 run_test 18 "test nodemap mapped_noadmin fileops"
1742 [ "$MDS1_VERSION" -lt $(version_code 2.11.55) ]; then
1743 skip "Need MDS >= 2.11.55"
1746 nodemap_version_check || return 0
1749 trap nodemap_test_cleanup EXIT
1750 nodemap_clients_admin_trusted 1 1
1751 test_fops trusted_admin 1
1752 nodemap_test_cleanup
1754 run_test 19 "test nodemap trusted_admin fileops"
1758 [ "$MDS1_VERSION" -lt $(version_code 2.11.55) ]; then
1759 skip "Need MDS >= 2.11.55"
1762 nodemap_version_check || return 0
1765 trap nodemap_test_cleanup EXIT
1766 nodemap_clients_admin_trusted 1 0
1767 test_fops mapped_admin 1
1768 nodemap_test_cleanup
1770 run_test 20 "test nodemap mapped_admin fileops"
1774 [ "$MDS1_VERSION" -lt $(version_code 2.11.55) ]; then
1775 skip "Need MDS >= 2.11.55"
1778 nodemap_version_check || return 0
1781 trap nodemap_test_cleanup EXIT
1784 for client in $clients; do
1785 do_facet mgs $LCTL nodemap_modify --name c${i} \
1786 --property admin --value 0
1787 do_facet mgs $LCTL nodemap_modify --name c${i} \
1788 --property trusted --value $x
1792 wait_nm_sync c$((i - 1)) trusted_nodemap
1794 test_fops mapped_trusted_noadmin
1795 nodemap_test_cleanup
1797 run_test 21 "test nodemap mapped_trusted_noadmin fileops"
1801 [ "$MDS1_VERSION" -lt $(version_code 2.11.55) ]; then
1802 skip "Need MDS >= 2.11.55"
1805 nodemap_version_check || return 0
1808 trap nodemap_test_cleanup EXIT
1811 for client in $clients; do
1812 do_facet mgs $LCTL nodemap_modify --name c${i} \
1813 --property admin --value 1
1814 do_facet mgs $LCTL nodemap_modify --name c${i} \
1815 --property trusted --value $x
1819 wait_nm_sync c$((i - 1)) trusted_nodemap
1821 test_fops mapped_trusted_admin
1822 nodemap_test_cleanup
1824 run_test 22 "test nodemap mapped_trusted_admin fileops"
1826 # acl test directory needs to be initialized on a privileged client
1827 nodemap_acl_test_setup() {
1828 local admin=$(do_facet mgs $LCTL get_param -n \
1829 nodemap.c0.admin_nodemap)
1830 local trust=$(do_facet mgs $LCTL get_param -n \
1831 nodemap.c0.trusted_nodemap)
1833 do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 1
1834 do_facet mgs $LCTL nodemap_modify --name c0 --property trusted --value 1
1836 wait_nm_sync c0 admin_nodemap
1837 wait_nm_sync c0 trusted_nodemap
1839 do_node ${clients_arr[0]} rm -rf $DIR/$tdir
1841 do_node ${clients_arr[0]} chmod a+rwx $DIR/$tdir ||
1842 error unable to chmod a+rwx test dir $DIR/$tdir
1844 do_facet mgs $LCTL nodemap_modify --name c0 \
1845 --property admin --value $admin
1846 do_facet mgs $LCTL nodemap_modify --name c0 \
1847 --property trusted --value $trust
1849 wait_nm_sync c0 trusted_nodemap
1852 # returns 0 if the number of ACLs does not change on the second (mapped) client
1853 # after being set on the first client
1854 nodemap_acl_test() {
1856 local set_client="$2"
1857 local get_client="$3"
1858 local check_setfacl="$4"
1859 local setfacl_error=0
1860 local testfile=$DIR/$tdir/$tfile
1861 local RUNAS_USER="$RUNAS_CMD -u $user"
1863 local acl_count_post=0
1865 nodemap_acl_test_setup
1868 do_node $set_client $RUNAS_USER touch $testfile
1869 # remove from cache, otherwise ACLs will not be fetched from server
1870 do_rpc_nodes $set_client cancel_lru_locks
1871 do_node $set_client "sync ; echo 3 > /proc/sys/vm/drop_caches"
1873 # ACL masks aren't filtered by nodemap code, so we ignore them
1874 acl_count=$(do_node $get_client getfacl $testfile | grep -v mask |
1876 # remove from cache, otherwise ACLs will not be fetched from server
1877 do_rpc_nodes $get_client cancel_lru_locks
1878 do_node $get_client "sync ; echo 3 > /proc/sys/vm/drop_caches"
1879 do_node $set_client $RUNAS_USER setfacl -m $user:rwx $testfile ||
1881 # remove from cache, otherwise ACLs will not be fetched from server
1882 do_rpc_nodes $set_client cancel_lru_locks
1883 do_node $set_client "sync ; echo 3 > /proc/sys/vm/drop_caches"
1885 # if check setfacl is set to 1, then it's supposed to error
1886 if [ "$check_setfacl" == "1" ]; then
1887 [ "$setfacl_error" != "1" ] && return 1
1890 [ "$setfacl_error" == "1" ] && echo "WARNING: unable to setfacl"
1892 acl_count_post=$(do_node $get_client getfacl $testfile | grep -v mask |
1894 # remove from cache, otherwise ACLs will not be fetched from server
1895 do_rpc_nodes $get_client cancel_lru_locks
1896 do_node $get_client "sync ; echo 3 > /proc/sys/vm/drop_caches"
1897 [ $acl_count -eq $acl_count_post ] && return 0
1902 [ $num_clients -lt 2 ] && skip "Need 2 clients at least" && return
1903 nodemap_version_check || return 0
1906 trap nodemap_test_cleanup EXIT
1907 # 1 trusted cluster, 1 mapped cluster
1908 local unmapped_fs=$((IDBASE+0))
1909 local unmapped_c1=$((IDBASE+5))
1910 local mapped_fs=$((IDBASE+2))
1911 local mapped_c0=$((IDBASE+4))
1912 local mapped_c1=$((IDBASE+6))
1914 do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 1
1915 do_facet mgs $LCTL nodemap_modify --name c0 --property trusted --value 1
1917 do_facet mgs $LCTL nodemap_modify --name c1 --property admin --value 0
1918 do_facet mgs $LCTL nodemap_modify --name c1 --property trusted --value 0
1920 wait_nm_sync c1 trusted_nodemap
1922 # setfacl on trusted cluster to unmapped user, verify it's not seen
1923 nodemap_acl_test $unmapped_fs ${clients_arr[0]} ${clients_arr[1]} ||
1924 error "acl count (1)"
1926 # setfacl on trusted cluster to mapped user, verify it's seen
1927 nodemap_acl_test $mapped_fs ${clients_arr[0]} ${clients_arr[1]} &&
1928 error "acl count (2)"
1930 # setfacl on mapped cluster to mapped user, verify it's seen
1931 nodemap_acl_test $mapped_c1 ${clients_arr[1]} ${clients_arr[0]} &&
1932 error "acl count (3)"
1934 # setfacl on mapped cluster to unmapped user, verify error
1935 nodemap_acl_test $unmapped_fs ${clients_arr[1]} ${clients_arr[0]} 1 ||
1936 error "acl count (4)"
1939 do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 0
1940 do_facet mgs $LCTL nodemap_modify --name c0 --property trusted --value 0
1942 wait_nm_sync c0 trusted_nodemap
1944 # setfacl to mapped user on c1, also mapped to c0, verify it's seen
1945 nodemap_acl_test $mapped_c1 ${clients_arr[1]} ${clients_arr[0]} &&
1946 error "acl count (5)"
1948 # setfacl to mapped user on c1, not mapped to c0, verify not seen
1949 nodemap_acl_test $unmapped_c1 ${clients_arr[1]} ${clients_arr[0]} ||
1950 error "acl count (6)"
1952 nodemap_test_cleanup
1954 run_test 23a "test mapped regular ACLs"
1956 test_23b() { #LU-9929
1957 [ $num_clients -lt 2 ] && skip "Need 2 clients at least"
1958 [ "$MGS_VERSION" -lt $(version_code 2.10.53) ] &&
1959 skip "Need MGS >= 2.10.53"
1961 export SK_UNIQUE_NM=true
1963 trap nodemap_test_cleanup EXIT
1965 local testdir=$DIR/$tdir
1966 local fs_id=$((IDBASE+10))
1971 do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 1
1972 wait_nm_sync c0 admin_nodemap
1973 do_facet mgs $LCTL nodemap_modify --name c1 --property admin --value 1
1974 wait_nm_sync c1 admin_nodemap
1975 do_facet mgs $LCTL nodemap_modify --name c1 --property trusted --value 1
1976 wait_nm_sync c1 trusted_nodemap
1978 # Add idmap $ID0:$fs_id (500:60010)
1979 do_facet mgs $LCTL nodemap_add_idmap --name c0 --idtype gid \
1980 --idmap $ID0:$fs_id ||
1981 error "add idmap $ID0:$fs_id to nodemap c0 failed"
1982 wait_nm_sync c0 idmap
1984 # set/getfacl default acl on client 1 (unmapped gid=500)
1985 do_node ${clients_arr[0]} rm -rf $testdir
1986 do_node ${clients_arr[0]} mkdir -p $testdir
1987 # Here, USER0=$(getent passwd | grep :$ID0:$ID0: | cut -d: -f1)
1988 do_node ${clients_arr[0]} setfacl -R -d -m group:$USER0:rwx $testdir ||
1989 error "setfacl $testdir on ${clients_arr[0]} failed"
1990 unmapped_id=$(do_node ${clients_arr[0]} getfacl $testdir |
1991 grep -E "default:group:.*:rwx" | awk -F: '{print $3}')
1992 [ "$unmapped_id" = "$USER0" ] ||
1993 error "gid=$ID0 was not unmapped correctly on ${clients_arr[0]}"
1995 # getfacl default acl on client 2 (mapped gid=60010)
1996 mapped_id=$(do_node ${clients_arr[1]} getfacl $testdir |
1997 grep -E "default:group:.*:rwx" | awk -F: '{print $3}')
1998 fs_user=$(do_node ${clients_arr[1]} getent passwd |
1999 grep :$fs_id:$fs_id: | cut -d: -f1)
2000 [ -z "$fs_user" ] && fs_user=$fs_id
2001 [ $mapped_id -eq $fs_id -o "$mapped_id" = "$fs_user" ] ||
2002 error "Should return gid=$fs_id or $fs_user on client2"
2005 nodemap_test_cleanup
2006 export SK_UNIQUE_NM=false
2008 run_test 23b "test mapped default ACLs"
2013 trap nodemap_test_cleanup EXIT
2014 do_nodes $(comma_list $(all_server_nodes)) $LCTL get_param -R nodemap
2016 nodemap_test_cleanup
2018 run_test 24 "check nodemap proc files for LBUGs and Oopses"
2021 local tmpfile=$(mktemp)
2022 local tmpfile2=$(mktemp)
2023 local tmpfile3=$(mktemp)
2024 local tmpfile4=$(mktemp)
2028 nodemap_version_check || return 0
2030 # stop clients for this test
2031 zconf_umount_clients $CLIENTS $MOUNT ||
2032 error "unable to umount clients $CLIENTS"
2034 export SK_UNIQUE_NM=true
2037 # enable trusted/admin for setquota call in cleanup_and_setup_lustre()
2039 for client in $clients; do
2040 do_facet mgs $LCTL nodemap_modify --name c${i} \
2041 --property admin --value 1
2042 do_facet mgs $LCTL nodemap_modify --name c${i} \
2043 --property trusted --value 1
2046 wait_nm_sync c$((i - 1)) trusted_nodemap
2048 trap nodemap_test_cleanup EXIT
2050 # create a new, empty nodemap, and add fileset info to it
2051 do_facet mgs $LCTL nodemap_add test25 ||
2052 error "unable to create nodemap $testname"
2053 do_facet mgs $LCTL set_param -P nodemap.$testname.fileset=/$subdir ||
2054 error "unable to add fileset info to nodemap test25"
2056 wait_nm_sync test25 id
2058 do_facet mgs $LCTL nodemap_info > $tmpfile
2059 do_facet mds $LCTL nodemap_info > $tmpfile2
2061 if ! $SHARED_KEY; then
2062 # will conflict with SK's nodemaps
2063 cleanup_and_setup_lustre
2065 # stop clients for this test
2066 zconf_umount_clients $CLIENTS $MOUNT ||
2067 error "unable to umount clients $CLIENTS"
2069 do_facet mgs $LCTL nodemap_info > $tmpfile3
2070 diff -q $tmpfile3 $tmpfile >& /dev/null ||
2071 error "nodemap_info diff on MGS after remount"
2073 do_facet mds $LCTL nodemap_info > $tmpfile4
2074 diff -q $tmpfile4 $tmpfile2 >& /dev/null ||
2075 error "nodemap_info diff on MDS after remount"
2078 do_facet mgs $LCTL nodemap_del test25 ||
2079 error "cannot delete nodemap test25 from config"
2080 nodemap_test_cleanup
2081 # restart clients previously stopped
2082 zconf_mount_clients $CLIENTS $MOUNT ||
2083 error "unable to mount clients $CLIENTS"
2085 rm -f $tmpfile $tmpfile2
2086 export SK_UNIQUE_NM=false
2088 run_test 25 "test save and reload nodemap config"
2091 nodemap_version_check || return 0
2095 do_facet mgs "seq -f 'c%g' $large_i | xargs -n1 $LCTL nodemap_add"
2096 wait_nm_sync c$large_i admin_nodemap
2098 do_facet mgs "seq -f 'c%g' $large_i | xargs -n1 $LCTL nodemap_del"
2099 wait_nm_sync c$large_i admin_nodemap
2101 run_test 26 "test transferring very large nodemap"
2103 nodemap_exercise_fileset() {
2106 local check_proj=true
2108 (( $MDS1_VERSION >= $(version_code 2.14.52) )) || check_proj=false
2111 if [ "$nm" == "default" ]; then
2112 do_facet mgs $LCTL nodemap_activate 1
2114 do_facet mgs $LCTL nodemap_modify --name default \
2115 --property admin --value 1
2116 do_facet mgs $LCTL nodemap_modify --name default \
2117 --property trusted --value 1
2118 wait_nm_sync default admin_nodemap
2119 wait_nm_sync default trusted_nodemap
2124 if $SHARED_KEY; then
2125 export SK_UNIQUE_NM=true
2127 # will conflict with SK's nodemaps
2128 trap "fileset_test_cleanup $nm" EXIT
2130 fileset_test_setup "$nm"
2132 # add fileset info to $nm nodemap
2133 if ! combined_mgs_mds; then
2134 do_facet mgs $LCTL set_param nodemap.${nm}.fileset=/$subdir ||
2135 error "unable to add fileset info to $nm nodemap on MGS"
2137 do_facet mgs $LCTL set_param -P nodemap.${nm}.fileset=/$subdir ||
2138 error "unable to add fileset info to $nm nodemap for servers"
2139 wait_nm_sync $nm fileset "nodemap.${nm}.fileset=/$subdir"
2141 if $check_proj; then
2142 do_facet mgs $LCTL nodemap_modify --name $nm \
2143 --property admin --value 1
2144 wait_nm_sync $nm admin_nodemap
2145 do_facet mgs $LCTL nodemap_modify --name $nm \
2146 --property trusted --value 0
2147 wait_nm_sync $nm trusted_nodemap
2148 do_facet mgs $LCTL nodemap_modify --name $nm \
2149 --property map_mode --value projid
2150 wait_nm_sync $nm map_mode
2151 do_facet mgs $LCTL nodemap_add_idmap --name $nm \
2152 --idtype projid --idmap 1:1
2153 do_facet mgs $LCTL nodemap_modify --name $nm \
2154 --property deny_unknown --value 1
2155 wait_nm_sync $nm deny_unknown
2159 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2160 error "unable to umount client ${clients_arr[0]}"
2161 # set some generic fileset to trigger SSK code
2163 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2164 error "unable to remount client ${clients_arr[0]}"
2167 # test mount point content
2168 do_node ${clients_arr[0]} test -f $MOUNT/this_is_$subdir ||
2169 error "fileset not taken into account"
2171 if $check_proj; then
2172 do_node ${clients_arr[0]} $LFS setquota -p 1 -b 10000 -B 11000 \
2173 -i 0 -I 0 $MOUNT || error "setquota -p 1 failed"
2174 do_node ${clients_arr[0]} $LFS setquota -p 2 -b 10000 -B 11000 \
2175 -i 0 -I 0 $MOUNT && error "setquota -p 2 should fail"
2178 # re-mount client with sub-subdir
2179 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2180 error "unable to umount client ${clients_arr[0]}"
2181 export FILESET=/$subsubdir
2182 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2183 error "unable to remount client ${clients_arr[0]}"
2186 # test mount point content
2187 do_node ${clients_arr[0]} test -f $MOUNT/this_is_$subsubdir ||
2188 error "subdir of fileset not taken into account"
2190 # remove fileset info from nodemap
2191 do_facet mgs $LCTL nodemap_set_fileset --name $nm --fileset clear ||
2192 error "unable to delete fileset info on $nm nodemap"
2193 wait_update_facet mgs "$LCTL get_param nodemap.${nm}.fileset" \
2194 "nodemap.${nm}.fileset=" ||
2195 error "fileset info still not cleared on $nm nodemap"
2196 do_facet mgs $LCTL set_param -P nodemap.${nm}.fileset=clear ||
2197 error "unable to reset fileset info on $nm nodemap"
2198 wait_nm_sync $nm fileset "nodemap.${nm}.fileset="
2199 do_facet mgs $LCTL set_param -P -d nodemap.${nm}.fileset ||
2200 error "unable to remove fileset rule on $nm nodemap"
2203 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2204 error "unable to umount client ${clients_arr[0]}"
2205 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2206 error "unable to remount client ${clients_arr[0]}"
2208 # test mount point content
2209 if ! $(do_node ${clients_arr[0]} test -d $MOUNT/$subdir); then
2211 error "fileset not cleared on $nm nodemap"
2214 # back to non-nodemap setup
2215 if $SHARED_KEY; then
2216 export SK_UNIQUE_NM=false
2217 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2218 error "unable to umount client ${clients_arr[0]}"
2220 fileset_test_cleanup "$nm"
2221 if [ "$nm" == "default" ]; then
2222 do_facet mgs $LCTL nodemap_modify --name default \
2223 --property admin --value 0
2224 do_facet mgs $LCTL nodemap_modify --name default \
2225 --property trusted --value 0
2226 wait_nm_sync default admin_nodemap
2227 wait_nm_sync default trusted_nodemap
2228 do_facet mgs $LCTL nodemap_activate 0
2229 wait_nm_sync active 0
2231 export SK_UNIQUE_NM=false
2233 nodemap_test_cleanup
2235 if $SHARED_KEY; then
2236 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2237 error "unable to remount client ${clients_arr[0]}"
2242 [ "$MDS1_VERSION" -lt $(version_code 2.11.50) ] &&
2243 skip "Need MDS >= 2.11.50"
2245 # if servers run on the same node, it is impossible to tell if they get
2246 # synced with the mgs, so this test needs to be skipped
2247 if [ $(facet_active_host mgs) == $(facet_active_host mds) ] &&
2248 [ $(facet_active_host mgs) == $(facet_active_host ost1) ]; then
2249 skip "local mode not supported"
2252 for nm in "default" "c0"; do
2253 local subdir="subdir_${nm}"
2254 local subsubdir="subsubdir_${nm}"
2256 if [ "$nm" == "default" ] && [ "$SHARED_KEY" == "true" ]; then
2257 echo "Skipping nodemap $nm with SHARED_KEY";
2261 echo "Exercising fileset for nodemap $nm"
2262 nodemap_exercise_fileset "$nm"
2265 run_test 27a "test fileset in various nodemaps"
2267 test_27b() { #LU-10703
2268 [ "$MDS1_VERSION" -lt $(version_code 2.11.50) ] &&
2269 skip "Need MDS >= 2.11.50"
2270 [[ $MDSCOUNT -lt 2 ]] && skip "needs >= 2 MDTs"
2272 # if servers run on the same node, it is impossible to tell if they get
2273 # synced with the mgs, so this test needs to be skipped
2274 if [ $(facet_active_host mgs) == $(facet_active_host mds) ] &&
2275 [ $(facet_active_host mgs) == $(facet_active_host ost1) ]; then
2276 skip "local mode not supported"
2280 trap nodemap_test_cleanup EXIT
2282 # Add the nodemaps and set their filesets
2283 for i in $(seq 1 $MDSCOUNT); do
2284 do_facet mgs $LCTL nodemap_del nm$i 2>/dev/null
2285 do_facet mgs $LCTL nodemap_add nm$i ||
2286 error "add nodemap nm$i failed"
2287 wait_nm_sync nm$i "" "" "-N"
2289 if ! combined_mgs_mds; then
2291 $LCTL set_param nodemap.nm$i.fileset=/dir$i ||
2292 error "set nm$i.fileset=/dir$i failed on MGS"
2294 do_facet mgs $LCTL set_param -P nodemap.nm$i.fileset=/dir$i ||
2295 error "set nm$i.fileset=/dir$i failed on servers"
2296 wait_nm_sync nm$i fileset "nodemap.nm$i.fileset=/dir$i"
2299 # Check if all the filesets are correct
2300 for i in $(seq 1 $MDSCOUNT); do
2301 fileset=$(do_facet mds$i \
2302 $LCTL get_param -n nodemap.nm$i.fileset)
2303 [ "$fileset" = "/dir$i" ] ||
2304 error "nm$i.fileset $fileset != /dir$i on mds$i"
2305 do_facet mgs $LCTL set_param -P -d nodemap.nm$i.fileset ||
2306 error "unable to remove fileset rule for nm$i nodemap"
2307 do_facet mgs $LCTL nodemap_del nm$i ||
2308 error "delete nodemap nm$i failed"
2311 nodemap_test_cleanup
2313 run_test 27b "The new nodemap won't clear the old nodemap's fileset"
2316 if ! $SHARED_KEY; then
2317 skip "need shared key feature for this test" && return
2319 mkdir -p $DIR/$tdir || error "mkdir failed"
2320 touch $DIR/$tdir/$tdir.out || error "touch failed"
2321 if [ ! -f $DIR/$tdir/$tdir.out ]; then
2322 error "read before rotation failed"
2324 # check srpc_contexts is valid YAML
2325 $LCTL get_param -n *.*.srpc_contexts 2>/dev/null | verify_yaml ||
2326 error "srpc_contexts is not valid YAML"
2327 # store top key identity to ensure rotation has occurred
2328 SK_IDENTITY_OLD=$($LCTL get_param -n *.*.srpc_contexts 2>/dev/null |
2329 head -n 1 | awk 'BEGIN{RS=", "} $1=="expire:"{print $2}')
2330 do_facet $SINGLEMDS lfs flushctx ||
2331 error "could not run flushctx on $SINGLEMDS"
2333 lfs flushctx || error "could not run flushctx on client"
2335 # verify new key is in place
2336 SK_IDENTITY_NEW=$($LCTL get_param -n *.*.srpc_contexts 2>/dev/null |
2337 head -n 1 | awk 'BEGIN{RS=", "} $1=="expire:"{print $2}')
2338 if [ $SK_IDENTITY_OLD == $SK_IDENTITY_NEW ]; then
2339 error "key did not rotate correctly"
2341 if [ ! -f $DIR/$tdir/$tdir.out ]; then
2342 error "read after rotation failed"
2345 run_test 28 "check shared key rotation method"
2348 if ! $SHARED_KEY; then
2349 skip "need shared key feature for this test" && return
2351 if [ $SK_FLAVOR != "ski" ] && [ $SK_FLAVOR != "skpi" ]; then
2352 skip "test only valid if integrity is active"
2355 mkdir $DIR/$tdir || error "mkdir"
2356 touch $DIR/$tdir/$tfile || error "touch"
2357 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2358 error "unable to umount clients"
2359 do_node ${clients_arr[0]} "keyctl show |
2360 awk '/lustre/ { print \\\$1 }' | xargs -IX keyctl unlink X"
2361 OLD_SK_PATH=$SK_PATH
2362 export SK_PATH=/dev/null
2363 if zconf_mount_clients ${clients_arr[0]} $MOUNT; then
2364 export SK_PATH=$OLD_SK_PATH
2365 do_node ${clients_arr[0]} "ls $DIR/$tdir/$tfile"
2366 if [ $? -eq 0 ]; then
2367 error "able to mount and read without key"
2369 error "able to mount without key"
2372 export SK_PATH=$OLD_SK_PATH
2373 do_node ${clients_arr[0]} "keyctl show |
2374 awk '/lustre/ { print \\\$1 }' |
2375 xargs -IX keyctl unlink X"
2377 zconf_mount_clients ${clients_arr[0]} $MOUNT ||
2378 error "unable to mount clients"
2380 run_test 29 "check for missing shared key"
2383 if ! $SHARED_KEY; then
2384 skip "need shared key feature for this test" && return
2386 if [ $SK_FLAVOR != "ski" ] && [ $SK_FLAVOR != "skpi" ]; then
2387 skip "test only valid if integrity is active"
2389 mkdir -p $DIR/$tdir || error "mkdir failed"
2390 touch $DIR/$tdir/$tdir.out || error "touch failed"
2391 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2392 error "unable to umount clients"
2393 # unload keys from ring
2394 do_node ${clients_arr[0]} "keyctl show |
2395 awk '/lustre/ { print \\\$1 }' | xargs -IX keyctl unlink X"
2396 # generate key with bogus filesystem name
2397 do_node ${clients_arr[0]} "$LGSS_SK -w $SK_PATH/$FSNAME-bogus.key \
2398 -f $FSNAME.bogus -t client -d /dev/urandom" ||
2399 error "lgss_sk failed (1)"
2400 do_facet $SINGLEMDS lfs flushctx || error "could not run flushctx"
2401 OLD_SK_PATH=$SK_PATH
2402 export SK_PATH=$SK_PATH/$FSNAME-bogus.key
2403 if zconf_mount_clients ${clients_arr[0]} $MOUNT; then
2404 SK_PATH=$OLD_SK_PATH
2405 do_node ${clients_arr[0]} "ls $DIR/$tdir/$tdir.out"
2406 if [ $? -eq 0 ]; then
2407 error "mount and read file with invalid key"
2409 error "mount with invalid key"
2412 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2413 error "unable to umount clients"
2414 # unload keys from ring
2415 do_node ${clients_arr[0]} "keyctl show |
2416 awk '/lustre/ { print \\\$1 }' | xargs -IX keyctl unlink X"
2418 SK_PATH=$OLD_SK_PATH
2419 zconf_mount_clients ${clients_arr[0]} $MOUNT ||
2420 error "unable to mount clients"
2422 run_test 30 "check for invalid shared key"
2427 mkdir -p $DIR/$tdir/dir0 || error "mkdir $flvr"
2428 touch $DIR/$tdir/dir0/f0 || error "touch $flvr"
2429 ls $DIR/$tdir/dir0 || error "ls $flvr"
2430 dd if=/dev/zero of=$DIR/$tdir/dir0/f0 conv=fsync bs=1M count=10 \
2431 >& /dev/null || error "dd $flvr"
2432 rm -f $DIR/$tdir/dir0/f0 || error "rm $flvr"
2433 rmdir $DIR/$tdir/dir0 || error "rmdir $flvr"
2436 echo 3 > /proc/sys/vm/drop_caches
2440 # restore clients' idle_timeout
2441 for c in ${clients//,/ }; do
2442 param=IDLETIME_$(echo $c | cut -d'.' -f1 | sed s+-+_+g)
2443 do_node $c "lctl set_param osc.*.idle_timeout=${!param}"
2448 local save_flvr=$SK_FLAVOR
2450 if ! $SHARED_KEY; then
2451 skip "need shared key feature for this test"
2454 # save clients' idle_timeout, and set all to 0 for this test,
2455 # as we do not want connections to go idle
2456 for c in ${clients//,/ }; do
2457 param=IDLETIME_$(echo $c | cut -d'.' -f1 | sed s+-+_+g)
2458 idle=$(do_node $c lctl get_param -n osc.*.idle_timeout |
2460 eval export $param=\$idle
2461 do_node $c lctl set_param osc.*.idle_timeout=0
2464 stack_trap cleanup_30b EXIT
2465 stack_trap restore_to_default_flavor EXIT
2467 lfs mkdir -i 0 -c 1 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2468 lfs setstripe -c -1 $DIR/$tdir/fileA ||
2469 error "setstripe $DIR/$tdir/fileA failed"
2470 echo 30b > $DIR/$tdir/fileA ||
2471 error "wrtie to $DIR/$tdir/fileA failed"
2473 for flvr in skn ska ski skpi; do
2476 restore_to_default_flavor || error "cannot set $flvr flavor"
2477 SK_FLAVOR=$save_flvr
2482 run_test 30b "basic test of all different SSK flavors"
2485 local failover_mds1=$1
2488 zconf_umount $HOSTNAME $MOUNT || error "unable to umount client"
2490 # necessary to do writeconf in order to de-register
2491 # @${NETTYPE}999 nid for targets
2493 export KEEP_ZPOOL="true"
2495 LOAD_MODULES_REMOTE=true unload_modules
2496 LOAD_MODULES_REMOTE=true load_modules
2498 do_facet mds1 $TUNEFS --erase-param failover.node $(mdsdevname 1)
2499 if [ -n "$failover_mds1" ]; then
2500 do_facet mds1 $TUNEFS \
2501 --servicenode=$failover_mds1 $(mdsdevname 1)
2503 # If no service node previously existed, setting one in test_31
2504 # added the no_primnode flag to the target. To remove everything
2505 # and clear the flag, add a meaningless failnode and remove it.
2506 do_facet mds1 $TUNEFS \
2507 --failnode=$(do_facet mds1 $LCTL list_nids | head -1) \
2509 do_facet mds1 $TUNEFS \
2510 --erase-param failover.node $(mdsdevname 1)
2513 export SK_MOUNTED=false
2516 export KEEP_ZPOOL="$KZPOOL"
2520 local nid=$(lctl list_nids | grep ${NETTYPE} | head -n1)
2521 local addr=${nid%@*}
2523 local net2=${NETTYPE}999
2524 local mdsnid=$(do_facet mds1 $LCTL list_nids | head -1)
2525 local addr1=${mdsnid%@*}
2526 local addr2 failover_mds1
2528 export LNETCTL=$(which lnetctl 2> /dev/null)
2530 [ -z "$LNETCTL" ] && skip "without lnetctl support." && return
2531 local_mode && skip "in local mode."
2533 if $SHARED_KEY; then
2534 skip "Conflicting test with SSK"
2537 if [[ $addr1 =~ ^([0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}$ ]]; then
2538 local tmp=$(printf "%x" $(((0x${addr1##*:} + 11) % 65536)))
2540 addr2=${addr1%:*}:${tmp}
2541 elif [[ $addr1 =~ ^([0-9]{1,3}\.){3,3}[0-9]{1,3}$ ]]; then
2542 addr2=${addr1%.*}.$(((${addr1##*.} + 11) % 256))
2543 elif [[ $addr1 =~ ^[0-9]+$ ]]; then
2544 addr2=$((addr1 + 11))
2547 # build list of interface on nodes
2548 for node in $(all_nodes); do
2549 infname=inf_$(echo $node | cut -d'.' -f1 | sed s+-+_+g)
2550 itf=$(do_node $node $LNETCTL net show --net $net |
2552 {if (inf==1) { print $2; exit; } fi} /interfaces/{inf=1}')
2557 local mgsnid_orig=$MGSNID
2558 # compute new MGSNID
2559 local mgsnid_new=${MGSNID%@*}@$net2
2561 # save mds failover nids for restore at cleanup
2562 failover_mds1=$(do_facet mds1 $TUNEFS --dryrun $(mdsdevname 1))
2563 if [ -n "$failover_mds1" ]; then
2564 failover_mds1=${failover_mds1##*Parameters:}
2565 failover_mds1=${failover_mds1%%exiting*}
2566 failover_mds1=$(echo $failover_mds1 | tr ' ' '\n' |
2567 grep failover.node | cut -d'=' -f2-)
2569 stack_trap "cleanup_31 $failover_mds1" EXIT
2572 if [ "$MOUNT_2" ] && $(grep -q $MOUNT2' ' /proc/mounts); then
2573 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
2575 if $(grep -q $MOUNT' ' /proc/mounts); then
2576 umount_client $MOUNT || error "umount $MOUNT failed"
2579 # check exports on servers are empty for client
2580 do_facet mgs "lctl get_param *.MGS*.exports.*.export"
2581 do_facet mgs "lctl get_param -n *.MGS*.exports.'$nid'.uuid 2>/dev/null |
2582 grep -q -" && error "export on MGS should be empty"
2583 do_nodes $(comma_list $(mdts_nodes) $(osts_nodes)) \
2584 "lctl get_param -n *.${FSNAME}*.exports.'$nid'.uuid \
2585 2>/dev/null | grep -q -" &&
2586 error "export on servers should be empty"
2589 export KEEP_ZPOOL="true"
2590 stopall || error "stopall failed"
2591 LOAD_MODULES_REMOTE=true unload_modules ||
2592 error "Failed to unload modules"
2594 # add network $net2 on all nodes
2595 do_rpc_nodes $(comma_list $(all_nodes)) load_modules ||
2596 error "unable to load modules on $(all_nodes)"
2597 for node in $(all_nodes); do
2598 do_node $node "$LNETCTL lnet configure" ||
2599 error "unable to configure lnet on node $node"
2600 infname=inf_$(echo $node | cut -d'.' -f1 | sed s+-+_+g)
2601 do_node $node "$LNETCTL net add --if ${!infname} --net $net2" ||
2602 error "unable to configure NID on $net2 for node $node"
2605 LOAD_MODULES_REMOTE=true load_modules || error "failed to load modules"
2607 # necessary to do writeconf in order to register
2608 # new @$net2 nid for targets
2609 export SK_MOUNTED=false
2610 writeconf_all || error "writeconf failed"
2612 nids="${addr1}@$net,${addr1}@$net2:${addr2}@$net,${addr2}@$net2"
2613 do_facet mds1 "$TUNEFS --servicenode="$nids" $(mdsdevname 1)" ||
2614 error "tunefs failed"
2616 setupall server_only || error "setupall failed"
2617 export KEEP_ZPOOL="$KZPOOL"
2621 stack_trap "MGSNID=$mgsnid_orig" EXIT
2623 # on client, reconfigure LNet and turn LNet Dynamic Discovery off
2624 $LUSTRE_RMMOD || error "$LUSTRE_RMMOD failed (1)"
2625 load_modules || error "Failed to load modules"
2626 $LNETCTL set discovery 0 || error "Failed to disable discovery"
2627 $LNETCTL lnet configure ||
2628 error "unable to configure lnet on client"
2629 infname=inf_$(echo $(hostname -s) | sed s+-+_+g)
2630 $LNETCTL net add --if ${!infname} --net $net2 ||
2631 error "unable to configure NID on $net2 on client (1)"
2633 # mount client with -o network=$net2 option
2634 mount_client $MOUNT ${MOUNT_OPTS},network=$net2 ||
2635 error "unable to remount client"
2637 # check export on MGS
2638 do_facet mgs "lctl get_param *.MGS*.exports.*.export"
2639 do_facet mgs "lctl get_param -n *.MGS*.exports.'$nid'.uuid 2>/dev/null |
2641 [ $? -ne 0 ] || error "export for $nid on MGS should not exist"
2644 "lctl get_param -n *.MGS*.exports.'${addr}@$net2'.uuid \
2645 2>/dev/null | grep -"
2647 error "export for ${addr}@$net2 on MGS should exist"
2649 # check {mdc,osc} imports
2650 lctl get_param mdc.${FSNAME}-*.import | grep current_connection |
2653 error "import for mdc should use ${addr1}@$net2"
2654 lctl get_param osc.${FSNAME}-*.import | grep current_connection |
2657 error "import for osc should use ${addr1}@$net2"
2659 # no NIDs on other networks should be listed
2660 lctl get_param mdc.${FSNAME}-*.import | grep failover_nids |
2661 grep -w ".*@$net" &&
2662 error "MDC import shouldn't have failnids at @$net"
2664 # failover NIDs on net999 should be listed
2665 lctl get_param mdc.${FSNAME}-*.import | grep failover_nids |
2666 grep ${addr2}@$net2 ||
2667 error "MDC import should have failnid ${addr2}@$net2"
2670 zconf_umount $HOSTNAME $MOUNT || error "unable to umount client"
2672 # on client, configure LNet and turn LNet Dynamic Discovery on (default)
2673 $LUSTRE_RMMOD || error "$LUSTRE_RMMOD failed (2)"
2674 load_modules || error "Failed to load modules"
2675 $LNETCTL lnet configure ||
2676 error "unable to configure lnet on client"
2677 infname=inf_$(echo $(hostname -s) | sed s+-+_+g)
2678 $LNETCTL net add --if ${!infname} --net $net2 ||
2679 error "unable to configure NID on $net2 on client (2)"
2681 # mount client with -o network=$net2 option:
2682 # should fail because of LNet Dynamic Discovery
2683 mount_client $MOUNT ${MOUNT_OPTS},network=$net2 &&
2684 error "client mount with '-o network' option should be refused"
2688 run_test 31 "client mount option '-o network'"
2692 zconf_umount_clients ${clients_arr[0]} $MOUNT
2694 # disable sk flavor enforcement on MGS
2695 set_rule _mgs any any null
2697 # stop gss daemon on MGS
2698 send_sigint $mgs_HOST lsvcgssd
2700 # re-start gss daemon on MDS if necessary
2701 if combined_mgs_mds ; then
2702 start_gss_daemons $mds_HOST $LSVCGSSD "-vvv -s -m -o -z"
2705 # restore MGS NIDs in key on MGS
2706 do_nodes $mgs_HOST "$LGSS_SK -g $MGSNID -m \
2707 $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2708 error "could not modify keyfile on MGS (3)"
2710 # load modified key file on MGS
2711 do_nodes $mgs_HOST "$LGSS_SK -l $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2712 error "could not load keyfile on MGS (3)"
2714 # restore MGS NIDs in key on client
2715 do_nodes ${clients_arr[0]} "$LGSS_SK -g $MGSNID -m \
2716 $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2717 error "could not modify keyfile on client (3)"
2720 MOUNT_OPTS=$(add_sk_mntflag $MOUNT_OPTS)
2723 restore_to_default_flavor
2727 local mgsnid2=$(host_nids_address $ost1_HOST $NETTYPE)@${MGSNID#*@}
2728 local mgsorig=$MGSNID
2730 if ! $SHARED_KEY; then
2731 skip "need shared key feature for this test"
2734 stack_trap cleanup_32 EXIT
2736 # restore to default null flavor
2737 save_flvr=$SK_FLAVOR
2739 restore_to_default_flavor || error "cannot set null flavor"
2740 SK_FLAVOR=$save_flvr
2743 if [ "$MOUNT_2" ] && $(grep -q $MOUNT2' ' /proc/mounts); then
2744 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
2746 if $(grep -q $MOUNT' ' /proc/mounts); then
2747 umount_client $MOUNT || error "umount $MOUNT failed"
2750 # kill daemon on MGS to start afresh
2751 send_sigint $mgs_HOST lsvcgssd
2753 # start gss daemon on MGS
2754 if combined_mgs_mds ; then
2755 start_gss_daemons $mgs_HOST $LSVCGSSD "-vvv -s -g -m -o -z"
2757 start_gss_daemons $mgs_HOST $LSVCGSSD "-vvv -s -g"
2760 # add mgs key type and MGS NIDs in key on MGS
2761 do_nodes $mgs_HOST "$LGSS_SK -t mgs,server -g $MGSNID -m \
2762 $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2763 error "could not modify keyfile on MGS (1)"
2765 # load modified key file on MGS
2766 do_nodes $mgs_HOST "$LGSS_SK -l $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2767 error "could not load keyfile on MGS (1)"
2769 # add MGS NIDs in key on client
2770 do_nodes ${clients_arr[0]} "$LGSS_SK -g $MGSNID -m \
2771 $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2772 error "could not modify keyfile on client (1)"
2774 # set perms for per-nodemap keys else permission denied
2775 do_nodes $(comma_list $(all_nodes)) \
2776 "keyctl show | grep lustre | cut -c1-11 |
2778 xargs -IX keyctl setperm X 0x3f3f3f3f"
2780 # re-mount client with mgssec=skn
2781 save_opts=$MOUNT_OPTS
2782 stack_trap "MOUNT_OPTS=$save_opts" EXIT
2783 if [ -z "$MOUNT_OPTS" ]; then
2784 MOUNT_OPTS="-o mgssec=skn"
2786 MOUNT_OPTS="$MOUNT_OPTS,mgssec=skn"
2788 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2789 error "mount ${clients_arr[0]} with mgssec=skn failed"
2790 MOUNT_OPTS=$save_opts
2793 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2794 error "umount ${clients_arr[0]} failed"
2796 # enforce ska flavor on MGS
2797 set_rule _mgs any any ska
2799 # re-mount client without mgssec
2800 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS &&
2801 error "mount ${clients_arr[0]} without mgssec should fail"
2803 # re-mount client with mgssec=skn
2804 save_opts=$MOUNT_OPTS
2805 if [ -z "$MOUNT_OPTS" ]; then
2806 MOUNT_OPTS="-o mgssec=skn"
2808 MOUNT_OPTS="$MOUNT_OPTS,mgssec=skn"
2810 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS &&
2811 error "mount ${clients_arr[0]} with mgssec=skn should fail"
2812 MOUNT_OPTS=$save_opts
2814 # re-mount client with mgssec=ska
2815 save_opts=$MOUNT_OPTS
2816 if [ -z "$MOUNT_OPTS" ]; then
2817 MOUNT_OPTS="-o mgssec=ska"
2819 MOUNT_OPTS="$MOUNT_OPTS,mgssec=ska"
2821 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2822 error "mount ${clients_arr[0]} with mgssec=ska failed"
2824 MGSNID=$mgsnid2:$mgsorig
2825 stack_trap "MGSNID=$mgsorig" EXIT
2828 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2829 error "umount ${clients_arr[0]} failed"
2831 # add MGS NIDs in key on MGS
2832 do_nodes $mgs_HOST "$LGSS_SK -g ${MGSNID//:/,} -m \
2833 $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2834 error "could not modify keyfile on MGS (2)"
2836 # load modified key file on MGS
2837 do_nodes $mgs_HOST "$LGSS_SK -l $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2838 error "could not load keyfile on MGS (2)"
2840 # add MGS NIDs in key on client
2841 do_nodes ${clients_arr[0]} "$LGSS_SK -g ${MGSNID//:/,} -m \
2842 $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2843 error "could not modify keyfile on client (2)"
2845 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2846 error "mount ${clients_arr[0]} with alternate mgsnid failed"
2848 run_test 32 "check for mgssec"
2851 # disable sk flavor enforcement
2852 set_rule $FSNAME any cli2mdt null
2853 wait_flavor cli2mdt null
2856 zconf_umount_clients ${clients_arr[0]} $MOUNT
2858 # stop gss daemon on MGS
2859 send_sigint $mgs_HOST lsvcgssd
2861 # re-start gss daemon on MDS if necessary
2862 if combined_mgs_mds ; then
2863 start_gss_daemons $mds_HOST $LSVCGSSD "-vvv -s -m -o -z"
2867 MOUNT_OPTS=$(add_sk_mntflag $MOUNT_OPTS)
2870 restore_to_default_flavor
2874 if ! $SHARED_KEY; then
2875 skip "need shared key feature for this test"
2878 stack_trap cleanup_33 EXIT
2880 # restore to default null flavor
2881 save_flvr=$SK_FLAVOR
2883 restore_to_default_flavor || error "cannot set null flavor"
2884 SK_FLAVOR=$save_flvr
2887 if [ "$MOUNT_2" ] && $(grep -q $MOUNT2' ' /proc/mounts); then
2888 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
2890 if $(grep -q $MOUNT' ' /proc/mounts); then
2891 umount_client $MOUNT || error "umount $MOUNT failed"
2894 # kill daemon on MGS to start afresh
2895 send_sigint $mgs_HOST lsvcgssd
2897 # start gss daemon on MGS
2898 if combined_mgs_mds ; then
2899 start_gss_daemons $mgs_HOST $LSVCGSSD "-vvv -s -g -m -o -z"
2901 start_gss_daemons $mgs_HOST $LSVCGSSD "-vvv -s -g"
2904 # add mgs key type and MGS NIDs in key on MGS
2905 do_nodes $mgs_HOST "$LGSS_SK -t mgs,server -g $MGSNID -m \
2906 $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2907 error "could not modify keyfile on MGS"
2909 # load modified key file on MGS
2910 do_nodes $mgs_HOST "$LGSS_SK -l $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2911 error "could not load keyfile on MGS"
2913 # add MGS NIDs in key on client
2914 do_nodes ${clients_arr[0]} "$LGSS_SK -g $MGSNID -m \
2915 $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2916 error "could not modify keyfile on MGS"
2918 # set perms for per-nodemap keys else permission denied
2919 do_nodes $(comma_list $(all_nodes)) \
2920 "keyctl show | grep lustre | cut -c1-11 |
2922 xargs -IX keyctl setperm X 0x3f3f3f3f"
2924 # re-mount client with mgssec=skn
2925 save_opts=$MOUNT_OPTS
2926 if [ -z "$MOUNT_OPTS" ]; then
2927 MOUNT_OPTS="-o mgssec=skn"
2929 MOUNT_OPTS="$MOUNT_OPTS,mgssec=skn"
2931 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2932 error "mount ${clients_arr[0]} with mgssec=skn failed"
2933 MOUNT_OPTS=$save_opts
2935 # enforce ska flavor for cli2mdt
2936 set_rule $FSNAME any cli2mdt ska
2937 wait_flavor cli2mdt ska
2939 # check error message
2940 $LCTL dk | grep "faked source" &&
2941 error "MGS connection srpc flags incorrect"
2945 run_test 33 "correct srpc flags for MGS connection"
2948 # restore deny_unknown
2949 do_facet mgs $LCTL nodemap_modify --name default \
2950 --property deny_unknown --value $denydefault
2951 if [ $? -ne 0 ]; then
2952 error_noexit "cannot reset deny_unknown on default nodemap"
2956 wait_nm_sync default deny_unknown
2963 [ $MGS_VERSION -lt $(version_code 2.12.51) ] &&
2964 skip "deny_unknown on default nm not supported before 2.12.51"
2966 activedefault=$(do_facet mgs $LCTL get_param -n nodemap.active)
2968 if [[ "$activedefault" != "1" ]]; then
2969 do_facet mgs $LCTL nodemap_activate 1
2971 stack_trap cleanup_active EXIT
2974 denydefault=$(do_facet mgs $LCTL get_param -n \
2975 nodemap.default.deny_unknown)
2976 [ -z "$denydefault" ] &&
2977 error "cannot get deny_unknown on default nodemap"
2978 if [ "$denydefault" -eq 0 ]; then
2984 do_facet mgs $LCTL nodemap_modify --name default \
2985 --property deny_unknown --value $denynew ||
2986 error "cannot set deny_unknown on default nodemap"
2988 [ "$(do_facet mgs $LCTL get_param -n nodemap.default.deny_unknown)" \
2990 error "setting deny_unknown on default nodemap did not work"
2992 stack_trap cleanup_34_deny EXIT
2994 wait_nm_sync default deny_unknown
2996 run_test 34 "deny_unknown on default nodemap"
2999 (( $MDS1_VERSION >= $(version_code 2.13.50) )) ||
3000 skip "Need MDS >= 2.13.50"
3002 # activate changelogs
3003 changelog_register || error "changelog_register failed"
3004 local cl_user="${CL_USERS[$SINGLEMDS]%% *}"
3005 changelog_users $SINGLEMDS | grep -q $cl_user ||
3006 error "User $cl_user not found in changelog_users"
3007 changelog_chmask ALL
3010 mkdir $DIR/$tdir || error "failed to mkdir $tdir"
3011 touch $DIR/$tdir/$tfile || error "failed to touch $tfile"
3013 # access changelogs with root
3014 changelog_dump || error "failed to dump changelogs"
3015 changelog_clear 0 || error "failed to clear changelogs"
3017 # put clients in non-admin nodemap
3019 stack_trap nodemap_test_cleanup EXIT
3020 for i in $(seq 0 $((num_clients-1))); do
3021 do_facet mgs $LCTL nodemap_modify --name c${i} \
3022 --property admin --value 0
3024 for i in $(seq 0 $((num_clients-1))); do
3025 wait_nm_sync c${i} admin_nodemap
3028 # access with mapped root
3029 changelog_dump && error "dump changelogs should have failed"
3030 changelog_clear 0 && error "clear changelogs should have failed"
3034 run_test 35 "Check permissions when accessing changelogs"
3037 local mode='\x00\x00\x00\x00'
3038 local raw="$(printf ""\\\\x%02x"" {0..63})"
3042 [[ $(lscpu) =~ Byte\ Order.*Little ]] && size='\x40\x00\x00\x00' ||
3043 size='\x00\x00\x00\x40'
3044 key="${mode}${raw}${size}"
3045 echo -n -e "${key}" | keyctl padd logon fscrypt:4242424242424242 @s
3050 sync ; echo 3 > /proc/sys/vm/drop_caches
3057 $LCTL set_param -n ldlm.namespaces.*.lru_size=clear
3058 sync ; echo 3 > /proc/sys/vm/drop_caches
3059 dummy_key=$(keyctl show | awk '$7 ~ "^fscrypt:" {print $1}')
3060 if [ -n "$dummy_key" ]; then
3061 keyctl revoke $dummy_key
3066 remount_client_normally() {
3067 # remount client without dummy encryption key
3068 if is_mounted $MOUNT; then
3069 umount_client $MOUNT || error "umount $MOUNT failed"
3071 mount_client $MOUNT ${MOUNT_OPTS} ||
3072 error "remount failed"
3074 if is_mounted $MOUNT2; then
3075 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
3077 if [ "$MOUNT_2" ]; then
3078 mount_client $MOUNT2 ${MOUNT_OPTS} ||
3079 error "remount failed"
3086 remount_client_dummykey() {
3089 # remount client with dummy encryption key
3090 if is_mounted $MOUNT; then
3091 umount_client $MOUNT || error "umount $MOUNT failed"
3093 mount_client $MOUNT ${MOUNT_OPTS},test_dummy_encryption ||
3094 error "remount failed"
3099 setup_for_enc_tests() {
3100 # remount client with test_dummy_encryption option
3101 if is_mounted $MOUNT; then
3102 umount_client $MOUNT || error "umount $MOUNT failed"
3104 mount_client $MOUNT ${MOUNT_OPTS},test_dummy_encryption ||
3105 error "mount with '-o test_dummy_encryption' failed"
3109 # this directory will be encrypted, because of dummy mode
3113 cleanup_for_enc_tests() {
3114 rm -rf $DIR/$tdir $*
3116 remount_client_normally
3119 cleanup_nodemap_after_enc_tests() {
3120 umount_client $MOUNT || true
3122 if (( MGS_VERSION >= $(version_code 2.13.55) )); then
3123 do_facet mgs $LCTL nodemap_modify --name default \
3124 --property forbid_encryption --value 0
3125 if (( MGS_VERSION >= $(version_code 2.15.51) )); then
3126 do_facet mgs $LCTL nodemap_modify --name default \
3127 --property readonly_mount --value 0
3130 do_facet mgs $LCTL nodemap_modify --name default \
3131 --property trusted --value 0
3132 do_facet mgs $LCTL nodemap_modify --name default \
3133 --property admin --value 0
3134 do_facet mgs $LCTL nodemap_activate 0
3136 if (( MGS_VERSION >= $(version_code 2.13.55) )); then
3137 wait_nm_sync default forbid_encryption '' inactive
3138 if (( MGS_VERSION >= $(version_code 2.15.51) )); then
3139 wait_nm_sync default readonly_mount '' inactive
3142 wait_nm_sync default trusted_nodemap '' inactive
3143 wait_nm_sync default admin_nodemap '' inactive
3146 mount_client $MOUNT ${MOUNT_OPTS} || error "re-mount failed"
3151 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3152 skip "client encryption not supported"
3154 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3155 skip "need dummy encryption support"
3157 stack_trap cleanup_for_enc_tests EXIT
3159 # first make sure it is possible to enable encryption
3160 # when nodemap is not active
3163 umount_client $MOUNT || error "umount $MOUNT failed (1)"
3165 # then activate nodemap, and retry
3166 # should succeed as encryption is not forbidden on default nodemap
3168 stack_trap cleanup_nodemap_after_enc_tests EXIT
3169 do_facet mgs $LCTL nodemap_activate 1
3171 forbid=$(do_facet mgs lctl get_param -n nodemap.default.forbid_encryption)
3172 [ $forbid -eq 0 ] || error "wrong default value for forbid_encryption"
3173 mount_client $MOUNT ${MOUNT_OPTS},test_dummy_encryption ||
3174 error "mount '-o test_dummy_encryption' failed with default"
3175 umount_client $MOUNT || error "umount $MOUNT failed (2)"
3177 # then forbid encryption, and retry
3178 do_facet mgs $LCTL nodemap_modify --name default \
3179 --property forbid_encryption --value 1
3180 wait_nm_sync default forbid_encryption
3181 mount_client $MOUNT ${MOUNT_OPTS},test_dummy_encryption &&
3182 error "mount '-o test_dummy_encryption' should have failed"
3185 run_test 36 "control if clients can use encryption"
3188 local testfile=$DIR/$tdir/$tfile
3189 local tmpfile=$TMP/abc
3190 local objdump=$TMP/objdump
3192 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3193 skip "client encryption not supported"
3195 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3196 skip "need dummy encryption support"
3198 [ "$ost1_FSTYPE" = ldiskfs ] || skip "ldiskfs only test (using debugfs)"
3200 stack_trap cleanup_for_enc_tests EXIT
3203 # write a few bytes in file
3204 echo "abc" > $tmpfile
3205 $LFS setstripe -c1 -i0 $testfile
3206 dd if=$tmpfile of=$testfile bs=4 count=1 conv=fsync
3207 do_facet ost1 "sync; sync"
3209 # check that content on ost is encrypted
3210 local fid=($($LFS getstripe $testfile | grep 0x))
3211 local seq=${fid[3]#0x}
3215 if [ $seq == 0 ]; then
3218 oid_hex=${fid[2]#0x}
3220 do_facet ost1 "$DEBUGFS -c -R 'cat O/$seq/d$(($oid % 32))/$oid_hex' \
3221 $(ostdevname 1)" > $objdump
3222 cmp -s $objdump $tmpfile &&
3223 error "file $testfile is not encrypted on ost"
3225 # check that in-memory representation of file is correct
3226 cmp -bl ${tmpfile} ${testfile} ||
3227 error "file $testfile is corrupted in memory"
3229 cancel_lru_locks osc ; cancel_lru_locks mdc
3231 # check that file read from server is correct
3232 cmp -bl ${tmpfile} ${testfile} ||
3233 error "file $testfile is corrupted on server"
3235 rm -f $tmpfile $objdump
3237 run_test 37 "simple encrypted file"
3240 local testfile=$DIR/$tdir/$tfile
3241 local tmpfile=$TMP/abc
3245 local pagesz=$(getconf PAGE_SIZE)
3247 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3248 skip "client encryption not supported"
3250 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3251 skip "need dummy encryption support"
3253 stack_trap cleanup_for_enc_tests EXIT
3256 # get block size on ost
3257 blksz=$($LCTL get_param osc.$FSNAME*.import |
3258 awk '/grant_block_size:/ { print $2; exit; }')
3259 # write a few bytes in file at offset $blksz
3260 echo "abc" > $tmpfile
3261 $LFS setstripe -c1 -i0 $testfile
3262 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$blksz \
3263 oflag=seek_bytes conv=fsync
3265 blksz=$(($blksz > $pagesz ? $blksz : $pagesz))
3266 # check that in-memory representation of file is correct
3267 bsize=$(stat --format=%B $testfile)
3268 filesz=$(stat --format=%b $testfile)
3269 filesz=$((filesz*bsize))
3270 [ $filesz -le $blksz ] ||
3271 error "file $testfile is $filesz long in memory"
3273 cancel_lru_locks osc ; cancel_lru_locks mdc
3275 # check that file read from server is correct
3276 bsize=$(stat --format=%B $testfile)
3277 filesz=$(stat --format=%b $testfile)
3278 filesz=$((filesz*bsize))
3279 [ $filesz -le $blksz ] ||
3280 error "file $testfile is $filesz long on server"
3284 run_test 38 "encrypted file with hole"
3287 local testfile=$DIR/$tdir/$tfile
3288 local tmpfile=$TMP/abc
3290 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3291 skip "client encryption not supported"
3293 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3294 skip "need dummy encryption support"
3296 stack_trap cleanup_for_enc_tests EXIT
3299 # write a few bytes in file
3300 echo "abc" > $tmpfile
3301 $LFS setstripe -c1 -i0 $testfile
3302 dd if=$tmpfile of=$testfile bs=4 count=1 conv=fsync
3304 # write a few more bytes in the same page
3305 dd if=$tmpfile of=$testfile bs=4 count=1 seek=1024 oflag=seek_bytes \
3308 dd if=$tmpfile of=$tmpfile bs=4 count=1 seek=1024 oflag=seek_bytes \
3311 # check that in-memory representation of file is correct
3312 cmp -bl $tmpfile $testfile ||
3313 error "file $testfile is corrupted in memory"
3315 cancel_lru_locks osc ; cancel_lru_locks mdc
3317 # check that file read from server is correct
3318 cmp -bl $tmpfile $testfile ||
3319 error "file $testfile is corrupted on server"
3323 run_test 39 "rewrite data in already encrypted page"
3326 local testfile=$DIR/$tdir/$tfile
3327 local tmpfile=$TMP/abc
3328 local tmpfile2=$TMP/abc2
3331 #define LUSTRE_ENCRYPTION_UNIT_SIZE (1 << 12)
3332 local UNIT_SIZE=$((1 << 12))
3335 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3336 skip "client encryption not supported"
3338 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3339 skip "need dummy encryption support"
3341 [[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
3343 stack_trap cleanup_for_enc_tests EXIT
3346 # write a few bytes in file
3347 echo "abc" > $tmpfile
3348 $LFS setstripe -c1 -i0 $testfile
3349 dd if=$tmpfile of=$testfile bs=4 count=1 conv=fsync
3351 # check that in-memory representation of file is correct
3352 cmp -bl $tmpfile $testfile ||
3353 error "file $testfile is corrupted in memory (1)"
3355 cancel_lru_locks osc ; cancel_lru_locks mdc
3357 # check that file read from server is correct
3358 cmp -bl $tmpfile $testfile ||
3359 error "file $testfile is corrupted on server (1)"
3361 # write a few other bytes in same page
3362 dd if=$tmpfile of=$testfile bs=4 count=1 seek=256 oflag=seek_bytes \
3365 dd if=$tmpfile of=$tmpfile bs=4 count=1 seek=256 oflag=seek_bytes \
3368 # check that in-memory representation of file is correct
3369 cmp -bl $tmpfile $testfile ||
3370 error "file $testfile is corrupted in memory (2)"
3372 cancel_lru_locks osc ; cancel_lru_locks mdc
3374 # check that file read from server is correct
3375 cmp -bl $tmpfile $testfile ||
3376 error "file $testfile is corrupted on server (2)"
3378 rm -f $testfile $tmpfile
3379 cancel_lru_locks osc ; cancel_lru_locks mdc
3381 # write a few bytes in file, at end of first page
3382 echo "abc" > $tmpfile
3383 $LFS setstripe -c1 -i0 $testfile
3384 seek=$(getconf PAGESIZE)
3386 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3389 # write a few other bytes at beginning of first page
3390 dd if=$tmpfile of=$testfile bs=4 count=1 conv=fsync,notrunc
3392 dd if=$tmpfile of=$tmpfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3395 # check that in-memory representation of file is correct
3396 cmp -bl $tmpfile $testfile ||
3397 error "file $testfile is corrupted in memory (3)"
3399 cancel_lru_locks osc ; cancel_lru_locks mdc
3401 # check that file read from server is correct
3402 cmp -bl $tmpfile $testfile ||
3403 error "file $testfile is corrupted on server (3)"
3405 rm -f $testfile $tmpfile
3406 cancel_lru_locks osc ; cancel_lru_locks mdc
3408 # write a few bytes in file, at beginning of second page
3409 echo "abc" > $tmpfile
3410 $LFS setstripe -c1 -i0 $testfile
3411 seek=$(getconf PAGESIZE)
3412 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3414 dd if=$tmpfile of=$tmpfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3417 # write a few other bytes at end of first page
3419 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3421 dd if=$tmpfile of=$tmpfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3424 # check that in-memory representation of file is correct
3425 cmp -bl $tmpfile2 $testfile ||
3426 error "file $testfile is corrupted in memory (4)"
3428 cancel_lru_locks osc ; cancel_lru_locks mdc
3430 # check that file read from server is correct
3431 cmp -bl $tmpfile2 $testfile ||
3432 error "file $testfile is corrupted on server (4)"
3434 rm -f $testfile $tmpfile $tmpfile2
3435 cancel_lru_locks osc ; cancel_lru_locks mdc
3437 # write a few bytes in file, at beginning of first stripe
3438 echo "abc" > $tmpfile
3439 $LFS setstripe -S 256k -c2 $testfile
3440 dd if=$tmpfile of=$testfile bs=4 count=1 conv=fsync,notrunc
3442 # write a few other bytes, at beginning of second stripe
3443 dd if=$tmpfile of=$testfile bs=4 count=1 seek=262144 oflag=seek_bytes \
3445 dd if=$tmpfile of=$tmpfile bs=4 count=1 seek=262144 oflag=seek_bytes \
3448 # check that in-memory representation of file is correct
3449 cmp -bl $tmpfile $testfile ||
3450 error "file $testfile is corrupted in memory (5)"
3452 cancel_lru_locks osc ; cancel_lru_locks mdc
3454 # check that file read from server is correct
3455 cmp -bl $tmpfile $testfile ||
3456 error "file $testfile is corrupted on server (5)"
3458 filesz=$(stat --format=%s $testfile)
3459 filesz=$(((filesz+UNIT_SIZE-1)/UNIT_SIZE * UNIT_SIZE))
3461 # remount without dummy encryption key
3462 remount_client_normally
3464 scrambledfile=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -type f)
3465 [ $(stat --format=%s $scrambledfile) -eq $filesz ] ||
3466 error "file size without key should be rounded up"
3470 run_test 40 "exercise size of encrypted file"
3473 local testfile=$DIR/$tdir/$tfile
3474 local tmpfile=$TMP/abc
3475 local tmpfile2=$TMP/abc2
3478 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3479 skip "client encryption not supported"
3481 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3482 skip "need dummy encryption support"
3484 stack_trap cleanup_for_enc_tests EXIT
3487 echo "abc" > $tmpfile
3488 seek=$(getconf PAGESIZE)
3489 seek=$((seek - 204))
3490 dd if=$tmpfile of=$tmpfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3492 seek=$(getconf PAGESIZE)
3493 seek=$((seek + 1092))
3494 dd if=$tmpfile of=$tmpfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3497 # write a few bytes in file
3498 $LFS setstripe -c1 -i0 -S 256k $testfile
3499 seek=$(getconf PAGESIZE)
3500 seek=$((seek - 204))
3501 #define OBD_FAIL_OST_WR_ATTR_DELAY 0x250
3502 do_facet ost1 "$LCTL set_param fail_loc=0x250 fail_val=15"
3503 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3507 # write a few other bytes, at a different offset
3508 seek=$(getconf PAGESIZE)
3509 seek=$((seek + 1092))
3510 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3511 conv=fsync,notrunc &
3513 do_facet ost1 "$LCTL set_param fail_loc=0x0"
3515 # check that in-memory representation of file is correct
3516 cmp -bl $tmpfile2 $testfile ||
3517 error "file $testfile is corrupted in memory (1)"
3519 cancel_lru_locks osc ; cancel_lru_locks mdc
3521 # check that file read from server is correct
3522 cmp -bl $tmpfile2 $testfile ||
3523 error "file $testfile is corrupted on server (1)"
3525 rm -f $tmpfile $tmpfile2
3527 run_test 41 "test race on encrypted file size (1)"
3530 local testfile=$DIR/$tdir/$tfile
3531 local testfile2=$DIR2/$tdir/$tfile
3532 local tmpfile=$TMP/abc
3533 local tmpfile2=$TMP/abc2
3534 local pagesz=$(getconf PAGESIZE)
3537 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3538 skip "client encryption not supported"
3540 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3541 skip "need dummy encryption support"
3543 stack_trap cleanup_for_enc_tests EXIT
3546 if is_mounted $MOUNT2; then
3547 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
3549 mount_client $MOUNT2 ${MOUNT_OPTS},test_dummy_encryption ||
3550 error "mount2 with '-o test_dummy_encryption' failed"
3552 # create file by writting one whole page
3553 $LFS setstripe -c1 -i0 -S 256k $testfile
3554 dd if=/dev/zero of=$testfile bs=$pagesz count=1 conv=fsync
3556 # read file from 2nd mount point
3557 cat $testfile2 > /dev/null
3559 echo "abc" > $tmpfile
3560 dd if=/dev/zero of=$tmpfile2 bs=$pagesz count=1 conv=fsync
3561 seek=$((2*pagesz - 204))
3562 dd if=$tmpfile of=$tmpfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3564 seek=$((2*pagesz + 1092))
3565 dd if=$tmpfile of=$tmpfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3568 # write a few bytes in file from 1st mount point
3569 seek=$((2*pagesz - 204))
3570 #define OBD_FAIL_OST_WR_ATTR_DELAY 0x250
3571 do_facet ost1 "$LCTL set_param fail_loc=0x250 fail_val=15"
3572 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3573 conv=fsync,notrunc &
3576 # write a few other bytes, at a different offset from 2nd mount point
3577 seek=$((2*pagesz + 1092))
3578 dd if=$tmpfile of=$testfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3579 conv=fsync,notrunc &
3581 do_facet ost1 "$LCTL set_param fail_loc=0x0"
3583 # check that in-memory representation of file is correct
3584 cmp -bl $tmpfile2 $testfile ||
3585 error "file $testfile is corrupted in memory (1)"
3587 # check that in-memory representation of file is correct
3588 cmp -bl $tmpfile2 $testfile2 ||
3589 error "file $testfile is corrupted in memory (2)"
3591 cancel_lru_locks osc ; cancel_lru_locks mdc
3593 # check that file read from server is correct
3594 cmp -bl $tmpfile2 $testfile ||
3595 error "file $testfile is corrupted on server (1)"
3597 rm -f $tmpfile $tmpfile2
3599 run_test 42 "test race on encrypted file size (2)"
3602 local testfile=$DIR/$tdir/$tfile
3603 local testfile2=$DIR2/$tdir/$tfile
3604 local tmpfile=$TMP/abc
3605 local tmpfile2=$TMP/abc2
3606 local resfile=$TMP/res
3607 local pagesz=$(getconf PAGESIZE)
3610 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3611 skip "client encryption not supported"
3613 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3614 skip "need dummy encryption support"
3616 stack_trap cleanup_for_enc_tests EXIT
3619 if is_mounted $MOUNT2; then
3620 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
3622 mount_client $MOUNT2 ${MOUNT_OPTS},test_dummy_encryption ||
3623 error "mount2 with '-o test_dummy_encryption' failed"
3626 tr '\0' '1' < /dev/zero |
3627 dd of=$tmpfile bs=1 count=$pagesz conv=fsync
3628 $LFS setstripe -c1 -i0 -S 256k $testfile
3629 cp $tmpfile $testfile
3631 # read file from 2nd mount point
3632 cat $testfile2 > /dev/null
3634 # write a few bytes in file from 1st mount point
3635 echo "abc" > $tmpfile2
3636 seek=$((2*pagesz - 204))
3637 #define OBD_FAIL_OST_WR_ATTR_DELAY 0x250
3638 do_facet ost1 "$LCTL set_param fail_loc=0x250 fail_val=15"
3639 dd if=$tmpfile2 of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3640 conv=fsync,notrunc &
3643 # read file from 2nd mount point
3644 dd if=$testfile2 of=$resfile bs=$pagesz count=1 conv=fsync,notrunc
3645 cmp -bl $tmpfile $resfile ||
3646 error "file $testfile is corrupted in memory (1)"
3649 do_facet ost1 "$LCTL set_param fail_loc=0x0"
3651 # check that in-memory representation of file is correct
3652 dd if=$tmpfile2 of=$tmpfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3654 cmp -bl $tmpfile $testfile2 ||
3655 error "file $testfile is corrupted in memory (2)"
3657 cancel_lru_locks osc ; cancel_lru_locks mdc
3659 # check that file read from server is correct
3660 cmp -bl $tmpfile $testfile ||
3661 error "file $testfile is corrupted on server (1)"
3663 rm -f $tmpfile $tmpfile2
3665 run_test 43 "test race on encrypted file size (3)"
3668 local testfile=$DIR/$tdir/$tfile
3669 local tmpfile=$TMP/abc
3670 local resfile=$TMP/resfile
3671 local pagesz=$(getconf PAGESIZE)
3674 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3675 skip "client encryption not supported"
3677 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3678 skip "need dummy encryption support"
3680 which vmtouch || skip "This test needs vmtouch utility"
3682 # Direct I/O is now supported on encrypted files.
3684 stack_trap cleanup_for_enc_tests EXIT
3687 $LFS setstripe -c1 -i0 $testfile
3688 dd if=/dev/urandom of=$tmpfile bs=$pagesz count=2 conv=fsync
3689 dd if=$tmpfile of=$testfile bs=$pagesz count=2 oflag=direct ||
3690 error "could not write to file with O_DIRECT (1)"
3692 respage=$(vmtouch $testfile | awk '/Resident Pages:/ {print $3}')
3693 [ "$respage" == "0/2" ] ||
3694 error "write to enc file fell back to buffered IO"
3698 dd if=$testfile of=$resfile bs=$pagesz count=2 iflag=direct ||
3699 error "could not read from file with O_DIRECT (1)"
3701 respage=$(vmtouch $testfile | awk '/Resident Pages:/ {print $3}')
3702 [ "$respage" == "0/2" ] ||
3703 error "read from enc file fell back to buffered IO"
3705 cmp -bl $tmpfile $resfile ||
3706 error "file $testfile is corrupted (1)"
3710 $TRUNCATE $tmpfile $pagesz
3711 dd if=$tmpfile of=$testfile bs=$pagesz count=1 seek=13 oflag=direct ||
3712 error "could not write to file with O_DIRECT (2)"
3716 dd if=$testfile of=$resfile bs=$pagesz count=1 skip=13 iflag=direct ||
3717 error "could not read from file with O_DIRECT (2)"
3718 cmp -bl $tmpfile $resfile ||
3719 error "file $testfile is corrupted (2)"
3721 rm -f $testfile $resfile
3722 $LFS setstripe -c1 -i0 $testfile
3724 $TRUNCATE $tmpfile $((pagesz/2 - 5))
3725 cp $tmpfile $testfile
3729 dd if=$testfile of=$resfile bs=$pagesz count=1 iflag=direct ||
3730 error "could not read from file with O_DIRECT (3)"
3731 cmp -bl $tmpfile $resfile ||
3732 error "file $testfile is corrupted (3)"
3734 rm -f $tmpfile $resfile $testfile
3736 if [ $OSTCOUNT -ge 2 ]; then
3737 dd if=/dev/urandom of=$tmpfile bs=$pagesz count=1 conv=fsync
3738 $LFS setstripe -S 256k -c2 $testfile
3740 # write in file, at beginning of first stripe, buffered IO
3741 dd if=$tmpfile of=$testfile bs=$pagesz count=1 \
3744 # write at beginning of second stripe, direct IO
3745 dd if=$tmpfile of=$testfile bs=$pagesz count=1 seek=256k \
3746 oflag=seek_bytes,direct conv=fsync,notrunc
3750 # read at beginning of first stripe, direct IO
3751 dd if=$testfile of=$resfile bs=$pagesz count=1 \
3752 iflag=direct conv=fsync
3754 cmp -bl $tmpfile $resfile ||
3755 error "file $testfile is corrupted (4)"
3757 # read at beginning of second stripe, buffered IO
3758 dd if=$testfile of=$resfile bs=$pagesz count=1 skip=256k \
3759 iflag=skip_bytes conv=fsync
3761 cmp -bl $tmpfile $resfile ||
3762 error "file $testfile is corrupted (5)"
3764 rm -f $tmpfile $resfile
3767 run_test 44 "encrypted file access semantics: direct IO"
3770 local testfile=$DIR/$tdir/$tfile
3771 local tmpfile=$TMP/junk
3773 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3774 skip "client encryption not supported"
3776 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3777 skip "need dummy encryption support"
3779 stack_trap cleanup_for_enc_tests EXIT
3782 $LFS setstripe -c1 -i0 $testfile
3783 dd if=/dev/zero of=$testfile bs=512K count=1
3784 $MULTIOP $testfile OSMRUc || error "$MULTIOP $testfile failed (1)"
3785 $MULTIOP $testfile OSMWUc || error "$MULTIOP $testfile failed (2)"
3787 dd if=/dev/zero of=$tmpfile bs=512K count=1
3788 $MULTIOP $tmpfile OSMWUc || error "$MULTIOP $tmpfile failed"
3789 $MMAP_CAT $tmpfile > ${tmpfile}2
3793 $MULTIOP $testfile OSMRUc
3794 $MMAP_CAT $testfile > ${testfile}2
3795 cmp -bl ${tmpfile}2 ${testfile}2 ||
3796 error "file $testfile is corrupted"
3798 rm -f $tmpfile ${tmpfile}2
3800 run_test 45 "encrypted file access semantics: MMAP"
3803 local testdir=$DIR/$tdir/mydir
3804 local testfile=$testdir/myfile
3805 local testdir2=$DIR/$tdir/mydirwithaveryverylongnametotestcodebehaviour0
3806 local testfile2=$testdir/myfilewithaveryverylongnametotestcodebehaviour0
3807 # testdir3, testfile3, testhl3 and testsl3 names are 255 bytes long
3808 local testdir3=$testdir2/dir_abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz012345678
3809 local testfile3=$testdir2/file_abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01234567
3810 local testhl3=$testdir2/hl_abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789
3811 local testsl3=$testdir2/sl_abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789
3812 local lsfile=$TMP/lsfile
3817 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3818 skip "client encryption not supported"
3820 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3821 skip "need dummy encryption support"
3823 stack_trap cleanup_for_enc_tests EXIT
3826 touch $DIR/$tdir/$tfile
3828 echo test > $testfile
3829 echo othertest > $testfile2
3830 if [[ $MDSCOUNT -gt 1 ]]; then
3831 $LFS setdirstripe -c1 -i1 $testdir2
3835 inum=$(stat -c %i $testdir2)
3836 if [ "$mds1_FSTYPE" = ldiskfs ]; then
3837 # For now, restrict this part of the test to ldiskfs backend,
3838 # as osd-zfs does not support 255 byte-long encrypted names.
3839 mkdir $testdir3 || error "cannot mkdir $testdir3"
3840 touch $testfile3 || error "cannot touch $testfile3"
3841 ln $testfile3 $testhl3 || error "cannot ln $testhl3"
3842 ln -s $testfile3 $testsl3 || error "cannot ln $testsl3"
3844 sync ; echo 3 > /proc/sys/vm/drop_caches
3846 # remount without dummy encryption key
3847 remount_client_normally
3850 scrambleddir=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -inum $inum)
3851 stat $scrambleddir || error "stat $scrambleddir failed"
3852 if [ "$mds1_FSTYPE" = ldiskfs ]; then
3853 stat $scrambleddir/* || error "cannot stat in $scrambleddir"
3854 rm -rf $scrambleddir/* || error "cannot clean in $scrambleddir"
3856 rmdir $scrambleddir || error "rmdir $scrambleddir failed"
3858 scrambleddir=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -type d)
3859 ls -1 $scrambleddir > $lsfile || error "ls $testdir failed (1)"
3861 scrambledfile=$scrambleddir/$(head -n 1 $lsfile)
3862 stat $scrambledfile || error "stat $scrambledfile failed (1)"
3865 cat $scrambledfile && error "cat $scrambledfile should have failed (1)"
3866 rm -f $scrambledfile || error "rm $scrambledfile failed (1)"
3868 ls -1 $scrambleddir > $lsfile || error "ls $testdir failed (2)"
3869 scrambledfile=$scrambleddir/$(head -n 1 $lsfile)
3870 stat $scrambledfile || error "stat $scrambledfile failed (2)"
3872 cat $scrambledfile && error "cat $scrambledfile should have failed (2)"
3874 touch $scrambleddir/otherfile &&
3875 error "touch otherfile should have failed"
3876 ls $scrambleddir/otherfile && error "otherfile should not exist"
3877 mkdir $scrambleddir/otherdir &&
3878 error "mkdir otherdir should have failed"
3879 ls -d $scrambleddir/otherdir && error "otherdir should not exist"
3882 rm -f $scrambledfile || error "rm $scrambledfile failed (2)"
3883 rmdir $scrambleddir || error "rmdir $scrambleddir failed"
3886 run_test 46 "encrypted file access semantics without key"
3889 local testfile=$DIR/$tdir/$tfile
3890 local testfile2=$DIR/$tdir/${tfile}.2
3891 local tmpfile=$DIR/junk
3896 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3897 skip "client encryption not supported"
3899 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3900 skip "need dummy encryption support"
3902 $LCTL get_param mdc.*.connect_flags | grep -q name_encryption ||
3905 stack_trap cleanup_for_enc_tests EXIT
3908 dd if=/dev/urandom of=$tmpfile bs=512K count=1
3909 mrename $tmpfile $testfile &&
3910 error "rename from unencrypted to encrypted dir should fail"
3912 ln $tmpfile $testfile &&
3913 error "link from encrypted to unencrypted dir should fail"
3915 cp $tmpfile $testfile ||
3916 error "cp from unencrypted to encrypted dir should succeed"
3919 mrename $testfile $testfile2 ||
3920 error "rename from within encrypted dir should succeed"
3922 ln $testfile2 $testfile ||
3923 error "link from within encrypted dir should succeed"
3924 cmp -bl $testfile2 $testfile ||
3925 error "cannot read from hard link (1.1)"
3926 echo a >> $testfile || error "cannot write to hard link (1)"
3928 cmp -bl $testfile2 $testfile ||
3929 error "cannot read from hard link (1.2)"
3932 ln $testfile2 $tmpfile ||
3933 error "link from unencrypted to encrypted dir should succeed"
3935 cmp -bl $testfile2 $tmpfile ||
3936 error "cannot read from hard link (2.1)"
3937 echo a >> $tmpfile || error "cannot write to hard link (2)"
3939 cmp -bl $testfile2 $tmpfile ||
3940 error "cannot read from hard link (2.2)"
3943 if [ $name_enc -eq 1 ]; then
3944 # check we are limited in the number of hard links
3945 # we can create for encrypted files, to what can fit into LinkEA
3946 for i in $(seq 1 160); do
3947 ln $testfile2 ${testfile}_$i || break
3949 [ $i -lt 160 ] || error "hard link $i should fail"
3953 mrename $testfile2 $tmpfile &&
3954 error "rename from encrypted to unencrypted dir should fail"
3956 dd if=/dev/urandom of=$tmpfile bs=512K count=1
3958 dd if=/dev/urandom of=$testfile bs=512K count=1
3959 mkdir $DIR/$tdir/mydir
3961 ln -s $testfile ${testfile}.sym ||
3962 error "symlink from within encrypted dir should succeed"
3964 cmp -bl $testfile ${testfile}.sym ||
3965 error "cannot read from sym link (1.1)"
3966 echo a >> ${testfile}.sym || error "cannot write to sym link (1)"
3968 cmp -bl $testfile ${testfile}.sym ||
3969 error "cannot read from sym link (1.2)"
3970 [ $(stat -c %s ${testfile}.sym) -eq ${#testfile} ] ||
3971 error "wrong symlink size (1)"
3973 ln -s $tmpfile ${testfile}.sl ||
3974 error "symlink from encrypted to unencrypted dir should succeed"
3976 cmp -bl $tmpfile ${testfile}.sl ||
3977 error "cannot read from sym link (2.1)"
3978 echo a >> ${testfile}.sl || error "cannot write to sym link (2)"
3980 cmp -bl $tmpfile ${testfile}.sl ||
3981 error "cannot read from sym link (2.2)"
3982 [ $(stat -c %s ${testfile}.sl) -eq ${#tmpfile} ] ||
3983 error "wrong symlink size (2)"
3984 rm -f ${testfile}.sl
3986 sync ; echo 3 > /proc/sys/vm/drop_caches
3988 # remount without dummy encryption key
3989 remount_client_normally
3991 scrambleddir=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -type d)
3992 scrambledfile=$(find $DIR/$tdir/ -maxdepth 1 -type f)
3993 scrambledlink=$(find $DIR/$tdir/ -maxdepth 1 -type l)
3994 ln $scrambledfile $scrambleddir/linkfile &&
3995 error "ln linkfile should have failed"
3996 mrename $scrambledfile $DIR/onefile2 &&
3997 error "mrename from $scrambledfile should have failed"
3999 mrename $DIR/onefile $scrambleddir/otherfile &&
4000 error "mrename to $scrambleddir should have failed"
4001 readlink $scrambledlink ||
4002 error "link should be read without key"
4003 [ $(stat -c %s $scrambledlink) -eq \
4004 $(expr length "$(readlink $scrambledlink)") ] ||
4005 error "wrong symlink size without key"
4006 if [ $name_enc -eq 1 ]; then
4007 readlink -e $scrambledlink &&
4008 error "link should not point to anywhere useful"
4010 ln -s $scrambledfile ${scrambledfile}.sym &&
4011 error "symlink without key should fail (1)"
4012 ln -s $tmpfile ${scrambledfile}.sl &&
4013 error "symlink without key should fail (2)"
4015 rm -f $tmpfile $DIR/onefile
4017 run_test 47 "encrypted file access semantics: rename/link"
4020 local save="$TMP/$TESTSUITE-$TESTNAME.parameters"
4021 local testfile=$DIR/$tdir/$tfile
4022 local tmpfile=$TMP/111
4023 local tmpfile2=$TMP/abc
4024 local pagesz=$(getconf PAGESIZE)
4029 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4030 skip "client encryption not supported"
4032 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4033 skip "need dummy encryption support"
4035 stack_trap cleanup_for_enc_tests EXIT
4038 # create file, 4 x PAGE_SIZE long
4039 tr '\0' '1' < /dev/zero |
4040 dd of=$tmpfile bs=1 count=4x$pagesz conv=fsync
4041 $LFS setstripe -c1 -i0 $testfile
4042 cp $tmpfile $testfile
4043 echo "abc" > $tmpfile2
4045 # decrease size: truncate to PAGE_SIZE
4046 $TRUNCATE $tmpfile $pagesz
4047 $TRUNCATE $testfile $pagesz
4048 cancel_lru_locks osc ; cancel_lru_locks mdc
4049 cmp -bl $tmpfile $testfile ||
4050 error "file $testfile is corrupted (1)"
4052 # increase size: truncate to 2 x PAGE_SIZE
4054 $TRUNCATE $tmpfile $sz
4055 $TRUNCATE $testfile $sz
4056 cancel_lru_locks osc ; cancel_lru_locks mdc
4057 cmp -bl $tmpfile $testfile ||
4058 error "file $testfile is corrupted (2)"
4061 seek=$((pagesz+100))
4062 dd if=$tmpfile2 of=$tmpfile bs=4 count=1 seek=$seek oflag=seek_bytes \
4064 dd if=$tmpfile2 of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
4066 cancel_lru_locks osc ; cancel_lru_locks mdc
4067 cmp -bl $tmpfile $testfile ||
4068 error "file $testfile is corrupted (3)"
4070 # truncate to PAGE_SIZE / 2
4072 $TRUNCATE $tmpfile $sz
4073 $TRUNCATE $testfile $sz
4074 cancel_lru_locks osc ; cancel_lru_locks mdc
4075 cmp -bl $tmpfile $testfile ||
4076 error "file $testfile is corrupted (4)"
4078 # truncate to a smaller, non-multiple of PAGE_SIZE, non-multiple of 16
4080 $TRUNCATE $tmpfile $sz
4081 $TRUNCATE $testfile $sz
4082 cancel_lru_locks osc ; cancel_lru_locks mdc
4083 cmp -bl $tmpfile $testfile ||
4084 error "file $testfile is corrupted (5)"
4086 # truncate to a larger, non-multiple of PAGE_SIZE, non-multiple of 16
4088 $TRUNCATE $tmpfile $sz
4089 $TRUNCATE $testfile $sz
4090 cancel_lru_locks osc ; cancel_lru_locks mdc
4091 cmp -bl $tmpfile $testfile ||
4092 error "file $testfile is corrupted (6)"
4094 # truncate to a larger, non-multiple of PAGE_SIZE, in a different page
4095 sz=$((sz+pagesz+30))
4096 $TRUNCATE $tmpfile $sz
4097 $TRUNCATE $testfile $sz
4098 cancel_lru_locks osc ; cancel_lru_locks mdc
4099 cmp -bl $tmpfile $testfile ||
4100 error "file $testfile is corrupted (7)"
4102 sync ; echo 3 > /proc/sys/vm/drop_caches
4104 # remount without dummy encryption key
4105 remount_client_normally
4107 scrambledfile=$(find $DIR/$tdir/ -maxdepth 1 -type f)
4108 $TRUNCATE $scrambledfile 0 &&
4109 error "truncate $scrambledfile should have failed without key"
4111 rm -f $tmpfile $tmpfile2
4113 run_test 48a "encrypted file access semantics: truncate"
4115 cleanup_for_enc_tests_othercli() {
4118 # remount othercli normally
4119 zconf_umount $othercli $MOUNT ||
4120 error "umount $othercli $MOUNT failed"
4121 zconf_mount $othercli $MOUNT ||
4122 error "remount $othercli $MOUNT failed"
4128 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4129 skip "client encryption not supported"
4131 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4132 skip "need dummy encryption support"
4134 [ "$num_clients" -ge 2 ] || skip "Need at least 2 clients"
4136 if [ "$HOSTNAME" == ${clients_arr[0]} ]; then
4137 othercli=${clients_arr[1]}
4139 othercli=${clients_arr[0]}
4142 stack_trap cleanup_for_enc_tests EXIT
4143 stack_trap "cleanup_for_enc_tests_othercli $othercli" EXIT
4145 zconf_umount $othercli $MOUNT ||
4146 error "umount $othercli $MOUNT failed"
4148 cp /bin/sleep $DIR/$tdir/
4149 cancel_lru_locks osc ; cancel_lru_locks mdc
4150 $DIR/$tdir/sleep 30 &
4151 # mount and IOs must be done in the same shell session, otherwise
4152 # encryption key in session keyring is missing
4153 do_node $othercli "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
4154 $MGSNID:/$FSNAME $MOUNT && \
4155 $TRUNCATE $DIR/$tdir/sleep 7"
4156 wait || error "wait error"
4157 cmp --silent /bin/sleep $DIR/$tdir/sleep ||
4158 error "/bin/sleep and $DIR/$tdir/sleep differ"
4160 run_test 48b "encrypted file: concurrent truncate"
4166 $LCTL set_param debug=+info
4171 [ $? -eq 0 ] || error "$cmd failed"
4173 if [ -z "$MATCHING_STRING" ]; then
4174 $LCTL dk | grep -E "get xattr 'encryption.c'|get xattrs"
4176 $LCTL dk | grep -E "$MATCHING_STRING"
4178 [ $? -ne 0 ] || error "get xattr event was triggered"
4182 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4183 skip "client encryption not supported"
4185 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4186 skip "need dummy encryption support"
4188 stack_trap cleanup_for_enc_tests EXIT
4191 local dirname=$DIR/$tdir/subdir
4195 trace_cmd stat $dirname
4196 trace_cmd echo a > $dirname/f1
4197 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
4198 trace_cmd stat $dirname/f1
4199 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
4200 trace_cmd cat $dirname/f1
4201 dd if=/dev/zero of=$dirname/f1 bs=1M count=10 conv=fsync
4202 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
4203 MATCHING_STRING="get xattr 'encryption.c'" \
4204 trace_cmd $TRUNCATE $dirname/f1 10240
4205 trace_cmd $LFS setstripe -E -1 -S 4M $dirname/f2
4206 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
4207 trace_cmd $LFS migrate -E -1 -S 256K $dirname/f2
4209 if [[ $MDSCOUNT -gt 1 ]]; then
4210 trace_cmd $LFS setdirstripe -i 1 $dirname/d2
4211 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
4212 trace_cmd $LFS migrate -m 0 $dirname/d2
4213 echo b > $dirname/d2/subf
4214 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
4215 if (( "$MDS1_VERSION" > $(version_code 2.14.54.54) )); then
4216 # migrate a non-empty encrypted dir
4217 trace_cmd $LFS migrate -m 1 $dirname/d2
4218 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
4219 [ -f $dirname/d2/subf ] || error "migrate failed (1)"
4220 [ $(cat $dirname/d2/subf) == "b" ] ||
4221 error "migrate failed (2)"
4224 $LFS setdirstripe -i 1 -c 1 $dirname/d3
4225 dirname=$dirname/d3/subdir
4227 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
4228 trace_cmd stat $dirname
4229 trace_cmd echo c > $dirname/f1
4230 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
4231 trace_cmd stat $dirname/f1
4232 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
4233 trace_cmd cat $dirname/f1
4234 dd if=/dev/zero of=$dirname/f1 bs=1M count=10 conv=fsync
4235 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
4236 MATCHING_STRING="get xattr 'encryption.c'" \
4237 trace_cmd $TRUNCATE $dirname/f1 10240
4238 trace_cmd $LFS setstripe -E -1 -S 4M $dirname/f2
4239 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
4240 trace_cmd $LFS migrate -E -1 -S 256K $dirname/f2
4242 skip_noexit "2nd part needs >= 2 MDTs"
4245 run_test 49 "Avoid getxattr for encryption context"
4248 local testfile=$DIR/$tdir/$tfile
4249 local tmpfile=$TMP/abc
4250 local pagesz=$(getconf PAGESIZE)
4253 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4254 skip "client encryption not supported"
4256 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4257 skip "need dummy encryption support"
4259 stack_trap cleanup_for_enc_tests EXIT
4262 # write small file, data on MDT only
4263 tr '\0' '1' < /dev/zero |
4264 dd of=$tmpfile bs=1 count=5000 conv=fsync
4265 $LFS setstripe -E 1M -L mdt -E EOF $testfile
4266 cp $tmpfile $testfile
4268 # check that in-memory representation of file is correct
4269 cmp -bl $tmpfile $testfile ||
4270 error "file $testfile is corrupted in memory"
4272 remove_enc_key ; insert_enc_key
4274 # check that file read from server is correct
4275 cmp -bl $tmpfile $testfile ||
4276 error "file $testfile is corrupted on server"
4278 # decrease size: truncate to PAGE_SIZE
4279 $TRUNCATE $tmpfile $pagesz
4280 $TRUNCATE $testfile $pagesz
4281 remove_enc_key ; insert_enc_key
4282 cmp -bl $tmpfile $testfile ||
4283 error "file $testfile is corrupted (1)"
4285 # increase size: truncate to 2 x PAGE_SIZE
4287 $TRUNCATE $tmpfile $sz
4288 $TRUNCATE $testfile $sz
4289 remove_enc_key ; insert_enc_key
4290 cmp -bl $tmpfile $testfile ||
4291 error "file $testfile is corrupted (2)"
4293 # truncate to PAGE_SIZE / 2
4295 $TRUNCATE $tmpfile $sz
4296 $TRUNCATE $testfile $sz
4297 remove_enc_key ; insert_enc_key
4298 cmp -bl $tmpfile $testfile ||
4299 error "file $testfile is corrupted (3)"
4301 # truncate to a smaller, non-multiple of PAGE_SIZE, non-multiple of 16
4303 $TRUNCATE $tmpfile $sz
4304 $TRUNCATE $testfile $sz
4305 remove_enc_key ; insert_enc_key
4306 cmp -bl $tmpfile $testfile ||
4307 error "file $testfile is corrupted (4)"
4309 # truncate to a larger, non-multiple of PAGE_SIZE, non-multiple of 16
4311 $TRUNCATE $tmpfile $sz
4312 $TRUNCATE $testfile $sz
4313 remove_enc_key ; insert_enc_key
4314 cmp -bl $tmpfile $testfile ||
4315 error "file $testfile is corrupted (5)"
4317 # truncate to a larger, non-multiple of PAGE_SIZE, in a different page
4318 sz=$((sz+pagesz+30))
4319 $TRUNCATE $tmpfile $sz
4320 $TRUNCATE $testfile $sz
4321 remove_enc_key ; insert_enc_key
4322 cmp -bl $tmpfile $testfile ||
4323 error "file $testfile is corrupted (6)"
4326 remove_enc_key ; insert_enc_key
4328 # write hole in file, data spread on MDT and OST
4329 tr '\0' '2' < /dev/zero |
4330 dd of=$tmpfile bs=1 count=1539 seek=1539074 conv=fsync,notrunc
4331 $LFS setstripe -E 1M -L mdt -E EOF $testfile
4332 cp --sparse=always $tmpfile $testfile
4334 # check that in-memory representation of file is correct
4335 cmp -bl $tmpfile $testfile ||
4336 error "file $testfile is corrupted in memory"
4338 remove_enc_key ; insert_enc_key
4340 # check that file read from server is correct
4341 cmp -bl $tmpfile $testfile ||
4342 error "file $testfile is corrupted on server"
4344 # truncate to a smaller, non-multiple of PAGE_SIZE, non-multiple of 16,
4345 # inside OST part of data
4346 sz=$((1024*1024+13))
4347 $TRUNCATE $tmpfile $sz
4348 $TRUNCATE $testfile $sz
4349 remove_enc_key ; insert_enc_key
4350 cmp -bl $tmpfile $testfile ||
4351 error "file $testfile is corrupted (7)"
4353 # truncate to a smaller, non-multiple of PAGE_SIZE, non-multiple of 16,
4354 # inside MDT part of data
4356 $TRUNCATE $tmpfile $sz
4357 $TRUNCATE $testfile $sz
4358 remove_enc_key ; insert_enc_key
4359 cmp -bl $tmpfile $testfile ||
4360 error "file $testfile is corrupted (8)"
4362 # truncate to a larger, non-multiple of PAGE_SIZE, non-multiple of 16,
4363 # inside MDT part of data
4364 sz=$((1024*1024-13))
4365 $TRUNCATE $tmpfile $sz
4366 $TRUNCATE $testfile $sz
4367 remove_enc_key ; insert_enc_key
4368 cmp -bl $tmpfile $testfile ||
4369 error "file $testfile is corrupted (9)"
4371 # truncate to a larger, non-multiple of PAGE_SIZE, non-multiple of 16,
4372 # inside OST part of data
4374 $TRUNCATE $tmpfile $sz
4375 $TRUNCATE $testfile $sz
4376 remove_enc_key ; insert_enc_key
4377 cmp -bl $tmpfile $testfile ||
4378 error "file $testfile is corrupted (10)"
4382 run_test 50 "DoM encrypted file"
4385 (( "$MDS1_VERSION" >= $(version_code v2_13_55-38-gf05edf8e2b) )) ||
4386 skip "Need MDS version at least 2.13.55.38"
4388 mkdir $DIR/$tdir || error "mkdir $tdir"
4389 local mdts=$(comma_list $(mdts_nodes))
4390 local cap_param=mdt.*.enable_cap_mask
4392 old_cap=($(do_nodes $mdts $LCTL get_param -n $cap_param 2>/dev/null))
4393 if [[ -n "$old_cap" ]]; then
4394 do_nodes $mdts $LCTL set_param $cap_param=0xf
4395 stack_trap "do_nodes $mdts $LCTL set_param $cap_param=$old_cap"
4398 touch $DIR/$tdir/$tfile || error "touch $tfile"
4399 cp $(which chown) $DIR/$tdir || error "cp chown"
4400 $RUNAS_CMD -u $ID0 $DIR/$tdir/chown $ID0 $DIR/$tdir/$tfile &&
4401 error "chown $tfile should fail"
4402 setcap 'CAP_CHOWN=ep' $DIR/$tdir/chown || error "setcap CAP_CHOWN"
4403 $RUNAS_CMD -u $ID0 $DIR/$tdir/chown $ID0 $DIR/$tdir/$tfile ||
4404 error "chown $tfile"
4405 rm $DIR/$tdir/$tfile || error "rm $tfile"
4407 touch $DIR/$tdir/$tfile || error "touch $tfile"
4408 cp $(which touch) $DIR/$tdir || error "cp touch"
4409 $RUNAS_CMD -u $ID0 $DIR/$tdir/touch $DIR/$tdir/$tfile &&
4410 error "touch should fail"
4411 setcap 'CAP_FOWNER=ep' $DIR/$tdir/touch || error "setcap CAP_FOWNER"
4412 $RUNAS_CMD -u $ID0 $DIR/$tdir/touch $DIR/$tdir/$tfile ||
4413 error "touch $tfile"
4414 rm $DIR/$tdir/$tfile || error "rm $tfile"
4417 for cap in "CAP_DAC_OVERRIDE" "CAP_DAC_READ_SEARCH"; do
4418 touch $DIR/$tdir/$tfile || error "touch $tfile"
4419 chmod 600 $DIR/$tdir/$tfile || error "chmod $tfile"
4420 cp $(which cat) $DIR/$tdir || error "cp cat"
4421 $RUNAS_CMD -u $ID0 $DIR/$tdir/cat $DIR/$tdir/$tfile &&
4422 error "cat should fail"
4423 setcap $cap=ep $DIR/$tdir/cat || error "setcap $cap"
4424 $RUNAS_CMD -u $ID0 $DIR/$tdir/cat $DIR/$tdir/$tfile ||
4426 rm $DIR/$tdir/$tfile || error "rm $tfile"
4429 run_test 51 "FS capabilities ==============="
4432 local testfile=$DIR/$tdir/$tfile
4433 local tmpfile=$TMP/$tfile
4434 local mirror1=$TMP/$tfile.mirror1
4435 local mirror2=$TMP/$tfile.mirror2
4437 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4438 skip "client encryption not supported"
4440 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4441 skip "need dummy encryption support"
4443 [[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
4445 stack_trap "cleanup_for_enc_tests $tmpfile $mirror1 $mirror2" EXIT
4448 dd if=/dev/urandom of=$tmpfile bs=5000 count=1 conv=fsync
4450 $LFS mirror create -N -i0 -N -i1 $testfile ||
4451 error "could not create mirror"
4453 dd if=$tmpfile of=$testfile bs=5000 count=1 conv=fsync ||
4454 error "could not write to $testfile"
4456 $LFS mirror resync $testfile ||
4457 error "could not resync mirror"
4459 $LFS mirror verify -v $testfile ||
4460 error "verify mirror failed"
4462 $LFS mirror read -N 1 -o $mirror1 $testfile ||
4463 error "could not read from mirror 1"
4465 cmp -bl $tmpfile $mirror1 ||
4466 error "mirror 1 is corrupted"
4468 $LFS mirror read -N 2 -o $mirror2 $testfile ||
4469 error "could not read from mirror 2"
4471 cmp -bl $tmpfile $mirror2 ||
4472 error "mirror 2 is corrupted"
4474 tr '\0' '2' < /dev/zero |
4475 dd of=$tmpfile bs=1 count=9000 conv=fsync
4477 $LFS mirror write -N 1 -i $tmpfile $testfile ||
4478 error "could not write to mirror 1"
4480 $LFS mirror verify -v $testfile &&
4481 error "mirrors should be different"
4483 rm -f $testfile $mirror1 $mirror2
4485 $LFS setstripe -c1 -i0 $testfile
4486 dd if=$tmpfile of=$testfile bs=9000 count=1 conv=fsync ||
4487 error "write to $testfile failed"
4488 $LFS getstripe $testfile
4491 $LFS migrate -i1 $testfile ||
4492 error "migrate $testfile failed"
4493 $LFS getstripe $testfile
4494 stripe=$($LFS getstripe -i $testfile)
4495 [ $stripe -eq 1 ] || error "migrate file $testfile failed"
4498 cmp -bl $tmpfile $testfile ||
4499 error "migrated file is corrupted"
4501 $LFS mirror extend -N -i0 $testfile ||
4502 error "mirror extend $testfile failed"
4503 $LFS getstripe $testfile
4504 mirror_count=$($LFS getstripe -N $testfile)
4505 [ $mirror_count -eq 2 ] ||
4506 error "mirror extend file $testfile failed (1)"
4507 stripe=$($LFS getstripe --mirror-id=1 -i $testfile)
4508 [ $stripe -eq 1 ] || error "mirror extend file $testfile failed (2)"
4509 stripe=$($LFS getstripe --mirror-id=2 -i $testfile)
4510 [ $stripe -eq 0 ] || error "mirror extend file $testfile failed (3)"
4513 $LFS mirror verify -v $testfile ||
4514 error "mirror verify failed"
4515 $LFS mirror read -N 1 -o $mirror1 $testfile ||
4516 error "read from mirror 1 failed"
4517 cmp -bl $tmpfile $mirror1 ||
4518 error "corruption of mirror 1"
4519 $LFS mirror read -N 2 -o $mirror2 $testfile ||
4520 error "read from mirror 2 failed"
4521 cmp -bl $tmpfile $mirror2 ||
4522 error "corruption of mirror 2"
4524 $LFS mirror split --mirror-id 1 -f ${testfile}.mirror $testfile &&
4525 error "mirror split -f should fail"
4527 $LFS mirror split --mirror-id 1 $testfile &&
4528 error "mirror split without -d should fail"
4530 $LFS mirror split --mirror-id 1 -d $testfile ||
4531 error "mirror split failed"
4532 $LFS getstripe $testfile
4533 mirror_count=$($LFS getstripe -N $testfile)
4534 [ $mirror_count -eq 1 ] ||
4535 error "mirror split file $testfile failed (1)"
4536 stripe=$($LFS getstripe --mirror-id=1 -i $testfile)
4537 [ -z "$stripe" ] || error "mirror extend file $testfile failed (2)"
4538 stripe=$($LFS getstripe --mirror-id=2 -i $testfile)
4539 [ $stripe -eq 0 ] || error "mirror extend file $testfile failed (3)"
4542 cmp -bl $tmpfile $testfile ||
4543 error "extended/split file is corrupted"
4545 run_test 52 "Mirrored encrypted file"
4548 local testfile=$DIR/$tdir/$tfile
4549 local testfile2=$DIR2/$tdir/$tfile
4550 local tmpfile=$TMP/$tfile.tmp
4551 local resfile=$TMP/$tfile.res
4555 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4556 skip "client encryption not supported"
4558 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4559 skip "need dummy encryption support"
4561 pagesz=$(getconf PAGESIZE)
4562 [[ $pagesz == 65536 ]] || skip "Need 64K PAGE_SIZE client"
4564 do_node $mds1_HOST \
4565 "mount.lustre --help |& grep -q 'test_dummy_encryption:'" ||
4566 skip "need dummy encryption support on MDS client mount"
4568 # this test is probably useless now, but may turn out to be useful when
4569 # Lustre supports servers with PAGE_SIZE != 4KB
4570 pagesz=$(do_node $mds1_HOST getconf PAGESIZE)
4571 [[ $pagesz == 4096 ]] || skip "Need 4K PAGE_SIZE MDS client"
4573 stack_trap cleanup_for_enc_tests EXIT
4574 stack_trap "zconf_umount $mds1_HOST $MOUNT2" EXIT
4577 $LFS setstripe -c1 -i0 $testfile
4579 # write from 1st client
4580 cat /dev/urandom | tr -dc 'a-zA-Z0-9' |
4581 dd of=$tmpfile bs=$((pagesz+3)) count=2 conv=fsync
4582 dd if=$tmpfile of=$testfile bs=$((pagesz+3)) count=2 conv=fsync ||
4583 error "could not write to $testfile (1)"
4585 # read from 2nd client
4586 # mount and IOs must be done in the same shell session, otherwise
4587 # encryption key in session keyring is missing
4588 do_node $mds1_HOST "mkdir -p $MOUNT2"
4589 do_node $mds1_HOST \
4590 "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
4591 $MGSNID:/$FSNAME $MOUNT2 && \
4592 dd if=$testfile2 of=$resfile bs=$((pagesz+3)) count=2" ||
4593 error "could not read from $testfile2 (1)"
4596 filemd5=$(do_node $mds1_HOST md5sum $resfile | awk '{print $1}')
4597 [ $filemd5 = $(md5sum $tmpfile | awk '{print $1}') ] ||
4598 error "file is corrupted (1)"
4599 do_node $mds1_HOST rm -f $resfile
4602 # truncate from 2nd client
4603 $TRUNCATE $tmpfile $((pagesz+3))
4604 zconf_umount $mds1_HOST $MOUNT2 ||
4605 error "umount $mds1_HOST $MOUNT2 failed (1)"
4606 do_node $mds1_HOST "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
4607 $MGSNID:/$FSNAME $MOUNT2 && \
4608 $TRUNCATE $testfile2 $((pagesz+3))" ||
4609 error "could not truncate $testfile2 (1)"
4612 cmp -bl $tmpfile $testfile ||
4613 error "file is corrupted (2)"
4614 rm -f $tmpfile $testfile
4616 zconf_umount $mds1_HOST $MOUNT2 ||
4617 error "umount $mds1_HOST $MOUNT2 failed (2)"
4620 do_node $mds1_HOST \
4621 dd if=/dev/urandom of=$tmpfile bs=$((pagesz+3)) count=2 conv=fsync
4622 # write from 2nd client
4623 do_node $mds1_HOST \
4624 "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
4625 $MGSNID:/$FSNAME $MOUNT2 && \
4626 dd if=$tmpfile of=$testfile2 bs=$((pagesz+3)) count=2 conv=fsync" ||
4627 error "could not write to $testfile2 (2)"
4629 # read from 1st client
4630 dd if=$testfile of=$resfile bs=$((pagesz+3)) count=2 ||
4631 error "could not read from $testfile (2)"
4634 filemd5=$(do_node $mds1_HOST md5sum -b $tmpfile | awk '{print $1}')
4635 [ $filemd5 = $(md5sum -b $resfile | awk '{print $1}') ] ||
4636 error "file is corrupted (3)"
4640 # truncate from 1st client
4641 do_node $mds1_HOST "$TRUNCATE $tmpfile $((pagesz+3))"
4642 $TRUNCATE $testfile $((pagesz+3)) ||
4643 error "could not truncate $testfile (2)"
4646 zconf_umount $mds1_HOST $MOUNT2 ||
4647 error "umount $mds1_HOST $MOUNT2 failed (3)"
4648 do_node $mds1_HOST "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
4649 $MGSNID:/$FSNAME $MOUNT2 && \
4650 cmp -bl $tmpfile $testfile2" ||
4651 error "file is corrupted (4)"
4653 do_node $mds1_HOST rm -f $tmpfile
4656 run_test 53 "Mixed PAGE_SIZE clients"
4659 local testdir=$DIR/$tdir/$ID0
4660 local testdir2=$DIR2/$tdir/$ID0
4661 local testfile=$testdir/$tfile
4662 local testfile2=$testdir/${tfile}withveryverylongnametoexercisecode
4663 local testfile3=$testdir/_${tfile}
4664 local tmpfile=$TMP/${tfile}.tmp
4665 local resfile=$TMP/${tfile}.res
4670 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4671 skip "client encryption not supported"
4673 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4674 skip "need dummy encryption support"
4676 which fscrypt || skip "This test needs fscrypt userspace tool"
4678 yes | fscrypt setup --force --verbose ||
4679 error "fscrypt global setup failed"
4680 sed -i 's/\(.*\)policy_version\(.*\):\(.*\)\"[0-9]*\"\(.*\)/\1policy_version\2:\3"2"\4/' \
4682 yes | fscrypt setup --verbose $MOUNT ||
4683 error "fscrypt setup $MOUNT failed"
4685 chown -R $ID0:$ID0 $testdir
4687 echo -e 'mypass\nmypass' | su - $USER0 -c "fscrypt encrypt --verbose \
4688 --source=custom_passphrase --name=protector $testdir" ||
4689 error "fscrypt encrypt failed"
4691 echo -e 'mypass\nmypass' | su - $USER0 -c "fscrypt encrypt --verbose \
4692 --source=custom_passphrase --name=protector2 $testdir" &&
4693 error "second fscrypt encrypt should have failed"
4695 mkdir -p ${testdir}2 || error "mkdir ${testdir}2 failed"
4696 touch ${testdir}2/f || error "mkdir ${testdir}2/f failed"
4699 echo -e 'mypass\nmypass' | fscrypt encrypt --verbose \
4700 --source=custom_passphrase --name=protector3 ${testdir}2 &&
4701 error "fscrypt encrypt on non-empty dir should have failed"
4703 $RUNAS dd if=/dev/urandom of=$testfile bs=127 count=1 conv=fsync ||
4704 error "write to encrypted file $testfile failed"
4705 cp $testfile $tmpfile
4706 $RUNAS dd if=/dev/urandom of=$testfile2 bs=127 count=1 conv=fsync ||
4707 error "write to encrypted file $testfile2 failed"
4708 $RUNAS dd if=/dev/urandom of=$testfile3 bs=127 count=1 conv=fsync ||
4709 error "write to encrypted file $testfile3 failed"
4710 $RUNAS mkdir $testdir/subdir || error "mkdir subdir failed"
4711 $RUNAS touch $testdir/subdir/subfile || error "mkdir subdir failed"
4713 $RUNAS fscrypt lock --verbose $testdir ||
4714 error "fscrypt lock $testdir failed (1)"
4716 $RUNAS ls -R $testdir || error "ls -R $testdir failed"
4717 local filecount=$($RUNAS find $testdir -type f | wc -l)
4718 [ $filecount -eq 4 ] || error "found $filecount files"
4720 # check enable_filename_encryption default value
4721 # tunable only available for client built against embedded llcrypt
4722 $LCTL get_param mdc.*.connect_flags | grep -q name_encryption &&
4723 nameenc=$(lctl get_param -n llite.*.enable_filename_encryption |
4725 # If client is built against in-kernel fscrypt, it is not possible
4726 # to decide to encrypt file names or not: they are always encrypted.
4727 if [ -n "$nameenc" ]; then
4728 [ $nameenc -eq 0 ] ||
4729 error "enable_filename_encryption should be 0 by default"
4731 # $testfile, $testfile2 and $testfile3 should exist because
4732 # names are not encrypted
4734 error "$testfile should exist because name not encrypted"
4735 [ -f $testfile2 ] ||
4736 error "$testfile2 should exist because name not encrypted"
4737 [ -f $testfile3 ] ||
4738 error "$testfile3 should exist because name not encrypted"
4740 [ $? -eq 0 ] || error "cannot stat $testfile3 without key"
4743 scrambledfiles=( $(find $testdir/ -maxdepth 1 -type f) )
4744 $RUNAS hexdump -C ${scrambledfiles[0]} &&
4745 error "reading ${scrambledfiles[0]} should fail without key"
4747 $RUNAS touch ${testfile}.nokey &&
4748 error "touch ${testfile}.nokey should have failed without key"
4750 echo mypass | $RUNAS fscrypt unlock --verbose $testdir ||
4751 error "fscrypt unlock $testdir failed (1)"
4753 $RUNAS cat $testfile > $resfile ||
4754 error "reading $testfile failed"
4756 cmp -bl $tmpfile $resfile || error "file read differs from file written"
4758 [ $? -eq 0 ] || error "cannot stat $testfile3 with key"
4760 $RUNAS fscrypt lock --verbose $testdir ||
4761 error "fscrypt lock $testdir failed (2)"
4763 $RUNAS hexdump -C ${scrambledfiles[1]} &&
4764 error "reading ${scrambledfiles[1]} should fail without key"
4766 # server local client incompatible with SSK keys installed
4767 if [ "$SHARED_KEY" != true ]; then
4769 stack_trap umount_mds_client EXIT
4770 do_facet $SINGLEMDS touch $DIR2/$tdir/newfile
4771 mdsscrambledfile=$(do_facet $SINGLEMDS find $testdir2/ \
4772 -maxdepth 1 -type f | head -n1)
4773 [ -n "$mdsscrambledfile" ] || error "could not find file"
4774 do_facet $SINGLEMDS cat "$mdsscrambledfile" &&
4775 error "reading $mdsscrambledfile should fail on MDS"
4776 do_facet $SINGLEMDS "echo aaa >> \"$mdsscrambledfile\"" &&
4777 error "writing $mdsscrambledfile should fail on MDS"
4778 do_facet $SINGLEMDS $MULTIOP $testdir2/fileA m &&
4779 error "creating $testdir2/fileA should fail on MDS"
4780 do_facet $SINGLEMDS mkdir $testdir2/dirA &&
4781 error "mkdir $testdir2/dirA should fail on MDS"
4782 do_facet $SINGLEMDS ln -s $DIR2/$tdir/newfile $testdir2/sl1 &&
4783 error "ln -s $testdir2/sl1 should fail on MDS"
4784 do_facet $SINGLEMDS ln $DIR2/$tdir/newfile $testdir2/hl1 &&
4785 error "ln $testdir2/hl1 should fail on MDS"
4786 do_facet $SINGLEMDS mv "$mdsscrambledfile" $testdir2/fB &&
4787 error "mv $mdsscrambledfile should fail on MDS"
4788 do_facet $SINGLEMDS mrename "$mdsscrambledfile" $testdir2/fB &&
4789 error "mrename $mdsscrambledfile should fail on MDS"
4790 do_facet $SINGLEMDS rm -f $DIR2/$tdir/newfile
4793 echo mypass | $RUNAS fscrypt unlock --verbose $testdir ||
4794 error "fscrypt unlock $testdir failed (2)"
4797 $RUNAS fscrypt lock --verbose $testdir ||
4798 error "fscrypt lock $testdir failed (3)"
4800 rm -rf $tmpfile $resfile $testdir ${testdir}2 $MOUNT/.fscrypt
4802 # remount client with subdirectory mount
4803 umount_client $MOUNT || error "umount $MOUNT failed (1)"
4804 export FILESET=/$tdir
4805 mount_client $MOUNT ${MOUNT_OPTS} || error "remount failed (1)"
4809 # setup encryption from inside this subdir mount
4810 # the .fscrypt directory is going to be created at the real fs root
4811 yes | fscrypt setup --verbose $MOUNT ||
4812 error "fscrypt setup $MOUNT failed (2)"
4813 testdir=$MOUNT/vault
4815 chown -R $ID0:$ID0 $testdir
4816 fid1=$(path2fid $MOUNT/.fscrypt)
4817 echo "With FILESET $tdir, .fscrypt FID is $fid1"
4819 # enable name encryption, only valid if built against embedded llcrypt
4820 if [ -n "$nameenc" ]; then
4821 do_facet mgs $LCTL set_param -P \
4822 llite.*.enable_filename_encryption=1
4824 error "set_param -P \
4825 llite.*.enable_filename_encryption failed"
4827 wait_update_facet --verbose client \
4828 "$LCTL get_param -n llite.*.enable_filename_encryption \
4830 error "enable_filename_encryption not set on client"
4833 # encrypt 'vault' dir inside the subdir mount
4834 echo -e 'mypass\nmypass' | su - $USER0 -c "fscrypt encrypt --verbose \
4835 --source=custom_passphrase --name=protector $testdir" ||
4836 error "fscrypt encrypt failed"
4840 $RUNAS cp $tmpfile $testdir/encfile
4842 $RUNAS fscrypt lock --verbose $testdir ||
4843 error "fscrypt lock $testdir failed (4)"
4845 # encfile should actually have its name encrypted
4846 if [ -n "$nameenc" ]; then
4847 [ -f $testdir/encfile ] &&
4848 error "encfile name should be encrypted"
4850 filecount=$(find $testdir -type f | wc -l)
4851 [ $filecount -eq 1 ] || error "found $filecount files instead of 1"
4853 # remount client with encrypted dir as subdirectory mount
4854 umount_client $MOUNT || error "umount $MOUNT failed (2)"
4855 export FILESET=/$tdir/vault
4856 mount_client $MOUNT ${MOUNT_OPTS} || error "remount failed (2)"
4860 fid2=$(path2fid $MOUNT/.fscrypt)
4861 echo "With FILESET $tdir/vault, .fscrypt FID is $fid2"
4862 [ "$fid1" == "$fid2" ] || error "fid1 $fid1 != fid2 $fid2 (1)"
4864 # all content seen by this mount is encrypted, but .fscrypt is virtually
4865 # presented, letting us call fscrypt lock/unlock
4866 echo mypass | $RUNAS fscrypt unlock --verbose $MOUNT ||
4867 error "fscrypt unlock $MOUNT failed (3)"
4870 [ $(cat $MOUNT/encfile) == "abc" ] || error "cat encfile failed"
4872 # remount client without subdir mount
4873 umount_client $MOUNT || error "umount $MOUNT failed (3)"
4874 mount_client $MOUNT ${MOUNT_OPTS} || error "remount failed (3)"
4877 fid2=$(path2fid $MOUNT/.fscrypt)
4878 echo "Without FILESET, .fscrypt FID is $fid2"
4879 [ "$fid1" == "$fid2" ] || error "fid1 $fid1 != fid2 $fid2 (2)"
4881 # because .fscrypt was actually created at the real root of the fs,
4882 # we can call fscrypt lock/unlock on the encrypted dir
4883 echo mypass | $RUNAS fscrypt unlock --verbose $DIR/$tdir/vault ||
4884 error "fscrypt unlock $$DIR/$tdir/vault failed (4)"
4887 echo c >> $DIR/$tdir/vault/encfile || error "write to encfile failed"
4889 rm -rf $DIR/$tdir/vault/*
4890 $RUNAS fscrypt lock --verbose $DIR/$tdir/vault ||
4891 error "fscrypt lock $DIR/$tdir/vault failed (5)"
4893 # disable name encryption, only valid if built against embedded llcrypt
4894 if [ -n "$nameenc" ]; then
4895 do_facet mgs $LCTL set_param -P \
4896 llite.*.enable_filename_encryption=0
4898 error "set_param -P \
4899 llite.*.enable_filename_encryption failed"
4901 wait_update_facet --verbose client \
4902 "$LCTL get_param -n llite.*.enable_filename_encryption \
4904 error "enable_filename_encryption not set back to default"
4907 rm -rf $tmpfile $MOUNT/.fscrypt
4909 run_test 54 "Encryption policies with fscrypt"
4913 if is_mounted $MOUNT; then
4914 umount_client $MOUNT || error "umount $MOUNT failed"
4917 do_facet mgs $LCTL nodemap_del c0
4918 do_facet mgs $LCTL nodemap_modify --name default \
4919 --property admin --value 0
4920 do_facet mgs $LCTL nodemap_modify --name default \
4921 --property trusted --value 0
4922 wait_nm_sync default admin_nodemap
4923 wait_nm_sync default trusted_nodemap
4925 do_facet mgs $LCTL nodemap_activate 0
4926 wait_nm_sync active 0
4928 if $SHARED_KEY; then
4929 export SK_UNIQUE_NM=false
4933 mount_client $MOUNT ${MOUNT_OPTS} || error "remount failed"
4934 if [ "$MOUNT_2" ]; then
4935 mount_client $MOUNT2 ${MOUNT_OPTS} || error "remount failed"
4941 (( $MDS1_VERSION > $(version_code 2.12.6.2) )) ||
4942 skip "Need MDS version at least 2.12.6.3"
4947 mkdir -p $DIR/$tdir/$USER0/testdir_groups
4948 chown root:$USER0 $DIR/$tdir/$USER0
4949 chmod 770 $DIR/$tdir/$USER0
4950 chmod g+s $DIR/$tdir/$USER0
4951 chown $USER0:$USER0 $DIR/$tdir/$USER0/testdir_groups
4952 chmod 770 $DIR/$tdir/$USER0/testdir_groups
4953 chmod g+s $DIR/$tdir/$USER0/testdir_groups
4955 # unmount client completely
4956 umount_client $MOUNT || error "umount $MOUNT failed"
4957 if is_mounted $MOUNT2; then
4958 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
4961 do_nodes $(comma_list $(all_mdts_nodes)) \
4962 $LCTL set_param mdt.*.identity_upcall=NONE
4964 stack_trap cleanup_55 EXIT
4966 do_facet mgs $LCTL nodemap_activate 1
4969 do_facet mgs $LCTL nodemap_del c0 || true
4970 wait_nm_sync c0 id ''
4972 do_facet mgs $LCTL nodemap_modify --name default \
4973 --property admin --value 1
4974 do_facet mgs $LCTL nodemap_modify --name default \
4975 --property trusted --value 1
4976 wait_nm_sync default admin_nodemap
4977 wait_nm_sync default trusted_nodemap
4979 client_ip=$(host_nids_address $HOSTNAME $NETTYPE)
4980 client_nid=$(h2nettype $client_ip)
4981 do_facet mgs $LCTL nodemap_add c0
4982 do_facet mgs $LCTL nodemap_add_range \
4983 --name c0 --range $client_nid
4984 do_facet mgs $LCTL nodemap_modify --name c0 \
4985 --property admin --value 0
4986 do_facet mgs $LCTL nodemap_modify --name c0 \
4987 --property trusted --value 1
4988 wait_nm_sync c0 admin_nodemap
4989 wait_nm_sync c0 trusted_nodemap
4991 if $SHARED_KEY; then
4992 export SK_UNIQUE_NM=true
4993 # set some generic fileset to trigger SSK code
4997 # remount client to take nodemap into account
4998 zconf_mount_clients $HOSTNAME $MOUNT $MOUNT_OPTS ||
4999 error "remount failed"
5003 euid_access $USER0 $DIR/$tdir/$USER0/testdir_groups/file
5005 run_test 55 "access with seteuid"
5008 local testfile=$DIR/$tdir/$tfile
5010 [[ $(facet_fstype ost1) == zfs ]] && skip "skip ZFS backend"
5012 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5013 skip "client encryption not supported"
5015 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5016 skip "need dummy encryption support"
5018 [[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
5020 stack_trap cleanup_for_enc_tests EXIT
5023 $LFS setstripe -c1 $testfile
5024 dd if=/dev/urandom of=$testfile bs=1M count=3 conv=fsync
5025 filefrag -v $testfile || error "filefrag $testfile failed"
5026 (( $(filefrag -v $testfile | grep -c encrypted) >= 1 )) ||
5027 error "filefrag $testfile does not show encrypted flag"
5028 (( $(filefrag -v $testfile | grep -c encoded) >= 1 )) ||
5029 error "filefrag $testfile does not show encoded flag"
5031 run_test 56 "FIEMAP on encrypted file"
5034 local testdir=$DIR/$tdir/mytestdir
5035 local testfile=$DIR/$tdir/$tfile
5037 [[ $(facet_fstype ost1) == zfs ]] && skip "skip ZFS backend"
5039 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5040 skip "client encryption not supported"
5042 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5043 skip "need dummy encryption support"
5047 setfattr -n security.c -v myval $testdir &&
5048 error "setting xattr on $testdir should have failed (1.1)"
5049 setfattr -n encryption.c -v myval $testdir &&
5050 error "setting xattr on $testdir should have failed (1.2)"
5052 setfattr -n security.c -v myval $testfile &&
5053 error "setting xattr on $testfile should have failed (1.1)"
5054 setfattr -n encryption.c -v myval $testfile &&
5055 error "setting xattr on $testfile should have failed (1.2)"
5059 stack_trap cleanup_for_enc_tests EXIT
5063 if [ $(getfattr -n security.c $testdir 2>&1 |
5064 grep -ci "Operation not permitted") -eq 0 ]; then
5065 error "getting xattr on $testdir should have failed (1.1)"
5067 if [ $(getfattr -n encryption.c $testdir 2>&1 |
5068 grep -ci "Operation not supported") -eq 0 ]; then
5069 error "getting xattr on $testdir should have failed (1.2)"
5071 getfattr -d -m - $testdir 2>&1 | grep security\.c &&
5072 error "listing xattrs on $testdir should not expose security.c"
5073 getfattr -d -m - $testdir 2>&1 | grep encryption\.c &&
5074 error "listing xattrs on $testdir should not expose encryption.c"
5075 if [ $(setfattr -n security.c -v myval $testdir 2>&1 |
5076 grep -ci "Operation not permitted") -eq 0 ]; then
5077 error "setting xattr on $testdir should have failed (2.1)"
5079 if [ $(setfattr -n encryption.c -v myval $testdir 2>&1 |
5080 grep -ci "Operation not supported") -eq 0 ]; then
5081 error "setting xattr on $testdir should have failed (2.2)"
5084 if [ $(getfattr -n security.c $testfile 2>&1 |
5085 grep -ci "Operation not permitted") -eq 0 ]; then
5086 error "getting xattr on $testfile should have failed (1.1)"
5088 if [ $(getfattr -n encryption.c $testfile 2>&1 |
5089 grep -ci "Operation not supported") -eq 0 ]; then
5090 error "getting xattr on $testfile should have failed (1.2)"
5092 getfattr -d -m - $testfile 2>&1 | grep security\.c &&
5093 error "listing xattrs on $testfile should not expose security.c"
5094 getfattr -d -m - $testfile 2>&1 | grep encryption\.c &&
5095 error "listing xattrs on $testfile should not expose encryption.c"
5096 if [ $(setfattr -n security.c -v myval $testfile 2>&1 |
5097 grep -ci "Operation not permitted") -eq 0 ]; then
5098 error "setting xattr on $testfile should have failed (2.1)"
5100 if [ $(setfattr -n encryption.c -v myval $testfile 2>&1 |
5101 grep -ci "Operation not supported") -eq 0 ]; then
5102 error "setting xattr on $testfile should have failed (2.2)"
5106 run_test 57 "security.c/encryption.c xattr protection"
5109 local testdir=$DIR/$tdir/mytestdir
5110 local testfile=$DIR/$tdir/$tfile
5112 [[ $(facet_fstype ost1) == zfs ]] && skip "skip ZFS backend"
5114 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5115 skip "client encryption not supported"
5117 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5118 skip "need dummy encryption support"
5120 stack_trap cleanup_for_enc_tests EXIT
5123 touch $DIR/$tdir/$tfile
5124 mkdir $DIR/$tdir/subdir
5128 echo 3 > /proc/sys/vm/drop_caches
5130 ll_decode_linkea $DIR/$tdir/$tfile || error "cannot read $tfile linkea"
5131 ll_decode_linkea $DIR/$tdir/subdir || error "cannot read subdir linkea"
5133 for ((i = 0; i < 1000; i = $((i+1)))); do
5134 mkdir -p $DIR/$tdir/d${i}
5135 touch $DIR/$tdir/f${i}
5136 createmany -m $DIR/$tdir/d${i}/f 5 > /dev/null
5141 echo 3 > /proc/sys/vm/drop_caches
5144 ls -ailR $DIR/$tdir > /dev/null || error "fail to ls"
5146 run_test 58 "access to enc file's xattrs"
5149 local mirror1=$TMP/$tfile.mirror1
5150 local mirror2=$TMP/$tfile.mirror2
5154 $LFS mirror verify -vvv $testfile ||
5155 error "verifying mirror failed (1)"
5156 if [ $($LFS mirror verify -v $testfile 2>&1 |
5157 grep -ci "only valid") -ne 0 ]; then
5158 error "verifying mirror failed (2)"
5161 $LFS mirror read -N 1 -o $mirror1 $testfile ||
5162 error "read from mirror 1 failed"
5163 cmp -bl $reffile $mirror1 ||
5164 error "corruption of mirror 1"
5165 $LFS mirror read -N 2 -o $mirror2 $testfile ||
5166 error "read from mirror 2 failed"
5167 cmp -bl $reffile $mirror2 ||
5168 error "corruption of mirror 2"
5172 local testfile=$DIR/$tdir/$tfile
5173 local tmpfile=$TMP/$tfile
5174 local mirror1=$TMP/$tfile.mirror1
5175 local mirror2=$TMP/$tfile.mirror2
5178 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5179 skip "client encryption not supported"
5181 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5182 skip "need dummy encryption support"
5184 [[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
5186 stack_trap "cleanup_for_enc_tests $tmpfile $mirror1 $mirror2" EXIT
5189 dd if=/dev/urandom of=$tmpfile bs=5000 count=1 conv=fsync
5191 $LFS mirror create -N -i0 -N -i1 $testfile ||
5192 error "could not create mirror"
5193 dd if=$tmpfile of=$testfile bs=5000 count=1 conv=fsync ||
5194 error "could not write to $testfile"
5195 $LFS getstripe $testfile
5197 # remount without dummy encryption key
5198 remount_client_normally
5200 scrambledfile=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -type f)
5201 $LFS mirror resync $scrambledfile ||
5202 error "could not resync mirror"
5204 $LFS mirror verify -vvv $scrambledfile ||
5205 error "mirror verify failed (1)"
5206 if [ $($LFS mirror verify -v $scrambledfile 2>&1 |
5207 grep -ci "only valid") -ne 0 ]; then
5208 error "mirror verify failed (2)"
5211 $LFS mirror read -N 1 -o $mirror1 $scrambledfile &&
5212 error "read from mirror should fail"
5215 remount_client_dummykey
5216 verify_mirror $testfile $tmpfile
5218 run_test 59a "mirror resync of encrypted files without key"
5221 local testfile=$DIR/$tdir/$tfile
5222 local tmpfile=$TMP/$tfile
5223 local mirror1=$TMP/$tfile.mirror1
5224 local mirror2=$TMP/$tfile.mirror2
5227 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5228 skip "client encryption not supported"
5230 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5231 skip "need dummy encryption support"
5233 [[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
5235 stack_trap "cleanup_for_enc_tests $tmpfile $mirror1 $mirror2" EXIT
5238 tr '\0' '2' < /dev/zero |
5239 dd of=$tmpfile bs=1 count=9000 conv=fsync
5241 $LFS setstripe -c1 -i0 $testfile
5242 dd if=$tmpfile of=$testfile bs=9000 count=1 conv=fsync ||
5243 error "write to $testfile failed"
5244 $LFS getstripe $testfile
5246 # remount without dummy encryption key
5247 remount_client_normally
5249 scrambledfile=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -type f)
5250 $LFS migrate -i1 $scrambledfile ||
5251 error "migrate $scrambledfile failed"
5252 $LFS getstripe $scrambledfile
5253 stripe=$($LFS getstripe -i $scrambledfile)
5254 [ $stripe -eq 1 ] || error "migrate file $scrambledfile failed"
5258 remount_client_dummykey
5259 cmp -bl $tmpfile $testfile ||
5260 error "migrated file is corrupted"
5262 # remount without dummy encryption key
5263 remount_client_normally
5265 $LFS mirror extend -N -i0 $scrambledfile ||
5266 error "mirror extend $scrambledfile failed (1)"
5267 $LFS getstripe $scrambledfile
5268 mirror_count=$($LFS getstripe -N $scrambledfile)
5269 [ $mirror_count -eq 2 ] ||
5270 error "mirror extend file $scrambledfile failed (2)"
5271 stripe=$($LFS getstripe --mirror-id=1 -i $scrambledfile)
5272 [ $stripe -eq 1 ] ||
5273 error "mirror extend file $scrambledfile failed (3)"
5274 stripe=$($LFS getstripe --mirror-id=2 -i $scrambledfile)
5275 [ $stripe -eq 0 ] ||
5276 error "mirror extend file $scrambledfile failed (4)"
5278 $LFS mirror verify -vvv $scrambledfile ||
5279 error "mirror verify failed (1)"
5280 if [ $($LFS mirror verify -v $scrambledfile 2>&1 |
5281 grep -ci "only valid") -ne 0 ]; then
5282 error "mirror verify failed (2)"
5286 remount_client_dummykey
5287 verify_mirror $testfile $tmpfile
5289 # remount without dummy encryption key
5290 remount_client_normally
5292 $LFS mirror split --mirror-id 1 -d $scrambledfile ||
5293 error "mirror split file $scrambledfile failed (1)"
5294 $LFS getstripe $scrambledfile
5295 mirror_count=$($LFS getstripe -N $scrambledfile)
5296 [ $mirror_count -eq 1 ] ||
5297 error "mirror split file $scrambledfile failed (2)"
5298 stripe=$($LFS getstripe --mirror-id=1 -i $scrambledfile)
5299 [ -z "$stripe" ] || error "mirror split file $scrambledfile failed (3)"
5300 stripe=$($LFS getstripe --mirror-id=2 -i $scrambledfile)
5301 [ $stripe -eq 0 ] || error "mirror split file $scrambledfile failed (4)"
5304 remount_client_dummykey
5306 cmp -bl $tmpfile $testfile ||
5307 error "extended/split file is corrupted"
5309 run_test 59b "migrate/extend/split of encrypted files without key"
5312 local dirname=$DIR/$tdir/subdir
5315 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5316 skip "client encryption not supported"
5318 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5319 skip "need dummy encryption support"
5321 [[ $MDSCOUNT -ge 2 ]] || skip_env "needs >= 2 MDTs"
5323 (( "$MDS1_VERSION" > $(version_code 2.14.54.54) )) ||
5324 skip "MDT migration not supported with older server"
5326 stack_trap cleanup_for_enc_tests EXIT
5329 $LFS setdirstripe -i 0 $dirname
5330 echo b > $dirname/subf
5332 # remount without dummy encryption key
5333 remount_client_normally
5335 scrambleddir=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -type d)
5337 # migrate a non-empty encrypted dir
5338 $LFS migrate -m 1 $scrambleddir ||
5339 error "migrate $scrambleddir between MDTs failed (1)"
5341 stripe=$($LFS getdirstripe -i $scrambleddir)
5342 [ $stripe -eq 1 ] ||
5343 error "migrate $scrambleddir between MDTs failed (2)"
5347 [ -f $dirname/subf ] ||
5348 error "migrate $scrambleddir between MDTs failed (3)"
5349 [ $(cat $dirname/subf) == "b" ] ||
5350 error "migrate $scrambleddir between MDTs failed (4)"
5352 run_test 59c "MDT migrate of encrypted files without key"
5355 local testdir=$DIR/$tdir/mytestdir
5356 local testfile=$DIR/$tdir/$tfile
5358 (( $MDS1_VERSION > $(version_code 2.14.53) )) ||
5359 skip "Need MDS version at least 2.14.53"
5361 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5362 skip "client encryption not supported"
5364 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5365 skip "need dummy encryption support"
5367 stack_trap cleanup_for_enc_tests EXIT
5370 echo a > $DIR/$tdir/file1
5371 mkdir $DIR/$tdir/subdir
5372 echo b > $DIR/$tdir/subdir/subfile1
5375 # unmount client completely
5376 umount_client $MOUNT || error "umount $MOUNT failed"
5377 if is_mounted $MOUNT2; then
5378 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
5381 # remount client with subdirectory mount
5382 export FILESET=/$tdir
5383 mount_client $MOUNT ${MOUNT_OPTS} || error "remount failed"
5384 if [ "$MOUNT_2" ]; then
5385 mount_client $MOUNT2 ${MOUNT_OPTS} || error "remount failed"
5389 ls -Rl $DIR || error "ls -Rl $DIR failed (1)"
5392 remount_client_dummykey
5395 ls -Rl $DIR || error "ls -Rl $DIR failed (2)"
5396 cat $DIR/file1 || error "cat $DIR/$tdir/file1 failed"
5397 cat $DIR/subdir/subfile1 ||
5398 error "cat $DIR/$tdir/subdir/subfile1 failed"
5400 run_test 60 "Subdirmount of encrypted dir"
5403 if $SHARED_KEY; then
5404 export SK_UNIQUE_NM=true
5408 do_facet mgs $LCTL nodemap_activate 1
5411 do_facet mgs $LCTL nodemap_del c0 || true
5412 wait_nm_sync c0 id ''
5414 do_facet mgs $LCTL nodemap_modify --name default \
5415 --property admin --value 1
5416 do_facet mgs $LCTL nodemap_modify --name default \
5417 --property trusted --value 1
5418 wait_nm_sync default admin_nodemap
5419 wait_nm_sync default trusted_nodemap
5421 client_ip=$(host_nids_address $HOSTNAME $NETTYPE)
5422 client_nid=$(h2nettype $client_ip)
5423 [[ "$client_nid" =~ ":" ]] && client_nid+="/128"
5424 do_facet mgs $LCTL nodemap_add c0
5425 do_facet mgs $LCTL nodemap_add_range \
5426 --name c0 --range $client_nid || {
5427 do_facet mgs $LCTL nodemap_del c0
5430 do_facet mgs $LCTL nodemap_modify --name c0 \
5431 --property admin --value 1
5432 do_facet mgs $LCTL nodemap_modify --name c0 \
5433 --property trusted --value 1
5434 wait_nm_sync c0 admin_nodemap
5435 wait_nm_sync c0 trusted_nodemap
5439 do_facet mgs $LCTL nodemap_del c0
5440 do_facet mgs $LCTL nodemap_modify --name default \
5441 --property admin --value 0
5442 do_facet mgs $LCTL nodemap_modify --name default \
5443 --property trusted --value 0
5444 wait_nm_sync default admin_nodemap
5445 wait_nm_sync default trusted_nodemap
5447 do_facet mgs $LCTL nodemap_activate 0
5448 wait_nm_sync active 0
5450 if $SHARED_KEY; then
5452 export SK_UNIQUE_NM=false
5455 mount_client $MOUNT ${MOUNT_OPTS} || error "re-mount failed"
5460 local testfile=$DIR/$tdir/$tfile
5463 readonly=$(do_facet mgs \
5464 lctl get_param -n nodemap.default.readonly_mount)
5465 [ -n "$readonly" ] ||
5466 skip "Server does not have readonly_mount nodemap flag"
5468 stack_trap cleanup_61 EXIT
5469 for idx in $(seq 1 $MDSCOUNT); do
5470 wait_recovery_complete mds$idx
5472 umount_client $MOUNT || error "umount $MOUNT failed (1)"
5474 # Activate nodemap, and mount rw.
5475 # Should succeed as rw mount is not forbidden by default.
5477 readonly=$(do_facet mgs \
5478 lctl get_param -n nodemap.default.readonly_mount)
5479 [ $readonly -eq 0 ] ||
5480 error "wrong default value for readonly_mount on default nodemap"
5481 readonly=$(do_facet mgs \
5482 lctl get_param -n nodemap.c0.readonly_mount)
5483 [ $readonly -eq 0 ] ||
5484 error "wrong default value for readonly_mount on nodemap c0"
5486 zconf_mount_clients $HOSTNAME $MOUNT ${MOUNT_OPTS},rw ||
5487 error "mount '-o rw' failed with default"
5489 findmnt $MOUNT --output=options -n -f | grep -q "rw," ||
5490 error "should be rw mount"
5491 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
5492 echo a > $testfile || error "write $testfile failed"
5493 umount_client $MOUNT || error "umount $MOUNT failed (2)"
5495 # Now enforce read-only, and retry.
5496 do_facet mgs $LCTL nodemap_modify --name c0 \
5497 --property readonly_mount --value 1
5498 wait_nm_sync c0 readonly_mount
5500 # mount without option should turn into ro
5501 zconf_mount_clients $HOSTNAME $MOUNT ${MOUNT_OPTS} ||
5502 error "mount failed (1)"
5503 findmnt $MOUNT --output=options -n -f | grep -q "ro," ||
5504 error "mount should have been turned into ro"
5505 cat $testfile || error "read $testfile failed (1)"
5506 echo b > $testfile && error "write $testfile should fail (1)"
5507 umount_client $MOUNT || error "umount $MOUNT failed (3)"
5509 # mount rw should turn into ro
5510 zconf_mount_clients $HOSTNAME $MOUNT ${MOUNT_OPTS},rw ||
5511 error "mount '-o rw' failed"
5512 findmnt $MOUNT --output=options -n -f | grep -q "ro," ||
5513 error "mount rw should have been turned into ro"
5514 cat $testfile || error "read $testfile failed (2)"
5515 echo b > $testfile && error "write $testfile should fail (2)"
5516 umount_client $MOUNT || error "umount $MOUNT failed (4)"
5518 # mount ro should work as expected
5519 zconf_mount_clients $HOSTNAME $MOUNT ${MOUNT_OPTS},ro ||
5520 error "mount '-o ro' failed"
5522 cat $testfile || error "read $testfile failed (3)"
5523 echo b > $testfile && error "write $testfile should fail (3)"
5524 umount_client $MOUNT || error "umount $MOUNT failed (5)"
5526 # remount rw should not work
5527 zconf_mount_clients $HOSTNAME $MOUNT ${MOUNT_OPTS} ||
5528 error "mount failed (2)"
5529 mount_client $MOUNT remount,rw || error "remount failed"
5530 findmnt $MOUNT --output=options -n -f | grep -q "ro," ||
5531 error "remount rw should have been turned into ro"
5532 cat $testfile || error "read $testfile failed (4)"
5533 echo b > $testfile && error "write $testfile should fail (4)"
5534 umount_client $MOUNT || error "umount $MOUNT failed (6)"
5536 run_test 61 "Nodemap enforces read-only mount"
5539 local testdir=$DIR/$tdir/mytestdir
5540 local testfile=$DIR/$tdir/$tfile
5542 [[ $(facet_fstype ost1) == zfs ]] && skip "skip ZFS backend"
5544 (( $MDS1_VERSION > $(version_code 2.15.51) )) ||
5545 skip "Need MDS version at least 2.15.51"
5547 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5548 skip "client encryption not supported"
5550 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5551 skip "need dummy encryption support"
5553 stack_trap cleanup_for_enc_tests EXIT
5556 lfs setstripe -c -1 $DIR/$tdir
5557 touch $DIR/$tdir/${tfile}_1 || error "touch ${tfile}_1 failed"
5558 dd if=/dev/zero of=$DIR/$tdir/${tfile}_2 bs=1 count=1 conv=fsync ||
5559 error "dd ${tfile}_2 failed"
5561 # unmount the Lustre filesystem
5562 stopall || error "stopping for e2fsck run"
5564 # run e2fsck on the MDT and OST devices
5565 local mds_host=$(facet_active_host $SINGLEMDS)
5566 local ost_host=$(facet_active_host ost1)
5567 local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
5568 local ost_dev=$(ostdevname 1)
5570 run_e2fsck $mds_host $mds_dev "-n"
5571 run_e2fsck $ost_host $ost_dev "-n"
5573 # mount the Lustre filesystem
5574 setupall || error "remounting the filesystem failed"
5576 run_test 62 "e2fsck with encrypted files"
5581 for path in "${paths[@]}"; do
5589 for path in "${paths[@]}"; do
5590 fids+=("$(lfs path2fid $path)")
5595 for fid in "${fids[@]}"; do
5597 respath=$(lfs fid2path $MOUNT $fid)
5598 echo -e "\t" $respath
5599 ls -li $respath >/dev/null
5600 [ $? -eq 0 ] || error "fid2path $fid failed"
5607 local vaultdir1=$DIR/$tdir/vault1==dir
5608 local vaultdir2=$DIR/$tdir/vault2==dir
5609 local longfname1="longfilenamewitha=inthemiddletotestbehaviorregardingthedigestedform"
5610 local longdname="longdirectorynamewitha=inthemiddletotestbehaviorregardingthedigestedform"
5611 local longfname2="$longdname/${longfname1}2"
5613 (( $MDS1_VERSION > $(version_code 2.15.53) )) ||
5614 skip "Need MDS version at least 2.15.53"
5616 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5617 skip "client encryption not supported"
5619 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5620 skip "need dummy encryption support"
5622 which fscrypt || skip "This test needs fscrypt userspace tool"
5624 yes | fscrypt setup --force --verbose ||
5625 echo "fscrypt global setup already done"
5626 sed -i 's/\(.*\)policy_version\(.*\):\(.*\)\"[0-9]*\"\(.*\)/\1policy_version\2:\3"2"\4/' \
5628 yes | fscrypt setup --verbose $MOUNT ||
5629 echo "fscrypt setup $MOUNT already done"
5631 # enable_filename_encryption tunable only available for client
5632 # built against embedded llcrypt. If client is built against in-kernel
5633 # fscrypt, file names are always encrypted.
5634 $LCTL get_param mdc.*.connect_flags | grep -q name_encryption &&
5635 nameenc=$(lctl get_param -n llite.*.enable_filename_encryption |
5637 if [ -n "$nameenc" ]; then
5638 do_facet mgs $LCTL set_param -P \
5639 llite.*.enable_filename_encryption=1
5641 error "set_param -P \
5642 llite.*.enable_filename_encryption=1 failed"
5644 wait_update_facet --verbose client \
5645 "$LCTL get_param -n llite.*.enable_filename_encryption \
5647 error "enable_filename_encryption not set on client"
5651 echo -e 'mypass\nmypass' | fscrypt encrypt --verbose \
5652 --source=custom_passphrase --name=protector_63_1 $vaultdir1 ||
5653 error "fscrypt encrypt $vaultdir1 failed"
5655 mkdir $vaultdir1/dirA
5656 mkdir $vaultdir1/$longdname
5657 paths=("$vaultdir1/fileA")
5658 paths+=("$vaultdir1/dirA/fileB")
5659 paths+=("$vaultdir1/$longfname1")
5660 paths+=("$vaultdir1/$longfname2")
5663 paths+=("$vaultdir1/dirA")
5664 paths+=("$vaultdir1/$longdname")
5669 fscrypt lock --verbose $vaultdir1 ||
5670 error "fscrypt lock $vaultdir1 failed (1)"
5674 if [ -z "$nameenc" ]; then
5675 echo "Rest of the test requires disabling name encryption"
5679 # disable name encryption
5680 do_facet mgs $LCTL set_param -P llite.*.enable_filename_encryption=0
5682 error "set_param -P llite.*.enable_filename_encryption=0 failed"
5684 wait_update_facet --verbose client \
5685 "$LCTL get_param -n llite.*.enable_filename_encryption \
5687 error "enable_filename_encryption not set back to default"
5690 echo -e 'mypass\nmypass' | fscrypt encrypt --verbose \
5691 --source=custom_passphrase --name=protector_63_2 $vaultdir2 ||
5692 error "fscrypt encrypt $vaultdir2 failed"
5694 mkdir $vaultdir2/dirA
5695 mkdir $vaultdir2/$longdname
5698 paths=("$vaultdir2/fileA")
5699 paths+=("$vaultdir2/dirA/fileB")
5700 paths+=("$vaultdir2/$longfname1")
5701 paths+=("$vaultdir2/$longfname2")
5704 paths+=("$vaultdir2/dirA")
5705 paths+=("$vaultdir2/$longdname")
5710 fscrypt lock --verbose $vaultdir2 ||
5711 error "fscrypt lock $vaultdir2 failed (2)"
5715 rm -rf $MOUNT/.fscrypt
5717 run_test 63 "fid2path with encrypted files"
5720 do_facet mgs $LCTL nodemap_activate 1
5723 do_facet mgs $LCTL nodemap_del c0 || true
5724 wait_nm_sync c0 id ''
5726 do_facet mgs $LCTL nodemap_modify --name default \
5727 --property admin --value 1
5728 do_facet mgs $LCTL nodemap_modify --name default \
5729 --property trusted --value 1
5730 wait_nm_sync default admin_nodemap
5731 wait_nm_sync default trusted_nodemap
5733 client_ip=$(host_nids_address $HOSTNAME $NETTYPE)
5734 client_nid=$(h2nettype $client_ip)
5735 do_facet mgs $LCTL nodemap_add c0
5736 do_facet mgs $LCTL nodemap_add_range \
5737 --name c0 --range $client_nid
5738 do_facet mgs $LCTL nodemap_modify --name c0 \
5739 --property admin --value 1
5740 do_facet mgs $LCTL nodemap_modify --name c0 \
5741 --property trusted --value 1
5742 wait_nm_sync c0 admin_nodemap
5743 wait_nm_sync c0 trusted_nodemap
5747 do_facet mgs $LCTL nodemap_del c0
5748 do_facet mgs $LCTL nodemap_modify --name default \
5749 --property admin --value 0
5750 do_facet mgs $LCTL nodemap_modify --name default \
5751 --property trusted --value 0
5752 wait_nm_sync default admin_nodemap
5753 wait_nm_sync default trusted_nodemap
5755 do_facet mgs $LCTL nodemap_activate 0
5756 wait_nm_sync active 0
5760 local testfile=$DIR/$tdir/$tfile
5763 (( MDS1_VERSION >= $(version_code 2.15.54) )) ||
5764 skip "Need MDS >= 2.15.54 for role-based controls"
5766 stack_trap cleanup_64 EXIT
5767 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
5770 # check default value for rbac is all
5771 rbac=$(do_facet mds $LCTL get_param -n nodemap.c0.rbac)
5772 for role in file_perms \
5780 [[ "$rbac" =~ "$role" ]] ||
5781 error "role '$role' not in default '$rbac'"
5784 do_facet mgs $LCTL nodemap_modify --name c0 \
5785 --property rbac --value file_perms
5786 wait_nm_sync c0 rbac
5788 stack_trap "set +vx"
5790 chmod 777 $testfile || error "chmod failed"
5791 chown $TSTUSR:$TSTUSR $testfile || error "chown failed"
5792 chgrp $TSTUSR $testfile || error "chgrp failed"
5793 $LFS project -p 1000 $testfile || error "setting project failed"
5796 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
5797 wait_nm_sync c0 rbac
5800 chmod 777 $testfile && error "chmod should fail"
5801 chown $TSTUSR:$TSTUSR $testfile && error "chown should fail"
5802 chgrp $TSTUSR $testfile && error "chgrp should fail"
5803 $LFS project -p 1000 $testfile && error "setting project should fail"
5806 run_test 64a "Nodemap enforces file_perms RBAC roles"
5809 local testdir=$DIR/$tdir/${tfile}.d
5812 (( MDS1_VERSION >= $(version_code 2.15.54) )) ||
5813 skip "Need MDS >= 2.15.54 for role-based controls"
5815 (( MDSCOUNT >= 2 )) || skip "mdt count $MDSCOUNT, skipping dne_ops role"
5817 stack_trap cleanup_64 EXIT
5818 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
5821 dir_restripe=$(do_node $mds1_HOST \
5822 "$LCTL get_param -n mdt.*MDT0000.enable_dir_restripe")
5823 [ -n "$dir_restripe" ] || dir_restripe=0
5824 do_nodes $(comma_list $(all_mdts_nodes)) \
5825 $LCTL set_param mdt.*.enable_dir_restripe=1 ||
5826 error "enabling dir_restripe failed"
5827 stack_trap "do_nodes $(comma_list $(all_mdts_nodes)) \
5828 $LCTL set_param mdt.*.enable_dir_restripe=$dir_restripe" EXIT
5829 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac \
5831 wait_nm_sync c0 rbac
5832 $LFS mkdir -i 0 ${testdir}_for_migr ||
5833 error "$LFS mkdir ${testdir}_for_migr failed (1)"
5834 touch ${testdir}_for_migr/file001 ||
5835 error "touch ${testdir}_for_migr/file001 failed (1)"
5836 $LFS mkdir -i 0 ${testdir}_mdt0 ||
5837 error "$LFS mkdir ${testdir}_mdt0 failed (1)"
5838 $LFS mkdir -i 1 ${testdir}_mdt1 ||
5839 error "$LFS mkdir ${testdir}_mdt1 failed (1)"
5841 $LFS mkdir -i 1 $testdir || error "$LFS mkdir failed (1)"
5843 $LFS mkdir -c 2 $testdir || error "$LFS mkdir failed (2)"
5846 $LFS setdirstripe -c 2 $testdir || error "$LFS setdirstripe failed"
5848 $LFS migrate -m 1 ${testdir}_for_migr || error "$LFS migrate failed"
5849 touch ${testdir}_mdt0/fileA || error "touch fileA failed (1)"
5850 mv ${testdir}_mdt0/fileA ${testdir}_mdt1/ || error "mv failed (1)"
5853 $LFS mkdir -i 0 ${testdir}_for_migr ||
5854 error "$LFS mkdir ${testdir}_for_migr failed (2)"
5855 touch ${testdir}_for_migr/file001 ||
5856 error "touch ${testdir}_for_migr/file001 failed (2)"
5857 $LFS mkdir -i 0 ${testdir}_mdt0 ||
5858 error "$LFS mkdir ${testdir}_mdt0 failed (2)"
5859 $LFS mkdir -i 1 ${testdir}_mdt1 ||
5860 error "$LFS mkdir ${testdir}_mdt1 failed (2)"
5862 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
5863 wait_nm_sync c0 rbac
5865 $LFS mkdir -i 1 $testdir && error "$LFS mkdir should fail (1)"
5866 $LFS mkdir -c 2 $testdir && error "$LFS mkdir should fail (2)"
5868 $LFS setdirstripe -c 2 $testdir && error "$LFS setdirstripe should fail"
5870 $LFS migrate -m 1 ${testdir}_for_migr &&
5871 error "$LFS migrate should fail"
5872 touch ${testdir}_mdt0/fileA || error "touch fileA failed (2)"
5873 mv ${testdir}_mdt0/fileA ${testdir}_mdt1/ || error "mv failed (2)"
5876 run_test 64b "Nodemap enforces dne_ops RBAC roles"
5879 (( MDS1_VERSION >= $(version_code 2.15.54) )) ||
5880 skip "Need MDS >= 2.15.54 for role-based controls"
5882 stack_trap cleanup_64 EXIT
5883 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
5886 do_facet mgs $LCTL nodemap_modify --name c0 \
5887 --property rbac --value quota_ops
5888 wait_nm_sync c0 rbac
5890 $LFS setquota -u $USER0 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT ||
5891 error "lfs setquota -u failed"
5892 $LFS setquota -u $USER0 --delete $MOUNT
5893 $LFS setquota -g $USER0 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT ||
5894 error "lfs setquota -g failed"
5895 $LFS setquota -g $USER0 --delete $MOUNT
5896 $LFS setquota -p 1000 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT ||
5897 error "lfs setquota -p failed"
5898 $LFS setquota -p 1000 --delete $MOUNT
5900 $LFS setquota -U -b 10G -B 11G -i 100K -I 105K $MOUNT ||
5901 error "lfs setquota -U failed"
5902 $LFS setquota -U -b 0 -B 0 -i 0 -I 0 $MOUNT
5903 $LFS setquota -G -b 10G -B 11G -i 100K -I 105K $MOUNT ||
5904 error "lfs setquota -G failed"
5905 $LFS setquota -G -b 0 -B 0 -i 0 -I 0 $MOUNT
5906 $LFS setquota -P -b 10G -B 11G -i 100K -I 105K $MOUNT ||
5907 error "lfs setquota -P failed"
5908 $LFS setquota -P -b 0 -B 0 -i 0 -I 0 $MOUNT
5909 $LFS setquota -u $USER0 -D $MOUNT ||
5910 error "lfs setquota -u -D failed"
5911 $LFS setquota -u $USER0 --delete $MOUNT
5912 $LFS setquota -g $USER0 -D $MOUNT ||
5913 error "lfs setquota -g -D failed"
5914 $LFS setquota -g $USER0 --delete $MOUNT
5915 $LFS setquota -p 1000 -D $MOUNT ||
5916 error "lfs setquota -p -D failed"
5917 $LFS setquota -p 1000 --delete $MOUNT
5920 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
5921 wait_nm_sync c0 rbac
5924 $LFS setquota -u $USER0 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT &&
5925 error "lfs setquota -u should fail"
5926 $LFS setquota -u $USER0 --delete $MOUNT
5927 $LFS setquota -g $USER0 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT &&
5928 error "lfs setquota -g should fail"
5929 $LFS setquota -g $USER0 --delete $MOUNT
5930 $LFS setquota -p 1000 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT &&
5931 error "lfs setquota -p should fail"
5932 $LFS setquota -p 1000 --delete $MOUNT
5934 $LFS setquota -U -b 10G -B 11G -i 100K -I 105K $MOUNT &&
5935 error "lfs setquota -U should fail"
5936 $LFS setquota -G -b 10G -B 11G -i 100K -I 105K $MOUNT &&
5937 error "lfs setquota -G should fail"
5938 $LFS setquota -P -b 10G -B 11G -i 100K -I 105K $MOUNT &&
5939 error "lfs setquota -P should fail"
5940 $LFS setquota -u $USER0 -D $MOUNT &&
5941 error "lfs setquota -u -D should fail"
5942 $LFS setquota -u $USER0 --delete $MOUNT
5943 $LFS setquota -g $USER0 -D $MOUNT &&
5944 error "lfs setquota -g -D should fail"
5945 $LFS setquota -g $USER0 --delete $MOUNT
5946 $LFS setquota -p 1000 -D $MOUNT &&
5947 error "lfs setquota -p -D should fail"
5948 $LFS setquota -p 1000 --delete $MOUNT
5951 run_test 64c "Nodemap enforces quota_ops RBAC roles"
5954 local testfile=$DIR/$tdir/$tfile
5957 (( MDS1_VERSION >= $(version_code 2.15.54) )) ||
5958 skip "Need MDS >= 2.15.54 for role-based controls"
5960 stack_trap cleanup_64 EXIT
5961 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
5964 do_facet mgs $LCTL nodemap_modify --name c0 \
5965 --property rbac --value byfid_ops
5966 wait_nm_sync c0 rbac
5969 fid=$(lfs path2fid $testfile)
5971 $LFS fid2path $MOUNT $fid || error "fid2path $fid failed (1)"
5972 cat $MOUNT/.lustre/fid/$fid || error "cat by fid failed"
5973 lfs rmfid $MOUNT $fid || error "lfs rmfid failed"
5976 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
5977 wait_nm_sync c0 rbac
5980 fid=$(lfs path2fid $testfile)
5982 $LFS fid2path $MOUNT $fid || error "fid2path $fid failed (2)"
5983 cat $MOUNT/.lustre/fid/$fid && error "cat by fid should fail"
5984 lfs rmfid $MOUNT $fid && error "lfs rmfid should fail"
5988 run_test 64d "Nodemap enforces byfid_ops RBAC roles"
5991 local testfile=$DIR/$tdir/$tfile
5992 local testdir=$DIR/$tdir/${tfile}.d
5994 (( MDS1_VERSION >= $(version_code 2.15.54) )) ||
5995 skip "Need MDS >= 2.15.54 for role-based controls"
5997 stack_trap cleanup_64 EXIT
5998 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
6001 # activate changelogs
6002 changelog_register || error "changelog_register failed"
6003 local cl_user="${CL_USERS[$SINGLEMDS]%% *}"
6004 changelog_users $SINGLEMDS | grep -q $cl_user ||
6005 error "User $cl_user not found in changelog_users"
6006 changelog_chmask ALL
6009 mkdir $testdir || error "failed to mkdir $testdir"
6010 touch $testfile || error "failed to touch $testfile"
6012 do_facet mgs $LCTL nodemap_modify --name c0 \
6013 --property rbac --value chlg_ops
6014 wait_nm_sync c0 rbac
6017 echo "changelogs dump"
6018 changelog_dump || error "failed to dump changelogs"
6019 echo "changelogs clear"
6020 changelog_clear 0 || error "failed to clear changelogs"
6022 rm -rf $testdir $testfile || error "rm -rf $testdir $testfile failed"
6024 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
6025 wait_nm_sync c0 rbac
6028 mkdir $testdir || error "failed to mkdir $testdir"
6029 touch $testfile || error "failed to touch $testfile"
6032 echo "changelogs dump"
6033 changelog_dump && error "dump changelogs should fail"
6034 echo "changelogs clear"
6035 changelog_clear 0 && error "clear changelogs should fail"
6036 rm -rf $testdir $testfile
6038 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value all
6039 wait_nm_sync c0 rbac
6041 run_test 64e "Nodemap enforces chlg_ops RBAC roles"
6044 local vaultdir=$DIR/$tdir/vault
6049 (( MDS1_VERSION >= $(version_code 2.15.54) )) ||
6050 skip "Need MDS >= 2.15.54 for role-based controls"
6052 cli_enc=$($LCTL get_param mdc.*.import | grep client_encryption)
6053 [ -n "$cli_enc" ] || skip "Need enc support, skip fscrypt_admin role"
6054 which fscrypt || skip "Need fscrypt, skip fscrypt_admin role"
6056 stack_trap cleanup_64 EXIT
6057 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
6060 yes | fscrypt setup --force --verbose ||
6061 echo "fscrypt global setup already done"
6062 sed -i 's/\(.*\)policy_version\(.*\):\(.*\)\"[0-9]*\"\(.*\)/\1policy_version\2:\3"2"\4/' \
6064 yes | fscrypt setup --verbose $MOUNT ||
6065 echo "fscrypt setup $MOUNT already done"
6066 stack_trap "rm -rf $MOUNT/.fscrypt"
6068 # file_perms is required because fscrypt uses chmod/chown
6069 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac \
6070 --value fscrypt_admin,file_perms
6071 wait_nm_sync c0 rbac
6075 echo -e 'mypass\nmypass' | fscrypt encrypt --verbose \
6076 --source=custom_passphrase --name=protector_64 $vaultdir ||
6077 error "fscrypt encrypt $vaultdir failed"
6078 fscrypt lock $vaultdir || error "fscrypt lock $vaultdir failed (1)"
6079 policy=$(fscrypt status $vaultdir | awk '$1 == "Policy:"{print $2}')
6080 [ -n "$policy" ] || error "could not get enc policy"
6081 protector=$(fscrypt status $vaultdir |
6082 awk 'BEGIN {found=0} { if (found == 1) { print $1 }} \
6083 $1 == "PROTECTOR" {found=1}')
6084 [ -n "$protector" ] || error "could not get enc protector"
6088 # file_perms is required because fscrypt uses chmod/chown
6089 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac \
6091 wait_nm_sync c0 rbac
6094 echo mypass | fscrypt unlock $vaultdir ||
6095 error "fscrypt unlock $vaultdir failed"
6096 fscrypt lock $vaultdir || error "fscrypt lock $vaultdir failed (2)"
6097 fscrypt metadata destroy --protector=$MOUNT:$protector --force &&
6098 error "destroy protector should fail"
6099 fscrypt metadata destroy --policy=$MOUNT:$policy --force &&
6100 error "destroy policy should fail"
6101 mkdir -p ${vaultdir}2
6102 echo -e 'mypass\nmypass' | fscrypt encrypt --verbose \
6103 --source=custom_passphrase \
6104 --name=protector_64bis ${vaultdir}2 &&
6105 error "fscrypt encrypt ${vaultdir}2 should fail"
6109 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value all
6110 wait_nm_sync c0 rbac
6113 fscrypt metadata destroy --protector=$MOUNT:$protector --force ||
6114 error "destroy protector failed"
6115 fscrypt metadata destroy --policy=$MOUNT:$policy --force ||
6116 error "destroy policy failed"
6121 run_test 64f "Nodemap enforces fscrypt_admin RBAC roles"
6130 (( neg == 1 )) || neg=""
6131 $LFS find -type f ${neg:+"!"} --attrs $pattern $path > $TMP/res
6133 res=$(cat $TMP/res | wc -l)
6134 (( res == $expected )) ||
6135 error "Find $pattern $path: found $res, expected $expected"
6139 local dirbis=$DIR/${tdir}_bis
6140 local testfile=$DIR/$tdir/$tfile
6143 $LCTL get_param mdc.*.import | grep -q client_encryption ||
6144 skip "client encryption not supported"
6146 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
6147 skip "need dummy encryption support"
6149 # $dirbis is not going to be encrypted, as client
6150 # is not mounted with -o test_dummy_encryption yet
6152 stack_trap "rm -rf $dirbis" EXIT
6153 touch $dirbis/$tfile.1
6154 touch $dirbis/$tfile.2
6155 chattr +i $dirbis/$tfile.2
6156 stack_trap "chattr -i $dirbis/$tfile.2" EXIT
6158 stack_trap cleanup_for_enc_tests EXIT
6161 # All files/dirs under $DIR/$tdir are encrypted
6164 chattr +i $testfile.2
6165 stack_trap "chattr -i $testfile.2" EXIT
6167 $LFS find -printf "%p %LA\n" $dirbis/$tfile.1
6168 res=$($LFS find -printf "%LA" $dirbis/$tfile.1)
6169 [ "$res" == "---" ] ||
6170 error "$dirbis/$tfile.1 should have no attr, showed $res (1)"
6171 $LFS find -printf "%p %La\n" $dirbis/$tfile.1
6172 res=$($LFS find -printf "%La" $dirbis/$tfile.1)
6173 [ "$res" == "---" ] ||
6174 error "$dirbis/$tfile.1 should have no attr, showed $res (2)"
6175 $LFS find -printf "%p %LA\n" $dirbis/$tfile.2
6176 res=$($LFS find -printf "%LA" $dirbis/$tfile.2)
6177 [ "$res" == "Immutable" ] ||
6178 error "$dirbis/$tfile.2 should be Immutable, showed $res"
6179 $LFS find -printf "%p %La\n" $dirbis/$tfile.2
6180 res=$($LFS find -printf "%La" $dirbis/$tfile.2)
6181 [ "$res" == "i" ] ||
6182 error "$dirbis/$tfile.2 should be 'i', showed $res"
6183 $LFS find -printf "%p %LA\n" $testfile.1
6184 res=$($LFS find -printf "%LA" $testfile.1)
6185 [ "$res" == "Encrypted" ] ||
6186 error "$testfile.1 should be Encrypted, showed $res"
6187 $LFS find -printf "%p %La\n" $testfile.1
6188 res=$($LFS find -printf "%La" $testfile.1)
6189 [ "$res" == "E" ] ||
6190 error "$testfile.1 should be 'E', showed $res"
6191 $LFS find -printf "%p %LA\n" $testfile.2
6192 res=$($LFS find -printf "%LA" $testfile.2)
6193 [ "$res" == "Immutable,Encrypted" ] ||
6194 error "$testfile.2 should be Immutable,Encrypted, showed $res"
6195 $LFS find -printf "%p %La\n" $testfile.2
6196 res=$($LFS find -printf "%La" $testfile.2)
6197 [ "$res" == "iE" ] ||
6198 error "$testfile.2 should be 'iE', showed $res"
6200 echo Expecting to find 2 encrypted files
6201 look_for_files Encrypted 0 "$DIR/${tdir}*" 2
6202 echo Expecting to find 2 encrypted files
6203 look_for_files E 0 "$DIR/${tdir}*" 2
6205 echo Expecting to find 2 non-encrypted files
6206 look_for_files Encrypted 1 "$DIR/${tdir}*" 2
6207 echo Expecting to find 2 non-encrypted files
6208 look_for_files E 1 "$DIR/${tdir}*" 2
6210 echo Expecting to find 1 encrypted+immutable file
6211 look_for_files "Encrypted,Immutable" 0 "$DIR/${tdir}*" 1
6212 echo Expecting to find 1 encrypted+immutable file
6213 look_for_files "Ei" 0 "$DIR/${tdir}*" 1
6215 echo Expecting to find 1 encrypted+^immutable file
6216 look_for_files "Encrypted,^Immutable" 0 "$DIR/${tdir}*" 1
6217 echo Expecting to find 1 encrypted+^immutable file
6218 look_for_files "E^i" 0 "$DIR/${tdir}*" 1
6220 echo Expecting to find 1 ^encrypted+immutable file
6221 look_for_files "^Encrypted,Immutable" 0 "$DIR/${tdir}*" 1
6222 echo Expecting to find 1 ^encrypted+immutable file
6223 look_for_files "^Ei" 0 "$DIR/${tdir}*" 1
6225 echo Expecting to find 1 ^encrypted+^immutable file
6226 look_for_files "^Encrypted,^Immutable" 0 "$DIR/${tdir}*" 1
6227 echo Expecting to find 1 ^encrypted+^immutable file
6228 look_for_files "^E^i" 0 "$DIR/${tdir}*" 1
6230 run_test 65 "lfs find -printf %La and --attrs support"
6233 lctl set_param fail_loc=0 fail_val=0
6234 mount_client $MOUNT ${MOUNT_OPTS} || error "re-mount $MOUNT failed"
6235 if is_mounted $MOUNT2; then
6236 mount_client $MOUNT2 ${MOUNT_OPTS} ||
6237 error "re-mount $MOUNT2 failed"
6242 stack_trap cleanup_68 EXIT
6244 # unmount client completely
6245 umount_client $MOUNT || error "umount $MOUNT failed"
6246 if is_mounted $MOUNT2; then
6247 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
6250 #define CFS_FAIL_ONCE|OBD_FAIL_PTLRPC_DROP_MGS 0x51d
6251 lctl set_param fail_loc=0x8000051d fail_val=20
6253 zconf_mount_clients $HOSTNAME $MOUNT $MOUNT_OPTS ||
6254 error "mount failed"
6256 umount_client $MOUNT || error "re-umount $MOUNT failed"
6258 run_test 68 "all config logs are processed"
6261 local mdt="$(mdtname_from_index 0 $MOUNT)"
6265 param="mdt.$mdt.identity_upcall"
6266 orig="$(do_facet mds1 "$LCTL get_param -n $param")"
6267 stack_trap "do_facet mds1 $LCTL set_param $param=$orig" EXIT
6269 # identity_upcall accepts a path to an executable,
6270 # or NONE (case insensitive)
6271 do_facet mds1 $LCTL set_param $param=/path/to/prog ||
6272 error "set_param $param=/path/to/prog failed"
6273 do_facet mds1 $LCTL set_param $param=prog &&
6274 error "set_param $param=prog should failed"
6275 do_facet mds1 $LCTL set_param $param=NONE ||
6276 error "set_param $param=NONE failed"
6277 do_facet mds1 $LCTL set_param $param=none ||
6278 error "set_param $param=none failed"
6281 param="sptlrpc.gss.rsi_upcall"
6282 orig="$(do_facet mds1 "$LCTL get_param -n $param")"
6283 stack_trap "do_facet mds1 $LCTL set_param $param=$orig" EXIT
6285 # rsi_upcall only accepts a path to an executable
6286 do_facet mds1 $LCTL set_param $param=prog &&
6287 error "set_param $param=prog should failed"
6288 do_facet mds1 $LCTL set_param $param=NONE &&
6289 error "set_param $param=NONE should fail"
6290 do_facet mds1 $LCTL set_param $param=/path/to/prog ||
6291 error "set_param $param=/path/to/prog failed"
6294 run_test 69 "check upcall incorrect values"
6297 local param_mgs=$(mktemp $TMP/$tfile-mgs.XXXXXX)
6298 local param_copy=$(mktemp $TMP/$tfile-copy.XXXXXX)
6300 stack_trap "rm -f $param_mgs $param_copy" EXIT
6302 (( $MDS1_VERSION > $(version_code 2.15.61) )) ||
6303 skip "Need MDS version at least 2.15.61"
6305 if ! $SHARED_KEY; then
6306 skip "need shared key feature for this test"
6309 [[ "$ost1_FSTYPE" == ldiskfs ]] ||
6310 skip "ldiskfs only test (using debugfs)"
6312 # unmount then remount the Lustre filesystem, to make sure llogs
6313 # are copied locally
6314 export SK_NO_KEY=false
6315 stopall || error "stopall failed"
6317 mountmgs || error "mountmgs failed"
6318 mountmds || error "mountmds failed"
6319 mountoss || error "mountoss failed"
6320 mountcli || error "mountcli failed"
6324 do_facet mgs "sync ; sync"
6325 do_facet mgs "$DEBUGFS -c -R 'ls CONFIGS/' $(mgsdevname)"
6326 do_facet mgs "$DEBUGFS -c -R 'dump CONFIGS/$FSNAME-sptlrpc $param_mgs' \
6328 do_facet mgs "llog_reader $param_mgs" | grep -vE "SKIP|marker" |
6329 grep "^#" > $param_mgs
6332 if ! combined_mgs_mds; then
6333 do_facet mds1 "sync ; sync"
6334 do_facet mds1 "$DEBUGFS -c -R 'ls CONFIGS/' $(mdsdevname 1)"
6335 do_facet mds1 "$DEBUGFS -c -R 'dump CONFIGS/$FSNAME-sptlrpc \
6336 $param_copy' $(mdsdevname 1)"
6337 do_facet mds1 "llog_reader $param_copy" |
6338 grep -vE "SKIP|marker" | grep "^#" > $param_copy
6340 cmp -bl $param_mgs $param_copy ||
6341 error "sptlrpc llog differ in mds"
6345 do_facet ost1 "sync ; sync"
6346 do_facet ost1 "$DEBUGFS -c -R 'ls CONFIGS/' $(ostdevname 1)"
6347 do_facet ost1 "$DEBUGFS -c -R 'dump CONFIGS/$FSNAME-sptlrpc \
6348 $param_copy' $(ostdevname 1)"
6349 do_facet ost1 "llog_reader $param_copy" | grep -vE "SKIP|marker" |
6350 grep "^#" > $param_copy
6352 cmp -bl $param_mgs $param_copy ||
6353 error "sptlrpc llog differ in oss"
6355 run_test 70 "targets have local copy of sptlrpc llog"
6357 log "cleanup: ======================================================"
6360 for ((num = 1; num <= $MDSCOUNT; num++)); do
6361 if [[ "${identity_old[$num]}" == 1 ]]; then
6362 switch_identity $num false || identity_old[$num]=$?
6366 $RUNAS_CMD -u $ID0 ls $DIR
6367 $RUNAS_CMD -u $ID1 ls $DIR
6371 complete_test $SECONDS
6372 check_and_cleanup_lustre