3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
11 LUSTRE=${LUSTRE:-$(dirname $0)/..}
12 . $LUSTRE/tests/test-framework.sh
17 ALWAYS_EXCEPT="$SANITY_SEC_EXCEPT "
18 # bug number for skipped test:
20 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
22 [ "$SLOW" = "no" ] && EXCEPT_SLOW="26"
24 NODEMAP_TESTS=$(seq 7 26)
26 if ! check_versions; then
27 echo "It is NOT necessary to test nodemap under interoperation mode"
28 EXCEPT="$EXCEPT $NODEMAP_TESTS"
33 RUNAS_CMD=${RUNAS_CMD:-runas}
35 WTL=${WTL:-"$LUSTRE/tests/write_time_limit"}
38 PERM_CONF=$CONFDIR/perm.conf
40 HOSTNAME_CHECKSUM=$(hostname | sum | awk '{ print $1 }')
41 SUBNET_CHECKSUM=$(expr $HOSTNAME_CHECKSUM % 250 + 1)
43 require_dsh_mds || exit 0
44 require_dsh_ost || exit 0
46 clients=${CLIENTS//,/ }
47 num_clients=$(get_node_count ${clients})
48 clients_arr=($clients)
50 echo "was USER0=$(getent passwd | grep :${ID0:-500}:)"
51 echo "was USER1=$(getent passwd | grep :${ID1:-501}:)"
56 echo "now USER0=$USER0=$ID0:$(id -g $USER0), USER1=$USER1=$ID1:$(id -g $USER1)"
58 if [ "$SLOW" == "yes" ]; then
61 NODEMAP_IPADDR_LIST="1 10 64 128 200 250"
66 NODEMAP_IPADDR_LIST="1 250"
69 NODEMAP_MAX_ID=$((ID0 + NODEMAP_ID_COUNT))
72 skip "need to add user0 ($ID0:$ID0)" && exit 0
75 skip "need to add user1 ($ID1:$ID1)" && exit 0
77 IDBASE=${IDBASE:-60000}
79 # changes to mappings must be reflected in test 23
81 [0]="$((IDBASE+3)):$((IDBASE+0)) $((IDBASE+4)):$((IDBASE+2))"
82 [1]="$((IDBASE+5)):$((IDBASE+1)) $((IDBASE+6)):$((IDBASE+2))"
85 check_and_setup_lustre
90 GSS_REF=$(lsmod | grep ^ptlrpc_gss | awk '{print $3}')
91 if [ ! -z "$GSS_REF" -a "$GSS_REF" != "0" ]; then
93 echo "with GSS support"
96 echo "without GSS support"
99 MDT=$(do_facet $SINGLEMDS lctl get_param -N "mdt.\*MDT0000" |
101 [ -z "$MDT" ] && error "fail to get MDT device" && exit 1
102 do_facet $SINGLEMDS "mkdir -p $CONFDIR"
103 IDENTITY_FLUSH=mdt.$MDT.identity_flush
104 IDENTITY_UPCALL=mdt.$MDT.identity_upcall
113 if ! $RUNAS_CMD -u $user krb5_login.sh; then
114 error "$user login kerberos failed."
118 if ! $RUNAS_CMD -u $user -g $group ls $DIR > /dev/null 2>&1; then
119 $RUNAS_CMD -u $user lfs flushctx -k
120 $RUNAS_CMD -u $user krb5_login.sh
121 if ! $RUNAS_CMD -u$user -g$group ls $DIR > /dev/null 2>&1; then
122 error "init $user $group failed."
128 declare -a identity_old
131 for num in $(seq $MDSCOUNT); do
132 switch_identity $num true || identity_old[$num]=$?
135 if ! $RUNAS_CMD -u $ID0 ls $DIR > /dev/null 2>&1; then
136 sec_login $USER0 $USER0
139 if ! $RUNAS_CMD -u $ID1 ls $DIR > /dev/null 2>&1; then
140 sec_login $USER1 $USER1
145 # run as different user
149 chmod 0755 $DIR || error "chmod (1)"
150 rm -rf $DIR/$tdir || error "rm (1)"
151 mkdir -p $DIR/$tdir || error "mkdir (1)"
152 chown $USER0 $DIR/$tdir || error "chown (2)"
153 $RUNAS_CMD -u $ID0 ls $DIR || error "ls (1)"
154 rm -f $DIR/f0 || error "rm (2)"
155 $RUNAS_CMD -u $ID0 touch $DIR/f0 && error "touch (1)"
156 $RUNAS_CMD -u $ID0 touch $DIR/$tdir/f1 || error "touch (2)"
157 $RUNAS_CMD -u $ID1 touch $DIR/$tdir/f2 && error "touch (3)"
158 touch $DIR/$tdir/f3 || error "touch (4)"
159 chown root $DIR/$tdir || error "chown (3)"
160 chgrp $USER0 $DIR/$tdir || error "chgrp (1)"
161 chmod 0775 $DIR/$tdir || error "chmod (2)"
162 $RUNAS_CMD -u $ID0 touch $DIR/$tdir/f4 || error "touch (5)"
163 $RUNAS_CMD -u $ID1 touch $DIR/$tdir/f5 && error "touch (6)"
164 touch $DIR/$tdir/f6 || error "touch (7)"
165 rm -rf $DIR/$tdir || error "rm (3)"
167 run_test 0 "uid permission ============================="
171 [ $GSS_SUP = 0 ] && skip "without GSS support." && return
174 mkdir_on_mdt0 $DIR/$tdir
176 chown $USER0 $DIR/$tdir || error "chown (1)"
177 $RUNAS_CMD -u $ID1 -v $ID0 touch $DIR/$tdir/f0 && error "touch (2)"
178 echo "enable uid $ID1 setuid"
179 do_facet $SINGLEMDS "echo '* $ID1 setuid' >> $PERM_CONF"
180 do_facet $SINGLEMDS "lctl set_param -n $IDENTITY_FLUSH=-1"
181 $RUNAS_CMD -u $ID1 -v $ID0 touch $DIR/$tdir/f1 || error "touch (3)"
183 chown root $DIR/$tdir || error "chown (4)"
184 chgrp $USER0 $DIR/$tdir || error "chgrp (5)"
185 chmod 0770 $DIR/$tdir || error "chmod (6)"
186 $RUNAS_CMD -u $ID1 -g $ID1 touch $DIR/$tdir/f2 && error "touch (7)"
187 $RUNAS_CMD -u$ID1 -g$ID1 -j$ID0 touch $DIR/$tdir/f3 && error "touch (8)"
188 echo "enable uid $ID1 setuid,setgid"
189 do_facet $SINGLEMDS "echo '* $ID1 setuid,setgid' > $PERM_CONF"
190 do_facet $SINGLEMDS "lctl set_param -n $IDENTITY_FLUSH=-1"
191 $RUNAS_CMD -u $ID1 -g $ID1 -j $ID0 touch $DIR/$tdir/f4 ||
193 $RUNAS_CMD -u $ID1 -v $ID0 -g $ID1 -j $ID0 touch $DIR/$tdir/f5 ||
198 do_facet $SINGLEMDS "rm -f $PERM_CONF"
199 do_facet $SINGLEMDS "lctl set_param -n $IDENTITY_FLUSH=-1"
201 run_test 1 "setuid/gid ============================="
203 # bug 3285 - supplementary group should always succeed.
204 # NB: the supplementary groups are set for local client only,
205 # as for remote client, the groups of the specified uid on MDT
206 # will be obtained by upcall /sbin/l_getidentity and used.
208 [[ "$MDS1_VERSION" -ge $(version_code 2.6.93) ]] ||
209 [[ "$MDS1_VERSION" -ge $(version_code 2.5.35) &&
210 "$MDS1_VERSION" -lt $(version_code 2.5.50) ]] ||
211 skip "Need MDS version at least 2.6.93 or 2.5.35"
215 chmod 0771 $DIR/$tdir
216 chgrp $ID0 $DIR/$tdir
217 $RUNAS_CMD -u $ID0 ls $DIR/$tdir || error "setgroups (1)"
218 do_facet $SINGLEMDS "echo '* $ID1 setgrp' > $PERM_CONF"
219 do_facet $SINGLEMDS "lctl set_param -n $IDENTITY_FLUSH=-1"
220 $RUNAS_CMD -u $ID1 -G1,2,$ID0 ls $DIR/$tdir ||
221 error "setgroups (2)"
222 $RUNAS_CMD -u $ID1 -G1,2 ls $DIR/$tdir && error "setgroups (3)"
225 do_facet $SINGLEMDS "rm -f $PERM_CONF"
226 do_facet $SINGLEMDS "lctl set_param -n $IDENTITY_FLUSH=-1"
228 run_test 4 "set supplementary group ==============="
234 squash_id default 99 0
235 wait_nm_sync default squash_uid '' inactive
236 squash_id default 99 1
237 wait_nm_sync default squash_gid '' inactive
238 for (( i = 0; i < NODEMAP_COUNT; i++ )); do
239 local csum=${HOSTNAME_CHECKSUM}_${i}
241 do_facet mgs $LCTL nodemap_add $csum
243 if [ $rc -ne 0 ]; then
244 echo "nodemap_add $csum failed with $rc"
248 wait_update_facet --verbose mgs \
249 "$LCTL get_param nodemap.$csum.id 2>/dev/null | \
250 grep -c $csum || true" 1 30 ||
253 for (( i = 0; i < NODEMAP_COUNT; i++ )); do
254 local csum=${HOSTNAME_CHECKSUM}_${i}
256 wait_nm_sync $csum id '' inactive
264 for ((i = 0; i < NODEMAP_COUNT; i++)); do
265 local csum=${HOSTNAME_CHECKSUM}_${i}
267 if ! do_facet mgs $LCTL nodemap_del $csum; then
268 error "nodemap_del $csum failed with $?"
272 wait_update_facet --verbose mgs \
273 "$LCTL get_param nodemap.$csum.id 2>/dev/null | \
274 grep -c $csum || true" 0 30 ||
277 for (( i = 0; i < NODEMAP_COUNT; i++ )); do
278 local csum=${HOSTNAME_CHECKSUM}_${i}
280 wait_nm_sync $csum id '' inactive
287 local cmd="$LCTL nodemap_add_range"
291 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
292 range="$SUBNET_CHECKSUM.${2}.${j}.[1-253]@tcp"
293 if ! do_facet mgs $cmd --name $1 --range $range; then
302 local cmd="$LCTL nodemap_del_range"
306 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
307 range="$SUBNET_CHECKSUM.${2}.${j}.[1-253]@tcp"
308 if ! do_facet mgs $cmd --name $1 --range $range; then
318 local cmd="$LCTL nodemap_add_idmap"
322 (( $MDS1_VERSION >= $(version_code 2.14.52) )) || do_proj=false
324 echo "Start to add idmaps ..."
325 for ((i = 0; i < NODEMAP_COUNT; i++)); do
328 for ((j = $ID0; j < NODEMAP_MAX_ID; j++)); do
329 local csum=${HOSTNAME_CHECKSUM}_${i}
331 local fs_id=$((j + 1))
333 if ! do_facet mgs $cmd --name $csum --idtype uid \
334 --idmap $client_id:$fs_id; then
337 if ! do_facet mgs $cmd --name $csum --idtype gid \
338 --idmap $client_id:$fs_id; then
342 if ! do_facet mgs $cmd --name $csum \
343 --idtype projid --idmap \
344 $client_id:$fs_id; then
354 update_idmaps() { #LU-10040
355 [ "$MGS_VERSION" -lt $(version_code 2.10.55) ] &&
356 skip "Need MGS >= 2.10.55"
358 local csum=${HOSTNAME_CHECKSUM}_0
359 local old_id_client=$ID0
360 local old_id_fs=$((ID0 + 1))
361 local new_id=$((ID0 + 100))
368 echo "Start to update idmaps ..."
370 #Inserting an existed idmap should return error
371 cmd="$LCTL nodemap_add_idmap --name $csum --idtype uid"
373 $cmd --idmap $old_id_client:$old_id_fs 2>/dev/null; then
374 error "insert idmap {$old_id_client:$old_id_fs} " \
375 "should return error"
380 #Update id_fs and check it
381 if ! do_facet mgs $cmd --idmap $old_id_client:$new_id; then
382 error "$cmd --idmap $old_id_client:$new_id failed"
386 tmp_id=$(do_facet mgs $LCTL get_param -n nodemap.$csum.idmap |
387 awk '{ print $7 }' | sed -n '2p')
388 [ $tmp_id != $new_id ] && { error "new id_fs $tmp_id != $new_id"; \
389 rc=$((rc + 1)); return $rc; }
391 #Update id_client and check it
392 if ! do_facet mgs $cmd --idmap $new_id:$new_id; then
393 error "$cmd --idmap $new_id:$new_id failed"
397 tmp_id=$(do_facet mgs $LCTL get_param -n nodemap.$csum.idmap |
398 awk '{ print $5 }' | sed -n "$((NODEMAP_ID_COUNT + 1)) p")
399 tmp_id=$(echo ${tmp_id%,*}) #e.g. "501,"->"501"
400 [ $tmp_id != $new_id ] && { error "new id_client $tmp_id != $new_id"; \
401 rc=$((rc + 1)); return $rc; }
403 #Delete above updated idmap
404 cmd="$LCTL nodemap_del_idmap --name $csum --idtype uid"
405 if ! do_facet mgs $cmd --idmap $new_id:$new_id; then
406 error "$cmd --idmap $new_id:$new_id failed"
411 #restore the idmaps to make delete_idmaps work well
412 cmd="$LCTL nodemap_add_idmap --name $csum --idtype uid"
413 if ! do_facet mgs $cmd --idmap $old_id_client:$old_id_fs; then
414 error "$cmd --idmap $old_id_client:$old_id_fs failed"
424 local cmd="$LCTL nodemap_del_idmap"
428 (( $MDS1_VERSION >= $(version_code 2.14.52) )) || do_proj=false
430 echo "Start to delete idmaps ..."
431 for ((i = 0; i < NODEMAP_COUNT; i++)); do
434 for ((j = $ID0; j < NODEMAP_MAX_ID; j++)); do
435 local csum=${HOSTNAME_CHECKSUM}_${i}
437 local fs_id=$((j + 1))
439 if ! do_facet mgs $cmd --name $csum --idtype uid \
440 --idmap $client_id:$fs_id; then
443 if ! do_facet mgs $cmd --name $csum --idtype gid \
444 --idmap $client_id:$fs_id; then
448 if ! do_facet mgs $cmd --name $csum \
449 --idtype projid --idmap \
450 $client_id:$fs_id; then
464 local cmd="$LCTL nodemap_modify"
467 proc[0]="admin_nodemap"
468 proc[1]="trusted_nodemap"
472 for ((idx = 0; idx < 2; idx++)); do
473 if ! do_facet mgs $cmd --name $1 --property ${option[$idx]} \
478 if ! do_facet mgs $cmd --name $1 --property ${option[$idx]} \
488 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
489 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
493 cmd[0]="$LCTL nodemap_modify --property squash_uid"
494 cmd[1]="$LCTL nodemap_modify --property squash_gid"
495 cmd[2]="$LCTL nodemap_modify --property squash_projid"
497 if ! do_facet mgs ${cmd[$3]} --name $1 --value $2; then
502 # ensure that the squash defaults are the expected defaults
503 squash_id default 99 0
504 wait_nm_sync default squash_uid '' inactive
505 squash_id default 99 1
506 wait_nm_sync default squash_gid '' inactive
507 if [ "$MDS1_VERSION" -ge $(version_code 2.14.50) ]; then
508 squash_id default 99 2
509 wait_nm_sync default squash_projid '' inactive
515 cmd="$LCTL nodemap_test_nid"
517 nid=$(do_facet mgs $cmd $1)
519 if [ $nid == $2 ]; then
527 # restore activation state
528 do_facet mgs $LCTL nodemap_activate 0
534 local cmd="$LCTL nodemap_test_id"
537 echo "Start to test idmaps ..."
538 ## nodemap deactivated
539 if ! do_facet mgs $LCTL nodemap_activate 0; then
542 for ((id = $ID0; id < NODEMAP_MAX_ID; id++)); do
545 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
546 local nid="$SUBNET_CHECKSUM.0.${j}.100@tcp"
547 local fs_id=$(do_facet mgs $cmd --nid $nid \
548 --idtype uid --id $id)
549 if [ $fs_id != $id ]; then
550 echo "expected $id, got $fs_id"
557 if ! do_facet mgs $LCTL nodemap_activate 1; then
561 for ((id = $ID0; id < NODEMAP_MAX_ID; id++)); do
562 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
563 nid="$SUBNET_CHECKSUM.0.${j}.100@tcp"
564 fs_id=$(do_facet mgs $cmd --nid $nid \
565 --idtype uid --id $id)
566 expected_id=$((id + 1))
567 if [ $fs_id != $expected_id ]; then
568 echo "expected $expected_id, got $fs_id"
575 for ((i = 0; i < NODEMAP_COUNT; i++)); do
576 local csum=${HOSTNAME_CHECKSUM}_${i}
578 if ! do_facet mgs $LCTL nodemap_modify --name $csum \
579 --property trusted --value 1; then
580 error "nodemap_modify $csum failed with $?"
585 for ((id = $ID0; id < NODEMAP_MAX_ID; id++)); do
586 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
587 nid="$SUBNET_CHECKSUM.0.${j}.100@tcp"
588 fs_id=$(do_facet mgs $cmd --nid $nid \
589 --idtype uid --id $id)
590 if [ $fs_id != $id ]; then
591 echo "expected $id, got $fs_id"
597 ## ensure allow_root_access is enabled
598 for ((i = 0; i < NODEMAP_COUNT; i++)); do
599 local csum=${HOSTNAME_CHECKSUM}_${i}
601 if ! do_facet mgs $LCTL nodemap_modify --name $csum \
602 --property admin --value 1; then
603 error "nodemap_modify $csum failed with $?"
608 ## check that root allowed
609 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
610 nid="$SUBNET_CHECKSUM.0.${j}.100@tcp"
611 fs_id=$(do_facet mgs $cmd --nid $nid --idtype uid --id 0)
612 if [ $fs_id != 0 ]; then
613 echo "root allowed expected 0, got $fs_id"
618 ## ensure allow_root_access is disabled
619 for ((i = 0; i < NODEMAP_COUNT; i++)); do
620 local csum=${HOSTNAME_CHECKSUM}_${i}
622 if ! do_facet mgs $LCTL nodemap_modify --name $csum \
623 --property admin --value 0; then
624 error "nodemap_modify ${HOSTNAME_CHECKSUM}_${i} "
630 ## check that root is mapped to 99
631 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
632 nid="$SUBNET_CHECKSUM.0.${j}.100@tcp"
633 fs_id=$(do_facet mgs $cmd --nid $nid --idtype uid --id 0)
634 if [ $fs_id != 99 ]; then
635 error "root squash expected 99, got $fs_id"
640 ## reset client trust to 0
641 for ((i = 0; i < NODEMAP_COUNT; i++)); do
642 if ! do_facet mgs $LCTL nodemap_modify \
643 --name ${HOSTNAME_CHECKSUM}_${i} \
644 --property trusted --value 0; then
645 error "nodemap_modify ${HOSTNAME_CHECKSUM}_${i} "
657 remote_mgs_nodsh && skip "remote MGS with nodsh"
658 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
659 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
663 [[ $rc != 0 ]] && error "nodemap_add failed with $rc"
667 [[ $rc != 0 ]] && error "nodemap_del failed with $rc"
671 run_test 7 "nodemap create and delete"
676 remote_mgs_nodsh && skip "remote MGS with nodsh"
677 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
678 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
684 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
690 [[ $rc == 0 ]] && error "duplicate nodemap_add allowed with $rc" &&
696 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 3
700 run_test 8 "nodemap reject duplicates"
706 remote_mgs_nodsh && skip "remote MGS with nodsh"
707 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
708 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
713 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
716 for ((i = 0; i < NODEMAP_COUNT; i++)); do
717 if ! add_range ${HOSTNAME_CHECKSUM}_${i} $i; then
721 [[ $rc != 0 ]] && error "nodemap_add_range failed with $rc" && return 2
724 for ((i = 0; i < NODEMAP_COUNT; i++)); do
725 if ! delete_range ${HOSTNAME_CHECKSUM}_${i} $i; then
729 [[ $rc != 0 ]] && error "nodemap_del_range failed with $rc" && return 4
734 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 4
738 run_test 9 "nodemap range add"
743 remote_mgs_nodsh && skip "remote MGS with nodsh"
744 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
745 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
750 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
753 for ((i = 0; i < NODEMAP_COUNT; i++)); do
754 if ! add_range ${HOSTNAME_CHECKSUM}_${i} $i; then
758 [[ $rc != 0 ]] && error "nodemap_add_range failed with $rc" && return 2
761 for ((i = 0; i < NODEMAP_COUNT; i++)); do
762 if ! add_range ${HOSTNAME_CHECKSUM}_${i} $i; then
766 [[ $rc == 0 ]] && error "nodemap_add_range duplicate add with $rc" &&
771 for ((i = 0; i < NODEMAP_COUNT; i++)); do
772 if ! delete_range ${HOSTNAME_CHECKSUM}_${i} $i; then
776 [[ $rc != 0 ]] && error "nodemap_del_range failed with $rc" && return 4
780 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 5
784 run_test 10a "nodemap reject duplicate ranges"
787 [ "$MGS_VERSION" -lt $(version_code 2.10.53) ] &&
788 skip "Need MGS >= 2.10.53"
792 local nids="192.168.19.[0-255]@o2ib20"
794 do_facet mgs $LCTL nodemap_del $nm1 2>/dev/null
795 do_facet mgs $LCTL nodemap_del $nm2 2>/dev/null
797 do_facet mgs $LCTL nodemap_add $nm1 || error "Add $nm1 failed"
798 do_facet mgs $LCTL nodemap_add $nm2 || error "Add $nm2 failed"
799 do_facet mgs $LCTL nodemap_add_range --name $nm1 --range $nids ||
800 error "Add range $nids to $nm1 failed"
801 [ -n "$(do_facet mgs $LCTL get_param nodemap.$nm1.* |
802 grep start_nid)" ] || error "No range was found"
803 do_facet mgs $LCTL nodemap_del_range --name $nm2 --range $nids &&
804 error "Deleting range $nids from $nm2 should fail"
805 [ -n "$(do_facet mgs $LCTL get_param nodemap.$nm1.* |
806 grep start_nid)" ] || error "Range $nids should be there"
808 do_facet mgs $LCTL nodemap_del $nm1 || error "Delete $nm1 failed"
809 do_facet mgs $LCTL nodemap_del $nm2 || error "Delete $nm2 failed"
812 run_test 10b "delete range from the correct nodemap"
814 test_10c() { #LU-8912
815 [ "$MGS_VERSION" -lt $(version_code 2.10.57) ] &&
816 skip "Need MGS >= 2.10.57"
818 local nm="nodemap_lu8912"
819 local nid_range="10.210.[32-47].[0-255]@o2ib3"
820 local start_nid="10.210.32.0@o2ib3"
821 local end_nid="10.210.47.255@o2ib3"
822 local start_nid_found
825 do_facet mgs $LCTL nodemap_del $nm 2>/dev/null
826 do_facet mgs $LCTL nodemap_add $nm || error "Add $nm failed"
827 do_facet mgs $LCTL nodemap_add_range --name $nm --range $nid_range ||
828 error "Add range $nid_range to $nm failed"
830 start_nid_found=$(do_facet mgs $LCTL get_param nodemap.$nm.* |
831 awk -F '[,: ]' /start_nid/'{ print $9 }')
832 [ "$start_nid" == "$start_nid_found" ] ||
833 error "start_nid: $start_nid_found != $start_nid"
834 end_nid_found=$(do_facet mgs $LCTL get_param nodemap.$nm.* |
835 awk -F '[,: ]' /end_nid/'{ print $13 }')
836 [ "$end_nid" == "$end_nid_found" ] ||
837 error "end_nid: $end_nid_found != $end_nid"
839 do_facet mgs $LCTL nodemap_del $nm || error "Delete $nm failed"
842 run_test 10c "verfify contiguous range support"
844 test_10d() { #LU-8913
845 [ "$MGS_VERSION" -lt $(version_code 2.10.59) ] &&
846 skip "Need MGS >= 2.10.59"
848 local nm="nodemap_lu8913"
849 local nid_range="*@o2ib3"
850 local start_nid="0.0.0.0@o2ib3"
851 local end_nid="255.255.255.255@o2ib3"
852 local start_nid_found
855 do_facet mgs $LCTL nodemap_del $nm 2>/dev/null
856 do_facet mgs $LCTL nodemap_add $nm || error "Add $nm failed"
857 do_facet mgs $LCTL nodemap_add_range --name $nm --range $nid_range ||
858 error "Add range $nid_range to $nm failed"
860 start_nid_found=$(do_facet mgs $LCTL get_param nodemap.$nm.* |
861 awk -F '[,: ]' /start_nid/'{ print $9 }')
862 [ "$start_nid" == "$start_nid_found" ] ||
863 error "start_nid: $start_nid_found != $start_nid"
864 end_nid_found=$(do_facet mgs $LCTL get_param nodemap.$nm.* |
865 awk -F '[,: ]' /end_nid/'{ print $13 }')
866 [ "$end_nid" == "$end_nid_found" ] ||
867 error "end_nid: $end_nid_found != $end_nid"
869 do_facet mgs $LCTL nodemap_del $nm || error "Delete $nm failed"
872 run_test 10d "verfify nodemap range format '*@<net>' support"
877 remote_mgs_nodsh && skip "remote MGS with nodsh"
878 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
879 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
884 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
887 for ((i = 0; i < NODEMAP_COUNT; i++)); do
888 if ! modify_flags ${HOSTNAME_CHECKSUM}_${i}; then
892 [[ $rc != 0 ]] && error "nodemap_modify with $rc" && return 2
897 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 3
901 run_test 11 "nodemap modify"
906 remote_mgs_nodsh && skip "remote MGS with nodsh"
907 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
908 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
913 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
916 for ((i = 0; i < NODEMAP_COUNT; i++)); do
917 if ! squash_id ${HOSTNAME_CHECKSUM}_${i} 88 0; then
921 [[ $rc != 0 ]] && error "nodemap squash_uid with $rc" && return 2
924 for ((i = 0; i < NODEMAP_COUNT; i++)); do
925 if ! squash_id ${HOSTNAME_CHECKSUM}_${i} 88 1; then
929 [[ $rc != 0 ]] && error "nodemap squash_gid with $rc" && return 3
932 if (( $MDS1_VERSION >= $(version_code 2.14.52) )); then
933 for ((i = 0; i < NODEMAP_COUNT; i++)); do
934 if ! squash_id ${HOSTNAME_CHECKSUM}_${i} 88 2; then
939 [[ $rc != 0 ]] && error "nodemap squash_projid with $rc" && return 5
944 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 4
948 run_test 12 "nodemap set squash ids"
953 remote_mgs_nodsh && skip "remote MGS with nodsh"
954 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
955 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
960 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
963 for ((i = 0; i < NODEMAP_COUNT; i++)); do
964 if ! add_range ${HOSTNAME_CHECKSUM}_${i} $i; then
968 [[ $rc != 0 ]] && error "nodemap_add_range failed with $rc" && return 2
971 for ((i = 0; i < NODEMAP_COUNT; i++)); do
972 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
973 for k in $NODEMAP_IPADDR_LIST; do
974 if ! test_nid $SUBNET_CHECKSUM.$i.$j.$k \
975 ${HOSTNAME_CHECKSUM}_${i}; then
981 [[ $rc != 0 ]] && error "nodemap_test_nid failed with $rc" && return 3
986 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 4
990 run_test 13 "test nids"
995 remote_mgs_nodsh && skip "remote MGS with nodsh"
996 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
997 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
1002 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
1005 for ((i = 0; i < NODEMAP_COUNT; i++)); do
1006 for ((j = 0; j < NODEMAP_RANGE_COUNT; j++)); do
1007 for k in $NODEMAP_IPADDR_LIST; do
1008 if ! test_nid $SUBNET_CHECKSUM.$i.$j.$k \
1015 [[ $rc != 0 ]] && error "nodemap_test_nid failed with $rc" && return 3
1020 [[ $rc != 0 ]] && error "nodemap_del failed with $rc" && return 4
1024 run_test 14 "test default nodemap nid lookup"
1029 remote_mgs_nodsh && skip "remote MGS with nodsh"
1030 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
1031 skip "No nodemap on $MGS_VERSION MGS < 2.5.53"
1036 [[ $rc != 0 ]] && error "nodemap_add failed with $rc" && return 1
1038 for (( i = 0; i < NODEMAP_COUNT; i++ )); do
1039 local csum=${HOSTNAME_CHECKSUM}_${i}
1041 if ! do_facet mgs $LCTL nodemap_modify --name $csum \
1042 --property admin --value 0; then
1045 if ! do_facet mgs $LCTL nodemap_modify --name $csum \
1046 --property trusted --value 0; then
1050 [[ $rc != 0 ]] && error "nodemap_modify failed with $rc" && return 1
1053 for ((i = 0; i < NODEMAP_COUNT; i++)); do
1054 if ! add_range ${HOSTNAME_CHECKSUM}_${i} $i; then
1058 [[ $rc != 0 ]] && error "nodemap_add_range failed with $rc" && return 2
1063 [[ $rc != 0 ]] && error "nodemap_add_idmap failed with $rc" && return 3
1065 activedefault=$(do_facet mgs $LCTL get_param -n nodemap.active)
1066 if [[ "$activedefault" != "1" ]]; then
1067 stack_trap cleanup_active EXIT
1073 [[ $rc != 0 ]] && error "nodemap_test_id failed with $rc" && return 4
1078 [[ $rc != 0 ]] && error "update_idmaps failed with $rc" && return 5
1083 [[ $rc != 0 ]] && error "nodemap_del_idmap failed with $rc" && return 6
1088 [[ $rc != 0 ]] && error "nodemap_delete failed with $rc" && return 7
1092 run_test 15 "test id mapping"
1094 create_fops_nodemaps() {
1097 for client in $clients; do
1098 local client_ip=$(host_nids_address $client $NETTYPE)
1099 local client_nid=$(h2nettype $client_ip)
1100 do_facet mgs $LCTL nodemap_add c${i} || return 1
1101 do_facet mgs $LCTL nodemap_add_range \
1102 --name c${i} --range $client_nid || return 1
1103 for map in ${FOPS_IDMAPS[i]}; do
1104 do_facet mgs $LCTL nodemap_add_idmap --name c${i} \
1105 --idtype uid --idmap ${map} || return 1
1106 do_facet mgs $LCTL nodemap_add_idmap --name c${i} \
1107 --idtype gid --idmap ${map} || return 1
1110 wait_nm_sync c$i idmap
1117 delete_fops_nodemaps() {
1120 for client in $clients; do
1121 do_facet mgs $LCTL nodemap_del c${i} || return 1
1129 if [ $MDSCOUNT -le 1 ]; then
1130 do_node ${clients_arr[0]} mkdir -p $DIR/$tdir
1132 # round-robin MDTs to test DNE nodemap support
1133 [ ! -d $DIR ] && do_node ${clients_arr[0]} mkdir -p $DIR
1134 do_node ${clients_arr[0]} $LFS setdirstripe -c 1 -i \
1135 $((fops_mds_index % MDSCOUNT)) $DIR/$tdir
1136 ((fops_mds_index++))
1140 # acl test directory needs to be initialized on a privileged client
1142 local admin=$(do_facet mgs $LCTL get_param -n nodemap.c0.admin_nodemap)
1143 local trust=$(do_facet mgs $LCTL get_param -n \
1144 nodemap.c0.trusted_nodemap)
1146 do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 1
1147 do_facet mgs $LCTL nodemap_modify --name c0 --property trusted --value 1
1149 wait_nm_sync c0 admin_nodemap
1150 wait_nm_sync c0 trusted_nodemap
1152 do_node ${clients_arr[0]} rm -rf $DIR/$tdir
1154 do_node ${clients_arr[0]} chown $user $DIR/$tdir
1156 do_facet mgs $LCTL nodemap_modify --name c0 \
1157 --property admin --value $admin
1158 do_facet mgs $LCTL nodemap_modify --name c0 \
1159 --property trusted --value $trust
1161 # flush MDT locks to make sure they are reacquired before test
1162 do_node ${clients_arr[0]} $LCTL set_param \
1163 ldlm.namespaces.$FSNAME-MDT*.lru_size=clear
1165 wait_nm_sync c0 admin_nodemap
1166 wait_nm_sync c0 trusted_nodemap
1169 # fileset test directory needs to be initialized on a privileged client
1170 fileset_test_setup() {
1173 if [ -n "$FILESET" -a -z "$SKIP_FILESET" ]; then
1174 cleanup_mount $MOUNT
1175 FILESET="" zconf_mount_clients $CLIENTS $MOUNT
1178 local admin=$(do_facet mgs $LCTL get_param -n \
1179 nodemap.${nm}.admin_nodemap)
1180 local trust=$(do_facet mgs $LCTL get_param -n \
1181 nodemap.${nm}.trusted_nodemap)
1183 do_facet mgs $LCTL nodemap_modify --name $nm --property admin --value 1
1184 do_facet mgs $LCTL nodemap_modify --name $nm --property trusted \
1187 wait_nm_sync $nm admin_nodemap
1188 wait_nm_sync $nm trusted_nodemap
1190 # create directory and populate it for subdir mount
1191 do_node ${clients_arr[0]} mkdir $MOUNT/$subdir ||
1192 error "unable to create dir $MOUNT/$subdir"
1193 do_node ${clients_arr[0]} touch $MOUNT/$subdir/this_is_$subdir ||
1194 error "unable to create file $MOUNT/$subdir/this_is_$subdir"
1195 do_node ${clients_arr[0]} mkdir $MOUNT/$subdir/$subsubdir ||
1196 error "unable to create dir $MOUNT/$subdir/$subsubdir"
1197 do_node ${clients_arr[0]} touch \
1198 $MOUNT/$subdir/$subsubdir/this_is_$subsubdir ||
1199 error "unable to create file \
1200 $MOUNT/$subdir/$subsubdir/this_is_$subsubdir"
1202 do_facet mgs $LCTL nodemap_modify --name $nm \
1203 --property admin --value $admin
1204 do_facet mgs $LCTL nodemap_modify --name $nm \
1205 --property trusted --value $trust
1207 # flush MDT locks to make sure they are reacquired before test
1208 do_node ${clients_arr[0]} $LCTL set_param \
1209 ldlm.namespaces.$FSNAME-MDT*.lru_size=clear
1211 wait_nm_sync $nm admin_nodemap
1212 wait_nm_sync $nm trusted_nodemap
1215 # fileset test directory needs to be initialized on a privileged client
1216 fileset_test_cleanup() {
1218 local admin=$(do_facet mgs $LCTL get_param -n \
1219 nodemap.${nm}.admin_nodemap)
1220 local trust=$(do_facet mgs $LCTL get_param -n \
1221 nodemap.${nm}.trusted_nodemap)
1223 do_facet mgs $LCTL nodemap_modify --name $nm --property admin --value 1
1224 do_facet mgs $LCTL nodemap_modify --name $nm --property trusted \
1227 wait_nm_sync $nm admin_nodemap
1228 wait_nm_sync $nm trusted_nodemap
1230 # cleanup directory created for subdir mount
1231 do_node ${clients_arr[0]} rm -rf $MOUNT/$subdir ||
1232 error "unable to remove dir $MOUNT/$subdir"
1234 do_facet mgs $LCTL nodemap_modify --name $nm \
1235 --property admin --value $admin
1236 do_facet mgs $LCTL nodemap_modify --name $nm \
1237 --property trusted --value $trust
1239 # flush MDT locks to make sure they are reacquired before test
1240 do_node ${clients_arr[0]} $LCTL set_param \
1241 ldlm.namespaces.$FSNAME-MDT*.lru_size=clear
1243 wait_nm_sync $nm admin_nodemap
1244 wait_nm_sync $nm trusted_nodemap
1245 if [ -n "$FILESET" -a -z "$SKIP_FILESET" ]; then
1246 cleanup_mount $MOUNT
1247 zconf_mount_clients $CLIENTS $MOUNT
1251 do_create_delete() {
1254 local testfile=$DIR/$tdir/$tfile
1258 if $run_u touch $testfile >& /dev/null; then
1260 $run_u rm $testfile && d=1
1264 local expected=$(get_cr_del_expected $key)
1265 [ "$res" != "$expected" ] &&
1266 error "test $key, wanted $expected, got $res" && rc=$((rc + 1))
1270 nodemap_check_quota() {
1272 $run_u lfs quota -q $DIR | awk '{ print $2; exit; }'
1275 do_fops_quota_test() {
1277 # fuzz quota used to account for possible indirect blocks, etc
1278 local quota_fuzz=$(fs_log_size)
1279 local qused_orig=$(nodemap_check_quota "$run_u")
1280 local qused_high=$((qused_orig + quota_fuzz))
1281 local qused_low=$((qused_orig - quota_fuzz))
1282 local testfile=$DIR/$tdir/$tfile
1283 $run_u dd if=/dev/zero of=$testfile oflag=sync bs=1M count=1 \
1284 >& /dev/null || error "unable to write quota test file"
1285 sync; sync_all_data || true
1287 local qused_new=$(nodemap_check_quota "$run_u")
1288 [ $((qused_new)) -lt $((qused_low + 1024)) -o \
1289 $((qused_new)) -gt $((qused_high + 1024)) ] &&
1290 error "$qused_new != $qused_orig + 1M after write, " \
1291 "fuzz is $quota_fuzz"
1292 $run_u rm $testfile || error "unable to remove quota test file"
1293 wait_delete_completed_mds
1295 qused_new=$(nodemap_check_quota "$run_u")
1296 [ $((qused_new)) -lt $((qused_low)) \
1297 -o $((qused_new)) -gt $((qused_high)) ] &&
1298 error "quota not reclaimed, expect $qused_orig, " \
1299 "got $qused_new, fuzz $quota_fuzz"
1302 get_fops_mapped_user() {
1305 for ((i=0; i < ${#FOPS_IDMAPS[@]}; i++)); do
1306 for map in ${FOPS_IDMAPS[i]}; do
1307 if [ $(cut -d: -f1 <<< "$map") == $cli_user ]; then
1308 cut -d: -f2 <<< "$map"
1316 get_cr_del_expected() {
1318 IFS=":" read -a key <<< "$1"
1319 local mapmode="${key[0]}"
1320 local mds_user="${key[1]}"
1321 local cluster="${key[2]}"
1322 local cli_user="${key[3]}"
1323 local mode="0${key[4]}"
1330 [[ $mapmode == *mapped* ]] && mapped=1
1331 # only c1 is mapped in these test cases
1332 [[ $mapmode == mapped_trusted* ]] && [ "$cluster" == "c0" ] && mapped=0
1333 [[ $mapmode == *noadmin* ]] && noadmin=1
1335 # o+wx works as long as the user isn't mapped
1336 if [ $((mode & 3)) -eq 3 ]; then
1340 # if client user is root, check if root is squashed
1341 if [ "$cli_user" == "0" ]; then
1342 # squash root succeed, if other bit is on
1345 1) [ "$other" == "1" ] && echo $SUCCESS
1346 [ "$other" == "0" ] && echo $FAILURE;;
1350 if [ "$mapped" == "0" ]; then
1351 [ "$other" == "1" ] && echo $SUCCESS
1352 [ "$other" == "0" ] && echo $FAILURE
1356 # if mapped user is mds user, check for u+wx
1357 mapped_user=$(get_fops_mapped_user $cli_user)
1358 [ "$mapped_user" == "-1" ] &&
1359 error "unable to find mapping for client user $cli_user"
1361 if [ "$mapped_user" == "$mds_user" -a \
1362 $(((mode & 0300) == 0300)) -eq 1 ]; then
1366 if [ "$mapped_user" != "$mds_user" -a "$other" == "1" ]; then
1373 test_fops_admin_cli_i=""
1374 test_fops_chmod_dir() {
1375 local current_cli_i=$1
1377 local dir_to_chmod=$3
1378 local new_admin_cli_i=""
1380 # do we need to set up a new admin client?
1381 [ "$current_cli_i" == "0" ] && [ "$test_fops_admin_cli_i" != "1" ] &&
1383 [ "$current_cli_i" != "0" ] && [ "$test_fops_admin_cli_i" != "0" ] &&
1386 # if only one client, and non-admin, need to flip admin everytime
1387 if [ "$num_clients" == "1" ]; then
1388 test_fops_admin_client=$clients
1389 test_fops_admin_val=$(do_facet mgs $LCTL get_param -n \
1390 nodemap.c0.admin_nodemap)
1391 if [ "$test_fops_admin_val" != "1" ]; then
1392 do_facet mgs $LCTL nodemap_modify \
1396 wait_nm_sync c0 admin_nodemap
1398 elif [ "$new_admin_cli_i" != "" ]; then
1399 # restore admin val to old admin client
1400 if [ "$test_fops_admin_cli_i" != "" ] &&
1401 [ "$test_fops_admin_val" != "1" ]; then
1402 do_facet mgs $LCTL nodemap_modify \
1403 --name c${test_fops_admin_cli_i} \
1405 --value $test_fops_admin_val
1406 wait_nm_sync c${test_fops_admin_cli_i} admin_nodemap
1409 test_fops_admin_cli_i=$new_admin_cli_i
1410 test_fops_admin_client=${clients_arr[$new_admin_cli_i]}
1411 test_fops_admin_val=$(do_facet mgs $LCTL get_param -n \
1412 nodemap.c${new_admin_cli_i}.admin_nodemap)
1414 if [ "$test_fops_admin_val" != "1" ]; then
1415 do_facet mgs $LCTL nodemap_modify \
1416 --name c${new_admin_cli_i} \
1419 wait_nm_sync c${new_admin_cli_i} admin_nodemap
1423 do_node $test_fops_admin_client chmod $perm_bits $DIR/$tdir || return 1
1425 # remove admin for single client if originally non-admin
1426 if [ "$num_clients" == "1" ] && [ "$test_fops_admin_val" != "1" ]; then
1427 do_facet mgs $LCTL nodemap_modify --name c0 --property admin \
1429 wait_nm_sync c0 admin_nodemap
1437 local single_client="$2"
1438 local client_user_list=([0]="0 $((IDBASE+3))"
1439 [1]="0 $((IDBASE+5))")
1440 local mds_users="-1 0"
1443 local perm_bit_list="3 $((0300))"
1444 # SLOW tests 000-007, 010-070, 100-700 (octal modes)
1445 if [ "$SLOW" == "yes" ]; then
1446 perm_bit_list="0 $(seq 1 7) $(seq 8 8 63) $(seq 64 64 511) \
1448 client_user_list=([0]="0 $((IDBASE+3)) $((IDBASE+4))"
1449 [1]="0 $((IDBASE+5)) $((IDBASE+6))")
1450 mds_users="-1 0 1 2"
1453 # force single_client to speed up test
1454 [ "$SLOW" == "yes" ] ||
1456 # step through mds users. -1 means root
1457 for mds_i in $mds_users; do
1458 local user=$((mds_i + IDBASE))
1462 [ "$mds_i" == "-1" ] && user=0
1464 echo mkdir -p $DIR/$tdir
1467 for client in $clients; do
1469 for u in ${client_user_list[$cli_i]}; do
1470 local run_u="do_node $client \
1471 $RUNAS_CMD -u$u -g$u -G$u"
1472 for perm_bits in $perm_bit_list; do
1473 local mode=$(printf %03o $perm_bits)
1475 key="$mapmode:$user:c$cli_i:$u:$mode"
1476 test_fops_chmod_dir $cli_i $mode \
1478 error cannot chmod $key
1479 do_create_delete "$run_u" "$key"
1483 test_fops_chmod_dir $cli_i 777 $DIR/$tdir ||
1484 error cannot chmod $key
1485 do_fops_quota_test "$run_u"
1488 cli_i=$((cli_i + 1))
1489 [ "$single_client" == "1" ] && break
1496 nodemap_version_check () {
1497 remote_mgs_nodsh && skip "remote MGS with nodsh" && return 1
1498 [ "$MGS_VERSION" -lt $(version_code 2.5.53) ] &&
1499 skip "No nodemap on $MGS_VERSION MGS < 2.5.53" &&
1504 nodemap_test_setup() {
1506 local active_nodemap=1
1508 [ "$1" == "0" ] && active_nodemap=0
1510 do_nodes $(comma_list $(all_mdts_nodes)) \
1511 $LCTL set_param mdt.*.identity_upcall=NONE
1514 create_fops_nodemaps
1516 [[ $rc != 0 ]] && error "adding fops nodemaps failed $rc"
1518 do_facet mgs $LCTL nodemap_activate $active_nodemap
1521 do_facet mgs $LCTL nodemap_modify --name default \
1522 --property admin --value 1
1523 wait_nm_sync default admin_nodemap
1524 do_facet mgs $LCTL nodemap_modify --name default \
1525 --property trusted --value 1
1526 wait_nm_sync default trusted_nodemap
1529 nodemap_test_cleanup() {
1531 delete_fops_nodemaps
1533 [[ $rc != 0 ]] && error "removing fops nodemaps failed $rc"
1535 do_facet mgs $LCTL nodemap_modify --name default \
1536 --property admin --value 0
1537 wait_nm_sync default admin_nodemap
1538 do_facet mgs $LCTL nodemap_modify --name default \
1539 --property trusted --value 0
1540 wait_nm_sync default trusted_nodemap
1542 do_facet mgs $LCTL nodemap_activate 0
1543 wait_nm_sync active 0
1545 export SK_UNIQUE_NM=false
1549 nodemap_clients_admin_trusted() {
1553 for client in $clients; do
1554 do_facet mgs $LCTL nodemap_modify --name c0 \
1555 --property admin --value $admin
1556 do_facet mgs $LCTL nodemap_modify --name c0 \
1557 --property trusted --value $tr
1560 wait_nm_sync c$((i - 1)) admin_nodemap
1561 wait_nm_sync c$((i - 1)) trusted_nodemap
1565 nodemap_version_check || return 0
1566 nodemap_test_setup 0
1568 trap nodemap_test_cleanup EXIT
1570 nodemap_test_cleanup
1572 run_test 16 "test nodemap all_off fileops"
1576 [ "$MDS1_VERSION" -lt $(version_code 2.11.55) ]; then
1577 skip "Need MDS >= 2.11.55"
1579 local check_proj=true
1581 (( $MDS1_VERSION >= $(version_code 2.14.52) )) || check_proj=false
1583 nodemap_version_check || return 0
1586 trap nodemap_test_cleanup EXIT
1587 nodemap_clients_admin_trusted 0 1
1588 test_fops trusted_noadmin 1
1589 if $check_proj; then
1590 do_facet mgs $LCTL nodemap_modify --name c0 \
1591 --property map_mode --value projid
1592 wait_nm_sync c0 map_mode
1594 test_fops trusted_noadmin 1
1595 nodemap_test_cleanup
1597 run_test 17 "test nodemap trusted_noadmin fileops"
1601 [ "$MDS1_VERSION" -lt $(version_code 2.11.55) ]; then
1602 skip "Need MDS >= 2.11.55"
1605 nodemap_version_check || return 0
1608 trap nodemap_test_cleanup EXIT
1609 nodemap_clients_admin_trusted 0 0
1610 test_fops mapped_noadmin 1
1611 nodemap_test_cleanup
1613 run_test 18 "test nodemap mapped_noadmin fileops"
1617 [ "$MDS1_VERSION" -lt $(version_code 2.11.55) ]; then
1618 skip "Need MDS >= 2.11.55"
1621 nodemap_version_check || return 0
1624 trap nodemap_test_cleanup EXIT
1625 nodemap_clients_admin_trusted 1 1
1626 test_fops trusted_admin 1
1627 nodemap_test_cleanup
1629 run_test 19 "test nodemap trusted_admin fileops"
1633 [ "$MDS1_VERSION" -lt $(version_code 2.11.55) ]; then
1634 skip "Need MDS >= 2.11.55"
1637 nodemap_version_check || return 0
1640 trap nodemap_test_cleanup EXIT
1641 nodemap_clients_admin_trusted 1 0
1642 test_fops mapped_admin 1
1643 nodemap_test_cleanup
1645 run_test 20 "test nodemap mapped_admin fileops"
1649 [ "$MDS1_VERSION" -lt $(version_code 2.11.55) ]; then
1650 skip "Need MDS >= 2.11.55"
1653 nodemap_version_check || return 0
1656 trap nodemap_test_cleanup EXIT
1659 for client in $clients; do
1660 do_facet mgs $LCTL nodemap_modify --name c${i} \
1661 --property admin --value 0
1662 do_facet mgs $LCTL nodemap_modify --name c${i} \
1663 --property trusted --value $x
1667 wait_nm_sync c$((i - 1)) trusted_nodemap
1669 test_fops mapped_trusted_noadmin
1670 nodemap_test_cleanup
1672 run_test 21 "test nodemap mapped_trusted_noadmin fileops"
1676 [ "$MDS1_VERSION" -lt $(version_code 2.11.55) ]; then
1677 skip "Need MDS >= 2.11.55"
1680 nodemap_version_check || return 0
1683 trap nodemap_test_cleanup EXIT
1686 for client in $clients; do
1687 do_facet mgs $LCTL nodemap_modify --name c${i} \
1688 --property admin --value 1
1689 do_facet mgs $LCTL nodemap_modify --name c${i} \
1690 --property trusted --value $x
1694 wait_nm_sync c$((i - 1)) trusted_nodemap
1696 test_fops mapped_trusted_admin
1697 nodemap_test_cleanup
1699 run_test 22 "test nodemap mapped_trusted_admin fileops"
1701 # acl test directory needs to be initialized on a privileged client
1702 nodemap_acl_test_setup() {
1703 local admin=$(do_facet mgs $LCTL get_param -n \
1704 nodemap.c0.admin_nodemap)
1705 local trust=$(do_facet mgs $LCTL get_param -n \
1706 nodemap.c0.trusted_nodemap)
1708 do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 1
1709 do_facet mgs $LCTL nodemap_modify --name c0 --property trusted --value 1
1711 wait_nm_sync c0 admin_nodemap
1712 wait_nm_sync c0 trusted_nodemap
1714 do_node ${clients_arr[0]} rm -rf $DIR/$tdir
1716 do_node ${clients_arr[0]} chmod a+rwx $DIR/$tdir ||
1717 error unable to chmod a+rwx test dir $DIR/$tdir
1719 do_facet mgs $LCTL nodemap_modify --name c0 \
1720 --property admin --value $admin
1721 do_facet mgs $LCTL nodemap_modify --name c0 \
1722 --property trusted --value $trust
1724 wait_nm_sync c0 trusted_nodemap
1727 # returns 0 if the number of ACLs does not change on the second (mapped) client
1728 # after being set on the first client
1729 nodemap_acl_test() {
1731 local set_client="$2"
1732 local get_client="$3"
1733 local check_setfacl="$4"
1734 local setfacl_error=0
1735 local testfile=$DIR/$tdir/$tfile
1736 local RUNAS_USER="$RUNAS_CMD -u $user"
1738 local acl_count_post=0
1740 nodemap_acl_test_setup
1743 do_node $set_client $RUNAS_USER touch $testfile
1745 # ACL masks aren't filtered by nodemap code, so we ignore them
1746 acl_count=$(do_node $get_client getfacl $testfile | grep -v mask |
1748 do_node $set_client $RUNAS_USER setfacl -m $user:rwx $testfile ||
1751 # if check setfacl is set to 1, then it's supposed to error
1752 if [ "$check_setfacl" == "1" ]; then
1753 [ "$setfacl_error" != "1" ] && return 1
1756 [ "$setfacl_error" == "1" ] && echo "WARNING: unable to setfacl"
1758 acl_count_post=$(do_node $get_client getfacl $testfile | grep -v mask |
1760 [ $acl_count -eq $acl_count_post ] && return 0
1765 [ $num_clients -lt 2 ] && skip "Need 2 clients at least" && return
1766 nodemap_version_check || return 0
1769 trap nodemap_test_cleanup EXIT
1770 # 1 trusted cluster, 1 mapped cluster
1771 local unmapped_fs=$((IDBASE+0))
1772 local unmapped_c1=$((IDBASE+5))
1773 local mapped_fs=$((IDBASE+2))
1774 local mapped_c0=$((IDBASE+4))
1775 local mapped_c1=$((IDBASE+6))
1777 do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 1
1778 do_facet mgs $LCTL nodemap_modify --name c0 --property trusted --value 1
1780 do_facet mgs $LCTL nodemap_modify --name c1 --property admin --value 0
1781 do_facet mgs $LCTL nodemap_modify --name c1 --property trusted --value 0
1783 wait_nm_sync c1 trusted_nodemap
1785 # setfacl on trusted cluster to unmapped user, verify it's not seen
1786 nodemap_acl_test $unmapped_fs ${clients_arr[0]} ${clients_arr[1]} ||
1787 error "acl count (1)"
1789 # setfacl on trusted cluster to mapped user, verify it's seen
1790 nodemap_acl_test $mapped_fs ${clients_arr[0]} ${clients_arr[1]} &&
1791 error "acl count (2)"
1793 # setfacl on mapped cluster to mapped user, verify it's seen
1794 nodemap_acl_test $mapped_c1 ${clients_arr[1]} ${clients_arr[0]} &&
1795 error "acl count (3)"
1797 # setfacl on mapped cluster to unmapped user, verify error
1798 nodemap_acl_test $unmapped_fs ${clients_arr[1]} ${clients_arr[0]} 1 ||
1799 error "acl count (4)"
1802 do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 0
1803 do_facet mgs $LCTL nodemap_modify --name c0 --property trusted --value 0
1805 wait_nm_sync c0 trusted_nodemap
1807 # setfacl to mapped user on c1, also mapped to c0, verify it's seen
1808 nodemap_acl_test $mapped_c1 ${clients_arr[1]} ${clients_arr[0]} &&
1809 error "acl count (5)"
1811 # setfacl to mapped user on c1, not mapped to c0, verify not seen
1812 nodemap_acl_test $unmapped_c1 ${clients_arr[1]} ${clients_arr[0]} ||
1813 error "acl count (6)"
1815 nodemap_test_cleanup
1817 run_test 23a "test mapped regular ACLs"
1819 test_23b() { #LU-9929
1820 [ $num_clients -lt 2 ] && skip "Need 2 clients at least"
1821 [ "$MGS_VERSION" -lt $(version_code 2.10.53) ] &&
1822 skip "Need MGS >= 2.10.53"
1824 export SK_UNIQUE_NM=true
1826 trap nodemap_test_cleanup EXIT
1828 local testdir=$DIR/$tdir
1829 local fs_id=$((IDBASE+10))
1834 do_facet mgs $LCTL nodemap_modify --name c0 --property admin --value 1
1835 wait_nm_sync c0 admin_nodemap
1836 do_facet mgs $LCTL nodemap_modify --name c1 --property admin --value 1
1837 wait_nm_sync c1 admin_nodemap
1838 do_facet mgs $LCTL nodemap_modify --name c1 --property trusted --value 1
1839 wait_nm_sync c1 trusted_nodemap
1841 # Add idmap $ID0:$fs_id (500:60010)
1842 do_facet mgs $LCTL nodemap_add_idmap --name c0 --idtype gid \
1843 --idmap $ID0:$fs_id ||
1844 error "add idmap $ID0:$fs_id to nodemap c0 failed"
1845 wait_nm_sync c0 idmap
1847 # set/getfacl default acl on client 1 (unmapped gid=500)
1848 do_node ${clients_arr[0]} rm -rf $testdir
1849 do_node ${clients_arr[0]} mkdir -p $testdir
1850 # Here, USER0=$(getent passwd | grep :$ID0:$ID0: | cut -d: -f1)
1851 do_node ${clients_arr[0]} setfacl -R -d -m group:$USER0:rwx $testdir ||
1852 error "setfacl $testdir on ${clients_arr[0]} failed"
1853 unmapped_id=$(do_node ${clients_arr[0]} getfacl $testdir |
1854 grep -E "default:group:.*:rwx" | awk -F: '{print $3}')
1855 [ "$unmapped_id" = "$USER0" ] ||
1856 error "gid=$ID0 was not unmapped correctly on ${clients_arr[0]}"
1858 # getfacl default acl on client 2 (mapped gid=60010)
1859 mapped_id=$(do_node ${clients_arr[1]} getfacl $testdir |
1860 grep -E "default:group:.*:rwx" | awk -F: '{print $3}')
1861 fs_user=$(do_node ${clients_arr[1]} getent passwd |
1862 grep :$fs_id:$fs_id: | cut -d: -f1)
1863 [ -z "$fs_user" ] && fs_user=$fs_id
1864 [ $mapped_id -eq $fs_id -o "$mapped_id" = "$fs_user" ] ||
1865 error "Should return gid=$fs_id or $fs_user on client2"
1868 nodemap_test_cleanup
1869 export SK_UNIQUE_NM=false
1871 run_test 23b "test mapped default ACLs"
1876 trap nodemap_test_cleanup EXIT
1877 do_nodes $(comma_list $(all_server_nodes)) $LCTL get_param -R nodemap
1879 nodemap_test_cleanup
1881 run_test 24 "check nodemap proc files for LBUGs and Oopses"
1884 local tmpfile=$(mktemp)
1885 local tmpfile2=$(mktemp)
1886 local tmpfile3=$(mktemp)
1887 local tmpfile4=$(mktemp)
1891 nodemap_version_check || return 0
1893 # stop clients for this test
1894 zconf_umount_clients $CLIENTS $MOUNT ||
1895 error "unable to umount clients $CLIENTS"
1897 export SK_UNIQUE_NM=true
1900 # enable trusted/admin for setquota call in cleanup_and_setup_lustre()
1902 for client in $clients; do
1903 do_facet mgs $LCTL nodemap_modify --name c${i} \
1904 --property admin --value 1
1905 do_facet mgs $LCTL nodemap_modify --name c${i} \
1906 --property trusted --value 1
1909 wait_nm_sync c$((i - 1)) trusted_nodemap
1911 trap nodemap_test_cleanup EXIT
1913 # create a new, empty nodemap, and add fileset info to it
1914 do_facet mgs $LCTL nodemap_add test25 ||
1915 error "unable to create nodemap $testname"
1916 do_facet mgs $LCTL set_param -P nodemap.$testname.fileset=/$subdir ||
1917 error "unable to add fileset info to nodemap test25"
1919 wait_nm_sync test25 id
1921 do_facet mgs $LCTL nodemap_info > $tmpfile
1922 do_facet mds $LCTL nodemap_info > $tmpfile2
1924 if ! $SHARED_KEY; then
1925 # will conflict with SK's nodemaps
1926 cleanup_and_setup_lustre
1928 # stop clients for this test
1929 zconf_umount_clients $CLIENTS $MOUNT ||
1930 error "unable to umount clients $CLIENTS"
1932 do_facet mgs $LCTL nodemap_info > $tmpfile3
1933 diff -q $tmpfile3 $tmpfile >& /dev/null ||
1934 error "nodemap_info diff on MGS after remount"
1936 do_facet mds $LCTL nodemap_info > $tmpfile4
1937 diff -q $tmpfile4 $tmpfile2 >& /dev/null ||
1938 error "nodemap_info diff on MDS after remount"
1941 do_facet mgs $LCTL nodemap_del test25 ||
1942 error "cannot delete nodemap test25 from config"
1943 nodemap_test_cleanup
1944 # restart clients previously stopped
1945 zconf_mount_clients $CLIENTS $MOUNT ||
1946 error "unable to mount clients $CLIENTS"
1948 rm -f $tmpfile $tmpfile2
1949 export SK_UNIQUE_NM=false
1951 run_test 25 "test save and reload nodemap config"
1954 nodemap_version_check || return 0
1958 do_facet mgs "seq -f 'c%g' $large_i | xargs -n1 $LCTL nodemap_add"
1959 wait_nm_sync c$large_i admin_nodemap
1961 do_facet mgs "seq -f 'c%g' $large_i | xargs -n1 $LCTL nodemap_del"
1962 wait_nm_sync c$large_i admin_nodemap
1964 run_test 26 "test transferring very large nodemap"
1966 nodemap_exercise_fileset() {
1969 local check_proj=true
1971 (( $MDS1_VERSION >= $(version_code 2.14.52) )) || check_proj=false
1974 if [ "$nm" == "default" ]; then
1975 do_facet mgs $LCTL nodemap_activate 1
1977 do_facet mgs $LCTL nodemap_modify --name default \
1978 --property admin --value 1
1979 do_facet mgs $LCTL nodemap_modify --name default \
1980 --property trusted --value 1
1981 wait_nm_sync default admin_nodemap
1982 wait_nm_sync default trusted_nodemap
1987 if $SHARED_KEY; then
1988 export SK_UNIQUE_NM=true
1990 # will conflict with SK's nodemaps
1991 trap "fileset_test_cleanup $nm" EXIT
1993 fileset_test_setup "$nm"
1995 # add fileset info to $nm nodemap
1996 if ! combined_mgs_mds; then
1997 do_facet mgs $LCTL set_param nodemap.${nm}.fileset=/$subdir ||
1998 error "unable to add fileset info to $nm nodemap on MGS"
2000 do_facet mgs $LCTL set_param -P nodemap.${nm}.fileset=/$subdir ||
2001 error "unable to add fileset info to $nm nodemap for servers"
2002 wait_nm_sync $nm fileset "nodemap.${nm}.fileset=/$subdir"
2004 if $check_proj; then
2005 do_facet mgs $LCTL nodemap_modify --name $nm \
2006 --property admin --value 1
2007 wait_nm_sync $nm admin_nodemap
2008 do_facet mgs $LCTL nodemap_modify --name $nm \
2009 --property trusted --value 0
2010 wait_nm_sync $nm trusted_nodemap
2011 do_facet mgs $LCTL nodemap_modify --name $nm \
2012 --property map_mode --value projid
2013 wait_nm_sync $nm map_mode
2014 do_facet mgs $LCTL nodemap_add_idmap --name $nm \
2015 --idtype projid --idmap 1:1
2016 do_facet mgs $LCTL nodemap_modify --name $nm \
2017 --property deny_unknown --value 1
2018 wait_nm_sync $nm deny_unknown
2022 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2023 error "unable to umount client ${clients_arr[0]}"
2024 # set some generic fileset to trigger SSK code
2026 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2027 error "unable to remount client ${clients_arr[0]}"
2030 # test mount point content
2031 do_node ${clients_arr[0]} test -f $MOUNT/this_is_$subdir ||
2032 error "fileset not taken into account"
2034 if $check_proj; then
2035 do_node ${clients_arr[0]} $LFS setquota -p 1 -b 10000 -B 11000 \
2036 -i 0 -I 0 $MOUNT || error "setquota -p 1 failed"
2037 do_node ${clients_arr[0]} $LFS setquota -p 2 -b 10000 -B 11000 \
2038 -i 0 -I 0 $MOUNT && error "setquota -p 2 should fail"
2041 # re-mount client with sub-subdir
2042 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2043 error "unable to umount client ${clients_arr[0]}"
2044 export FILESET=/$subsubdir
2045 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2046 error "unable to remount client ${clients_arr[0]}"
2049 # test mount point content
2050 do_node ${clients_arr[0]} test -f $MOUNT/this_is_$subsubdir ||
2051 error "subdir of fileset not taken into account"
2053 # remove fileset info from nodemap
2054 do_facet mgs $LCTL nodemap_set_fileset --name $nm --fileset clear ||
2055 error "unable to delete fileset info on $nm nodemap"
2056 wait_update_facet mgs "$LCTL get_param nodemap.${nm}.fileset" \
2057 "nodemap.${nm}.fileset=" ||
2058 error "fileset info still not cleared on $nm nodemap"
2059 do_facet mgs $LCTL set_param -P nodemap.${nm}.fileset=clear ||
2060 error "unable to reset fileset info on $nm nodemap"
2061 wait_nm_sync $nm fileset "nodemap.${nm}.fileset="
2064 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2065 error "unable to umount client ${clients_arr[0]}"
2066 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2067 error "unable to remount client ${clients_arr[0]}"
2069 # test mount point content
2070 if ! $(do_node ${clients_arr[0]} test -d $MOUNT/$subdir); then
2072 error "fileset not cleared on $nm nodemap"
2075 # back to non-nodemap setup
2076 if $SHARED_KEY; then
2077 export SK_UNIQUE_NM=false
2078 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2079 error "unable to umount client ${clients_arr[0]}"
2081 fileset_test_cleanup "$nm"
2082 if [ "$nm" == "default" ]; then
2083 do_facet mgs $LCTL nodemap_modify --name default \
2084 --property admin --value 0
2085 do_facet mgs $LCTL nodemap_modify --name default \
2086 --property trusted --value 0
2087 wait_nm_sync default admin_nodemap
2088 wait_nm_sync default trusted_nodemap
2089 do_facet mgs $LCTL nodemap_activate 0
2090 wait_nm_sync active 0
2092 export SK_UNIQUE_NM=false
2094 nodemap_test_cleanup
2096 if $SHARED_KEY; then
2097 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2098 error "unable to remount client ${clients_arr[0]}"
2103 [ "$MDS1_VERSION" -lt $(version_code 2.11.50) ] &&
2104 skip "Need MDS >= 2.11.50"
2106 for nm in "default" "c0"; do
2107 local subdir="subdir_${nm}"
2108 local subsubdir="subsubdir_${nm}"
2110 if [ "$nm" == "default" ] && [ "$SHARED_KEY" == "true" ]; then
2111 echo "Skipping nodemap $nm with SHARED_KEY";
2115 echo "Exercising fileset for nodemap $nm"
2116 nodemap_exercise_fileset "$nm"
2119 run_test 27a "test fileset in various nodemaps"
2121 test_27b() { #LU-10703
2122 [ "$MDS1_VERSION" -lt $(version_code 2.11.50) ] &&
2123 skip "Need MDS >= 2.11.50"
2124 [[ $MDSCOUNT -lt 2 ]] && skip "needs >= 2 MDTs"
2127 trap nodemap_test_cleanup EXIT
2129 # Add the nodemaps and set their filesets
2130 for i in $(seq 1 $MDSCOUNT); do
2131 do_facet mgs $LCTL nodemap_del nm$i 2>/dev/null
2132 do_facet mgs $LCTL nodemap_add nm$i ||
2133 error "add nodemap nm$i failed"
2134 wait_nm_sync nm$i "" "" "-N"
2136 if ! combined_mgs_mds; then
2138 $LCTL set_param nodemap.nm$i.fileset=/dir$i ||
2139 error "set nm$i.fileset=/dir$i failed on MGS"
2141 do_facet mgs $LCTL set_param -P nodemap.nm$i.fileset=/dir$i ||
2142 error "set nm$i.fileset=/dir$i failed on servers"
2143 wait_nm_sync nm$i fileset "nodemap.nm$i.fileset=/dir$i"
2146 # Check if all the filesets are correct
2147 for i in $(seq 1 $MDSCOUNT); do
2148 fileset=$(do_facet mds$i \
2149 $LCTL get_param -n nodemap.nm$i.fileset)
2150 [ "$fileset" = "/dir$i" ] ||
2151 error "nm$i.fileset $fileset != /dir$i on mds$i"
2152 do_facet mgs $LCTL nodemap_del nm$i ||
2153 error "delete nodemap nm$i failed"
2156 nodemap_test_cleanup
2158 run_test 27b "The new nodemap won't clear the old nodemap's fileset"
2161 if ! $SHARED_KEY; then
2162 skip "need shared key feature for this test" && return
2164 mkdir -p $DIR/$tdir || error "mkdir failed"
2165 touch $DIR/$tdir/$tdir.out || error "touch failed"
2166 if [ ! -f $DIR/$tdir/$tdir.out ]; then
2167 error "read before rotation failed"
2169 # store top key identity to ensure rotation has occurred
2170 SK_IDENTITY_OLD=$(lctl get_param *.*.*srpc* | grep "expire" |
2171 head -1 | awk '{print $15}' | cut -c1-8)
2172 do_facet $SINGLEMDS lfs flushctx ||
2173 error "could not run flushctx on $SINGLEMDS"
2175 lfs flushctx || error "could not run flushctx on client"
2177 # verify new key is in place
2178 SK_IDENTITY_NEW=$(lctl get_param *.*.*srpc* | grep "expire" |
2179 head -1 | awk '{print $15}' | cut -c1-8)
2180 if [ $SK_IDENTITY_OLD == $SK_IDENTITY_NEW ]; then
2181 error "key did not rotate correctly"
2183 if [ ! -f $DIR/$tdir/$tdir.out ]; then
2184 error "read after rotation failed"
2187 run_test 28 "check shared key rotation method"
2190 if ! $SHARED_KEY; then
2191 skip "need shared key feature for this test" && return
2193 if [ $SK_FLAVOR != "ski" ] && [ $SK_FLAVOR != "skpi" ]; then
2194 skip "test only valid if integrity is active"
2197 mkdir $DIR/$tdir || error "mkdir"
2198 touch $DIR/$tdir/$tfile || error "touch"
2199 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2200 error "unable to umount clients"
2201 do_node ${clients_arr[0]} "keyctl show |
2202 awk '/lustre/ { print \\\$1 }' | xargs -IX keyctl unlink X"
2203 OLD_SK_PATH=$SK_PATH
2204 export SK_PATH=/dev/null
2205 if zconf_mount_clients ${clients_arr[0]} $MOUNT; then
2206 export SK_PATH=$OLD_SK_PATH
2207 do_node ${clients_arr[0]} "ls $DIR/$tdir/$tfile"
2208 if [ $? -eq 0 ]; then
2209 error "able to mount and read without key"
2211 error "able to mount without key"
2214 export SK_PATH=$OLD_SK_PATH
2215 do_node ${clients_arr[0]} "keyctl show |
2216 awk '/lustre/ { print \\\$1 }' |
2217 xargs -IX keyctl unlink X"
2219 zconf_mount_clients ${clients_arr[0]} $MOUNT ||
2220 error "unable to mount clients"
2222 run_test 29 "check for missing shared key"
2225 if ! $SHARED_KEY; then
2226 skip "need shared key feature for this test" && return
2228 if [ $SK_FLAVOR != "ski" ] && [ $SK_FLAVOR != "skpi" ]; then
2229 skip "test only valid if integrity is active"
2231 mkdir -p $DIR/$tdir || error "mkdir failed"
2232 touch $DIR/$tdir/$tdir.out || error "touch failed"
2233 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2234 error "unable to umount clients"
2235 # unload keys from ring
2236 do_node ${clients_arr[0]} "keyctl show |
2237 awk '/lustre/ { print \\\$1 }' | xargs -IX keyctl unlink X"
2238 # generate key with bogus filesystem name
2239 do_node ${clients_arr[0]} "lgss_sk -w $SK_PATH/$FSNAME-bogus.key \
2240 -f $FSNAME.bogus -t client -d /dev/urandom" ||
2241 error "lgss_sk failed (1)"
2242 do_facet $SINGLEMDS lfs flushctx || error "could not run flushctx"
2243 OLD_SK_PATH=$SK_PATH
2244 export SK_PATH=$SK_PATH/$FSNAME-bogus.key
2245 if zconf_mount_clients ${clients_arr[0]} $MOUNT; then
2246 SK_PATH=$OLD_SK_PATH
2247 do_node ${clients_arr[0]} "ls $DIR/$tdir/$tdir.out"
2248 if [ $? -eq 0 ]; then
2249 error "mount and read file with invalid key"
2251 error "mount with invalid key"
2254 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2255 error "unable to umount clients"
2256 # unload keys from ring
2257 do_node ${clients_arr[0]} "keyctl show |
2258 awk '/lustre/ { print \\\$1 }' | xargs -IX keyctl unlink X"
2260 SK_PATH=$OLD_SK_PATH
2261 zconf_mount_clients ${clients_arr[0]} $MOUNT ||
2262 error "unable to mount clients"
2264 run_test 30 "check for invalid shared key"
2269 mkdir -p $DIR/$tdir || error "mkdir $flvr"
2270 touch $DIR/$tdir/f0 || error "touch $flvr"
2271 ls $DIR/$tdir || error "ls $flvr"
2272 dd if=/dev/zero of=$DIR/$tdir/f0 conv=fsync bs=1M count=10 \
2273 >& /dev/null || error "dd $flvr"
2274 rm -f $DIR/$tdir/f0 || error "rm $flvr"
2275 rmdir $DIR/$tdir || error "rmdir $flvr"
2278 echo 3 > /proc/sys/vm/drop_caches
2282 local save_flvr=$SK_FLAVOR
2284 if ! $SHARED_KEY; then
2285 skip "need shared key feature for this test"
2288 stack_trap restore_to_default_flavor EXIT
2290 for flvr in skn ska ski skpi; do
2293 restore_to_default_flavor || error "cannot set $flvr flavor"
2294 SK_FLAVOR=$save_flvr
2299 run_test 30b "basic test of all different SSK flavors"
2303 zconf_umount $HOSTNAME $MOUNT || error "unable to umount client"
2305 # remove ${NETTYPE}999 network on all nodes
2306 do_nodes $(comma_list $(all_nodes)) \
2307 "$LNETCTL net del --net ${NETTYPE}999 && \
2308 $LNETCTL lnet unconfigure 2>/dev/null || true"
2310 # necessary to do writeconf in order to de-register
2311 # @${NETTYPE}999 nid for targets
2313 export KEEP_ZPOOL="true"
2315 export SK_MOUNTED=false
2318 export KEEP_ZPOOL="$KZPOOL"
2322 local nid=$(lctl list_nids | grep ${NETTYPE} | head -n1)
2323 local addr=${nid%@*}
2325 local net2=${NETTYPE}999
2326 local mdsnid=$(do_facet mds1 $LCTL list_nids | head -1)
2327 local addr1=${mdsnid%@*}
2328 local addr2=${addr1%.*}.$(((${addr1##*.} + 11) % 256))
2330 export LNETCTL=$(which lnetctl 2> /dev/null)
2332 [ -z "$LNETCTL" ] && skip "without lnetctl support." && return
2333 local_mode && skip "in local mode."
2335 stack_trap cleanup_31 EXIT
2338 if [ "$MOUNT_2" ] && $(grep -q $MOUNT2' ' /proc/mounts); then
2339 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
2341 if $(grep -q $MOUNT' ' /proc/mounts); then
2342 umount_client $MOUNT || error "umount $MOUNT failed"
2345 # check exports on servers are empty for client
2346 do_facet mgs "lctl get_param -n *.MGS*.exports.'$nid'.uuid 2>/dev/null |
2347 grep -q -" && error "export on MGS should be empty"
2348 do_nodes $(comma_list $(mdts_nodes) $(osts_nodes)) \
2349 "lctl get_param -n *.${FSNAME}*.exports.'$nid'.uuid \
2350 2>/dev/null | grep -q -" &&
2351 error "export on servers should be empty"
2353 # add network $net2 on all nodes
2354 do_nodes $(comma_list $(all_nodes)) \
2355 "$LNETCTL lnet configure && $LNETCTL net add --if \
2356 \$($LNETCTL net show --net $net | awk 'BEGIN{inf=0} \
2357 {if (inf==1) print \$2; fi; inf=0} /interfaces/{inf=1}') \
2359 error "unable to configure NID $net2"
2361 # necessary to do writeconf in order to register
2362 # new @$net2 nid for targets
2364 export KEEP_ZPOOL="true"
2366 export SK_MOUNTED=false
2369 nids="${addr1}@$net,${addr1}@$net2:${addr2}@$net,${addr2}@$net2"
2370 do_facet mds1 "$TUNEFS --servicenode="$nids" $(mdsdevname 1)" ||
2371 error "tunefs failed"
2373 setupall server_only || echo 1
2374 export KEEP_ZPOOL="$KZPOOL"
2377 local mgsnid_orig=$MGSNID
2378 # compute new MGSNID
2379 MGSNID=$(do_facet mgs "$LCTL list_nids | grep $net2")
2381 # on client, turn LNet Dynamic Discovery on
2382 lnetctl set discovery 1
2384 # mount client with -o network=$net2 option:
2385 # should fail because of LNet Dynamic Discovery
2386 mount_client $MOUNT ${MOUNT_OPTS},network=$net2 &&
2387 error "client mount with '-o network' option should be refused"
2389 # on client, reconfigure LNet and turn LNet Dynamic Discovery off
2390 $LNETCTL net del --net $net2 && lnetctl lnet unconfigure
2393 lnetctl set discovery 0
2395 $LNETCTL lnet configure && $LNETCTL net add --if \
2396 $($LNETCTL net show --net $net | awk 'BEGIN{inf=0} \
2397 {if (inf==1) print $2; fi; inf=0} /interfaces/{inf=1}') \
2399 error "unable to configure NID $net2 on client"
2401 # mount client with -o network=$net2 option
2402 mount_client $MOUNT ${MOUNT_OPTS},network=$net2 ||
2403 error "unable to remount client"
2408 # check export on MGS
2409 do_facet mgs "lctl get_param -n *.MGS*.exports.'$nid'.uuid 2>/dev/null |
2411 [ $? -ne 0 ] || error "export for $nid on MGS should not exist"
2414 "lctl get_param -n *.MGS*.exports.'${addr}@$net2'.uuid \
2415 2>/dev/null | grep -"
2417 error "export for ${addr}@$net2 on MGS should exist"
2419 # check {mdc,osc} imports
2420 lctl get_param mdc.${FSNAME}-*.import | grep current_connection |
2423 error "import for mdc should use ${addr1}@$net2"
2424 lctl get_param osc.${FSNAME}-*.import | grep current_connection |
2427 error "import for osc should use ${addr1}@$net2"
2429 # no NIDs on other networks should be listed
2430 lctl get_param mdc.${FSNAME}-*.import | grep failover_nids |
2431 grep -w ".*@$net" &&
2432 error "MDC import shouldn't have failnids at @$net"
2434 # failover NIDs on net999 should be listed
2435 lctl get_param mdc.${FSNAME}-*.import | grep failover_nids |
2436 grep ${addr2}@$net2 ||
2437 error "MDC import should have failnid ${addr2}@$net2"
2439 run_test 31 "client mount option '-o network'"
2443 zconf_umount_clients ${clients_arr[0]} $MOUNT
2445 # disable sk flavor enforcement on MGS
2446 set_rule _mgs any any null
2448 # stop gss daemon on MGS
2449 if ! combined_mgs_mds ; then
2450 send_sigint $mgs_HOST lsvcgssd
2454 MOUNT_OPTS=$(add_sk_mntflag $MOUNT_OPTS)
2457 restore_to_default_flavor
2461 if ! $SHARED_KEY; then
2462 skip "need shared key feature for this test"
2465 stack_trap cleanup_32 EXIT
2467 # restore to default null flavor
2468 save_flvr=$SK_FLAVOR
2470 restore_to_default_flavor || error "cannot set null flavor"
2471 SK_FLAVOR=$save_flvr
2474 if [ "$MOUNT_2" ] && $(grep -q $MOUNT2' ' /proc/mounts); then
2475 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
2477 if $(grep -q $MOUNT' ' /proc/mounts); then
2478 umount_client $MOUNT || error "umount $MOUNT failed"
2481 # start gss daemon on MGS
2482 if combined_mgs_mds ; then
2483 send_sigint $mds_HOST lsvcgssd
2485 start_gss_daemons $mgs_HOST "$LSVCGSSD -vvv -s -g"
2487 # add mgs key type and MGS NIDs in key on MGS
2488 do_nodes $mgs_HOST "lgss_sk -t mgs,server -g $MGSNID -m \
2489 $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2490 error "could not modify keyfile on MGS"
2492 # load modified key file on MGS
2493 do_nodes $mgs_HOST "lgss_sk -l $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2494 error "could not load keyfile on MGS"
2496 # add MGS NIDs in key on client
2497 do_nodes ${clients_arr[0]} "lgss_sk -g $MGSNID -m \
2498 $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2499 error "could not modify keyfile on MGS"
2501 # set perms for per-nodemap keys else permission denied
2502 do_nodes $(comma_list $(all_nodes)) \
2503 "keyctl show | grep lustre | cut -c1-11 |
2505 xargs -IX keyctl setperm X 0x3f3f3f3f"
2507 # re-mount client with mgssec=skn
2508 save_opts=$MOUNT_OPTS
2509 if [ -z "$MOUNT_OPTS" ]; then
2510 MOUNT_OPTS="-o mgssec=skn"
2512 MOUNT_OPTS="$MOUNT_OPTS,mgssec=skn"
2514 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2515 error "mount ${clients_arr[0]} with mgssec=skn failed"
2516 MOUNT_OPTS=$save_opts
2519 zconf_umount_clients ${clients_arr[0]} $MOUNT ||
2520 error "umount ${clients_arr[0]} failed"
2522 # enforce ska flavor on MGS
2523 set_rule _mgs any any ska
2525 # re-mount client without mgssec
2526 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS &&
2527 error "mount ${clients_arr[0]} without mgssec should fail"
2529 # re-mount client with mgssec=skn
2530 save_opts=$MOUNT_OPTS
2531 if [ -z "$MOUNT_OPTS" ]; then
2532 MOUNT_OPTS="-o mgssec=skn"
2534 MOUNT_OPTS="$MOUNT_OPTS,mgssec=skn"
2536 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS &&
2537 error "mount ${clients_arr[0]} with mgssec=skn should fail"
2538 MOUNT_OPTS=$save_opts
2540 # re-mount client with mgssec=ska
2541 save_opts=$MOUNT_OPTS
2542 if [ -z "$MOUNT_OPTS" ]; then
2543 MOUNT_OPTS="-o mgssec=ska"
2545 MOUNT_OPTS="$MOUNT_OPTS,mgssec=ska"
2547 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2548 error "mount ${clients_arr[0]} with mgssec=ska failed"
2549 MOUNT_OPTS=$save_opts
2553 run_test 32 "check for mgssec"
2556 # disable sk flavor enforcement
2557 set_rule $FSNAME any cli2mdt null
2558 wait_flavor cli2mdt null
2561 zconf_umount_clients ${clients_arr[0]} $MOUNT
2563 # stop gss daemon on MGS
2564 if ! combined_mgs_mds ; then
2565 send_sigint $mgs_HOST lsvcgssd
2569 MOUNT_OPTS=$(add_sk_mntflag $MOUNT_OPTS)
2572 restore_to_default_flavor
2576 if ! $SHARED_KEY; then
2577 skip "need shared key feature for this test"
2580 stack_trap cleanup_33 EXIT
2582 # restore to default null flavor
2583 save_flvr=$SK_FLAVOR
2585 restore_to_default_flavor || error "cannot set null flavor"
2586 SK_FLAVOR=$save_flvr
2589 if [ "$MOUNT_2" ] && $(grep -q $MOUNT2' ' /proc/mounts); then
2590 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
2592 if $(grep -q $MOUNT' ' /proc/mounts); then
2593 umount_client $MOUNT || error "umount $MOUNT failed"
2596 # start gss daemon on MGS
2597 if combined_mgs_mds ; then
2598 send_sigint $mds_HOST lsvcgssd
2600 start_gss_daemons $mgs_HOST "$LSVCGSSD -vvv -s -g"
2602 # add mgs key type and MGS NIDs in key on MGS
2603 do_nodes $mgs_HOST "lgss_sk -t mgs,server -g $MGSNID -m \
2604 $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2605 error "could not modify keyfile on MGS"
2607 # load modified key file on MGS
2608 do_nodes $mgs_HOST "lgss_sk -l $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2609 error "could not load keyfile on MGS"
2611 # add MGS NIDs in key on client
2612 do_nodes ${clients_arr[0]} "lgss_sk -g $MGSNID -m \
2613 $SK_PATH/$FSNAME.key >/dev/null 2>&1" ||
2614 error "could not modify keyfile on MGS"
2616 # set perms for per-nodemap keys else permission denied
2617 do_nodes $(comma_list $(all_nodes)) \
2618 "keyctl show | grep lustre | cut -c1-11 |
2620 xargs -IX keyctl setperm X 0x3f3f3f3f"
2622 # re-mount client with mgssec=skn
2623 save_opts=$MOUNT_OPTS
2624 if [ -z "$MOUNT_OPTS" ]; then
2625 MOUNT_OPTS="-o mgssec=skn"
2627 MOUNT_OPTS="$MOUNT_OPTS,mgssec=skn"
2629 zconf_mount_clients ${clients_arr[0]} $MOUNT $MOUNT_OPTS ||
2630 error "mount ${clients_arr[0]} with mgssec=skn failed"
2631 MOUNT_OPTS=$save_opts
2633 # enforce ska flavor for cli2mdt
2634 set_rule $FSNAME any cli2mdt ska
2635 wait_flavor cli2mdt ska
2637 # check error message
2638 $LCTL dk | grep "faked source" &&
2639 error "MGS connection srpc flags incorrect"
2643 run_test 33 "correct srpc flags for MGS connection"
2646 # restore deny_unknown
2647 do_facet mgs $LCTL nodemap_modify --name default \
2648 --property deny_unknown --value $denydefault
2649 if [ $? -ne 0 ]; then
2650 error_noexit "cannot reset deny_unknown on default nodemap"
2654 wait_nm_sync default deny_unknown
2661 [ $MGS_VERSION -lt $(version_code 2.12.51) ] &&
2662 skip "deny_unknown on default nm not supported before 2.12.51"
2664 activedefault=$(do_facet mgs $LCTL get_param -n nodemap.active)
2666 if [[ "$activedefault" != "1" ]]; then
2667 do_facet mgs $LCTL nodemap_activate 1
2669 stack_trap cleanup_active EXIT
2672 denydefault=$(do_facet mgs $LCTL get_param -n \
2673 nodemap.default.deny_unknown)
2674 [ -z "$denydefault" ] &&
2675 error "cannot get deny_unknown on default nodemap"
2676 if [ "$denydefault" -eq 0 ]; then
2682 do_facet mgs $LCTL nodemap_modify --name default \
2683 --property deny_unknown --value $denynew ||
2684 error "cannot set deny_unknown on default nodemap"
2686 [ "$(do_facet mgs $LCTL get_param -n nodemap.default.deny_unknown)" \
2688 error "setting deny_unknown on default nodemap did not work"
2690 stack_trap cleanup_34_deny EXIT
2692 wait_nm_sync default deny_unknown
2694 run_test 34 "deny_unknown on default nodemap"
2697 (( $MDS1_VERSION >= $(version_code 2.13.50) )) ||
2698 skip "Need MDS >= 2.13.50"
2700 # activate changelogs
2701 changelog_register || error "changelog_register failed"
2702 local cl_user="${CL_USERS[$SINGLEMDS]%% *}"
2703 changelog_users $SINGLEMDS | grep -q $cl_user ||
2704 error "User $cl_user not found in changelog_users"
2705 changelog_chmask ALL
2708 mkdir $DIR/$tdir || error "failed to mkdir $tdir"
2709 touch $DIR/$tdir/$tfile || error "failed to touch $tfile"
2711 # access changelogs with root
2712 changelog_dump || error "failed to dump changelogs"
2713 changelog_clear 0 || error "failed to clear changelogs"
2715 # put clients in non-admin nodemap
2717 stack_trap nodemap_test_cleanup EXIT
2718 for i in $(seq 0 $((num_clients-1))); do
2719 do_facet mgs $LCTL nodemap_modify --name c${i} \
2720 --property admin --value 0
2722 for i in $(seq 0 $((num_clients-1))); do
2723 wait_nm_sync c${i} admin_nodemap
2726 # access with mapped root
2727 changelog_dump && error "dump changelogs should have failed"
2728 changelog_clear 0 && error "clear changelogs should have failed"
2732 run_test 35 "Check permissions when accessing changelogs"
2735 local mode='\x00\x00\x00\x00'
2736 local raw="$(printf ""\\\\x%02x"" {0..63})"
2740 [[ $(lscpu) =~ Byte\ Order.*Little ]] && size='\x40\x00\x00\x00' ||
2741 size='\x00\x00\x00\x40'
2742 key="${mode}${raw}${size}"
2743 echo -n -e "${key}" | keyctl padd logon fscrypt:4242424242424242 @s
2748 sync ; echo 3 > /proc/sys/vm/drop_caches
2755 $LCTL set_param -n ldlm.namespaces.*.lru_size=clear
2756 sync ; echo 3 > /proc/sys/vm/drop_caches
2757 dummy_key=$(keyctl show | awk '$7 ~ "^fscrypt:" {print $1}')
2758 if [ -n "$dummy_key" ]; then
2759 keyctl revoke $dummy_key
2765 # wait for SSK flavor to be applied if necessary
2768 wait_flavor all2all $SK_FLAVOR
2770 wait_flavor cli2mdt $SK_FLAVOR
2771 wait_flavor cli2ost $SK_FLAVOR
2776 remount_client_normally() {
2777 # remount client without dummy encryption key
2778 if is_mounted $MOUNT; then
2779 umount_client $MOUNT || error "umount $MOUNT failed"
2781 mount_client $MOUNT ${MOUNT_OPTS} ||
2782 error "remount failed"
2784 if is_mounted $MOUNT2; then
2785 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
2787 if [ "$MOUNT_2" ]; then
2788 mount_client $MOUNT2 ${MOUNT_OPTS} ||
2789 error "remount failed"
2796 remount_client_dummykey() {
2799 # remount client with dummy encryption key
2800 if is_mounted $MOUNT; then
2801 umount_client $MOUNT || error "umount $MOUNT failed"
2803 mount_client $MOUNT ${MOUNT_OPTS},test_dummy_encryption ||
2804 error "remount failed"
2809 setup_for_enc_tests() {
2810 # remount client with test_dummy_encryption option
2811 if is_mounted $MOUNT; then
2812 umount_client $MOUNT || error "umount $MOUNT failed"
2814 mount_client $MOUNT ${MOUNT_OPTS},test_dummy_encryption ||
2815 error "mount with '-o test_dummy_encryption' failed"
2819 # this directory will be encrypted, because of dummy mode
2823 cleanup_for_enc_tests() {
2824 rm -rf $DIR/$tdir $*
2826 remount_client_normally
2829 cleanup_nodemap_after_enc_tests() {
2830 umount_client $MOUNT || true
2832 if (( MGS_VERSION >= $(version_code 2.13.55) )); then
2833 do_facet mgs $LCTL nodemap_modify --name default \
2834 --property forbid_encryption --value 0
2835 if (( MGS_VERSION >= $(version_code 2.15.51) )); then
2836 do_facet mgs $LCTL nodemap_modify --name default \
2837 --property readonly_mount --value 0
2840 do_facet mgs $LCTL nodemap_modify --name default \
2841 --property trusted --value 0
2842 do_facet mgs $LCTL nodemap_modify --name default \
2843 --property admin --value 0
2844 do_facet mgs $LCTL nodemap_activate 0
2846 if (( MGS_VERSION >= $(version_code 2.13.55) )); then
2847 wait_nm_sync default forbid_encryption '' inactive
2848 if (( MGS_VERSION >= $(version_code 2.15.51) )); then
2849 wait_nm_sync default readonly_mount '' inactive
2852 wait_nm_sync default trusted_nodemap '' inactive
2853 wait_nm_sync default admin_nodemap '' inactive
2856 mount_client $MOUNT ${MOUNT_OPTS} || error "re-mount failed"
2861 $LCTL get_param mdc.*.import | grep -q client_encryption ||
2862 skip "client encryption not supported"
2864 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
2865 skip "need dummy encryption support"
2867 stack_trap cleanup_for_enc_tests EXIT
2869 # first make sure it is possible to enable encryption
2870 # when nodemap is not active
2873 umount_client $MOUNT || error "umount $MOUNT failed (1)"
2875 # then activate nodemap, and retry
2876 # should succeed as encryption is not forbidden on default nodemap
2878 stack_trap cleanup_nodemap_after_enc_tests EXIT
2879 do_facet mgs $LCTL nodemap_activate 1
2881 forbid=$(do_facet mgs lctl get_param -n nodemap.default.forbid_encryption)
2882 [ $forbid -eq 0 ] || error "wrong default value for forbid_encryption"
2883 mount_client $MOUNT ${MOUNT_OPTS},test_dummy_encryption ||
2884 error "mount '-o test_dummy_encryption' failed with default"
2885 umount_client $MOUNT || error "umount $MOUNT failed (2)"
2887 # then forbid encryption, and retry
2888 do_facet mgs $LCTL nodemap_modify --name default \
2889 --property forbid_encryption --value 1
2890 wait_nm_sync default forbid_encryption
2891 mount_client $MOUNT ${MOUNT_OPTS},test_dummy_encryption &&
2892 error "mount '-o test_dummy_encryption' should have failed"
2895 run_test 36 "control if clients can use encryption"
2898 local testfile=$DIR/$tdir/$tfile
2899 local tmpfile=$TMP/abc
2900 local objdump=$TMP/objdump
2902 $LCTL get_param mdc.*.import | grep -q client_encryption ||
2903 skip "client encryption not supported"
2905 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
2906 skip "need dummy encryption support"
2908 [ "$ost1_FSTYPE" = ldiskfs ] || skip "ldiskfs only test (using debugfs)"
2910 stack_trap cleanup_for_enc_tests EXIT
2913 # write a few bytes in file
2914 echo "abc" > $tmpfile
2915 $LFS setstripe -c1 -i0 $testfile
2916 dd if=$tmpfile of=$testfile bs=4 count=1 conv=fsync
2917 do_facet ost1 "sync; sync"
2919 # check that content on ost is encrypted
2920 local fid=($($LFS getstripe $testfile | grep 0x))
2921 local seq=${fid[3]#0x}
2925 if [ $seq == 0 ]; then
2928 oid_hex=${fid[2]#0x}
2930 do_facet ost1 "$DEBUGFS -c -R 'cat O/$seq/d$(($oid % 32))/$oid_hex' \
2931 $(ostdevname 1)" > $objdump
2932 cmp -s $objdump $tmpfile &&
2933 error "file $testfile is not encrypted on ost"
2935 # check that in-memory representation of file is correct
2936 cmp -bl ${tmpfile} ${testfile} ||
2937 error "file $testfile is corrupted in memory"
2939 cancel_lru_locks osc ; cancel_lru_locks mdc
2941 # check that file read from server is correct
2942 cmp -bl ${tmpfile} ${testfile} ||
2943 error "file $testfile is corrupted on server"
2945 rm -f $tmpfile $objdump
2947 run_test 37 "simple encrypted file"
2950 local testfile=$DIR/$tdir/$tfile
2951 local tmpfile=$TMP/abc
2957 local pagesz=$(getconf PAGE_SIZE)
2959 $LCTL get_param mdc.*.import | grep -q client_encryption ||
2960 skip "client encryption not supported"
2962 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
2963 skip "need dummy encryption support"
2965 stack_trap cleanup_for_enc_tests EXIT
2968 # get block size on ost
2969 blksz=$($LCTL get_param osc.$FSNAME*.import |
2970 awk '/grant_block_size:/ { print $2; exit; }')
2971 # write a few bytes in file at offset $blksz
2972 echo "abc" > $tmpfile
2973 $LFS setstripe -c1 -i0 $testfile
2974 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$blksz \
2975 oflag=seek_bytes conv=fsync
2977 blksz=$(($blksz > $pagesz ? $blksz : $pagesz))
2978 # check that in-memory representation of file is correct
2979 bsize=$(stat --format=%B $testfile)
2980 filesz=$(stat --format=%b $testfile)
2981 filesz=$((filesz*bsize))
2982 [ $filesz -le $blksz ] ||
2983 error "file $testfile is $filesz long in memory"
2985 cancel_lru_locks osc ; cancel_lru_locks mdc
2987 # check that file read from server is correct
2988 bsize=$(stat --format=%B $testfile)
2989 filesz=$(stat --format=%b $testfile)
2990 filesz=$((filesz*bsize))
2991 [ $filesz -le $blksz ] ||
2992 error "file $testfile is $filesz long on server"
2996 run_test 38 "encrypted file with hole"
2999 local testfile=$DIR/$tdir/$tfile
3000 local tmpfile=$TMP/abc
3002 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3003 skip "client encryption not supported"
3005 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3006 skip "need dummy encryption support"
3008 stack_trap cleanup_for_enc_tests EXIT
3011 # write a few bytes in file
3012 echo "abc" > $tmpfile
3013 $LFS setstripe -c1 -i0 $testfile
3014 dd if=$tmpfile of=$testfile bs=4 count=1 conv=fsync
3016 # write a few more bytes in the same page
3017 dd if=$tmpfile of=$testfile bs=4 count=1 seek=1024 oflag=seek_bytes \
3020 dd if=$tmpfile of=$tmpfile bs=4 count=1 seek=1024 oflag=seek_bytes \
3023 # check that in-memory representation of file is correct
3024 cmp -bl $tmpfile $testfile ||
3025 error "file $testfile is corrupted in memory"
3027 cancel_lru_locks osc ; cancel_lru_locks mdc
3029 # check that file read from server is correct
3030 cmp -bl $tmpfile $testfile ||
3031 error "file $testfile is corrupted on server"
3035 run_test 39 "rewrite data in already encrypted page"
3038 local testfile=$DIR/$tdir/$tfile
3039 local tmpfile=$TMP/abc
3040 local tmpfile2=$TMP/abc2
3043 #define LUSTRE_ENCRYPTION_UNIT_SIZE (1 << 12)
3044 local UNIT_SIZE=$((1 << 12))
3047 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3048 skip "client encryption not supported"
3050 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3051 skip "need dummy encryption support"
3053 [[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
3055 stack_trap cleanup_for_enc_tests EXIT
3058 # write a few bytes in file
3059 echo "abc" > $tmpfile
3060 $LFS setstripe -c1 -i0 $testfile
3061 dd if=$tmpfile of=$testfile bs=4 count=1 conv=fsync
3063 # check that in-memory representation of file is correct
3064 cmp -bl $tmpfile $testfile ||
3065 error "file $testfile is corrupted in memory (1)"
3067 cancel_lru_locks osc ; cancel_lru_locks mdc
3069 # check that file read from server is correct
3070 cmp -bl $tmpfile $testfile ||
3071 error "file $testfile is corrupted on server (1)"
3073 # write a few other bytes in same page
3074 dd if=$tmpfile of=$testfile bs=4 count=1 seek=256 oflag=seek_bytes \
3077 dd if=$tmpfile of=$tmpfile bs=4 count=1 seek=256 oflag=seek_bytes \
3080 # check that in-memory representation of file is correct
3081 cmp -bl $tmpfile $testfile ||
3082 error "file $testfile is corrupted in memory (2)"
3084 cancel_lru_locks osc ; cancel_lru_locks mdc
3086 # check that file read from server is correct
3087 cmp -bl $tmpfile $testfile ||
3088 error "file $testfile is corrupted on server (2)"
3090 rm -f $testfile $tmpfile
3091 cancel_lru_locks osc ; cancel_lru_locks mdc
3093 # write a few bytes in file, at end of first page
3094 echo "abc" > $tmpfile
3095 $LFS setstripe -c1 -i0 $testfile
3096 seek=$(getconf PAGESIZE)
3098 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3101 # write a few other bytes at beginning of first page
3102 dd if=$tmpfile of=$testfile bs=4 count=1 conv=fsync,notrunc
3104 dd if=$tmpfile of=$tmpfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3107 # check that in-memory representation of file is correct
3108 cmp -bl $tmpfile $testfile ||
3109 error "file $testfile is corrupted in memory (3)"
3111 cancel_lru_locks osc ; cancel_lru_locks mdc
3113 # check that file read from server is correct
3114 cmp -bl $tmpfile $testfile ||
3115 error "file $testfile is corrupted on server (3)"
3117 rm -f $testfile $tmpfile
3118 cancel_lru_locks osc ; cancel_lru_locks mdc
3120 # write a few bytes in file, at beginning of second page
3121 echo "abc" > $tmpfile
3122 $LFS setstripe -c1 -i0 $testfile
3123 seek=$(getconf PAGESIZE)
3124 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3126 dd if=$tmpfile of=$tmpfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3129 # write a few other bytes at end of first page
3131 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3133 dd if=$tmpfile of=$tmpfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3136 # check that in-memory representation of file is correct
3137 cmp -bl $tmpfile2 $testfile ||
3138 error "file $testfile is corrupted in memory (4)"
3140 cancel_lru_locks osc ; cancel_lru_locks mdc
3142 # check that file read from server is correct
3143 cmp -bl $tmpfile2 $testfile ||
3144 error "file $testfile is corrupted on server (4)"
3146 rm -f $testfile $tmpfile $tmpfile2
3147 cancel_lru_locks osc ; cancel_lru_locks mdc
3149 # write a few bytes in file, at beginning of first stripe
3150 echo "abc" > $tmpfile
3151 $LFS setstripe -S 256k -c2 $testfile
3152 dd if=$tmpfile of=$testfile bs=4 count=1 conv=fsync,notrunc
3154 # write a few other bytes, at beginning of second stripe
3155 dd if=$tmpfile of=$testfile bs=4 count=1 seek=262144 oflag=seek_bytes \
3157 dd if=$tmpfile of=$tmpfile bs=4 count=1 seek=262144 oflag=seek_bytes \
3160 # check that in-memory representation of file is correct
3161 cmp -bl $tmpfile $testfile ||
3162 error "file $testfile is corrupted in memory (5)"
3164 cancel_lru_locks osc ; cancel_lru_locks mdc
3166 # check that file read from server is correct
3167 cmp -bl $tmpfile $testfile ||
3168 error "file $testfile is corrupted on server (5)"
3170 filesz=$(stat --format=%s $testfile)
3171 filesz=$(((filesz+UNIT_SIZE-1)/UNIT_SIZE * UNIT_SIZE))
3173 # remount without dummy encryption key
3174 remount_client_normally
3176 scrambledfile=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -type f)
3177 [ $(stat --format=%s $scrambledfile) -eq $filesz ] ||
3178 error "file size without key should be rounded up"
3182 run_test 40 "exercise size of encrypted file"
3185 local testfile=$DIR/$tdir/$tfile
3186 local tmpfile=$TMP/abc
3187 local tmpfile2=$TMP/abc2
3190 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3191 skip "client encryption not supported"
3193 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3194 skip "need dummy encryption support"
3196 stack_trap cleanup_for_enc_tests EXIT
3199 echo "abc" > $tmpfile
3200 seek=$(getconf PAGESIZE)
3201 seek=$((seek - 204))
3202 dd if=$tmpfile of=$tmpfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3204 seek=$(getconf PAGESIZE)
3205 seek=$((seek + 1092))
3206 dd if=$tmpfile of=$tmpfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3209 # write a few bytes in file
3210 $LFS setstripe -c1 -i0 -S 256k $testfile
3211 seek=$(getconf PAGESIZE)
3212 seek=$((seek - 204))
3213 #define OBD_FAIL_OST_WR_ATTR_DELAY 0x250
3214 do_facet ost1 "$LCTL set_param fail_loc=0x250 fail_val=15"
3215 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3219 # write a few other bytes, at a different offset
3220 seek=$(getconf PAGESIZE)
3221 seek=$((seek + 1092))
3222 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3223 conv=fsync,notrunc &
3225 do_facet ost1 "$LCTL set_param fail_loc=0x0"
3227 # check that in-memory representation of file is correct
3228 cmp -bl $tmpfile2 $testfile ||
3229 error "file $testfile is corrupted in memory (1)"
3231 cancel_lru_locks osc ; cancel_lru_locks mdc
3233 # check that file read from server is correct
3234 cmp -bl $tmpfile2 $testfile ||
3235 error "file $testfile is corrupted on server (1)"
3237 rm -f $tmpfile $tmpfile2
3239 run_test 41 "test race on encrypted file size (1)"
3242 local testfile=$DIR/$tdir/$tfile
3243 local testfile2=$DIR2/$tdir/$tfile
3244 local tmpfile=$TMP/abc
3245 local tmpfile2=$TMP/abc2
3246 local pagesz=$(getconf PAGESIZE)
3249 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3250 skip "client encryption not supported"
3252 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3253 skip "need dummy encryption support"
3255 stack_trap cleanup_for_enc_tests EXIT
3258 if is_mounted $MOUNT2; then
3259 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
3261 mount_client $MOUNT2 ${MOUNT_OPTS},test_dummy_encryption ||
3262 error "mount2 with '-o test_dummy_encryption' failed"
3264 # create file by writting one whole page
3265 $LFS setstripe -c1 -i0 -S 256k $testfile
3266 dd if=/dev/zero of=$testfile bs=$pagesz count=1 conv=fsync
3268 # read file from 2nd mount point
3269 cat $testfile2 > /dev/null
3271 echo "abc" > $tmpfile
3272 dd if=/dev/zero of=$tmpfile2 bs=$pagesz count=1 conv=fsync
3273 seek=$((2*pagesz - 204))
3274 dd if=$tmpfile of=$tmpfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3276 seek=$((2*pagesz + 1092))
3277 dd if=$tmpfile of=$tmpfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3280 # write a few bytes in file from 1st mount point
3281 seek=$((2*pagesz - 204))
3282 #define OBD_FAIL_OST_WR_ATTR_DELAY 0x250
3283 do_facet ost1 "$LCTL set_param fail_loc=0x250 fail_val=15"
3284 dd if=$tmpfile of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3285 conv=fsync,notrunc &
3288 # write a few other bytes, at a different offset from 2nd mount point
3289 seek=$((2*pagesz + 1092))
3290 dd if=$tmpfile of=$testfile2 bs=4 count=1 seek=$seek oflag=seek_bytes \
3291 conv=fsync,notrunc &
3293 do_facet ost1 "$LCTL set_param fail_loc=0x0"
3295 # check that in-memory representation of file is correct
3296 cmp -bl $tmpfile2 $testfile ||
3297 error "file $testfile is corrupted in memory (1)"
3299 # check that in-memory representation of file is correct
3300 cmp -bl $tmpfile2 $testfile2 ||
3301 error "file $testfile is corrupted in memory (2)"
3303 cancel_lru_locks osc ; cancel_lru_locks mdc
3305 # check that file read from server is correct
3306 cmp -bl $tmpfile2 $testfile ||
3307 error "file $testfile is corrupted on server (1)"
3309 rm -f $tmpfile $tmpfile2
3311 run_test 42 "test race on encrypted file size (2)"
3314 local testfile=$DIR/$tdir/$tfile
3315 local testfile2=$DIR2/$tdir/$tfile
3316 local tmpfile=$TMP/abc
3317 local tmpfile2=$TMP/abc2
3318 local resfile=$TMP/res
3319 local pagesz=$(getconf PAGESIZE)
3322 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3323 skip "client encryption not supported"
3325 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3326 skip "need dummy encryption support"
3328 stack_trap cleanup_for_enc_tests EXIT
3331 if is_mounted $MOUNT2; then
3332 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
3334 mount_client $MOUNT2 ${MOUNT_OPTS},test_dummy_encryption ||
3335 error "mount2 with '-o test_dummy_encryption' failed"
3338 tr '\0' '1' < /dev/zero |
3339 dd of=$tmpfile bs=1 count=$pagesz conv=fsync
3340 $LFS setstripe -c1 -i0 -S 256k $testfile
3341 cp $tmpfile $testfile
3343 # read file from 2nd mount point
3344 cat $testfile2 > /dev/null
3346 # write a few bytes in file from 1st mount point
3347 echo "abc" > $tmpfile2
3348 seek=$((2*pagesz - 204))
3349 #define OBD_FAIL_OST_WR_ATTR_DELAY 0x250
3350 do_facet ost1 "$LCTL set_param fail_loc=0x250 fail_val=15"
3351 dd if=$tmpfile2 of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3352 conv=fsync,notrunc &
3355 # read file from 2nd mount point
3356 dd if=$testfile2 of=$resfile bs=$pagesz count=1 conv=fsync,notrunc
3357 cmp -bl $tmpfile $resfile ||
3358 error "file $testfile is corrupted in memory (1)"
3361 do_facet ost1 "$LCTL set_param fail_loc=0x0"
3363 # check that in-memory representation of file is correct
3364 dd if=$tmpfile2 of=$tmpfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3366 cmp -bl $tmpfile $testfile2 ||
3367 error "file $testfile is corrupted in memory (2)"
3369 cancel_lru_locks osc ; cancel_lru_locks mdc
3371 # check that file read from server is correct
3372 cmp -bl $tmpfile $testfile ||
3373 error "file $testfile is corrupted on server (1)"
3375 rm -f $tmpfile $tmpfile2
3377 run_test 43 "test race on encrypted file size (3)"
3380 local testfile=$DIR/$tdir/$tfile
3381 local tmpfile=$TMP/abc
3382 local resfile=$TMP/resfile
3383 local pagesz=$(getconf PAGESIZE)
3386 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3387 skip "client encryption not supported"
3389 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3390 skip "need dummy encryption support"
3392 which vmtouch || skip "This test needs vmtouch utility"
3394 # Direct I/O is now supported on encrypted files.
3396 stack_trap cleanup_for_enc_tests EXIT
3399 $LFS setstripe -c1 -i0 $testfile
3400 dd if=/dev/urandom of=$tmpfile bs=$pagesz count=2 conv=fsync
3401 dd if=$tmpfile of=$testfile bs=$pagesz count=2 oflag=direct ||
3402 error "could not write to file with O_DIRECT (1)"
3404 respage=$(vmtouch $testfile | awk '/Resident Pages:/ {print $3}')
3405 [ "$respage" == "0/2" ] ||
3406 error "write to enc file fell back to buffered IO"
3410 dd if=$testfile of=$resfile bs=$pagesz count=2 iflag=direct ||
3411 error "could not read from file with O_DIRECT (1)"
3413 respage=$(vmtouch $testfile | awk '/Resident Pages:/ {print $3}')
3414 [ "$respage" == "0/2" ] ||
3415 error "read from enc file fell back to buffered IO"
3417 cmp -bl $tmpfile $resfile ||
3418 error "file $testfile is corrupted (1)"
3422 $TRUNCATE $tmpfile $pagesz
3423 dd if=$tmpfile of=$testfile bs=$pagesz count=1 seek=13 oflag=direct ||
3424 error "could not write to file with O_DIRECT (2)"
3428 dd if=$testfile of=$resfile bs=$pagesz count=1 skip=13 iflag=direct ||
3429 error "could not read from file with O_DIRECT (2)"
3430 cmp -bl $tmpfile $resfile ||
3431 error "file $testfile is corrupted (2)"
3433 rm -f $testfile $resfile
3434 $LFS setstripe -c1 -i0 $testfile
3436 $TRUNCATE $tmpfile $((pagesz/2 - 5))
3437 cp $tmpfile $testfile
3441 dd if=$testfile of=$resfile bs=$pagesz count=1 iflag=direct ||
3442 error "could not read from file with O_DIRECT (3)"
3443 cmp -bl $tmpfile $resfile ||
3444 error "file $testfile is corrupted (3)"
3446 rm -f $tmpfile $resfile $testfile
3448 if [ $OSTCOUNT -ge 2 ]; then
3449 dd if=/dev/urandom of=$tmpfile bs=$pagesz count=1 conv=fsync
3450 $LFS setstripe -S 256k -c2 $testfile
3452 # write in file, at beginning of first stripe, buffered IO
3453 dd if=$tmpfile of=$testfile bs=$pagesz count=1 \
3456 # write at beginning of second stripe, direct IO
3457 dd if=$tmpfile of=$testfile bs=$pagesz count=1 seek=256k \
3458 oflag=seek_bytes,direct conv=fsync,notrunc
3462 # read at beginning of first stripe, direct IO
3463 dd if=$testfile of=$resfile bs=$pagesz count=1 \
3464 iflag=direct conv=fsync
3466 cmp -bl $tmpfile $resfile ||
3467 error "file $testfile is corrupted (4)"
3469 # read at beginning of second stripe, buffered IO
3470 dd if=$testfile of=$resfile bs=$pagesz count=1 skip=256k \
3471 iflag=skip_bytes conv=fsync
3473 cmp -bl $tmpfile $resfile ||
3474 error "file $testfile is corrupted (5)"
3476 rm -f $tmpfile $resfile
3479 run_test 44 "encrypted file access semantics: direct IO"
3482 local testfile=$DIR/$tdir/$tfile
3483 local tmpfile=$TMP/junk
3485 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3486 skip "client encryption not supported"
3488 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3489 skip "need dummy encryption support"
3491 stack_trap cleanup_for_enc_tests EXIT
3494 $LFS setstripe -c1 -i0 $testfile
3495 dd if=/dev/zero of=$testfile bs=512K count=1
3496 $MULTIOP $testfile OSMRUc || error "$MULTIOP $testfile failed (1)"
3497 $MULTIOP $testfile OSMWUc || error "$MULTIOP $testfile failed (2)"
3499 dd if=/dev/zero of=$tmpfile bs=512K count=1
3500 $MULTIOP $tmpfile OSMWUc || error "$MULTIOP $tmpfile failed"
3501 $MMAP_CAT $tmpfile > ${tmpfile}2
3505 $MULTIOP $testfile OSMRUc
3506 $MMAP_CAT $testfile > ${testfile}2
3507 cmp -bl ${tmpfile}2 ${testfile}2 ||
3508 error "file $testfile is corrupted"
3510 rm -f $tmpfile ${tmpfile}2
3512 run_test 45 "encrypted file access semantics: MMAP"
3515 local testdir=$DIR/$tdir/mydir
3516 local testfile=$testdir/myfile
3517 local testdir2=$DIR/$tdir/mydirwithaveryverylongnametotestcodebehaviour0
3518 local testfile2=$testdir/myfilewithaveryverylongnametotestcodebehaviour0
3519 # testdir3, testfile3, testhl3 and testsl3 names are 255 bytes long
3520 local testdir3=$testdir2/dir_abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz012345678
3521 local testfile3=$testdir2/file_abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01234567
3522 local testhl3=$testdir2/hl_abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789
3523 local testsl3=$testdir2/sl_abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789
3524 local lsfile=$TMP/lsfile
3529 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3530 skip "client encryption not supported"
3532 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3533 skip "need dummy encryption support"
3535 stack_trap cleanup_for_enc_tests EXIT
3538 touch $DIR/$tdir/$tfile
3540 echo test > $testfile
3541 echo othertest > $testfile2
3542 if [[ $MDSCOUNT -gt 1 ]]; then
3543 $LFS setdirstripe -c1 -i1 $testdir2
3547 inum=$(stat -c %i $testdir2)
3548 if [ "$mds1_FSTYPE" = ldiskfs ]; then
3549 # For now, restrict this part of the test to ldiskfs backend,
3550 # as osd-zfs does not support 255 byte-long encrypted names.
3551 mkdir $testdir3 || error "cannot mkdir $testdir3"
3552 touch $testfile3 || error "cannot touch $testfile3"
3553 ln $testfile3 $testhl3 || error "cannot ln $testhl3"
3554 ln -s $testfile3 $testsl3 || error "cannot ln $testsl3"
3556 sync ; echo 3 > /proc/sys/vm/drop_caches
3558 # remount without dummy encryption key
3559 remount_client_normally
3562 scrambleddir=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -inum $inum)
3563 stat $scrambleddir || error "stat $scrambleddir failed"
3564 if [ "$mds1_FSTYPE" = ldiskfs ]; then
3565 stat $scrambleddir/* || error "cannot stat in $scrambleddir"
3566 rm -rf $scrambleddir/* || error "cannot clean in $scrambleddir"
3568 rmdir $scrambleddir || error "rmdir $scrambleddir failed"
3570 scrambleddir=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -type d)
3571 ls -1 $scrambleddir > $lsfile || error "ls $testdir failed (1)"
3573 scrambledfile=$scrambleddir/$(head -n 1 $lsfile)
3574 stat $scrambledfile || error "stat $scrambledfile failed (1)"
3577 cat $scrambledfile && error "cat $scrambledfile should have failed (1)"
3578 rm -f $scrambledfile || error "rm $scrambledfile failed (1)"
3580 ls -1 $scrambleddir > $lsfile || error "ls $testdir failed (2)"
3581 scrambledfile=$scrambleddir/$(head -n 1 $lsfile)
3582 stat $scrambledfile || error "stat $scrambledfile failed (2)"
3584 cat $scrambledfile && error "cat $scrambledfile should have failed (2)"
3586 touch $scrambleddir/otherfile &&
3587 error "touch otherfile should have failed"
3588 ls $scrambleddir/otherfile && error "otherfile should not exist"
3589 mkdir $scrambleddir/otherdir &&
3590 error "mkdir otherdir should have failed"
3591 ls -d $scrambleddir/otherdir && error "otherdir should not exist"
3594 rm -f $scrambledfile || error "rm $scrambledfile failed (2)"
3595 rmdir $scrambleddir || error "rmdir $scrambleddir failed"
3598 run_test 46 "encrypted file access semantics without key"
3601 local testfile=$DIR/$tdir/$tfile
3602 local testfile2=$DIR/$tdir/${tfile}.2
3603 local tmpfile=$DIR/junk
3608 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3609 skip "client encryption not supported"
3611 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3612 skip "need dummy encryption support"
3614 $LCTL get_param mdc.*.connect_flags | grep -q name_encryption ||
3617 stack_trap cleanup_for_enc_tests EXIT
3620 dd if=/dev/urandom of=$tmpfile bs=512K count=1
3621 mrename $tmpfile $testfile &&
3622 error "rename from unencrypted to encrypted dir should fail"
3624 ln $tmpfile $testfile &&
3625 error "link from encrypted to unencrypted dir should fail"
3627 cp $tmpfile $testfile ||
3628 error "cp from unencrypted to encrypted dir should succeed"
3631 mrename $testfile $testfile2 ||
3632 error "rename from within encrypted dir should succeed"
3634 ln $testfile2 $testfile ||
3635 error "link from within encrypted dir should succeed"
3636 cmp -bl $testfile2 $testfile ||
3637 error "cannot read from hard link (1.1)"
3638 echo a >> $testfile || error "cannot write to hard link (1)"
3640 cmp -bl $testfile2 $testfile ||
3641 error "cannot read from hard link (1.2)"
3644 ln $testfile2 $tmpfile ||
3645 error "link from unencrypted to encrypted dir should succeed"
3647 cmp -bl $testfile2 $tmpfile ||
3648 error "cannot read from hard link (2.1)"
3649 echo a >> $tmpfile || error "cannot write to hard link (2)"
3651 cmp -bl $testfile2 $tmpfile ||
3652 error "cannot read from hard link (2.2)"
3655 if [ $name_enc -eq 1 ]; then
3656 # check we are limited in the number of hard links
3657 # we can create for encrypted files, to what can fit into LinkEA
3658 for i in $(seq 1 160); do
3659 ln $testfile2 ${testfile}_$i || break
3661 [ $i -lt 160 ] || error "hard link $i should fail"
3665 mrename $testfile2 $tmpfile &&
3666 error "rename from encrypted to unencrypted dir should fail"
3668 dd if=/dev/urandom of=$tmpfile bs=512K count=1
3670 dd if=/dev/urandom of=$testfile bs=512K count=1
3671 mkdir $DIR/$tdir/mydir
3673 ln -s $testfile ${testfile}.sym ||
3674 error "symlink from within encrypted dir should succeed"
3676 cmp -bl $testfile ${testfile}.sym ||
3677 error "cannot read from sym link (1.1)"
3678 echo a >> ${testfile}.sym || error "cannot write to sym link (1)"
3680 cmp -bl $testfile ${testfile}.sym ||
3681 error "cannot read from sym link (1.2)"
3682 [ $(stat -c %s ${testfile}.sym) -eq ${#testfile} ] ||
3683 error "wrong symlink size (1)"
3685 ln -s $tmpfile ${testfile}.sl ||
3686 error "symlink from encrypted to unencrypted dir should succeed"
3688 cmp -bl $tmpfile ${testfile}.sl ||
3689 error "cannot read from sym link (2.1)"
3690 echo a >> ${testfile}.sl || error "cannot write to sym link (2)"
3692 cmp -bl $tmpfile ${testfile}.sl ||
3693 error "cannot read from sym link (2.2)"
3694 [ $(stat -c %s ${testfile}.sl) -eq ${#tmpfile} ] ||
3695 error "wrong symlink size (2)"
3696 rm -f ${testfile}.sl
3698 sync ; echo 3 > /proc/sys/vm/drop_caches
3700 # remount without dummy encryption key
3701 remount_client_normally
3703 scrambleddir=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -type d)
3704 scrambledfile=$(find $DIR/$tdir/ -maxdepth 1 -type f)
3705 scrambledlink=$(find $DIR/$tdir/ -maxdepth 1 -type l)
3706 ln $scrambledfile $scrambleddir/linkfile &&
3707 error "ln linkfile should have failed"
3708 mrename $scrambledfile $DIR/onefile2 &&
3709 error "mrename from $scrambledfile should have failed"
3711 mrename $DIR/onefile $scrambleddir/otherfile &&
3712 error "mrename to $scrambleddir should have failed"
3713 readlink $scrambledlink ||
3714 error "link should be read without key"
3715 [ $(stat -c %s $scrambledlink) -eq \
3716 $(expr length "$(readlink $scrambledlink)") ] ||
3717 error "wrong symlink size without key"
3718 if [ $name_enc -eq 1 ]; then
3719 readlink -e $scrambledlink &&
3720 error "link should not point to anywhere useful"
3722 ln -s $scrambledfile ${scrambledfile}.sym &&
3723 error "symlink without key should fail (1)"
3724 ln -s $tmpfile ${scrambledfile}.sl &&
3725 error "symlink without key should fail (2)"
3727 rm -f $tmpfile $DIR/onefile
3729 run_test 47 "encrypted file access semantics: rename/link"
3732 local save="$TMP/$TESTSUITE-$TESTNAME.parameters"
3733 local testfile=$DIR/$tdir/$tfile
3734 local tmpfile=$TMP/111
3735 local tmpfile2=$TMP/abc
3736 local pagesz=$(getconf PAGESIZE)
3741 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3742 skip "client encryption not supported"
3744 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3745 skip "need dummy encryption support"
3747 stack_trap cleanup_for_enc_tests EXIT
3750 # create file, 4 x PAGE_SIZE long
3751 tr '\0' '1' < /dev/zero |
3752 dd of=$tmpfile bs=1 count=4x$pagesz conv=fsync
3753 $LFS setstripe -c1 -i0 $testfile
3754 cp $tmpfile $testfile
3755 echo "abc" > $tmpfile2
3757 # decrease size: truncate to PAGE_SIZE
3758 $TRUNCATE $tmpfile $pagesz
3759 $TRUNCATE $testfile $pagesz
3760 cancel_lru_locks osc ; cancel_lru_locks mdc
3761 cmp -bl $tmpfile $testfile ||
3762 error "file $testfile is corrupted (1)"
3764 # increase size: truncate to 2 x PAGE_SIZE
3766 $TRUNCATE $tmpfile $sz
3767 $TRUNCATE $testfile $sz
3768 cancel_lru_locks osc ; cancel_lru_locks mdc
3769 cmp -bl $tmpfile $testfile ||
3770 error "file $testfile is corrupted (2)"
3773 seek=$((pagesz+100))
3774 dd if=$tmpfile2 of=$tmpfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3776 dd if=$tmpfile2 of=$testfile bs=4 count=1 seek=$seek oflag=seek_bytes \
3778 cancel_lru_locks osc ; cancel_lru_locks mdc
3779 cmp -bl $tmpfile $testfile ||
3780 error "file $testfile is corrupted (3)"
3782 # truncate to PAGE_SIZE / 2
3784 $TRUNCATE $tmpfile $sz
3785 $TRUNCATE $testfile $sz
3786 cancel_lru_locks osc ; cancel_lru_locks mdc
3787 cmp -bl $tmpfile $testfile ||
3788 error "file $testfile is corrupted (4)"
3790 # truncate to a smaller, non-multiple of PAGE_SIZE, non-multiple of 16
3792 $TRUNCATE $tmpfile $sz
3793 $TRUNCATE $testfile $sz
3794 cancel_lru_locks osc ; cancel_lru_locks mdc
3795 cmp -bl $tmpfile $testfile ||
3796 error "file $testfile is corrupted (5)"
3798 # truncate to a larger, non-multiple of PAGE_SIZE, non-multiple of 16
3800 $TRUNCATE $tmpfile $sz
3801 $TRUNCATE $testfile $sz
3802 cancel_lru_locks osc ; cancel_lru_locks mdc
3803 cmp -bl $tmpfile $testfile ||
3804 error "file $testfile is corrupted (6)"
3806 # truncate to a larger, non-multiple of PAGE_SIZE, in a different page
3807 sz=$((sz+pagesz+30))
3808 $TRUNCATE $tmpfile $sz
3809 $TRUNCATE $testfile $sz
3810 cancel_lru_locks osc ; cancel_lru_locks mdc
3811 cmp -bl $tmpfile $testfile ||
3812 error "file $testfile is corrupted (7)"
3814 sync ; echo 3 > /proc/sys/vm/drop_caches
3816 # remount without dummy encryption key
3817 remount_client_normally
3819 scrambledfile=$(find $DIR/$tdir/ -maxdepth 1 -type f)
3820 $TRUNCATE $scrambledfile 0 &&
3821 error "truncate $scrambledfile should have failed without key"
3823 rm -f $tmpfile $tmpfile2
3825 run_test 48a "encrypted file access semantics: truncate"
3827 cleanup_for_enc_tests_othercli() {
3830 # remount othercli normally
3831 zconf_umount $othercli $MOUNT ||
3832 error "umount $othercli $MOUNT failed"
3833 zconf_mount $othercli $MOUNT ||
3834 error "remount $othercli $MOUNT failed"
3840 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3841 skip "client encryption not supported"
3843 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3844 skip "need dummy encryption support"
3846 [ "$num_clients" -ge 2 ] || skip "Need at least 2 clients"
3848 if [ "$HOSTNAME" == ${clients_arr[0]} ]; then
3849 othercli=${clients_arr[1]}
3851 othercli=${clients_arr[0]}
3854 stack_trap cleanup_for_enc_tests EXIT
3855 stack_trap "cleanup_for_enc_tests_othercli $othercli" EXIT
3857 zconf_umount $othercli $MOUNT ||
3858 error "umount $othercli $MOUNT failed"
3860 cp /bin/sleep $DIR/$tdir/
3861 cancel_lru_locks osc ; cancel_lru_locks mdc
3862 $DIR/$tdir/sleep 30 &
3863 # mount and IOs must be done in the same shell session, otherwise
3864 # encryption key in session keyring is missing
3865 do_node $othercli "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
3866 $MGSNID:/$FSNAME $MOUNT && \
3867 $TRUNCATE $DIR/$tdir/sleep 7"
3868 wait || error "wait error"
3869 cmp --silent /bin/sleep $DIR/$tdir/sleep ||
3870 error "/bin/sleep and $DIR/$tdir/sleep differ"
3872 run_test 48b "encrypted file: concurrent truncate"
3878 $LCTL set_param debug=+info
3883 [ $? -eq 0 ] || error "$cmd failed"
3885 if [ -z "$MATCHING_STRING" ]; then
3886 $LCTL dk | grep -E "get xattr 'encryption.c'|get xattrs"
3888 $LCTL dk | grep -E "$MATCHING_STRING"
3890 [ $? -ne 0 ] || error "get xattr event was triggered"
3894 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3895 skip "client encryption not supported"
3897 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3898 skip "need dummy encryption support"
3900 stack_trap cleanup_for_enc_tests EXIT
3903 local dirname=$DIR/$tdir/subdir
3907 trace_cmd stat $dirname
3908 trace_cmd echo a > $dirname/f1
3909 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
3910 trace_cmd stat $dirname/f1
3911 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
3912 trace_cmd cat $dirname/f1
3913 dd if=/dev/zero of=$dirname/f1 bs=1M count=10 conv=fsync
3914 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
3915 MATCHING_STRING="get xattr 'encryption.c'" \
3916 trace_cmd $TRUNCATE $dirname/f1 10240
3917 trace_cmd $LFS setstripe -E -1 -S 4M $dirname/f2
3918 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
3919 trace_cmd $LFS migrate -E -1 -S 256K $dirname/f2
3921 if [[ $MDSCOUNT -gt 1 ]]; then
3922 trace_cmd $LFS setdirstripe -i 1 $dirname/d2
3923 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
3924 trace_cmd $LFS migrate -m 0 $dirname/d2
3925 echo b > $dirname/d2/subf
3926 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
3927 if (( "$MDS1_VERSION" > $(version_code 2.14.54.54) )); then
3928 # migrate a non-empty encrypted dir
3929 trace_cmd $LFS migrate -m 1 $dirname/d2
3930 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
3931 [ -f $dirname/d2/subf ] || error "migrate failed (1)"
3932 [ $(cat $dirname/d2/subf) == "b" ] ||
3933 error "migrate failed (2)"
3936 $LFS setdirstripe -i 1 -c 1 $dirname/d3
3937 dirname=$dirname/d3/subdir
3939 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
3940 trace_cmd stat $dirname
3941 trace_cmd echo c > $dirname/f1
3942 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
3943 trace_cmd stat $dirname/f1
3944 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
3945 trace_cmd cat $dirname/f1
3946 dd if=/dev/zero of=$dirname/f1 bs=1M count=10 conv=fsync
3947 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
3948 MATCHING_STRING="get xattr 'encryption.c'" \
3949 trace_cmd $TRUNCATE $dirname/f1 10240
3950 trace_cmd $LFS setstripe -E -1 -S 4M $dirname/f2
3951 sync ; sync ; echo 3 > /proc/sys/vm/drop_caches
3952 trace_cmd $LFS migrate -E -1 -S 256K $dirname/f2
3954 skip_noexit "2nd part needs >= 2 MDTs"
3957 run_test 49 "Avoid getxattr for encryption context"
3960 local testfile=$DIR/$tdir/$tfile
3961 local tmpfile=$TMP/abc
3962 local pagesz=$(getconf PAGESIZE)
3965 $LCTL get_param mdc.*.import | grep -q client_encryption ||
3966 skip "client encryption not supported"
3968 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
3969 skip "need dummy encryption support"
3971 stack_trap cleanup_for_enc_tests EXIT
3974 # write small file, data on MDT only
3975 tr '\0' '1' < /dev/zero |
3976 dd of=$tmpfile bs=1 count=5000 conv=fsync
3977 $LFS setstripe -E 1M -L mdt -E EOF $testfile
3978 cp $tmpfile $testfile
3980 # check that in-memory representation of file is correct
3981 cmp -bl $tmpfile $testfile ||
3982 error "file $testfile is corrupted in memory"
3984 cancel_lru_locks osc ; cancel_lru_locks mdc
3986 # check that file read from server is correct
3987 cmp -bl $tmpfile $testfile ||
3988 error "file $testfile is corrupted on server"
3990 # decrease size: truncate to PAGE_SIZE
3991 $TRUNCATE $tmpfile $pagesz
3992 $TRUNCATE $testfile $pagesz
3993 cancel_lru_locks osc ; cancel_lru_locks mdc
3994 cmp -bl $tmpfile $testfile ||
3995 error "file $testfile is corrupted (1)"
3997 # increase size: truncate to 2 x PAGE_SIZE
3999 $TRUNCATE $tmpfile $sz
4000 $TRUNCATE $testfile $sz
4001 cancel_lru_locks osc ; cancel_lru_locks mdc
4002 cmp -bl $tmpfile $testfile ||
4003 error "file $testfile is corrupted (2)"
4005 # truncate to PAGE_SIZE / 2
4007 $TRUNCATE $tmpfile $sz
4008 $TRUNCATE $testfile $sz
4009 cancel_lru_locks osc ; cancel_lru_locks mdc
4010 cmp -bl $tmpfile $testfile ||
4011 error "file $testfile is corrupted (3)"
4013 # truncate to a smaller, non-multiple of PAGE_SIZE, non-multiple of 16
4015 $TRUNCATE $tmpfile $sz
4016 $TRUNCATE $testfile $sz
4017 cancel_lru_locks osc ; cancel_lru_locks mdc
4018 cmp -bl $tmpfile $testfile ||
4019 error "file $testfile is corrupted (4)"
4021 # truncate to a larger, non-multiple of PAGE_SIZE, non-multiple of 16
4023 $TRUNCATE $tmpfile $sz
4024 $TRUNCATE $testfile $sz
4025 cancel_lru_locks osc ; cancel_lru_locks mdc
4026 cmp -bl $tmpfile $testfile ||
4027 error "file $testfile is corrupted (5)"
4029 # truncate to a larger, non-multiple of PAGE_SIZE, in a different page
4030 sz=$((sz+pagesz+30))
4031 $TRUNCATE $tmpfile $sz
4032 $TRUNCATE $testfile $sz
4033 cancel_lru_locks osc ; cancel_lru_locks mdc
4034 cmp -bl $tmpfile $testfile ||
4035 error "file $testfile is corrupted (6)"
4038 cancel_lru_locks osc ; cancel_lru_locks mdc
4040 # write hole in file, data spread on MDT and OST
4041 tr '\0' '2' < /dev/zero |
4042 dd of=$tmpfile bs=1 count=1539 seek=1539074 conv=fsync,notrunc
4043 $LFS setstripe -E 1M -L mdt -E EOF $testfile
4044 cp --sparse=always $tmpfile $testfile
4046 # check that in-memory representation of file is correct
4047 cmp -bl $tmpfile $testfile ||
4048 error "file $testfile is corrupted in memory"
4050 cancel_lru_locks osc ; cancel_lru_locks mdc
4052 # check that file read from server is correct
4053 cmp -bl $tmpfile $testfile ||
4054 error "file $testfile is corrupted on server"
4056 # truncate to a smaller, non-multiple of PAGE_SIZE, non-multiple of 16,
4057 # inside OST part of data
4058 sz=$((1024*1024+13))
4059 $TRUNCATE $tmpfile $sz
4060 $TRUNCATE $testfile $sz
4061 cancel_lru_locks osc ; cancel_lru_locks mdc
4062 cmp -bl $tmpfile $testfile ||
4063 error "file $testfile is corrupted (7)"
4065 # truncate to a smaller, non-multiple of PAGE_SIZE, non-multiple of 16,
4066 # inside MDT part of data
4068 $TRUNCATE $tmpfile $sz
4069 $TRUNCATE $testfile $sz
4070 cancel_lru_locks osc ; cancel_lru_locks mdc
4071 cmp -bl $tmpfile $testfile ||
4072 error "file $testfile is corrupted (8)"
4074 # truncate to a larger, non-multiple of PAGE_SIZE, non-multiple of 16,
4075 # inside MDT part of data
4076 sz=$((1024*1024-13))
4077 $TRUNCATE $tmpfile $sz
4078 $TRUNCATE $testfile $sz
4079 cancel_lru_locks osc ; cancel_lru_locks mdc
4080 cmp -bl $tmpfile $testfile ||
4081 error "file $testfile is corrupted (9)"
4083 # truncate to a larger, non-multiple of PAGE_SIZE, non-multiple of 16,
4084 # inside OST part of data
4086 $TRUNCATE $tmpfile $sz
4087 $TRUNCATE $testfile $sz
4088 cancel_lru_locks osc ; cancel_lru_locks mdc
4089 cmp -bl $tmpfile $testfile ||
4090 error "file $testfile is corrupted (10)"
4094 run_test 50 "DoM encrypted file"
4097 [ "$MDS1_VERSION" -gt $(version_code 2.13.53) ] ||
4098 skip "Need MDS version at least 2.13.53"
4100 mkdir $DIR/$tdir || error "mkdir $tdir"
4102 touch $DIR/$tdir/$tfile || error "touch $tfile"
4103 cp $(which chown) $DIR/$tdir || error "cp chown"
4104 $RUNAS_CMD -u $ID0 $DIR/$tdir/chown $ID0 $DIR/$tdir/$tfile &&
4105 error "chown $tfile should fail"
4106 setcap 'CAP_CHOWN=ep' $DIR/$tdir/chown || error "setcap CAP_CHOWN"
4107 $RUNAS_CMD -u $ID0 $DIR/$tdir/chown $ID0 $DIR/$tdir/$tfile ||
4108 error "chown $tfile"
4109 rm $DIR/$tdir/$tfile || error "rm $tfile"
4111 touch $DIR/$tdir/$tfile || error "touch $tfile"
4112 cp $(which touch) $DIR/$tdir || error "cp touch"
4113 $RUNAS_CMD -u $ID0 $DIR/$tdir/touch $DIR/$tdir/$tfile &&
4114 error "touch should fail"
4115 setcap 'CAP_FOWNER=ep' $DIR/$tdir/touch || error "setcap CAP_FOWNER"
4116 $RUNAS_CMD -u $ID0 $DIR/$tdir/touch $DIR/$tdir/$tfile ||
4117 error "touch $tfile"
4118 rm $DIR/$tdir/$tfile || error "rm $tfile"
4121 for cap in "CAP_DAC_OVERRIDE" "CAP_DAC_READ_SEARCH"; do
4122 touch $DIR/$tdir/$tfile || error "touch $tfile"
4123 chmod 600 $DIR/$tdir/$tfile || error "chmod $tfile"
4124 cp $(which cat) $DIR/$tdir || error "cp cat"
4125 $RUNAS_CMD -u $ID0 $DIR/$tdir/cat $DIR/$tdir/$tfile &&
4126 error "cat should fail"
4127 setcap $cap=ep $DIR/$tdir/cat || error "setcap $cap"
4128 $RUNAS_CMD -u $ID0 $DIR/$tdir/cat $DIR/$tdir/$tfile ||
4130 rm $DIR/$tdir/$tfile || error "rm $tfile"
4133 run_test 51 "FS capabilities ==============="
4136 local testfile=$DIR/$tdir/$tfile
4137 local tmpfile=$TMP/$tfile
4138 local mirror1=$TMP/$tfile.mirror1
4139 local mirror2=$TMP/$tfile.mirror2
4141 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4142 skip "client encryption not supported"
4144 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4145 skip "need dummy encryption support"
4147 [[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
4149 stack_trap "cleanup_for_enc_tests $tmpfile $mirror1 $mirror2" EXIT
4152 dd if=/dev/urandom of=$tmpfile bs=5000 count=1 conv=fsync
4154 $LFS mirror create -N -i0 -N -i1 $testfile ||
4155 error "could not create mirror"
4157 dd if=$tmpfile of=$testfile bs=5000 count=1 conv=fsync ||
4158 error "could not write to $testfile"
4160 $LFS mirror resync $testfile ||
4161 error "could not resync mirror"
4163 $LFS mirror verify -v $testfile ||
4164 error "verify mirror failed"
4166 $LFS mirror read -N 1 -o $mirror1 $testfile ||
4167 error "could not read from mirror 1"
4169 cmp -bl $tmpfile $mirror1 ||
4170 error "mirror 1 is corrupted"
4172 $LFS mirror read -N 2 -o $mirror2 $testfile ||
4173 error "could not read from mirror 2"
4175 cmp -bl $tmpfile $mirror2 ||
4176 error "mirror 2 is corrupted"
4178 tr '\0' '2' < /dev/zero |
4179 dd of=$tmpfile bs=1 count=9000 conv=fsync
4181 $LFS mirror write -N 1 -i $tmpfile $testfile ||
4182 error "could not write to mirror 1"
4184 $LFS mirror verify -v $testfile &&
4185 error "mirrors should be different"
4187 rm -f $testfile $mirror1 $mirror2
4189 $LFS setstripe -c1 -i0 $testfile
4190 dd if=$tmpfile of=$testfile bs=9000 count=1 conv=fsync ||
4191 error "write to $testfile failed"
4192 $LFS getstripe $testfile
4195 $LFS migrate -i1 $testfile ||
4196 error "migrate $testfile failed"
4197 $LFS getstripe $testfile
4198 stripe=$($LFS getstripe -i $testfile)
4199 [ $stripe -eq 1 ] || error "migrate file $testfile failed"
4202 cmp -bl $tmpfile $testfile ||
4203 error "migrated file is corrupted"
4205 $LFS mirror extend -N -i0 $testfile ||
4206 error "mirror extend $testfile failed"
4207 $LFS getstripe $testfile
4208 mirror_count=$($LFS getstripe -N $testfile)
4209 [ $mirror_count -eq 2 ] ||
4210 error "mirror extend file $testfile failed (1)"
4211 stripe=$($LFS getstripe --mirror-id=1 -i $testfile)
4212 [ $stripe -eq 1 ] || error "mirror extend file $testfile failed (2)"
4213 stripe=$($LFS getstripe --mirror-id=2 -i $testfile)
4214 [ $stripe -eq 0 ] || error "mirror extend file $testfile failed (3)"
4217 $LFS mirror verify -v $testfile ||
4218 error "mirror verify failed"
4219 $LFS mirror read -N 1 -o $mirror1 $testfile ||
4220 error "read from mirror 1 failed"
4221 cmp -bl $tmpfile $mirror1 ||
4222 error "corruption of mirror 1"
4223 $LFS mirror read -N 2 -o $mirror2 $testfile ||
4224 error "read from mirror 2 failed"
4225 cmp -bl $tmpfile $mirror2 ||
4226 error "corruption of mirror 2"
4228 $LFS mirror split --mirror-id 1 -f ${testfile}.mirror $testfile &&
4229 error "mirror split -f should fail"
4231 $LFS mirror split --mirror-id 1 $testfile &&
4232 error "mirror split without -d should fail"
4234 $LFS mirror split --mirror-id 1 -d $testfile ||
4235 error "mirror split failed"
4236 $LFS getstripe $testfile
4237 mirror_count=$($LFS getstripe -N $testfile)
4238 [ $mirror_count -eq 1 ] ||
4239 error "mirror split file $testfile failed (1)"
4240 stripe=$($LFS getstripe --mirror-id=1 -i $testfile)
4241 [ -z "$stripe" ] || error "mirror extend file $testfile failed (2)"
4242 stripe=$($LFS getstripe --mirror-id=2 -i $testfile)
4243 [ $stripe -eq 0 ] || error "mirror extend file $testfile failed (3)"
4246 cmp -bl $tmpfile $testfile ||
4247 error "extended/split file is corrupted"
4249 run_test 52 "Mirrored encrypted file"
4252 local testfile=$DIR/$tdir/$tfile
4253 local testfile2=$DIR2/$tdir/$tfile
4254 local tmpfile=$TMP/$tfile.tmp
4255 local resfile=$TMP/$tfile.res
4259 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4260 skip "client encryption not supported"
4262 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4263 skip "need dummy encryption support"
4265 pagesz=$(getconf PAGESIZE)
4266 [[ $pagesz == 65536 ]] || skip "Need 64K PAGE_SIZE client"
4268 do_node $mds1_HOST \
4269 "mount.lustre --help |& grep -q 'test_dummy_encryption:'" ||
4270 skip "need dummy encryption support on MDS client mount"
4272 # this test is probably useless now, but may turn out to be useful when
4273 # Lustre supports servers with PAGE_SIZE != 4KB
4274 pagesz=$(do_node $mds1_HOST getconf PAGESIZE)
4275 [[ $pagesz == 4096 ]] || skip "Need 4K PAGE_SIZE MDS client"
4277 stack_trap cleanup_for_enc_tests EXIT
4278 stack_trap "zconf_umount $mds1_HOST $MOUNT2" EXIT
4281 $LFS setstripe -c1 -i0 $testfile
4283 # write from 1st client
4284 cat /dev/urandom | tr -dc 'a-zA-Z0-9' |
4285 dd of=$tmpfile bs=$((pagesz+3)) count=2 conv=fsync
4286 dd if=$tmpfile of=$testfile bs=$((pagesz+3)) count=2 conv=fsync ||
4287 error "could not write to $testfile (1)"
4289 # read from 2nd client
4290 # mount and IOs must be done in the same shell session, otherwise
4291 # encryption key in session keyring is missing
4292 do_node $mds1_HOST "mkdir -p $MOUNT2"
4293 do_node $mds1_HOST \
4294 "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
4295 $MGSNID:/$FSNAME $MOUNT2 && \
4296 dd if=$testfile2 of=$resfile bs=$((pagesz+3)) count=2" ||
4297 error "could not read from $testfile2 (1)"
4300 filemd5=$(do_node $mds1_HOST md5sum $resfile | awk '{print $1}')
4301 [ $filemd5 = $(md5sum $tmpfile | awk '{print $1}') ] ||
4302 error "file is corrupted (1)"
4303 do_node $mds1_HOST rm -f $resfile
4306 # truncate from 2nd client
4307 $TRUNCATE $tmpfile $((pagesz+3))
4308 zconf_umount $mds1_HOST $MOUNT2 ||
4309 error "umount $mds1_HOST $MOUNT2 failed (1)"
4310 do_node $mds1_HOST "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
4311 $MGSNID:/$FSNAME $MOUNT2 && \
4312 $TRUNCATE $testfile2 $((pagesz+3))" ||
4313 error "could not truncate $testfile2 (1)"
4316 cmp -bl $tmpfile $testfile ||
4317 error "file is corrupted (2)"
4318 rm -f $tmpfile $testfile
4320 zconf_umount $mds1_HOST $MOUNT2 ||
4321 error "umount $mds1_HOST $MOUNT2 failed (2)"
4324 do_node $mds1_HOST \
4325 dd if=/dev/urandom of=$tmpfile bs=$((pagesz+3)) count=2 conv=fsync
4326 # write from 2nd client
4327 do_node $mds1_HOST \
4328 "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
4329 $MGSNID:/$FSNAME $MOUNT2 && \
4330 dd if=$tmpfile of=$testfile2 bs=$((pagesz+3)) count=2 conv=fsync" ||
4331 error "could not write to $testfile2 (2)"
4333 # read from 1st client
4334 dd if=$testfile of=$resfile bs=$((pagesz+3)) count=2 ||
4335 error "could not read from $testfile (2)"
4338 filemd5=$(do_node $mds1_HOST md5sum -b $tmpfile | awk '{print $1}')
4339 [ $filemd5 = $(md5sum -b $resfile | awk '{print $1}') ] ||
4340 error "file is corrupted (3)"
4344 # truncate from 1st client
4345 do_node $mds1_HOST "$TRUNCATE $tmpfile $((pagesz+3))"
4346 $TRUNCATE $testfile $((pagesz+3)) ||
4347 error "could not truncate $testfile (2)"
4350 zconf_umount $mds1_HOST $MOUNT2 ||
4351 error "umount $mds1_HOST $MOUNT2 failed (3)"
4352 do_node $mds1_HOST "$MOUNT_CMD -o ${MOUNT_OPTS},test_dummy_encryption \
4353 $MGSNID:/$FSNAME $MOUNT2 && \
4354 cmp -bl $tmpfile $testfile2" ||
4355 error "file is corrupted (4)"
4357 do_node $mds1_HOST rm -f $tmpfile
4360 run_test 53 "Mixed PAGE_SIZE clients"
4363 local testdir=$DIR/$tdir/$ID0
4364 local testdir2=$DIR2/$tdir/$ID0
4365 local testfile=$testdir/$tfile
4366 local testfile2=$testdir/${tfile}withveryverylongnametoexercisecode
4367 local testfile3=$testdir/_${tfile}
4368 local tmpfile=$TMP/${tfile}.tmp
4369 local resfile=$TMP/${tfile}.res
4374 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4375 skip "client encryption not supported"
4377 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4378 skip "need dummy encryption support"
4380 which fscrypt || skip "This test needs fscrypt userspace tool"
4382 yes | fscrypt setup --force --verbose ||
4383 error "fscrypt global setup failed"
4384 sed -i 's/\(.*\)policy_version\(.*\):\(.*\)\"[0-9]*\"\(.*\)/\1policy_version\2:\3"2"\4/' \
4386 yes | fscrypt setup --verbose $MOUNT ||
4387 error "fscrypt setup $MOUNT failed"
4389 chown -R $ID0:$ID0 $testdir
4391 echo -e 'mypass\nmypass' | su - $USER0 -c "fscrypt encrypt --verbose \
4392 --source=custom_passphrase --name=protector $testdir" ||
4393 error "fscrypt encrypt failed"
4395 echo -e 'mypass\nmypass' | su - $USER0 -c "fscrypt encrypt --verbose \
4396 --source=custom_passphrase --name=protector2 $testdir" &&
4397 error "second fscrypt encrypt should have failed"
4399 mkdir -p ${testdir}2 || error "mkdir ${testdir}2 failed"
4400 touch ${testdir}2/f || error "mkdir ${testdir}2/f failed"
4403 echo -e 'mypass\nmypass' | fscrypt encrypt --verbose \
4404 --source=custom_passphrase --name=protector3 ${testdir}2 &&
4405 error "fscrypt encrypt on non-empty dir should have failed"
4407 $RUNAS dd if=/dev/urandom of=$testfile bs=127 count=1 conv=fsync ||
4408 error "write to encrypted file $testfile failed"
4409 cp $testfile $tmpfile
4410 $RUNAS dd if=/dev/urandom of=$testfile2 bs=127 count=1 conv=fsync ||
4411 error "write to encrypted file $testfile2 failed"
4412 $RUNAS dd if=/dev/urandom of=$testfile3 bs=127 count=1 conv=fsync ||
4413 error "write to encrypted file $testfile3 failed"
4414 $RUNAS mkdir $testdir/subdir || error "mkdir subdir failed"
4415 $RUNAS touch $testdir/subdir/subfile || error "mkdir subdir failed"
4417 $RUNAS fscrypt lock --verbose $testdir ||
4418 error "fscrypt lock $testdir failed (1)"
4420 $RUNAS ls -R $testdir || error "ls -R $testdir failed"
4421 local filecount=$($RUNAS find $testdir -type f | wc -l)
4422 [ $filecount -eq 4 ] || error "found $filecount files"
4424 # check enable_filename_encryption default value
4425 # tunable only available for client built against embedded llcrypt
4426 $LCTL get_param mdc.*.connect_flags | grep -q name_encryption &&
4427 nameenc=$(lctl get_param -n llite.*.enable_filename_encryption |
4429 # If client is built against in-kernel fscrypt, it is not possible
4430 # to decide to encrypt file names or not: they are always encrypted.
4431 if [ -n "$nameenc" ]; then
4432 [ $nameenc -eq 0 ] ||
4433 error "enable_filename_encryption should be 0 by default"
4435 # $testfile, $testfile2 and $testfile3 should exist because
4436 # names are not encrypted
4438 error "$testfile should exist because name not encrypted"
4439 [ -f $testfile2 ] ||
4440 error "$testfile2 should exist because name not encrypted"
4441 [ -f $testfile3 ] ||
4442 error "$testfile3 should exist because name not encrypted"
4444 [ $? -eq 0 ] || error "cannot stat $testfile3 without key"
4447 scrambledfiles=( $(find $testdir/ -maxdepth 1 -type f) )
4448 $RUNAS hexdump -C ${scrambledfiles[0]} &&
4449 error "reading ${scrambledfiles[0]} should fail without key"
4451 $RUNAS touch ${testfile}.nokey &&
4452 error "touch ${testfile}.nokey should have failed without key"
4454 echo mypass | $RUNAS fscrypt unlock --verbose $testdir ||
4455 error "fscrypt unlock $testdir failed (1)"
4457 $RUNAS cat $testfile > $resfile ||
4458 error "reading $testfile failed"
4460 cmp -bl $tmpfile $resfile || error "file read differs from file written"
4462 [ $? -eq 0 ] || error "cannot stat $testfile3 with key"
4464 $RUNAS fscrypt lock --verbose $testdir ||
4465 error "fscrypt lock $testdir failed (2)"
4467 $RUNAS hexdump -C ${scrambledfiles[1]} &&
4468 error "reading ${scrambledfiles[1]} should fail without key"
4470 # server local client incompatible with SSK keys installed
4471 if [ "$SHARED_KEY" != true ]; then
4473 stack_trap umount_mds_client EXIT
4474 do_facet $SINGLEMDS touch $DIR2/$tdir/newfile
4475 mdsscrambledfile=$(do_facet $SINGLEMDS find $testdir2/ \
4476 -maxdepth 1 -type f | head -n1)
4477 [ -n "$mdsscrambledfile" ] || error "could not find file"
4478 do_facet $SINGLEMDS cat "$mdsscrambledfile" &&
4479 error "reading $mdsscrambledfile should fail on MDS"
4480 do_facet $SINGLEMDS "echo aaa >> \"$mdsscrambledfile\"" &&
4481 error "writing $mdsscrambledfile should fail on MDS"
4482 do_facet $SINGLEMDS $MULTIOP $testdir2/fileA m &&
4483 error "creating $testdir2/fileA should fail on MDS"
4484 do_facet $SINGLEMDS mkdir $testdir2/dirA &&
4485 error "mkdir $testdir2/dirA should fail on MDS"
4486 do_facet $SINGLEMDS ln -s $DIR2/$tdir/newfile $testdir2/sl1 &&
4487 error "ln -s $testdir2/sl1 should fail on MDS"
4488 do_facet $SINGLEMDS ln $DIR2/$tdir/newfile $testdir2/hl1 &&
4489 error "ln $testdir2/hl1 should fail on MDS"
4490 do_facet $SINGLEMDS mv "$mdsscrambledfile" $testdir2/fB &&
4491 error "mv $mdsscrambledfile should fail on MDS"
4492 do_facet $SINGLEMDS mrename "$mdsscrambledfile" $testdir2/fB &&
4493 error "mrename $mdsscrambledfile should fail on MDS"
4494 do_facet $SINGLEMDS rm -f $DIR2/$tdir/newfile
4497 echo mypass | $RUNAS fscrypt unlock --verbose $testdir ||
4498 error "fscrypt unlock $testdir failed (2)"
4501 $RUNAS fscrypt lock --verbose $testdir ||
4502 error "fscrypt lock $testdir failed (3)"
4504 rm -rf $tmpfile $resfile $testdir ${testdir}2 $MOUNT/.fscrypt
4506 # remount client with subdirectory mount
4507 umount_client $MOUNT || error "umount $MOUNT failed (1)"
4508 export FILESET=/$tdir
4509 mount_client $MOUNT ${MOUNT_OPTS} || error "remount failed (1)"
4513 # setup encryption from inside this subdir mount
4514 # the .fscrypt directory is going to be created at the real fs root
4515 yes | fscrypt setup --verbose $MOUNT ||
4516 error "fscrypt setup $MOUNT failed (2)"
4517 testdir=$MOUNT/vault
4519 chown -R $ID0:$ID0 $testdir
4520 fid1=$(path2fid $MOUNT/.fscrypt)
4521 echo "With FILESET $tdir, .fscrypt FID is $fid1"
4523 # enable name encryption, only valid if built against embedded llcrypt
4524 if [ -n "$nameenc" ]; then
4525 do_facet mgs $LCTL set_param -P \
4526 llite.*.enable_filename_encryption=1
4528 error "set_param -P \
4529 llite.*.enable_filename_encryption failed"
4531 wait_update_facet --verbose client \
4532 "$LCTL get_param -n llite.*.enable_filename_encryption \
4534 error "enable_filename_encryption not set on client"
4537 # encrypt 'vault' dir inside the subdir mount
4538 echo -e 'mypass\nmypass' | su - $USER0 -c "fscrypt encrypt --verbose \
4539 --source=custom_passphrase --name=protector $testdir" ||
4540 error "fscrypt encrypt failed"
4544 $RUNAS cp $tmpfile $testdir/encfile
4546 $RUNAS fscrypt lock --verbose $testdir ||
4547 error "fscrypt lock $testdir failed (4)"
4549 # encfile should actually have its name encrypted
4550 if [ -n "$nameenc" ]; then
4551 [ -f $testdir/encfile ] &&
4552 error "encfile name should be encrypted"
4554 filecount=$(find $testdir -type f | wc -l)
4555 [ $filecount -eq 1 ] || error "found $filecount files instead of 1"
4557 # remount client with encrypted dir as subdirectory mount
4558 umount_client $MOUNT || error "umount $MOUNT failed (2)"
4559 export FILESET=/$tdir/vault
4560 mount_client $MOUNT ${MOUNT_OPTS} || error "remount failed (2)"
4564 fid2=$(path2fid $MOUNT/.fscrypt)
4565 echo "With FILESET $tdir/vault, .fscrypt FID is $fid2"
4566 [ "$fid1" == "$fid2" ] || error "fid1 $fid1 != fid2 $fid2 (1)"
4568 # all content seen by this mount is encrypted, but .fscrypt is virtually
4569 # presented, letting us call fscrypt lock/unlock
4570 echo mypass | $RUNAS fscrypt unlock --verbose $MOUNT ||
4571 error "fscrypt unlock $MOUNT failed (3)"
4574 [ $(cat $MOUNT/encfile) == "abc" ] || error "cat encfile failed"
4576 # remount client without subdir mount
4577 umount_client $MOUNT || error "umount $MOUNT failed (3)"
4578 mount_client $MOUNT ${MOUNT_OPTS} || error "remount failed (3)"
4581 fid2=$(path2fid $MOUNT/.fscrypt)
4582 echo "Without FILESET, .fscrypt FID is $fid2"
4583 [ "$fid1" == "$fid2" ] || error "fid1 $fid1 != fid2 $fid2 (2)"
4585 # because .fscrypt was actually created at the real root of the fs,
4586 # we can call fscrypt lock/unlock on the encrypted dir
4587 echo mypass | $RUNAS fscrypt unlock --verbose $DIR/$tdir/vault ||
4588 error "fscrypt unlock $$DIR/$tdir/vault failed (4)"
4591 echo c >> $DIR/$tdir/vault/encfile || error "write to encfile failed"
4593 rm -rf $DIR/$tdir/vault/*
4594 $RUNAS fscrypt lock --verbose $DIR/$tdir/vault ||
4595 error "fscrypt lock $DIR/$tdir/vault failed (5)"
4597 # disable name encryption, only valid if built against embedded llcrypt
4598 if [ -n "$nameenc" ]; then
4599 do_facet mgs $LCTL set_param -P \
4600 llite.*.enable_filename_encryption=0
4602 error "set_param -P \
4603 llite.*.enable_filename_encryption failed"
4605 wait_update_facet --verbose client \
4606 "$LCTL get_param -n llite.*.enable_filename_encryption \
4608 error "enable_filename_encryption not set back to default"
4611 rm -rf $tmpfile $MOUNT/.fscrypt
4613 run_test 54 "Encryption policies with fscrypt"
4617 if is_mounted $MOUNT; then
4618 umount_client $MOUNT || error "umount $MOUNT failed"
4621 do_facet mgs $LCTL nodemap_del c0
4622 do_facet mgs $LCTL nodemap_modify --name default \
4623 --property admin --value 0
4624 do_facet mgs $LCTL nodemap_modify --name default \
4625 --property trusted --value 0
4626 wait_nm_sync default admin_nodemap
4627 wait_nm_sync default trusted_nodemap
4629 do_facet mgs $LCTL nodemap_activate 0
4630 wait_nm_sync active 0
4632 if $SHARED_KEY; then
4633 export SK_UNIQUE_NM=false
4637 mount_client $MOUNT ${MOUNT_OPTS} || error "remount failed"
4638 if [ "$MOUNT_2" ]; then
4639 mount_client $MOUNT2 ${MOUNT_OPTS} || error "remount failed"
4645 (( $MDS1_VERSION > $(version_code 2.12.6.2) )) ||
4646 skip "Need MDS version at least 2.12.6.3"
4651 mkdir -p $DIR/$tdir/$USER0/testdir_groups
4652 chown root:$USER0 $DIR/$tdir/$USER0
4653 chmod 770 $DIR/$tdir/$USER0
4654 chmod g+s $DIR/$tdir/$USER0
4655 chown $USER0:$USER0 $DIR/$tdir/$USER0/testdir_groups
4656 chmod 770 $DIR/$tdir/$USER0/testdir_groups
4657 chmod g+s $DIR/$tdir/$USER0/testdir_groups
4659 # unmount client completely
4660 umount_client $MOUNT || error "umount $MOUNT failed"
4661 if is_mounted $MOUNT2; then
4662 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
4665 do_nodes $(comma_list $(all_mdts_nodes)) \
4666 $LCTL set_param mdt.*.identity_upcall=NONE
4668 stack_trap cleanup_55 EXIT
4670 do_facet mgs $LCTL nodemap_activate 1
4673 do_facet mgs $LCTL nodemap_del c0 || true
4674 wait_nm_sync c0 id ''
4676 do_facet mgs $LCTL nodemap_modify --name default \
4677 --property admin --value 1
4678 do_facet mgs $LCTL nodemap_modify --name default \
4679 --property trusted --value 1
4680 wait_nm_sync default admin_nodemap
4681 wait_nm_sync default trusted_nodemap
4683 client_ip=$(host_nids_address $HOSTNAME $NETTYPE)
4684 client_nid=$(h2nettype $client_ip)
4685 do_facet mgs $LCTL nodemap_add c0
4686 do_facet mgs $LCTL nodemap_add_range \
4687 --name c0 --range $client_nid
4688 do_facet mgs $LCTL nodemap_modify --name c0 \
4689 --property admin --value 0
4690 do_facet mgs $LCTL nodemap_modify --name c0 \
4691 --property trusted --value 1
4692 wait_nm_sync c0 admin_nodemap
4693 wait_nm_sync c0 trusted_nodemap
4695 if $SHARED_KEY; then
4696 export SK_UNIQUE_NM=true
4697 # set some generic fileset to trigger SSK code
4701 # remount client to take nodemap into account
4702 zconf_mount_clients $HOSTNAME $MOUNT $MOUNT_OPTS ||
4703 error "remount failed"
4707 euid_access $USER0 $DIR/$tdir/$USER0/testdir_groups/file
4709 run_test 55 "access with seteuid"
4712 local testfile=$DIR/$tdir/$tfile
4714 [[ $(facet_fstype ost1) == zfs ]] && skip "skip ZFS backend"
4716 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4717 skip "client encryption not supported"
4719 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4720 skip "need dummy encryption support"
4722 [[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
4724 stack_trap cleanup_for_enc_tests EXIT
4727 $LFS setstripe -c1 $testfile
4728 dd if=/dev/urandom of=$testfile bs=1M count=3 conv=fsync
4729 filefrag -v $testfile || error "filefrag $testfile failed"
4730 (( $(filefrag -v $testfile | grep -c encrypted) >= 1 )) ||
4731 error "filefrag $testfile does not show encrypted flag"
4732 (( $(filefrag -v $testfile | grep -c encoded) >= 1 )) ||
4733 error "filefrag $testfile does not show encoded flag"
4735 run_test 56 "FIEMAP on encrypted file"
4738 local testdir=$DIR/$tdir/mytestdir
4739 local testfile=$DIR/$tdir/$tfile
4741 [[ $(facet_fstype ost1) == zfs ]] && skip "skip ZFS backend"
4743 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4744 skip "client encryption not supported"
4746 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4747 skip "need dummy encryption support"
4751 setfattr -n security.c -v myval $testdir &&
4752 error "setting xattr on $testdir should have failed (1.1)"
4753 setfattr -n encryption.c -v myval $testdir &&
4754 error "setting xattr on $testdir should have failed (1.2)"
4756 setfattr -n security.c -v myval $testfile &&
4757 error "setting xattr on $testfile should have failed (1.1)"
4758 setfattr -n encryption.c -v myval $testfile &&
4759 error "setting xattr on $testfile should have failed (1.2)"
4763 stack_trap cleanup_for_enc_tests EXIT
4767 if [ $(getfattr -n security.c $testdir 2>&1 |
4768 grep -ci "Operation not permitted") -eq 0 ]; then
4769 error "getting xattr on $testdir should have failed (1.1)"
4771 if [ $(getfattr -n encryption.c $testdir 2>&1 |
4772 grep -ci "Operation not supported") -eq 0 ]; then
4773 error "getting xattr on $testdir should have failed (1.2)"
4775 getfattr -d -m - $testdir 2>&1 | grep security\.c &&
4776 error "listing xattrs on $testdir should not expose security.c"
4777 getfattr -d -m - $testdir 2>&1 | grep encryption\.c &&
4778 error "listing xattrs on $testdir should not expose encryption.c"
4779 if [ $(setfattr -n security.c -v myval $testdir 2>&1 |
4780 grep -ci "Operation not permitted") -eq 0 ]; then
4781 error "setting xattr on $testdir should have failed (2.1)"
4783 if [ $(setfattr -n encryption.c -v myval $testdir 2>&1 |
4784 grep -ci "Operation not supported") -eq 0 ]; then
4785 error "setting xattr on $testdir should have failed (2.2)"
4788 if [ $(getfattr -n security.c $testfile 2>&1 |
4789 grep -ci "Operation not permitted") -eq 0 ]; then
4790 error "getting xattr on $testfile should have failed (1.1)"
4792 if [ $(getfattr -n encryption.c $testfile 2>&1 |
4793 grep -ci "Operation not supported") -eq 0 ]; then
4794 error "getting xattr on $testfile should have failed (1.2)"
4796 getfattr -d -m - $testfile 2>&1 | grep security\.c &&
4797 error "listing xattrs on $testfile should not expose security.c"
4798 getfattr -d -m - $testfile 2>&1 | grep encryption\.c &&
4799 error "listing xattrs on $testfile should not expose encryption.c"
4800 if [ $(setfattr -n security.c -v myval $testfile 2>&1 |
4801 grep -ci "Operation not permitted") -eq 0 ]; then
4802 error "setting xattr on $testfile should have failed (2.1)"
4804 if [ $(setfattr -n encryption.c -v myval $testfile 2>&1 |
4805 grep -ci "Operation not supported") -eq 0 ]; then
4806 error "setting xattr on $testfile should have failed (2.2)"
4810 run_test 57 "security.c/encryption.c xattr protection"
4813 local testdir=$DIR/$tdir/mytestdir
4814 local testfile=$DIR/$tdir/$tfile
4816 [[ $(facet_fstype ost1) == zfs ]] && skip "skip ZFS backend"
4818 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4819 skip "client encryption not supported"
4821 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4822 skip "need dummy encryption support"
4824 stack_trap cleanup_for_enc_tests EXIT
4827 touch $DIR/$tdir/$tfile
4828 mkdir $DIR/$tdir/subdir
4832 echo 3 > /proc/sys/vm/drop_caches
4834 ll_decode_linkea $DIR/$tdir/$tfile || error "cannot read $tfile linkea"
4835 ll_decode_linkea $DIR/$tdir/subdir || error "cannot read subdir linkea"
4837 for ((i = 0; i < 1000; i = $((i+1)))); do
4838 mkdir -p $DIR/$tdir/d${i}
4839 touch $DIR/$tdir/f${i}
4840 createmany -m $DIR/$tdir/d${i}/f 5 > /dev/null
4845 echo 3 > /proc/sys/vm/drop_caches
4848 ls -ailR $DIR/$tdir > /dev/null || error "fail to ls"
4850 run_test 58 "access to enc file's xattrs"
4853 local mirror1=$TMP/$tfile.mirror1
4854 local mirror2=$TMP/$tfile.mirror2
4858 $LFS mirror verify -vvv $testfile ||
4859 error "verifying mirror failed (1)"
4860 if [ $($LFS mirror verify -v $testfile 2>&1 |
4861 grep -ci "only valid") -ne 0 ]; then
4862 error "verifying mirror failed (2)"
4865 $LFS mirror read -N 1 -o $mirror1 $testfile ||
4866 error "read from mirror 1 failed"
4867 cmp -bl $reffile $mirror1 ||
4868 error "corruption of mirror 1"
4869 $LFS mirror read -N 2 -o $mirror2 $testfile ||
4870 error "read from mirror 2 failed"
4871 cmp -bl $reffile $mirror2 ||
4872 error "corruption of mirror 2"
4876 local testfile=$DIR/$tdir/$tfile
4877 local tmpfile=$TMP/$tfile
4878 local mirror1=$TMP/$tfile.mirror1
4879 local mirror2=$TMP/$tfile.mirror2
4882 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4883 skip "client encryption not supported"
4885 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4886 skip "need dummy encryption support"
4888 [[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
4890 stack_trap "cleanup_for_enc_tests $tmpfile $mirror1 $mirror2" EXIT
4893 dd if=/dev/urandom of=$tmpfile bs=5000 count=1 conv=fsync
4895 $LFS mirror create -N -i0 -N -i1 $testfile ||
4896 error "could not create mirror"
4897 dd if=$tmpfile of=$testfile bs=5000 count=1 conv=fsync ||
4898 error "could not write to $testfile"
4899 $LFS getstripe $testfile
4901 # remount without dummy encryption key
4902 remount_client_normally
4904 scrambledfile=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -type f)
4905 $LFS mirror resync $scrambledfile ||
4906 error "could not resync mirror"
4908 $LFS mirror verify -vvv $scrambledfile ||
4909 error "mirror verify failed (1)"
4910 if [ $($LFS mirror verify -v $scrambledfile 2>&1 |
4911 grep -ci "only valid") -ne 0 ]; then
4912 error "mirror verify failed (2)"
4915 $LFS mirror read -N 1 -o $mirror1 $scrambledfile &&
4916 error "read from mirror should fail"
4919 remount_client_dummykey
4920 verify_mirror $testfile $tmpfile
4922 run_test 59a "mirror resync of encrypted files without key"
4925 local testfile=$DIR/$tdir/$tfile
4926 local tmpfile=$TMP/$tfile
4927 local mirror1=$TMP/$tfile.mirror1
4928 local mirror2=$TMP/$tfile.mirror2
4931 $LCTL get_param mdc.*.import | grep -q client_encryption ||
4932 skip "client encryption not supported"
4934 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
4935 skip "need dummy encryption support"
4937 [[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
4939 stack_trap "cleanup_for_enc_tests $tmpfile $mirror1 $mirror2" EXIT
4942 tr '\0' '2' < /dev/zero |
4943 dd of=$tmpfile bs=1 count=9000 conv=fsync
4945 $LFS setstripe -c1 -i0 $testfile
4946 dd if=$tmpfile of=$testfile bs=9000 count=1 conv=fsync ||
4947 error "write to $testfile failed"
4948 $LFS getstripe $testfile
4950 # remount without dummy encryption key
4951 remount_client_normally
4953 scrambledfile=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -type f)
4954 $LFS migrate -i1 $scrambledfile ||
4955 error "migrate $scrambledfile failed"
4956 $LFS getstripe $scrambledfile
4957 stripe=$($LFS getstripe -i $scrambledfile)
4958 [ $stripe -eq 1 ] || error "migrate file $scrambledfile failed"
4962 remount_client_dummykey
4963 cmp -bl $tmpfile $testfile ||
4964 error "migrated file is corrupted"
4966 # remount without dummy encryption key
4967 remount_client_normally
4969 $LFS mirror extend -N -i0 $scrambledfile ||
4970 error "mirror extend $scrambledfile failed (1)"
4971 $LFS getstripe $scrambledfile
4972 mirror_count=$($LFS getstripe -N $scrambledfile)
4973 [ $mirror_count -eq 2 ] ||
4974 error "mirror extend file $scrambledfile failed (2)"
4975 stripe=$($LFS getstripe --mirror-id=1 -i $scrambledfile)
4976 [ $stripe -eq 1 ] ||
4977 error "mirror extend file $scrambledfile failed (3)"
4978 stripe=$($LFS getstripe --mirror-id=2 -i $scrambledfile)
4979 [ $stripe -eq 0 ] ||
4980 error "mirror extend file $scrambledfile failed (4)"
4982 $LFS mirror verify -vvv $scrambledfile ||
4983 error "mirror verify failed (1)"
4984 if [ $($LFS mirror verify -v $scrambledfile 2>&1 |
4985 grep -ci "only valid") -ne 0 ]; then
4986 error "mirror verify failed (2)"
4990 remount_client_dummykey
4991 verify_mirror $testfile $tmpfile
4993 # remount without dummy encryption key
4994 remount_client_normally
4996 $LFS mirror split --mirror-id 1 -d $scrambledfile ||
4997 error "mirror split file $scrambledfile failed (1)"
4998 $LFS getstripe $scrambledfile
4999 mirror_count=$($LFS getstripe -N $scrambledfile)
5000 [ $mirror_count -eq 1 ] ||
5001 error "mirror split file $scrambledfile failed (2)"
5002 stripe=$($LFS getstripe --mirror-id=1 -i $scrambledfile)
5003 [ -z "$stripe" ] || error "mirror split file $scrambledfile failed (3)"
5004 stripe=$($LFS getstripe --mirror-id=2 -i $scrambledfile)
5005 [ $stripe -eq 0 ] || error "mirror split file $scrambledfile failed (4)"
5008 remount_client_dummykey
5010 cmp -bl $tmpfile $testfile ||
5011 error "extended/split file is corrupted"
5013 run_test 59b "migrate/extend/split of encrypted files without key"
5016 local dirname=$DIR/$tdir/subdir
5019 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5020 skip "client encryption not supported"
5022 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5023 skip "need dummy encryption support"
5025 [[ $MDSCOUNT -ge 2 ]] || skip_env "needs >= 2 MDTs"
5027 (( "$MDS1_VERSION" > $(version_code 2.14.54.54) )) ||
5028 skip "MDT migration not supported with older server"
5030 stack_trap cleanup_for_enc_tests EXIT
5033 $LFS setdirstripe -i 0 $dirname
5034 echo b > $dirname/subf
5036 # remount without dummy encryption key
5037 remount_client_normally
5039 scrambleddir=$(find $DIR/$tdir/ -maxdepth 1 -mindepth 1 -type d)
5041 # migrate a non-empty encrypted dir
5042 $LFS migrate -m 1 $scrambleddir ||
5043 error "migrate $scrambleddir between MDTs failed (1)"
5045 stripe=$($LFS getdirstripe -i $scrambleddir)
5046 [ $stripe -eq 1 ] ||
5047 error "migrate $scrambleddir between MDTs failed (2)"
5051 [ -f $dirname/subf ] ||
5052 error "migrate $scrambleddir between MDTs failed (3)"
5053 [ $(cat $dirname/subf) == "b" ] ||
5054 error "migrate $scrambleddir between MDTs failed (4)"
5056 run_test 59c "MDT migrate of encrypted files without key"
5059 local testdir=$DIR/$tdir/mytestdir
5060 local testfile=$DIR/$tdir/$tfile
5062 (( $MDS1_VERSION > $(version_code 2.14.53) )) ||
5063 skip "Need MDS version at least 2.14.53"
5065 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5066 skip "client encryption not supported"
5068 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5069 skip "need dummy encryption support"
5071 stack_trap cleanup_for_enc_tests EXIT
5074 echo a > $DIR/$tdir/file1
5075 mkdir $DIR/$tdir/subdir
5076 echo b > $DIR/$tdir/subdir/subfile1
5079 # unmount client completely
5080 umount_client $MOUNT || error "umount $MOUNT failed"
5081 if is_mounted $MOUNT2; then
5082 umount_client $MOUNT2 || error "umount $MOUNT2 failed"
5085 # remount client with subdirectory mount
5086 export FILESET=/$tdir
5087 mount_client $MOUNT ${MOUNT_OPTS} || error "remount failed"
5088 if [ "$MOUNT_2" ]; then
5089 mount_client $MOUNT2 ${MOUNT_OPTS} || error "remount failed"
5093 ls -Rl $DIR || error "ls -Rl $DIR failed (1)"
5096 remount_client_dummykey
5099 ls -Rl $DIR || error "ls -Rl $DIR failed (2)"
5100 cat $DIR/file1 || error "cat $DIR/$tdir/file1 failed"
5101 cat $DIR/subdir/subfile1 ||
5102 error "cat $DIR/$tdir/subdir/subfile1 failed"
5104 run_test 60 "Subdirmount of encrypted dir"
5107 local testfile=$DIR/$tdir/$tfile
5110 readonly=$(do_facet mgs \
5111 lctl get_param -n nodemap.default.readonly_mount)
5112 [ -n "$readonly" ] ||
5113 skip "Server does not have readonly_mount nodemap flag"
5115 stack_trap cleanup_nodemap_after_enc_tests EXIT
5116 umount_client $MOUNT || error "umount $MOUNT failed (1)"
5118 # Activate nodemap, and mount rw.
5119 # Should succeed as rw mount is not forbidden on default nodemap
5121 do_facet mgs $LCTL nodemap_activate 1
5123 do_facet mgs $LCTL nodemap_modify --name default \
5124 --property admin --value 1
5125 do_facet mgs $LCTL nodemap_modify --name default \
5126 --property trusted --value 1
5127 wait_nm_sync default admin_nodemap
5128 wait_nm_sync default trusted_nodemap
5129 readonly=$(do_facet mgs \
5130 lctl get_param -n nodemap.default.readonly_mount)
5131 [ $readonly -eq 0 ] || error "wrong default value for readonly_mount"
5133 mount_client $MOUNT ${MOUNT_OPTS},rw ||
5134 error "mount '-o rw' failed with default"
5136 findmnt $MOUNT --output=options -n -f | grep -q "rw," ||
5137 error "should be rw mount"
5138 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
5139 echo a > $testfile || error "write $testfile failed"
5140 umount_client $MOUNT || error "umount $MOUNT failed (2)"
5142 # Now enforce read-only, and retry.
5143 do_facet mgs $LCTL nodemap_modify --name default \
5144 --property readonly_mount --value 1
5145 wait_nm_sync default readonly_mount
5146 mount_client $MOUNT ${MOUNT_OPTS} ||
5147 error "mount failed"
5148 findmnt $MOUNT --output=options -n -f | grep -q "ro," ||
5149 error "mount should have been turned into ro"
5150 cat $testfile || error "read $testfile failed (1)"
5151 echo b > $testfile && error "write $testfile should fail (1)"
5152 umount_client $MOUNT || error "umount $MOUNT failed (3)"
5153 mount_client $MOUNT ${MOUNT_OPTS},rw ||
5154 error "mount '-o rw' failed"
5155 findmnt $MOUNT --output=options -n -f | grep -q "ro," ||
5156 error "mount rw should have been turned into ro"
5157 cat $testfile || error "read $testfile failed (2)"
5158 echo b > $testfile && error "write $testfile should fail (2)"
5159 umount_client $MOUNT || error "umount $MOUNT failed (4)"
5160 mount_client $MOUNT ${MOUNT_OPTS},ro ||
5161 error "mount '-o ro' failed"
5163 cat $testfile || error "read $testfile failed (3)"
5164 echo b > $testfile && error "write $testfile should fail (3)"
5165 umount_client $MOUNT || error "umount $MOUNT failed (5)"
5167 run_test 61 "Nodemap enforces read-only mount"
5170 local testdir=$DIR/$tdir/mytestdir
5171 local testfile=$DIR/$tdir/$tfile
5173 [[ $(facet_fstype ost1) == zfs ]] && skip "skip ZFS backend"
5175 (( $MDS1_VERSION > $(version_code 2.15.51) )) ||
5176 skip "Need MDS version at least 2.15.51"
5178 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5179 skip "client encryption not supported"
5181 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5182 skip "need dummy encryption support"
5184 stack_trap cleanup_for_enc_tests EXIT
5187 lfs setstripe -c -1 $DIR/$tdir
5188 touch $DIR/$tdir/${tfile}_1 || error "touch ${tfile}_1 failed"
5189 dd if=/dev/zero of=$DIR/$tdir/${tfile}_2 bs=1 count=1 conv=fsync ||
5190 error "dd ${tfile}_2 failed"
5192 # unmount the Lustre filesystem
5193 stopall || error "stopping for e2fsck run"
5195 # run e2fsck on the MDT and OST devices
5196 local mds_host=$(facet_active_host $SINGLEMDS)
5197 local ost_host=$(facet_active_host ost1)
5198 local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
5199 local ost_dev=$(ostdevname 1)
5201 run_e2fsck $mds_host $mds_dev "-n"
5202 run_e2fsck $ost_host $ost_dev "-n"
5204 # mount the Lustre filesystem
5205 setupall || error "remounting the filesystem failed"
5207 run_test 62 "e2fsck with encrypted files"
5212 for path in "${paths[@]}"; do
5220 for path in "${paths[@]}"; do
5221 fids+=("$(lfs path2fid $path)")
5226 for fid in "${fids[@]}"; do
5228 respath=$(lfs fid2path $MOUNT $fid)
5229 echo -e "\t" $respath
5230 ls -li $respath >/dev/null
5231 [ $? -eq 0 ] || error "fid2path $fid failed"
5238 local vaultdir1=$DIR/$tdir/vault1==dir
5239 local vaultdir2=$DIR/$tdir/vault2==dir
5240 local longfname1="longfilenamewitha=inthemiddletotestbehaviorregardingthedigestedform"
5241 local longdname="longdirectorynamewitha=inthemiddletotestbehaviorregardingthedigestedform"
5242 local longfname2="$longdname/${longfname1}2"
5244 (( $MDS1_VERSION > $(version_code 2.15.53) )) ||
5245 skip "Need MDS version at least 2.15.53"
5247 $LCTL get_param mdc.*.import | grep -q client_encryption ||
5248 skip "client encryption not supported"
5250 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
5251 skip "need dummy encryption support"
5253 which fscrypt || skip "This test needs fscrypt userspace tool"
5255 yes | fscrypt setup --force --verbose ||
5256 echo "fscrypt global setup already done"
5257 sed -i 's/\(.*\)policy_version\(.*\):\(.*\)\"[0-9]*\"\(.*\)/\1policy_version\2:\3"2"\4/' \
5259 yes | fscrypt setup --verbose $MOUNT ||
5260 echo "fscrypt setup $MOUNT already done"
5262 # enable_filename_encryption tunable only available for client
5263 # built against embedded llcrypt. If client is built against in-kernel
5264 # fscrypt, file names are always encrypted.
5265 $LCTL get_param mdc.*.connect_flags | grep -q name_encryption &&
5266 nameenc=$(lctl get_param -n llite.*.enable_filename_encryption |
5268 if [ -n "$nameenc" ]; then
5269 do_facet mgs $LCTL set_param -P \
5270 llite.*.enable_filename_encryption=1
5272 error "set_param -P \
5273 llite.*.enable_filename_encryption=1 failed"
5275 wait_update_facet --verbose client \
5276 "$LCTL get_param -n llite.*.enable_filename_encryption \
5278 error "enable_filename_encryption not set on client"
5282 echo -e 'mypass\nmypass' | fscrypt encrypt --verbose \
5283 --source=custom_passphrase --name=protector_63_1 $vaultdir1 ||
5284 error "fscrypt encrypt $vaultdir1 failed"
5286 mkdir $vaultdir1/dirA
5287 mkdir $vaultdir1/$longdname
5288 paths=("$vaultdir1/fileA")
5289 paths+=("$vaultdir1/dirA/fileB")
5290 paths+=("$vaultdir1/$longfname1")
5291 paths+=("$vaultdir1/$longfname2")
5294 paths+=("$vaultdir1/dirA")
5295 paths+=("$vaultdir1/$longdname")
5300 fscrypt lock --verbose $vaultdir1 ||
5301 error "fscrypt lock $vaultdir1 failed (1)"
5305 if [ -z "$nameenc" ]; then
5306 echo "Rest of the test requires disabling name encryption"
5310 # disable name encryption
5311 do_facet mgs $LCTL set_param -P llite.*.enable_filename_encryption=0
5313 error "set_param -P llite.*.enable_filename_encryption=0 failed"
5315 wait_update_facet --verbose client \
5316 "$LCTL get_param -n llite.*.enable_filename_encryption \
5318 error "enable_filename_encryption not set back to default"
5321 echo -e 'mypass\nmypass' | fscrypt encrypt --verbose \
5322 --source=custom_passphrase --name=protector_63_2 $vaultdir2 ||
5323 error "fscrypt encrypt $vaultdir2 failed"
5325 mkdir $vaultdir2/dirA
5326 mkdir $vaultdir2/$longdname
5329 paths=("$vaultdir2/fileA")
5330 paths+=("$vaultdir2/dirA/fileB")
5331 paths+=("$vaultdir2/$longfname1")
5332 paths+=("$vaultdir2/$longfname2")
5335 paths+=("$vaultdir2/dirA")
5336 paths+=("$vaultdir2/$longdname")
5341 fscrypt lock --verbose $vaultdir2 ||
5342 error "fscrypt lock $vaultdir2 failed (2)"
5346 rm -rf $MOUNT/.fscrypt
5348 run_test 63 "fid2path with encrypted files"
5351 do_facet mgs $LCTL nodemap_activate 1
5354 do_facet mgs $LCTL nodemap_del c0 || true
5355 wait_nm_sync c0 id ''
5357 do_facet mgs $LCTL nodemap_modify --name default \
5358 --property admin --value 1
5359 do_facet mgs $LCTL nodemap_modify --name default \
5360 --property trusted --value 1
5361 wait_nm_sync default admin_nodemap
5362 wait_nm_sync default trusted_nodemap
5364 client_ip=$(host_nids_address $HOSTNAME $NETTYPE)
5365 client_nid=$(h2nettype $client_ip)
5366 do_facet mgs $LCTL nodemap_add c0
5367 do_facet mgs $LCTL nodemap_add_range \
5368 --name c0 --range $client_nid
5369 do_facet mgs $LCTL nodemap_modify --name c0 \
5370 --property admin --value 1
5371 do_facet mgs $LCTL nodemap_modify --name c0 \
5372 --property trusted --value 1
5373 wait_nm_sync c0 admin_nodemap
5374 wait_nm_sync c0 trusted_nodemap
5378 do_facet mgs $LCTL nodemap_del c0
5379 do_facet mgs $LCTL nodemap_modify --name default \
5380 --property admin --value 0
5381 do_facet mgs $LCTL nodemap_modify --name default \
5382 --property trusted --value 0
5383 wait_nm_sync default admin_nodemap
5384 wait_nm_sync default trusted_nodemap
5386 do_facet mgs $LCTL nodemap_activate 0
5387 wait_nm_sync active 0
5391 local testfile=$DIR/$tdir/$tfile
5394 (( MDS1_VERSION >= $(version_code 2.15.54) )) ||
5395 skip "Need MDS >= 2.15.54 for role-based controls"
5397 stack_trap cleanup_64 EXIT
5398 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
5401 # check default value for rbac is all
5402 rbac=$(do_facet mds $LCTL get_param -n nodemap.c0.rbac)
5403 for role in file_perms \
5411 [[ "$rbac" =~ "$role" ]] ||
5412 error "role '$role' not in default '$rbac'"
5415 do_facet mgs $LCTL nodemap_modify --name c0 \
5416 --property rbac --value file_perms
5417 wait_nm_sync c0 rbac
5419 stack_trap "set +vx"
5421 chmod 777 $testfile || error "chmod failed"
5422 chown $TSTUSR:$TSTUSR $testfile || error "chown failed"
5423 chgrp $TSTUSR $testfile || error "chgrp failed"
5424 $LFS project -p 1000 $testfile || error "setting project failed"
5427 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
5428 wait_nm_sync c0 rbac
5431 chmod 777 $testfile && error "chmod should fail"
5432 chown $TSTUSR:$TSTUSR $testfile && error "chown should fail"
5433 chgrp $TSTUSR $testfile && error "chgrp should fail"
5434 $LFS project -p 1000 $testfile && error "setting project should fail"
5437 run_test 64a "Nodemap enforces file_perms RBAC roles"
5440 local testdir=$DIR/$tdir/${tfile}.d
5443 (( MDS1_VERSION >= $(version_code 2.15.54) )) ||
5444 skip "Need MDS >= 2.15.54 for role-based controls"
5446 (( MDSCOUNT >= 2 )) || skip "mdt count $MDSCOUNT, skipping dne_ops role"
5448 stack_trap cleanup_64 EXIT
5449 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
5452 dir_restripe=$(do_node $mds1_HOST \
5453 "$LCTL get_param -n mdt.*MDT0000.enable_dir_restripe")
5454 [ -n "$dir_restripe" ] || dir_restripe=0
5455 do_nodes $(comma_list $(all_mdts_nodes)) \
5456 $LCTL set_param mdt.*.enable_dir_restripe=1 ||
5457 error "enabling dir_restripe failed"
5458 stack_trap "do_nodes $(comma_list $(all_mdts_nodes)) \
5459 $LCTL set_param mdt.*.enable_dir_restripe=$dir_restripe" EXIT
5460 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac \
5462 wait_nm_sync c0 rbac
5463 $LFS mkdir -i 0 ${testdir}_for_migr ||
5464 error "$LFS mkdir ${testdir}_for_migr failed (1)"
5465 touch ${testdir}_for_migr/file001 ||
5466 error "touch ${testdir}_for_migr/file001 failed (1)"
5467 $LFS mkdir -i 0 ${testdir}_mdt0 ||
5468 error "$LFS mkdir ${testdir}_mdt0 failed (1)"
5469 $LFS mkdir -i 1 ${testdir}_mdt1 ||
5470 error "$LFS mkdir ${testdir}_mdt1 failed (1)"
5472 $LFS mkdir -i 1 $testdir || error "$LFS mkdir failed (1)"
5474 $LFS mkdir -c 2 $testdir || error "$LFS mkdir failed (2)"
5477 $LFS setdirstripe -c 2 $testdir || error "$LFS setdirstripe failed"
5479 $LFS migrate -m 1 ${testdir}_for_migr || error "$LFS migrate failed"
5480 touch ${testdir}_mdt0/fileA || error "touch fileA failed (1)"
5481 mv ${testdir}_mdt0/fileA ${testdir}_mdt1/ || error "mv failed (1)"
5484 $LFS mkdir -i 0 ${testdir}_for_migr ||
5485 error "$LFS mkdir ${testdir}_for_migr failed (2)"
5486 touch ${testdir}_for_migr/file001 ||
5487 error "touch ${testdir}_for_migr/file001 failed (2)"
5488 $LFS mkdir -i 0 ${testdir}_mdt0 ||
5489 error "$LFS mkdir ${testdir}_mdt0 failed (2)"
5490 $LFS mkdir -i 1 ${testdir}_mdt1 ||
5491 error "$LFS mkdir ${testdir}_mdt1 failed (2)"
5493 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
5494 wait_nm_sync c0 rbac
5496 $LFS mkdir -i 1 $testdir && error "$LFS mkdir should fail (1)"
5497 $LFS mkdir -c 2 $testdir && error "$LFS mkdir should fail (2)"
5499 $LFS setdirstripe -c 2 $testdir && error "$LFS setdirstripe should fail"
5501 $LFS migrate -m 1 ${testdir}_for_migr &&
5502 error "$LFS migrate should fail"
5503 touch ${testdir}_mdt0/fileA || error "touch fileA failed (2)"
5504 mv ${testdir}_mdt0/fileA ${testdir}_mdt1/ || error "mv failed (2)"
5507 run_test 64b "Nodemap enforces dne_ops RBAC roles"
5510 (( MDS1_VERSION >= $(version_code 2.15.54) )) ||
5511 skip "Need MDS >= 2.15.54 for role-based controls"
5513 stack_trap cleanup_64 EXIT
5514 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
5517 do_facet mgs $LCTL nodemap_modify --name c0 \
5518 --property rbac --value quota_ops
5519 wait_nm_sync c0 rbac
5521 $LFS setquota -u $USER0 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT ||
5522 error "lfs setquota -u failed"
5523 $LFS setquota -u $USER0 --delete $MOUNT
5524 $LFS setquota -g $USER0 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT ||
5525 error "lfs setquota -g failed"
5526 $LFS setquota -g $USER0 --delete $MOUNT
5527 $LFS setquota -p 1000 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT ||
5528 error "lfs setquota -p failed"
5529 $LFS setquota -p 1000 --delete $MOUNT
5531 $LFS setquota -U -b 10G -B 11G -i 100K -I 105K $MOUNT ||
5532 error "lfs setquota -U failed"
5533 $LFS setquota -U -b 0 -B 0 -i 0 -I 0 $MOUNT
5534 $LFS setquota -G -b 10G -B 11G -i 100K -I 105K $MOUNT ||
5535 error "lfs setquota -G failed"
5536 $LFS setquota -G -b 0 -B 0 -i 0 -I 0 $MOUNT
5537 $LFS setquota -P -b 10G -B 11G -i 100K -I 105K $MOUNT ||
5538 error "lfs setquota -P failed"
5539 $LFS setquota -P -b 0 -B 0 -i 0 -I 0 $MOUNT
5540 $LFS setquota -u $USER0 -D $MOUNT ||
5541 error "lfs setquota -u -D failed"
5542 $LFS setquota -u $USER0 --delete $MOUNT
5543 $LFS setquota -g $USER0 -D $MOUNT ||
5544 error "lfs setquota -g -D failed"
5545 $LFS setquota -g $USER0 --delete $MOUNT
5546 $LFS setquota -p 1000 -D $MOUNT ||
5547 error "lfs setquota -p -D failed"
5548 $LFS setquota -p 1000 --delete $MOUNT
5551 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
5552 wait_nm_sync c0 rbac
5555 $LFS setquota -u $USER0 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT &&
5556 error "lfs setquota -u should fail"
5557 $LFS setquota -u $USER0 --delete $MOUNT
5558 $LFS setquota -g $USER0 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT &&
5559 error "lfs setquota -g should fail"
5560 $LFS setquota -g $USER0 --delete $MOUNT
5561 $LFS setquota -p 1000 -b 307200 -B 309200 -i 10000 -I 11000 $MOUNT &&
5562 error "lfs setquota -p should fail"
5563 $LFS setquota -p 1000 --delete $MOUNT
5565 $LFS setquota -U -b 10G -B 11G -i 100K -I 105K $MOUNT &&
5566 error "lfs setquota -U should fail"
5567 $LFS setquota -G -b 10G -B 11G -i 100K -I 105K $MOUNT &&
5568 error "lfs setquota -G should fail"
5569 $LFS setquota -P -b 10G -B 11G -i 100K -I 105K $MOUNT &&
5570 error "lfs setquota -P should fail"
5571 $LFS setquota -u $USER0 -D $MOUNT &&
5572 error "lfs setquota -u -D should fail"
5573 $LFS setquota -u $USER0 --delete $MOUNT
5574 $LFS setquota -g $USER0 -D $MOUNT &&
5575 error "lfs setquota -g -D should fail"
5576 $LFS setquota -g $USER0 --delete $MOUNT
5577 $LFS setquota -p 1000 -D $MOUNT &&
5578 error "lfs setquota -p -D should fail"
5579 $LFS setquota -p 1000 --delete $MOUNT
5582 run_test 64c "Nodemap enforces quota_ops RBAC roles"
5585 local testfile=$DIR/$tdir/$tfile
5588 (( MDS1_VERSION >= $(version_code 2.15.54) )) ||
5589 skip "Need MDS >= 2.15.54 for role-based controls"
5591 stack_trap cleanup_64 EXIT
5592 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
5595 do_facet mgs $LCTL nodemap_modify --name c0 \
5596 --property rbac --value byfid_ops
5597 wait_nm_sync c0 rbac
5600 fid=$(lfs path2fid $testfile)
5602 $LFS fid2path $MOUNT $fid || error "fid2path $fid failed (1)"
5603 cat $MOUNT/.lustre/fid/$fid || error "cat by fid failed"
5604 lfs rmfid $MOUNT $fid || error "lfs rmfid failed"
5607 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
5608 wait_nm_sync c0 rbac
5611 fid=$(lfs path2fid $testfile)
5613 $LFS fid2path $MOUNT $fid || error "fid2path $fid failed (2)"
5614 cat $MOUNT/.lustre/fid/$fid && error "cat by fid should fail"
5615 lfs rmfid $MOUNT $fid && error "lfs rmfid should fail"
5619 run_test 64d "Nodemap enforces byfid_ops RBAC roles"
5622 local testfile=$DIR/$tdir/$tfile
5623 local testdir=$DIR/$tdir/${tfile}.d
5625 (( MDS1_VERSION >= $(version_code 2.15.54) )) ||
5626 skip "Need MDS >= 2.15.54 for role-based controls"
5628 stack_trap cleanup_64 EXIT
5629 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
5632 # activate changelogs
5633 changelog_register || error "changelog_register failed"
5634 local cl_user="${CL_USERS[$SINGLEMDS]%% *}"
5635 changelog_users $SINGLEMDS | grep -q $cl_user ||
5636 error "User $cl_user not found in changelog_users"
5637 changelog_chmask ALL
5640 mkdir $testdir || error "failed to mkdir $testdir"
5641 touch $testfile || error "failed to touch $testfile"
5643 do_facet mgs $LCTL nodemap_modify --name c0 \
5644 --property rbac --value chlg_ops
5645 wait_nm_sync c0 rbac
5648 echo "changelogs dump"
5649 changelog_dump || error "failed to dump changelogs"
5650 echo "changelogs clear"
5651 changelog_clear 0 || error "failed to clear changelogs"
5653 rm -rf $testdir $testfile || error "rm -rf $testdir $testfile failed"
5655 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value none
5656 wait_nm_sync c0 rbac
5659 mkdir $testdir || error "failed to mkdir $testdir"
5660 touch $testfile || error "failed to touch $testfile"
5663 echo "changelogs dump"
5664 changelog_dump && error "dump changelogs should fail"
5665 echo "changelogs clear"
5666 changelog_clear 0 && error "clear changelogs should fail"
5667 rm -rf $testdir $testfile
5669 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value all
5670 wait_nm_sync c0 rbac
5672 run_test 64e "Nodemap enforces chlg_ops RBAC roles"
5675 local vaultdir=$DIR/$tdir/vault
5680 (( MDS1_VERSION >= $(version_code 2.15.54) )) ||
5681 skip "Need MDS >= 2.15.54 for role-based controls"
5683 cli_enc=$($LCTL get_param mdc.*.import | grep client_encryption)
5684 [ -n "$cli_enc" ] || skip "Need enc support, skip fscrypt_admin role"
5685 which fscrypt || skip "Need fscrypt, skip fscrypt_admin role"
5687 stack_trap cleanup_64 EXIT
5688 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
5691 yes | fscrypt setup --force --verbose ||
5692 echo "fscrypt global setup already done"
5693 sed -i 's/\(.*\)policy_version\(.*\):\(.*\)\"[0-9]*\"\(.*\)/\1policy_version\2:\3"2"\4/' \
5695 yes | fscrypt setup --verbose $MOUNT ||
5696 echo "fscrypt setup $MOUNT already done"
5697 stack_trap "rm -rf $MOUNT/.fscrypt"
5699 # file_perms is required because fscrypt uses chmod/chown
5700 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac \
5701 --value fscrypt_admin,file_perms
5702 wait_nm_sync c0 rbac
5706 echo -e 'mypass\nmypass' | fscrypt encrypt --verbose \
5707 --source=custom_passphrase --name=protector_64 $vaultdir ||
5708 error "fscrypt encrypt $vaultdir failed"
5709 fscrypt lock $vaultdir || error "fscrypt lock $vaultdir failed (1)"
5710 policy=$(fscrypt status $vaultdir | awk '$1 == "Policy:"{print $2}')
5711 [ -n "$policy" ] || error "could not get enc policy"
5712 protector=$(fscrypt status $vaultdir |
5713 awk 'BEGIN {found=0} { if (found == 1) { print $1 }} \
5714 $1 == "PROTECTOR" {found=1}')
5715 [ -n "$protector" ] || error "could not get enc protector"
5719 # file_perms is required because fscrypt uses chmod/chown
5720 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac \
5722 wait_nm_sync c0 rbac
5725 echo mypass | fscrypt unlock $vaultdir ||
5726 error "fscrypt unlock $vaultdir failed"
5727 fscrypt lock $vaultdir || error "fscrypt lock $vaultdir failed (2)"
5728 fscrypt metadata destroy --protector=$MOUNT:$protector --force &&
5729 error "destroy protector should fail"
5730 fscrypt metadata destroy --policy=$MOUNT:$policy --force &&
5731 error "destroy policy should fail"
5732 mkdir -p ${vaultdir}2
5733 echo -e 'mypass\nmypass' | fscrypt encrypt --verbose \
5734 --source=custom_passphrase \
5735 --name=protector_64bis ${vaultdir}2 &&
5736 error "fscrypt encrypt ${vaultdir}2 should fail"
5740 do_facet mgs $LCTL nodemap_modify --name c0 --property rbac --value all
5741 wait_nm_sync c0 rbac
5744 fscrypt metadata destroy --protector=$MOUNT:$protector --force ||
5745 error "destroy protector failed"
5746 fscrypt metadata destroy --policy=$MOUNT:$policy --force ||
5747 error "destroy policy failed"
5752 run_test 64f "Nodemap enforces fscrypt_admin RBAC roles"
5754 log "cleanup: ======================================================"
5757 for num in $(seq $MDSCOUNT); do
5758 if [ "${identity_old[$num]}" = 1 ]; then
5759 switch_identity $num false || identity_old[$num]=$?
5763 $RUNAS_CMD -u $ID0 ls $DIR
5764 $RUNAS_CMD -u $ID1 ls $DIR
5769 check_and_cleanup_lustre