3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
11 LUSTRE=${LUSTRE:-$(dirname $0)/..}
12 . $LUSTRE/tests/test-framework.sh
16 ALWAYS_EXCEPT="$SANITY_FLR_EXCEPT "
17 # Bug number for skipped test: LU-14818 LU-11381 LU-14765
18 ALWAYS_EXCEPT+=" 6 201 44c "
19 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
21 # skip all tests for PPC until we can get sanity-pfl to pass
22 if [[ $(uname -m) = ppc64 ]]; then
23 skip "Skip FLR testing for PPC clients"
26 if [[ "$ost1_FSTYPE" == "zfs" ]]; then
28 ALWAYS_EXCEPT+=" 49a "
33 [[ "$MDS1_VERSION" -ge $(version_code 2.10.56) ]] ||
34 skip "Need MDS version at least 2.10.56"
36 check_and_setup_lustre
39 rm -rf $DIR/[Rdfs][0-9]*
41 [ $UID -eq 0 -a $RUNAS_ID -eq 0 ] &&
42 error "\$RUNAS_ID set to 0, but \$UID is also 0!"
44 check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS
47 # global array to store mirror IDs
48 declare -a mirror_array
55 for id in $($LFS getstripe $tf | awk '/lcme_id/{print $2}'); do
56 array[${#array[@]}]=$((id >> 16))
59 mirror_array=($(printf "%s\n" "${array[@]}" | sort -u))
61 echo ${#mirror_array[@]}
65 echo 3 > /proc/sys/vm/drop_caches
76 wait_osc_import_state client ost$idx "\(DISCONN\|IDLE\)"
84 start ost$idx $(ostdevname $idx) $OST_MOUNT_OPTS ||
85 error "start ost$idx failed"
89 wait_recovery_complete ost$idx
94 # Verify mirror count with an expected value for a given file.
96 verify_mirror_count() {
99 local mirror_count=$($LFS getstripe -N $tf)
101 [[ $mirror_count = $expected ]] || {
102 $LFS getstripe -v $tf
103 error "verify mirror count failed on $tf:" \
104 "$mirror_count != $expected"
109 # Verify component count with an expected value for a given file.
110 # $1 coposited layout file
111 # $2 expected component number
113 verify_comp_count() {
116 local comp_count=$($LFS getstripe --component-count $tf)
118 [[ $comp_count = $expected ]] || {
119 $LFS getstripe -v $tf
120 error "verify component count failed on $tf:" \
121 "$comp_count != $expected"
126 # Verify component attribute with an expected value for a given file
134 local cmd="$LFS getstripe -I$comp_id"
135 local getstripe_cmd="$cmd -v"
139 stripe-size) cmd+=" -S $tf" ;;
140 stripe-count) cmd+=" -c $tf" ;;
141 stripe-index) cmd+=" -i $tf" ;;
142 pool) cmd+=" -p $tf" ;;
143 comp-start) cmd+=" --component-start $tf" ;;
144 comp-end) cmd+=" --component-end $tf" ;;
145 lcme_flags) cmd+=" $tf | awk '/lcme_flags:/ { print \$2 }'" ;;
146 *) error "invalid attribute $attr";;
151 [ $attr = lcme_flags ] && {
153 local expected_list=$(comma_list $expected)
154 for fl in ${expected_list//,/ }; do
157 [[ ${fl:0:1} = "^" ]] && neg=1
158 [[ $neg = 1 ]] && fl=${fl:1}
160 $(echo $value | grep -q $fl)
162 # 0: matched; 1: not matched
164 if [[ $neg = 0 && $match != 0 ||
165 $neg = 1 && $match = 0 ]]; then
167 [[ $neg = 0 ]] && # expect the flag
168 error "expected flag '$fl' not in $comp_id"
169 [[ $neg = 1 ]] && # not expect the flag
170 error "not expected flag '$fl' in $comp_id"
176 [[ $value = $expected ]] || {
178 error "verify $attr failed on $tf: $value != $expected"
183 # Verify component extent with expected start and end extent values
184 # for a given file and component ID.
186 verify_comp_extent() {
189 local expected_start=$3
190 local expected_end=$4
192 verify_comp_attr comp-start $tf $comp_id $expected_start
193 verify_comp_attr comp-end $tf $comp_id $expected_end
197 # Verify component attribute with parent directory for a given file
200 verify_comp_attr_with_parent() {
204 local td=$(cd $(dirname $tf); echo $PWD)
205 local tf_cmd="$LFS getstripe -I$comp_id"
206 local td_cmd="$LFS getstripe"
212 stripe-size) opt="-S" ;;
213 stripe-count) opt="-c" ;;
215 *) error "invalid attribute $attr";;
218 expected=$($td_cmd $opt $td)
219 [[ $expected = -1 ]] && expected=$OSTCOUNT
221 value=$($tf_cmd $opt $tf)
222 [[ $value = -1 ]] && value=$OSTCOUNT
224 [[ $value = $expected ]] || {
227 error "verify $attr failed with parent on $tf:" \
228 "$value != $expected"
233 # Verify component attribute with filesystem-wide default value for a given file
236 verify_comp_attr_with_default() {
240 local tf_cmd="$LFS getstripe -I$comp_id"
248 expected=$($LCTL get_param -n \
249 lov.$FSNAME-clilov-*.stripesize)
253 expected=$($LCTL get_param -n \
254 lov.$FSNAME-clilov-*.stripecount)
255 [[ $expected = -1 ]] && expected=$OSTCOUNT
257 *) error "invalid attribute $attr";;
260 value=$($tf_cmd $opt $tf)
261 [[ $value = -1 ]] && value=$OSTCOUNT
263 [[ $value = $expected ]] || {
265 error "verify $attr failed with default value on $tf:" \
266 "$value != $expected"
271 # Verify unspecified component attributes for a given file
274 # This will only verify the inherited attributes:
275 # stripe size, stripe count and OST pool name
277 verify_comp_attrs() {
281 verify_comp_attr_with_default stripe-size $tf $comp_id
282 verify_comp_attr_with_default stripe-count $tf $comp_id
283 verify_comp_attr_with_parent pool $tf $comp_id
289 local expected_state=$2
291 local state=$($LFS getstripe -v $tf | awk '/lcm_flags/{ print $2 }')
292 [ $expected_state = $state ] ||
293 error "expected: $expected_state, actual $state"
296 # command line test cases
300 local mirror_count=16 # LUSTRE_MIRROR_COUNT_MAX
301 local mirror_cmd="$LFS mirror create"
306 # create parent directory
307 mkdir $td || error "mkdir $td failed"
309 $mirror_cmd $tf &> /dev/null && error "miss -N option"
311 $mirror_cmd -N $tf || error "create mirrored file $tf failed"
312 verify_mirror_count $tf 1
313 id=$($LFS getstripe -I $tf)
314 verify_comp_attrs $tf $id
315 verify_comp_extent $tf $id 0 EOF
317 $mirror_cmd -N0 $tf-1 &> /dev/null && error "invalid mirror count 0"
318 $mirror_cmd -N$((mirror_count + 1)) $tf-1 &> /dev/null &&
319 error "invalid mirror count $((mirror_count + 1))"
321 $mirror_cmd -N$mirror_count $tf-1 ||
322 error "create mirrored file $tf-1 failed"
323 verify_mirror_count $tf-1 $mirror_count
324 ids=($($LFS getstripe $tf-1 | awk '/lcme_id/{print $2}' | tr '\n' ' '))
325 for ((i = 0; i < $mirror_count; i++)); do
326 verify_comp_attrs $tf-1 ${ids[$i]}
327 verify_comp_extent $tf-1 ${ids[$i]} 0 EOF
330 $mirror_cmd -N -N2 -N3 -N4 $tf-2 ||
331 error "create mirrored file $tf-2 failed"
332 verify_mirror_count $tf-2 10
333 ids=($($LFS getstripe $tf-2 | awk '/lcme_id/{print $2}' | tr '\n' ' '))
334 for ((i = 0; i < 10; i++)); do
335 verify_comp_attrs $tf-2 ${ids[$i]}
336 verify_comp_extent $tf-2 ${ids[$i]} 0 EOF
339 run_test 0a "lfs mirror create with -N option"
342 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
346 local mirror_cmd="$LFS mirror create"
350 # create a new OST pool
351 local pool_name=$TESTNAME
352 create_pool $FSNAME.$pool_name ||
353 error "create OST pool $pool_name failed"
355 # add OSTs into the pool
356 pool_add_targets $pool_name 0 $((OSTCOUNT - 1)) ||
357 error "add OSTs into pool $pool_name failed"
359 # create parent directory
360 mkdir $td || error "mkdir $td failed"
361 $LFS setstripe -S 8M -c -1 -p $pool_name $td ||
362 error "$LFS setstripe $td failed"
364 # create a mirrored file with plain layout mirrors
365 $mirror_cmd -N -N -S 4M -c 2 -p flash -i 2 -o 2,3 \
366 -N -S 16M -N -c -1 -N -p archive -N -p none $tf ||
367 error "create mirrored file $tf failed"
368 verify_mirror_count $tf 6
369 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
370 for ((i = 0; i < 6; i++)); do
371 verify_comp_extent $tf ${ids[$i]} 0 EOF
374 # verify component ${ids[0]}
375 verify_comp_attrs $tf ${ids[0]}
377 # verify component ${ids[1]}
378 verify_comp_attr stripe-size $tf ${ids[1]} 4194304
379 verify_comp_attr stripe-count $tf ${ids[1]} 2
380 verify_comp_attr stripe-index $tf ${ids[1]} 2
382 # verify component ${ids[2]}
383 verify_comp_attr stripe-size $tf ${ids[2]} 16777216
384 verify_comp_attr stripe-count $tf ${ids[2]} 2
385 verify_comp_attr pool $tf ${ids[2]} flash
387 # verify component ${ids[3]}
388 verify_comp_attr stripe-size $tf ${ids[3]} 16777216
389 verify_comp_attr stripe-count $tf ${ids[3]} $OSTCOUNT
390 verify_comp_attr pool $tf ${ids[3]} flash
392 # verify component ${ids[4]}
393 verify_comp_attr stripe-size $tf ${ids[4]} 16777216
394 verify_comp_attr stripe-count $tf ${ids[4]} $OSTCOUNT
395 verify_comp_attr pool $tf ${ids[4]} archive
397 # verify component ${ids[5]}
398 verify_comp_attr stripe-size $tf ${ids[5]} 16777216
399 verify_comp_attr stripe-count $tf ${ids[5]} $OSTCOUNT
400 verify_comp_attr_with_parent pool $tf ${ids[5]}
402 if [ $MDS1_VERSION -ge $(version_code 2.12.55) ]; then
403 # LU-11022 - remove mirror by pool name
404 local=cnt cnt=$($LFS getstripe $tf | grep archive | wc -l)
405 [ "$cnt" != "1" ] && error "unexpected mirror count $cnt"
406 $LFS mirror delete --pool archive $tf || error "delete mirror"
407 cnt=$($LFS getstripe $tf | grep archive | wc -l)
408 [ "$cnt" != "0" ] && error "mirror count after removal: $cnt"
414 run_test 0b "lfs mirror create plain layout mirrors"
417 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
421 local mirror_cmd="$LFS mirror create"
425 # create a new OST pool
426 local pool_name=$TESTNAME
427 create_pool $FSNAME.$pool_name ||
428 error "create OST pool $pool_name failed"
430 # add OSTs into the pool
431 pool_add_targets $pool_name 0 $((OSTCOUNT - 1)) ||
432 error "add OSTs into pool $pool_name failed"
434 # create parent directory
435 mkdir $td || error "mkdir $td failed"
436 $LFS setstripe -E 32M -S 8M -c -1 -p $pool_name -E eof -S 16M $td ||
437 error "$LFS setstripe $td failed"
439 # create a mirrored file with composite layout mirrors
440 $mirror_cmd -N2 -E 4M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
442 -N3 -E 512M -S 16M -p archive -E -1 -i -1 -c -1 $tf ||
443 error "create mirrored file $tf failed"
444 verify_mirror_count $tf 6
445 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
447 # verify components ${ids[0]} and ${ids[2]}
449 verify_comp_attr_with_default stripe-size $tf ${ids[$i]}
450 verify_comp_attr stripe-count $tf ${ids[$i]} 2
451 verify_comp_attr stripe-index $tf ${ids[$i]} 1
452 verify_comp_extent $tf ${ids[$i]} 0 4194304
455 # verify components ${ids[1]} and ${ids[3]}
457 verify_comp_attr stripe-size $tf ${ids[$i]} 4194304
458 verify_comp_attr stripe-count $tf ${ids[$i]} 2
459 verify_comp_attr pool $tf ${ids[$i]} flash
460 verify_comp_extent $tf ${ids[$i]} 4194304 EOF
463 # verify component ${ids[4]}
464 verify_comp_attr stripe-size $tf ${ids[4]} 4194304
465 verify_comp_attr stripe-count $tf ${ids[4]} 4
466 verify_comp_attr_with_parent pool $tf ${ids[4]}
467 verify_comp_extent $tf ${ids[4]} 0 EOF
469 # verify components ${ids[5]}, ${ids[7]} and ${ids[9]}
471 verify_comp_attr stripe-size $tf ${ids[$i]} 16777216
472 verify_comp_attr stripe-count $tf ${ids[$i]} 4
473 verify_comp_attr pool $tf ${ids[$i]} archive
474 verify_comp_extent $tf ${ids[$i]} 0 536870912
477 # verify components ${ids[6]}, ${ids[8]} and ${ids[10]}
479 verify_comp_attr stripe-size $tf ${ids[$i]} 16777216
480 verify_comp_attr stripe-count $tf ${ids[$i]} -1
481 verify_comp_attr pool $tf ${ids[$i]} archive
482 verify_comp_extent $tf ${ids[$i]} 536870912 EOF
488 run_test 0c "lfs mirror create composite layout mirrors"
493 local mirror_count=16 # LUSTRE_MIRROR_COUNT_MAX
494 local mirror_cmd="$LFS mirror extend"
498 # create parent directory
499 mkdir $td || error "mkdir $td failed"
501 $mirror_cmd $tf &> /dev/null && error "miss -N option"
502 $mirror_cmd -N $tf &> /dev/null && error "$tf does not exist"
504 # create a non-mirrored file, convert it to a mirrored file and extend
505 touch $tf || error "touch $tf failed"
506 $mirror_cmd -N $tf || error "convert and extend $tf failed"
507 verify_mirror_count $tf 2
508 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
509 for ((i = 0; i < 2; i++)); do
510 verify_comp_attrs $tf ${ids[$i]}
511 verify_comp_extent $tf ${ids[$i]} 0 EOF
514 lfsck_verify_pfid $tf || error "PFID is not set"
516 # create a mirrored file and extend it
517 $LFS mirror create -N $tf-1 || error "create mirrored file $tf-1 failed"
518 $LFS mirror create -N $tf-2 || error "create mirrored file $tf-2 failed"
519 $LFS mirror create -N $tf-3 || error "create mirrored file $tf-3 failed"
521 $mirror_cmd -N -S 4M -N -f $tf-2 $tf-1 ||
522 error "extend mirror with -f failed"
524 $mirror_cmd -N$((mirror_count - 1)) $tf-3 ||
525 error "extend mirrored file $tf-3 failed"
526 verify_mirror_count $tf-3 $mirror_count
527 ids=($($LFS getstripe $tf-3 | awk '/lcme_id/{print $2}' | tr '\n' ' '))
528 for ((i = 0; i < $mirror_count; i++)); do
529 verify_comp_attrs $tf-3 ${ids[$i]}
530 verify_comp_extent $tf-3 ${ids[$i]} 0 EOF
533 $mirror_cmd -N $tf-3 &> /dev/null
536 error "exceeded maximum mirror count returns $rc not ERANGE(34)"
538 run_test 0d "lfs mirror extend with -N option"
541 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
545 local mirror_cmd="$LFS mirror extend"
549 # create parent directory
550 mkdir $td || error "mkdir $td failed"
552 # create a mirrored file with plain layout mirrors
553 $LFS mirror create -N -S 32M -c 3 -p ssd -i 1 -o 1,2,3 $tf ||
554 error "create mirrored file $tf failed"
556 # extend the mirrored file with plain layout mirrors
557 $mirror_cmd -N -S 4M -c 2 -p flash -i 2 -o 2,3 \
558 -N -S 16M -N -c -1 -N -p archive -N -p none $tf ||
559 error "extend mirrored file $tf failed"
560 verify_mirror_count $tf 6
561 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
562 for ((i = 0; i < 6; i++)); do
563 verify_comp_extent $tf ${ids[$i]} 0 EOF
566 # verify component ${ids[0]}
567 verify_comp_attr stripe-size $tf ${ids[0]} 33554432
568 verify_comp_attr stripe-count $tf ${ids[0]} 3
569 verify_comp_attr stripe-index $tf ${ids[0]} 1
571 # verify component ${ids[1]}
572 verify_comp_attr stripe-size $tf ${ids[1]} 4194304
573 verify_comp_attr stripe-count $tf ${ids[1]} 2
574 verify_comp_attr stripe-index $tf ${ids[1]} 2
576 # verify component ${ids[2]}
577 verify_comp_attr stripe-size $tf ${ids[2]} 16777216
578 verify_comp_attr stripe-count $tf ${ids[2]} 2
579 verify_comp_attr pool $tf ${ids[2]} flash
581 # verify component ${ids[3]}
582 verify_comp_attr stripe-size $tf ${ids[3]} 16777216
583 verify_comp_attr stripe-count $tf ${ids[3]} $OSTCOUNT
584 verify_comp_attr pool $tf ${ids[3]} flash
586 # verify component ${ids[4]}
587 verify_comp_attr stripe-size $tf ${ids[4]} 16777216
588 verify_comp_attr stripe-count $tf ${ids[4]} $OSTCOUNT
589 verify_comp_attr pool $tf ${ids[4]} archive
591 # verify component ${ids[5]}
592 verify_comp_attr stripe-size $tf ${ids[5]} 16777216
593 verify_comp_attr stripe-count $tf ${ids[5]} $OSTCOUNT
594 verify_comp_attr_with_parent pool $tf ${ids[5]}
596 run_test 0e "lfs mirror extend plain layout mirrors"
599 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
603 local mirror_cmd="$LFS mirror extend"
607 # create parent directory
608 mkdir $td || error "mkdir $td failed"
610 # create a mirrored file with composite layout mirror
611 $LFS mirror create -N -E 32M -S 16M -p ssd -E eof -S 32M $tf ||
612 error "create mirrored file $tf failed"
614 # extend the mirrored file with composite layout mirrors
615 $mirror_cmd -N -p archive \
616 -N2 -E 4M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
618 -N3 -E 512M -S 16M -p archive -E -1 -i -1 -c -1 $tf ||
619 error "extend mirrored file $tf failed"
620 verify_mirror_count $tf 8
621 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
623 # verify component ${ids[0]}
624 verify_comp_attr stripe-size $tf ${ids[0]} 16777216
625 verify_comp_attr_with_default stripe-count $tf ${ids[0]}
626 verify_comp_attr pool $tf ${ids[0]} ssd
627 verify_comp_extent $tf ${ids[0]} 0 33554432
629 # verify component ${ids[1]}
630 verify_comp_attr stripe-size $tf ${ids[1]} 33554432
631 verify_comp_attr_with_default stripe-count $tf ${ids[1]}
632 verify_comp_attr pool $tf ${ids[1]} ssd
633 verify_comp_extent $tf ${ids[1]} 33554432 EOF
635 # verify component ${ids[2]}
636 verify_comp_attr stripe-size $tf ${ids[0]} 16777216
637 verify_comp_attr_with_default stripe-count $tf ${ids[2]}
638 verify_comp_attr pool $tf ${ids[2]} archive
639 verify_comp_extent $tf ${ids[2]} 0 EOF
641 # verify components ${ids[3]} and ${ids[5]}
643 verify_comp_attr_with_default stripe-size $tf ${ids[$i]}
644 verify_comp_attr stripe-count $tf ${ids[$i]} 2
645 verify_comp_attr stripe-index $tf ${ids[$i]} 1
646 verify_comp_extent $tf ${ids[$i]} 0 4194304
649 # verify components ${ids[4]} and ${ids[6]}
651 verify_comp_attr stripe-size $tf ${ids[$i]} 4194304
652 verify_comp_attr stripe-count $tf ${ids[$i]} 2
653 verify_comp_attr pool $tf ${ids[$i]} flash
654 verify_comp_extent $tf ${ids[$i]} 4194304 EOF
657 # verify component ${ids[7]}
658 verify_comp_attr stripe-size $tf ${ids[7]} 4194304
659 verify_comp_attr stripe-count $tf ${ids[7]} $OSTCOUNT
660 verify_comp_attr_with_parent pool $tf ${ids[7]}
661 verify_comp_extent $tf ${ids[7]} 0 EOF
663 # verify components ${ids[8]}, ${ids[10]} and ${ids[12]}
665 verify_comp_attr stripe-size $tf ${ids[$i]} 16777216
666 verify_comp_attr stripe-count $tf ${ids[$i]} $OSTCOUNT
667 verify_comp_attr pool $tf ${ids[$i]} archive
668 verify_comp_extent $tf ${ids[$i]} 0 536870912
671 # verify components ${ids[9]}, ${ids[11]} and ${ids[13]}
673 verify_comp_attr stripe-size $tf ${ids[$i]} 16777216
674 verify_comp_attr stripe-count $tf ${ids[$i]} -1
675 verify_comp_attr pool $tf ${ids[$i]} archive
676 verify_comp_extent $tf ${ids[$i]} 536870912 EOF
679 run_test 0f "lfs mirror extend composite layout mirrors"
684 ! $LFS mirror create --flags prefer $tf ||
685 error "creating $tf w/ --flags but w/o -N option should fail"
687 ! $LFS mirror create -N --flags foo $tf ||
688 error "creating $tf with '--flags foo' should fail"
690 ! $LFS mirror create -N --flags stale $tf ||
691 error "creating $tf with '--flags stale' should fail"
693 ! $LFS mirror create -N --flags prefer,init $tf ||
694 error "creating $tf with '--flags prefer,init' should fail"
696 ! $LFS mirror create -N --flags ^prefer $tf ||
697 error "creating $tf with '--flags ^prefer' should fail"
699 $LFS mirror create -N -E 1M -S 1M -o0 --flags=prefer -E eof -o1 \
700 -N -o1 $tf || error "create mirrored file $tf failed"
702 verify_comp_attr lcme_flags $tf 0x10001 prefer
703 verify_comp_attr lcme_flags $tf 0x10002 prefer
705 # write to the mirrored file and check primary
706 cp /etc/hosts $tf || error "error writing file '$tf'"
708 verify_comp_attr lcme_flags $tf 0x20003 stale
710 # resync file and check prefer flag
711 $LFS mirror resync $tf || error "error resync-ing file '$tf'"
714 $LCTL set_param osc.*.stats=clear
715 cat $tf &> /dev/null || error "error reading file '$tf'"
717 # verify that the data was provided by OST1 where mirror 1 resides
718 local nr_read=$($LCTL get_param -n osc.$FSNAME-OST0000-osc-[-0-9a-f]*.stats |
719 awk '/ost_read/{print $2}')
720 [ -n "$nr_read" ] || error "read was not provided by OST1"
722 run_test 0g "lfs mirror create flags support"
725 [ $MDS1_VERSION -lt $(version_code 2.11.57) ] &&
726 skip "Need MDS version at least 2.11.57"
733 # create parent directory
734 test_mkdir $td || error "mkdir $td failed"
736 $LFS setstripe -N -E 1M -S 1M --flags=prefer -E eof -N2 $td ||
737 error "set default mirrored layout on directory $td failed"
739 # verify flags are inherited from the directory
742 verify_comp_attr lcme_flags $tf 0x10001 prefer
743 verify_comp_attr lcme_flags $tf 0x10002 prefer
745 # set flags to the first component
746 ! $LFS setstripe --comp-set -I 0x10001 --comp-flags=^prefer,foo $tf ||
747 error "setting '^prefer,foo' flags should fail"
749 ! $LFS getstripe --component-flags=prefer,foo $tf ||
750 error "getting component(s) with 'prefer,foo' flags should fail"
752 $LFS setstripe --comp-set -I 0x10001 --comp-flags=^prefer,stale $tf
754 verify_comp_attr lcme_flags $tf 0x10001 stale
755 verify_comp_attr lcme_flags $tf 0x10002 prefer
757 $LFS setstripe --comp-set -I0x10001 --comp-flags=^stale $tf &&
758 error "clearing 'stale' should fail"
760 # write and resync file. It can't resync the file directly because the
761 # file state is still 'ro'
762 cp /etc/hosts $tf || error "error writing file '$tf'"
763 $LFS mirror resync $tf || error "error resync-ing file '$tf'"
765 $LFS setstripe --comp-set -I 0x20003 --comp-flags=prefer $tf ||
766 error "error setting flag prefer"
768 verify_comp_attr lcme_flags $tf 0x20003 prefer
770 $LFS setstripe --comp-set -I 0x20003 --comp-flags=^prefer $tf ||
771 error "error clearing prefer flag from component 0x20003"
773 # MDS disallows setting stale flag on the last non-stale mirror
774 [[ "$MDS1_VERSION" -ge $(version_code 2.12.57) ]] || return 0
776 cp /etc/hosts $tf || error "error writing file '$tf'"
778 verify_comp_attr lcme_flags $tf 0x10002 prefer
779 verify_comp_attr lcme_flags $tf 0x20003 stale
780 verify_comp_attr lcme_flags $tf 0x30004 stale
782 ! $LFS setstripe --comp-set -I 0x10002 --comp-flags=^prefer,stale $tf \
784 error "setting stale flag on component 0x10002 should fail"
786 $LFS mirror resync $tf || error "error resync-ing file '$tf'"
788 $LFS setstripe --comp-set -I 0x10001 --comp-flags=stale $tf ||
789 error "error setting stale flag on component 0x10001"
790 $LFS setstripe --comp-set -I 0x20003 --comp-flags=stale $tf ||
791 error "error setting stale flag on component 0x20003"
793 ! $LFS setstripe --comp-set -I 0x30004 --comp-flags=stale $tf \
795 error "setting stale flag on component 0x30004 should fail"
797 $LFS mirror resync $tf || error "error resync-ing file '$tf'"
799 run_test 0h "set, clear and test flags for FLR files"
802 $LFS mirror create -N2 $DIR/$tfile || error "create $DIR/$tfile failed"
804 cp /etc/hosts $DIR/$tfile || error "write to $DIR/$tfile failed"
805 $LFS mirror resync $DIR/$tfile || error "resync $DIR/$tfile failed"
806 cmp /etc/hosts $DIR/$tfile || error "cmp with /etc/hosts failed"
808 $LFS mirror read -N2 -o $TMP/$tfile $DIR/$tfile || error "read mirror failed"
809 stack_trap "rm -f $TMP/$tfile"
810 cmp $TMP/$tfile $DIR/$tfile || error "cmp with $TMP/$tfile failed"
811 $LFS mirror write -N2 -i /etc/passwd $DIR/$tfile || error "write failed"
812 $LFS setstripe --comp-set -I 65537 --comp-flags=stale $DIR/$tfile ||
813 error "set component 1 stale failed"
814 $LFS mirror resync $DIR/$tfile || error "resync $DIR/$tfile failed"
815 cmp /etc/passwd $DIR/$tfile || error "cmp with /etc/passwd failed"
817 run_test 0j "test lfs mirror read/write commands"
820 [[ $OSTCOUNT -lt 3 ]] && skip "need >= 3 OSTs" && return
823 # default FLR with 2 mirrors
824 $LFS setstripe -N -c1 -i0 -N -c1 -i1 $DIR/$tdir ||
825 error "set default layout failed"
827 # plain file extension under default FLR dir
828 echo $tfile >> $DIR/$tdir/$tfile.append || error "create $tfile.append failed"
829 echo -n " before extend $tfile.append, mirror count = "
830 $LFS getstripe -N $DIR/$tdir/$tfile.append
832 $LFS mirror extend -N -c1 -i2 $DIR/$tdir/$tfile.append ||
833 error "mirror extend failed"
834 echo -n " after extend $tfile.append, mirror count = "
835 $LFS getstripe -N $DIR/$tdir/$tfile.append
837 # normal file extension under default FLR dir
838 touch $DIR/$tdir/$tfile || error "create $tfile failed"
839 echo -n " before extend $tfile, mirror count = "
840 $LFS getstripe -N $DIR/$tdir/$tfile
842 $LFS mirror extend -N -c1 -i2 $DIR/$tdir/$tfile ||
843 error "mirror extend $tfile failed"
844 echo -n " after extend $tfile, mirror count = "
845 $LFS getstripe -N $DIR/$tdir/$tfile
847 run_test 0k "mirroring a file in directory with default FLR layout"
851 local mirror_count=16 # LUSTRE_MIRROR_COUNT_MAX
852 local mirror_create_cmd="$LFS mirror create"
853 local stripes[0]=$OSTCOUNT
855 mirror_create_cmd+=" -N -c ${stripes[0]}"
856 for ((i = 1; i < $mirror_count; i++)); do
857 # add mirrors with different stripes to the file
858 stripes[$i]=$((RANDOM % OSTCOUNT))
859 [ ${stripes[$i]} -eq 0 ] && stripes[$i]=1
861 mirror_create_cmd+=" -N -c ${stripes[$i]}"
864 $mirror_create_cmd $tf || error "create mirrored file $tf failed"
865 verify_mirror_count $tf $mirror_count
867 # can't create mirrors exceeding LUSTRE_MIRROR_COUNT_MAX
868 $LFS mirror extend -N $tf &&
869 error "Creating the $((mirror_count+1))th mirror succeeded"
871 local ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' |
874 # verify the range of components and stripe counts
875 for ((i = 0; i < $mirror_count; i++)); do
876 verify_comp_attr stripe-count $tf ${ids[$i]} ${stripes[$i]}
877 verify_comp_extent $tf ${ids[$i]} 0 EOF
880 run_test 1 "create components with setstripe options"
884 local tf2=$DIR/$tfile-2
886 $LFS setstripe -E 1M -S 1M -E EOF -c 1 $tf
887 $LFS setstripe -E 2M -S 1M -E EOF -c -1 $tf2
889 $LFS mirror extend -N -f $tf2 $tf ||
890 error "merging $tf2 into $tf failed"
892 verify_mirror_count $tf 2
893 [[ ! -e $tf2 ]] || error "$tf2 was not unlinked"
895 run_test 2 "create components from existing files"
898 [[ $MDSCOUNT -lt 2 ]] && skip "need >= 2 MDTs" && return
900 for ((i = 0; i < 2; i++)); do
901 $LFS mkdir -i $i $DIR/$tdir-$i
902 $LFS setstripe -E -1 $DIR/$tdir-$i/$tfile
905 $LFS mirror extend -N -f $DIR/$tdir-1/$tfile \
906 $DIR/$tdir-0/$tfile || error "creating mirrors"
908 # mdt doesn't support to cancel layout lock for remote objects, do
912 # make sure the mirrorted file was created successfully
913 [[ $($LFS getstripe --component-count $DIR/$tdir-0/$tfile) -eq 2 ]] ||
914 { $LFS getstripe $DIR/$tdir-0/$tfile;
915 error "expected 2 components"; }
920 run_test 3 "create components from files located on different MDTs"
923 local tf=$DIR/$tdir/$tfile
926 test_mkdir $DIR/$tdir
928 # set mirror with setstripe options to directory
929 $LFS mirror create -N2 -E 1M -S 1M -E eof $DIR/$tdir ||
930 error "set mirror to directory error"
932 [ x$($LFS getstripe -v $DIR/$tdir | awk '/lcm_flags/{print $2}') = \
933 x"mirrored" ] || error "failed to create mirrored dir"
936 verify_mirror_count $tf 2
938 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
939 verify_comp_extent $tf ${ids[0]} 0 1048576
940 verify_comp_extent $tf ${ids[1]} 1048576 EOF
942 # sub directory should inherit mirror setting from parent
943 test_mkdir $DIR/$tdir/td
944 [ x$($LFS getstripe -v $DIR/$tdir/td | awk '/lcm_flags/{print $2}') = \
945 x"mirrored" ] || error "failed to inherit mirror from parent"
947 # mirror extend won't be applied to directory
948 $LFS mirror extend -N2 $DIR/$tdir &&
949 error "expecting mirror extend failure"
952 run_test 4 "Make sure mirror attributes can be inhertied from directory"
958 $MULTIOP $tf oO_RDWR:O_CREAT:O_LOV_DELAY_CREATE:T12345c ||
959 error "failed to create file with non-empty layout"
960 $CHECKSTAT -t file -s 12345 $tf || error "size error: expecting 12345"
962 $LFS mirror create -N3 $tf || error "failed to attach mirror layout"
963 verify_mirror_count $tf 3
965 $CHECKSTAT -t file -s 12345 $tf ||
966 error "size error after attaching layout "
968 run_test 5 "Make sure init size work for mirrored layout"
971 (( $MDS1_VERSION >= $(version_code 2.12.58) )) ||
972 skip "MDS version older than 2.12.58"
976 $LFS mirror create -N -E 1M -L mdt -E eof -S 1M -N -E eof $tf ||
977 error "failure to create DoM file with mirror"
979 $LFS mirror create -N -E 1M -S 1M -E eof -N -E 1M -L mdt -E eof $tf ||
980 error "failure to create mirrored file with DoM"
982 $LFS setstripe -E 1M -L mdt -E eof -S 1M $tf ||
983 error "failure to create PFL with DoM file"
984 $LFS mirror extend -N2 $tf ||
985 error "failure to extend mirror with DoM"
987 $LFS setstripe -E 1M -L mdt -E eof -S 1M $tf-1 ||
988 error "failure to create PFL with DoM file"
989 $LFS mirror create -N2 -E 1M -S 1M -E eof $tf-2 ||
990 error "failure to create mirrored file"
991 $LFS mirror extend -N -f $tf-1 $tf-2 ||
992 error "failure to extend mirrored file with DoM extent"
994 run_test 6 "DoM and FLR work together"
999 # create DoM with setting stripe_size == component size
1000 $LFS mirror create -N -E1M -S1M -L mdt -Eeof $tf ||
1001 error "failure to create DoM with stripe_size == comp size"
1002 rm -f $tf || error "delete $tf"
1004 # DoM should not inherit previous component stripe_size
1005 $LFS mirror create -N -E4M -S2M -Eeof -N -E1M -L mdt -Eeof $tf ||
1006 error "DoM component shouldn't inherit previous stripe_size"
1008 run_test 7 "Create mirror with DoM component"
1011 local tf=$DIR/$tfile
1012 local tf2=$DIR/$tfile-2
1014 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
1016 stack_trap "rm -f $tf $tf2"
1018 $LFS setstripe -E EOF -o 0 $tf
1019 $LFS setstripe -E EOF -o 1 $tf2
1021 local dd_count=$((RANDOM % 20 + 1))
1022 dd if=/dev/zero of=$tf bs=1M count=$dd_count oflag=sync
1023 dd if=/dev/zero of=$tf2 bs=1M count=1 seek=$((dd_count - 1)) oflag=sync
1025 # for zfs - sync OST dataset so that du below will return
1027 [ "$FSTYPE" = "zfs" ] &&
1028 do_nodes $(comma_list $(osts_nodes)) "$ZPOOL sync"
1030 local blocks=$(du -kc $tf $tf2 | awk '/total/{print $1}')
1033 $LFS mirror extend -N -f $tf2 $tf ||
1034 error "merging $tf2 into $tf failed"
1036 # cancel layout lock
1037 cancel_lru_locks mdc
1039 local new_blocks=$(du -k $tf | awk '{print $1}')
1040 [ $new_blocks -eq $blocks ] ||
1041 error "i_blocks error expected: $blocks, actual: $new_blocks"
1043 run_test 21 "glimpse should report accurate i_blocks"
1045 get_osc_lock_count() {
1052 osc_name=${FSNAME}-OST$(printf "%04x" $((idx-1)))-osc-'[-0-9a-f]*'
1053 count=$($LCTL get_param -n ldlm.namespaces.$osc_name.lock_count)
1054 lock_count=$((lock_count + count))
1060 local tf=$DIR/$tfile
1062 stack_trap "rm -f $tf"
1064 $LFS setstripe -E EOF -o 0 $tf
1065 dd if=/dev/zero of=$tf bs=1M count=$((RANDOM % 20 + 1))
1067 # add component, two mirrors located on the same OST ;-)
1068 $LFS mirror extend -N -o 0 $tf ||
1069 error "extending mirrored file $tf failed"
1071 size_blocks=$(stat --format="%b %s" $tf)
1073 cancel_lru_locks mdc
1074 cancel_lru_locks osc
1076 local new_size_blocks=$(stat --format="%b %s" $tf)
1078 # make sure there is no lock cached
1079 [ $(get_osc_lock_count 1) -eq 0 ] || error "glimpse requests were sent"
1081 [ "$new_size_blocks" = "$size_blocks" ] ||
1082 echo "size expected: $size_blocks, actual: $new_size_blocks"
1086 run_test 22 "no glimpse to OSTs for READ_ONLY files"
1089 local tf=$DIR/$tfile
1091 $LFS mirror create -N -o 0 -N -o 1 $tf ||
1092 error "creating mirrored file $tf failed"
1095 for ((ost_idx = 1; ost_idx <= 2; ost_idx++)); do
1096 cancel_lru_locks osc
1099 local tmpfile=$(mktemp)
1100 stat --format="%b %s" $tf > $tmpfile &
1104 while [ $cnt -le 5 ]; do
1105 kill -0 $pid > /dev/null 2>&1 || break
1109 kill -0 $pid > /dev/null 2>&1 &&
1110 error "stat process stuck due to unavailable OSTs"
1112 # make sure glimpse request has been sent
1113 [ $(get_osc_lock_count 1 2) -ne 0 ] ||
1114 error "OST $ost_idx: no glimpse request was sent"
1119 run_test 31 "make sure glimpse request can be retried"
1122 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
1123 rm -f $DIR/$tfile $DIR/$tfile-2
1125 stack_trap "rm -f $DIR/$tfile"
1127 $LFS setstripe -E EOF -o 0 $DIR/$tfile
1128 dd if=/dev/urandom of=$DIR/$tfile bs=1M count=$((RANDOM % 10 + 2))
1130 local fsize=$(stat -c %s $DIR/$tfile)
1131 [[ $fsize -ne 0 ]] || error "file size is (wrongly) zero"
1133 local cksum=$(md5sum $DIR/$tfile)
1135 # create a new mirror in sync mode
1136 $LFS mirror extend -N -o 1 $DIR/$tfile ||
1137 error "extending mirrored file $DIR/$tfile failed"
1139 # make sure the mirrored file was created successfully
1140 [ $($LFS getstripe -N $DIR/$tfile) -eq 2 ] ||
1141 { $LFS getstripe $DIR/$tfile; error "expected 2 mirrors"; }
1146 # check size is correct, glimpse request should go to the 2nd mirror
1147 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
1148 error "file size error $fsize vs. $(stat -c %s $DIR/$tfile)"
1150 echo "reading file from the 2nd mirror and verify checksum"
1151 [[ "$cksum" == "$(md5sum $DIR/$tfile)" ]] ||
1152 error "checksum error: expected $cksum"
1156 run_test 32 "data should be mirrored to newly created mirror"
1159 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
1161 rm -f $DIR/$tfile $DIR/$tfile-2
1163 # create a file with two mirrors
1164 $LFS setstripe -E EOF -o 0 $DIR/$tfile
1167 while [ $count -lt $max_count ]; do
1168 echo "ost1" >> $DIR/$tfile
1169 count=$((count + 1));
1172 # tmp file that will be used as mirror
1173 $LFS setstripe -E EOF -o 1 $DIR/$tfile-2
1175 while [ $count -lt $max_count ]; do
1176 echo "ost2" >> $DIR/$tfile-2
1177 count=$((count + 1));
1180 # create a mirrored file
1181 $LFS mirror extend -N -f $DIR/$tfile-2 $DIR/$tfile &&
1182 error "merging $DIR/$tfile-2 into $DIR/$tfile" \
1183 "with verification should fail"
1184 $LFS mirror extend --no-verify -N -f $DIR/$tfile-2 $DIR/$tfile ||
1185 error "merging $DIR/$tfile-2 into $DIR/$tfile" \
1186 "without verification failed"
1188 # make sure that $tfile has two mirrors and $tfile-2 does not exist
1189 [ $($LFS getstripe -N $DIR/$tfile) -eq 2 ] ||
1190 { $LFS getstripe $DIR/$tfile; error "expected count 2"; }
1192 [[ ! -e $DIR/$tfile-2 ]] || error "$DIR/$tfile-2 was not unlinked"
1194 # execpted file size
1195 local fsize=$((5 * max_count))
1196 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
1197 error "mirrored file size is not $fsize"
1199 # read file - all OSTs are available
1200 echo "reading file (data can be provided by any ost)... "
1201 local rs=$(cat $DIR/$tfile | head -1)
1202 [[ "$rs" == "ost1" || "$rs" == "ost2" ]] ||
1203 error "file content error: expected: \"ost1\" or \"ost2\""
1205 # read file again with ost1 failed
1209 echo "reading file (data should be provided by ost2)..."
1210 local rs=$(cat $DIR/$tfile | head -1)
1211 [[ "$rs" == "ost2" ]] ||
1212 error "file content error: expected: \"ost2\", actual: \"$rs\""
1217 # read file again with ost2 failed
1221 # check size, glimpse should work
1222 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
1223 error "mirrored file size is not $fsize"
1225 echo "reading file (data should be provided by ost1)..."
1226 local rs=$(cat $DIR/$tfile | head -1)
1227 [[ "$rs" == "ost1" ]] ||
1228 error "file content error: expected: \"ost1\", actual: \"$rs\""
1232 run_test 33a "read can choose available mirror to read"
1235 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
1239 stack_trap "rm -f $DIR/$tfile" EXIT
1241 # create a file with two mirrors on OST0000 and OST0001
1242 $LFS setstripe -N -Eeof -o0 -N -Eeof -o1 $DIR/$tfile
1244 # make sure that $tfile has two mirrors
1245 [ $($LFS getstripe -N $DIR/$tfile) -eq 2 ] ||
1246 { $LFS getstripe $DIR/$tfile; error "expected count 2"; }
1249 dd if=/dev/urandom of=$DIR/$tfile bs=2M count=25 ||
1250 error "write failed for $DIR/$tfile"
1251 $LFS mirror resync $DIR/$tfile || error "resync failed for $DIR/$tfile"
1252 verify_flr_state $DIR/$tfile "ro"
1257 # read file - all OSTs are available
1258 echo "reading file (data can be provided by any ost)... "
1260 time cat $DIR/$tfile > /dev/null || error "read all"
1264 # read file again with ost1 {OST0000} failed
1267 echo "reading file (data should be provided by ost2)..."
1269 time cat $DIR/$tfile > /dev/null || error "read ost2"
1276 # read file again with ost2 {OST0001} failed
1280 echo "reading file (data should be provided by ost1)..."
1282 time cat $DIR/$tfile > /dev/null || error "read ost1"
1289 [ $((r1 * 100)) -gt $((ra * 105)) -a $r1 -gt $((ra + 2)) ] &&
1290 error "read mirror too slow without ost1, from $ra to $r1"
1291 [ $((r2 * 100)) -gt $((ra * 105)) -a $r2 -gt $((ra + 2)) ] &&
1292 error "read mirror too slow without ost2, from $ra to $r2"
1294 wait_osc_import_ready client ost2
1296 run_test 33b "avoid reading from unhealthy mirror"
1299 [[ $OSTCOUNT -lt 3 ]] && skip "need >= 3 OSTs" && return
1303 stack_trap "rm -f $DIR/$tfile" EXIT
1305 # create a file with two mirrors
1306 # mirror1: {OST0000, OST0001}
1307 # mirror2: {OST0001, OST0002}
1308 $LFS setstripe -N -Eeof -c2 -o0,1 -N -Eeof -c2 -o1,2 $DIR/$tfile
1310 # make sure that $tfile has two mirrors
1311 [ $($LFS getstripe -N $DIR/$tfile) -eq 2 ] ||
1312 { $LFS getstripe $DIR/$tfile; error "expected count 2"; }
1315 dd if=/dev/urandom of=$DIR/$tfile bs=2M count=25 ||
1316 error "write failed for $DIR/$tfile"
1317 $LFS mirror resync $DIR/$tfile || error "resync failed for $DIR/$tfile"
1318 verify_flr_state $DIR/$tfile "ro"
1323 # read file - all OSTs are available
1324 echo "reading file (data can be provided by any ost)... "
1325 time cat $DIR/$tfile > /dev/null || error "read all"
1327 # read file again with ost2 (OST0001) failed
1331 echo "reading file (data should be provided by ost1 and ost3)..."
1332 time cat $DIR/$tfile > /dev/null || error "read ost1 & ost3"
1337 wait_osc_import_ready client ost2
1339 run_test 33c "keep reading among unhealthy mirrors"
1342 (( $OSTCOUNT >= 4 )) || skip "need >= 4 OSTs"
1344 stack_trap "rm -f $DIR/$tfile $DIR/$tfile-2 $DIR/$tfile-ref"
1347 $LFS setstripe -o 0 $DIR/$tfile-ref
1348 dd if=/dev/urandom of=$DIR/$tfile-ref bs=1M count=3
1350 # create a file with two mirrors
1351 $LFS setstripe -E -1 -o 0,1 -S 1M $DIR/$tfile
1352 dd if=$DIR/$tfile-ref of=$DIR/$tfile bs=1M
1354 $LFS setstripe -E -1 -o 2,3 -S 1M $DIR/$tfile-2
1355 dd if=$DIR/$tfile-ref of=$DIR/$tfile-2 bs=1M
1357 $CHECKSTAT -t file -s $((3 * 1024 * 1024)) $DIR/$tfile ||
1358 error "mirrored file size is not 3M"
1360 # merge a mirrored file
1361 $LFS mirror extend -N -f $DIR/$tfile-2 $DIR/$tfile ||
1362 error "merging $DIR/$tfile-2 into $DIR/$tfile failed"
1364 cancel_lru_locks osc
1366 # stop two OSTs, so the 2nd stripe of the 1st mirror and
1367 # the 1st stripe of the 2nd mirror will be inaccessible, ...
1370 echo "comparing files ... "
1372 # however, read can still return the correct data. It should return
1373 # the 1st stripe from mirror 1 and 2st stripe from mirror 2.
1374 cmp -n 2097152 <(rwv -f $DIR/$tfile -r -o -n 1 2097152) \
1375 $DIR/$tfile-ref || error "file reading error"
1379 run_test 34a "read mirrored file with multiple stripes"
1382 (( $OSTCOUNT >= 4 )) || skip "need >= 4 OSTs"
1384 stack_trap "rm -f $DIR/$tfile $DIR/$tfile-2 $DIR/$tfile-ref"
1387 $LFS setstripe -o 0 $DIR/$tfile-ref
1388 dd if=/dev/urandom of=$DIR/$tfile-ref bs=1M count=3
1390 $LFS setstripe -E 1M -S 1M -o 0 -E eof -o 1 $DIR/$tfile
1391 dd if=$DIR/$tfile-ref of=$DIR/$tfile bs=1M
1393 $LFS setstripe -E 1M -S 1M -o 2 -E eof -o 3 $DIR/$tfile-2
1394 dd if=$DIR/$tfile-ref of=$DIR/$tfile-2 bs=1M
1396 $CHECKSTAT -t file -s $((3 * 1024 * 1024)) $DIR/$tfile ||
1397 error "mirrored file size is not 3M"
1399 # merge a mirrored file
1400 $LFS mirror extend -N -f $DIR/$tfile-2 $DIR/$tfile ||
1401 error "merging $DIR/$tfile-2 into $DIR/$tfile failed"
1403 cancel_lru_locks osc
1405 # stop two OSTs, so the 2nd component of the 1st mirror and
1406 # the 1st component of the 2nd mirror will be inaccessible, ...
1409 echo "comparing files ... "
1411 # however, read can still return the correct data. It should return
1412 # the 1st stripe from mirror 1 and 2st stripe from mirror 2.
1413 cmp -n 2097152 <(rwv -f $DIR/$tfile -r -o -n 1 2097152) \
1414 $DIR/$tfile-ref || error "file reading error"
1418 run_test 34b "read mirrored file with multiple components"
1421 local tf=$DIR/$tfile
1423 $LFS setstripe -E eof $tf
1425 # add an out-of-sync mirror to the file
1426 $LFS mirror extend -N -c 2 $tf ||
1427 error "extending mirrored file $tf failed"
1429 $MULTIOP $tf oO_WRONLY:c ||
1430 error "write open a mirrored file failed"
1432 # truncate file should return error
1433 $TRUNCATE $tf 100 || error "error truncating a mirrored file"
1435 run_test 35 "allow to write to mirrored files"
1437 get_file_layout_version() {
1438 $LFS getstripe $1 | awk '/lcm_layout_gen/{print $2}'
1441 get_ost_layout_version() {
1442 $MULTIOP $1 oXc | awk '/ostlayoutversion/{print $2}'
1445 verify_ost_layout_version() {
1448 # get file layout version
1449 local flv=$(get_file_layout_version $tf)
1451 # layout version from OST objects
1452 local olv=$(get_ost_layout_version $tf)
1454 (( flv >= olv )) || error "layout version mismatch: $flv vs. $olv"
1461 $LFS setstripe -E 1M -S 1M -E 2M -E 4M -E eof -c -1 $tf
1462 $LFS setstripe -E 3M -S 1M -E 6M -E eof -c -1 $tf-tmp
1464 $LFS mirror extend -N -f $tf-tmp $tf ||
1465 error "merging $tf-tmp into $tf failed"
1470 local tf=$DIR/$tfile
1472 stack_trap "rm -f $tf $tf-2 $tf-3"
1474 create_file_36 $tf $tf-2 $tf-3
1476 [ $($LFS getstripe -N $tf) -gt 1 ] || error "wrong mirror count"
1478 # test case 1 - check file write and verify layout version
1479 $MULTIOP $tf oO_WRONLY:c ||
1480 error "write open a mirrored file failed"
1482 # write open file should not return error
1483 $MULTIOP $tf oO_WRONLY:w1024Yc || error "write mirrored file error"
1485 # instantiate components should work
1486 dd if=/dev/zero of=$tf bs=1M count=12 || error "write file error"
1488 # verify OST layout version
1489 verify_ost_layout_version $tf
1492 local mds_facet=mds$(($($LFS getstripe -m $tf-2) + 1))
1495 do_facet $mds_facet $LCTL set_param fail_val=$delay_sec
1497 #define OBD_FAIL_FLR_LV_DELAY 0x1A01
1498 do_facet $mds_facet $LCTL set_param fail_loc=0x1A01
1500 # write should take at least $fail_loc seconds and succeed
1501 local st=$(date +%s)
1502 $MULTIOP $tf-2 oO_WRONLY:w1024Yc || error "write mirrored file error"
1504 # verify OST layout version
1505 verify_ost_layout_version $tf
1507 do_facet $mds_facet $LCTL set_param fail_loc=0
1509 run_test 36a "write to mirrored files"
1512 local tf=$DIR/$tfile
1514 (( OST1_VERSION >= $(version_code 2.15.50) )) ||
1515 skip "Need OST version at least 2.15.50"
1517 (( OSTCOUNT >= 2 )) || skip "need >= 2 OSTs"
1519 # create 2 mirrors using different OSTs
1520 $LFS setstripe -N -c1 -i0 --flags=prefer -N -c1 -i1 $tf ||
1521 error "create mirrored file"
1523 # write 1M data to one mirror
1524 dd if=/dev/zero of=$tf bs=1M count=1 || error "write file error"
1527 # set prefer mirror to another mirror
1528 $LFS setstripe --comp-set -I0x10001 --comp-flags=^prefer $tf ||
1529 error "clear prefer mirror error"
1530 $LFS setstripe --comp-set -I0x20002 --comp-flags=prefer $tf ||
1531 error "set prefer mirror error"
1533 # the second write should not hung
1534 dd if=/dev/zero of=$tf bs=1M count=1 || error "write file error"
1536 run_test 36b "write should not hung when prefered mirror is stale"
1539 local tf=$DIR/$tfile
1541 (( OST1_VERSION >= $(version_code 2.15.50) )) ||
1542 skip "Need OST version at least 2.15.50"
1544 (( OSTCOUNT >= 2 )) || skip "need >= 2 OSTs"
1546 # create 2 mirrors using different OSTs
1547 $LFS setstripe -N -c1 -i0 --flags=prefer -N -c1 -i1 $tf ||
1548 error "create mirrored file"
1549 stack_trap "rm -f $tf"
1551 # write it in the background
1552 dd if=/dev/zero of=$tf bs=1M count=600 &
1557 $LFS setstripe --comp-set -I0x10001 --comp-flags=^prefer $tf ||
1558 error "clear prefer mirror error"
1559 $LFS setstripe --comp-set -I0x20002 --comp-flags=prefer $tf ||
1560 error "set prefer mirror error"
1564 run_test 36c "change prefer mirror during write shouldn't hung"
1567 local tf=$DIR/$tfile
1569 (( OST1_VERSION >= $(version_code 2.15.50) )) ||
1570 skip "Need OST version at least 2.15.50"
1572 echo " ** create $tf"
1573 $LFS mirror create -N $tf || error "create $tf failed"
1576 echo " ** mirror extend $tf ($i/2)"
1577 $LFS mirror extend -N $tf || error "mirror extend $tf failed"
1578 flv=$(get_file_layout_version $tf)
1579 olv=$(get_ost_layout_version $tf)
1580 echo " flv=$flv olv=$olv"
1584 echo " ** write $tf ($i/2)"
1585 dd if=/dev/zero of=$tf bs=1k count=1 || error "write $tf failed"
1586 flv=$(get_file_layout_version $tf)
1587 olv=$(get_ost_layout_version $tf)
1588 echo " flv=$flv olv=$olv"
1590 error "write update OST layout failed $flv/$olv"
1593 echo " ** resync $tf"
1594 $LFS mirror resync $tf || error "mirror resync $tf failed"
1595 flv=$(get_file_layout_version $tf)
1596 olv=$(get_ost_layout_version $tf)
1597 echo " flv=$flv olv=$olv"
1600 echo " ** truncate $tf ($i/2)"
1601 $TRUNCATE $tf $((1024 * 1024)) || error "truncate $tf fails"
1602 flv=$(get_file_layout_version $tf)
1603 olv=$(get_ost_layout_version $tf)
1604 echo " flv=$flv olv=$olv"
1605 (( flv == olv || flv == olv + 1 )) ||
1606 error "truncate update OST layout failed $flv/$olv"
1609 echo " ** resync $tf"
1610 $LFS mirror resync $tf || error "mirror resync $tf failed"
1611 flv=$(get_file_layout_version $tf)
1612 olv=$(get_ost_layout_version $tf)
1613 echo " flv=$flv olv=$olv"
1616 echo " ** write $tf ($i/2)"
1617 dd if=/dev/zero of=$tf bs=1k count=1 || error "write $tf failed"
1618 flv=$(get_file_layout_version $tf)
1619 olv=$(get_ost_layout_version $tf)
1620 echo " flv=$flv olv=$olv"
1622 error "write update OST layout failed $flv/$olv"
1625 run_test 36d "write/punch FLR file update OST layout version"
1631 echo "create test files with size $fsize .."
1635 $LFS setstripe -E 1M -S 1M -c 1 -E eof -c -1 $tf
1637 dd if=/dev/urandom of=$tf bs=1M count=16 &> /dev/null
1638 $TRUNCATE $tf $fsize
1644 [ $MDS1_VERSION -lt $(version_code 2.11.57) ] &&
1645 skip "Need MDS version at least 2.11.57"
1647 local tf=$DIR/$tfile
1648 local tf2=$DIR/$tfile-2
1649 local tf3=$DIR/$tfile-3
1650 local tf4=$DIR/$tfile-4
1652 stack_trap "rm -f $tf $tf2 $tf3 $tf4"
1654 create_files_37 $((RANDOM + 15 * 1048576)) $tf $tf2 $tf3
1657 # assume the mirror id will be 1, 2, and 3
1658 declare -A checksums
1659 checksums[1]=$(cat $tf | md5sum)
1660 checksums[2]=$(cat $tf2 | md5sum)
1661 checksums[3]=$(cat $tf3 | md5sum)
1663 printf '%s\n' "${checksums[@]}"
1665 # merge these files into a mirrored file
1666 $LFS mirror extend --no-verify -N -f $tf2 $tf ||
1667 error "merging $tf2 into $tf failed"
1668 $LFS mirror extend --no-verify -N -f $tf3 $tf ||
1669 error "merging $tf3 into $tf failed"
1673 # verify mirror read, checksums should equal to the original files'
1674 echo "Verifying mirror read .."
1677 for i in "${mirror_array[@]}"; do
1678 $LCTL set_param ldlm.namespaces.*.lru_size=clear > /dev/null
1679 sum=$($LFS mirror read -N $i $tf | md5sum)
1680 [ "$sum" = "${checksums[$i]}" ] ||
1681 error "$i: mismatch: \'${checksums[$i]}\' vs. \'$sum\'"
1684 # verify mirror write
1685 echo "Verifying mirror write .."
1686 $LFS mirror write -N2 $tf < $tf4
1688 sum=$($LFS mirror read -N2 $tf | md5sum)
1689 [[ "$sum" = "${checksums[1]}" ]] ||
1690 error "2: mismatch \'${checksums[1]}\' vs. \'$sum\'"
1692 # verify mirror copy, write to this mirrored file will invalidate
1693 # the other two mirrors
1694 echo "Verifying mirror copy .."
1696 local osts=$(comma_list $(osts_nodes))
1698 $LFS mirror copy -i ${mirror_array[0]} -o-1 $tf ||
1699 error "mirror copy error"
1701 # verify copying is successful by checking checksums
1702 remount_client $MOUNT
1703 for i in "${mirror_array[@]}"; do
1704 sum=$($LFS mirror read -N $i $tf | md5sum)
1705 [ "$sum" = "${checksums[1]}" ] ||
1706 error "$i: mismatch checksum after copy \'$sum\'"
1711 run_test 37 "mirror I/O API verification"
1714 local tf=$DIR/$tfile
1715 local ref=$DIR/${tfile}-ref
1717 stack_trap "rm -f $tf $ref"
1719 $LFS setstripe -E 1M -S 1M -c 1 -E 4M -c 2 -E eof -c -1 $tf ||
1720 error "creating $tf failed"
1721 $LFS setstripe -E 2M -S 1M -c 1 -E 6M -c 2 -E 8M -c -1 -E eof -c -1 \
1722 $tf-2 || error "creating $tf-2 failed"
1723 $LFS setstripe -E 4M -c 1 -E 8M -c 2 -E eof -c -1 $tf-3 ||
1724 error "creating $tf-3 failed"
1726 # instantiate all components
1727 $LFS mirror extend -N -f $tf-2 $tf ||
1728 error "merging $tf-2 into $tf failed"
1729 $LFS mirror extend -N -f $tf-3 $tf ||
1730 error "merging $tf-3 into $tf failed"
1731 $LFS mirror extend -N -c 1 $tf ||
1732 error "extending mirrored file $tf failed"
1734 verify_flr_state $tf "ro"
1736 dd if=/dev/urandom of=$ref bs=1M count=16 &> /dev/null
1738 local fsize=$((RANDOM << 8 + 1048576))
1739 $TRUNCATE $ref $fsize
1741 local ref_cksum=$(cat $ref | md5sum)
1743 # case 1: verify write to mirrored file & resync work
1744 cp $ref $tf || error "copy from $ref to $f error"
1745 verify_flr_state $tf "wp"
1747 local file_cksum=$(cat $tf | md5sum)
1748 [ "$file_cksum" = "$ref_cksum" ] || error "write failed, cksum mismatch"
1751 echo "mirror IDs: ${mirror_array[*]}"
1753 local valid_mirror stale_mirror id mirror_cksum
1754 for id in "${mirror_array[@]}"; do
1755 mirror_cksum=$($LFS mirror read -N $id $tf | md5sum)
1756 [ "$ref_cksum" == "$mirror_cksum" ] &&
1757 { valid_mirror=$id; continue; }
1762 [ -z "$stale_mirror" ] && error "stale mirror doesn't exist"
1763 [ -z "$valid_mirror" ] && error "valid mirror doesn't exist"
1765 $LFS mirror resync $tf || error "resync failed"
1766 verify_flr_state $tf "ro"
1768 mirror_cksum=$($LFS mirror read -N $stale_mirror $tf | md5sum)
1769 [ "$file_cksum" = "$ref_cksum" ] || error "resync failed"
1771 # case 2: inject an error to make mirror_io exit after changing
1772 # the file state to sync_pending so that we can start a concurrent
1774 $MULTIOP $tf oO_WRONLY:w$((RANDOM % 1048576 + 1024))c
1775 verify_flr_state $tf "wp"
1777 mirror_io resync -e resync_start $tf && error "resync succeeded"
1778 verify_flr_state $tf "sp"
1780 # from sync_pending to write_pending
1781 $MULTIOP $tf oO_WRONLY:w$((RANDOM % 1048576 + 1024))c
1782 verify_flr_state $tf "wp"
1784 mirror_io resync -e resync_start $tf && error "resync succeeded"
1785 verify_flr_state $tf "sp"
1787 # from sync_pending to read_only
1788 $LFS mirror resync $tf || error "resync failed"
1789 verify_flr_state $tf "ro"
1791 run_test 38 "resync"
1794 local tf=$DIR/$tfile
1797 $LFS mirror create -N2 -E1m -c1 -S1M -E-1 $tf ||
1798 error "create PFL file $tf failed"
1800 verify_mirror_count $tf 2
1801 verify_comp_count $tf 4
1803 rm -f $tf || error "delete $tf failed"
1805 run_test 39 "check FLR+PFL (a.k.a. PFLR) creation"
1808 local tf=$DIR/$tfile
1811 for ops in "conv=notrunc" ""; do
1814 $LFS mirror create -N -E 2M -S 1M -E 4M -E -1 --flags=prefer \
1815 -N -E 1M -E 2M -E 4M -E -1 $tf ||
1816 error "create PFLR file $tf failed"
1817 dd if=/dev/zero of=$tf $ops bs=1M seek=2 count=1 ||
1818 error "write PFLR file $tf failed"
1820 lfs getstripe -vy $tf
1824 # file mirror state should be write_pending
1825 flags=$($LFS getstripe -v $tf | awk '/lcm_flags:/ { print $2 }')
1827 error "file mirror state $flags"
1828 # the 1st component (in mirror 1) should be inited
1829 verify_comp_attr lcme_flags $tf 0x10001 init
1830 # the 2nd component (in mirror 1) should be inited
1831 verify_comp_attr lcme_flags $tf 0x10002 init
1832 # the 3rd component (in mirror 1) should be uninited
1833 verify_comp_attr lcme_flags $tf 0x10003 prefer
1834 # the 4th component (in mirror 2) should be inited
1835 verify_comp_attr lcme_flags $tf 0x20004 init
1836 # the 5th component (in mirror 2) should be uninited
1837 verify_comp_attr lcme_flags $tf 0x20005 0
1838 # the 6th component (in mirror 2) should be stale
1839 verify_comp_attr lcme_flags $tf 0x20006 stale
1840 # the 7th component (in mirror 2) should be uninited
1841 if [[ x$ops = "xconv=notrunc" ]]; then
1842 verify_comp_attr lcme_flags $tf 0x20007 0
1843 elif [[ x$ops = "x" ]]; then
1844 verify_comp_attr lcme_flags $tf 0x20007 stale
1848 rm -f $tf || error "delete $tf failed"
1850 run_test 40 "PFLR rdonly state instantiation check"
1853 local tf=$DIR/$tfile
1855 stack_trap "rm -f $tf $tf-1"
1858 echo " **create two FLR files $tf $tf-1"
1859 $LFS mirror create -N -E 2M -S 1M -E 4M -E -1 \
1860 -N -E 1M -E 2M -E 3M -E -1 $tf ||
1861 error "create PFLR file $tf failed"
1862 $LFS mirror create -N -E 2M -S 1M -E eof \
1863 -N -E 1M -E eof --flags prefer \
1864 -N -E 4m -E eof $tf-1 ||
1865 error "create PFLR file $tf-1 failed"
1867 # file should be in ro status
1868 echo " **verify files be RDONLY"
1869 verify_flr_state $tf "ro"
1870 verify_flr_state $tf-1 "ro"
1872 # write data in [0, 2M)
1873 dd if=/dev/zero of=$tf bs=1M count=2 conv=notrunc ||
1874 error "writing $tf failed"
1875 dd if=/dev/urandom of=$tf-1 bs=1M count=4 conv=notrunc ||
1876 error "writing $tf-1 failed"
1878 local sum0=$(cat $tf-1 | md5sum)
1880 echo " **verify files be WRITE_PENDING"
1881 verify_flr_state $tf "wp"
1882 verify_flr_state $tf-1 "wp"
1884 # file should have stale component
1885 echo " **verify files have stale component"
1886 $LFS getstripe $tf | grep lcme_flags | grep stale > /dev/null ||
1887 error "after writing $tf, it does not contain stale component"
1888 $LFS getstripe $tf-1 | grep lcme_flags | grep stale > /dev/null ||
1889 error "after writing $tf-1, it does not contain stale component"
1891 echo " **full resync"
1892 $LFS mirror resync $tf $tf-1 || error "mirror resync $tf $tf-1 failed"
1894 echo " **verify $tf-1 data consistency in all mirrors"
1896 local sum=$($LFS mirror read -N$i $tf-1 | md5sum)
1897 [[ "$sum" = "$sum0" ]] ||
1898 error "$tf-1.$i: checksum mismatch: $sum != $sum0"
1901 echo " **verify files be RDONLY"
1902 verify_flr_state $tf "ro"
1903 verify_flr_state $tf-1 "ro"
1905 # file should not have stale component
1906 echo " **verify files do not contain stale component"
1907 $LFS getstripe $tf | grep lcme_flags | grep stale &&
1908 error "after resyncing $tf, it contains stale component"
1909 $LFS getstripe $tf-1 | grep lcme_flags | grep stale &&
1910 error "after resyncing $tf, it contains stale component"
1912 # verify partial resync
1913 echo " **write $tf-1 for partial resync test"
1914 dd if=/dev/zero of=$tf-1 bs=1M count=2 conv=notrunc ||
1915 error "writing $tf-1 failed"
1917 echo " **only resync mirror 2"
1918 verify_flr_state $tf-1 "wp"
1919 $LFS mirror resync --only 2 $tf-1 ||
1920 error "resync mirror 2 of $tf-1 failed"
1921 verify_flr_state $tf "ro"
1923 # resync synced mirror
1924 echo " **resync mirror 2 again"
1925 $LFS mirror resync --only 2 $tf-1 ||
1926 error "resync mirror 2 of $tf-1 failed"
1927 verify_flr_state $tf "ro"
1928 echo " **verify $tf-1 contains stale component"
1929 $LFS getstripe $tf-1 | grep lcme_flags | grep stale > /dev/null ||
1930 error "after writing $tf-1, it does not contain stale component"
1932 echo " **full resync $tf-1"
1933 $LFS mirror resync $tf-1 || error "resync of $tf-1 failed"
1934 verify_flr_state $tf "ro"
1935 echo " **full resync $tf-1 again"
1936 $LFS mirror resync $tf-1 || error "resync of $tf-1 failed"
1937 echo " **verify $tf-1 does not contain stale component"
1938 $LFS getstripe $tf | grep lcme_flags | grep stale &&
1939 error "after resyncing $tf, it contains stale component"
1943 run_test 41 "lfs mirror resync check"
1946 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
1950 local mirror_cmd="$LFS mirror verify"
1953 stack_trap "rm -rf $td"
1955 # create parent directory
1956 mkdir $td || error "mkdir $td failed"
1958 $mirror_cmd &> /dev/null && error "no file name given"
1959 $mirror_cmd $tf &> /dev/null && error "cannot stat file $tf"
1960 $mirror_cmd $td &> /dev/null && error "$td is not a regular file"
1962 # create mirrored files
1963 $LFS mirror create -N -E 4M -S 1M -E 10M -E EOF $tf ||
1964 error "create mirrored file $tf failed"
1965 $LFS mirror create -N -E 2M -S 1M -E EOF \
1966 -N -E 6M -E 8M -E EOF \
1967 -N -E 16M -E EOF $tf-1 ||
1968 error "create mirrored file $tf-1 failed"
1969 $LFS mirror create -N -c 2 -o 1,3 -N -S 2M -c -1 $tf-2 ||
1970 error "create mirrored file $tf-2 failed"
1972 # write data in [0, 10M)
1973 for i in $tf $tf-1 $tf-2; do
1974 yes | dd of=$i bs=1M count=10 iflag=fullblock conv=notrunc ||
1975 error "write $i failed"
1978 # resync the mirrored files
1979 $LFS mirror resync $tf-1 $tf-2 ||
1980 error "resync $tf-1 $tf-2 failed"
1982 # verify the mirrored files
1983 $mirror_cmd $tf-1 $tf-2 ||
1984 error "verify $tf-1 $tf-2 failed"
1986 get_mirror_ids $tf-1
1987 $mirror_cmd --only ${mirror_array[0]} $tf-1 &> /dev/null &&
1988 error "at least 2 mirror ids needed with '--only' option"
1989 $mirror_cmd --only ${mirror_array[0]},${mirror_array[1]} $tf-1 $tf-2 \
1991 error "'--only' option cannot be used upon multiple files"
1992 $mirror_cmd --only 65534,${mirror_array[0]},65535 $tf-1 &&
1993 error "invalid specified mirror ids"
1995 # change the content of $tf and merge it into $tf-1
1997 echo a | dd of=$tf bs=1M seek=$i conv=notrunc ||
1998 error "change $tf with seek=$i failed"
1999 echo b | dd of=$tf-1 bs=1M seek=$i conv=notrunc ||
2000 error "change $tf-1 with seek=$i failed"
2003 $LFS mirror resync $tf-1 || error "resync $tf-1 failed"
2004 $LFS mirror extend --no-verify -N -f $tf $tf-1 ||
2005 error "merge $tf into $tf-1 failed"
2007 # verify the mirrored files
2008 echo "Verify $tf-1 without -v option:"
2009 $mirror_cmd $tf-1 &&
2010 error "verify $tf-1 should fail" || echo "PASS"
2012 echo "Verify $tf-1 with -v option:"
2013 $mirror_cmd -v $tf-1 &&
2014 error "verify $tf-1 should fail"
2016 get_mirror_ids $tf-1
2017 echo "Verify $tf-1 with --only option:"
2018 $mirror_cmd -v --only ${mirror_array[1]},${mirror_array[-1]} $tf-1 &&
2019 error "verify $tf-1 with mirror ${mirror_array[1]} and" \
2020 "${mirror_array[-1]} should fail"
2022 $mirror_cmd --only ${mirror_array[0]},${mirror_array[1]} $tf-1 ||
2023 error "verify $tf-1 with mirror ${mirror_array[0]} and" \
2024 "${mirror_array[1]} should succeed"
2026 # set stale components in $tf-1
2027 for i in 0x40002 0x40003; do
2028 $LFS setstripe --comp-set -I$i --comp-flags=stale $tf-1 ||
2029 error "set stale flag on component $i failed"
2032 # verify the mirrored file
2033 echo "Verify $tf-1 with stale components:"
2034 $mirror_cmd -vvv $tf-1 ||
2035 error "verify $tf-1 with stale components should succeed"
2037 echo "Verify $tf-1 with stale components and --only option:"
2038 $mirror_cmd -vvv --only ${mirror_array[1]},${mirror_array[-1]} $tf-1 ||
2039 error "verify $tf-1 with mirror ${mirror_array[1]} and" \
2040 "${mirror_array[-1]} should succeed"
2042 run_test 42 "lfs mirror verify"
2044 # inactivate one OST && write && restore the OST
2048 local PARAM="osp.${FSNAME}-OST000${ost}-osc-M*.active"
2051 wait=$(do_facet $SINGLEMDS \
2052 "$LCTL get_param -n lod.*MDT0000-*.qos_maxage")
2053 wait=${wait%%[^0-9]*}
2055 echo " **deactivate OST$ost, waiting for $((wait*2+2)) seconds"
2056 $(do_facet $SINGLEMDS "$LCTL set_param -n $PARAM 0")
2057 # lod_qos_statfs_update needs 2*$wait seconds to refresh targets statfs
2058 sleep $(($wait * 2 + 2))
2059 echo " **write $file"
2060 dd if=/dev/zero of=$file bs=1M count=1 || error "write $file failed"
2061 echo " **restore activating OST$ost, waiting for $((wait*2+2)) seconds"
2062 $(do_facet $SINGLEMDS "$LCTL set_param -n $PARAM 1")
2063 sleep $((wait * 2 + 2))
2065 local flags=$($LFS getstripe -v $file | awk '/lcm_flags:/ { print $2 }')
2066 [ $flags = wp ] || error "file mirror state $flags != wp"
2070 [ $OSTCOUNT -lt 3 ] && skip "needs >= 3 OSTs" && return
2072 local tf=$DIR/$tfile
2075 stack_trap "rm -f $tf"
2077 ## mirror 0 ost (0, 1)
2078 ## mirror 1 ost (1, 2)
2079 ## mirror 2 ost (2, 0)
2080 $LFS mirror create -N -Eeof -c2 -o0,1 -N -Eeof -c2 -o1,2 \
2081 -N -Eeof -c2 -o2,0 $tf ||
2082 error "create 3 mirrors file $tf failed"
2084 ################## OST0 ###########################################
2086 echo " **verify components"
2087 verify_comp_attr lcme_flags $tf 0x10001 init,stale
2088 verify_comp_attr lcme_flags $tf 0x20002 init
2089 verify_comp_attr lcme_flags $tf 0x30003 init,stale
2092 echo " **resync $tf"
2093 $LFS mirror resync $tf
2094 flags=$($LFS getstripe -v $tf | awk '/lcm_flags:/ { print $2 }')
2095 [ $flags = ro ] || error "file mirror state $flags != ro"
2097 ################## OST1 ###########################################
2099 echo " **verify components"
2100 verify_comp_attr lcme_flags $tf 0x10001 init,stale
2101 verify_comp_attr lcme_flags $tf 0x20002 init,stale
2102 verify_comp_attr lcme_flags $tf 0x30003 init
2105 echo " **resync $tf"
2106 $LFS mirror resync $tf
2107 flags=$($LFS getstripe -v $tf | awk '/lcm_flags:/ { print $2 }')
2108 [ $flags = ro ] || error "file mirror state $flags != ro"
2110 ################## OST2 ###########################################
2112 echo " **verify components"
2113 verify_comp_attr lcme_flags $tf 0x10001 init
2114 verify_comp_attr lcme_flags $tf 0x20002 init,stale
2115 verify_comp_attr lcme_flags $tf 0x30003 init,stale
2117 run_test 43a "mirror pick on write"
2120 local tf=$DIR/$tdir/$tfile
2122 test_mkdir $DIR/$tdir
2124 stack_trap "rm -rf $tf"
2126 # create 3 mirrors FLR file, the first 2 mirrors are preferred
2127 $LFS setstripe -N -Eeof --flags=prefer -N -Eeof --flags=prefer \
2128 -N -Eeof $tf || error "create 3 mirrors file $tf failed"
2129 verify_flr_state $tf "ro"
2131 echo " ** write to $tf"
2132 dd if=/dev/zero of=$tf bs=1M count=1 || error "write $tf failed"
2133 verify_flr_state $tf "wp"
2135 echo " ** resync $tf"
2136 $LFS mirror resync $tf || error "resync $tf failed"
2137 verify_flr_state $tf "ro"
2139 run_test 43b "allow writing to multiple preferred mirror file"
2142 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
2145 local tf=$DIR/$tdir/$tfile
2146 local tf1=$DIR/$tdir-1/$tfile-1
2148 stack_trap "rm -rf $tf $tf1"
2150 $LFS setdirstripe -i 0 -c 1 $DIR/$tdir ||
2151 error "create directory failed"
2152 $LFS setdirstripe -i 1 -c 1 $DIR/$tdir-1 ||
2153 error "create remote directory failed"
2154 rm -f $tf $tf1 $tf.mirror~2
2155 # create file with 4 mirrors
2156 $LFS mirror create -N -E 2M -S 1M -E 4M -E -1 \
2157 -N -E 1M -E 2M -E 3M -E -1 -N2 $tf ||
2158 error "create PFLR file $tf failed"
2160 # file should be in ro status
2161 verify_flr_state $tf "ro"
2163 # write data in [0, 3M)
2164 dd if=/dev/urandom of=$tf bs=1M count=3 conv=notrunc ||
2165 error "writing $tf failed"
2167 verify_flr_state $tf "wp"
2169 # disallow destroying the last non-stale mirror
2170 ! $LFS mirror delete --mirror-id 1 $tf > /dev/null 2>&1 ||
2171 error "destroying mirror 1 should fail"
2173 # synchronize all mirrors of the file
2174 $LFS mirror resync $tf || error "mirror resync $tf failed"
2176 verify_flr_state $tf "ro"
2179 $LFS mirror split --mirror-id 1 -f $tf1 $tf ||
2180 error "split to $tf1 failed"
2182 local idx0=$($LFS getstripe -m $tf)
2183 local idx1=$($LFS getstripe -m $tf1)
2185 [[ x$idx0 == x0 ]] || error "$tf is not on MDT0"
2186 [[ x$idx1 == x1 ]] || error "$tf1 is not on MDT1"
2188 # verify mirror count
2189 verify_mirror_count $tf 3
2190 verify_mirror_count $tf1 1
2192 $LFS mirror split --mirror-id 2 $tf ||
2193 error "split mirror 2 failed"
2195 verify_mirror_count $tf 2
2196 verify_mirror_count $tf.mirror~2 1
2198 $LFS setstripe --comp-set -I 0x30008 --comp-flags=stale $tf ||
2199 error "setting stale flag on component 0x30008 failed"
2201 # disallow destroying the last non-stale mirror
2202 ! $LFS mirror split --mirror-id 4 -d $tf > /dev/null 2>&1 ||
2203 error "destroying mirror 4 should fail"
2205 $LFS mirror resync $tf || error "resynchronizing $tf failed"
2207 $LFS mirror split --mirror-id 3 -d $tf ||
2208 error "destroying mirror 3 failed"
2209 verify_mirror_count $tf 1
2211 # verify splitted file contains the same content as the orig file does
2212 diff $tf $tf1 || error "splited file $tf1 diffs from $tf"
2213 diff $tf $tf.mirror~2 ||
2214 error "splited file $tf.mirror~2 diffs from $tf"
2216 run_test 44a "lfs mirror split check"
2219 (( $MDS1_VERSION >= $(version_code 2.14.56) )) ||
2220 skip "Need MDS version at least 2.14.56"
2223 local tf=$DIR/$tdir/$tfile
2225 mkdir -p $DIR/$tdir || error "create directory failed"
2229 # create 2 mirrors file
2230 $LFS mirror extend -N -c1 $tf
2234 verify_flr_state $tf "wp"
2236 local str=$(cat $tf)
2238 [[ $str == "YYY" ]] || error "$tf content is not YYY"
2240 # get the non-stale mirror id
2241 local ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' |
2243 local mirror_ids=($($LFS getstripe $tf |
2244 awk '/lcme_mirror_id/{print $2}' | tr '\n' ' '))
2245 for ((i = 0; i < 2; i++)); do
2246 $LFS getstripe -I${ids[$i]} --component-flags $tf |
2247 grep stale > /dev/null || break
2250 [[ $i -ge 2 ]] && ( $LFS getstripe $tf; error "no stale mirror" )
2254 # split the updated mirror, should fail
2255 echo "split mirror_id ${mirror_ids[$i]} id ${ids[$i]}, should fail"
2256 $LFS mirror split --mirror-id=${mirror_ids[$i]} $tf &> /dev/null &&
2257 error "split --mirror-id=${mirror_ids[$i]} $tf should fail"
2260 # split the stale mirror
2261 echo "split mirror_id ${mirror_ids[$i]} id ${ids[$i]}"
2262 $LFS mirror split --mirror-id=${mirror_ids[$i]} -d $tf ||
2263 error "mirror split --mirror-id=${mirror_ids[$i]} $tf failed"
2265 echo "make sure there's no stale comp in the file"
2266 # make sure there's no stale comp in the file
2267 $LFS getstripe $tf | awk '/lcme_flags/{print $2}' | grep stale &&
2268 ( $LFS getstripe $tf; error "stale mirror file" )
2271 [[ $str == "YYY" ]] ||
2272 ( cat $tf; error "$tf content is not YYY after split" )
2274 run_test 44b "mirror split does not create stale file"
2277 local tf=$DIR/$tdir/$tfile
2279 stack_trap "rm -f $tf"
2281 [ $MDS1_VERSION -ge $(version_code 2.14.52) ] ||
2282 skip "Need MDS version at least 2.14.52"
2284 [ "$FSTYPE" != "zfs" ] || skip "ZFS file's block number is not accurate"
2286 mkdir -p $DIR/$tdir || error "create directroy failed"
2288 dd if=/dev/zero of=$tf bs=1M count=10 || error "dd write $tfile failed"
2290 block1=$(( $(stat -c "%b*%B" $tf) ))
2291 echo " ** before mirror ops, file blocks=$((block1/1024)) KiB"
2293 $LFS mirror extend -N2 -c1 $tf || error "mirror extend $tfile failed"
2295 block2=$(( $(stat -c "%b*%B" $tf) ))
2296 echo " ** after mirror extend, file blocks=$((block2/1024)) KiB"
2298 $LFS mirror split -d --mirror-id=2 $tf ||
2299 error "mirror split $tfile failed"
2300 $LFS mirror split -d --mirror-id=3 $tf ||
2301 error "mirror split $tfile failed"
2304 block3=$(( $(stat -c "%b*%B" $tf) ))
2305 echo " ** after mirror split, file blocks=$((block3/1024)) KiB"
2307 [[ $block1 -eq $block3 ]] ||
2308 error "mirror split does not reduce block# $block3 != $block1"
2310 run_test 44c "lfs mirror split reduces block size of a file"
2313 local tf=$DIR/$tdir/$tfile
2319 stack_trap "rm -f $tf"
2321 mkdir -p $DIR/$tdir || error "create directroy failed"
2323 dd if=/dev/zero of=$tf bs=1M count=10 || error "dd write $tfile failed"
2325 size1=$(stat -c "%s" $tf)
2326 echo " ** before mirror ops, file size=$size1"
2328 $LFS mirror extend -N2 -c1 $tf || error "mirror extend $tfile failed"
2330 size2=$(stat -c "%s" $tf)
2331 echo " ** after mirror extend, file size=$size2"
2333 (($size1 == $size2)) ||
2334 error "mirror extend should not change size, before: $size1, after $size2"
2336 $LFS mirror split -d --mirror-id=2 $tf ||
2337 error "mirror split $tfile failed"
2339 size2=$(stat -c "%s" $tf)
2340 echo " ** after mirror split, file size=$size2"
2341 (($size1 == $size2)) ||
2342 error "mirror split should not change size, before: $size1, after $size2"
2344 # Remount client to clear cached size information
2345 remount_client $MOUNT
2346 size2=$(stat -c "%s" $tf)
2347 echo " ** after mirror split & remount, file size=$size2"
2348 (($size1 == $size2)) ||
2349 error "mirror extend should not change size, before: $size1, after $size2"
2352 run_test 44d "lfs mirror split does not break size"
2355 local tf=$DIR/$tdir/$tfile
2356 local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
2360 test_mkdir $DIR/$tdir
2361 [ $MDS1_VERSION -ge $(version_code 2.14.52) ] ||
2362 skip "Need MDS version at least 2.14.52"
2364 $LFS mirror create -N2 $tf || error "create mirrored file $tf failed"
2366 # Disable xattr caching so we can repeatedly check SOM with lfs getsom
2367 $LCTL set_param llite.*.xattr_cache=0
2368 stack_trap "$LCTL set_param llite.*.xattr_cache=1"
2369 stack_trap "rm -rf $tf"
2371 dd if=/dev/zero of=$tf bs=1M count=10 || error "dd write $tfile failed"
2373 size1=$(stat -c "%s" $tf)
2374 echo " ** before mirror resync, file size=$size1"
2376 $LFS mirror resync $tf || error "mirror resync file $tf failed"
2377 size1=$(stat -c "%s" $tf)
2378 size2=$($LFS getsom -s $tf)
2382 ((size1 == size2)) ||
2383 error "mirrored file with strict SOM $size1 != disabled SOM $size2"
2385 # Remount client to clear cached size information
2386 remount_client $MOUNT
2388 save_lustre_params $(get_facets MDS) mdt.*MDT*.enable_strict_som > $p
2389 stack_trap "restore_lustre_params < $p; rm -f $p"
2390 local mds_facet=mds$(($($LFS getstripe -m $tf) + 1))
2392 do_facet $mds_facet $LCTL set_param mdt.*MDT*.enable_strict_som=0
2394 size2=$(stat -c "%s" $tf)
2395 # 'getsom' here is just for debugging
2398 ((size2 == size1)) ||
2399 error "mirrored file in sync with som disabled, size with som disabled ($size2) and without som disabled ($size1) should agree"
2401 run_test 44e "basic FLR SOM tests + disable SOM"
2404 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
2406 local file=$DIR/$tdir/$tfile
2407 local dir=$DIR/$tdir/$dir
2408 local temp=$DIR/$tdir/template
2410 test_mkdir $DIR/$tdir
2412 $LFS setstripe -N -E1m -S1m -c2 -o0,1 -E2m -Eeof -N -E4m -Eeof \
2413 -N -E3m -S1m -Eeof -N -E8m -Eeof $file ||
2414 error "Create $file failed"
2416 verify_yaml_layout $file $file.copy $temp "1. FLR file"
2417 rm -f $file $file.copy
2419 $LFS setstripe -N -E1m -S1m -c2 -o0,1 -E2m -Eeof -N -E4m -Eeof \
2420 -N -E3m -S1m -Eeof -N -E8m --flags=prefer -Eeof $file ||
2421 error "Create $file failed"
2423 verify_yaml_layout $file $file.copy $temp "2. FLR file with flags"
2425 run_test 45 "Verify setstripe/getstripe with YAML with FLR file"
2432 $LFS setstripe --copy=$src $dst || error "setstripe $dst failed"
2434 local layout1=$(get_layout_param $src)
2435 local layout2=$(get_layout_param $dst)
2436 # compare their layout info
2437 [ "$layout1" == "$layout2" ] ||
2438 error "$msg_prefix $src <=> $dst layouts are not equal"
2442 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" && return
2444 local file=$DIR/$tdir/$tfile
2445 test_mkdir $DIR/$tdir
2447 ########################### 1. PFL file #############################
2448 echo " ** 1. PFL file"
2450 $LFS setstripe -E1m -S 1M -c2 -o0,1 -E2m -c2 -E3m -o1,0 -E4m -c1 -E-1 \
2451 $file || error "1. Create PFL $file failed"
2454 verify_46 $file $file.copy "1. PFL file"
2456 ########################### 2. plain file ###########################
2457 echo " ** 2. plain file"
2459 $LFS setstripe -c2 -o0,1 -i1 $file ||
2460 error "2. Create plain $file failed"
2463 verify_46 $file $file.copy "2. plain file"
2465 ########################### 3. FLR file #############################
2466 echo " ** 3. FLR file"
2468 $LFS setstripe -N -E1m -S 1M -c2 -o0,1 -E4m -c1 -Eeof -N -E16m -Eeof \
2469 $file || error "3. Create FLR $file failed"
2472 verify_46 $file $file.copy "3. FLR file"
2474 local dir=$DIR/$tdir/dir
2475 ########################### 4. PFL dir ##############################
2476 echo " ** 4. PFL dir"
2478 $LFS setstripe -E1m -S 1M -c2 -E2m -c1 -E-1 $dir ||
2479 error "4. setstripe PFL $dir failed"
2481 test_mkdir $dir.copy
2482 verify_46 $dir $dir.copy "4. PFL dir"
2484 ########################### 5. plain dir ############################
2485 echo " ** 5. plain dir"
2486 $LFS setstripe -c2 -i-1 $dir || error "5. setstripe plain $dir failed"
2488 verify_46 $dir $dir.copy "5. plain dir"
2490 ########################### 6. FLR dir ##############################
2491 echo " ** 6. FLR dir"
2492 $LFS setstripe -N -E1m -S 1M -c2 -E2m -c1 -Eeof -N -E4m -Eeof $dir ||
2493 error "6. setstripe FLR $dir failed"
2495 verify_46 $dir $dir.copy "6. FLR dir"
2497 (( $MDS1_VERSION >= $(version_code 2.13.53.205) )) ||
2498 echo "server version $MDS1_VERSION does not support SEL" &&
2501 ########################### 7. SEL file ##############################
2502 echo " ** 7. SEL file"
2504 $LFS setstripe -E256M -S 1M -c2 -o0,1 -z 64M -E-1 -o1,0 -z 128M \
2505 $file || error "Create $file failed"
2508 verify_46 $file $file.copy "7. SEL file"
2510 ########################### 8. SEL dir ##############################
2511 echo " ** 8. SEL dir"
2512 $LFS setstripe -E256M -S 1M -c2 -z 64M -E-1 -z 128M \
2513 $dir || error "setstripe $dir failed"
2515 verify_46 $dir $dir.copy "8. SEL dir"
2517 ########################### 9. FLR SEL file ##########################
2518 echo " ** 9. FLR SEL file"
2520 $LFS setstripe -N -E256M -c2 -z 64M -E-1 -z 128M \
2521 -N -E1G -c4 -z128M -E-1 -z256M $file || error "Create $file failed"
2524 verify_46 $file $file.copy "9. SEL file"
2526 ########################### 10. FLR SEL dir #########################
2527 echo " ** 10. FLR SEL dir"
2528 $LFS setstripe -N -E256M -c2 -z 64M -E-1 -z 128M \
2529 -N -E1G -c4 -z128M -E-1 -z256M $dir || error "Create $file failed"
2531 verify_46 $dir $dir.copy "10. SEL dir"
2533 run_test 46 "Verify setstripe --copy option"
2536 [ $OSTCOUNT -lt 3 ] && skip "needs >= 3 OSTs" && return
2538 local file=$DIR/$tdir/$tfile
2543 test_mkdir $DIR/$tdir
2544 stack_trap "rm -f $file"
2548 # mirror1: [comp0]ost0, [comp1]ost1 and ost2
2549 # mirror2: [comp2] , [comp3] should not use ost1 or ost2
2550 $LFS mirror create -N -E2m -c1 -o0 --flags=prefer -Eeof -c2 -o1,2 \
2551 -N -E2m -c1 -Eeof -c1 $file || error "create FLR $file failed"
2552 ids=($($LFS getstripe $file | awk '/lcme_id/{print $2}' | tr '\n' ' '))
2554 dd if=/dev/zero of=$file bs=1M count=3 || error "dd $file failed"
2555 $LFS mirror resync $file || error "resync $file failed"
2557 ost=$($LFS getstripe -I${ids[2]} $file | awk '/l_ost_idx/{print $5}')
2558 if [[ x$ost == "x0," ]]; then
2559 $LFS getstripe $file
2560 error "component ${ids[2]} objects allocated on $ost " \
2564 ost=$($LFS getstripe -I${ids[3]} $file | awk '/l_ost_idx/{print $5}')
2565 if [[ x$ost == "x1," || x$ost == "x2," ]]; then
2566 $LFS getstripe $file
2567 error "component ${ids[3]} objects allocated on $ost " \
2568 "shouldn't on OST1 or on OST2"
2573 # mirror1: [comp0] [comp1]
2574 # mirror2: [comp2] [comp3]
2575 # mirror3: [comp4] [comp5]
2576 # mirror4: [comp6] [comp7]
2577 $LFS mirror create -N4 -E1m -c1 -Eeof -c1 $file ||
2578 error "create FLR $file failed"
2579 ids=($($LFS getstripe $file | awk '/lcme_id/{print $2}' | tr '\n' ' '))
2581 dd if=/dev/zero of=$file bs=1M count=3 || error "dd $file failed"
2582 $LFS mirror resync $file || error "resync $file failed"
2584 for ((i = 0; i < 6; i++)); do
2585 osts[$i]=$($LFS getstripe -I${ids[$i]} $file |
2586 awk '/l_ost_idx/{print $5}')
2588 # comp[0],comp[2],comp[4] should use different osts
2589 if [[ ${osts[0]} == ${osts[2]} || ${osts[0]} == ${osts[4]} ||
2590 ${osts[2]} == ${osts[4]} ]]; then
2591 $LFS getstripe $file
2592 error "component ${ids[0]}, ${ids[2]}, ${ids[4]} have objects "\
2593 "allocated on duplicated OSTs"
2595 # comp[1],comp[3],comp[5] should use different osts
2596 if [[ ${osts[1]} == ${osts[3]} || ${osts[1]} == ${osts[5]} ||
2597 ${osts[3]} == ${osts[5]} ]]; then
2598 $LFS getstripe $file
2599 error "component ${ids[1]}, ${ids[3]}, ${ids[5]} have objects "\
2600 "allocated on duplicated OSTs"
2605 run_test 47 "Verify mirror obj alloc"
2608 [ $MDS1_VERSION -lt $(version_code 2.11.55) ] &&
2609 skip "Need MDS version at least 2.11.55"
2611 local tf=$DIR/$tfile
2613 stack_trap "rm -f $tf"
2615 echo " ** create 2 mirrors FLR file $tf"
2616 $LFS mirror create -N -E2M -Eeof --flags prefer \
2617 -N -E1M -Eeof $tf ||
2618 error "create FLR file $tf failed"
2621 dd if=/dev/urandom of=$tf bs=1M count=3 || error "write $tf failed"
2622 verify_flr_state $tf "wp"
2624 local sum0=$(md5sum < $tf)
2626 echo " ** resync the file"
2627 $LFS mirror resync $tf
2629 echo " ** snapshot mirror 2"
2630 $LFS setstripe --comp-set -I 0x20003 --comp-flags=nosync $tf
2632 echo " ** write it again"
2633 dd if=/dev/urandom of=$tf bs=1M count=3 || error "write $tf failed"
2634 echo " ** resync it again"
2635 $LFS mirror resync $tf
2637 verify_flr_state $tf "wp"
2638 verify_comp_attr lcme_flags $tf 0x20003 nosync,stale
2640 local sum1=$($LFS mirror read -N1 $tf | md5sum)
2641 local sum2=$($LFS mirror read -N2 $tf | md5sum)
2643 echo " ** verify mirror 2 doesn't change"
2644 echo "original checksum: $sum0"
2645 echo "mirror 1 checksum: $sum1"
2646 echo "mirror 2 checksum: $sum2"
2647 [[ $sum0 = $sum2 ]] ||
2648 error "original checksum: $sum0, mirror 2 checksum: $sum2"
2649 echo " ** mirror 2 stripe info"
2650 $LFS getstripe -v --mirror-index=2 $tf
2652 echo " ** resync mirror 2"
2653 $LFS mirror resync --only 2 $tf
2655 verify_flr_state $tf "ro"
2656 verify_comp_attr lcme_flags $tf 0x20003 nosync,^stale
2658 sum1=$($LFS mirror read -N1 $tf | md5sum)
2659 sum2=$($LFS mirror read -N2 $tf | md5sum)
2661 echo " ** verify mirror 2 resync-ed"
2662 echo "original checksum: $sum0"
2663 echo "mirror 1 checksum: $sum1"
2664 echo "mirror 2 checksum: $sum2"
2665 [[ $sum1 = $sum2 ]] ||
2666 error "mirror 1 checksum: $sum1, mirror 2 checksum: $sum2"
2667 echo " ** mirror 2 stripe info"
2668 $LFS getstripe -v --mirror-index=2 $tf
2670 run_test 48 "Verify snapshot mirror"
2679 (( "$OSTCOUNT" >= "2" )) || skip "needs >= 2 OSTs"
2680 local filefrag_op=$(filefrag -l 2>&1 | grep "invalid option")
2681 [[ -z "$filefrag_op" ]] || skip_env "filefrag missing logical ordering"
2682 [[ "$ost1_FSTYPE" != "zfs" ]] ||
2683 skip "LU-1941: FIEMAP unimplemented on ZFS"
2685 trap cleanup_49 EXIT RETURN
2687 local file=$DIR/$tfile
2689 $LFS setstripe -N -E eof -c1 -o1 -N -E eof -c1 -o0 $file ||
2690 error "setstripe on $file"
2691 stack_trap "rm -f $file"
2693 dd if=/dev/zero of=$file bs=1M count=1 || error "dd failed for $file"
2694 $LFS mirror resync $file
2696 filefrag -ves $file || error "filefrag $file failed"
2697 filefrag_op=$(filefrag -ve -k $file |
2698 sed -n '/ext:/,/found/{/ext:/d; /found/d; p}')
2700 #Filesystem type is: bd00bd0
2701 #File size of /mnt/lustre/f49a.sanity-flr is 1048576 (1024 blocks of 1024 bytes)
2702 # ext: device_logical: physical_offset: length: dev: flags:
2703 # 0: 0.. 1023: 1572864.. 1573887: 1024: 0001: net,eof
2704 # 1: 0.. 1023: 1572864.. 1573887: 1024: 0000: last,net,eof
2705 #/mnt/lustre/f49a.sanity-flr: 2 extents found
2707 last_lun=$(echo $filefrag_op | cut -d: -f5)
2711 for line in $filefrag_op; do
2712 frag_lun=$(echo $line | cut -d: -f5)
2713 ext_len=$(echo $line | cut -d: -f4)
2714 if [[ "$frag_lun" != "$last_lun" ]]; then
2715 if (( tot_len != 1024 )); then
2717 error "$file: OST$last_lun $tot_len != 1024"
2723 (( tot_len += ext_len ))
2726 if (( num_luns != 2 || tot_len != 1024 )); then
2728 error "$file: $num_luns != 2, $tot_len != 1024 on OST$last_lun"
2731 echo "FIEMAP on $file succeeded"
2733 run_test 49a "FIEMAP upon FLR file"
2735 test_50A() { # EX-2179
2738 local file=$DIR/$tdir/$tfile
2740 $LFS setstripe -c1 -i0 $file || error "setstripe $file failed"
2742 $LFS mirror extend -N -c1 -i1 $file ||
2743 error "extending mirror for $file failed"
2745 local olv=$($LFS getstripe $file | awk '/lcm_layout_gen/{print $2}')
2749 $LFS mirror split -d --mirror-id=1 $file || error "split $file failed"
2751 local flv=$($LFS getstripe $file | awk '/lcm_layout_gen/{print $2}')
2753 echo "$file layout generation from $olv to $flv"
2754 (( $flv != ($olv + 1) )) &&
2755 error "split does not increase layout gen from $olv to $flv"
2757 dd if=/dev/zero of=$file bs=1M count=1 || error "write $file failed"
2759 $LFS getstripe -v $file || error "getstripe $file failed"
2761 run_test 50A "mirror split update layout generation"
2764 $LCTL get_param osc.*.import | grep -q 'connect_flags:.*seek' ||
2765 skip "OST does not support SEEK_HOLE"
2766 [ "$FSTYPE" != "zfs" ] ||
2767 skip "lseek for ZFS is not accurate if obj is not committed"
2769 local file=$DIR/$tdir/$tfile
2777 echo " ** create striped file $file"
2778 $LFS setstripe -E 1M -c1 -S 1M -E eof -c2 -S1M $file ||
2779 error "cannot create file with PFL layout"
2780 echo " ** write 1st data chunk at 1M boundary"
2781 dd if=/dev/urandom of=$file bs=1k count=20 seek=1021 ||
2782 error "cannot write data at 1M boundary"
2783 echo " ** write 2nd data chunk at 2M boundary"
2784 dd if=/dev/urandom of=$file bs=1k count=20 seek=2041 ||
2785 error "cannot write data at 2M boundary"
2786 echo " ** create hole at the file end"
2787 $TRUNCATE $file 3700000 || error "truncate fails"
2789 echo " ** verify sparseness"
2790 offset=$(lseek_test -d 1000 $file)
2791 echo " first data offset: $offset"
2792 [[ $offset == 1000 ]] &&
2793 error "src: data is not expected at offset $offset"
2794 offset=$(lseek_test -l 3500000 $file)
2795 echo " hole at the end: $offset"
2796 [[ $offset == 3500000 ]] ||
2797 error "src: hole is expected at offset $offset"
2799 echo " ** extend the file with new mirror"
2800 # migrate_copy_data() is used
2801 $LFS mirror extend -N -E 2M -S 1M -E 1G -S 2M -E eof $file ||
2802 error "cannot create mirror"
2803 $LFS getstripe $file | grep lcme_flags | grep stale > /dev/null &&
2804 error "$file still has stale component"
2806 # check migrate_data_copy() was correct
2807 sum_1=$($LFS mirror read -N 1 $file | md5sum)
2808 sum_2=$($LFS mirror read -N 2 $file | md5sum)
2809 [[ $sum_1 == $sum_2 ]] ||
2810 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
2812 # stale first mirror
2813 $LFS setstripe --comp-set -I0x10001 --comp-flags=stale $file
2814 $LFS setstripe --comp-set -I0x10002 --comp-flags=stale $file
2816 echo " ** verify mirror #2 sparseness"
2817 offset=$(lseek_test -d 1000 $file)
2818 echo " first data offset: $offset"
2819 [[ $offset == 1000 ]] &&
2820 error "dst: data is not expected at offset $offset"
2821 offset=$(lseek_test -l 3500000 $file)
2822 echo " hole at the end: $offset"
2823 [[ $offset == 3500000 ]] ||
2824 error "dst: hole is expected at offset $offset"
2826 echo " ** copy mirror #2 to mirror #1"
2827 $LFS mirror copy -i 2 -o 1 $file || error "mirror copy fails"
2828 $LFS getstripe $file | grep lcme_flags | grep stale > /dev/null &&
2829 error "$file still has stale component"
2831 # check llapi_mirror_copy_many correctness
2832 sum_1=$($LFS mirror read -N 1 $file | md5sum)
2833 sum_2=$($LFS mirror read -N 2 $file | md5sum)
2834 [[ $sum_1 == $sum_2 ]] ||
2835 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
2837 # stale 1st component of mirror #2 before lseek call
2838 $LFS setstripe --comp-set -I0x20001 --comp-flags=stale $file
2840 echo " ** verify mirror #1 sparseness again"
2841 offset=$(lseek_test -d 1000 $file)
2842 echo " first data offset: $offset"
2843 [[ $offset == 1000 ]] &&
2844 error "dst: data is not expected at offset $offset"
2845 offset=$(lseek_test -l 3500000 $file)
2846 echo " hole at the end: $offset"
2847 [[ $offset == 3500000 ]] ||
2848 error "dst: hole is expected at offset $offset"
2850 cancel_lru_locks osc
2852 blocks=$(stat -c%b $file)
2853 echo " ** final consumed blocks: $blocks"
2854 # for 3.5Mb file consumes ~6000 blocks, use 1000 to check
2855 # that file is still sparse
2856 (( blocks < 1000 )) ||
2857 error "Mirrored file consumes $blocks blocks"
2859 run_test 50a "mirror extend/copy preserves sparseness"
2862 $LCTL get_param osc.*.import | grep -q 'connect_flags:.*seek' ||
2863 skip "OST does not support SEEK_HOLE"
2864 [ "$FSTYPE" != "zfs" ] ||
2865 skip "lseek for ZFS is not accurate if obj is not committed"
2867 local file=$DIR/$tdir/$tfile
2874 stack_trap "rm -f $file"
2876 echo " ** create mirrored file $file"
2877 $LFS mirror create -N -E1M -c1 -S1M -E eof \
2878 -N -E2M -S1M -E eof -S2M $file ||
2879 error "cannot create mirrored file"
2880 echo " ** write data chunk at 1M boundary"
2881 dd if=/dev/urandom of=$file bs=1k count=20 seek=1021 ||
2882 error "cannot write data at 1M boundary"
2883 echo " ** create hole at the file end"
2884 $TRUNCATE $file 3700000 || error "truncate fails"
2886 echo " ** verify sparseness"
2887 offset=$(lseek_test -d 1000 $file)
2888 echo " first data offset: $offset"
2889 [[ $offset == 1000 ]] &&
2890 error "src: data is not expected at offset $offset"
2891 offset=$(lseek_test -l 3500000 $file)
2892 echo " hole at the end: $offset"
2893 [[ $offset == 3500000 ]] ||
2894 error "src: hole is expected at 3500000"
2896 echo " ** resync mirror #2 to mirror #1"
2897 $LFS mirror resync $file
2899 # check llapi_mirror_copy_many correctness
2900 sum_1=$($LFS mirror read -N 1 $file | md5sum)
2901 sum_2=$($LFS mirror read -N 2 $file | md5sum)
2902 [[ $sum_1 == $sum_2 ]] ||
2903 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
2905 cancel_lru_locks osc
2907 blocks=$(stat -c%b $file)
2908 echo " ** consumed blocks: $blocks"
2909 # without full punch() support the first component can be not sparse
2910 # but the last one should be, so file should use far fewer blocks
2911 (( blocks < 5000 )) ||
2912 error "Mirrored file consumes $blocks blocks"
2914 # stale first component in mirror #1
2915 $LFS setstripe --comp-set -I0x10001 --comp-flags=stale,nosync $file
2916 echo " ** truncate file down"
2918 echo " ** write data chunk at 2M boundary"
2919 dd if=/dev/urandom of=$file bs=1k count=20 seek=2041 conv=notrunc ||
2920 error "cannot write data at 2M boundary"
2921 echo " ** resync mirror #2 to mirror #1 with nosync 1st component"
2922 $LFS mirror resync $file || error "mirror rsync fails"
2923 # first component is still stale
2924 $LFS getstripe $file | grep 'lcme_flags:.*stale' > /dev/null ||
2925 error "$file still has no stale component"
2926 echo " ** resync mirror #2 to mirror #1 again"
2927 $LFS setstripe --comp-set -I0x10001 --comp-flags=stale,^nosync $file
2928 $LFS mirror resync $file || error "mirror rsync fails"
2929 $LFS getstripe $file | grep 'lcme_flags:.*stale' > /dev/null &&
2930 error "$file still has stale component"
2932 # check llapi_mirror_copy_many correctness
2933 sum_1=$($LFS mirror read -N 1 $file | md5sum)
2934 sum_2=$($LFS mirror read -N 2 $file | md5sum)
2935 [[ $sum_1 == $sum_2 ]] ||
2936 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
2938 cancel_lru_locks osc
2940 blocks=$(stat -c%b $file)
2941 echo " ** final consumed blocks: $blocks"
2942 # while the first component can lose sparseness, the last one should
2943 # not, so whole file should still use far fewer blocks in total
2944 (( blocks < 3000 )) ||
2945 error "Mirrored file consumes $blocks blocks"
2947 run_test 50b "mirror rsync handles sparseness"
2950 local tf=$DIR/$tdir/$tfile
2952 test_mkdir $DIR/$tdir
2954 $LFS setstripe -N2 -c-1 $tf || error "create FLR $tf failed"
2955 verify_flr_state $tf "ro"
2957 if [[ "$FSTYPE" == "ldiskfs" ]]; then
2958 # ZFS does not support fallocate for now
2959 out=$(fallocate -p -o 1MiB -l 1MiB $tf 2>&1) ||
2960 skip_eopnotsupp "$out|punch hole in $tf failed"
2961 verify_flr_state $tf "wp"
2964 dd if=/dev/zero of=$tf bs=4096 count=4 || error "write $tf failed"
2965 $LFS mirror resync $tf || error "mirror resync $tf failed"
2966 verify_flr_state $tf "ro"
2968 $MULTIOP $tf OSMWUc || error "$MULTIOP $tf failed"
2969 verify_flr_state $tf "wp"
2971 run_test 50c "punch_hole/mmap_write stale other mirrors"
2974 $LCTL get_param osc.*.import | grep -q 'connect_flags:.*seek' ||
2975 skip "OST does not support SEEK_HOLE"
2976 (( $LINUX_VERSION_CODE > $(version_code 3.0.0) )) ||
2977 skip "client kernel does not support SEEK_HOLE"
2978 [ "$FSTYPE" != "zfs" ] ||
2979 skip "lseek for ZFS is not accurate if obj is not committed"
2981 local file=$DIR/$tdir/$tfile
2988 echo " ** create mirrored file $file"
2989 $LFS mirror create -N -E1M -c1 -S1M -E eof \
2990 -N -E2M -S1M -E eof -S2M $file ||
2991 error "cannot create mirrored file"
2992 echo " ** write data chunk at 1M boundary"
2993 dd if=/dev/urandom of=$file bs=1k count=20 seek=1021 ||
2994 error "cannot write data at 1M boundary"
2995 echo " ** create hole at the file start"
2996 prt=$(fallocate -p -o 0 -l 1M $file 2>&1)
2999 if [[ $rc -eq 0 ]]; then
3000 verify_flr_state $file "wp"
3001 elif [[ ! $prt =~ unsupported ]]; then
3002 error "punch hole in $file failed: $prt"
3004 skip "Fallocate punch is not supported: $prt"
3007 echo " ** verify sparseness"
3008 offset=$(lseek_test -d 1000 $file)
3009 echo " first data offset: $offset"
3010 (( $offset >= 1024 * 1024 )) ||
3011 error "src: data is not expected at offset $offset"
3013 echo " ** resync mirror #2"
3014 $LFS mirror resync $file
3016 # check llapi_mirror_copy_many correctness
3017 sum_1=$($LFS mirror read -N 1 $file | md5sum)
3018 sum_2=$($LFS mirror read -N 2 $file | md5sum)
3019 [[ $sum_1 == $sum_2 ]] ||
3020 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
3022 cancel_lru_locks osc
3024 # stale first component in mirror #1
3025 $LFS setstripe --comp-set -I0x10001 --comp-flags=stale,nosync $file
3026 echo " ** verify sparseness of mirror #2"
3027 offset=$(lseek_test -d 1000 $file)
3028 echo " first data offset: $offset"
3029 (( $offset >= 1024 * 1024 )) ||
3030 error "src: data is not expected at offset $offset"
3032 run_test 50d "mirror rsync keep holes"
3035 $LCTL get_param osc.*.import | grep -q 'connect_flags:.*seek' ||
3036 skip "OST does not support SEEK_HOLE"
3038 local file=$DIR/$tdir/$tfile
3039 local old_size=2147483648 # 2GiB
3043 dd if=/dev/urandom of=$file bs=4096 count=1 seek=$((134217728 / 4096))
3044 $TRUNCATE $file $old_size
3046 $LFS mirror extend -N -c 1 $file
3047 dd if=/dev/urandom of=$file bs=4096 count=1 seek=$((134217728 / 4096)) conv=notrunc
3048 $LFS mirror resync $file
3050 new_size=$(stat --format='%s' $file)
3051 if ((new_size != old_size)); then
3052 error "new_size ($new_size) is not equal to old_size ($old_size)"
3055 run_test 60a "mirror extend sets correct size on sparse file"
3057 get_flr_layout_gen() {
3058 getfattr -n lustre.lov --only-values $tf 2>/dev/null |
3059 od -tx4 | awk '/000000/ { print "0x"$4; exit; }'
3062 check_layout_gen() {
3065 local v1=$(get_flr_layout_gen $tf)
3066 local v2=$($LFS getstripe -v $tf | awk '/lcm_layout_gen/ { print $2 }')
3068 [[ $v1 -eq $v2 ]] ||
3069 error "$tf in-memory layout gen $v1 != $v2 after $2"
3073 local tf=$DIR/$tdir/$tfile
3075 test_mkdir $DIR/$tdir
3077 $LFS setstripe -Eeof $tf || error "setstripe $tf failed"
3079 for ((i = 0; i < 20; i++)); do
3080 $LFS mirror extend -N $tf ||
3081 error "extending mirror for $tf failed"
3082 check_layout_gen $tf "extend"
3084 $LFS mirror split -d --mirror-id=$((i+1)) $tf ||
3085 error "split $tf failed"
3086 check_layout_gen $tf "split"
3089 run_test 60b "mirror merge/split cancel client's in-memory layout gen"
3092 stat --format='%X %Y %Z' $file || error "$file: cannot get times"
3097 local -a old=( $2 $3 $4 )
3100 new=( $(get_times_61 $file) )
3101 ((${old[0]} == ${new[0]})) ||
3102 error "$file: atime: old '${old[0]}' != new '${new[0]}'"
3104 ((${old[1]} == ${new[1]})) ||
3105 error "$file: mtime: old '${old[1]}' != new '${new[1]}'"
3108 test_61a() { # LU-14508
3109 local file=$DIR/$tdir/$tfile
3115 $LFS setstripe -E1M -Eeof $file || error "setstripe $file failed"
3116 echo "create $file-2"
3117 $LFS setstripe -E2M -Eeof $file-2 || error "setstripe $file-2 failed"
3119 echo XXX > $file || error "write $file failed"
3120 chown $RUNAS_ID $DIR/$tdir $file || error "chown $file failed"
3122 echo "sleep $nap seconds, then cat $tfile"
3124 cat $file || error "cat $file failed"
3126 echo "sleep $nap seconds, then re-write $tfile"
3128 echo XXXX > $file || error "write $file failed"
3129 cp -p $file $file-2 || error "copy $file-2 failed"
3131 echo "sleep $nap seconds"
3134 tim=( $(get_times_61 $file) )
3136 echo "mirror merge $tfile-2 to $tfile and test timestamps"
3137 $LFS mirror extend -N -f $file-2 $file ||
3138 error "cannot mirror merge $file-2 to $file"
3139 check_times_61 $file "${tim[@]}"
3141 echo "mirror extend $tfile and test timestamps"
3142 $LFS mirror extend -N -c1 -i1 $file ||
3143 error "cannot extend mirror $file"
3144 check_times_61 $file "${tim[@]}"
3146 echo "migrate $tfile and test timestamps"
3147 $LFS migrate -n $file || error "cannot migrate $file"
3148 check_times_61 $file "${tim[@]}"
3150 echo "normal user migrate $tfile and test timestamps"
3151 $RUNAS $LFS migrate -n $file || error "cannot migrate $file"
3152 check_times_61 $file "${tim[@]}"
3154 run_test 61a "mirror extend and migrate preserve timestamps"
3156 test_61b() { # LU-14508
3157 local file=$DIR/$tdir/$tfile
3163 echo XXX > $file || error "create $file failed"
3164 chown $RUNAS_ID $DIR/$tdir $file || error "chown $file failed"
3166 echo "sleep $nap seconds, then cat $tfile"
3168 cat $file || error "cat $file failed"
3170 echo "sleep $nap seconds, then re-write $tfile"
3172 echo XXXX > $file || error "write $file failed"
3174 echo "sleep $nap seconds, then test timestamps"
3177 tim=( $(get_times_61 $file) )
3179 echo "mirror extend $tfile and test timestamps"
3180 $LFS mirror extend -N -c1 -i1 $file ||
3181 error "cannot extend mirror $file"
3182 check_times_61 $file "${tim[@]}"
3184 echo "mirror split $tfile and test timestamps"
3185 $LFS mirror split -d --mirror-id=1 $file ||
3186 error "cannot split mirror 1 off $file"
3187 check_times_61 $file "${tim[@]}"
3189 echo "normal user mirror extend $tfile and test timestamps"
3190 $RUNAS $LFS mirror extend -N -c1 -i1 $file ||
3191 error "cannot extend mirror $file"
3192 check_times_61 $file "${tim[@]}"
3194 run_test 61b "mirror extend and split preserve timestamps"
3196 test_61c() { # LU-14508
3197 local file=$DIR/$tdir/$tfile
3203 echo XXX > $file || error "create $file failed"
3204 chown $RUNAS_ID $DIR/$tdir $file || error "chown $file failed"
3206 echo "sleep $nap seconds, then cat $tfile"
3208 cat $file || error "cat $file failed"
3210 echo "sleep $nap seconds, then mirror extend $tfile and write it"
3212 $LFS mirror extend -N -c1 -i1 $file ||
3213 error "cannot extend mirror $file"
3214 echo XXXX > $file || error "write $file failed"
3216 echo "sleep $nap seconds, then resync $tfile and test timestamps"
3217 tim=( $(get_times_61 $file) )
3219 $LFS mirror resync $file || error "cannot resync mirror $file"
3220 check_times_61 $file "${tim[@]}"
3222 echo XXXXXX > $file || error "write $tfile failed"
3224 echo "normal user resync $tfile and test timestamps"
3225 tim=( $(get_times_61 $file) )
3226 $RUNAS $LFS mirror resync $file || error "cannot resync mirror $file"
3227 check_times_61 $file "${tim[@]}"
3229 run_test 61c "mirror resync preserves timestamps"
3232 local tf=$DIR/$tdir/$tfile
3234 test_mkdir $DIR/$tdir
3235 stack_trap "rm -f $tf"
3239 $LFS mirror create -N -E 1M -c -1 -E eof -N $tf
3243 echo "mirror create pid $c_pid"
3246 $LFS mirror split -d --mirror-id=1 $tf &> /dev/null
3249 echo "mirror split pid $s_pid"
3251 echo "mirror create and split race for 60 seconds, should not crash"
3253 kill -9 $c_pid &> /dev/null
3254 kill -9 $s_pid &> /dev/null
3258 run_test 70 "mirror create and split race"
3261 local tf=$DIR/$tdir/$tfile
3263 (( $OST1_VERSION >= $(version_code 2.14.51) )) ||
3264 skip "Need OST version at least 2.14.51"
3267 test_mkdir $DIR/$tdir
3268 stack_trap "rm -f $tf"
3270 $LFS setstripe -N -E1M -c-1 -Eeof -c-1 $tf ||
3271 error "setstripe $tf failed"
3273 FSXNUM=${FSXNUM:-1000}
3274 $FSX -p 5 -N $FSXNUM -S 0 -M $tf || error "fsx FLR file $tf failed"
3276 run_test 100 "flr mode fsx test"
3281 local fsize=$(stat --printf=%s $tf)
3283 while [ -f $ctrl_file ]; do
3284 local off=$((RANDOM << 8))
3285 local len=$((RANDOM << 5 + 131072))
3287 [ $((off + len)) -gt $fsize ] && {
3288 fsize=$((off + len))
3289 echo "Extending file size to $fsize .."
3292 flock -s $lock_file -c \
3293 "$MULTIOP $tf oO_WRONLY:z${off}w${len}c" ||
3295 error "failed writing to $off:$len"; }
3296 sleep 0.$((RANDOM % 2 + 1))
3303 while [ -f $ctrl_file ]; do
3304 flock -s $lock_file -c "cat $tf &> /dev/null" ||
3305 { rm -f $ctrl_file; error "read failed"; }
3306 sleep 0.$((RANDOM % 2 + 1))
3313 options=("" "-e resync_start" "-e delay_before_copy -d 1" "" "")
3315 exec 200<>$lock_file
3316 while [ -f $ctrl_file ]; do
3317 local lock_taken=false
3318 local index=$((RANDOM % ${#options[@]}))
3319 local cmd="mirror_io resync ${options[$index]}"
3321 [ "${options[$index]}" = "" ] && cmd="$LFS mirror resync"
3323 [ $((RANDOM % 4)) -eq 0 ] && {
3329 echo -n "resync file $tf with '$cmd' .."
3331 if [[ $lock_taken = "true" ]]; then
3333 $cmd $tf &> /dev/null && echo "done" || echo "failed"
3336 $cmd $tf &> /dev/null && echo "done" || echo "failed"
3339 sleep 0.$((RANDOM % 8 + 1))
3343 # this was test_200 before adding "b" and "c" subtests
3345 local tf=$DIR/$tfile
3346 local tf2=$DIR2/$tfile
3347 local tf3=$DIR3/$tfile
3349 ctrl_file=$(mktemp /tmp/CTRL.XXXXXX)
3350 lock_file=$(mktemp /var/lock/FLR.XXXXXX)
3351 stack_trap "rm -f $ctrl_file $lock_file $tf $tf-2 $tf-3"
3353 $LFS setstripe -E 1M -S 1M -E 2M -c 2 -E 4M -E 16M -E eof $tf
3354 $LFS setstripe -E 2M -S 1M -E 6M -c 2 -E 8M -E 32M -E eof $tf-2
3355 $LFS setstripe -E 4M -c 2 -E 8M -E 64M -E eof $tf-3
3357 $LFS mirror extend -N -f $tf-2 $tf ||
3358 error "merging $tf-2 into $tf failed"
3359 $LFS mirror extend -N -f $tf-3 $tf ||
3360 error "merging $tf-3 into $tf failed"
3362 mkdir -p $MOUNT2 && mount_client $MOUNT2
3364 mkdir -p $MOUNT3 && mount_client $MOUNT3
3366 verify_flr_state $tf3 "ro"
3368 #define OBD_FAIL_FLR_RANDOM_PICK_MIRROR 0x1A03
3369 $LCTL set_param fail_loc=0x1A03
3371 local mds_idx=mds$(($($LFS getstripe -m $tf) + 1))
3372 do_facet $mds_idx $LCTL set_param fail_loc=0x1A03
3376 write_file_200 $tf &
3382 write_file_200 $tf2 &
3385 read_file_200 $tf2 &
3388 resync_file_200 $tf3 &
3392 [ "$SLOW" = "yes" ] && sleep_time=360
3393 while [ $sleep_time -gt 0 -a -f $ctrl_file ]; do
3400 echo "Waiting ${pids[*]}"
3403 umount_client $MOUNT2
3404 umount_client $MOUNT3
3406 # resync and verify mirrors
3407 $LFS mirror resync $tf || error "final resync failed"
3410 local csum=$($LFS mirror read -N "${mirror_array[0]}" $tf | md5sum)
3412 for id in "${mirror_array[@]:1}"; do
3413 [ "$($LFS mirror read -N $id $tf | md5sum)" = "$csum" ] ||
3414 error "checksum error for mirror $id"
3419 run_test 200a "stress test"
3422 local tf=$DIR/$tfile
3423 local tf2=$DIR2/$tfile
3424 local tf3=$DIR3/$tfile
3426 ctrl_file=$(mktemp /tmp/CTRL.XXXXXX)
3427 lock_file=$(mktemp /var/lock/FLR.XXXXXX)
3428 stack_trap "rm -f $ctrl_file $lock_file $tf $tf-2 $tf-3"
3430 $LFS setstripe -E 1M -S 1M -E 2M -c 2 -E 4M -E 16M -E eof $tf
3431 $LFS setstripe -E 2M -S 1M -E 6M -c 2 -E 8M -E 32M -E eof $tf-2
3432 $LFS setstripe -E 4M -c 2 -E 8M -E 64M -E eof $tf-3
3434 $LFS mirror extend -N -f $tf-2 $tf ||
3435 error "merging $tf-2 into $tf failed"
3436 $LFS mirror extend -N -f $tf-3 $tf ||
3437 error "merging $tf-3 into $tf failed"
3439 mkdir -p $MOUNT2 && mount_client $MOUNT2
3441 mkdir -p $MOUNT3 && mount_client $MOUNT3
3443 verify_flr_state $tf3 "ro"
3445 #define OBD_FAIL_LLITE_PANIC_ON_ESTALE 0x1423
3446 $LCTL set_param fail_loc=0x1423
3450 write_file_200 $tf &
3456 write_file_200 $tf2 &
3459 read_file_200 $tf2 &
3462 resync_file_200 $tf3 &
3466 [ "$SLOW" = "yes" ] && sleep_time=400
3470 echo "Waiting ${pids[@]}"
3473 umount_client $MOUNT2
3474 umount_client $MOUNT3
3476 # resync and verify mirrors
3477 $LFS mirror resync $tf || {
3479 error "final resync failed"
3483 local csum=$($LFS mirror read -N ${mirror_array[0]} $tf | md5sum)
3484 for id in ${mirror_array[@]:1}; do
3485 [ "$($LFS mirror read -N $id $tf | md5sum)" = "$csum" ] ||
3486 error "checksum error for mirror $id"
3491 run_test 200b "racing IO, mirror extend and resync"
3494 local tf=$DIR/$tfile
3495 local tf2=$DIR2/$tfile
3497 mkdir -p $MOUNT2 && mount_client $MOUNT2
3498 stack_trap "umount_client $MOUNT2"
3499 stack_trap "rm -f $tf"
3503 dd if=/dev/urandom of=$tf bs=1M count=2 || error "can't write"
3505 mdt_idx=$($LFS getstripe -m $tf)
3507 cancel_lru_locks mdc
3508 cancel_lru_locks osc
3510 # start a process modifying file, block it just
3511 # before layout lock acquisition
3512 #define OBD_FAIL_MDS_DELAY_OPEN 0x175
3513 do_facet mds$((mdt_idx+1)) $LCTL set_param fail_loc=0x80000175 fail_val=10
3514 #log "dd to stale replica"
3515 dd if=/dev/urandom of=$tf bs=1M count=2 oflag=direct conv=notrunc &
3521 $LFS mirror extend -N -c -1 $tf2 || {
3523 error "can't mirror"
3525 log "mirror extend done"
3526 do_facet mds$((mdt_idx+1)) $LCTL set_param fail_loc=0 fail_val=0
3528 # wait for blocking dd to complete and modify file
3529 wait $PID || error "2nd dd failed"
3532 verify_mirror_count $tf 2
3534 $LFS getstripe $tf | grep -q lcme_flags.*stale || {
3537 error "both replicas are still in sync"
3540 $LFS mirror verify -vvv $tf || {
3542 error "corrupted in-sync file"
3545 run_test 200c "layout change racing with open: LOVEA changes"
3547 cleanup_test_201() {
3548 do_facet $SINGLEMDS $LCTL --device $MDT0 changelog_deregister $CL_USER
3550 umount_client $MOUNT2
3554 local delay=${RESYNC_DELAY:-5}
3556 MDT0=$($LCTL get_param -n mdc.*.mds_server_uuid |
3557 awk '{ gsub(/_UUID/,""); print $1 }' | head -n1)
3559 stack_trap cleanup_test_201 EXIT
3561 CL_USER=$(do_facet $SINGLEMDS $LCTL --device $MDT0 \
3562 changelog_register -n)
3564 mkdir -p $MOUNT2 && mount_client $MOUNT2
3568 local log=$($LFS changelog $MDT0 $index | grep FLRW)
3569 [ -z "$log" ] && { sleep 1; continue; }
3571 index=$(echo $log | awk '{print $1}')
3572 local ts=$(date -d "$(echo $log | awk '{print $3}')" "+%s" -u)
3573 local fid=$(echo $log | awk '{print $6}' | sed -e 's/t=//')
3574 local file=$($LFS fid2path $MOUNT2 $fid 2> /dev/null)
3577 [ -z "$file" ] && continue
3579 local now=$(date +%s)
3581 echo "file: $file $fid was modified at $ts, now: $now, " \
3582 "will be resynced at $((ts+delay))"
3584 [ $now -lt $((ts + delay)) ] && sleep $((ts + delay - now))
3586 $LFS mirror resync $file
3587 echo "$file resync done"
3590 run_test 201 "FLR data mover"
3593 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
3595 local tf=$DIR/$tfile
3598 $LFS setstripe -E 1M -S 1M -c 1 $tf
3599 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
3600 verify_comp_attr stripe-count $tf ${ids[0]} 1
3602 $LFS setstripe --component-add -E 2M -c -1 $tf
3603 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
3604 verify_comp_attr stripe-count $tf ${ids[0]} 1
3605 verify_comp_attr stripe-count $tf ${ids[1]} -1
3607 dd if=/dev/zero of=$tf bs=1M count=2
3608 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
3609 verify_comp_attr stripe-count $tf ${ids[0]} 1
3610 verify_comp_attr stripe-count $tf ${ids[1]} $OSTCOUNT
3612 run_test 202 "lfs setstripe --add-component wide striping"
3615 [ $MDS1_VERSION -lt $(version_code 2.11.55) ] &&
3616 skip "Need MDS version at least 2.11.55"
3617 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs"
3619 local tf=$DIR/$tfile
3622 $LFS mirror create -N2 -c1 $tf || error "create FLR file $tf"
3623 #delete first mirror
3624 $LFS mirror delete --mirror-id=1 $tf || error "delete first mirror"
3627 local old_id=$($LFS getstripe --mirror-id=2 -I $tf)
3628 local count=$($LFS getstripe --mirror-id=2 -c $tf) ||
3629 error "getstripe count of mirror 2"
3630 [[ x$count = x1 ]] || error "mirror 2 stripe count $count is not 1"
3632 #extend a mirror with 2 OSTs
3633 $LFS mirror extend -N -c2 $tf || error "extend mirror"
3636 local new_id=$($LFS getstripe --mirror-id=2 -I $tf)
3637 count=$($LFS getstripe --mirror-id=2 -c $tf) ||
3638 error "getstripe count of mirror 2"
3639 [[ x$old_id = x$new_id ]] ||
3640 error "mirror 2 changed ID from $old_id to $new_id"
3641 [[ x$count = x1 ]] || error "mirror 2 stripe count $count is not 1"
3643 count=$($LFS getstripe --mirror-id=3 -c $tf) ||
3644 error "getstripe count of mirror 3"
3645 [[ x$count = x2 ]] || error "mirror 3 stripe count $count is not 2"
3647 run_test 203 "mirror file preserve mirror ID"
3649 # Simple test of FLR + self-extending layout, SEL in non-primary mirror
3651 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3652 skip "skipped for lustre < $SEL_VER"
3654 local comp_file=$DIR/$tdir/$tfile
3658 test_mkdir $DIR/$tdir
3659 stack_trap "rm -f $comp_file"
3661 # first mirror is 0-10M, then 10M-(-1), second mirror is 1M followed
3662 # by extension space to -1
3663 $LFS setstripe -N -E 10M -E-1 -N -E 1M -E-1 -z64M $comp_file ||
3664 error "Create $comp_file failed"
3666 # Write to first component, extending & staling second mirror
3667 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc ||
3668 error "dd to extend + stale failed"
3670 $LFS getstripe $comp_file
3672 flg_opts="--component-flags init,stale"
3673 found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
3674 [ $found -eq 1 ] || error "write: Second comp end incorrect"
3676 flg_opts="--component-flags extension"
3677 found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
3678 [ $found -eq 1 ] || error "write: Third comp start incorrect"
3680 # mirror resync should not change the extents
3681 $LFS mirror resync $comp_file
3683 flg_opts="--component-flags init"
3684 found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
3685 [ $found -eq 1 ] || error "resync: Second comp end incorrect"
3687 flg_opts="--component-flags extension"
3688 found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
3689 [ $found -eq 1 ] || error "resync: Third comp start incorrect"
3691 sel_layout_sanity $comp_file 5
3693 run_test 204a "FLR write/stale/resync tests with self-extending mirror"
3695 # Simple test of FLR + self-extending layout, SEL in primary mirror
3697 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3698 skip "skipped for lustre < $SEL_VER"
3700 local comp_file=$DIR/$tdir/$tfile
3704 test_mkdir $DIR/$tdir
3705 stack_trap "rm -f $comp_file"
3707 # first mirror is 1M followed by extension space to -1, second mirror
3708 # is 0-10M, then 10M-(-1),
3709 $LFS setstripe -N -E 1M -E-1 -z64M -N -E 10M -E-1 $comp_file ||
3710 error "Create $comp_file failed"
3712 # Write to first component, extending first component & staling
3714 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc ||
3715 error "dd to extend + stale failed"
3717 $LFS getstripe $comp_file
3719 flg_opts="--component-flags init"
3720 found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
3721 [ $found -eq 1 ] || error "write: First comp end incorrect"
3723 flg_opts="--component-flags extension"
3724 found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
3725 [ $found -eq 1 ] || error "write: Second comp start incorrect"
3727 flg_opts="--component-flags init,stale"
3728 found=$($LFS find --component-end 10M $flg_opts $comp_file | wc -l)
3729 [ $found -eq 1 ] || error "write: First mirror comp flags incorrect"
3731 # This component is staled because it overlaps the extended first
3732 # component of the primary mirror, even though it doesn't overlap
3733 # the actual write - thus not inited.
3734 flg_opts="--component-flags stale"
3735 found=$($LFS find --component-start 10M $flg_opts $comp_file | wc -l)
3736 [ $found -eq 1 ] || error "write: Second mirror comp flags incorrect"
3738 # mirror resync should not change the extents
3739 $LFS mirror resync $comp_file
3741 $LFS getstripe $comp_file
3743 flg_opts="--component-flags init"
3744 found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
3745 [ $found -eq 1 ] || error "resync: First comp end incorrect"
3747 flg_opts="--component-flags extension"
3748 found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
3749 [ $found -eq 1 ] || error "resync: Second comp start incorrect"
3751 flg_opts="--component-flags init"
3752 found=$($LFS find --component-end 10M $flg_opts $comp_file | wc -l)
3753 [ $found -eq 1 ] || error "resync: First mirror comp flags incorrect"
3755 flg_opts="--component-flags init"
3756 found=$($LFS find --component-start 10M $flg_opts $comp_file | wc -l)
3757 [ $found -eq 1 ] || error "resync: Second mirror comp flags incorrect"
3759 sel_layout_sanity $comp_file 5
3761 run_test 204b "FLR write/stale/resync tests with self-extending primary"
3763 # FLR + SEL failed extension & component removal
3764 # extension space in second mirror
3766 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
3767 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3768 skip "skipped for lustre < $SEL_VER"
3770 local comp_file=$DIR/$tdir/$tfile
3773 local ost_name=$(ostname_from_index $ost_idx1)
3775 test_mkdir $DIR/$tdir
3776 stack_trap "rm -f $comp_file"
3778 # first mirror is is 0-10M, then 10M-(-1), second mirror is 0-1M, then
3779 # extension space from 1M to 1G, then normal space to -1
3780 $LFS setstripe -N -E 10M -E-1 -N -E 1M -E 1G -i $ost_idx1 -z 64M \
3781 -E -1 $comp_file || error "Create $comp_file failed"
3783 do_facet ost1 $LCTL set_param -n obdfilter.$ost_name.degraded=1
3786 # write to first comp (0 - 10M) of mirror 1, extending + staling
3787 # first + second comp of mirror 2
3788 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc
3791 do_facet ost1 $LCTL set_param -n obdfilter.$ost_name.degraded=0
3794 [ $RC -eq 0 ] || error "dd to extend + stale failed"
3796 $LFS getstripe $comp_file
3798 found=$($LFS find --component-start 0m --component-end 1m \
3799 --comp-flags init,stale $comp_file | wc -l)
3800 [ $found -eq 1 ] || error "write: First mirror comp incorrect"
3802 found=$($LFS find --component-start 1m --component-end EOF \
3803 --comp-flags stale,^init $comp_file | wc -l)
3804 [ $found -eq 1 ] || error "write: Second mirror comp incorrect"
3806 local mirror_id=$($LFS getstripe --component-start=1m \
3807 --component-end=EOF $comp_file | \
3808 grep lcme_mirror_id | awk '{ print $2 }')
3810 [[ $mirror_id -eq 2 ]] ||
3811 error "component not in correct mirror? $mirror_id"
3813 $LFS mirror resync $comp_file
3815 $LFS getstripe $comp_file
3817 # component dimensions should not change from resync
3818 found=$($LFS find --component-start 1m --component-end EOF \
3819 --component-flags init $comp_file | wc -l)
3820 [ $found -eq 1 ] || error "resync: Second mirror comp incorrect"
3822 sel_layout_sanity $comp_file 4
3824 run_test 204c "FLR write/stale/resync test with component removal"
3826 # Successful repeated component in primary mirror
3828 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
3829 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3830 skip "skipped for lustre < $SEL_VER"
3832 local comp_file=$DIR/$tdir/$tfile
3835 wait_delete_completed
3837 test_mkdir $DIR/$tdir
3838 stack_trap "rm -f $comp_file"
3840 # first mirror is 64M followed by extension space to -1, second mirror
3841 # is 0-10M, then 10M-(-1)
3842 $LFS setstripe -N -E-1 -z64M -N -E 10M -E-1 $comp_file ||
3843 error "Create $comp_file failed"
3845 local ost_idx1=$($LFS getstripe -I65537 -i $comp_file)
3846 local ost_name=$(ostname_from_index $ost_idx1)
3847 # degrade OST for first comp so we won't extend there
3848 do_facet ost$((ost_idx1+1)) $LCTL set_param -n \
3849 obdfilter.$ost_name.degraded=1
3852 # Write beyond first component, causing repeat & stale second mirror
3853 dd if=/dev/zero bs=1M count=1 seek=66 of=$comp_file conv=notrunc
3856 do_facet ost$((ost_idx1+1)) $LCTL set_param -n \
3857 obdfilter.$ost_name.degraded=0
3860 [ $RC -eq 0 ] || error "dd to repeat & stale failed"
3862 $LFS getstripe $comp_file
3864 found=$($LFS find --component-start 64m --component-end 128m \
3865 --component-flags init $comp_file | wc -l)
3866 [ $found -eq 1 ] || error "write: Repeat comp incorrect"
3868 local ost_idx2=$($LFS getstripe --component-start=64m \
3869 --component-end=128m --component-flags=init \
3871 [[ $ost_idx1 -eq $ost_idx2 ]] && error "$ost_idx1 == $ost_idx2"
3872 local mirror_id=$($LFS getstripe --component-start=64m \
3873 --component-end=128m --component-flags=init \
3874 $comp_file | grep lcme_mirror_id | awk '{ print $2 }')
3875 [[ $mirror_id -eq 1 ]] ||
3876 error "component not in correct mirror: $mirror_id, not 1"
3878 $LFS mirror resync $comp_file
3880 $LFS getstripe $comp_file
3882 # component dimensions should not change from resync
3883 found=$($LFS find --component-start 0m --component-end 64m \
3884 --component-flags init $comp_file | wc -l)
3885 [ $found -eq 1 ] || error "resync: first comp incorrect"
3886 found=$($LFS find --component-start 64m --component-end 128m \
3887 --component-flags init $comp_file | wc -l)
3888 [ $found -eq 1 ] || error "resync: repeat comp incorrect"
3890 sel_layout_sanity $comp_file 5
3892 run_test 204d "FLR write/stale/resync sel test with repeated comp"
3894 # Successful repeated component, SEL in non-primary mirror
3896 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
3897 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3898 skip "skipped for lustre < $SEL_VER"
3900 local comp_file=$DIR/$tdir/$tfile
3903 wait_delete_completed
3906 test_mkdir $DIR/$tdir
3907 stack_trap "rm -f $comp_file"
3909 # first mirror is is 0-100M, then 100M-(-1), second mirror is extension
3910 # space to -1 (-z 64M, so first comp is 0-64M)
3911 # Note: we have to place both 1st components on OST0, otherwise 2 OSTs
3912 # will be not enough - one will be degraded, the other is used on
3913 # an overlapping mirror.
3914 $LFS setstripe -N -E 100M -i 0 -E-1 -N -E-1 -i 0 -z 64M $comp_file ||
3915 error "Create $comp_file failed"
3917 local ost_idx1=$($LFS getstripe --component-start=0 \
3918 --component-end=64m -i $comp_file)
3919 local ost_name=$(ostname_from_index $ost_idx1)
3920 # degrade OST for first comp of 2nd mirror so we won't extend there
3921 do_facet ost$((ost_idx1+1)) $LCTL set_param -n \
3922 obdfilter.$ost_name.degraded=1
3925 $LFS getstripe $comp_file
3927 # Write to first component, stale & instantiate second mirror components
3928 # overlapping with the written component (0-100M);
3929 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc
3932 do_facet ost$((ost_idx1+1)) $LCTL set_param -n \
3933 obdfilter.$ost_name.degraded=0
3935 $LFS getstripe $comp_file
3937 [ $RC -eq 0 ] || error "dd to repeat & stale failed"
3939 found=$($LFS find --component-start 0m --component-end 64m \
3940 --component-flags init,stale $comp_file | wc -l)
3941 [ $found -eq 1 ] || error "write: first comp incorrect"
3943 # was repeated due to degraded ost
3944 found=$($LFS find --component-start 64m --component-end 128m \
3945 --component-flags init,stale $comp_file | wc -l)
3946 [ $found -eq 1 ] || error "write: repeated comp incorrect"
3948 local ost_idx2=$($LFS getstripe --component-start=64m \
3949 --component-end=128m --component-flags=init \
3951 [[ $ost_idx1 -eq $ost_idx2 ]] && error "$ost_idx1 == $ost_idx2"
3952 local mirror_id=$($LFS getstripe --component-start=0m \
3953 --component-end=64m --component-flags=init \
3954 $comp_file | grep lcme_mirror_id | awk '{ print $2 }')
3955 [[ $mirror_id -eq 2 ]] ||
3956 error "component not in correct mirror? $mirror_id"
3958 $LFS mirror resync $comp_file
3960 $LFS getstripe $comp_file
3962 # component dimensions should not change from resync
3963 found=$($LFS find --component-start 0m --component-end 64m \
3964 --component-flags init,^stale $comp_file | wc -l)
3965 [ $found -eq 1 ] || error "resync: first comp incorrect"
3966 found=$($LFS find --component-start 64m --component-end 128m \
3967 --component-flags init,^stale $comp_file | wc -l)
3968 [ $found -eq 1 ] || error "resync: repeated comp incorrect"
3970 sel_layout_sanity $comp_file 5
3972 run_test 204e "FLR write/stale/resync sel test with repeated comp"
3974 # FLR + SEL: failed repeated component, SEL in non-primary mirror
3976 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
3977 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3978 skip "skipped for lustre < $SEL_VER"
3980 local comp_file=$DIR/$tdir/$tfile
3983 wait_delete_completed
3985 test_mkdir $DIR/$tdir
3986 stack_trap "rm -f $comp_file"
3988 pool_add $TESTNAME || error "Pool creation failed"
3989 pool_add_targets $TESTNAME 0 1 || error "Pool add targets failed"
3991 # first mirror is is 0-100M, then 100M-(-1), second mirror is extension
3992 # space to -1 (-z 64M, so first comp is 0-64M)
3993 $LFS setstripe -N -E 100M -E-1 -N --pool="$TESTNAME" \
3994 -E-1 -c 1 -z 64M $comp_file || error "Create $comp_file failed"
3996 local ost_name0=$(ostname_from_index 0)
3997 local ost_name1=$(ostname_from_index 1)
3999 # degrade both OSTs in pool, so we'll try to repeat, then fail and
4000 # extend original comp
4001 do_facet ost1 $LCTL set_param -n obdfilter.$ost_name0.degraded=1
4002 do_facet ost2 $LCTL set_param -n obdfilter.$ost_name1.degraded=1
4005 # a write to the 1st component, 100M length, which will try to stale
4006 # the first 100M of mirror 2, attempting to extend its 0-64M component
4007 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc
4010 do_facet ost1 $LCTL set_param -n obdfilter.$ost_name0.degraded=0
4011 do_facet ost2 $LCTL set_param -n obdfilter.$ost_name1.degraded=0
4014 [ $RC -eq 0 ] || error "dd to extend mirror comp failed"
4016 $LFS getstripe $comp_file
4018 found=$($LFS find --component-start 0m --component-end 128m \
4019 --component-flags init,stale $comp_file | wc -l)
4020 [ $found -eq 1 ] || error "write: First mirror comp incorrect"
4022 local mirror_id=$($LFS getstripe --component-start=0m \
4023 --component-end=128m --component-flags=init \
4024 $comp_file | grep lcme_mirror_id | awk '{ print $2 }')
4026 [[ $mirror_id -eq 2 ]] ||
4027 error "component not in correct mirror? $mirror_id, not 2"
4029 $LFS mirror resync $comp_file
4031 $LFS getstripe $comp_file
4033 # component dimensions should not change from resync
4034 found=$($LFS find --component-start 0m --component-end 128m \
4035 --component-flags init,^stale $comp_file | wc -l)
4036 [ $found -eq 1 ] || error "resync: First mirror comp incorrect"
4038 sel_layout_sanity $comp_file 4
4040 run_test 204f "FLR write/stale/resync sel w/forced extension"
4042 function test_205() {
4043 local tf=$DIR/$tfile
4046 $LFS setstripe -c1 $tf
4047 $LFS mirror extend -N $tf
4048 mirrors=$($LFS getstripe $tf | grep lcme_mirror_id | wc -l )
4049 (( $mirrors == 2 )) || error "no new mirror was created?"
4051 $LFS mirror extend -N --flags=prefer $tf
4052 mirrors=$($LFS getstripe $tf | grep lcme_mirror_id | wc -l )
4053 (( $mirrors == 3 )) || error "no new mirror was created?"
4055 $($LFS getstripe $tf | grep lcme_flags: | tail -1 | grep -q prefer) ||
4056 error "prefer flag was not set on the new mirror"
4058 run_test 205 "lfs mirror extend to set prefer flag"
4060 function test_206() {
4061 # create a new OST pool
4062 local pool_name=$TESTNAME
4064 create_pool $FSNAME.$pool_name ||
4065 error "create OST pool $pool_name failed"
4066 # add OSTs into the pool
4067 pool_add_targets $pool_name 0 1 ||
4068 error "add OSTs into pool $pool_name failed"
4070 $LFS setstripe -c1 --pool=$pool_name $DIR/$tfile ||
4071 error "can't setstripe"
4072 $LFS mirror extend -N $DIR/$tfile ||
4073 error "can't create replica"
4074 if $LFS getstripe $DIR/$tfile | grep -q prefer ; then
4075 $LFS getstripe $DIR/$tfile
4076 error "prefer found"
4078 $LFS setstripe --comp-set --comp-flags=prefer -p $pool_name $DIR/$tfile || {
4079 $LFS getstripe $DIR/$tfile
4080 error "can't setstripe prefer"
4083 if ! $LFS getstripe $DIR/$tfile | grep -q prefer ; then
4084 $LFS getstripe $DIR/$tfile
4085 error "no prefer found"
4091 run_test 206 "lfs setstripe -pool .. --comp-flags=.. "
4094 local file=$DIR/$tfile
4095 local tmpfile=$DIR/$tfile-tt
4097 [ $MDS1_VERSION -lt $(version_code 2.14.50) ] &&
4098 skip "Need MDS version at least 2.14.50"
4100 stack_trap "rm -f $tmpfile $file"
4102 # generate data for verification
4103 dd if=/dev/urandom of=$tmpfile bs=1M count=1 ||
4104 error "can't generate file with random data"
4106 # create a mirrored file with one stale replica
4107 $LFS mirror create -N -S 4M -c 2 -N -S 1M -c -1 $file ||
4108 error "create mirrored file $file failed"
4109 get_mirror_ids $file
4110 echo "mirror IDs: ${mirror_array[*]}"
4112 dd if=$tmpfile of=$file bs=1M || error "can't copy"
4113 get_mirror_ids $file
4114 echo "mirror IDs: ${mirror_array[*]}"
4117 cmp $tmpfile $file || error "files don't match"
4118 get_mirror_ids $file
4119 echo "mirror IDs: ${mirror_array[*]}"
4121 # mirror creation should work fine
4122 $LFS mirror extend -N -S 8M -c -1 $file ||
4123 error "mirror extend $file failed"
4125 get_mirror_ids $file
4126 echo "mirror IDs: ${mirror_array[*]}"
4129 $LFS mirror verify -v $file || error "verification failed"
4130 cmp $tmpfile $file || error "files don't match"
4132 run_test 207 "create another replica with existing out-of-sync one"
4134 function check_ost_used() {
4143 cancel_lru_locks osc # to drop pages
4144 cancel_lru_locks mdc # to refresh layout
4145 # XXX: cancel_lru_locks mdc doesn't work
4146 # XXX: need a better way to reload the layout
4147 umount_client $MOUNT || error "umount failed"
4148 mount_client $MOUNT || error "mount failed"
4150 # refresh non-rotation status on MDTs
4152 touch $DIR/$tfile-temp
4153 rm -f $DIR/$tfile-temp
4154 # refresh non-rotational status on the client
4158 $LCTL set_param osc.*.stats=clear >/dev/null
4159 if [[ $io == "read" ]]; then
4160 ddarg="if=$file of=/dev/null"
4161 elif [[ $io == "write" ]]; then
4162 ddarg="if=/dev/zero of=$file"
4164 error "unknown type $io"
4166 dd $ddarg bs=2M count=1 || error "can't $io $file"
4167 cancel_lru_locks osc
4169 # check only specified OSTs got reads
4170 for ((ost = 0; ost < $OSTCOUNT; ost++)); do
4171 local nr=$($LCTL get_param -n \
4172 osc.$FSNAME-OST000$ost-osc-[-0-9a-f]*.stats |
4173 awk "/ost_$io/{print \$2}")
4175 if [[ " $* " =~ $ost ]]; then
4176 (( nr > 0 )) || error "expected reads on $ost"
4178 (( nr == 0 )) || error "unexpected $nr reads on $ost"
4184 local tf=$DIR/$tfile
4185 local osts=$(comma_list $(osts_nodes))
4187 (( $OSTCOUNT >= 4 )) || skip "needs >= 4 OSTs"
4188 (( $MDS1_VERSION >= $(version_code 2.14.55) )) ||
4189 skip "Need MDS version at least 2.14.55"
4191 local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
4193 save_lustre_params $(get_facets OST) osd*.*OST*.nonrotational > $p
4194 stack_trap "restore_lustre_params < $p; rm -f $p"
4196 stack_trap "rm -f $tf"
4197 $LFS setstripe -i0 -c1 $tf || error "can't setstripe"
4198 dd if=/dev/zero of=$tf bs=2M count=1 || error "can't dd (1)"
4199 $LFS mirror extend -N -c1 -o1 $tf || error "can't create mirror"
4200 $LFS mirror extend -N -c2 -o 2,3 $tf || error "can't create mirror"
4201 $LFS mirror resync $tf || error "can't resync"
4204 log "set OST0000 non-rotational"
4206 $LCTL set_param osd*.*OST*.nonrotational=0
4208 $LCTL set_param osd*.*OST0000*.nonrotational=1
4209 check_ost_used $tf read 0
4211 log "set OST0002 and OST0003 non-rotational, two fast OSTs is better"
4213 $LCTL set_param osd*.*OST*.nonrotational=0
4215 $LCTL set_param osd*.*OST0002*.nonrotational=1 \
4216 osd*.*OST0003*.nonrotational=1
4217 check_ost_used $tf read 2 3
4219 log "set mirror 1 on OST0001 preferred"
4220 $LFS setstripe --comp-set -I 0x20001 --comp-flags=prefer $tf ||
4221 error "can't set prefer"
4222 check_ost_used $tf read 1
4224 run_test 208a "mirror selection to prefer non-rotational devices for reads"
4227 local tf=$DIR/$tfile
4228 local osts=$(comma_list $(osts_nodes))
4230 (( $OSTCOUNT >= 4 )) || skip "needs >= 4 OSTs"
4231 (( $MDS1_VERSION >= $(version_code 2.14.55) )) ||
4232 skip "Need MDS version at least 2.14.55"
4234 local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
4236 save_lustre_params $(get_facets OST) osd*.*OST*.nonrotational > $p
4237 stack_trap "restore_lustre_params < $p; rm -f $p"
4239 stack_trap "rm -f $tf"
4240 $LFS setstripe -i0 -c1 $tf || error "can't setstripe"
4241 dd if=/dev/zero of=$tf bs=2M count=1 || error "can't dd (1)"
4242 $LFS mirror extend -N -c1 -o1 $tf || error "can't create mirror"
4243 $LFS mirror extend -N -c2 -o 2,3 $tf || error "can't create mirror"
4244 $LFS mirror resync $tf || error "can't resync"
4245 $LFS getstripe $tf | grep -q flags.*stale && error "still stale"
4247 log "set OST0000 non-rotational"
4249 $LCTL set_param osd*.*OST*.nonrotational=0
4251 $LCTL set_param osd*.*OST0000*.nonrotational=1
4252 check_ost_used $tf write 0
4253 $LFS mirror resync $tf || error "can't resync"
4255 log "set OST0002 and OST0003 non-rotational, two fast OSTs is better"
4257 $LCTL set_param osd*.*OST*.nonrotational=0
4259 $LCTL set_param osd*.*OST0002*.nonrotational=1 \
4260 osd*.*OST0003*.nonrotational=1
4261 check_ost_used $tf write 2 3
4262 $LFS mirror resync $tf || error "can't resync"
4264 log "set mirror 1 on OST0001 preferred"
4265 $LFS setstripe --comp-set -I 0x20001 --comp-flags=prefer $tf ||
4266 error "can't set prefer"
4267 check_ost_used $tf write 1
4269 run_test 208b "mirror selection to prefer non-rotational devices for writes"
4272 check_and_cleanup_lustre