3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
11 LUSTRE=${LUSTRE:-$(dirname $0)/..}
12 . $LUSTRE/tests/test-framework.sh
16 ALWAYS_EXCEPT="$SANITY_FLR_EXCEPT "
17 # Bug number for skipped test: LU-14818 LU-11381 LU-14765
18 ALWAYS_EXCEPT+=" 6 201 44c "
19 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
21 # skip all tests for PPC until we can get sanity-pfl to pass
22 if [[ $(uname -m) = ppc64 ]]; then
23 skip "Skip FLR testing for PPC clients"
26 if [[ "$ost1_FSTYPE" == "zfs" ]]; then
28 ALWAYS_EXCEPT+=" 49a "
33 [[ "$MDS1_VERSION" -ge $(version_code 2.10.56) ]] ||
34 skip "Need MDS version at least 2.10.56"
36 check_and_setup_lustre
39 rm -rf $DIR/[Rdfs][0-9]*
41 [ $UID -eq 0 -a $RUNAS_ID -eq 0 ] &&
42 error "\$RUNAS_ID set to 0, but \$UID is also 0!"
44 check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS
47 # global array to store mirror IDs
48 declare -a mirror_array
55 for id in $($LFS getstripe $tf | awk '/lcme_id/{print $2}'); do
56 array[${#array[@]}]=$((id >> 16))
59 mirror_array=($(printf "%s\n" "${array[@]}" | sort -u))
61 echo ${#mirror_array[@]}
65 echo 3 > /proc/sys/vm/drop_caches
76 wait_osc_import_state client ost$idx "\(DISCONN\|IDLE\)"
84 start ost$idx $(ostdevname $idx) $OST_MOUNT_OPTS ||
85 error "start ost$idx failed"
89 wait_recovery_complete ost$idx
94 # Verify mirror count with an expected value for a given file.
96 verify_mirror_count() {
99 local mirror_count=$($LFS getstripe -N $tf)
101 [[ $mirror_count = $expected ]] || {
102 $LFS getstripe -v $tf
103 error "verify mirror count failed on $tf:" \
104 "$mirror_count != $expected"
109 # Verify component count with an expected value for a given file.
110 # $1 coposited layout file
111 # $2 expected component number
113 verify_comp_count() {
116 local comp_count=$($LFS getstripe --component-count $tf)
118 [[ $comp_count = $expected ]] || {
119 $LFS getstripe -v $tf
120 error "verify component count failed on $tf:" \
121 "$comp_count != $expected"
126 # Verify component attribute with an expected value for a given file
134 local cmd="$LFS getstripe -I$comp_id"
135 local getstripe_cmd="$cmd -v"
139 stripe-size) cmd+=" -S $tf" ;;
140 stripe-count) cmd+=" -c $tf" ;;
141 stripe-index) cmd+=" -i $tf" ;;
142 pool) cmd+=" -p $tf" ;;
143 comp-start) cmd+=" --component-start $tf" ;;
144 comp-end) cmd+=" --component-end $tf" ;;
145 lcme_flags) cmd+=" $tf | awk '/lcme_flags:/ { print \$2 }'" ;;
146 *) error "invalid attribute $attr";;
151 [ $attr = lcme_flags ] && {
153 local expected_list=$(comma_list $expected)
154 for fl in ${expected_list//,/ }; do
157 [[ ${fl:0:1} = "^" ]] && neg=1
158 [[ $neg = 1 ]] && fl=${fl:1}
160 $(echo $value | grep -q $fl)
162 # 0: matched; 1: not matched
164 if [[ $neg = 0 && $match != 0 ||
165 $neg = 1 && $match = 0 ]]; then
167 [[ $neg = 0 ]] && # expect the flag
168 error "expected flag '$fl' not in $comp_id"
169 [[ $neg = 1 ]] && # not expect the flag
170 error "not expected flag '$fl' in $comp_id"
176 [[ $value = $expected ]] || {
178 error "verify $attr failed on $tf: $value != $expected"
183 # Verify component extent with expected start and end extent values
184 # for a given file and component ID.
186 verify_comp_extent() {
189 local expected_start=$3
190 local expected_end=$4
192 verify_comp_attr comp-start $tf $comp_id $expected_start
193 verify_comp_attr comp-end $tf $comp_id $expected_end
197 # Verify component attribute with parent directory for a given file
200 verify_comp_attr_with_parent() {
204 local td=$(cd $(dirname $tf); echo $PWD)
205 local tf_cmd="$LFS getstripe -I$comp_id"
206 local td_cmd="$LFS getstripe"
212 stripe-size) opt="-S" ;;
213 stripe-count) opt="-c" ;;
215 *) error "invalid attribute $attr";;
218 expected=$($td_cmd $opt $td)
219 [[ $expected = -1 ]] && expected=$OSTCOUNT
221 value=$($tf_cmd $opt $tf)
222 [[ $value = -1 ]] && value=$OSTCOUNT
224 [[ $value = $expected ]] || {
227 error "verify $attr failed with parent on $tf:" \
228 "$value != $expected"
233 # Verify component attribute with filesystem-wide default value for a given file
236 verify_comp_attr_with_default() {
240 local tf_cmd="$LFS getstripe -I$comp_id"
248 expected=$($LCTL get_param -n \
249 lov.$FSNAME-clilov-*.stripesize)
253 expected=$($LCTL get_param -n \
254 lov.$FSNAME-clilov-*.stripecount)
255 [[ $expected = -1 ]] && expected=$OSTCOUNT
257 *) error "invalid attribute $attr";;
260 value=$($tf_cmd $opt $tf)
261 [[ $value = -1 ]] && value=$OSTCOUNT
263 [[ $value = $expected ]] || {
265 error "verify $attr failed with default value on $tf:" \
266 "$value != $expected"
271 # Verify unspecified component attributes for a given file
274 # This will only verify the inherited attributes:
275 # stripe size, stripe count and OST pool name
277 verify_comp_attrs() {
281 verify_comp_attr_with_default stripe-size $tf $comp_id
282 verify_comp_attr_with_default stripe-count $tf $comp_id
283 verify_comp_attr_with_parent pool $tf $comp_id
289 local expected_state=$2
291 local state=$($LFS getstripe -v $tf | awk '/lcm_flags/{ print $2 }')
292 [ $expected_state = $state ] ||
293 error "expected: $expected_state, actual $state"
296 # command line test cases
300 local mirror_count=16 # LUSTRE_MIRROR_COUNT_MAX
301 local mirror_cmd="$LFS mirror create"
306 # create parent directory
307 mkdir $td || error "mkdir $td failed"
309 $mirror_cmd $tf &> /dev/null && error "miss -N option"
311 $mirror_cmd -N $tf || error "create mirrored file $tf failed"
312 verify_mirror_count $tf 1
313 id=$($LFS getstripe -I $tf)
314 verify_comp_attrs $tf $id
315 verify_comp_extent $tf $id 0 EOF
317 $mirror_cmd -N0 $tf-1 &> /dev/null && error "invalid mirror count 0"
318 $mirror_cmd -N$((mirror_count + 1)) $tf-1 &> /dev/null &&
319 error "invalid mirror count $((mirror_count + 1))"
321 $mirror_cmd -N$mirror_count $tf-1 ||
322 error "create mirrored file $tf-1 failed"
323 verify_mirror_count $tf-1 $mirror_count
324 ids=($($LFS getstripe $tf-1 | awk '/lcme_id/{print $2}' | tr '\n' ' '))
325 for ((i = 0; i < $mirror_count; i++)); do
326 verify_comp_attrs $tf-1 ${ids[$i]}
327 verify_comp_extent $tf-1 ${ids[$i]} 0 EOF
330 $mirror_cmd -N -N2 -N3 -N4 $tf-2 ||
331 error "create mirrored file $tf-2 failed"
332 verify_mirror_count $tf-2 10
333 ids=($($LFS getstripe $tf-2 | awk '/lcme_id/{print $2}' | tr '\n' ' '))
334 for ((i = 0; i < 10; i++)); do
335 verify_comp_attrs $tf-2 ${ids[$i]}
336 verify_comp_extent $tf-2 ${ids[$i]} 0 EOF
339 run_test 0a "lfs mirror create with -N option"
342 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
346 local mirror_cmd="$LFS mirror create"
350 # create a new OST pool
351 local pool_name=$TESTNAME
352 create_pool $FSNAME.$pool_name ||
353 error "create OST pool $pool_name failed"
355 # add OSTs into the pool
356 pool_add_targets $pool_name 0 $((OSTCOUNT - 1)) ||
357 error "add OSTs into pool $pool_name failed"
359 # create parent directory
360 mkdir $td || error "mkdir $td failed"
361 $LFS setstripe -S 8M -c -1 -p $pool_name $td ||
362 error "$LFS setstripe $td failed"
364 # create a mirrored file with plain layout mirrors
365 $mirror_cmd -N -N -S 4M -c 2 -p flash -i 2 -o 2,3 \
366 -N -S 16M -N -c -1 -N -p archive -N -p none $tf ||
367 error "create mirrored file $tf failed"
368 verify_mirror_count $tf 6
369 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
370 for ((i = 0; i < 6; i++)); do
371 verify_comp_extent $tf ${ids[$i]} 0 EOF
374 # verify component ${ids[0]}
375 verify_comp_attrs $tf ${ids[0]}
377 # verify component ${ids[1]}
378 verify_comp_attr stripe-size $tf ${ids[1]} 4194304
379 verify_comp_attr stripe-count $tf ${ids[1]} 2
380 verify_comp_attr stripe-index $tf ${ids[1]} 2
382 # verify component ${ids[2]}
383 verify_comp_attr stripe-size $tf ${ids[2]} 16777216
384 verify_comp_attr stripe-count $tf ${ids[2]} 2
385 verify_comp_attr pool $tf ${ids[2]} flash
387 # verify component ${ids[3]}
388 verify_comp_attr stripe-size $tf ${ids[3]} 16777216
389 verify_comp_attr stripe-count $tf ${ids[3]} $OSTCOUNT
390 verify_comp_attr pool $tf ${ids[3]} flash
392 # verify component ${ids[4]}
393 verify_comp_attr stripe-size $tf ${ids[4]} 16777216
394 verify_comp_attr stripe-count $tf ${ids[4]} $OSTCOUNT
395 verify_comp_attr pool $tf ${ids[4]} archive
397 # verify component ${ids[5]}
398 verify_comp_attr stripe-size $tf ${ids[5]} 16777216
399 verify_comp_attr stripe-count $tf ${ids[5]} $OSTCOUNT
400 verify_comp_attr_with_parent pool $tf ${ids[5]}
402 if [ $MDS1_VERSION -ge $(version_code 2.12.55) ]; then
403 # LU-11022 - remove mirror by pool name
404 local=cnt cnt=$($LFS getstripe $tf | grep archive | wc -l)
405 [ "$cnt" != "1" ] && error "unexpected mirror count $cnt"
406 $LFS mirror delete --pool archive $tf || error "delete mirror"
407 cnt=$($LFS getstripe $tf | grep archive | wc -l)
408 [ "$cnt" != "0" ] && error "mirror count after removal: $cnt"
414 run_test 0b "lfs mirror create plain layout mirrors"
417 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
421 local mirror_cmd="$LFS mirror create"
425 # create a new OST pool
426 local pool_name=$TESTNAME
427 create_pool $FSNAME.$pool_name ||
428 error "create OST pool $pool_name failed"
430 # add OSTs into the pool
431 pool_add_targets $pool_name 0 $((OSTCOUNT - 1)) ||
432 error "add OSTs into pool $pool_name failed"
434 # create parent directory
435 mkdir $td || error "mkdir $td failed"
436 $LFS setstripe -E 32M -S 8M -c -1 -p $pool_name -E eof -S 16M $td ||
437 error "$LFS setstripe $td failed"
439 # create a mirrored file with composite layout mirrors
440 $mirror_cmd -N2 -E 4M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
442 -N3 -E 512M -S 16M -p archive -E -1 -i -1 -c -1 $tf ||
443 error "create mirrored file $tf failed"
444 verify_mirror_count $tf 6
445 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
447 # verify components ${ids[0]} and ${ids[2]}
449 verify_comp_attr_with_default stripe-size $tf ${ids[$i]}
450 verify_comp_attr stripe-count $tf ${ids[$i]} 2
451 verify_comp_attr stripe-index $tf ${ids[$i]} 1
452 verify_comp_extent $tf ${ids[$i]} 0 4194304
455 # verify components ${ids[1]} and ${ids[3]}
457 verify_comp_attr stripe-size $tf ${ids[$i]} 4194304
458 verify_comp_attr stripe-count $tf ${ids[$i]} 2
459 verify_comp_attr pool $tf ${ids[$i]} flash
460 verify_comp_extent $tf ${ids[$i]} 4194304 EOF
463 # verify component ${ids[4]}
464 verify_comp_attr stripe-size $tf ${ids[4]} 4194304
465 verify_comp_attr stripe-count $tf ${ids[4]} 4
466 verify_comp_attr_with_parent pool $tf ${ids[4]}
467 verify_comp_extent $tf ${ids[4]} 0 EOF
469 # verify components ${ids[5]}, ${ids[7]} and ${ids[9]}
471 verify_comp_attr stripe-size $tf ${ids[$i]} 16777216
472 verify_comp_attr stripe-count $tf ${ids[$i]} 4
473 verify_comp_attr pool $tf ${ids[$i]} archive
474 verify_comp_extent $tf ${ids[$i]} 0 536870912
477 # verify components ${ids[6]}, ${ids[8]} and ${ids[10]}
479 verify_comp_attr stripe-size $tf ${ids[$i]} 16777216
480 verify_comp_attr stripe-count $tf ${ids[$i]} -1
481 verify_comp_attr pool $tf ${ids[$i]} archive
482 verify_comp_extent $tf ${ids[$i]} 536870912 EOF
488 run_test 0c "lfs mirror create composite layout mirrors"
493 local mirror_count=16 # LUSTRE_MIRROR_COUNT_MAX
494 local mirror_cmd="$LFS mirror extend"
498 # create parent directory
499 mkdir $td || error "mkdir $td failed"
501 $mirror_cmd $tf &> /dev/null && error "miss -N option"
502 $mirror_cmd -N $tf &> /dev/null && error "$tf does not exist"
504 # create a non-mirrored file, convert it to a mirrored file and extend
505 touch $tf || error "touch $tf failed"
506 $mirror_cmd -N $tf || error "convert and extend $tf failed"
507 verify_mirror_count $tf 2
508 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
509 for ((i = 0; i < 2; i++)); do
510 verify_comp_attrs $tf ${ids[$i]}
511 verify_comp_extent $tf ${ids[$i]} 0 EOF
514 lfsck_verify_pfid $tf || error "PFID is not set"
516 # create a mirrored file and extend it
517 $LFS mirror create -N $tf-1 || error "create mirrored file $tf-1 failed"
518 $LFS mirror create -N $tf-2 || error "create mirrored file $tf-2 failed"
519 $LFS mirror create -N $tf-3 || error "create mirrored file $tf-3 failed"
521 $mirror_cmd -N -S 4M -N -f $tf-2 $tf-1 ||
522 error "extend mirror with -f failed"
524 $mirror_cmd -N$((mirror_count - 1)) $tf-3 ||
525 error "extend mirrored file $tf-3 failed"
526 verify_mirror_count $tf-3 $mirror_count
527 ids=($($LFS getstripe $tf-3 | awk '/lcme_id/{print $2}' | tr '\n' ' '))
528 for ((i = 0; i < $mirror_count; i++)); do
529 verify_comp_attrs $tf-3 ${ids[$i]}
530 verify_comp_extent $tf-3 ${ids[$i]} 0 EOF
533 $mirror_cmd -N $tf-3 &> /dev/null
536 error "exceeded maximum mirror count returns $rc not ERANGE(34)"
538 run_test 0d "lfs mirror extend with -N option"
541 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
545 local mirror_cmd="$LFS mirror extend"
549 # create parent directory
550 mkdir $td || error "mkdir $td failed"
552 # create a mirrored file with plain layout mirrors
553 $LFS mirror create -N -S 32M -c 3 -p ssd -i 1 -o 1,2,3 $tf ||
554 error "create mirrored file $tf failed"
556 # extend the mirrored file with plain layout mirrors
557 $mirror_cmd -N -S 4M -c 2 -p flash -i 2 -o 2,3 \
558 -N -S 16M -N -c -1 -N -p archive -N -p none $tf ||
559 error "extend mirrored file $tf failed"
560 verify_mirror_count $tf 6
561 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
562 for ((i = 0; i < 6; i++)); do
563 verify_comp_extent $tf ${ids[$i]} 0 EOF
566 # verify component ${ids[0]}
567 verify_comp_attr stripe-size $tf ${ids[0]} 33554432
568 verify_comp_attr stripe-count $tf ${ids[0]} 3
569 verify_comp_attr stripe-index $tf ${ids[0]} 1
571 # verify component ${ids[1]}
572 verify_comp_attr stripe-size $tf ${ids[1]} 4194304
573 verify_comp_attr stripe-count $tf ${ids[1]} 2
574 verify_comp_attr stripe-index $tf ${ids[1]} 2
576 # verify component ${ids[2]}
577 verify_comp_attr stripe-size $tf ${ids[2]} 16777216
578 verify_comp_attr stripe-count $tf ${ids[2]} 2
579 verify_comp_attr pool $tf ${ids[2]} flash
581 # verify component ${ids[3]}
582 verify_comp_attr stripe-size $tf ${ids[3]} 16777216
583 verify_comp_attr stripe-count $tf ${ids[3]} $OSTCOUNT
584 verify_comp_attr pool $tf ${ids[3]} flash
586 # verify component ${ids[4]}
587 verify_comp_attr stripe-size $tf ${ids[4]} 16777216
588 verify_comp_attr stripe-count $tf ${ids[4]} $OSTCOUNT
589 verify_comp_attr pool $tf ${ids[4]} archive
591 # verify component ${ids[5]}
592 verify_comp_attr stripe-size $tf ${ids[5]} 16777216
593 verify_comp_attr stripe-count $tf ${ids[5]} $OSTCOUNT
594 verify_comp_attr_with_parent pool $tf ${ids[5]}
596 run_test 0e "lfs mirror extend plain layout mirrors"
599 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
603 local mirror_cmd="$LFS mirror extend"
607 # create parent directory
608 mkdir $td || error "mkdir $td failed"
610 # create a mirrored file with composite layout mirror
611 $LFS mirror create -N -E 32M -S 16M -p ssd -E eof -S 32M $tf ||
612 error "create mirrored file $tf failed"
614 # extend the mirrored file with composite layout mirrors
615 $mirror_cmd -N -p archive \
616 -N2 -E 4M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
618 -N3 -E 512M -S 16M -p archive -E -1 -i -1 -c -1 $tf ||
619 error "extend mirrored file $tf failed"
620 verify_mirror_count $tf 8
621 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
623 # verify component ${ids[0]}
624 verify_comp_attr stripe-size $tf ${ids[0]} 16777216
625 verify_comp_attr_with_default stripe-count $tf ${ids[0]}
626 verify_comp_attr pool $tf ${ids[0]} ssd
627 verify_comp_extent $tf ${ids[0]} 0 33554432
629 # verify component ${ids[1]}
630 verify_comp_attr stripe-size $tf ${ids[1]} 33554432
631 verify_comp_attr_with_default stripe-count $tf ${ids[1]}
632 verify_comp_attr pool $tf ${ids[1]} ssd
633 verify_comp_extent $tf ${ids[1]} 33554432 EOF
635 # verify component ${ids[2]}
636 verify_comp_attr stripe-size $tf ${ids[0]} 16777216
637 verify_comp_attr_with_default stripe-count $tf ${ids[2]}
638 verify_comp_attr pool $tf ${ids[2]} archive
639 verify_comp_extent $tf ${ids[2]} 0 EOF
641 # verify components ${ids[3]} and ${ids[5]}
643 verify_comp_attr_with_default stripe-size $tf ${ids[$i]}
644 verify_comp_attr stripe-count $tf ${ids[$i]} 2
645 verify_comp_attr stripe-index $tf ${ids[$i]} 1
646 verify_comp_extent $tf ${ids[$i]} 0 4194304
649 # verify components ${ids[4]} and ${ids[6]}
651 verify_comp_attr stripe-size $tf ${ids[$i]} 4194304
652 verify_comp_attr stripe-count $tf ${ids[$i]} 2
653 verify_comp_attr pool $tf ${ids[$i]} flash
654 verify_comp_extent $tf ${ids[$i]} 4194304 EOF
657 # verify component ${ids[7]}
658 verify_comp_attr stripe-size $tf ${ids[7]} 4194304
659 verify_comp_attr stripe-count $tf ${ids[7]} $OSTCOUNT
660 verify_comp_attr_with_parent pool $tf ${ids[7]}
661 verify_comp_extent $tf ${ids[7]} 0 EOF
663 # verify components ${ids[8]}, ${ids[10]} and ${ids[12]}
665 verify_comp_attr stripe-size $tf ${ids[$i]} 16777216
666 verify_comp_attr stripe-count $tf ${ids[$i]} $OSTCOUNT
667 verify_comp_attr pool $tf ${ids[$i]} archive
668 verify_comp_extent $tf ${ids[$i]} 0 536870912
671 # verify components ${ids[9]}, ${ids[11]} and ${ids[13]}
673 verify_comp_attr stripe-size $tf ${ids[$i]} 16777216
674 verify_comp_attr stripe-count $tf ${ids[$i]} -1
675 verify_comp_attr pool $tf ${ids[$i]} archive
676 verify_comp_extent $tf ${ids[$i]} 536870912 EOF
679 run_test 0f "lfs mirror extend composite layout mirrors"
684 ! $LFS mirror create --flags prefer $tf ||
685 error "creating $tf w/ --flags but w/o -N option should fail"
687 ! $LFS mirror create -N --flags foo $tf ||
688 error "creating $tf with '--flags foo' should fail"
690 ! $LFS mirror create -N --flags stale $tf ||
691 error "creating $tf with '--flags stale' should fail"
693 ! $LFS mirror create -N --flags prefer,init $tf ||
694 error "creating $tf with '--flags prefer,init' should fail"
696 ! $LFS mirror create -N --flags ^prefer $tf ||
697 error "creating $tf with '--flags ^prefer' should fail"
699 $LFS mirror create -N -E 1M -S 1M -o0 --flags=prefer -E eof -o1 \
700 -N -o1 $tf || error "create mirrored file $tf failed"
702 verify_comp_attr lcme_flags $tf 0x10001 prefer
703 verify_comp_attr lcme_flags $tf 0x10002 prefer
705 # write to the mirrored file and check primary
706 cp /etc/hosts $tf || error "error writing file '$tf'"
708 verify_comp_attr lcme_flags $tf 0x20003 stale
710 # resync file and check prefer flag
711 $LFS mirror resync $tf || error "error resync-ing file '$tf'"
714 $LCTL set_param osc.*.stats=clear
715 cat $tf &> /dev/null || error "error reading file '$tf'"
717 # verify that the data was provided by OST1 where mirror 1 resides
718 local nr_read=$($LCTL get_param -n osc.$FSNAME-OST0000-osc-[-0-9a-f]*.stats |
719 awk '/ost_read/{print $2}')
720 [ -n "$nr_read" ] || error "read was not provided by OST1"
722 run_test 0g "lfs mirror create flags support"
725 [ $MDS1_VERSION -lt $(version_code 2.11.57) ] &&
726 skip "Need MDS version at least 2.11.57"
733 # create parent directory
734 test_mkdir $td || error "mkdir $td failed"
736 $LFS setstripe -N -E 1M -S 1M --flags=prefer -E eof -N2 $td ||
737 error "set default mirrored layout on directory $td failed"
739 # verify flags are inherited from the directory
742 verify_comp_attr lcme_flags $tf 0x10001 prefer
743 verify_comp_attr lcme_flags $tf 0x10002 prefer
745 # set flags to the first component
746 ! $LFS setstripe --comp-set -I 0x10001 --comp-flags=^prefer,foo $tf ||
747 error "setting '^prefer,foo' flags should fail"
749 ! $LFS getstripe --component-flags=prefer,foo $tf ||
750 error "getting component(s) with 'prefer,foo' flags should fail"
752 $LFS setstripe --comp-set -I 0x10001 --comp-flags=^prefer,stale $tf
754 verify_comp_attr lcme_flags $tf 0x10001 stale
755 verify_comp_attr lcme_flags $tf 0x10002 prefer
757 $LFS setstripe --comp-set -I0x10001 --comp-flags=^stale $tf &&
758 error "clearing 'stale' should fail"
760 # write and resync file. It can't resync the file directly because the
761 # file state is still 'ro'
762 cp /etc/hosts $tf || error "error writing file '$tf'"
763 $LFS mirror resync $tf || error "error resync-ing file '$tf'"
765 $LFS setstripe --comp-set -I 0x20003 --comp-flags=prefer $tf ||
766 error "error setting flag prefer"
768 verify_comp_attr lcme_flags $tf 0x20003 prefer
770 $LFS setstripe --comp-set -I 0x20003 --comp-flags=^prefer $tf ||
771 error "error clearing prefer flag from component 0x20003"
773 # MDS disallows setting stale flag on the last non-stale mirror
774 [[ "$MDS1_VERSION" -ge $(version_code 2.12.57) ]] || return 0
776 cp /etc/hosts $tf || error "error writing file '$tf'"
778 verify_comp_attr lcme_flags $tf 0x10002 prefer
779 verify_comp_attr lcme_flags $tf 0x20003 stale
780 verify_comp_attr lcme_flags $tf 0x30004 stale
782 ! $LFS setstripe --comp-set -I 0x10002 --comp-flags=^prefer,stale $tf \
784 error "setting stale flag on component 0x10002 should fail"
786 $LFS mirror resync $tf || error "error resync-ing file '$tf'"
788 $LFS setstripe --comp-set -I 0x10001 --comp-flags=stale $tf ||
789 error "error setting stale flag on component 0x10001"
790 $LFS setstripe --comp-set -I 0x20003 --comp-flags=stale $tf ||
791 error "error setting stale flag on component 0x20003"
793 ! $LFS setstripe --comp-set -I 0x30004 --comp-flags=stale $tf \
795 error "setting stale flag on component 0x30004 should fail"
797 $LFS mirror resync $tf || error "error resync-ing file '$tf'"
799 run_test 0h "set, clear and test flags for FLR files"
802 $LFS mirror create -N2 $DIR/$tfile || error "create $DIR/$tfile failed"
804 cp /etc/hosts $DIR/$tfile || error "write to $DIR/$tfile failed"
805 $LFS mirror resync $DIR/$tfile || error "resync $DIR/$tfile failed"
806 cmp /etc/hosts $DIR/$tfile || error "cmp with /etc/hosts failed"
808 $LFS mirror read -N2 -o $TMP/$tfile $DIR/$tfile || error "read mirror failed"
809 stack_trap "rm -f $TMP/$tfile"
810 cmp $TMP/$tfile $DIR/$tfile || error "cmp with $TMP/$tfile failed"
811 $LFS mirror write -N2 -i /etc/passwd $DIR/$tfile || error "write failed"
812 $LFS setstripe --comp-set -I 65537 --comp-flags=stale $DIR/$tfile ||
813 error "set component 1 stale failed"
814 $LFS mirror resync $DIR/$tfile || error "resync $DIR/$tfile failed"
815 cmp /etc/passwd $DIR/$tfile || error "cmp with /etc/passwd failed"
817 run_test 0j "test lfs mirror read/write commands"
820 [[ $OSTCOUNT -lt 3 ]] && skip "need >= 3 OSTs" && return
823 # default FLR with 2 mirrors
824 $LFS setstripe -N -c1 -i0 -N -c1 -i1 $DIR/$tdir ||
825 error "set default layout failed"
827 # plain file extension under default FLR dir
828 echo $tfile >> $DIR/$tdir/$tfile.append || error "create $tfile.append failed"
829 echo -n " before extend $tfile.append, mirror count = "
830 $LFS getstripe -N $DIR/$tdir/$tfile.append
832 $LFS mirror extend -N -c1 -i2 $DIR/$tdir/$tfile.append ||
833 error "mirror extend failed"
834 echo -n " after extend $tfile.append, mirror count = "
835 $LFS getstripe -N $DIR/$tdir/$tfile.append
837 # normal file extension under default FLR dir
838 touch $DIR/$tdir/$tfile || error "create $tfile failed"
839 echo -n " before extend $tfile, mirror count = "
840 $LFS getstripe -N $DIR/$tdir/$tfile
842 $LFS mirror extend -N -c1 -i2 $DIR/$tdir/$tfile ||
843 error "mirror extend $tfile failed"
844 echo -n " after extend $tfile, mirror count = "
845 $LFS getstripe -N $DIR/$tdir/$tfile
847 run_test 0k "mirroring a file in directory with default FLR layout"
851 local mirror_count=16 # LUSTRE_MIRROR_COUNT_MAX
852 local mirror_create_cmd="$LFS mirror create"
853 local stripes[0]=$OSTCOUNT
855 mirror_create_cmd+=" -N -c ${stripes[0]}"
856 for ((i = 1; i < $mirror_count; i++)); do
857 # add mirrors with different stripes to the file
858 stripes[$i]=$((RANDOM % OSTCOUNT))
859 [ ${stripes[$i]} -eq 0 ] && stripes[$i]=1
861 mirror_create_cmd+=" -N -c ${stripes[$i]}"
864 $mirror_create_cmd $tf || error "create mirrored file $tf failed"
865 verify_mirror_count $tf $mirror_count
867 # can't create mirrors exceeding LUSTRE_MIRROR_COUNT_MAX
868 $LFS mirror extend -N $tf &&
869 error "Creating the $((mirror_count+1))th mirror succeeded"
871 local ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' |
874 # verify the range of components and stripe counts
875 for ((i = 0; i < $mirror_count; i++)); do
876 verify_comp_attr stripe-count $tf ${ids[$i]} ${stripes[$i]}
877 verify_comp_extent $tf ${ids[$i]} 0 EOF
880 run_test 1 "create components with setstripe options"
884 local tf2=$DIR/$tfile-2
886 $LFS setstripe -E 1M -S 1M -E EOF -c 1 $tf
887 $LFS setstripe -E 2M -S 1M -E EOF -c -1 $tf2
889 $LFS mirror extend -N -f $tf2 $tf ||
890 error "merging $tf2 into $tf failed"
892 verify_mirror_count $tf 2
893 [[ ! -e $tf2 ]] || error "$tf2 was not unlinked"
895 run_test 2 "create components from existing files"
898 [[ $MDSCOUNT -lt 2 ]] && skip "need >= 2 MDTs" && return
900 for ((i = 0; i < 2; i++)); do
901 $LFS mkdir -i $i $DIR/$tdir-$i
902 $LFS setstripe -E -1 $DIR/$tdir-$i/$tfile
905 $LFS mirror extend -N -f $DIR/$tdir-1/$tfile \
906 $DIR/$tdir-0/$tfile || error "creating mirrors"
908 # mdt doesn't support to cancel layout lock for remote objects, do
912 # make sure the mirrorted file was created successfully
913 [[ $($LFS getstripe --component-count $DIR/$tdir-0/$tfile) -eq 2 ]] ||
914 { $LFS getstripe $DIR/$tdir-0/$tfile;
915 error "expected 2 components"; }
920 run_test 3 "create components from files located on different MDTs"
923 local tf=$DIR/$tdir/$tfile
926 test_mkdir $DIR/$tdir
928 # set mirror with setstripe options to directory
929 $LFS mirror create -N2 -E 1M -S 1M -E eof $DIR/$tdir ||
930 error "set mirror to directory error"
932 [ x$($LFS getstripe -v $DIR/$tdir | awk '/lcm_flags/{print $2}') = \
933 x"mirrored" ] || error "failed to create mirrored dir"
936 verify_mirror_count $tf 2
938 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
939 verify_comp_extent $tf ${ids[0]} 0 1048576
940 verify_comp_extent $tf ${ids[1]} 1048576 EOF
942 # sub directory should inherit mirror setting from parent
943 test_mkdir $DIR/$tdir/td
944 [ x$($LFS getstripe -v $DIR/$tdir/td | awk '/lcm_flags/{print $2}') = \
945 x"mirrored" ] || error "failed to inherit mirror from parent"
947 # mirror extend won't be applied to directory
948 $LFS mirror extend -N2 $DIR/$tdir &&
949 error "expecting mirror extend failure"
952 run_test 4 "Make sure mirror attributes can be inhertied from directory"
958 $MULTIOP $tf oO_RDWR:O_CREAT:O_LOV_DELAY_CREATE:T12345c ||
959 error "failed to create file with non-empty layout"
960 $CHECKSTAT -t file -s 12345 $tf || error "size error: expecting 12345"
962 $LFS mirror create -N3 $tf || error "failed to attach mirror layout"
963 verify_mirror_count $tf 3
965 $CHECKSTAT -t file -s 12345 $tf ||
966 error "size error after attaching layout "
968 run_test 5 "Make sure init size work for mirrored layout"
971 (( $MDS1_VERSION >= $(version_code 2.12.58) )) ||
972 skip "MDS version older than 2.12.58"
976 $LFS mirror create -N -E 1M -L mdt -E eof -S 1M -N -E eof $tf ||
977 error "failure to create DoM file with mirror"
979 $LFS mirror create -N -E 1M -S 1M -E eof -N -E 1M -L mdt -E eof $tf ||
980 error "failure to create mirrored file with DoM"
982 $LFS setstripe -E 1M -L mdt -E eof -S 1M $tf ||
983 error "failure to create PFL with DoM file"
984 $LFS mirror extend -N2 $tf ||
985 error "failure to extend mirror with DoM"
987 $LFS setstripe -E 1M -L mdt -E eof -S 1M $tf-1 ||
988 error "failure to create PFL with DoM file"
989 $LFS mirror create -N2 -E 1M -S 1M -E eof $tf-2 ||
990 error "failure to create mirrored file"
991 $LFS mirror extend -N -f $tf-1 $tf-2 ||
992 error "failure to extend mirrored file with DoM extent"
994 run_test 6 "DoM and FLR work together"
999 # create DoM with setting stripe_size == component size
1000 $LFS mirror create -N -E1M -S1M -L mdt -Eeof $tf ||
1001 error "failure to create DoM with stripe_size == comp size"
1002 rm -f $tf || error "delete $tf"
1004 # DoM should not inherit previous component stripe_size
1005 $LFS mirror create -N -E4M -S2M -Eeof -N -E1M -L mdt -Eeof $tf ||
1006 error "DoM component shouldn't inherit previous stripe_size"
1008 run_test 7 "Create mirror with DoM component"
1011 local tf=$DIR/$tfile
1012 local tf2=$DIR/$tfile-2
1014 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
1016 stack_trap "rm -f $tf $tf2"
1018 $LFS setstripe -E EOF -o 0 $tf
1019 $LFS setstripe -E EOF -o 1 $tf2
1021 local dd_count=$((RANDOM % 20 + 1))
1022 dd if=/dev/zero of=$tf bs=1M count=$dd_count oflag=sync
1023 dd if=/dev/zero of=$tf2 bs=1M count=1 seek=$((dd_count - 1)) oflag=sync
1025 # for zfs - sync OST dataset so that du below will return
1027 [ "$FSTYPE" = "zfs" ] &&
1028 do_nodes $(comma_list $(osts_nodes)) "$ZPOOL sync"
1030 local blocks=$(du -kc $tf $tf2 | awk '/total/{print $1}')
1033 $LFS mirror extend -N -f $tf2 $tf ||
1034 error "merging $tf2 into $tf failed"
1036 # cancel layout lock
1037 cancel_lru_locks mdc
1039 local new_blocks=$(du -k $tf | awk '{print $1}')
1040 [ $new_blocks -eq $blocks ] ||
1041 error "i_blocks error expected: $blocks, actual: $new_blocks"
1043 run_test 21 "glimpse should report accurate i_blocks"
1045 get_osc_lock_count() {
1052 osc_name=${FSNAME}-OST$(printf "%04x" $((idx-1)))-osc-'[-0-9a-f]*'
1053 count=$($LCTL get_param -n ldlm.namespaces.$osc_name.lock_count)
1054 lock_count=$((lock_count + count))
1060 local tf=$DIR/$tfile
1062 stack_trap "rm -f $tf"
1064 $LFS setstripe -E EOF -o 0 $tf
1065 dd if=/dev/zero of=$tf bs=1M count=$((RANDOM % 20 + 1))
1067 # add component, two mirrors located on the same OST ;-)
1068 $LFS mirror extend -N -o 0 $tf ||
1069 error "extending mirrored file $tf failed"
1071 size_blocks=$(stat --format="%b %s" $tf)
1073 cancel_lru_locks mdc
1074 cancel_lru_locks osc
1076 local new_size_blocks=$(stat --format="%b %s" $tf)
1078 # make sure there is no lock cached
1079 [ $(get_osc_lock_count 1) -eq 0 ] || error "glimpse requests were sent"
1081 [ "$new_size_blocks" = "$size_blocks" ] ||
1082 echo "size expected: $size_blocks, actual: $new_size_blocks"
1086 run_test 22 "no glimpse to OSTs for READ_ONLY files"
1089 local tf=$DIR/$tfile
1091 $LFS mirror create -N -o 0 -N -o 1 $tf ||
1092 error "creating mirrored file $tf failed"
1095 for ((ost_idx = 1; ost_idx <= 2; ost_idx++)); do
1096 cancel_lru_locks osc
1099 local tmpfile=$(mktemp)
1100 stat --format="%b %s" $tf > $tmpfile &
1104 while [ $cnt -le 5 ]; do
1105 kill -0 $pid > /dev/null 2>&1 || break
1109 kill -0 $pid > /dev/null 2>&1 &&
1110 error "stat process stuck due to unavailable OSTs"
1112 # make sure glimpse request has been sent
1113 [ $(get_osc_lock_count 1 2) -ne 0 ] ||
1114 error "OST $ost_idx: no glimpse request was sent"
1119 run_test 31 "make sure glimpse request can be retried"
1122 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
1123 rm -f $DIR/$tfile $DIR/$tfile-2
1125 stack_trap "rm -f $DIR/$tfile"
1127 $LFS setstripe -E EOF -o 0 $DIR/$tfile
1128 dd if=/dev/urandom of=$DIR/$tfile bs=1M count=$((RANDOM % 10 + 2))
1130 local fsize=$(stat -c %s $DIR/$tfile)
1131 [[ $fsize -ne 0 ]] || error "file size is (wrongly) zero"
1133 local cksum=$(md5sum $DIR/$tfile)
1135 # create a new mirror in sync mode
1136 $LFS mirror extend -N -o 1 $DIR/$tfile ||
1137 error "extending mirrored file $DIR/$tfile failed"
1139 # make sure the mirrored file was created successfully
1140 [ $($LFS getstripe -N $DIR/$tfile) -eq 2 ] ||
1141 { $LFS getstripe $DIR/$tfile; error "expected 2 mirrors"; }
1146 # check size is correct, glimpse request should go to the 2nd mirror
1147 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
1148 error "file size error $fsize vs. $(stat -c %s $DIR/$tfile)"
1150 echo "reading file from the 2nd mirror and verify checksum"
1151 [[ "$cksum" == "$(md5sum $DIR/$tfile)" ]] ||
1152 error "checksum error: expected $cksum"
1156 run_test 32 "data should be mirrored to newly created mirror"
1159 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
1161 rm -f $DIR/$tfile $DIR/$tfile-2
1163 # create a file with two mirrors
1164 $LFS setstripe -E EOF -o 0 $DIR/$tfile
1167 while [ $count -lt $max_count ]; do
1168 echo "ost1" >> $DIR/$tfile
1169 count=$((count + 1));
1172 # tmp file that will be used as mirror
1173 $LFS setstripe -E EOF -o 1 $DIR/$tfile-2
1175 while [ $count -lt $max_count ]; do
1176 echo "ost2" >> $DIR/$tfile-2
1177 count=$((count + 1));
1180 # create a mirrored file
1181 $LFS mirror extend -N -f $DIR/$tfile-2 $DIR/$tfile &&
1182 error "merging $DIR/$tfile-2 into $DIR/$tfile" \
1183 "with verification should fail"
1184 $LFS mirror extend --no-verify -N -f $DIR/$tfile-2 $DIR/$tfile ||
1185 error "merging $DIR/$tfile-2 into $DIR/$tfile" \
1186 "without verification failed"
1188 # make sure that $tfile has two mirrors and $tfile-2 does not exist
1189 [ $($LFS getstripe -N $DIR/$tfile) -eq 2 ] ||
1190 { $LFS getstripe $DIR/$tfile; error "expected count 2"; }
1192 [[ ! -e $DIR/$tfile-2 ]] || error "$DIR/$tfile-2 was not unlinked"
1194 # execpted file size
1195 local fsize=$((5 * max_count))
1196 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
1197 error "mirrored file size is not $fsize"
1199 # read file - all OSTs are available
1200 echo "reading file (data can be provided by any ost)... "
1201 local rs=$(cat $DIR/$tfile | head -1)
1202 [[ "$rs" == "ost1" || "$rs" == "ost2" ]] ||
1203 error "file content error: expected: \"ost1\" or \"ost2\""
1205 # read file again with ost1 failed
1209 echo "reading file (data should be provided by ost2)..."
1210 local rs=$(cat $DIR/$tfile | head -1)
1211 [[ "$rs" == "ost2" ]] ||
1212 error "file content error: expected: \"ost2\", actual: \"$rs\""
1217 # read file again with ost2 failed
1221 # check size, glimpse should work
1222 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
1223 error "mirrored file size is not $fsize"
1225 echo "reading file (data should be provided by ost1)..."
1226 local rs=$(cat $DIR/$tfile | head -1)
1227 [[ "$rs" == "ost1" ]] ||
1228 error "file content error: expected: \"ost1\", actual: \"$rs\""
1232 run_test 33a "read can choose available mirror to read"
1235 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
1239 stack_trap "rm -f $DIR/$tfile" EXIT
1241 # create a file with two mirrors on OST0000 and OST0001
1242 $LFS setstripe -N -Eeof -o0 -N -Eeof -o1 $DIR/$tfile
1244 # make sure that $tfile has two mirrors
1245 [ $($LFS getstripe -N $DIR/$tfile) -eq 2 ] ||
1246 { $LFS getstripe $DIR/$tfile; error "expected count 2"; }
1249 dd if=/dev/urandom of=$DIR/$tfile bs=2M count=25 ||
1250 error "write failed for $DIR/$tfile"
1251 $LFS mirror resync $DIR/$tfile || error "resync failed for $DIR/$tfile"
1252 verify_flr_state $DIR/$tfile "ro"
1257 # read file - all OSTs are available
1258 echo "reading file (data can be provided by any ost)... "
1260 time cat $DIR/$tfile > /dev/null || error "read all"
1264 # read file again with ost1 {OST0000} failed
1267 echo "reading file (data should be provided by ost2)..."
1269 time cat $DIR/$tfile > /dev/null || error "read ost2"
1276 # read file again with ost2 {OST0001} failed
1280 echo "reading file (data should be provided by ost1)..."
1282 time cat $DIR/$tfile > /dev/null || error "read ost1"
1289 [ $((r1 * 100)) -gt $((ra * 105)) -a $r1 -gt $((ra + 2)) ] &&
1290 error "read mirror too slow without ost1, from $ra to $r1"
1291 [ $((r2 * 100)) -gt $((ra * 105)) -a $r2 -gt $((ra + 2)) ] &&
1292 error "read mirror too slow without ost2, from $ra to $r2"
1294 wait_osc_import_ready client ost2
1296 run_test 33b "avoid reading from unhealthy mirror"
1299 [[ $OSTCOUNT -lt 3 ]] && skip "need >= 3 OSTs" && return
1303 stack_trap "rm -f $DIR/$tfile" EXIT
1305 # create a file with two mirrors
1306 # mirror1: {OST0000, OST0001}
1307 # mirror2: {OST0001, OST0002}
1308 $LFS setstripe -N -Eeof -c2 -o0,1 -N -Eeof -c2 -o1,2 $DIR/$tfile
1310 # make sure that $tfile has two mirrors
1311 [ $($LFS getstripe -N $DIR/$tfile) -eq 2 ] ||
1312 { $LFS getstripe $DIR/$tfile; error "expected count 2"; }
1315 dd if=/dev/urandom of=$DIR/$tfile bs=2M count=25 ||
1316 error "write failed for $DIR/$tfile"
1317 $LFS mirror resync $DIR/$tfile || error "resync failed for $DIR/$tfile"
1318 verify_flr_state $DIR/$tfile "ro"
1323 # read file - all OSTs are available
1324 echo "reading file (data can be provided by any ost)... "
1325 time cat $DIR/$tfile > /dev/null || error "read all"
1327 # read file again with ost2 (OST0001) failed
1331 echo "reading file (data should be provided by ost1 and ost3)..."
1332 time cat $DIR/$tfile > /dev/null || error "read ost1 & ost3"
1337 wait_osc_import_ready client ost2
1339 run_test 33c "keep reading among unhealthy mirrors"
1342 (( $OSTCOUNT >= 4 )) || skip "need >= 4 OSTs"
1344 stack_trap "rm -f $DIR/$tfile $DIR/$tfile-2 $DIR/$tfile-ref"
1347 $LFS setstripe -o 0 $DIR/$tfile-ref
1348 dd if=/dev/urandom of=$DIR/$tfile-ref bs=1M count=3
1350 # create a file with two mirrors
1351 $LFS setstripe -E -1 -o 0,1 -S 1M $DIR/$tfile
1352 dd if=$DIR/$tfile-ref of=$DIR/$tfile bs=1M
1354 $LFS setstripe -E -1 -o 2,3 -S 1M $DIR/$tfile-2
1355 dd if=$DIR/$tfile-ref of=$DIR/$tfile-2 bs=1M
1357 $CHECKSTAT -t file -s $((3 * 1024 * 1024)) $DIR/$tfile ||
1358 error "mirrored file size is not 3M"
1360 # merge a mirrored file
1361 $LFS mirror extend -N -f $DIR/$tfile-2 $DIR/$tfile ||
1362 error "merging $DIR/$tfile-2 into $DIR/$tfile failed"
1364 cancel_lru_locks osc
1366 # stop two OSTs, so the 2nd stripe of the 1st mirror and
1367 # the 1st stripe of the 2nd mirror will be inaccessible, ...
1370 echo "comparing files ... "
1372 # however, read can still return the correct data. It should return
1373 # the 1st stripe from mirror 1 and 2st stripe from mirror 2.
1374 cmp -n 2097152 <(rwv -f $DIR/$tfile -r -o -n 1 2097152) \
1375 $DIR/$tfile-ref || error "file reading error"
1379 run_test 34a "read mirrored file with multiple stripes"
1382 (( $OSTCOUNT >= 4 )) || skip "need >= 4 OSTs"
1384 stack_trap "rm -f $DIR/$tfile $DIR/$tfile-2 $DIR/$tfile-ref"
1387 $LFS setstripe -o 0 $DIR/$tfile-ref
1388 dd if=/dev/urandom of=$DIR/$tfile-ref bs=1M count=3
1390 $LFS setstripe -E 1M -S 1M -o 0 -E eof -o 1 $DIR/$tfile
1391 dd if=$DIR/$tfile-ref of=$DIR/$tfile bs=1M
1393 $LFS setstripe -E 1M -S 1M -o 2 -E eof -o 3 $DIR/$tfile-2
1394 dd if=$DIR/$tfile-ref of=$DIR/$tfile-2 bs=1M
1396 $CHECKSTAT -t file -s $((3 * 1024 * 1024)) $DIR/$tfile ||
1397 error "mirrored file size is not 3M"
1399 # merge a mirrored file
1400 $LFS mirror extend -N -f $DIR/$tfile-2 $DIR/$tfile ||
1401 error "merging $DIR/$tfile-2 into $DIR/$tfile failed"
1403 cancel_lru_locks osc
1405 # stop two OSTs, so the 2nd component of the 1st mirror and
1406 # the 1st component of the 2nd mirror will be inaccessible, ...
1409 echo "comparing files ... "
1411 # however, read can still return the correct data. It should return
1412 # the 1st stripe from mirror 1 and 2st stripe from mirror 2.
1413 cmp -n 2097152 <(rwv -f $DIR/$tfile -r -o -n 1 2097152) \
1414 $DIR/$tfile-ref || error "file reading error"
1418 run_test 34b "read mirrored file with multiple components"
1421 local tf=$DIR/$tfile
1423 $LFS setstripe -E eof $tf
1425 # add an out-of-sync mirror to the file
1426 $LFS mirror extend -N -c 2 $tf ||
1427 error "extending mirrored file $tf failed"
1429 $MULTIOP $tf oO_WRONLY:c ||
1430 error "write open a mirrored file failed"
1432 # truncate file should return error
1433 $TRUNCATE $tf 100 || error "error truncating a mirrored file"
1435 run_test 35 "allow to write to mirrored files"
1437 get_file_layout_version() {
1438 $LFS getstripe $1 | awk '/lcm_layout_gen/{print $2}'
1441 get_ost_layout_version() {
1442 $MULTIOP $1 oXc | awk '/ostlayoutversion/{print $2}'
1445 verify_ost_layout_version() {
1448 # get file layout version
1449 local flv=$(get_file_layout_version $tf)
1451 # layout version from OST objects
1452 local olv=$(get_ost_layout_version $tf)
1454 (( flv >= olv )) || error "layout version mismatch: $flv vs. $olv"
1461 $LFS setstripe -E 1M -S 1M -E 2M -E 4M -E eof -c -1 $tf
1462 $LFS setstripe -E 3M -S 1M -E 6M -E eof -c -1 $tf-tmp
1464 $LFS mirror extend -N -f $tf-tmp $tf ||
1465 error "merging $tf-tmp into $tf failed"
1470 local tf=$DIR/$tfile
1472 stack_trap "rm -f $tf $tf-2 $tf-3"
1474 create_file_36 $tf $tf-2 $tf-3
1476 [ $($LFS getstripe -N $tf) -gt 1 ] || error "wrong mirror count"
1478 # test case 1 - check file write and verify layout version
1479 $MULTIOP $tf oO_WRONLY:c ||
1480 error "write open a mirrored file failed"
1482 # write open file should not return error
1483 $MULTIOP $tf oO_WRONLY:w1024Yc || error "write mirrored file error"
1485 # instantiate components should work
1486 dd if=/dev/zero of=$tf bs=1M count=12 || error "write file error"
1488 # verify OST layout version
1489 verify_ost_layout_version $tf
1492 local mds_facet=mds$(($($LFS getstripe -m $tf-2) + 1))
1495 do_facet $mds_facet $LCTL set_param fail_val=$delay_sec
1497 #define OBD_FAIL_FLR_LV_DELAY 0x1A01
1498 do_facet $mds_facet $LCTL set_param fail_loc=0x1A01
1500 # write should take at least $fail_loc seconds and succeed
1501 local st=$(date +%s)
1502 $MULTIOP $tf-2 oO_WRONLY:w1024Yc || error "write mirrored file error"
1504 # verify OST layout version
1505 verify_ost_layout_version $tf
1507 do_facet $mds_facet $LCTL set_param fail_loc=0
1509 run_test 36a "write to mirrored files"
1512 local tf=$DIR/$tfile
1514 (( OST1_VERSION >= $(version_code 2.15.50) )) ||
1515 skip "Need OST version at least 2.15.50"
1517 (( OSTCOUNT >= 2 )) || skip "need >= 2 OSTs"
1519 # create 2 mirrors using different OSTs
1520 $LFS setstripe -N -c1 -i0 --flags=prefer -N -c1 -i1 $tf ||
1521 error "create mirrored file"
1523 # write 1M data to one mirror
1524 dd if=/dev/zero of=$tf bs=1M count=1 || error "write file error"
1527 # set prefer mirror to another mirror
1528 $LFS setstripe --comp-set -I0x10001 --comp-flags=^prefer $tf ||
1529 error "clear prefer mirror error"
1530 $LFS setstripe --comp-set -I0x20002 --comp-flags=prefer $tf ||
1531 error "set prefer mirror error"
1533 # the second write should not hung
1534 dd if=/dev/zero of=$tf bs=1M count=1 || error "write file error"
1536 run_test 36b "write should not hung when prefered mirror is stale"
1539 local tf=$DIR/$tfile
1541 (( OST1_VERSION >= $(version_code 2.15.50) )) ||
1542 skip "Need OST version at least 2.15.50"
1544 (( OSTCOUNT >= 2 )) || skip "need >= 2 OSTs"
1546 # create 2 mirrors using different OSTs
1547 $LFS setstripe -N -c1 -i0 --flags=prefer -N -c1 -i1 $tf ||
1548 error "create mirrored file"
1549 stack_trap "rm -f $tf"
1551 # write it in the background
1552 dd if=/dev/zero of=$tf bs=1M count=600 &
1557 $LFS setstripe --comp-set -I0x10001 --comp-flags=^prefer $tf ||
1558 error "clear prefer mirror error"
1559 $LFS setstripe --comp-set -I0x20002 --comp-flags=prefer $tf ||
1560 error "set prefer mirror error"
1564 run_test 36c "change prefer mirror during write shouldn't hung"
1567 local tf=$DIR/$tfile
1569 (( OST1_VERSION >= $(version_code 2.15.50) )) ||
1570 skip "Need OST version at least 2.15.50"
1572 echo " ** create $tf"
1573 $LFS mirror create -N $tf || error "create $tf failed"
1576 echo " ** mirror extend $tf ($i/2)"
1577 $LFS mirror extend -N $tf || error "mirror extend $tf failed"
1578 flv=$(get_file_layout_version $tf)
1579 olv=$(get_ost_layout_version $tf)
1580 echo " flv=$flv olv=$olv"
1584 echo " ** write $tf ($i/2)"
1585 dd if=/dev/zero of=$tf bs=1k count=1 || error "write $tf failed"
1586 flv=$(get_file_layout_version $tf)
1587 olv=$(get_ost_layout_version $tf)
1588 echo " flv=$flv olv=$olv"
1590 error "write update OST layout failed $flv/$olv"
1593 echo " ** resync $tf"
1594 $LFS mirror resync $tf || error "mirror resync $tf failed"
1595 flv=$(get_file_layout_version $tf)
1596 olv=$(get_ost_layout_version $tf)
1597 echo " flv=$flv olv=$olv"
1600 echo " ** truncate $tf ($i/2)"
1601 $TRUNCATE $tf $((1024 * 1024)) || error "truncate $tf fails"
1602 flv=$(get_file_layout_version $tf)
1603 olv=$(get_ost_layout_version $tf)
1604 echo " flv=$flv olv=$olv"
1605 (( flv == olv || flv == olv + 1 )) ||
1606 error "truncate update OST layout failed $flv/$olv"
1609 echo " ** resync $tf"
1610 $LFS mirror resync $tf || error "mirror resync $tf failed"
1611 flv=$(get_file_layout_version $tf)
1612 olv=$(get_ost_layout_version $tf)
1613 echo " flv=$flv olv=$olv"
1616 echo " ** write $tf ($i/2)"
1617 dd if=/dev/zero of=$tf bs=1k count=1 || error "write $tf failed"
1618 flv=$(get_file_layout_version $tf)
1619 olv=$(get_ost_layout_version $tf)
1620 echo " flv=$flv olv=$olv"
1622 error "write update OST layout failed $flv/$olv"
1625 run_test 36d "write/punch FLR file update OST layout version"
1631 echo "create test files with size $fsize .."
1635 $LFS setstripe -E 1M -S 1M -c 1 -E eof -c -1 $tf
1637 dd if=/dev/urandom of=$tf bs=1M count=16 &> /dev/null
1638 $TRUNCATE $tf $fsize
1644 [ $MDS1_VERSION -lt $(version_code 2.11.57) ] &&
1645 skip "Need MDS version at least 2.11.57"
1647 local tf=$DIR/$tfile
1648 local tf2=$DIR/$tfile-2
1649 local tf3=$DIR/$tfile-3
1650 local tf4=$DIR/$tfile-4
1652 stack_trap "rm -f $tf $tf2 $tf3 $tf4"
1654 create_files_37 $((RANDOM + 15 * 1048576)) $tf $tf2 $tf3
1657 # assume the mirror id will be 1, 2, and 3
1658 declare -A checksums
1659 checksums[1]=$(cat $tf | md5sum)
1660 checksums[2]=$(cat $tf2 | md5sum)
1661 checksums[3]=$(cat $tf3 | md5sum)
1663 printf '%s\n' "${checksums[@]}"
1665 # merge these files into a mirrored file
1666 $LFS mirror extend --no-verify -N -f $tf2 $tf ||
1667 error "merging $tf2 into $tf failed"
1668 $LFS mirror extend --no-verify -N -f $tf3 $tf ||
1669 error "merging $tf3 into $tf failed"
1673 # verify mirror read, checksums should equal to the original files'
1674 echo "Verifying mirror read .."
1677 for i in "${mirror_array[@]}"; do
1678 $LCTL set_param ldlm.namespaces.*.lru_size=clear > /dev/null
1679 sum=$($LFS mirror read -N $i $tf | md5sum)
1680 [ "$sum" = "${checksums[$i]}" ] ||
1681 error "$i: mismatch: \'${checksums[$i]}\' vs. \'$sum\'"
1684 # verify mirror write
1685 echo "Verifying mirror write .."
1686 $LFS mirror write -N2 $tf < $tf4
1688 sum=$($LFS mirror read -N2 $tf | md5sum)
1689 [[ "$sum" = "${checksums[1]}" ]] ||
1690 error "2: mismatch \'${checksums[1]}\' vs. \'$sum\'"
1692 # verify mirror copy, write to this mirrored file will invalidate
1693 # the other two mirrors
1694 echo "Verifying mirror copy .."
1696 local osts=$(comma_list $(osts_nodes))
1698 $LFS mirror copy -i ${mirror_array[0]} -o-1 $tf ||
1699 error "mirror copy error"
1701 # verify copying is successful by checking checksums
1702 remount_client $MOUNT
1703 for i in "${mirror_array[@]}"; do
1704 sum=$($LFS mirror read -N $i $tf | md5sum)
1705 [ "$sum" = "${checksums[1]}" ] ||
1706 error "$i: mismatch checksum after copy \'$sum\'"
1711 run_test 37 "mirror I/O API verification"
1714 local tf=$DIR/$tfile
1715 local ref=$DIR/${tfile}-ref
1717 stack_trap "rm -f $tf $ref"
1719 $LFS setstripe -E 1M -S 1M -c 1 -E 4M -c 2 -E eof -c -1 $tf ||
1720 error "creating $tf failed"
1721 $LFS setstripe -E 2M -S 1M -c 1 -E 6M -c 2 -E 8M -c -1 -E eof -c -1 \
1722 $tf-2 || error "creating $tf-2 failed"
1723 $LFS setstripe -E 4M -c 1 -E 8M -c 2 -E eof -c -1 $tf-3 ||
1724 error "creating $tf-3 failed"
1726 # instantiate all components
1727 $LFS mirror extend -N -f $tf-2 $tf ||
1728 error "merging $tf-2 into $tf failed"
1729 $LFS mirror extend -N -f $tf-3 $tf ||
1730 error "merging $tf-3 into $tf failed"
1731 $LFS mirror extend -N -c 1 $tf ||
1732 error "extending mirrored file $tf failed"
1734 verify_flr_state $tf "ro"
1736 dd if=/dev/urandom of=$ref bs=1M count=16 &> /dev/null
1738 local fsize=$((RANDOM << 8 + 1048576))
1739 $TRUNCATE $ref $fsize
1741 local ref_cksum=$(cat $ref | md5sum)
1743 # case 1: verify write to mirrored file & resync work
1744 cp $ref $tf || error "copy from $ref to $f error"
1745 verify_flr_state $tf "wp"
1747 local file_cksum=$(cat $tf | md5sum)
1748 [ "$file_cksum" = "$ref_cksum" ] || error "write failed, cksum mismatch"
1751 echo "mirror IDs: ${mirror_array[*]}"
1753 local valid_mirror stale_mirror id mirror_cksum
1754 for id in "${mirror_array[@]}"; do
1755 mirror_cksum=$($LFS mirror read -N $id $tf | md5sum)
1756 [ "$ref_cksum" == "$mirror_cksum" ] &&
1757 { valid_mirror=$id; continue; }
1762 [ -z "$stale_mirror" ] && error "stale mirror doesn't exist"
1763 [ -z "$valid_mirror" ] && error "valid mirror doesn't exist"
1765 $LFS mirror resync $tf || error "resync failed"
1766 verify_flr_state $tf "ro"
1768 mirror_cksum=$($LFS mirror read -N $stale_mirror $tf | md5sum)
1769 [ "$file_cksum" = "$ref_cksum" ] || error "resync failed"
1771 # case 2: inject an error to make mirror_io exit after changing
1772 # the file state to sync_pending so that we can start a concurrent
1774 $MULTIOP $tf oO_WRONLY:w$((RANDOM % 1048576 + 1024))c
1775 verify_flr_state $tf "wp"
1777 mirror_io resync -e resync_start $tf && error "resync succeeded"
1778 verify_flr_state $tf "sp"
1780 # from sync_pending to write_pending
1781 $MULTIOP $tf oO_WRONLY:w$((RANDOM % 1048576 + 1024))c
1782 verify_flr_state $tf "wp"
1784 mirror_io resync -e resync_start $tf && error "resync succeeded"
1785 verify_flr_state $tf "sp"
1787 # from sync_pending to read_only
1788 $LFS mirror resync $tf || error "resync failed"
1789 verify_flr_state $tf "ro"
1791 run_test 38 "resync"
1794 local tf=$DIR/$tfile
1797 $LFS mirror create -N2 -E1m -c1 -S1M -E-1 $tf ||
1798 error "create PFL file $tf failed"
1800 verify_mirror_count $tf 2
1801 verify_comp_count $tf 4
1803 rm -f $tf || error "delete $tf failed"
1805 run_test 39 "check FLR+PFL (a.k.a. PFLR) creation"
1808 local tf=$DIR/$tfile
1811 for ops in "conv=notrunc" ""; do
1814 $LFS mirror create -N -E 2M -S 1M -E 4M -E -1 --flags=prefer \
1815 -N -E 1M -E 2M -E 4M -E -1 $tf ||
1816 error "create PFLR file $tf failed"
1817 dd if=/dev/zero of=$tf $ops bs=1M seek=2 count=1 ||
1818 error "write PFLR file $tf failed"
1820 lfs getstripe -vy $tf
1824 # file mirror state should be write_pending
1825 flags=$($LFS getstripe -v $tf | awk '/lcm_flags:/ { print $2 }')
1827 error "file mirror state $flags"
1828 # the 1st component (in mirror 1) should be inited
1829 verify_comp_attr lcme_flags $tf 0x10001 init
1830 # the 2nd component (in mirror 1) should be inited
1831 verify_comp_attr lcme_flags $tf 0x10002 init
1832 # the 3rd component (in mirror 1) should be uninited
1833 verify_comp_attr lcme_flags $tf 0x10003 prefer
1834 # the 4th component (in mirror 2) should be inited
1835 verify_comp_attr lcme_flags $tf 0x20004 init
1836 # the 5th component (in mirror 2) should be uninited
1837 verify_comp_attr lcme_flags $tf 0x20005 0
1838 # the 6th component (in mirror 2) should be stale
1839 verify_comp_attr lcme_flags $tf 0x20006 stale
1840 # the 7th component (in mirror 2) should be uninited
1841 if [[ x$ops = "xconv=notrunc" ]]; then
1842 verify_comp_attr lcme_flags $tf 0x20007 0
1843 elif [[ x$ops = "x" ]]; then
1844 verify_comp_attr lcme_flags $tf 0x20007 stale
1848 rm -f $tf || error "delete $tf failed"
1850 run_test 40 "PFLR rdonly state instantiation check"
1853 local tf=$DIR/$tfile
1855 stack_trap "rm -f $tf $tf-1"
1858 echo " **create two FLR files $tf $tf-1"
1859 $LFS mirror create -N -E 2M -S 1M -E 4M -E -1 \
1860 -N -E 1M -E 2M -E 3M -E -1 $tf ||
1861 error "create PFLR file $tf failed"
1862 $LFS mirror create -N -E 2M -S 1M -E eof \
1863 -N -E 1M -E eof --flags prefer \
1864 -N -E 4m -E eof $tf-1 ||
1865 error "create PFLR file $tf-1 failed"
1867 # file should be in ro status
1868 echo " **verify files be RDONLY"
1869 verify_flr_state $tf "ro"
1870 verify_flr_state $tf-1 "ro"
1872 # write data in [0, 2M)
1873 dd if=/dev/zero of=$tf bs=1M count=2 conv=notrunc ||
1874 error "writing $tf failed"
1875 dd if=/dev/urandom of=$tf-1 bs=1M count=4 conv=notrunc ||
1876 error "writing $tf-1 failed"
1878 local sum0=$(cat $tf-1 | md5sum)
1880 echo " **verify files be WRITE_PENDING"
1881 verify_flr_state $tf "wp"
1882 verify_flr_state $tf-1 "wp"
1884 # file should have stale component
1885 echo " **verify files have stale component"
1886 $LFS getstripe $tf | grep lcme_flags | grep stale > /dev/null ||
1887 error "after writing $tf, it does not contain stale component"
1888 $LFS getstripe $tf-1 | grep lcme_flags | grep stale > /dev/null ||
1889 error "after writing $tf-1, it does not contain stale component"
1891 echo " **full resync"
1892 $LFS mirror resync $tf $tf-1 || error "mirror resync $tf $tf-1 failed"
1894 echo " **verify $tf-1 data consistency in all mirrors"
1896 local sum=$($LFS mirror read -N$i $tf-1 | md5sum)
1897 [[ "$sum" = "$sum0" ]] ||
1898 error "$tf-1.$i: checksum mismatch: $sum != $sum0"
1901 echo " **verify files be RDONLY"
1902 verify_flr_state $tf "ro"
1903 verify_flr_state $tf-1 "ro"
1905 # file should not have stale component
1906 echo " **verify files do not contain stale component"
1907 $LFS getstripe $tf | grep lcme_flags | grep stale &&
1908 error "after resyncing $tf, it contains stale component"
1909 $LFS getstripe $tf-1 | grep lcme_flags | grep stale &&
1910 error "after resyncing $tf, it contains stale component"
1912 # verify partial resync
1913 echo " **write $tf-1 for partial resync test"
1914 dd if=/dev/zero of=$tf-1 bs=1M count=2 conv=notrunc ||
1915 error "writing $tf-1 failed"
1917 echo " **only resync mirror 2"
1918 verify_flr_state $tf-1 "wp"
1919 $LFS mirror resync --only 2 $tf-1 ||
1920 error "resync mirror 2 of $tf-1 failed"
1921 verify_flr_state $tf "ro"
1923 # resync synced mirror
1924 echo " **resync mirror 2 again"
1925 $LFS mirror resync --only 2 $tf-1 ||
1926 error "resync mirror 2 of $tf-1 failed"
1927 verify_flr_state $tf "ro"
1928 echo " **verify $tf-1 contains stale component"
1929 $LFS getstripe $tf-1 | grep lcme_flags | grep stale > /dev/null ||
1930 error "after writing $tf-1, it does not contain stale component"
1932 echo " **full resync $tf-1"
1933 $LFS mirror resync $tf-1 || error "resync of $tf-1 failed"
1934 verify_flr_state $tf "ro"
1935 echo " **full resync $tf-1 again"
1936 $LFS mirror resync $tf-1 || error "resync of $tf-1 failed"
1937 echo " **verify $tf-1 does not contain stale component"
1938 $LFS getstripe $tf | grep lcme_flags | grep stale &&
1939 error "after resyncing $tf, it contains stale component"
1943 run_test 41 "lfs mirror resync check"
1946 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
1950 local mirror_cmd="$LFS mirror verify"
1953 stack_trap "rm -rf $td"
1955 # create parent directory
1956 mkdir $td || error "mkdir $td failed"
1958 $mirror_cmd &> /dev/null && error "no file name given"
1959 $mirror_cmd $tf &> /dev/null && error "cannot stat file $tf"
1960 $mirror_cmd $td &> /dev/null && error "$td is not a regular file"
1962 # create mirrored files
1963 $LFS mirror create -N -E 4M -S 1M -E 10M -E EOF $tf ||
1964 error "create mirrored file $tf failed"
1965 $LFS mirror create -N -E 2M -S 1M -E EOF \
1966 -N -E 6M -E 8M -E EOF \
1967 -N -E 16M -E EOF $tf-1 ||
1968 error "create mirrored file $tf-1 failed"
1969 $LFS mirror create -N -c 2 -o 1,3 -N -S 2M -c -1 $tf-2 ||
1970 error "create mirrored file $tf-2 failed"
1972 # write data in [0, 10M)
1973 for i in $tf $tf-1 $tf-2; do
1974 yes | dd of=$i bs=1M count=10 iflag=fullblock conv=notrunc ||
1975 error "write $i failed"
1978 # resync the mirrored files
1979 $LFS mirror resync $tf-1 $tf-2 ||
1980 error "resync $tf-1 $tf-2 failed"
1982 # verify the mirrored files
1983 $mirror_cmd $tf-1 $tf-2 ||
1984 error "verify $tf-1 $tf-2 failed"
1986 get_mirror_ids $tf-1
1987 $mirror_cmd --only ${mirror_array[0]} $tf-1 &> /dev/null &&
1988 error "at least 2 mirror ids needed with '--only' option"
1989 $mirror_cmd --only ${mirror_array[0]},${mirror_array[1]} $tf-1 $tf-2 \
1991 error "'--only' option cannot be used upon multiple files"
1992 $mirror_cmd --only 65534,${mirror_array[0]},65535 $tf-1 &&
1993 error "invalid specified mirror ids"
1995 # change the content of $tf and merge it into $tf-1
1997 echo a | dd of=$tf bs=1M seek=$i conv=notrunc ||
1998 error "change $tf with seek=$i failed"
1999 echo b | dd of=$tf-1 bs=1M seek=$i conv=notrunc ||
2000 error "change $tf-1 with seek=$i failed"
2003 $LFS mirror resync $tf-1 || error "resync $tf-1 failed"
2004 $LFS mirror extend --no-verify -N -f $tf $tf-1 ||
2005 error "merge $tf into $tf-1 failed"
2007 # verify the mirrored files
2008 echo "Verify $tf-1 without -v option:"
2009 $mirror_cmd $tf-1 &&
2010 error "verify $tf-1 should fail" || echo "PASS"
2012 echo "Verify $tf-1 with -v option:"
2013 $mirror_cmd -v $tf-1 &&
2014 error "verify $tf-1 should fail"
2016 get_mirror_ids $tf-1
2017 echo "Verify $tf-1 with --only option:"
2018 $mirror_cmd -v --only ${mirror_array[1]},${mirror_array[-1]} $tf-1 &&
2019 error "verify $tf-1 with mirror ${mirror_array[1]} and" \
2020 "${mirror_array[-1]} should fail"
2022 $mirror_cmd --only ${mirror_array[0]},${mirror_array[1]} $tf-1 ||
2023 error "verify $tf-1 with mirror ${mirror_array[0]} and" \
2024 "${mirror_array[1]} should succeed"
2026 # set stale components in $tf-1
2027 for i in 0x40002 0x40003; do
2028 $LFS setstripe --comp-set -I$i --comp-flags=stale $tf-1 ||
2029 error "set stale flag on component $i failed"
2032 # verify the mirrored file
2033 echo "Verify $tf-1 with stale components:"
2034 $mirror_cmd -vvv $tf-1 ||
2035 error "verify $tf-1 with stale components should succeed"
2037 echo "Verify $tf-1 with stale components and --only option:"
2038 $mirror_cmd -vvv --only ${mirror_array[1]},${mirror_array[-1]} $tf-1 ||
2039 error "verify $tf-1 with mirror ${mirror_array[1]} and" \
2040 "${mirror_array[-1]} should succeed"
2042 run_test 42 "lfs mirror verify"
2044 # inactivate one OST && write && restore the OST
2048 local PARAM="osp.${FSNAME}-OST000${ost}-osc-M*.active"
2051 wait=$(do_facet $SINGLEMDS \
2052 "$LCTL get_param -n lod.*MDT0000-*.qos_maxage")
2053 wait=${wait%%[^0-9]*}
2055 echo " **deactivate OST$ost, waiting for $((wait*2+2)) seconds"
2056 $(do_facet $SINGLEMDS "$LCTL set_param -n $PARAM 0")
2057 # lod_qos_statfs_update needs 2*$wait seconds to refresh targets statfs
2058 sleep $(($wait * 2 + 2))
2059 echo " **write $file"
2060 dd if=/dev/zero of=$file bs=1M count=1 || error "write $file failed"
2061 echo " **restore activating OST$ost, waiting for $((wait*2+2)) seconds"
2062 $(do_facet $SINGLEMDS "$LCTL set_param -n $PARAM 1")
2063 sleep $((wait * 2 + 2))
2065 local flags=$($LFS getstripe -v $file | awk '/lcm_flags:/ { print $2 }')
2066 [ $flags = wp ] || error "file mirror state $flags != wp"
2070 [ $OSTCOUNT -lt 3 ] && skip "needs >= 3 OSTs" && return
2072 local tf=$DIR/$tfile
2075 stack_trap "rm -f $tf"
2077 ## mirror 0 ost (0, 1)
2078 ## mirror 1 ost (1, 2)
2079 ## mirror 2 ost (2, 0)
2080 $LFS mirror create -N -Eeof -c2 -o0,1 -N -Eeof -c2 -o1,2 \
2081 -N -Eeof -c2 -o2,0 $tf ||
2082 error "create 3 mirrors file $tf failed"
2084 ################## OST0 ###########################################
2086 echo " **verify components"
2087 verify_comp_attr lcme_flags $tf 0x10001 init,stale
2088 verify_comp_attr lcme_flags $tf 0x20002 init
2089 verify_comp_attr lcme_flags $tf 0x30003 init,stale
2092 echo " **resync $tf"
2093 $LFS mirror resync $tf
2094 flags=$($LFS getstripe -v $tf | awk '/lcm_flags:/ { print $2 }')
2095 [ $flags = ro ] || error "file mirror state $flags != ro"
2097 ################## OST1 ###########################################
2099 echo " **verify components"
2100 verify_comp_attr lcme_flags $tf 0x10001 init,stale
2101 verify_comp_attr lcme_flags $tf 0x20002 init,stale
2102 verify_comp_attr lcme_flags $tf 0x30003 init
2105 echo " **resync $tf"
2106 $LFS mirror resync $tf
2107 flags=$($LFS getstripe -v $tf | awk '/lcm_flags:/ { print $2 }')
2108 [ $flags = ro ] || error "file mirror state $flags != ro"
2110 ################## OST2 ###########################################
2112 echo " **verify components"
2113 verify_comp_attr lcme_flags $tf 0x10001 init
2114 verify_comp_attr lcme_flags $tf 0x20002 init,stale
2115 verify_comp_attr lcme_flags $tf 0x30003 init,stale
2117 run_test 43a "mirror pick on write"
2120 local tf=$DIR/$tdir/$tfile
2122 test_mkdir $DIR/$tdir
2124 stack_trap "rm -rf $tf"
2126 # create 3 mirrors FLR file, the first 2 mirrors are preferred
2127 $LFS setstripe -N -Eeof --flags=prefer -N -Eeof --flags=prefer \
2128 -N -Eeof $tf || error "create 3 mirrors file $tf failed"
2129 verify_flr_state $tf "ro"
2131 echo " ** write to $tf"
2132 dd if=/dev/zero of=$tf bs=1M count=1 || error "write $tf failed"
2133 verify_flr_state $tf "wp"
2135 echo " ** resync $tf"
2136 $LFS mirror resync $tf || error "resync $tf failed"
2137 verify_flr_state $tf "ro"
2139 run_test 43b "allow writing to multiple preferred mirror file"
2142 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
2145 local tf=$DIR/$tdir/$tfile
2146 local tf1=$DIR/$tdir-1/$tfile-1
2148 stack_trap "rm -rf $tf $tf1"
2150 $LFS setdirstripe -i 0 -c 1 $DIR/$tdir ||
2151 error "create directory failed"
2152 $LFS setdirstripe -i 1 -c 1 $DIR/$tdir-1 ||
2153 error "create remote directory failed"
2154 rm -f $tf $tf1 $tf.mirror~2
2155 # create file with 4 mirrors
2156 $LFS mirror create -N -E 2M -S 1M -E 4M -E -1 \
2157 -N -E 1M -E 2M -E 3M -E -1 -N2 $tf ||
2158 error "create PFLR file $tf failed"
2160 # file should be in ro status
2161 verify_flr_state $tf "ro"
2163 # write data in [0, 3M)
2164 dd if=/dev/urandom of=$tf bs=1M count=3 conv=notrunc ||
2165 error "writing $tf failed"
2167 verify_flr_state $tf "wp"
2169 # disallow destroying the last non-stale mirror
2170 ! $LFS mirror delete --mirror-id 1 $tf > /dev/null 2>&1 ||
2171 error "destroying mirror 1 should fail"
2173 # synchronize all mirrors of the file
2174 $LFS mirror resync $tf || error "mirror resync $tf failed"
2176 verify_flr_state $tf "ro"
2179 $LFS mirror split --mirror-id 1 -f $tf1 $tf ||
2180 error "split to $tf1 failed"
2182 local idx0=$($LFS getstripe -m $tf)
2183 local idx1=$($LFS getstripe -m $tf1)
2185 [[ x$idx0 == x0 ]] || error "$tf is not on MDT0"
2186 [[ x$idx1 == x1 ]] || error "$tf1 is not on MDT1"
2188 # verify mirror count
2189 verify_mirror_count $tf 3
2190 verify_mirror_count $tf1 1
2192 $LFS mirror split --mirror-id 2 $tf ||
2193 error "split mirror 2 failed"
2195 verify_mirror_count $tf 2
2196 verify_mirror_count $tf.mirror~2 1
2198 $LFS setstripe --comp-set -I 0x30008 --comp-flags=stale $tf ||
2199 error "setting stale flag on component 0x30008 failed"
2201 # disallow destroying the last non-stale mirror
2202 ! $LFS mirror split --mirror-id 4 -d $tf > /dev/null 2>&1 ||
2203 error "destroying mirror 4 should fail"
2205 $LFS mirror resync $tf || error "resynchronizing $tf failed"
2207 $LFS mirror split --mirror-id 3 -d $tf ||
2208 error "destroying mirror 3 failed"
2209 verify_mirror_count $tf 1
2211 # verify splitted file contains the same content as the orig file does
2212 diff $tf $tf1 || error "splited file $tf1 diffs from $tf"
2213 diff $tf $tf.mirror~2 ||
2214 error "splited file $tf.mirror~2 diffs from $tf"
2216 run_test 44a "lfs mirror split check"
2219 (( $MDS1_VERSION >= $(version_code 2.14.56) )) ||
2220 skip "Need MDS version at least 2.14.56"
2223 local tf=$DIR/$tdir/$tfile
2225 mkdir -p $DIR/$tdir || error "create directory failed"
2229 # create 2 mirrors file
2230 $LFS mirror extend -N -c1 $tf
2234 verify_flr_state $tf "wp"
2236 local str=$(cat $tf)
2238 [[ $str == "YYY" ]] || error "$tf content is not YYY"
2240 # get the non-stale mirror id
2241 local ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' |
2243 local mirror_ids=($($LFS getstripe $tf |
2244 awk '/lcme_mirror_id/{print $2}' | tr '\n' ' '))
2245 for ((i = 0; i < 2; i++)); do
2246 $LFS getstripe -I${ids[$i]} --component-flags $tf |
2247 grep stale > /dev/null || break
2250 [[ $i -ge 2 ]] && ( $LFS getstripe $tf; error "no stale mirror" )
2254 # split the updated mirror, should fail
2255 echo "split mirror_id ${mirror_ids[$i]} id ${ids[$i]}, should fail"
2256 $LFS mirror split --mirror-id=${mirror_ids[$i]} $tf &> /dev/null &&
2257 error "split --mirror-id=${mirror_ids[$i]} $tf should fail"
2260 # split the stale mirror
2261 echo "split mirror_id ${mirror_ids[$i]} id ${ids[$i]}"
2262 $LFS mirror split --mirror-id=${mirror_ids[$i]} -d $tf ||
2263 error "mirror split --mirror-id=${mirror_ids[$i]} $tf failed"
2265 echo "make sure there's no stale comp in the file"
2266 # make sure there's no stale comp in the file
2267 $LFS getstripe $tf | awk '/lcme_flags/{print $2}' | grep stale &&
2268 ( $LFS getstripe $tf; error "stale mirror file" )
2271 [[ $str == "YYY" ]] ||
2272 ( cat $tf; error "$tf content is not YYY after split" )
2274 run_test 44b "mirror split does not create stale file"
2277 local tf=$DIR/$tdir/$tfile
2279 stack_trap "rm -f $tf"
2281 [ $MDS1_VERSION -ge $(version_code 2.14.52) ] ||
2282 skip "Need MDS version at least 2.14.52"
2284 [ "$FSTYPE" != "zfs" ] || skip "ZFS file's block number is not accurate"
2286 mkdir -p $DIR/$tdir || error "create directroy failed"
2288 dd if=/dev/zero of=$tf bs=1M count=10 || error "dd write $tfile failed"
2290 block1=$(( $(stat -c "%b*%B" $tf) ))
2291 echo " ** before mirror ops, file blocks=$((block1/1024)) KiB"
2293 $LFS mirror extend -N2 -c1 $tf || error "mirror extend $tfile failed"
2295 block2=$(( $(stat -c "%b*%B" $tf) ))
2296 echo " ** after mirror extend, file blocks=$((block2/1024)) KiB"
2298 $LFS mirror split -d --mirror-id=2 $tf ||
2299 error "mirror split $tfile failed"
2300 $LFS mirror split -d --mirror-id=3 $tf ||
2301 error "mirror split $tfile failed"
2304 block3=$(( $(stat -c "%b*%B" $tf) ))
2305 echo " ** after mirror split, file blocks=$((block3/1024)) KiB"
2307 [[ $block1 -eq $block3 ]] ||
2308 error "mirror split does not reduce block# $block3 != $block1"
2310 run_test 44c "lfs mirror split reduces block size of a file"
2313 local tf=$DIR/$tdir/$tfile
2319 stack_trap "rm -f $tf"
2321 mkdir -p $DIR/$tdir || error "create directroy failed"
2323 dd if=/dev/zero of=$tf bs=1M count=10 || error "dd write $tfile failed"
2325 size1=$(stat -c "%s" $tf)
2326 echo " ** before mirror ops, file size=$size1"
2328 $LFS mirror extend -N2 -c1 $tf || error "mirror extend $tfile failed"
2330 size2=$(stat -c "%s" $tf)
2331 echo " ** after mirror extend, file size=$size2"
2333 (($size1 == $size2)) ||
2334 error "mirror extend should not change size, before: $size1, after $size2"
2336 $LFS mirror split -d --mirror-id=2 $tf ||
2337 error "mirror split $tfile failed"
2339 size2=$(stat -c "%s" $tf)
2340 echo " ** after mirror split, file size=$size2"
2341 (($size1 == $size2)) ||
2342 error "mirror split should not change size, before: $size1, after $size2"
2344 # Remount client to clear cached size information
2345 remount_client $MOUNT
2346 size2=$(stat -c "%s" $tf)
2347 echo " ** after mirror split & remount, file size=$size2"
2348 (($size1 == $size2)) ||
2349 error "mirror extend should not change size, before: $size1, after $size2"
2352 run_test 44d "lfs mirror split does not break size"
2355 local tf=$DIR/$tdir/$tfile
2356 local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
2360 test_mkdir $DIR/$tdir
2361 [ $MDS1_VERSION -ge $(version_code 2.14.52) ] ||
2362 skip "Need MDS version at least 2.14.52"
2364 $LFS mirror create -N2 $tf || error "create mirrored file $tf failed"
2366 # Disable xattr caching so we can repeatedly check SOM with lfs getsom
2367 $LCTL set_param llite.*.xattr_cache=0
2368 stack_trap "$LCTL set_param llite.*.xattr_cache=1"
2369 stack_trap "rm -rf $tf"
2371 dd if=/dev/zero of=$tf bs=1M count=10 || error "dd write $tfile failed"
2373 size1=$(stat -c "%s" $tf)
2374 echo " ** before mirror resync, file size=$size1"
2376 $LFS mirror resync $tf || error "mirror resync file $tf failed"
2377 size1=$(stat -c "%s" $tf)
2378 size2=$($LFS getsom -s $tf)
2382 ((size1 == size2)) ||
2383 error "mirrored file with strict SOM $size1 != disabled SOM $size2"
2385 # Remount client to clear cached size information
2386 remount_client $MOUNT
2388 save_lustre_params $(get_facets MDS) mdt.*MDT*.enable_strict_som > $p
2389 stack_trap "restore_lustre_params < $p; rm -f $p"
2390 local mds_facet=mds$(($($LFS getstripe -m $tf) + 1))
2392 do_facet $mds_facet $LCTL set_param mdt.*MDT*.enable_strict_som=0
2394 size2=$(stat -c "%s" $tf)
2395 # 'getsom' here is just for debugging
2398 ((size2 == size1)) ||
2399 error "mirrored file in sync with som disabled, size with som disabled ($size2) and without som disabled ($size1) should agree"
2401 run_test 44e "basic FLR SOM tests + disable SOM"
2404 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
2406 local file=$DIR/$tdir/$tfile
2407 local dir=$DIR/$tdir/$dir
2408 local temp=$DIR/$tdir/template
2410 test_mkdir $DIR/$tdir
2412 $LFS setstripe -N -E1m -S1m -c2 -o0,1 -E2m -Eeof -N -E4m -Eeof \
2413 -N -E3m -S1m -Eeof -N -E8m -Eeof $file ||
2414 error "Create $file failed"
2416 verify_yaml_layout $file $file.copy $temp "1. FLR file"
2417 rm -f $file $file.copy
2419 $LFS setstripe -N -E1m -S1m -c2 -o0,1 -E2m -Eeof -N -E4m -Eeof \
2420 -N -E3m -S1m -Eeof -N -E8m --flags=prefer -Eeof $file ||
2421 error "Create $file failed"
2423 verify_yaml_layout $file $file.copy $temp "2. FLR file with flags"
2425 run_test 45 "Verify setstripe/getstripe with YAML with FLR file"
2432 $LFS setstripe --copy=$src $dst || error "setstripe $dst failed"
2434 local layout1=$(get_layout_param $src)
2435 local layout2=$(get_layout_param $dst)
2436 # compare their layout info
2437 [ "$layout1" == "$layout2" ] ||
2438 error "$msg_prefix $src <=> $dst layouts are not equal"
2442 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" && return
2444 local file=$DIR/$tdir/$tfile
2445 test_mkdir $DIR/$tdir
2447 ########################### 1. PFL file #############################
2448 echo " ** 1. PFL file"
2450 $LFS setstripe -E1m -S 1M -c2 -o0,1 -E2m -c2 -E3m -o1,0 -E4m -c1 -E-1 \
2451 $file || error "1. Create PFL $file failed"
2454 verify_46 $file $file.copy "1. PFL file"
2456 ########################### 2. plain file ###########################
2457 echo " ** 2. plain file"
2459 $LFS setstripe -c2 -o0,1 -i1 $file ||
2460 error "2. Create plain $file failed"
2463 verify_46 $file $file.copy "2. plain file"
2465 ########################### 3. FLR file #############################
2466 echo " ** 3. FLR file"
2468 $LFS setstripe -N -E1m -S 1M -c2 -o0,1 -E4m -c1 -Eeof -N -E16m -Eeof \
2469 $file || error "3. Create FLR $file failed"
2472 verify_46 $file $file.copy "3. FLR file"
2474 local dir=$DIR/$tdir/dir
2475 ########################### 4. PFL dir ##############################
2476 echo " ** 4. PFL dir"
2478 $LFS setstripe -E1m -S 1M -c2 -E2m -c1 -E-1 $dir ||
2479 error "4. setstripe PFL $dir failed"
2481 test_mkdir $dir.copy
2482 verify_46 $dir $dir.copy "4. PFL dir"
2484 ########################### 5. plain dir ############################
2485 echo " ** 5. plain dir"
2486 $LFS setstripe -c2 -i-1 $dir || error "5. setstripe plain $dir failed"
2488 verify_46 $dir $dir.copy "5. plain dir"
2490 ########################### 6. FLR dir ##############################
2491 echo " ** 6. FLR dir"
2492 $LFS setstripe -N -E1m -S 1M -c2 -E2m -c1 -Eeof -N -E4m -Eeof $dir ||
2493 error "6. setstripe FLR $dir failed"
2495 verify_46 $dir $dir.copy "6. FLR dir"
2497 (( $MDS1_VERSION >= $(version_code 2.13.53.205) )) ||
2498 echo "server version $MDS1_VERSION does not support SEL" &&
2501 ########################### 7. SEL file ##############################
2502 echo " ** 7. SEL file"
2504 $LFS setstripe -E256M -S 1M -c2 -o0,1 -z 64M -E-1 -o1,0 -z 128M \
2505 $file || error "Create $file failed"
2508 verify_46 $file $file.copy "7. SEL file"
2510 ########################### 8. SEL dir ##############################
2511 echo " ** 8. SEL dir"
2512 $LFS setstripe -E256M -S 1M -c2 -z 64M -E-1 -z 128M \
2513 $dir || error "setstripe $dir failed"
2515 verify_46 $dir $dir.copy "8. SEL dir"
2517 ########################### 9. FLR SEL file ##########################
2518 echo " ** 9. FLR SEL file"
2520 $LFS setstripe -N -E256M -c2 -z 64M -E-1 -z 128M \
2521 -N -E1G -c4 -z128M -E-1 -z256M $file || error "Create $file failed"
2524 verify_46 $file $file.copy "9. SEL file"
2526 ########################### 10. FLR SEL dir #########################
2527 echo " ** 10. FLR SEL dir"
2528 $LFS setstripe -N -E256M -c2 -z 64M -E-1 -z 128M \
2529 -N -E1G -c4 -z128M -E-1 -z256M $dir || error "Create $file failed"
2531 verify_46 $dir $dir.copy "10. SEL dir"
2533 run_test 46 "Verify setstripe --copy option"
2536 [ $OSTCOUNT -lt 3 ] && skip "needs >= 3 OSTs" && return
2538 local file=$DIR/$tdir/$tfile
2543 test_mkdir $DIR/$tdir
2544 stack_trap "rm -f $file"
2548 # mirror1: [comp0]ost0, [comp1]ost1 and ost2
2549 # mirror2: [comp2] , [comp3] should not use ost1 or ost2
2550 $LFS mirror create -N -E2m -c1 -o0 --flags=prefer -Eeof -c2 -o1,2 \
2551 -N -E2m -c1 -Eeof -c1 $file || error "create FLR $file failed"
2552 ids=($($LFS getstripe $file | awk '/lcme_id/{print $2}' | tr '\n' ' '))
2554 dd if=/dev/zero of=$file bs=1M count=3 || error "dd $file failed"
2555 $LFS mirror resync $file || error "resync $file failed"
2557 ost=$($LFS getstripe -I${ids[2]} $file | awk '/l_ost_idx/{print $5}')
2558 if [[ x$ost == "x0," ]]; then
2559 $LFS getstripe $file
2560 error "component ${ids[2]} objects allocated on $ost " \
2564 ost=$($LFS getstripe -I${ids[3]} $file | awk '/l_ost_idx/{print $5}')
2565 if [[ x$ost == "x1," || x$ost == "x2," ]]; then
2566 $LFS getstripe $file
2567 error "component ${ids[3]} objects allocated on $ost " \
2568 "shouldn't on OST1 or on OST2"
2573 # mirror1: [comp0] [comp1]
2574 # mirror2: [comp2] [comp3]
2575 # mirror3: [comp4] [comp5]
2576 # mirror4: [comp6] [comp7]
2577 $LFS mirror create -N4 -E1m -c1 -Eeof -c1 $file ||
2578 error "create FLR $file failed"
2579 ids=($($LFS getstripe $file | awk '/lcme_id/{print $2}' | tr '\n' ' '))
2581 dd if=/dev/zero of=$file bs=1M count=3 || error "dd $file failed"
2582 $LFS mirror resync $file || error "resync $file failed"
2584 for ((i = 0; i < 6; i++)); do
2585 osts[$i]=$($LFS getstripe -I${ids[$i]} $file |
2586 awk '/l_ost_idx/{print $5}')
2588 # comp[0],comp[2],comp[4] should use different osts
2589 if [[ ${osts[0]} == ${osts[2]} || ${osts[0]} == ${osts[4]} ||
2590 ${osts[2]} == ${osts[4]} ]]; then
2591 $LFS getstripe $file
2592 error "component ${ids[0]}, ${ids[2]}, ${ids[4]} have objects "\
2593 "allocated on duplicated OSTs"
2595 # comp[1],comp[3],comp[5] should use different osts
2596 if [[ ${osts[1]} == ${osts[3]} || ${osts[1]} == ${osts[5]} ||
2597 ${osts[3]} == ${osts[5]} ]]; then
2598 $LFS getstripe $file
2599 error "component ${ids[1]}, ${ids[3]}, ${ids[5]} have objects "\
2600 "allocated on duplicated OSTs"
2605 run_test 47 "Verify mirror obj alloc"
2608 [ $MDS1_VERSION -lt $(version_code 2.11.55) ] &&
2609 skip "Need MDS version at least 2.11.55"
2611 local tf=$DIR/$tfile
2613 stack_trap "rm -f $tf"
2615 echo " ** create 2 mirrors FLR file $tf"
2616 $LFS mirror create -N -E2M -Eeof --flags prefer \
2617 -N -E1M -Eeof $tf ||
2618 error "create FLR file $tf failed"
2621 dd if=/dev/urandom of=$tf bs=1M count=3 || error "write $tf failed"
2622 verify_flr_state $tf "wp"
2624 local sum0=$(md5sum < $tf)
2626 echo " ** resync the file"
2627 $LFS mirror resync $tf
2629 echo " ** snapshot mirror 2"
2630 $LFS setstripe --comp-set -I 0x20003 --comp-flags=nosync $tf
2632 echo " ** write it again"
2633 dd if=/dev/urandom of=$tf bs=1M count=3 || error "write $tf failed"
2634 echo " ** resync it again"
2635 $LFS mirror resync $tf
2637 verify_flr_state $tf "wp"
2638 verify_comp_attr lcme_flags $tf 0x20003 nosync,stale
2640 local sum1=$($LFS mirror read -N1 $tf | md5sum)
2641 local sum2=$($LFS mirror read -N2 $tf | md5sum)
2643 echo " ** verify mirror 2 doesn't change"
2644 echo "original checksum: $sum0"
2645 echo "mirror 1 checksum: $sum1"
2646 echo "mirror 2 checksum: $sum2"
2647 [[ $sum0 = $sum2 ]] ||
2648 error "original checksum: $sum0, mirror 2 checksum: $sum2"
2649 echo " ** mirror 2 stripe info"
2650 $LFS getstripe -v --mirror-index=2 $tf
2652 echo " ** resync mirror 2"
2653 $LFS mirror resync --only 2 $tf
2655 verify_flr_state $tf "ro"
2656 verify_comp_attr lcme_flags $tf 0x20003 nosync,^stale
2658 sum1=$($LFS mirror read -N1 $tf | md5sum)
2659 sum2=$($LFS mirror read -N2 $tf | md5sum)
2661 echo " ** verify mirror 2 resync-ed"
2662 echo "original checksum: $sum0"
2663 echo "mirror 1 checksum: $sum1"
2664 echo "mirror 2 checksum: $sum2"
2665 [[ $sum1 = $sum2 ]] ||
2666 error "mirror 1 checksum: $sum1, mirror 2 checksum: $sum2"
2667 echo " ** mirror 2 stripe info"
2668 $LFS getstripe -v --mirror-index=2 $tf
2670 run_test 48 "Verify snapshot mirror"
2679 (( "$OSTCOUNT" >= "2" )) || skip "needs >= 2 OSTs"
2680 local filefrag_op=$(filefrag -l 2>&1 | grep "invalid option")
2681 [[ -z "$filefrag_op" ]] || skip_env "filefrag missing logical ordering"
2682 [[ "$ost1_FSTYPE" != "zfs" ]] ||
2683 skip "LU-1941: FIEMAP unimplemented on ZFS"
2685 trap cleanup_49 EXIT RETURN
2687 local file=$DIR/$tfile
2689 $LFS setstripe -N -E eof -c1 -o1 -N -E eof -c1 -o0 $file ||
2690 error "setstripe on $file"
2691 stack_trap "rm -f $file"
2693 dd if=/dev/zero of=$file bs=1M count=1 || error "dd failed for $file"
2694 $LFS mirror resync $file
2696 filefrag -ves $file || error "filefrag $file failed"
2697 filefrag_op=$(filefrag -ve -k $file |
2698 sed -n '/ext:/,/found/{/ext:/d; /found/d; p}')
2700 #Filesystem type is: bd00bd0
2701 #File size of /mnt/lustre/f49a.sanity-flr is 1048576 (1024 blocks of 1024 bytes)
2702 # ext: device_logical: physical_offset: length: dev: flags:
2703 # 0: 0.. 1023: 1572864.. 1573887: 1024: 0001: net,eof
2704 # 1: 0.. 1023: 1572864.. 1573887: 1024: 0000: last,net,eof
2705 #/mnt/lustre/f49a.sanity-flr: 2 extents found
2707 last_lun=$(echo $filefrag_op | cut -d: -f5)
2711 for line in $filefrag_op; do
2712 frag_lun=$(echo $line | cut -d: -f5)
2713 ext_len=$(echo $line | cut -d: -f4)
2714 if [[ "$frag_lun" != "$last_lun" ]]; then
2715 if (( tot_len != 1024 )); then
2717 error "$file: OST$last_lun $tot_len != 1024"
2723 (( tot_len += ext_len ))
2726 if (( num_luns != 2 || tot_len != 1024 )); then
2728 error "$file: $num_luns != 2, $tot_len != 1024 on OST$last_lun"
2731 echo "FIEMAP on $file succeeded"
2733 run_test 49a "FIEMAP upon FLR file"
2735 test_50A() { # EX-2179
2738 local file=$DIR/$tdir/$tfile
2740 $LFS setstripe -c1 -i0 $file || error "setstripe $file failed"
2742 $LFS mirror extend -N -c1 -i1 $file ||
2743 error "extending mirror for $file failed"
2745 local olv=$($LFS getstripe $file | awk '/lcm_layout_gen/{print $2}')
2749 $LFS mirror split -d --mirror-id=1 $file || error "split $file failed"
2751 local flv=$($LFS getstripe $file | awk '/lcm_layout_gen/{print $2}')
2753 echo "$file layout generation from $olv to $flv"
2754 (( $flv != ($olv + 1) )) &&
2755 error "split does not increase layout gen from $olv to $flv"
2757 dd if=/dev/zero of=$file bs=1M count=1 || error "write $file failed"
2759 $LFS getstripe -v $file || error "getstripe $file failed"
2761 run_test 50A "mirror split update layout generation"
2764 $LCTL get_param osc.*.import | grep -q 'connect_flags:.*seek' ||
2765 skip "OST does not support SEEK_HOLE"
2766 [ "$FSTYPE" != "zfs" ] ||
2767 skip "lseek for ZFS is not accurate if obj is not committed"
2769 local file=$DIR/$tdir/$tfile
2777 echo " ** create striped file $file"
2778 $LFS setstripe -E 1M -c1 -S 1M -E eof -c2 -S1M $file ||
2779 error "cannot create file with PFL layout"
2780 echo " ** write 1st data chunk at 1M boundary"
2781 dd if=/dev/urandom of=$file bs=1k count=20 seek=1021 ||
2782 error "cannot write data at 1M boundary"
2783 echo " ** write 2nd data chunk at 2M boundary"
2784 dd if=/dev/urandom of=$file bs=1k count=20 seek=2041 ||
2785 error "cannot write data at 2M boundary"
2786 echo " ** create hole at the file end"
2787 $TRUNCATE $file 3700000 || error "truncate fails"
2789 echo " ** verify sparseness"
2790 offset=$(lseek_test -d 1000 $file)
2791 echo " first data offset: $offset"
2792 [[ $offset == 1000 ]] &&
2793 error "src: data is not expected at offset $offset"
2794 offset=$(lseek_test -l 3500000 $file)
2795 echo " hole at the end: $offset"
2796 [[ $offset == 3500000 ]] ||
2797 error "src: hole is expected at offset $offset"
2799 echo " ** extend the file with new mirror"
2800 # migrate_copy_data() is used
2801 $LFS mirror extend -N -E 2M -S 1M -E 1G -S 2M -E eof $file ||
2802 error "cannot create mirror"
2803 $LFS getstripe $file | grep lcme_flags | grep stale > /dev/null &&
2804 error "$file still has stale component"
2806 # check migrate_data_copy() was correct
2807 sum_1=$($LFS mirror read -N 1 $file | md5sum)
2808 sum_2=$($LFS mirror read -N 2 $file | md5sum)
2809 [[ $sum_1 == $sum_2 ]] ||
2810 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
2812 # stale first mirror
2813 $LFS setstripe --comp-set -I0x10001 --comp-flags=stale $file
2814 $LFS setstripe --comp-set -I0x10002 --comp-flags=stale $file
2816 echo " ** verify mirror #2 sparseness"
2817 offset=$(lseek_test -d 1000 $file)
2818 echo " first data offset: $offset"
2819 [[ $offset == 1000 ]] &&
2820 error "dst: data is not expected at offset $offset"
2821 offset=$(lseek_test -l 3500000 $file)
2822 echo " hole at the end: $offset"
2823 [[ $offset == 3500000 ]] ||
2824 error "dst: hole is expected at offset $offset"
2826 echo " ** copy mirror #2 to mirror #1"
2827 $LFS mirror copy -i 2 -o 1 $file || error "mirror copy fails"
2828 $LFS getstripe $file | grep lcme_flags | grep stale > /dev/null &&
2829 error "$file still has stale component"
2831 # check llapi_mirror_copy_many correctness
2832 sum_1=$($LFS mirror read -N 1 $file | md5sum)
2833 sum_2=$($LFS mirror read -N 2 $file | md5sum)
2834 [[ $sum_1 == $sum_2 ]] ||
2835 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
2837 # stale 1st component of mirror #2 before lseek call
2838 $LFS setstripe --comp-set -I0x20001 --comp-flags=stale $file
2840 echo " ** verify mirror #1 sparseness again"
2841 offset=$(lseek_test -d 1000 $file)
2842 echo " first data offset: $offset"
2843 [[ $offset == 1000 ]] &&
2844 error "dst: data is not expected at offset $offset"
2845 offset=$(lseek_test -l 3500000 $file)
2846 echo " hole at the end: $offset"
2847 [[ $offset == 3500000 ]] ||
2848 error "dst: hole is expected at offset $offset"
2850 cancel_lru_locks osc
2852 blocks=$(stat -c%b $file)
2853 echo " ** final consumed blocks: $blocks"
2854 # for 3.5Mb file consumes ~6000 blocks, use 1000 to check
2855 # that file is still sparse
2856 (( blocks < 1000 )) ||
2857 error "Mirrored file consumes $blocks blocks"
2859 run_test 50a "mirror extend/copy preserves sparseness"
2862 $LCTL get_param osc.*.import | grep -q 'connect_flags:.*seek' ||
2863 skip "OST does not support SEEK_HOLE"
2864 [ "$FSTYPE" != "zfs" ] ||
2865 skip "lseek for ZFS is not accurate if obj is not committed"
2867 local file=$DIR/$tdir/$tfile
2874 stack_trap "rm -f $file"
2876 echo " ** create mirrored file $file"
2877 $LFS mirror create -N -E1M -c1 -S1M -E eof \
2878 -N -E2M -S1M -E eof -S2M $file ||
2879 error "cannot create mirrored file"
2880 echo " ** write data chunk at 1M boundary"
2881 dd if=/dev/urandom of=$file bs=1k count=20 seek=1021 ||
2882 error "cannot write data at 1M boundary"
2883 echo " ** create hole at the file end"
2884 $TRUNCATE $file 3700000 || error "truncate fails"
2886 echo " ** verify sparseness"
2887 offset=$(lseek_test -d 1000 $file)
2888 echo " first data offset: $offset"
2889 [[ $offset == 1000 ]] &&
2890 error "src: data is not expected at offset $offset"
2891 offset=$(lseek_test -l 3500000 $file)
2892 echo " hole at the end: $offset"
2893 [[ $offset == 3500000 ]] ||
2894 error "src: hole is expected at 3500000"
2896 echo " ** resync mirror #2 to mirror #1"
2897 $LFS mirror resync $file
2899 # check llapi_mirror_copy_many correctness
2900 sum_1=$($LFS mirror read -N 1 $file | md5sum)
2901 sum_2=$($LFS mirror read -N 2 $file | md5sum)
2902 [[ $sum_1 == $sum_2 ]] ||
2903 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
2905 cancel_lru_locks osc
2907 blocks=$(stat -c%b $file)
2908 echo " ** consumed blocks: $blocks"
2909 # without full punch() support the first component can be not sparse
2910 # but the last one should be, so file should use far fewer blocks
2911 (( blocks < 5000 )) ||
2912 error "Mirrored file consumes $blocks blocks"
2914 # stale first component in mirror #1
2915 $LFS setstripe --comp-set -I0x10001 --comp-flags=stale,nosync $file
2916 echo " ** truncate file down"
2918 echo " ** write data chunk at 2M boundary"
2919 dd if=/dev/urandom of=$file bs=1k count=20 seek=2041 conv=notrunc ||
2920 error "cannot write data at 2M boundary"
2921 echo " ** resync mirror #2 to mirror #1 with nosync 1st component"
2922 $LFS mirror resync $file || error "mirror rsync fails"
2923 # first component is still stale
2924 $LFS getstripe $file | grep 'lcme_flags:.*stale' > /dev/null ||
2925 error "$file still has no stale component"
2926 echo " ** resync mirror #2 to mirror #1 again"
2927 $LFS setstripe --comp-set -I0x10001 --comp-flags=stale,^nosync $file
2928 $LFS mirror resync $file || error "mirror rsync fails"
2929 $LFS getstripe $file | grep 'lcme_flags:.*stale' > /dev/null &&
2930 error "$file still has stale component"
2932 # check llapi_mirror_copy_many correctness
2933 sum_1=$($LFS mirror read -N 1 $file | md5sum)
2934 sum_2=$($LFS mirror read -N 2 $file | md5sum)
2935 [[ $sum_1 == $sum_2 ]] ||
2936 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
2938 cancel_lru_locks osc
2940 blocks=$(stat -c%b $file)
2941 echo " ** final consumed blocks: $blocks"
2942 # while the first component can lose sparseness, the last one should
2943 # not, so whole file should still use far fewer blocks in total
2944 (( blocks < 3000 )) ||
2945 error "Mirrored file consumes $blocks blocks"
2947 run_test 50b "mirror rsync handles sparseness"
2950 local tf=$DIR/$tdir/$tfile
2952 test_mkdir $DIR/$tdir
2954 $LFS setstripe -N2 -c-1 $tf || error "create FLR $tf failed"
2955 verify_flr_state $tf "ro"
2957 if [[ "$FSTYPE" == "ldiskfs" ]]; then
2958 # ZFS does not support fallocate for now
2959 out=$(fallocate -p -o 1MiB -l 1MiB $tf 2>&1) ||
2960 skip_eopnotsupp "$out|punch hole in $tf failed"
2961 verify_flr_state $tf "wp"
2964 dd if=/dev/zero of=$tf bs=4096 count=4 || error "write $tf failed"
2965 $LFS mirror resync $tf || error "mirror resync $tf failed"
2966 verify_flr_state $tf "ro"
2968 $MULTIOP $tf OSMWUc || error "$MULTIOP $tf failed"
2969 verify_flr_state $tf "wp"
2971 run_test 50c "punch_hole/mmap_write stale other mirrors"
2974 $LCTL get_param osc.*.import | grep -q 'connect_flags:.*seek' ||
2975 skip "OST does not support SEEK_HOLE"
2976 [ "$FSTYPE" != "zfs" ] ||
2977 skip "lseek for ZFS is not accurate if obj is not committed"
2979 local file=$DIR/$tdir/$tfile
2986 echo " ** create mirrored file $file"
2987 $LFS mirror create -N -E1M -c1 -S1M -E eof \
2988 -N -E2M -S1M -E eof -S2M $file ||
2989 error "cannot create mirrored file"
2990 echo " ** write data chunk at 1M boundary"
2991 dd if=/dev/urandom of=$file bs=1k count=20 seek=1021 ||
2992 error "cannot write data at 1M boundary"
2993 echo " ** create hole at the file start"
2994 prt=$(fallocate -p -o 0 -l 1M $file 2>&1)
2997 if [[ $rc -eq 0 ]]; then
2998 verify_flr_state $file "wp"
2999 elif [[ ! $prt =~ unsupported ]]; then
3000 error "punch hole in $file failed: $prt"
3002 skip "Fallocate punch is not supported: $prt"
3005 echo " ** verify sparseness"
3006 offset=$(lseek_test -d 1000 $file)
3007 echo " first data offset: $offset"
3008 (( $offset >= 1024 * 1024 )) ||
3009 error "src: data is not expected at offset $offset"
3011 echo " ** resync mirror #2"
3012 $LFS mirror resync $file
3014 # check llapi_mirror_copy_many correctness
3015 sum_1=$($LFS mirror read -N 1 $file | md5sum)
3016 sum_2=$($LFS mirror read -N 2 $file | md5sum)
3017 [[ $sum_1 == $sum_2 ]] ||
3018 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
3020 cancel_lru_locks osc
3022 # stale first component in mirror #1
3023 $LFS setstripe --comp-set -I0x10001 --comp-flags=stale,nosync $file
3024 echo " ** verify sparseness of mirror #2"
3025 offset=$(lseek_test -d 1000 $file)
3026 echo " first data offset: $offset"
3027 (( $offset >= 1024 * 1024 )) ||
3028 error "src: data is not expected at offset $offset"
3030 run_test 50d "mirror rsync keep holes"
3033 $LCTL get_param osc.*.import | grep -q 'connect_flags:.*seek' ||
3034 skip "OST does not support SEEK_HOLE"
3036 local file=$DIR/$tdir/$tfile
3037 local old_size=2147483648 # 2GiB
3041 dd if=/dev/urandom of=$file bs=4096 count=1 seek=$((134217728 / 4096))
3042 $TRUNCATE $file $old_size
3044 $LFS mirror extend -N -c 1 $file
3045 dd if=/dev/urandom of=$file bs=4096 count=1 seek=$((134217728 / 4096)) conv=notrunc
3046 $LFS mirror resync $file
3048 new_size=$(stat --format='%s' $file)
3049 if ((new_size != old_size)); then
3050 error "new_size ($new_size) is not equal to old_size ($old_size)"
3053 run_test 60a "mirror extend sets correct size on sparse file"
3055 get_flr_layout_gen() {
3056 getfattr -n lustre.lov --only-values $tf 2>/dev/null |
3057 od -tx4 | awk '/000000/ { print "0x"$4; exit; }'
3060 check_layout_gen() {
3063 local v1=$(get_flr_layout_gen $tf)
3064 local v2=$($LFS getstripe -v $tf | awk '/lcm_layout_gen/ { print $2 }')
3066 [[ $v1 -eq $v2 ]] ||
3067 error "$tf in-memory layout gen $v1 != $v2 after $2"
3071 local tf=$DIR/$tdir/$tfile
3073 test_mkdir $DIR/$tdir
3075 $LFS setstripe -Eeof $tf || error "setstripe $tf failed"
3077 for ((i = 0; i < 20; i++)); do
3078 $LFS mirror extend -N $tf ||
3079 error "extending mirror for $tf failed"
3080 check_layout_gen $tf "extend"
3082 $LFS mirror split -d --mirror-id=$((i+1)) $tf ||
3083 error "split $tf failed"
3084 check_layout_gen $tf "split"
3087 run_test 60b "mirror merge/split cancel client's in-memory layout gen"
3090 stat --format='%X %Y %Z' $file || error "$file: cannot get times"
3095 local -a old=( $2 $3 $4 )
3098 new=( $(get_times_61 $file) )
3099 ((${old[0]} == ${new[0]})) ||
3100 error "$file: atime: old '${old[0]}' != new '${new[0]}'"
3102 ((${old[1]} == ${new[1]})) ||
3103 error "$file: mtime: old '${old[1]}' != new '${new[1]}'"
3106 test_61a() { # LU-14508
3107 local file=$DIR/$tdir/$tfile
3113 $LFS setstripe -E1M -Eeof $file || error "setstripe $file failed"
3114 echo "create $file-2"
3115 $LFS setstripe -E2M -Eeof $file-2 || error "setstripe $file-2 failed"
3117 echo XXX > $file || error "write $file failed"
3118 chown $RUNAS_ID $DIR/$tdir $file || error "chown $file failed"
3120 echo "sleep $nap seconds, then cat $tfile"
3122 cat $file || error "cat $file failed"
3124 echo "sleep $nap seconds, then re-write $tfile"
3126 echo XXXX > $file || error "write $file failed"
3127 cp -p $file $file-2 || error "copy $file-2 failed"
3129 echo "sleep $nap seconds"
3132 tim=( $(get_times_61 $file) )
3134 echo "mirror merge $tfile-2 to $tfile and test timestamps"
3135 $LFS mirror extend -N -f $file-2 $file ||
3136 error "cannot mirror merge $file-2 to $file"
3137 check_times_61 $file "${tim[@]}"
3139 echo "mirror extend $tfile and test timestamps"
3140 $LFS mirror extend -N -c1 -i1 $file ||
3141 error "cannot extend mirror $file"
3142 check_times_61 $file "${tim[@]}"
3144 echo "migrate $tfile and test timestamps"
3145 $LFS migrate -n $file || error "cannot migrate $file"
3146 check_times_61 $file "${tim[@]}"
3148 echo "normal user migrate $tfile and test timestamps"
3149 $RUNAS $LFS migrate -n $file || error "cannot migrate $file"
3150 check_times_61 $file "${tim[@]}"
3152 run_test 61a "mirror extend and migrate preserve timestamps"
3154 test_61b() { # LU-14508
3155 local file=$DIR/$tdir/$tfile
3161 echo XXX > $file || error "create $file failed"
3162 chown $RUNAS_ID $DIR/$tdir $file || error "chown $file failed"
3164 echo "sleep $nap seconds, then cat $tfile"
3166 cat $file || error "cat $file failed"
3168 echo "sleep $nap seconds, then re-write $tfile"
3170 echo XXXX > $file || error "write $file failed"
3172 echo "sleep $nap seconds, then test timestamps"
3175 tim=( $(get_times_61 $file) )
3177 echo "mirror extend $tfile and test timestamps"
3178 $LFS mirror extend -N -c1 -i1 $file ||
3179 error "cannot extend mirror $file"
3180 check_times_61 $file "${tim[@]}"
3182 echo "mirror split $tfile and test timestamps"
3183 $LFS mirror split -d --mirror-id=1 $file ||
3184 error "cannot split mirror 1 off $file"
3185 check_times_61 $file "${tim[@]}"
3187 echo "normal user mirror extend $tfile and test timestamps"
3188 $RUNAS $LFS mirror extend -N -c1 -i1 $file ||
3189 error "cannot extend mirror $file"
3190 check_times_61 $file "${tim[@]}"
3192 run_test 61b "mirror extend and split preserve timestamps"
3194 test_61c() { # LU-14508
3195 local file=$DIR/$tdir/$tfile
3201 echo XXX > $file || error "create $file failed"
3202 chown $RUNAS_ID $DIR/$tdir $file || error "chown $file failed"
3204 echo "sleep $nap seconds, then cat $tfile"
3206 cat $file || error "cat $file failed"
3208 echo "sleep $nap seconds, then mirror extend $tfile and write it"
3210 $LFS mirror extend -N -c1 -i1 $file ||
3211 error "cannot extend mirror $file"
3212 echo XXXX > $file || error "write $file failed"
3214 echo "sleep $nap seconds, then resync $tfile and test timestamps"
3215 tim=( $(get_times_61 $file) )
3217 $LFS mirror resync $file || error "cannot resync mirror $file"
3218 check_times_61 $file "${tim[@]}"
3220 echo XXXXXX > $file || error "write $tfile failed"
3222 echo "normal user resync $tfile and test timestamps"
3223 tim=( $(get_times_61 $file) )
3224 $RUNAS $LFS mirror resync $file || error "cannot resync mirror $file"
3225 check_times_61 $file "${tim[@]}"
3227 run_test 61c "mirror resync preserves timestamps"
3230 local tf=$DIR/$tdir/$tfile
3232 test_mkdir $DIR/$tdir
3233 stack_trap "rm -f $tf"
3237 $LFS mirror create -N -E 1M -c -1 -E eof -N $tf
3241 echo "mirror create pid $c_pid"
3244 $LFS mirror split -d --mirror-id=1 $tf &> /dev/null
3247 echo "mirror split pid $s_pid"
3249 echo "mirror create and split race for 60 seconds, should not crash"
3251 kill -9 $c_pid &> /dev/null
3252 kill -9 $s_pid &> /dev/null
3256 run_test 70 "mirror create and split race"
3259 local tf=$DIR/$tdir/$tfile
3261 (( $OST1_VERSION >= $(version_code 2.14.51) )) ||
3262 skip "Need OST version at least 2.14.51"
3265 test_mkdir $DIR/$tdir
3266 stack_trap "rm -f $tf"
3268 $LFS setstripe -N -E1M -c-1 -Eeof -c-1 $tf ||
3269 error "setstripe $tf failed"
3271 FSXNUM=${FSXNUM:-1000}
3272 $FSX -p 1 -N $FSXNUM -S 0 -M $tf || error "fsx FLR file $tf failed"
3274 run_test 70a "flr mode fsx test"
3279 local fsize=$(stat --printf=%s $tf)
3281 while [ -f $ctrl_file ]; do
3282 local off=$((RANDOM << 8))
3283 local len=$((RANDOM << 5 + 131072))
3285 [ $((off + len)) -gt $fsize ] && {
3286 fsize=$((off + len))
3287 echo "Extending file size to $fsize .."
3290 flock -s $lock_file -c \
3291 "$MULTIOP $tf oO_WRONLY:z${off}w${len}c" ||
3293 error "failed writing to $off:$len"; }
3294 sleep 0.$((RANDOM % 2 + 1))
3301 while [ -f $ctrl_file ]; do
3302 flock -s $lock_file -c "cat $tf &> /dev/null" ||
3303 { rm -f $ctrl_file; error "read failed"; }
3304 sleep 0.$((RANDOM % 2 + 1))
3311 options=("" "-e resync_start" "-e delay_before_copy -d 1" "" "")
3313 exec 200<>$lock_file
3314 while [ -f $ctrl_file ]; do
3315 local lock_taken=false
3316 local index=$((RANDOM % ${#options[@]}))
3317 local cmd="mirror_io resync ${options[$index]}"
3319 [ "${options[$index]}" = "" ] && cmd="$LFS mirror resync"
3321 [ $((RANDOM % 4)) -eq 0 ] && {
3327 echo -n "resync file $tf with '$cmd' .."
3329 if [[ $lock_taken = "true" ]]; then
3331 $cmd $tf &> /dev/null && echo "done" || echo "failed"
3334 $cmd $tf &> /dev/null && echo "done" || echo "failed"
3337 sleep 0.$((RANDOM % 8 + 1))
3341 # this was test_200 before adding "b" and "c" subtests
3343 local tf=$DIR/$tfile
3344 local tf2=$DIR2/$tfile
3345 local tf3=$DIR3/$tfile
3347 ctrl_file=$(mktemp /tmp/CTRL.XXXXXX)
3348 lock_file=$(mktemp /var/lock/FLR.XXXXXX)
3349 stack_trap "rm -f $ctrl_file $lock_file $tf $tf-2 $tf-3"
3351 $LFS setstripe -E 1M -S 1M -E 2M -c 2 -E 4M -E 16M -E eof $tf
3352 $LFS setstripe -E 2M -S 1M -E 6M -c 2 -E 8M -E 32M -E eof $tf-2
3353 $LFS setstripe -E 4M -c 2 -E 8M -E 64M -E eof $tf-3
3355 $LFS mirror extend -N -f $tf-2 $tf ||
3356 error "merging $tf-2 into $tf failed"
3357 $LFS mirror extend -N -f $tf-3 $tf ||
3358 error "merging $tf-3 into $tf failed"
3360 mkdir -p $MOUNT2 && mount_client $MOUNT2
3362 mkdir -p $MOUNT3 && mount_client $MOUNT3
3364 verify_flr_state $tf3 "ro"
3366 #define OBD_FAIL_FLR_RANDOM_PICK_MIRROR 0x1A03
3367 $LCTL set_param fail_loc=0x1A03
3369 local mds_idx=mds$(($($LFS getstripe -m $tf) + 1))
3370 do_facet $mds_idx $LCTL set_param fail_loc=0x1A03
3374 write_file_200 $tf &
3380 write_file_200 $tf2 &
3383 read_file_200 $tf2 &
3386 resync_file_200 $tf3 &
3390 [ "$SLOW" = "yes" ] && sleep_time=360
3391 while [ $sleep_time -gt 0 -a -f $ctrl_file ]; do
3398 echo "Waiting ${pids[*]}"
3401 umount_client $MOUNT2
3402 umount_client $MOUNT3
3404 # resync and verify mirrors
3405 $LFS mirror resync $tf || error "final resync failed"
3408 local csum=$($LFS mirror read -N "${mirror_array[0]}" $tf | md5sum)
3410 for id in "${mirror_array[@]:1}"; do
3411 [ "$($LFS mirror read -N $id $tf | md5sum)" = "$csum" ] ||
3412 error "checksum error for mirror $id"
3417 run_test 200a "stress test"
3420 local tf=$DIR/$tfile
3421 local tf2=$DIR2/$tfile
3422 local tf3=$DIR3/$tfile
3424 ctrl_file=$(mktemp /tmp/CTRL.XXXXXX)
3425 lock_file=$(mktemp /var/lock/FLR.XXXXXX)
3426 stack_trap "rm -f $ctrl_file $lock_file $tf $tf-2 $tf-3"
3428 $LFS setstripe -E 1M -S 1M -E 2M -c 2 -E 4M -E 16M -E eof $tf
3429 $LFS setstripe -E 2M -S 1M -E 6M -c 2 -E 8M -E 32M -E eof $tf-2
3430 $LFS setstripe -E 4M -c 2 -E 8M -E 64M -E eof $tf-3
3432 $LFS mirror extend -N -f $tf-2 $tf ||
3433 error "merging $tf-2 into $tf failed"
3434 $LFS mirror extend -N -f $tf-3 $tf ||
3435 error "merging $tf-3 into $tf failed"
3437 mkdir -p $MOUNT2 && mount_client $MOUNT2
3439 mkdir -p $MOUNT3 && mount_client $MOUNT3
3441 verify_flr_state $tf3 "ro"
3443 #define OBD_FAIL_LLITE_PANIC_ON_ESTALE 0x1423
3444 $LCTL set_param fail_loc=0x1423
3448 write_file_200 $tf &
3454 write_file_200 $tf2 &
3457 read_file_200 $tf2 &
3460 resync_file_200 $tf3 &
3464 [ "$SLOW" = "yes" ] && sleep_time=400
3468 echo "Waiting ${pids[@]}"
3471 umount_client $MOUNT2
3472 umount_client $MOUNT3
3474 # resync and verify mirrors
3475 $LFS mirror resync $tf || {
3477 error "final resync failed"
3481 local csum=$($LFS mirror read -N ${mirror_array[0]} $tf | md5sum)
3482 for id in ${mirror_array[@]:1}; do
3483 [ "$($LFS mirror read -N $id $tf | md5sum)" = "$csum" ] ||
3484 error "checksum error for mirror $id"
3489 run_test 200b "racing IO, mirror extend and resync"
3492 (( MDS1_VERSION >= $(version_code 2.15.53) )) ||
3493 skip "Need MDS version at least 2.15.53"
3495 local tf=$DIR/$tfile
3496 local tf2=$DIR2/$tfile
3498 mkdir -p $MOUNT2 && mount_client $MOUNT2
3499 stack_trap "umount_client $MOUNT2"
3500 stack_trap "rm -f $tf"
3504 dd if=/dev/urandom of=$tf bs=1M count=2 || error "can't write"
3506 mdt_idx=$($LFS getstripe -m $tf)
3508 cancel_lru_locks mdc
3509 cancel_lru_locks osc
3511 # start a process modifying file, block it just
3512 # before layout lock acquisition
3513 #define OBD_FAIL_MDS_DELAY_OPEN 0x175
3514 do_facet mds$((mdt_idx+1)) $LCTL set_param fail_loc=0x80000175 fail_val=10
3515 #log "dd to stale replica"
3516 dd if=/dev/urandom of=$tf bs=1M count=2 oflag=direct conv=notrunc &
3522 $LFS mirror extend -N -c -1 $tf2 || {
3524 error "can't mirror"
3526 log "mirror extend done"
3527 do_facet mds$((mdt_idx+1)) $LCTL set_param fail_loc=0 fail_val=0
3529 # wait for blocking dd to complete and modify file
3530 wait $PID || error "2nd dd failed"
3533 verify_mirror_count $tf 2
3535 $LFS getstripe $tf | grep -q lcme_flags.*stale || {
3538 error "both replicas are still in sync"
3541 $LFS mirror verify -vvv $tf || {
3543 error "corrupted in-sync file"
3546 run_test 200c "layout change racing with open: LOVEA changes"
3548 cleanup_test_201() {
3549 do_facet $SINGLEMDS $LCTL --device $MDT0 changelog_deregister $CL_USER
3551 umount_client $MOUNT2
3555 local delay=${RESYNC_DELAY:-5}
3557 MDT0=$($LCTL get_param -n mdc.*.mds_server_uuid |
3558 awk '{ gsub(/_UUID/,""); print $1 }' | head -n1)
3560 stack_trap cleanup_test_201 EXIT
3562 CL_USER=$(do_facet $SINGLEMDS $LCTL --device $MDT0 \
3563 changelog_register -n)
3565 mkdir -p $MOUNT2 && mount_client $MOUNT2
3569 local log=$($LFS changelog $MDT0 $index | grep FLRW)
3570 [ -z "$log" ] && { sleep 1; continue; }
3572 index=$(echo $log | awk '{print $1}')
3573 local ts=$(date -d "$(echo $log | awk '{print $3}')" "+%s" -u)
3574 local fid=$(echo $log | awk '{print $6}' | sed -e 's/t=//')
3575 local file=$($LFS fid2path $MOUNT2 $fid 2> /dev/null)
3578 [ -z "$file" ] && continue
3580 local now=$(date +%s)
3582 echo "file: $file $fid was modified at $ts, now: $now, " \
3583 "will be resynced at $((ts+delay))"
3585 [ $now -lt $((ts + delay)) ] && sleep $((ts + delay - now))
3587 $LFS mirror resync $file
3588 echo "$file resync done"
3591 run_test 201 "FLR data mover"
3594 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
3596 local tf=$DIR/$tfile
3599 $LFS setstripe -E 1M -S 1M -c 1 $tf
3600 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
3601 verify_comp_attr stripe-count $tf ${ids[0]} 1
3603 $LFS setstripe --component-add -E 2M -c $OSTCOUNT $tf
3604 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
3605 verify_comp_attr stripe-count $tf ${ids[0]} 1
3606 verify_comp_attr stripe-count $tf ${ids[1]} $OSTCOUNT
3608 dd if=/dev/zero of=$tf bs=1M count=2
3609 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
3610 verify_comp_attr stripe-count $tf ${ids[0]} 1
3611 verify_comp_attr stripe-count $tf ${ids[1]} $OSTCOUNT
3613 run_test 202 "lfs setstripe --add-component wide striping"
3616 [ $MDS1_VERSION -lt $(version_code 2.11.55) ] &&
3617 skip "Need MDS version at least 2.11.55"
3618 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs"
3620 local tf=$DIR/$tfile
3623 $LFS mirror create -N2 -c1 $tf || error "create FLR file $tf"
3624 #delete first mirror
3625 $LFS mirror delete --mirror-id=1 $tf || error "delete first mirror"
3628 local old_id=$($LFS getstripe --mirror-id=2 -I $tf)
3629 local count=$($LFS getstripe --mirror-id=2 -c $tf) ||
3630 error "getstripe count of mirror 2"
3631 [[ x$count = x1 ]] || error "mirror 2 stripe count $count is not 1"
3633 #extend a mirror with 2 OSTs
3634 $LFS mirror extend -N -c2 $tf || error "extend mirror"
3637 local new_id=$($LFS getstripe --mirror-id=2 -I $tf)
3638 count=$($LFS getstripe --mirror-id=2 -c $tf) ||
3639 error "getstripe count of mirror 2"
3640 [[ x$old_id = x$new_id ]] ||
3641 error "mirror 2 changed ID from $old_id to $new_id"
3642 [[ x$count = x1 ]] || error "mirror 2 stripe count $count is not 1"
3644 count=$($LFS getstripe --mirror-id=3 -c $tf) ||
3645 error "getstripe count of mirror 3"
3646 [[ x$count = x2 ]] || error "mirror 3 stripe count $count is not 2"
3648 run_test 203 "mirror file preserve mirror ID"
3650 # Simple test of FLR + self-extending layout, SEL in non-primary mirror
3652 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3653 skip "skipped for lustre < $SEL_VER"
3655 local comp_file=$DIR/$tdir/$tfile
3659 test_mkdir $DIR/$tdir
3660 stack_trap "rm -f $comp_file"
3662 # first mirror is 0-10M, then 10M-(-1), second mirror is 1M followed
3663 # by extension space to -1
3664 $LFS setstripe -N -E 10M -E-1 -N -E 1M -E-1 -z64M $comp_file ||
3665 error "Create $comp_file failed"
3667 # Write to first component, extending & staling second mirror
3668 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc ||
3669 error "dd to extend + stale failed"
3671 $LFS getstripe $comp_file
3673 flg_opts="--component-flags init,stale"
3674 found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
3675 [ $found -eq 1 ] || error "write: Second comp end incorrect"
3677 flg_opts="--component-flags extension"
3678 found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
3679 [ $found -eq 1 ] || error "write: Third comp start incorrect"
3681 # mirror resync should not change the extents
3682 $LFS mirror resync $comp_file
3684 flg_opts="--component-flags init"
3685 found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
3686 [ $found -eq 1 ] || error "resync: Second comp end incorrect"
3688 flg_opts="--component-flags extension"
3689 found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
3690 [ $found -eq 1 ] || error "resync: Third comp start incorrect"
3692 sel_layout_sanity $comp_file 5
3694 run_test 204a "FLR write/stale/resync tests with self-extending mirror"
3696 # Simple test of FLR + self-extending layout, SEL in primary mirror
3698 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3699 skip "skipped for lustre < $SEL_VER"
3701 local comp_file=$DIR/$tdir/$tfile
3705 test_mkdir $DIR/$tdir
3706 stack_trap "rm -f $comp_file"
3708 # first mirror is 1M followed by extension space to -1, second mirror
3709 # is 0-10M, then 10M-(-1),
3710 $LFS setstripe -N -E 1M -E-1 -z64M -N -E 10M -E-1 $comp_file ||
3711 error "Create $comp_file failed"
3713 # Write to first component, extending first component & staling
3715 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc ||
3716 error "dd to extend + stale failed"
3718 $LFS getstripe $comp_file
3720 flg_opts="--component-flags init"
3721 found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
3722 [ $found -eq 1 ] || error "write: First comp end incorrect"
3724 flg_opts="--component-flags extension"
3725 found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
3726 [ $found -eq 1 ] || error "write: Second comp start incorrect"
3728 flg_opts="--component-flags init,stale"
3729 found=$($LFS find --component-end 10M $flg_opts $comp_file | wc -l)
3730 [ $found -eq 1 ] || error "write: First mirror comp flags incorrect"
3732 # This component is staled because it overlaps the extended first
3733 # component of the primary mirror, even though it doesn't overlap
3734 # the actual write - thus not inited.
3735 flg_opts="--component-flags stale"
3736 found=$($LFS find --component-start 10M $flg_opts $comp_file | wc -l)
3737 [ $found -eq 1 ] || error "write: Second mirror comp flags incorrect"
3739 # mirror resync should not change the extents
3740 $LFS mirror resync $comp_file
3742 $LFS getstripe $comp_file
3744 flg_opts="--component-flags init"
3745 found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
3746 [ $found -eq 1 ] || error "resync: First comp end incorrect"
3748 flg_opts="--component-flags extension"
3749 found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
3750 [ $found -eq 1 ] || error "resync: Second comp start incorrect"
3752 flg_opts="--component-flags init"
3753 found=$($LFS find --component-end 10M $flg_opts $comp_file | wc -l)
3754 [ $found -eq 1 ] || error "resync: First mirror comp flags incorrect"
3756 flg_opts="--component-flags init"
3757 found=$($LFS find --component-start 10M $flg_opts $comp_file | wc -l)
3758 [ $found -eq 1 ] || error "resync: Second mirror comp flags incorrect"
3760 sel_layout_sanity $comp_file 5
3762 run_test 204b "FLR write/stale/resync tests with self-extending primary"
3764 # FLR + SEL failed extension & component removal
3765 # extension space in second mirror
3767 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
3768 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3769 skip "skipped for lustre < $SEL_VER"
3771 local comp_file=$DIR/$tdir/$tfile
3774 local ost_name=$(ostname_from_index $ost_idx1)
3776 test_mkdir $DIR/$tdir
3777 stack_trap "rm -f $comp_file"
3779 # first mirror is is 0-10M, then 10M-(-1), second mirror is 0-1M, then
3780 # extension space from 1M to 1G, then normal space to -1
3781 $LFS setstripe -N -E 10M -E-1 -N -E 1M -E 1G -i $ost_idx1 -z 64M \
3782 -E -1 $comp_file || error "Create $comp_file failed"
3784 do_facet ost1 $LCTL set_param -n obdfilter.$ost_name.degraded=1
3787 # write to first comp (0 - 10M) of mirror 1, extending + staling
3788 # first + second comp of mirror 2
3789 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc
3792 do_facet ost1 $LCTL set_param -n obdfilter.$ost_name.degraded=0
3795 [ $RC -eq 0 ] || error "dd to extend + stale failed"
3797 $LFS getstripe $comp_file
3799 found=$($LFS find --component-start 0m --component-end 1m \
3800 --comp-flags init,stale $comp_file | wc -l)
3801 [ $found -eq 1 ] || error "write: First mirror comp incorrect"
3803 found=$($LFS find --component-start 1m --component-end EOF \
3804 --comp-flags stale,^init $comp_file | wc -l)
3805 [ $found -eq 1 ] || error "write: Second mirror comp incorrect"
3807 local mirror_id=$($LFS getstripe --component-start=1m \
3808 --component-end=EOF $comp_file | \
3809 grep lcme_mirror_id | awk '{ print $2 }')
3811 [[ $mirror_id -eq 2 ]] ||
3812 error "component not in correct mirror? $mirror_id"
3814 $LFS mirror resync $comp_file
3816 $LFS getstripe $comp_file
3818 # component dimensions should not change from resync
3819 found=$($LFS find --component-start 1m --component-end EOF \
3820 --component-flags init $comp_file | wc -l)
3821 [ $found -eq 1 ] || error "resync: Second mirror comp incorrect"
3823 sel_layout_sanity $comp_file 4
3825 run_test 204c "FLR write/stale/resync test with component removal"
3827 # Successful repeated component in primary mirror
3829 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
3830 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3831 skip "skipped for lustre < $SEL_VER"
3833 local comp_file=$DIR/$tdir/$tfile
3836 wait_delete_completed
3838 test_mkdir $DIR/$tdir
3839 stack_trap "rm -f $comp_file"
3841 # first mirror is 64M followed by extension space to -1, second mirror
3842 # is 0-10M, then 10M-(-1)
3843 $LFS setstripe -N -E-1 -z64M -N -E 10M -E-1 $comp_file ||
3844 error "Create $comp_file failed"
3846 local ost_idx1=$($LFS getstripe -I65537 -i $comp_file)
3847 local ost_name=$(ostname_from_index $ost_idx1)
3848 # degrade OST for first comp so we won't extend there
3849 do_facet ost$((ost_idx1+1)) $LCTL set_param -n \
3850 obdfilter.$ost_name.degraded=1
3853 # Write beyond first component, causing repeat & stale second mirror
3854 dd if=/dev/zero bs=1M count=1 seek=66 of=$comp_file conv=notrunc
3857 do_facet ost$((ost_idx1+1)) $LCTL set_param -n \
3858 obdfilter.$ost_name.degraded=0
3861 [ $RC -eq 0 ] || error "dd to repeat & stale failed"
3863 $LFS getstripe $comp_file
3865 found=$($LFS find --component-start 64m --component-end 128m \
3866 --component-flags init $comp_file | wc -l)
3867 [ $found -eq 1 ] || error "write: Repeat comp incorrect"
3869 local ost_idx2=$($LFS getstripe --component-start=64m \
3870 --component-end=128m --component-flags=init \
3872 [[ $ost_idx1 -eq $ost_idx2 ]] && error "$ost_idx1 == $ost_idx2"
3873 local mirror_id=$($LFS getstripe --component-start=64m \
3874 --component-end=128m --component-flags=init \
3875 $comp_file | grep lcme_mirror_id | awk '{ print $2 }')
3876 [[ $mirror_id -eq 1 ]] ||
3877 error "component not in correct mirror: $mirror_id, not 1"
3879 $LFS mirror resync $comp_file
3881 $LFS getstripe $comp_file
3883 # component dimensions should not change from resync
3884 found=$($LFS find --component-start 0m --component-end 64m \
3885 --component-flags init $comp_file | wc -l)
3886 [ $found -eq 1 ] || error "resync: first comp incorrect"
3887 found=$($LFS find --component-start 64m --component-end 128m \
3888 --component-flags init $comp_file | wc -l)
3889 [ $found -eq 1 ] || error "resync: repeat comp incorrect"
3891 sel_layout_sanity $comp_file 5
3893 run_test 204d "FLR write/stale/resync sel test with repeated comp"
3895 # Successful repeated component, SEL in non-primary mirror
3897 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
3898 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3899 skip "skipped for lustre < $SEL_VER"
3901 local comp_file=$DIR/$tdir/$tfile
3904 wait_delete_completed
3907 test_mkdir $DIR/$tdir
3908 stack_trap "rm -f $comp_file"
3910 # first mirror is is 0-100M, then 100M-(-1), second mirror is extension
3911 # space to -1 (-z 64M, so first comp is 0-64M)
3912 # Note: we have to place both 1st components on OST0, otherwise 2 OSTs
3913 # will be not enough - one will be degraded, the other is used on
3914 # an overlapping mirror.
3915 $LFS setstripe -N -E 100M -i 0 -E-1 -N -E-1 -i 0 -z 64M $comp_file ||
3916 error "Create $comp_file failed"
3918 local ost_idx1=$($LFS getstripe --component-start=0 \
3919 --component-end=64m -i $comp_file)
3920 local ost_name=$(ostname_from_index $ost_idx1)
3921 # degrade OST for first comp of 2nd mirror so we won't extend there
3922 do_facet ost$((ost_idx1+1)) $LCTL set_param -n \
3923 obdfilter.$ost_name.degraded=1
3926 $LFS getstripe $comp_file
3928 # Write to first component, stale & instantiate second mirror components
3929 # overlapping with the written component (0-100M);
3930 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc
3933 do_facet ost$((ost_idx1+1)) $LCTL set_param -n \
3934 obdfilter.$ost_name.degraded=0
3936 $LFS getstripe $comp_file
3938 [ $RC -eq 0 ] || error "dd to repeat & stale failed"
3940 found=$($LFS find --component-start 0m --component-end 64m \
3941 --component-flags init,stale $comp_file | wc -l)
3942 [ $found -eq 1 ] || error "write: first comp incorrect"
3944 # was repeated due to degraded ost
3945 found=$($LFS find --component-start 64m --component-end 128m \
3946 --component-flags init,stale $comp_file | wc -l)
3947 [ $found -eq 1 ] || error "write: repeated comp incorrect"
3949 local ost_idx2=$($LFS getstripe --component-start=64m \
3950 --component-end=128m --component-flags=init \
3952 [[ $ost_idx1 -eq $ost_idx2 ]] && error "$ost_idx1 == $ost_idx2"
3953 local mirror_id=$($LFS getstripe --component-start=0m \
3954 --component-end=64m --component-flags=init \
3955 $comp_file | grep lcme_mirror_id | awk '{ print $2 }')
3956 [[ $mirror_id -eq 2 ]] ||
3957 error "component not in correct mirror? $mirror_id"
3959 $LFS mirror resync $comp_file
3961 $LFS getstripe $comp_file
3963 # component dimensions should not change from resync
3964 found=$($LFS find --component-start 0m --component-end 64m \
3965 --component-flags init,^stale $comp_file | wc -l)
3966 [ $found -eq 1 ] || error "resync: first comp incorrect"
3967 found=$($LFS find --component-start 64m --component-end 128m \
3968 --component-flags init,^stale $comp_file | wc -l)
3969 [ $found -eq 1 ] || error "resync: repeated comp incorrect"
3971 sel_layout_sanity $comp_file 5
3973 run_test 204e "FLR write/stale/resync sel test with repeated comp"
3975 # FLR + SEL: failed repeated component, SEL in non-primary mirror
3977 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
3978 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3979 skip "skipped for lustre < $SEL_VER"
3981 local comp_file=$DIR/$tdir/$tfile
3984 wait_delete_completed
3986 test_mkdir $DIR/$tdir
3987 stack_trap "rm -f $comp_file"
3989 pool_add $TESTNAME || error "Pool creation failed"
3990 pool_add_targets $TESTNAME 0 1 || error "Pool add targets failed"
3992 # first mirror is is 0-100M, then 100M-(-1), second mirror is extension
3993 # space to -1 (-z 64M, so first comp is 0-64M)
3994 $LFS setstripe -N -E 100M -E-1 -N --pool="$TESTNAME" \
3995 -E-1 -c 1 -z 64M $comp_file || error "Create $comp_file failed"
3997 local ost_name0=$(ostname_from_index 0)
3998 local ost_name1=$(ostname_from_index 1)
4000 # degrade both OSTs in pool, so we'll try to repeat, then fail and
4001 # extend original comp
4002 do_facet ost1 $LCTL set_param -n obdfilter.$ost_name0.degraded=1
4003 do_facet ost2 $LCTL set_param -n obdfilter.$ost_name1.degraded=1
4006 # a write to the 1st component, 100M length, which will try to stale
4007 # the first 100M of mirror 2, attempting to extend its 0-64M component
4008 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc
4011 do_facet ost1 $LCTL set_param -n obdfilter.$ost_name0.degraded=0
4012 do_facet ost2 $LCTL set_param -n obdfilter.$ost_name1.degraded=0
4015 [ $RC -eq 0 ] || error "dd to extend mirror comp failed"
4017 $LFS getstripe $comp_file
4019 found=$($LFS find --component-start 0m --component-end 128m \
4020 --component-flags init,stale $comp_file | wc -l)
4021 [ $found -eq 1 ] || error "write: First mirror comp incorrect"
4023 local mirror_id=$($LFS getstripe --component-start=0m \
4024 --component-end=128m --component-flags=init \
4025 $comp_file | grep lcme_mirror_id | awk '{ print $2 }')
4027 [[ $mirror_id -eq 2 ]] ||
4028 error "component not in correct mirror? $mirror_id, not 2"
4030 $LFS mirror resync $comp_file
4032 $LFS getstripe $comp_file
4034 # component dimensions should not change from resync
4035 found=$($LFS find --component-start 0m --component-end 128m \
4036 --component-flags init,^stale $comp_file | wc -l)
4037 [ $found -eq 1 ] || error "resync: First mirror comp incorrect"
4039 sel_layout_sanity $comp_file 4
4041 run_test 204f "FLR write/stale/resync sel w/forced extension"
4043 function test_205() {
4044 local tf=$DIR/$tfile
4047 $LFS setstripe -c1 $tf
4048 $LFS mirror extend -N $tf
4049 mirrors=$($LFS getstripe $tf | grep lcme_mirror_id | wc -l )
4050 (( $mirrors == 2 )) || error "no new mirror was created?"
4052 $LFS mirror extend -N --flags=prefer $tf
4053 mirrors=$($LFS getstripe $tf | grep lcme_mirror_id | wc -l )
4054 (( $mirrors == 3 )) || error "no new mirror was created?"
4056 $($LFS getstripe $tf | grep lcme_flags: | tail -1 | grep -q prefer) ||
4057 error "prefer flag was not set on the new mirror"
4059 run_test 205 "lfs mirror extend to set prefer flag"
4061 function test_206() {
4062 # create a new OST pool
4063 local pool_name=$TESTNAME
4065 create_pool $FSNAME.$pool_name ||
4066 error "create OST pool $pool_name failed"
4067 # add OSTs into the pool
4068 pool_add_targets $pool_name 0 1 ||
4069 error "add OSTs into pool $pool_name failed"
4071 $LFS setstripe -c1 --pool=$pool_name $DIR/$tfile ||
4072 error "can't setstripe"
4073 $LFS mirror extend -N $DIR/$tfile ||
4074 error "can't create replica"
4075 if $LFS getstripe $DIR/$tfile | grep -q prefer ; then
4076 $LFS getstripe $DIR/$tfile
4077 error "prefer found"
4079 $LFS setstripe --comp-set --comp-flags=prefer -p $pool_name $DIR/$tfile || {
4080 $LFS getstripe $DIR/$tfile
4081 error "can't setstripe prefer"
4084 if ! $LFS getstripe $DIR/$tfile | grep -q prefer ; then
4085 $LFS getstripe $DIR/$tfile
4086 error "no prefer found"
4092 run_test 206 "lfs setstripe -pool .. --comp-flags=.. "
4095 local file=$DIR/$tfile
4096 local tmpfile=$DIR/$tfile-tt
4098 [ $MDS1_VERSION -lt $(version_code 2.14.50) ] &&
4099 skip "Need MDS version at least 2.14.50"
4101 stack_trap "rm -f $tmpfile $file"
4103 # generate data for verification
4104 dd if=/dev/urandom of=$tmpfile bs=1M count=1 ||
4105 error "can't generate file with random data"
4107 # create a mirrored file with one stale replica
4108 $LFS mirror create -N -S 4M -c 2 -N -S 1M -c -1 $file ||
4109 error "create mirrored file $file failed"
4110 get_mirror_ids $file
4111 echo "mirror IDs: ${mirror_array[*]}"
4113 dd if=$tmpfile of=$file bs=1M || error "can't copy"
4114 get_mirror_ids $file
4115 echo "mirror IDs: ${mirror_array[*]}"
4118 cmp $tmpfile $file || error "files don't match"
4119 get_mirror_ids $file
4120 echo "mirror IDs: ${mirror_array[*]}"
4122 # mirror creation should work fine
4123 $LFS mirror extend -N -S 8M -c -1 $file ||
4124 error "mirror extend $file failed"
4126 get_mirror_ids $file
4127 echo "mirror IDs: ${mirror_array[*]}"
4130 $LFS mirror verify -v $file || error "verification failed"
4131 cmp $tmpfile $file || error "files don't match"
4133 run_test 207 "create another replica with existing out-of-sync one"
4135 function check_ost_used() {
4144 cancel_lru_locks osc # to drop pages
4145 cancel_lru_locks mdc # to refresh layout
4146 # XXX: cancel_lru_locks mdc doesn't work
4147 # XXX: need a better way to reload the layout
4148 umount_client $MOUNT || error "umount failed"
4149 mount_client $MOUNT || error "mount failed"
4151 # refresh non-rotation status on MDTs
4153 touch $DIR/$tfile-temp
4154 rm -f $DIR/$tfile-temp
4155 # refresh non-rotational status on the client
4159 $LCTL set_param osc.*.stats=clear >/dev/null
4160 if [[ $io == "read" ]]; then
4161 ddarg="if=$file of=/dev/null"
4162 elif [[ $io == "write" ]]; then
4163 ddarg="if=/dev/zero of=$file"
4165 error "unknown type $io"
4167 dd $ddarg bs=2M count=1 || error "can't $io $file"
4168 cancel_lru_locks osc
4170 # check only specified OSTs got reads
4171 for ((ost = 0; ost < $OSTCOUNT; ost++)); do
4172 local nr=$($LCTL get_param -n \
4173 osc.$FSNAME-OST000$ost-osc-[-0-9a-f]*.stats |
4174 awk "/ost_$io/{print \$2}")
4176 if [[ " $* " =~ $ost ]]; then
4177 (( nr > 0 )) || error "expected reads on $ost"
4179 (( nr == 0 )) || error "unexpected $nr reads on $ost"
4185 local tf=$DIR/$tfile
4186 local osts=$(comma_list $(osts_nodes))
4188 (( $OSTCOUNT >= 4 )) || skip "needs >= 4 OSTs"
4189 (( $MDS1_VERSION >= $(version_code 2.14.55) )) ||
4190 skip "Need MDS version at least 2.14.55"
4192 local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
4194 save_lustre_params $(get_facets OST) osd*.*OST*.nonrotational > $p
4195 stack_trap "restore_lustre_params < $p; rm -f $p"
4197 stack_trap "rm -f $tf"
4198 $LFS setstripe -i0 -c1 $tf || error "can't setstripe"
4199 dd if=/dev/zero of=$tf bs=2M count=1 || error "can't dd (1)"
4200 $LFS mirror extend -N -c1 -o1 $tf || error "can't create mirror"
4201 $LFS mirror extend -N -c2 -o 2,3 $tf || error "can't create mirror"
4202 $LFS mirror resync $tf || error "can't resync"
4205 log "set OST0000 non-rotational"
4207 $LCTL set_param osd*.*OST*.nonrotational=0
4209 $LCTL set_param osd*.*OST0000*.nonrotational=1
4210 check_ost_used $tf read 0
4212 log "set OST0002 and OST0003 non-rotational, two fast OSTs is better"
4214 $LCTL set_param osd*.*OST*.nonrotational=0
4216 $LCTL set_param osd*.*OST0002*.nonrotational=1 \
4217 osd*.*OST0003*.nonrotational=1
4218 check_ost_used $tf read 2 3
4220 log "set mirror 1 on OST0001 preferred"
4221 $LFS setstripe --comp-set -I 0x20001 --comp-flags=prefer $tf ||
4222 error "can't set prefer"
4223 check_ost_used $tf read 1
4225 run_test 208a "mirror selection to prefer non-rotational devices for reads"
4228 local tf=$DIR/$tfile
4229 local osts=$(comma_list $(osts_nodes))
4231 (( $OSTCOUNT >= 4 )) || skip "needs >= 4 OSTs"
4232 (( $MDS1_VERSION >= $(version_code 2.14.55) )) ||
4233 skip "Need MDS version at least 2.14.55"
4235 local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
4237 save_lustre_params $(get_facets OST) osd*.*OST*.nonrotational > $p
4238 stack_trap "restore_lustre_params < $p; rm -f $p"
4240 stack_trap "rm -f $tf"
4241 $LFS setstripe -i0 -c1 $tf || error "can't setstripe"
4242 dd if=/dev/zero of=$tf bs=2M count=1 || error "can't dd (1)"
4243 $LFS mirror extend -N -c1 -o1 $tf || error "can't create mirror"
4244 $LFS mirror extend -N -c2 -o 2,3 $tf || error "can't create mirror"
4245 $LFS mirror resync $tf || error "can't resync"
4246 $LFS getstripe $tf | grep -q flags.*stale && error "still stale"
4248 log "set OST0000 non-rotational"
4250 $LCTL set_param osd*.*OST*.nonrotational=0
4252 $LCTL set_param osd*.*OST0000*.nonrotational=1
4253 check_ost_used $tf write 0
4254 $LFS mirror resync $tf || error "can't resync"
4256 log "set OST0002 and OST0003 non-rotational, two fast OSTs is better"
4258 $LCTL set_param osd*.*OST*.nonrotational=0
4260 $LCTL set_param osd*.*OST0002*.nonrotational=1 \
4261 osd*.*OST0003*.nonrotational=1
4262 check_ost_used $tf write 2 3
4263 $LFS mirror resync $tf || error "can't resync"
4265 log "set mirror 1 on OST0001 preferred"
4266 $LFS setstripe --comp-set -I 0x20001 --comp-flags=prefer $tf ||
4267 error "can't set prefer"
4268 check_ost_used $tf write 1
4270 run_test 208b "mirror selection to prefer non-rotational devices for writes"
4273 local tf=$DIR/$tfile
4274 local tmpfile="$TMP/$TESTSUITE-$TESTNAME-multiop.output"
4275 local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
4276 local osts=$(comma_list $(osts_nodes))
4278 stack_trap "rm -f $tmpfile"
4280 mkdir -p $MOUNT2 && mount_client $MOUNT2
4281 stack_trap "umount_client $MOUNT2"
4283 # to make replica on ost1 preferred for new writes
4284 save_lustre_params $(get_facets OST) osd*.*OST*.nonrotational > $p
4285 stack_trap "restore_lustre_params < $p; rm -f $p"
4287 $LCTL set_param osd*.*OST*.nonrotational=0
4289 $LCTL set_param osd*.*OST0001*.nonrotational=1
4291 $LFS setstripe -c1 -i0 $tf || errro "can't create $tf"
4293 $LFS mirror extend -N -o1 $tf || error "can't make replica"
4294 log "replicated file created"
4296 cancel_lru_locks mdc
4297 cancel_lru_locks osc
4299 log "open(O_RDONLY) and first read from OST"
4300 $MULTIOP $tf vvoO_RDONLY:r4_z0r4_z0r4c >$tmpfile &
4303 log "first read complete"
4305 echo "BBBB" | dd bs=1 count=4 of=$DIR2/$tfile conv=notrunc ||
4306 error "can't write BBBB"
4307 log "BBBB written which made replica on ost1 stale"
4309 log "fast read from pagecache in the original process"
4313 log "read via $DIR2 new open(2)"
4314 $MULTIOP $DIR2/$tfile vvoO_RDONLY:r4c
4316 log "fast read from pagecache after 5s in the original process"
4321 local nr=$(grep "BBBB" $tmpfile | wc -l)
4327 log "read via new open(2)"
4328 $MULTIOP $tf vvoO_RDONLY:r4c
4330 run_test 209a "skip fast reads after layout invalidation"
4332 function sum_ost_reads() {
4333 $LCTL get_param -n osc.$FSNAME-OST*-osc-[-0-9a-f]*.stats |
4334 awk '/^ost_read/{sum=sum+$2}END{print sum}'
4338 local tf=$DIR/$tfile
4340 dd if=/dev/zero of=$tf bs=4k count=2 || error "can't create file"
4341 cancel_lru_locks osc
4342 echo "the very first read"
4343 cat $tf >/dev/null || error "can't read"
4345 # cancel layout lock
4346 cancel_lru_locks mdc
4348 # now read again, data must be in the cache, so no ost reads
4349 $LCTL set_param osc.*.stats=clear >/dev/null
4350 echo "read with warm cache"
4351 cat $tf >/dev/null || error "can't read"
4353 (( nr == 0 )) || error "reads with warm cache"
4355 # now verify we can catch reads at all
4356 cancel_lru_locks osc
4357 cat $tf >/dev/null || error "can't read"
4359 (( nr > 0 )) || error "no reads with cold cache"
4361 run_test 209b "pagecache can be used after LL cancellation"
4363 complete_test $SECONDS
4364 check_and_cleanup_lustre