3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
11 LUSTRE=${LUSTRE:-$(dirname $0)/..}
12 . $LUSTRE/tests/test-framework.sh
16 ALWAYS_EXCEPT="$SANITY_FLR_EXCEPT "
17 # Bug number for skipped test: LU-14818 LU-11381 LU-14765
18 ALWAYS_EXCEPT+=" 6 201 44c "
19 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
21 # skip all tests for PPC until we can get sanity-pfl to pass
22 if [[ $(uname -m) = ppc64 ]]; then
23 skip "Skip FLR testing for PPC clients"
26 if [[ "$ost1_FSTYPE" == "zfs" ]]; then
28 ALWAYS_EXCEPT+=" 49a "
33 [[ "$MDS1_VERSION" -ge $(version_code 2.10.56) ]] ||
34 skip "Need MDS version at least 2.10.56"
36 check_and_setup_lustre
39 rm -rf $DIR/[Rdfs][0-9]*
41 [ $UID -eq 0 -a $RUNAS_ID -eq 0 ] &&
42 error "\$RUNAS_ID set to 0, but \$UID is also 0!"
44 check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS
47 # global array to store mirror IDs
48 declare -a mirror_array
55 for id in $($LFS getstripe $tf | awk '/lcme_id/{print $2}'); do
56 array[${#array[@]}]=$((id >> 16))
59 mirror_array=($(printf "%s\n" "${array[@]}" | sort -u))
61 echo ${#mirror_array[@]}
65 echo 3 > /proc/sys/vm/drop_caches
76 wait_osc_import_state client ost$idx "\(DISCONN\|IDLE\)"
84 start ost$idx $(ostdevname $idx) $OST_MOUNT_OPTS ||
85 error "start ost$idx failed"
89 wait_recovery_complete ost$idx
94 # Verify mirror count with an expected value for a given file.
96 verify_mirror_count() {
99 local mirror_count=$($LFS getstripe -N $tf)
101 [[ $mirror_count = $expected ]] || {
102 $LFS getstripe -v $tf
103 error "verify mirror count failed on $tf:" \
104 "$mirror_count != $expected"
109 # Verify component count with an expected value for a given file.
110 # $1 coposited layout file
111 # $2 expected component number
113 verify_comp_count() {
116 local comp_count=$($LFS getstripe --component-count $tf)
118 [[ $comp_count = $expected ]] || {
119 $LFS getstripe -v $tf
120 error "verify component count failed on $tf:" \
121 "$comp_count != $expected"
126 # Verify component attribute with an expected value for a given file
134 local cmd="$LFS getstripe -I$comp_id"
135 local getstripe_cmd="$cmd -v"
139 stripe-size) cmd+=" -S $tf" ;;
140 stripe-count) cmd+=" -c $tf" ;;
141 stripe-index) cmd+=" -i $tf" ;;
142 pool) cmd+=" -p $tf" ;;
143 comp-start) cmd+=" --component-start $tf" ;;
144 comp-end) cmd+=" --component-end $tf" ;;
145 lcme_flags) cmd+=" $tf | awk '/lcme_flags:/ { print \$2 }'" ;;
146 *) error "invalid attribute $attr";;
151 [ $attr = lcme_flags ] && {
153 local expected_list=$(comma_list $expected)
154 for fl in ${expected_list//,/ }; do
157 [[ ${fl:0:1} = "^" ]] && neg=1
158 [[ $neg = 1 ]] && fl=${fl:1}
160 $(echo $value | grep -q $fl)
162 # 0: matched; 1: not matched
164 if [[ $neg = 0 && $match != 0 ||
165 $neg = 1 && $match = 0 ]]; then
167 [[ $neg = 0 ]] && # expect the flag
168 error "expected flag '$fl' not in $comp_id"
169 [[ $neg = 1 ]] && # not expect the flag
170 error "not expected flag '$fl' in $comp_id"
176 [[ $value = $expected ]] || {
178 error "verify $attr failed on $tf: $value != $expected"
183 # Verify component extent with expected start and end extent values
184 # for a given file and component ID.
186 verify_comp_extent() {
189 local expected_start=$3
190 local expected_end=$4
192 verify_comp_attr comp-start $tf $comp_id $expected_start
193 verify_comp_attr comp-end $tf $comp_id $expected_end
197 # Verify component attribute with parent directory for a given file
200 verify_comp_attr_with_parent() {
204 local td=$(cd $(dirname $tf); echo $PWD)
205 local tf_cmd="$LFS getstripe -I$comp_id"
206 local td_cmd="$LFS getstripe"
212 stripe-size) opt="-S" ;;
213 stripe-count) opt="-c" ;;
215 *) error "invalid attribute $attr";;
218 expected=$($td_cmd $opt $td)
219 [[ $expected = -1 ]] && expected=$OSTCOUNT
221 value=$($tf_cmd $opt $tf)
222 [[ $value = -1 ]] && value=$OSTCOUNT
224 [[ $value = $expected ]] || {
227 error "verify $attr failed with parent on $tf:" \
228 "$value != $expected"
233 # Verify component attribute with filesystem-wide default value for a given file
236 verify_comp_attr_with_default() {
240 local tf_cmd="$LFS getstripe -I$comp_id"
248 expected=$($LCTL get_param -n \
249 lov.$FSNAME-clilov-*.stripesize)
253 expected=$($LCTL get_param -n \
254 lov.$FSNAME-clilov-*.stripecount)
255 [[ $expected = -1 ]] && expected=$OSTCOUNT
257 *) error "invalid attribute $attr";;
260 value=$($tf_cmd $opt $tf)
261 [[ $value = -1 ]] && value=$OSTCOUNT
263 [[ $value = $expected ]] || {
265 error "verify $attr failed with default value on $tf:" \
266 "$value != $expected"
271 # Verify unspecified component attributes for a given file
274 # This will only verify the inherited attributes:
275 # stripe size, stripe count and OST pool name
277 verify_comp_attrs() {
281 verify_comp_attr_with_default stripe-size $tf $comp_id
282 verify_comp_attr_with_default stripe-count $tf $comp_id
283 verify_comp_attr_with_parent pool $tf $comp_id
289 local expected_state=$2
291 local state=$($LFS getstripe -v $tf | awk '/lcm_flags/{ print $2 }')
292 [ $expected_state = $state ] ||
293 error "expected: $expected_state, actual $state"
296 # command line test cases
300 local mirror_count=16 # LUSTRE_MIRROR_COUNT_MAX
301 local mirror_cmd="$LFS mirror create"
306 # create parent directory
307 mkdir $td || error "mkdir $td failed"
309 $mirror_cmd $tf &> /dev/null && error "miss -N option"
311 $mirror_cmd -N $tf || error "create mirrored file $tf failed"
312 verify_mirror_count $tf 1
313 id=$($LFS getstripe -I $tf)
314 verify_comp_attrs $tf $id
315 verify_comp_extent $tf $id 0 EOF
317 $mirror_cmd -N0 $tf-1 &> /dev/null && error "invalid mirror count 0"
318 $mirror_cmd -N$((mirror_count + 1)) $tf-1 &> /dev/null &&
319 error "invalid mirror count $((mirror_count + 1))"
321 $mirror_cmd -N$mirror_count $tf-1 ||
322 error "create mirrored file $tf-1 failed"
323 verify_mirror_count $tf-1 $mirror_count
324 ids=($($LFS getstripe $tf-1 | awk '/lcme_id/{print $2}' | tr '\n' ' '))
325 for ((i = 0; i < $mirror_count; i++)); do
326 verify_comp_attrs $tf-1 ${ids[$i]}
327 verify_comp_extent $tf-1 ${ids[$i]} 0 EOF
330 $mirror_cmd -N -N2 -N3 -N4 $tf-2 ||
331 error "create mirrored file $tf-2 failed"
332 verify_mirror_count $tf-2 10
333 ids=($($LFS getstripe $tf-2 | awk '/lcme_id/{print $2}' | tr '\n' ' '))
334 for ((i = 0; i < 10; i++)); do
335 verify_comp_attrs $tf-2 ${ids[$i]}
336 verify_comp_extent $tf-2 ${ids[$i]} 0 EOF
339 run_test 0a "lfs mirror create with -N option"
342 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
346 local mirror_cmd="$LFS mirror create"
350 # create a new OST pool
351 local pool_name=$TESTNAME
352 create_pool $FSNAME.$pool_name ||
353 error "create OST pool $pool_name failed"
355 # add OSTs into the pool
356 pool_add_targets $pool_name 0 $((OSTCOUNT - 1)) ||
357 error "add OSTs into pool $pool_name failed"
359 # create parent directory
360 mkdir $td || error "mkdir $td failed"
361 $LFS setstripe -S 8M -c -1 -p $pool_name $td ||
362 error "$LFS setstripe $td failed"
364 # create a mirrored file with plain layout mirrors
365 $mirror_cmd -N -N -S 4M -c 2 -p flash -i 2 -o 2,3 \
366 -N -S 16M -N -c -1 -N -p archive -N -p none $tf ||
367 error "create mirrored file $tf failed"
368 verify_mirror_count $tf 6
369 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
370 for ((i = 0; i < 6; i++)); do
371 verify_comp_extent $tf ${ids[$i]} 0 EOF
374 # verify component ${ids[0]}
375 verify_comp_attrs $tf ${ids[0]}
377 # verify component ${ids[1]}
378 verify_comp_attr stripe-size $tf ${ids[1]} 4194304
379 verify_comp_attr stripe-count $tf ${ids[1]} 2
380 verify_comp_attr stripe-index $tf ${ids[1]} 2
382 # verify component ${ids[2]}
383 verify_comp_attr stripe-size $tf ${ids[2]} 16777216
384 verify_comp_attr stripe-count $tf ${ids[2]} 2
385 verify_comp_attr pool $tf ${ids[2]} flash
387 # verify component ${ids[3]}
388 verify_comp_attr stripe-size $tf ${ids[3]} 16777216
389 verify_comp_attr stripe-count $tf ${ids[3]} $OSTCOUNT
390 verify_comp_attr pool $tf ${ids[3]} flash
392 # verify component ${ids[4]}
393 verify_comp_attr stripe-size $tf ${ids[4]} 16777216
394 verify_comp_attr stripe-count $tf ${ids[4]} $OSTCOUNT
395 verify_comp_attr pool $tf ${ids[4]} archive
397 # verify component ${ids[5]}
398 verify_comp_attr stripe-size $tf ${ids[5]} 16777216
399 verify_comp_attr stripe-count $tf ${ids[5]} $OSTCOUNT
400 verify_comp_attr_with_parent pool $tf ${ids[5]}
402 if [ $MDS1_VERSION -ge $(version_code 2.12.55) ]; then
403 # LU-11022 - remove mirror by pool name
404 local=cnt cnt=$($LFS getstripe $tf | grep archive | wc -l)
405 [ "$cnt" != "1" ] && error "unexpected mirror count $cnt"
406 $LFS mirror delete --pool archive $tf || error "delete mirror"
407 cnt=$($LFS getstripe $tf | grep archive | wc -l)
408 [ "$cnt" != "0" ] && error "mirror count after removal: $cnt"
414 run_test 0b "lfs mirror create plain layout mirrors"
417 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
421 local mirror_cmd="$LFS mirror create"
425 # create a new OST pool
426 local pool_name=$TESTNAME
427 create_pool $FSNAME.$pool_name ||
428 error "create OST pool $pool_name failed"
430 # add OSTs into the pool
431 pool_add_targets $pool_name 0 $((OSTCOUNT - 1)) ||
432 error "add OSTs into pool $pool_name failed"
434 # create parent directory
435 mkdir $td || error "mkdir $td failed"
436 $LFS setstripe -E 32M -S 8M -c -1 -p $pool_name -E eof -S 16M $td ||
437 error "$LFS setstripe $td failed"
439 # create a mirrored file with composite layout mirrors
440 $mirror_cmd -N2 -E 4M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
442 -N3 -E 512M -S 16M -p archive -E -1 -i -1 -c -1 $tf ||
443 error "create mirrored file $tf failed"
444 verify_mirror_count $tf 6
445 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
447 # verify components ${ids[0]} and ${ids[2]}
449 verify_comp_attr_with_default stripe-size $tf ${ids[$i]}
450 verify_comp_attr stripe-count $tf ${ids[$i]} 2
451 verify_comp_attr stripe-index $tf ${ids[$i]} 1
452 verify_comp_extent $tf ${ids[$i]} 0 4194304
455 # verify components ${ids[1]} and ${ids[3]}
457 verify_comp_attr stripe-size $tf ${ids[$i]} 4194304
458 verify_comp_attr stripe-count $tf ${ids[$i]} 2
459 verify_comp_attr pool $tf ${ids[$i]} flash
460 verify_comp_extent $tf ${ids[$i]} 4194304 EOF
463 # verify component ${ids[4]}
464 verify_comp_attr stripe-size $tf ${ids[4]} 4194304
465 verify_comp_attr stripe-count $tf ${ids[4]} 4
466 verify_comp_attr_with_parent pool $tf ${ids[4]}
467 verify_comp_extent $tf ${ids[4]} 0 EOF
469 # verify components ${ids[5]}, ${ids[7]} and ${ids[9]}
471 verify_comp_attr stripe-size $tf ${ids[$i]} 16777216
472 verify_comp_attr stripe-count $tf ${ids[$i]} 4
473 verify_comp_attr pool $tf ${ids[$i]} archive
474 verify_comp_extent $tf ${ids[$i]} 0 536870912
477 # verify components ${ids[6]}, ${ids[8]} and ${ids[10]}
479 verify_comp_attr stripe-size $tf ${ids[$i]} 16777216
480 verify_comp_attr stripe-count $tf ${ids[$i]} -1
481 verify_comp_attr pool $tf ${ids[$i]} archive
482 verify_comp_extent $tf ${ids[$i]} 536870912 EOF
488 run_test 0c "lfs mirror create composite layout mirrors"
493 local mirror_count=16 # LUSTRE_MIRROR_COUNT_MAX
494 local mirror_cmd="$LFS mirror extend"
498 # create parent directory
499 mkdir $td || error "mkdir $td failed"
501 $mirror_cmd $tf &> /dev/null && error "miss -N option"
502 $mirror_cmd -N $tf &> /dev/null && error "$tf does not exist"
504 # create a non-mirrored file, convert it to a mirrored file and extend
505 touch $tf || error "touch $tf failed"
506 $mirror_cmd -N $tf || error "convert and extend $tf failed"
507 verify_mirror_count $tf 2
508 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
509 for ((i = 0; i < 2; i++)); do
510 verify_comp_attrs $tf ${ids[$i]}
511 verify_comp_extent $tf ${ids[$i]} 0 EOF
514 lfsck_verify_pfid $tf || error "PFID is not set"
516 # create a mirrored file and extend it
517 $LFS mirror create -N $tf-1 || error "create mirrored file $tf-1 failed"
518 $LFS mirror create -N $tf-2 || error "create mirrored file $tf-2 failed"
519 $LFS mirror create -N $tf-3 || error "create mirrored file $tf-3 failed"
521 $mirror_cmd -N -S 4M -N -f $tf-2 $tf-1 ||
522 error "extend mirror with -f failed"
524 $mirror_cmd -N$((mirror_count - 1)) $tf-3 ||
525 error "extend mirrored file $tf-3 failed"
526 verify_mirror_count $tf-3 $mirror_count
527 ids=($($LFS getstripe $tf-3 | awk '/lcme_id/{print $2}' | tr '\n' ' '))
528 for ((i = 0; i < $mirror_count; i++)); do
529 verify_comp_attrs $tf-3 ${ids[$i]}
530 verify_comp_extent $tf-3 ${ids[$i]} 0 EOF
533 $mirror_cmd -N $tf-3 &> /dev/null
536 error "exceeded maximum mirror count returns $rc not ERANGE(34)"
538 run_test 0d "lfs mirror extend with -N option"
541 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
545 local mirror_cmd="$LFS mirror extend"
549 # create parent directory
550 mkdir $td || error "mkdir $td failed"
552 # create a mirrored file with plain layout mirrors
553 $LFS mirror create -N -S 32M -c 3 -p ssd -i 1 -o 1,2,3 $tf ||
554 error "create mirrored file $tf failed"
556 # extend the mirrored file with plain layout mirrors
557 $mirror_cmd -N -S 4M -c 2 -p flash -i 2 -o 2,3 \
558 -N -S 16M -N -c -1 -N -p archive -N -p none $tf ||
559 error "extend mirrored file $tf failed"
560 verify_mirror_count $tf 6
561 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
562 for ((i = 0; i < 6; i++)); do
563 verify_comp_extent $tf ${ids[$i]} 0 EOF
566 # verify component ${ids[0]}
567 verify_comp_attr stripe-size $tf ${ids[0]} 33554432
568 verify_comp_attr stripe-count $tf ${ids[0]} 3
569 verify_comp_attr stripe-index $tf ${ids[0]} 1
571 # verify component ${ids[1]}
572 verify_comp_attr stripe-size $tf ${ids[1]} 4194304
573 verify_comp_attr stripe-count $tf ${ids[1]} 2
574 verify_comp_attr stripe-index $tf ${ids[1]} 2
576 # verify component ${ids[2]}
577 verify_comp_attr stripe-size $tf ${ids[2]} 16777216
578 verify_comp_attr stripe-count $tf ${ids[2]} 2
579 verify_comp_attr pool $tf ${ids[2]} flash
581 # verify component ${ids[3]}
582 verify_comp_attr stripe-size $tf ${ids[3]} 16777216
583 verify_comp_attr stripe-count $tf ${ids[3]} $OSTCOUNT
584 verify_comp_attr pool $tf ${ids[3]} flash
586 # verify component ${ids[4]}
587 verify_comp_attr stripe-size $tf ${ids[4]} 16777216
588 verify_comp_attr stripe-count $tf ${ids[4]} $OSTCOUNT
589 verify_comp_attr pool $tf ${ids[4]} archive
591 # verify component ${ids[5]}
592 verify_comp_attr stripe-size $tf ${ids[5]} 16777216
593 verify_comp_attr stripe-count $tf ${ids[5]} $OSTCOUNT
594 verify_comp_attr_with_parent pool $tf ${ids[5]}
596 run_test 0e "lfs mirror extend plain layout mirrors"
599 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
603 local mirror_cmd="$LFS mirror extend"
607 # create parent directory
608 mkdir $td || error "mkdir $td failed"
610 # create a mirrored file with composite layout mirror
611 $LFS mirror create -N -E 32M -S 16M -p ssd -E eof -S 32M $tf ||
612 error "create mirrored file $tf failed"
614 # extend the mirrored file with composite layout mirrors
615 $mirror_cmd -N -p archive \
616 -N2 -E 4M -c 2 -p flash -i 1 -o 1,3 -E eof -S 4M \
618 -N3 -E 512M -S 16M -p archive -E -1 -i -1 -c -1 $tf ||
619 error "extend mirrored file $tf failed"
620 verify_mirror_count $tf 8
621 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
623 # verify component ${ids[0]}
624 verify_comp_attr stripe-size $tf ${ids[0]} 16777216
625 verify_comp_attr_with_default stripe-count $tf ${ids[0]}
626 verify_comp_attr pool $tf ${ids[0]} ssd
627 verify_comp_extent $tf ${ids[0]} 0 33554432
629 # verify component ${ids[1]}
630 verify_comp_attr stripe-size $tf ${ids[1]} 33554432
631 verify_comp_attr_with_default stripe-count $tf ${ids[1]}
632 verify_comp_attr pool $tf ${ids[1]} ssd
633 verify_comp_extent $tf ${ids[1]} 33554432 EOF
635 # verify component ${ids[2]}
636 verify_comp_attr stripe-size $tf ${ids[0]} 16777216
637 verify_comp_attr_with_default stripe-count $tf ${ids[2]}
638 verify_comp_attr pool $tf ${ids[2]} archive
639 verify_comp_extent $tf ${ids[2]} 0 EOF
641 # verify components ${ids[3]} and ${ids[5]}
643 verify_comp_attr_with_default stripe-size $tf ${ids[$i]}
644 verify_comp_attr stripe-count $tf ${ids[$i]} 2
645 verify_comp_attr stripe-index $tf ${ids[$i]} 1
646 verify_comp_extent $tf ${ids[$i]} 0 4194304
649 # verify components ${ids[4]} and ${ids[6]}
651 verify_comp_attr stripe-size $tf ${ids[$i]} 4194304
652 verify_comp_attr stripe-count $tf ${ids[$i]} 2
653 verify_comp_attr pool $tf ${ids[$i]} flash
654 verify_comp_extent $tf ${ids[$i]} 4194304 EOF
657 # verify component ${ids[7]}
658 verify_comp_attr stripe-size $tf ${ids[7]} 4194304
659 verify_comp_attr stripe-count $tf ${ids[7]} $OSTCOUNT
660 verify_comp_attr_with_parent pool $tf ${ids[7]}
661 verify_comp_extent $tf ${ids[7]} 0 EOF
663 # verify components ${ids[8]}, ${ids[10]} and ${ids[12]}
665 verify_comp_attr stripe-size $tf ${ids[$i]} 16777216
666 verify_comp_attr stripe-count $tf ${ids[$i]} $OSTCOUNT
667 verify_comp_attr pool $tf ${ids[$i]} archive
668 verify_comp_extent $tf ${ids[$i]} 0 536870912
671 # verify components ${ids[9]}, ${ids[11]} and ${ids[13]}
673 verify_comp_attr stripe-size $tf ${ids[$i]} 16777216
674 verify_comp_attr stripe-count $tf ${ids[$i]} -1
675 verify_comp_attr pool $tf ${ids[$i]} archive
676 verify_comp_extent $tf ${ids[$i]} 536870912 EOF
679 run_test 0f "lfs mirror extend composite layout mirrors"
684 ! $LFS mirror create --flags prefer $tf ||
685 error "creating $tf w/ --flags but w/o -N option should fail"
687 ! $LFS mirror create -N --flags foo $tf ||
688 error "creating $tf with '--flags foo' should fail"
690 ! $LFS mirror create -N --flags stale $tf ||
691 error "creating $tf with '--flags stale' should fail"
693 ! $LFS mirror create -N --flags prefer,init $tf ||
694 error "creating $tf with '--flags prefer,init' should fail"
696 ! $LFS mirror create -N --flags ^prefer $tf ||
697 error "creating $tf with '--flags ^prefer' should fail"
699 $LFS mirror create -N -E 1M -S 1M -o0 --flags=prefer -E eof -o1 \
700 -N -o1 $tf || error "create mirrored file $tf failed"
702 verify_comp_attr lcme_flags $tf 0x10001 prefer
703 verify_comp_attr lcme_flags $tf 0x10002 prefer
705 # write to the mirrored file and check primary
706 cp /etc/hosts $tf || error "error writing file '$tf'"
708 verify_comp_attr lcme_flags $tf 0x20003 stale
710 # resync file and check prefer flag
711 $LFS mirror resync $tf || error "error resync-ing file '$tf'"
714 $LCTL set_param osc.*.stats=clear
715 cat $tf &> /dev/null || error "error reading file '$tf'"
717 # verify that the data was provided by OST1 where mirror 1 resides
718 local nr_read=$($LCTL get_param -n osc.$FSNAME-OST0000-osc-[-0-9a-f]*.stats |
719 awk '/ost_read/{print $2}')
720 [ -n "$nr_read" ] || error "read was not provided by OST1"
722 run_test 0g "lfs mirror create flags support"
725 [ $MDS1_VERSION -lt $(version_code 2.11.57) ] &&
726 skip "Need MDS version at least 2.11.57"
733 # create parent directory
734 test_mkdir $td || error "mkdir $td failed"
736 $LFS setstripe -N -E 1M -S 1M --flags=prefer -E eof -N2 $td ||
737 error "set default mirrored layout on directory $td failed"
739 # verify flags are inherited from the directory
742 verify_comp_attr lcme_flags $tf 0x10001 prefer
743 verify_comp_attr lcme_flags $tf 0x10002 prefer
745 # set flags to the first component
746 ! $LFS setstripe --comp-set -I 0x10001 --comp-flags=^prefer,foo $tf ||
747 error "setting '^prefer,foo' flags should fail"
749 ! $LFS getstripe --component-flags=prefer,foo $tf ||
750 error "getting component(s) with 'prefer,foo' flags should fail"
752 $LFS setstripe --comp-set -I 0x10001 --comp-flags=^prefer,stale $tf
754 verify_comp_attr lcme_flags $tf 0x10001 stale
755 verify_comp_attr lcme_flags $tf 0x10002 prefer
757 $LFS setstripe --comp-set -I0x10001 --comp-flags=^stale $tf &&
758 error "clearing 'stale' should fail"
760 # write and resync file. It can't resync the file directly because the
761 # file state is still 'ro'
762 cp /etc/hosts $tf || error "error writing file '$tf'"
763 $LFS mirror resync $tf || error "error resync-ing file '$tf'"
765 $LFS setstripe --comp-set -I 0x20003 --comp-flags=prefer $tf ||
766 error "error setting flag prefer"
768 verify_comp_attr lcme_flags $tf 0x20003 prefer
770 $LFS setstripe --comp-set -I 0x20003 --comp-flags=^prefer $tf ||
771 error "error clearing prefer flag from component 0x20003"
773 # MDS disallows setting stale flag on the last non-stale mirror
774 [[ "$MDS1_VERSION" -ge $(version_code 2.12.57) ]] || return 0
776 cp /etc/hosts $tf || error "error writing file '$tf'"
778 verify_comp_attr lcme_flags $tf 0x10002 prefer
779 verify_comp_attr lcme_flags $tf 0x20003 stale
780 verify_comp_attr lcme_flags $tf 0x30004 stale
782 ! $LFS setstripe --comp-set -I 0x10002 --comp-flags=^prefer,stale $tf \
784 error "setting stale flag on component 0x10002 should fail"
786 $LFS mirror resync $tf || error "error resync-ing file '$tf'"
788 $LFS setstripe --comp-set -I 0x10001 --comp-flags=stale $tf ||
789 error "error setting stale flag on component 0x10001"
790 $LFS setstripe --comp-set -I 0x20003 --comp-flags=stale $tf ||
791 error "error setting stale flag on component 0x20003"
793 ! $LFS setstripe --comp-set -I 0x30004 --comp-flags=stale $tf \
795 error "setting stale flag on component 0x30004 should fail"
797 $LFS mirror resync $tf || error "error resync-ing file '$tf'"
799 run_test 0h "set, clear and test flags for FLR files"
802 $LFS mirror create -N2 $DIR/$tfile || error "create $DIR/$tfile failed"
804 cp /etc/hosts $DIR/$tfile || error "write to $DIR/$tfile failed"
805 $LFS mirror resync $DIR/$tfile || error "resync $DIR/$tfile failed"
806 cmp /etc/hosts $DIR/$tfile || error "cmp with /etc/hosts failed"
808 $LFS mirror read -N2 -o $TMP/$tfile $DIR/$tfile || error "read mirror failed"
809 stack_trap "rm -f $TMP/$tfile"
810 cmp $TMP/$tfile $DIR/$tfile || error "cmp with $TMP/$tfile failed"
811 $LFS mirror write -N2 -i /etc/passwd $DIR/$tfile || error "write failed"
812 $LFS setstripe --comp-set -I 65537 --comp-flags=stale $DIR/$tfile ||
813 error "set component 1 stale failed"
814 $LFS mirror resync $DIR/$tfile || error "resync $DIR/$tfile failed"
815 cmp /etc/passwd $DIR/$tfile || error "cmp with /etc/passwd failed"
817 run_test 0j "test lfs mirror read/write commands"
820 [[ $OSTCOUNT -lt 3 ]] && skip "need >= 3 OSTs" && return
823 # default FLR with 2 mirrors
824 $LFS setstripe -N -c1 -i0 -N -c1 -i1 $DIR/$tdir ||
825 error "set default layout failed"
827 # plain file extension under default FLR dir
828 echo $tfile >> $DIR/$tdir/$tfile.append || error "create $tfile.append failed"
829 echo -n " before extend $tfile.append, mirror count = "
830 $LFS getstripe -N $DIR/$tdir/$tfile.append
832 $LFS mirror extend -N -c1 -i2 $DIR/$tdir/$tfile.append ||
833 error "mirror extend failed"
834 echo -n " after extend $tfile.append, mirror count = "
835 $LFS getstripe -N $DIR/$tdir/$tfile.append
837 # normal file extension under default FLR dir
838 touch $DIR/$tdir/$tfile || error "create $tfile failed"
839 echo -n " before extend $tfile, mirror count = "
840 $LFS getstripe -N $DIR/$tdir/$tfile
842 $LFS mirror extend -N -c1 -i2 $DIR/$tdir/$tfile ||
843 error "mirror extend $tfile failed"
844 echo -n " after extend $tfile, mirror count = "
845 $LFS getstripe -N $DIR/$tdir/$tfile
847 run_test 0k "mirroring a file in directory with default FLR layout"
851 local mirror_count=16 # LUSTRE_MIRROR_COUNT_MAX
852 local mirror_create_cmd="$LFS mirror create"
853 local stripes[0]=$OSTCOUNT
855 mirror_create_cmd+=" -N -c ${stripes[0]}"
856 for ((i = 1; i < $mirror_count; i++)); do
857 # add mirrors with different stripes to the file
858 stripes[$i]=$((RANDOM % OSTCOUNT))
859 [ ${stripes[$i]} -eq 0 ] && stripes[$i]=1
861 mirror_create_cmd+=" -N -c ${stripes[$i]}"
864 $mirror_create_cmd $tf || error "create mirrored file $tf failed"
865 verify_mirror_count $tf $mirror_count
867 # can't create mirrors exceeding LUSTRE_MIRROR_COUNT_MAX
868 $LFS mirror extend -N $tf &&
869 error "Creating the $((mirror_count+1))th mirror succeeded"
871 local ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' |
874 # verify the range of components and stripe counts
875 for ((i = 0; i < $mirror_count; i++)); do
876 verify_comp_attr stripe-count $tf ${ids[$i]} ${stripes[$i]}
877 verify_comp_extent $tf ${ids[$i]} 0 EOF
880 run_test 1 "create components with setstripe options"
884 local tf2=$DIR/$tfile-2
886 $LFS setstripe -E 1M -S 1M -E EOF -c 1 $tf
887 $LFS setstripe -E 2M -S 1M -E EOF -c -1 $tf2
889 $LFS mirror extend -N -f $tf2 $tf ||
890 error "merging $tf2 into $tf failed"
892 verify_mirror_count $tf 2
893 [[ ! -e $tf2 ]] || error "$tf2 was not unlinked"
895 run_test 2 "create components from existing files"
898 [[ $MDSCOUNT -lt 2 ]] && skip "need >= 2 MDTs" && return
900 for ((i = 0; i < 2; i++)); do
901 $LFS mkdir -i $i $DIR/$tdir-$i
902 $LFS setstripe -E -1 $DIR/$tdir-$i/$tfile
905 $LFS mirror extend -N -f $DIR/$tdir-1/$tfile \
906 $DIR/$tdir-0/$tfile || error "creating mirrors"
908 # mdt doesn't support to cancel layout lock for remote objects, do
912 # make sure the mirrorted file was created successfully
913 [[ $($LFS getstripe --component-count $DIR/$tdir-0/$tfile) -eq 2 ]] ||
914 { $LFS getstripe $DIR/$tdir-0/$tfile;
915 error "expected 2 components"; }
920 run_test 3 "create components from files located on different MDTs"
923 local tf=$DIR/$tdir/$tfile
926 test_mkdir $DIR/$tdir
928 # set mirror with setstripe options to directory
929 $LFS mirror create -N2 -E 1M -S 1M -E eof $DIR/$tdir ||
930 error "set mirror to directory error"
932 [ x$($LFS getstripe -v $DIR/$tdir | awk '/lcm_flags/{print $2}') = \
933 x"mirrored" ] || error "failed to create mirrored dir"
936 verify_mirror_count $tf 2
938 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
939 verify_comp_extent $tf ${ids[0]} 0 1048576
940 verify_comp_extent $tf ${ids[1]} 1048576 EOF
942 # sub directory should inherit mirror setting from parent
943 test_mkdir $DIR/$tdir/td
944 [ x$($LFS getstripe -v $DIR/$tdir/td | awk '/lcm_flags/{print $2}') = \
945 x"mirrored" ] || error "failed to inherit mirror from parent"
947 # mirror extend won't be applied to directory
948 $LFS mirror extend -N2 $DIR/$tdir &&
949 error "expecting mirror extend failure"
952 run_test 4 "Make sure mirror attributes can be inhertied from directory"
958 $MULTIOP $tf oO_RDWR:O_CREAT:O_LOV_DELAY_CREATE:T12345c ||
959 error "failed to create file with non-empty layout"
960 $CHECKSTAT -t file -s 12345 $tf || error "size error: expecting 12345"
962 $LFS mirror create -N3 $tf || error "failed to attach mirror layout"
963 verify_mirror_count $tf 3
965 $CHECKSTAT -t file -s 12345 $tf ||
966 error "size error after attaching layout "
968 run_test 5 "Make sure init size work for mirrored layout"
971 (( $MDS1_VERSION >= $(version_code 2.12.58) )) ||
972 skip "MDS version older than 2.12.58"
976 $LFS mirror create -N -E 1M -L mdt -E eof -S 1M -N -E eof $tf ||
977 error "failure to create DoM file with mirror"
979 $LFS mirror create -N -E 1M -S 1M -E eof -N -E 1M -L mdt -E eof $tf ||
980 error "failure to create mirrored file with DoM"
982 $LFS setstripe -E 1M -L mdt -E eof -S 1M $tf ||
983 error "failure to create PFL with DoM file"
984 $LFS mirror extend -N2 $tf ||
985 error "failure to extend mirror with DoM"
987 $LFS setstripe -E 1M -L mdt -E eof -S 1M $tf-1 ||
988 error "failure to create PFL with DoM file"
989 $LFS mirror create -N2 -E 1M -S 1M -E eof $tf-2 ||
990 error "failure to create mirrored file"
991 $LFS mirror extend -N -f $tf-1 $tf-2 ||
992 error "failure to extend mirrored file with DoM extent"
994 run_test 6 "DoM and FLR work together"
999 # create DoM with setting stripe_size == component size
1000 $LFS mirror create -N -E1M -S1M -L mdt -Eeof $tf ||
1001 error "failure to create DoM with stripe_size == comp size"
1002 rm -f $tf || error "delete $tf"
1004 # DoM should not inherit previous component stripe_size
1005 $LFS mirror create -N -E4M -S2M -Eeof -N -E1M -L mdt -Eeof $tf ||
1006 error "DoM component shouldn't inherit previous stripe_size"
1008 run_test 7 "Create mirror with DoM component"
1011 local tf=$DIR/$tfile
1012 local tf2=$DIR/$tfile-2
1014 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
1016 stack_trap "rm -f $tf $tf2"
1018 $LFS setstripe -E EOF -o 0 $tf
1019 $LFS setstripe -E EOF -o 1 $tf2
1021 local dd_count=$((RANDOM % 20 + 1))
1022 dd if=/dev/zero of=$tf bs=1M count=$dd_count oflag=sync
1023 dd if=/dev/zero of=$tf2 bs=1M count=1 seek=$((dd_count - 1)) oflag=sync
1025 # for zfs - sync OST dataset so that du below will return
1027 [ "$FSTYPE" = "zfs" ] &&
1028 do_nodes $(comma_list $(osts_nodes)) "$ZPOOL sync"
1030 local blocks=$(du -kc $tf $tf2 | awk '/total/{print $1}')
1033 $LFS mirror extend -N -f $tf2 $tf ||
1034 error "merging $tf2 into $tf failed"
1036 # cancel layout lock
1037 cancel_lru_locks mdc
1039 local new_blocks=$(du -k $tf | awk '{print $1}')
1040 [ $new_blocks -eq $blocks ] ||
1041 error "i_blocks error expected: $blocks, actual: $new_blocks"
1043 run_test 21 "glimpse should report accurate i_blocks"
1045 get_osc_lock_count() {
1052 osc_name=${FSNAME}-OST$(printf "%04x" $((idx-1)))-osc-'[-0-9a-f]*'
1053 count=$($LCTL get_param -n ldlm.namespaces.$osc_name.lock_count)
1054 lock_count=$((lock_count + count))
1060 local tf=$DIR/$tfile
1062 stack_trap "rm -f $tf"
1064 $LFS setstripe -E EOF -o 0 $tf
1065 dd if=/dev/zero of=$tf bs=1M count=$((RANDOM % 20 + 1))
1067 # add component, two mirrors located on the same OST ;-)
1068 $LFS mirror extend -N -o 0 $tf ||
1069 error "extending mirrored file $tf failed"
1071 size_blocks=$(stat --format="%b %s" $tf)
1073 cancel_lru_locks mdc
1074 cancel_lru_locks osc
1076 local new_size_blocks=$(stat --format="%b %s" $tf)
1078 # make sure there is no lock cached
1079 [ $(get_osc_lock_count 1) -eq 0 ] || error "glimpse requests were sent"
1081 [ "$new_size_blocks" = "$size_blocks" ] ||
1082 echo "size expected: $size_blocks, actual: $new_size_blocks"
1086 run_test 22 "no glimpse to OSTs for READ_ONLY files"
1089 local tf=$DIR/$tfile
1091 $LFS mirror create -N -o 0 -N -o 1 $tf ||
1092 error "creating mirrored file $tf failed"
1095 for ((ost_idx = 1; ost_idx <= 2; ost_idx++)); do
1096 cancel_lru_locks osc
1099 local tmpfile=$(mktemp)
1100 stat --format="%b %s" $tf > $tmpfile &
1104 while [ $cnt -le 5 ]; do
1105 kill -0 $pid > /dev/null 2>&1 || break
1109 kill -0 $pid > /dev/null 2>&1 &&
1110 error "stat process stuck due to unavailable OSTs"
1112 # make sure glimpse request has been sent
1113 [ $(get_osc_lock_count 1 2) -ne 0 ] ||
1114 error "OST $ost_idx: no glimpse request was sent"
1119 run_test 31 "make sure glimpse request can be retried"
1122 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
1123 rm -f $DIR/$tfile $DIR/$tfile-2
1125 stack_trap "rm -f $DIR/$tfile"
1127 $LFS setstripe -E EOF -o 0 $DIR/$tfile
1128 dd if=/dev/urandom of=$DIR/$tfile bs=1M count=$((RANDOM % 10 + 2))
1130 local fsize=$(stat -c %s $DIR/$tfile)
1131 [[ $fsize -ne 0 ]] || error "file size is (wrongly) zero"
1133 local cksum=$(md5sum $DIR/$tfile)
1135 # create a new mirror in sync mode
1136 $LFS mirror extend -N -o 1 $DIR/$tfile ||
1137 error "extending mirrored file $DIR/$tfile failed"
1139 # make sure the mirrored file was created successfully
1140 [ $($LFS getstripe -N $DIR/$tfile) -eq 2 ] ||
1141 { $LFS getstripe $DIR/$tfile; error "expected 2 mirrors"; }
1146 # check size is correct, glimpse request should go to the 2nd mirror
1147 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
1148 error "file size error $fsize vs. $(stat -c %s $DIR/$tfile)"
1150 echo "reading file from the 2nd mirror and verify checksum"
1151 [[ "$cksum" == "$(md5sum $DIR/$tfile)" ]] ||
1152 error "checksum error: expected $cksum"
1156 run_test 32 "data should be mirrored to newly created mirror"
1159 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
1161 rm -f $DIR/$tfile $DIR/$tfile-2
1163 # create a file with two mirrors
1164 $LFS setstripe -E EOF -o 0 $DIR/$tfile
1167 while [ $count -lt $max_count ]; do
1168 echo "ost1" >> $DIR/$tfile
1169 count=$((count + 1));
1172 # tmp file that will be used as mirror
1173 $LFS setstripe -E EOF -o 1 $DIR/$tfile-2
1175 while [ $count -lt $max_count ]; do
1176 echo "ost2" >> $DIR/$tfile-2
1177 count=$((count + 1));
1180 # create a mirrored file
1181 $LFS mirror extend -N -f $DIR/$tfile-2 $DIR/$tfile &&
1182 error "merging $DIR/$tfile-2 into $DIR/$tfile" \
1183 "with verification should fail"
1184 $LFS mirror extend --no-verify -N -f $DIR/$tfile-2 $DIR/$tfile ||
1185 error "merging $DIR/$tfile-2 into $DIR/$tfile" \
1186 "without verification failed"
1188 # make sure that $tfile has two mirrors and $tfile-2 does not exist
1189 [ $($LFS getstripe -N $DIR/$tfile) -eq 2 ] ||
1190 { $LFS getstripe $DIR/$tfile; error "expected count 2"; }
1192 [[ ! -e $DIR/$tfile-2 ]] || error "$DIR/$tfile-2 was not unlinked"
1194 # execpted file size
1195 local fsize=$((5 * max_count))
1196 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
1197 error "mirrored file size is not $fsize"
1199 # read file - all OSTs are available
1200 echo "reading file (data can be provided by any ost)... "
1201 local rs=$(cat $DIR/$tfile | head -1)
1202 [[ "$rs" == "ost1" || "$rs" == "ost2" ]] ||
1203 error "file content error: expected: \"ost1\" or \"ost2\""
1205 # read file again with ost1 failed
1209 echo "reading file (data should be provided by ost2)..."
1210 local rs=$(cat $DIR/$tfile | head -1)
1211 [[ "$rs" == "ost2" ]] ||
1212 error "file content error: expected: \"ost2\", actual: \"$rs\""
1217 # read file again with ost2 failed
1221 # check size, glimpse should work
1222 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
1223 error "mirrored file size is not $fsize"
1225 echo "reading file (data should be provided by ost1)..."
1226 local rs=$(cat $DIR/$tfile | head -1)
1227 [[ "$rs" == "ost1" ]] ||
1228 error "file content error: expected: \"ost1\", actual: \"$rs\""
1232 run_test 33a "read can choose available mirror to read"
1235 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
1239 stack_trap "rm -f $DIR/$tfile" EXIT
1241 # create a file with two mirrors on OST0000 and OST0001
1242 $LFS setstripe -N -Eeof -o0 -N -Eeof -o1 $DIR/$tfile
1244 # make sure that $tfile has two mirrors
1245 [ $($LFS getstripe -N $DIR/$tfile) -eq 2 ] ||
1246 { $LFS getstripe $DIR/$tfile; error "expected count 2"; }
1249 dd if=/dev/urandom of=$DIR/$tfile bs=2M count=25 ||
1250 error "write failed for $DIR/$tfile"
1251 $LFS mirror resync $DIR/$tfile || error "resync failed for $DIR/$tfile"
1252 verify_flr_state $DIR/$tfile "ro"
1257 # read file - all OSTs are available
1258 echo "reading file (data can be provided by any ost)... "
1260 time cat $DIR/$tfile > /dev/null || error "read all"
1264 # read file again with ost1 {OST0000} failed
1267 echo "reading file (data should be provided by ost2)..."
1269 time cat $DIR/$tfile > /dev/null || error "read ost2"
1276 # read file again with ost2 {OST0001} failed
1280 echo "reading file (data should be provided by ost1)..."
1282 time cat $DIR/$tfile > /dev/null || error "read ost1"
1289 [ $((r1 * 100)) -gt $((ra * 105)) -a $r1 -gt $((ra + 2)) ] &&
1290 error "read mirror too slow without ost1, from $ra to $r1"
1291 [ $((r2 * 100)) -gt $((ra * 105)) -a $r2 -gt $((ra + 2)) ] &&
1292 error "read mirror too slow without ost2, from $ra to $r2"
1294 wait_osc_import_ready client ost2
1296 run_test 33b "avoid reading from unhealthy mirror"
1299 [[ $OSTCOUNT -lt 3 ]] && skip "need >= 3 OSTs" && return
1303 stack_trap "rm -f $DIR/$tfile" EXIT
1305 # create a file with two mirrors
1306 # mirror1: {OST0000, OST0001}
1307 # mirror2: {OST0001, OST0002}
1308 $LFS setstripe -N -Eeof -c2 -o0,1 -N -Eeof -c2 -o1,2 $DIR/$tfile
1310 # make sure that $tfile has two mirrors
1311 [ $($LFS getstripe -N $DIR/$tfile) -eq 2 ] ||
1312 { $LFS getstripe $DIR/$tfile; error "expected count 2"; }
1315 dd if=/dev/urandom of=$DIR/$tfile bs=2M count=25 ||
1316 error "write failed for $DIR/$tfile"
1317 $LFS mirror resync $DIR/$tfile || error "resync failed for $DIR/$tfile"
1318 verify_flr_state $DIR/$tfile "ro"
1323 # read file - all OSTs are available
1324 echo "reading file (data can be provided by any ost)... "
1325 time cat $DIR/$tfile > /dev/null || error "read all"
1327 # read file again with ost2 (OST0001) failed
1331 echo "reading file (data should be provided by ost1 and ost3)..."
1332 time cat $DIR/$tfile > /dev/null || error "read ost1 & ost3"
1337 wait_osc_import_ready client ost2
1339 run_test 33c "keep reading among unhealthy mirrors"
1342 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
1344 rm -f $DIR/$tfile $DIR/$tfile-2 $DIR/$tfile-ref
1347 $LFS setstripe -o 0 $DIR/$tfile-ref
1348 dd if=/dev/urandom of=$DIR/$tfile-ref bs=1M count=3
1350 # create a file with two mirrors
1351 $LFS setstripe -E -1 -o 0,1 -S 1M $DIR/$tfile
1352 dd if=$DIR/$tfile-ref of=$DIR/$tfile bs=1M
1354 $LFS setstripe -E -1 -o 2,3 -S 1M $DIR/$tfile-2
1355 dd if=$DIR/$tfile-ref of=$DIR/$tfile-2 bs=1M
1357 $CHECKSTAT -t file -s $((3 * 1024 * 1024)) $DIR/$tfile ||
1358 error "mirrored file size is not 3M"
1360 # merge a mirrored file
1361 $LFS mirror extend -N -f $DIR/$tfile-2 $DIR/$tfile ||
1362 error "merging $DIR/$tfile-2 into $DIR/$tfile failed"
1364 cancel_lru_locks osc
1366 # stop two OSTs, so the 2nd stripe of the 1st mirror and
1367 # the 1st stripe of the 2nd mirror will be inaccessible, ...
1370 echo "comparing files ... "
1372 # however, read can still return the correct data. It should return
1373 # the 1st stripe from mirror 1 and 2st stripe from mirror 2.
1374 cmp -n 2097152 <(rwv -f $DIR/$tfile -r -o -n 1 2097152) \
1375 $DIR/$tfile-ref || error "file reading error"
1379 run_test 34a "read mirrored file with multiple stripes"
1382 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
1384 rm -f $DIR/$tfile $DIR/$tfile-2 $DIR/$tfile-ref
1387 $LFS setstripe -o 0 $DIR/$tfile-ref
1388 dd if=/dev/urandom of=$DIR/$tfile-ref bs=1M count=3
1390 $LFS setstripe -E 1M -S 1M -o 0 -E eof -o 1 $DIR/$tfile
1391 dd if=$DIR/$tfile-ref of=$DIR/$tfile bs=1M
1393 $LFS setstripe -E 1M -S 1M -o 2 -E eof -o 3 $DIR/$tfile-2
1394 dd if=$DIR/$tfile-ref of=$DIR/$tfile-2 bs=1M
1396 $CHECKSTAT -t file -s $((3 * 1024 * 1024)) $DIR/$tfile ||
1397 error "mirrored file size is not 3M"
1399 # merge a mirrored file
1400 $LFS mirror extend -N -f $DIR/$tfile-2 $DIR/$tfile ||
1401 error "merging $DIR/$tfile-2 into $DIR/$tfile failed"
1403 cancel_lru_locks osc
1405 # stop two OSTs, so the 2nd component of the 1st mirror and
1406 # the 1st component of the 2nd mirror will be inaccessible, ...
1409 echo "comparing files ... "
1411 # however, read can still return the correct data. It should return
1412 # the 1st stripe from mirror 1 and 2st stripe from mirror 2.
1413 cmp -n 2097152 <(rwv -f $DIR/$tfile -r -o -n 1 2097152) \
1414 $DIR/$tfile-ref || error "file reading error"
1418 run_test 34b "read mirrored file with multiple components"
1421 local tf=$DIR/$tfile
1423 $LFS setstripe -E eof $tf
1425 # add an out-of-sync mirror to the file
1426 $LFS mirror extend -N -c 2 $tf ||
1427 error "extending mirrored file $tf failed"
1429 $MULTIOP $tf oO_WRONLY:c ||
1430 error "write open a mirrored file failed"
1432 # truncate file should return error
1433 $TRUNCATE $tf 100 || error "error truncating a mirrored file"
1435 run_test 35 "allow to write to mirrored files"
1437 get_file_layout_version() {
1438 $LFS getstripe $1 | awk '/lcm_layout_gen/{print $2}'
1441 get_ost_layout_version() {
1442 $MULTIOP $1 oXc | awk '/ostlayoutversion/{print $2}'
1445 verify_ost_layout_version() {
1448 # get file layout version
1449 local flv=$(get_file_layout_version $tf)
1451 # layout version from OST objects
1452 local olv=$(get_ost_layout_version $tf)
1454 (( flv >= olv )) || error "layout version mismatch: $flv vs. $olv"
1461 $LFS setstripe -E 1M -S 1M -E 2M -E 4M -E eof -c -1 $tf
1462 $LFS setstripe -E 3M -S 1M -E 6M -E eof -c -1 $tf-tmp
1464 $LFS mirror extend -N -f $tf-tmp $tf ||
1465 error "merging $tf-tmp into $tf failed"
1470 local tf=$DIR/$tfile
1472 stack_trap "rm -f $tf $tf-2 $tf-3"
1474 create_file_36 $tf $tf-2 $tf-3
1476 [ $($LFS getstripe -N $tf) -gt 1 ] || error "wrong mirror count"
1478 # test case 1 - check file write and verify layout version
1479 $MULTIOP $tf oO_WRONLY:c ||
1480 error "write open a mirrored file failed"
1482 # write open file should not return error
1483 $MULTIOP $tf oO_WRONLY:w1024Yc || error "write mirrored file error"
1485 # instantiate components should work
1486 dd if=/dev/zero of=$tf bs=1M count=12 || error "write file error"
1488 # verify OST layout version
1489 verify_ost_layout_version $tf
1492 local mds_facet=mds$(($($LFS getstripe -m $tf-2) + 1))
1495 do_facet $mds_facet $LCTL set_param fail_val=$delay_sec
1497 #define OBD_FAIL_FLR_LV_DELAY 0x1A01
1498 do_facet $mds_facet $LCTL set_param fail_loc=0x1A01
1500 # write should take at least $fail_loc seconds and succeed
1501 local st=$(date +%s)
1502 $MULTIOP $tf-2 oO_WRONLY:w1024Yc || error "write mirrored file error"
1504 # verify OST layout version
1505 verify_ost_layout_version $tf
1507 do_facet $mds_facet $LCTL set_param fail_loc=0
1509 run_test 36a "write to mirrored files"
1512 local tf=$DIR/$tfile
1514 (( OST1_VERSION >= $(version_code 2.15.50) )) ||
1515 skip "Need OST version at least 2.15.50"
1517 (( OSTCOUNT >= 2 )) || skip "need >= 2 OSTs"
1519 # create 2 mirrors using different OSTs
1520 $LFS setstripe -N -c1 -i0 --flags=prefer -N -c1 -i1 $tf ||
1521 error "create mirrored file"
1523 # write 1M data to one mirror
1524 dd if=/dev/zero of=$tf bs=1M count=1 || error "write file error"
1527 # set prefer mirror to another mirror
1528 $LFS setstripe --comp-set -I0x10001 --comp-flags=^prefer $tf ||
1529 error "clear prefer mirror error"
1530 $LFS setstripe --comp-set -I0x20002 --comp-flags=prefer $tf ||
1531 error "set prefer mirror error"
1533 # the second write should not hung
1534 dd if=/dev/zero of=$tf bs=1M count=1 || error "write file error"
1536 run_test 36b "write should not hung when prefered mirror is stale"
1539 local tf=$DIR/$tfile
1541 (( OST1_VERSION >= $(version_code 2.15.50) )) ||
1542 skip "Need OST version at least 2.15.50"
1544 (( OSTCOUNT >= 2 )) || skip "need >= 2 OSTs"
1546 # create 2 mirrors using different OSTs
1547 $LFS setstripe -N -c1 -i0 --flags=prefer -N -c1 -i1 $tf ||
1548 error "create mirrored file"
1550 # write it in the background
1551 dd if=/dev/zero of=$tf bs=1M count=600 &
1556 $LFS setstripe --comp-set -I0x10001 --comp-flags=^prefer $tf ||
1557 error "clear prefer mirror error"
1558 $LFS setstripe --comp-set -I0x20002 --comp-flags=prefer $tf ||
1559 error "set prefer mirror error"
1563 run_test 36c "change prefer mirror during write shouldn't hung"
1566 local tf=$DIR/$tfile
1568 (( OST1_VERSION >= $(version_code 2.15.50) )) ||
1569 skip "Need OST version at least 2.15.50"
1571 echo " ** create $tf"
1572 $LFS mirror create -N $tf || error "create $tf failed"
1575 echo " ** mirror extend $tf ($i/2)"
1576 $LFS mirror extend -N $tf || error "mirror extend $tf failed"
1577 flv=$(get_file_layout_version $tf)
1578 olv=$(get_ost_layout_version $tf)
1579 echo " flv=$flv olv=$olv"
1583 echo " ** write $tf ($i/2)"
1584 dd if=/dev/zero of=$tf bs=1k count=1 || error "write $tf failed"
1585 flv=$(get_file_layout_version $tf)
1586 olv=$(get_ost_layout_version $tf)
1587 echo " flv=$flv olv=$olv"
1589 error "write update OST layout failed $flv/$olv"
1592 echo " ** resync $tf"
1593 $LFS mirror resync $tf || error "mirror resync $tf failed"
1594 flv=$(get_file_layout_version $tf)
1595 olv=$(get_ost_layout_version $tf)
1596 echo " flv=$flv olv=$olv"
1599 echo " ** truncate $tf ($i/2)"
1600 $TRUNCATE $tf $((1024 * 1024)) || error "truncate $tf fails"
1601 flv=$(get_file_layout_version $tf)
1602 olv=$(get_ost_layout_version $tf)
1603 echo " flv=$flv olv=$olv"
1604 (( flv == olv || flv == olv + 1 )) ||
1605 error "truncate update OST layout failed $flv/$olv"
1608 echo " ** resync $tf"
1609 $LFS mirror resync $tf || error "mirror resync $tf failed"
1610 flv=$(get_file_layout_version $tf)
1611 olv=$(get_ost_layout_version $tf)
1612 echo " flv=$flv olv=$olv"
1615 echo " ** write $tf ($i/2)"
1616 dd if=/dev/zero of=$tf bs=1k count=1 || error "write $tf failed"
1617 flv=$(get_file_layout_version $tf)
1618 olv=$(get_ost_layout_version $tf)
1619 echo " flv=$flv olv=$olv"
1621 error "write update OST layout failed $flv/$olv"
1624 run_test 36d "write/punch FLR file update OST layout version"
1630 echo "create test files with size $fsize .."
1634 $LFS setstripe -E 1M -S 1M -c 1 -E eof -c -1 $tf
1636 dd if=/dev/urandom of=$tf bs=1M count=16 &> /dev/null
1637 $TRUNCATE $tf $fsize
1643 [ $MDS1_VERSION -lt $(version_code 2.11.57) ] &&
1644 skip "Need MDS version at least 2.11.57"
1646 local tf=$DIR/$tfile
1647 local tf2=$DIR/$tfile-2
1648 local tf3=$DIR/$tfile-3
1649 local tf4=$DIR/$tfile-4
1651 stack_trap "rm -f $tf $tf2 $tf3 $tf4"
1653 create_files_37 $((RANDOM + 15 * 1048576)) $tf $tf2 $tf3
1657 # assume the mirror id will be 1, 2, and 3
1658 declare -A checksums
1659 checksums[1]=$(cat $tf | md5sum)
1660 checksums[2]=$(cat $tf2 | md5sum)
1661 checksums[3]=$(cat $tf3 | md5sum)
1663 printf '%s\n' "${checksums[@]}"
1665 # merge these files into a mirrored file
1666 $LFS mirror extend --no-verify -N -f $tf2 $tf ||
1667 error "merging $tf2 into $tf failed"
1668 $LFS mirror extend --no-verify -N -f $tf3 $tf ||
1669 error "merging $tf3 into $tf failed"
1673 # verify mirror read, checksums should equal to the original files'
1674 echo "Verifying mirror read .."
1677 for i in "${mirror_array[@]}"; do
1678 $LCTL set_param ldlm.namespaces.*.lru_size=clear > /dev/null
1679 sum=$($LFS mirror read -N $i $tf | md5sum)
1680 [ "$sum" = "${checksums[$i]}" ] ||
1681 error "$i: mismatch: \'${checksums[$i]}\' vs. \'$sum\'"
1684 # verify mirror write
1685 echo "Verifying mirror write .."
1686 $LFS mirror write -N2 $tf < $tf4
1688 sum=$($LFS mirror read -N2 $tf | md5sum)
1689 [[ "$sum" = "${checksums[1]}" ]] ||
1690 error "2: mismatch \'${checksums[1]}\' vs. \'$sum\'"
1692 # verify mirror copy, write to this mirrored file will invalidate
1693 # the other two mirrors
1694 echo "Verifying mirror copy .."
1696 local osts=$(comma_list $(osts_nodes))
1698 $LFS mirror copy -i ${mirror_array[0]} -o-1 $tf ||
1699 error "mirror copy error"
1701 # verify copying is successful by checking checksums
1702 remount_client $MOUNT
1703 for i in "${mirror_array[@]}"; do
1704 sum=$($LFS mirror read -N $i $tf | md5sum)
1705 [ "$sum" = "${checksums[1]}" ] ||
1706 error "$i: mismatch checksum after copy \'$sum\'"
1711 run_test 37 "mirror I/O API verification"
1714 local tf=$DIR/$tfile
1715 local ref=$DIR/${tfile}-ref
1717 stack_trap "rm -f $tf $ref"
1719 $LFS setstripe -E 1M -S 1M -c 1 -E 4M -c 2 -E eof -c -1 $tf ||
1720 error "creating $tf failed"
1721 $LFS setstripe -E 2M -S 1M -c 1 -E 6M -c 2 -E 8M -c -1 -E eof -c -1 \
1722 $tf-2 || error "creating $tf-2 failed"
1723 $LFS setstripe -E 4M -c 1 -E 8M -c 2 -E eof -c -1 $tf-3 ||
1724 error "creating $tf-3 failed"
1726 # instantiate all components
1727 $LFS mirror extend -N -f $tf-2 $tf ||
1728 error "merging $tf-2 into $tf failed"
1729 $LFS mirror extend -N -f $tf-3 $tf ||
1730 error "merging $tf-3 into $tf failed"
1731 $LFS mirror extend -N -c 1 $tf ||
1732 error "extending mirrored file $tf failed"
1734 verify_flr_state $tf "ro"
1736 dd if=/dev/urandom of=$ref bs=1M count=16 &> /dev/null
1738 local fsize=$((RANDOM << 8 + 1048576))
1739 $TRUNCATE $ref $fsize
1741 local ref_cksum=$(cat $ref | md5sum)
1743 # case 1: verify write to mirrored file & resync work
1744 cp $ref $tf || error "copy from $ref to $f error"
1745 verify_flr_state $tf "wp"
1747 local file_cksum=$(cat $tf | md5sum)
1748 [ "$file_cksum" = "$ref_cksum" ] || error "write failed, cksum mismatch"
1751 echo "mirror IDs: ${mirror_array[*]}"
1753 local valid_mirror stale_mirror id mirror_cksum
1754 for id in "${mirror_array[@]}"; do
1755 mirror_cksum=$($LFS mirror read -N $id $tf | md5sum)
1756 [ "$ref_cksum" == "$mirror_cksum" ] &&
1757 { valid_mirror=$id; continue; }
1762 [ -z "$stale_mirror" ] && error "stale mirror doesn't exist"
1763 [ -z "$valid_mirror" ] && error "valid mirror doesn't exist"
1765 $LFS mirror resync $tf || error "resync failed"
1766 verify_flr_state $tf "ro"
1768 mirror_cksum=$($LFS mirror read -N $stale_mirror $tf | md5sum)
1769 [ "$file_cksum" = "$ref_cksum" ] || error "resync failed"
1771 # case 2: inject an error to make mirror_io exit after changing
1772 # the file state to sync_pending so that we can start a concurrent
1774 $MULTIOP $tf oO_WRONLY:w$((RANDOM % 1048576 + 1024))c
1775 verify_flr_state $tf "wp"
1777 mirror_io resync -e resync_start $tf && error "resync succeeded"
1778 verify_flr_state $tf "sp"
1780 # from sync_pending to write_pending
1781 $MULTIOP $tf oO_WRONLY:w$((RANDOM % 1048576 + 1024))c
1782 verify_flr_state $tf "wp"
1784 mirror_io resync -e resync_start $tf && error "resync succeeded"
1785 verify_flr_state $tf "sp"
1787 # from sync_pending to read_only
1788 $LFS mirror resync $tf || error "resync failed"
1789 verify_flr_state $tf "ro"
1791 run_test 38 "resync"
1794 local tf=$DIR/$tfile
1797 $LFS mirror create -N2 -E1m -c1 -S1M -E-1 $tf ||
1798 error "create PFL file $tf failed"
1800 verify_mirror_count $tf 2
1801 verify_comp_count $tf 4
1803 rm -f $tf || error "delete $tf failed"
1805 run_test 39 "check FLR+PFL (a.k.a. PFLR) creation"
1808 local tf=$DIR/$tfile
1811 for ops in "conv=notrunc" ""; do
1814 $LFS mirror create -N -E 2M -S 1M -E 4M -E -1 --flags=prefer \
1815 -N -E 1M -E 2M -E 4M -E -1 $tf ||
1816 error "create PFLR file $tf failed"
1817 dd if=/dev/zero of=$tf $ops bs=1M seek=2 count=1 ||
1818 error "write PFLR file $tf failed"
1820 lfs getstripe -vy $tf
1824 # file mirror state should be write_pending
1825 flags=$($LFS getstripe -v $tf | awk '/lcm_flags:/ { print $2 }')
1827 error "file mirror state $flags"
1828 # the 1st component (in mirror 1) should be inited
1829 verify_comp_attr lcme_flags $tf 0x10001 init
1830 # the 2nd component (in mirror 1) should be inited
1831 verify_comp_attr lcme_flags $tf 0x10002 init
1832 # the 3rd component (in mirror 1) should be uninited
1833 verify_comp_attr lcme_flags $tf 0x10003 prefer
1834 # the 4th component (in mirror 2) should be inited
1835 verify_comp_attr lcme_flags $tf 0x20004 init
1836 # the 5th component (in mirror 2) should be uninited
1837 verify_comp_attr lcme_flags $tf 0x20005 0
1838 # the 6th component (in mirror 2) should be stale
1839 verify_comp_attr lcme_flags $tf 0x20006 stale
1840 # the 7th component (in mirror 2) should be uninited
1841 if [[ x$ops = "xconv=notrunc" ]]; then
1842 verify_comp_attr lcme_flags $tf 0x20007 0
1843 elif [[ x$ops = "x" ]]; then
1844 verify_comp_attr lcme_flags $tf 0x20007 stale
1848 rm -f $tf || error "delete $tf failed"
1850 run_test 40 "PFLR rdonly state instantiation check"
1853 local tf=$DIR/$tfile
1855 stack_trap "rm -f $tf $tf-1"
1858 echo " **create two FLR files $tf $tf-1"
1859 $LFS mirror create -N -E 2M -S 1M -E 4M -E -1 \
1860 -N -E 1M -E 2M -E 3M -E -1 $tf ||
1861 error "create PFLR file $tf failed"
1862 $LFS mirror create -N -E 2M -S 1M -E eof \
1863 -N -E 1M -E eof --flags prefer \
1864 -N -E 4m -E eof $tf-1 ||
1865 error "create PFLR file $tf-1 failed"
1867 # file should be in ro status
1868 echo " **verify files be RDONLY"
1869 verify_flr_state $tf "ro"
1870 verify_flr_state $tf-1 "ro"
1872 # write data in [0, 2M)
1873 dd if=/dev/zero of=$tf bs=1M count=2 conv=notrunc ||
1874 error "writing $tf failed"
1875 dd if=/dev/urandom of=$tf-1 bs=1M count=4 conv=notrunc ||
1876 error "writing $tf-1 failed"
1878 local sum0=$(cat $tf-1 | md5sum)
1880 echo " **verify files be WRITE_PENDING"
1881 verify_flr_state $tf "wp"
1882 verify_flr_state $tf-1 "wp"
1884 # file should have stale component
1885 echo " **verify files have stale component"
1886 $LFS getstripe $tf | grep lcme_flags | grep stale > /dev/null ||
1887 error "after writing $tf, it does not contain stale component"
1888 $LFS getstripe $tf-1 | grep lcme_flags | grep stale > /dev/null ||
1889 error "after writing $tf-1, it does not contain stale component"
1891 echo " **full resync"
1892 $LFS mirror resync $tf $tf-1 || error "mirror resync $tf $tf-1 failed"
1894 echo " **verify $tf-1 data consistency in all mirrors"
1896 local sum=$($LFS mirror read -N$i $tf-1 | md5sum)
1897 [[ "$sum" = "$sum0" ]] ||
1898 error "$tf-1.$i: checksum mismatch: $sum != $sum0"
1901 echo " **verify files be RDONLY"
1902 verify_flr_state $tf "ro"
1903 verify_flr_state $tf-1 "ro"
1905 # file should not have stale component
1906 echo " **verify files do not contain stale component"
1907 $LFS getstripe $tf | grep lcme_flags | grep stale &&
1908 error "after resyncing $tf, it contains stale component"
1909 $LFS getstripe $tf-1 | grep lcme_flags | grep stale &&
1910 error "after resyncing $tf, it contains stale component"
1912 # verify partial resync
1913 echo " **write $tf-1 for partial resync test"
1914 dd if=/dev/zero of=$tf-1 bs=1M count=2 conv=notrunc ||
1915 error "writing $tf-1 failed"
1917 echo " **only resync mirror 2"
1918 verify_flr_state $tf-1 "wp"
1919 $LFS mirror resync --only 2 $tf-1 ||
1920 error "resync mirror 2 of $tf-1 failed"
1921 verify_flr_state $tf "ro"
1923 # resync synced mirror
1924 echo " **resync mirror 2 again"
1925 $LFS mirror resync --only 2 $tf-1 ||
1926 error "resync mirror 2 of $tf-1 failed"
1927 verify_flr_state $tf "ro"
1928 echo " **verify $tf-1 contains stale component"
1929 $LFS getstripe $tf-1 | grep lcme_flags | grep stale > /dev/null ||
1930 error "after writing $tf-1, it does not contain stale component"
1932 echo " **full resync $tf-1"
1933 $LFS mirror resync $tf-1 || error "resync of $tf-1 failed"
1934 verify_flr_state $tf "ro"
1935 echo " **full resync $tf-1 again"
1936 $LFS mirror resync $tf-1 || error "resync of $tf-1 failed"
1937 echo " **verify $tf-1 does not contain stale component"
1938 $LFS getstripe $tf | grep lcme_flags | grep stale &&
1939 error "after resyncing $tf, it contains stale component"
1943 run_test 41 "lfs mirror resync check"
1946 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
1950 local mirror_cmd="$LFS mirror verify"
1953 stack_trap "rm -rf $td"
1955 # create parent directory
1956 mkdir $td || error "mkdir $td failed"
1958 $mirror_cmd &> /dev/null && error "no file name given"
1959 $mirror_cmd $tf &> /dev/null && error "cannot stat file $tf"
1960 $mirror_cmd $td &> /dev/null && error "$td is not a regular file"
1962 # create mirrored files
1963 $LFS mirror create -N -E 4M -S 1M -E 10M -E EOF $tf ||
1964 error "create mirrored file $tf failed"
1965 $LFS mirror create -N -E 2M -S 1M -E EOF \
1966 -N -E 6M -E 8M -E EOF \
1967 -N -E 16M -E EOF $tf-1 ||
1968 error "create mirrored file $tf-1 failed"
1969 $LFS mirror create -N -c 2 -o 1,3 -N -S 2M -c -1 $tf-2 ||
1970 error "create mirrored file $tf-2 failed"
1972 # write data in [0, 10M)
1973 for i in $tf $tf-1 $tf-2; do
1974 yes | dd of=$i bs=1M count=10 iflag=fullblock conv=notrunc ||
1975 error "write $i failed"
1978 # resync the mirrored files
1979 $LFS mirror resync $tf-1 $tf-2 ||
1980 error "resync $tf-1 $tf-2 failed"
1982 # verify the mirrored files
1983 $mirror_cmd $tf-1 $tf-2 ||
1984 error "verify $tf-1 $tf-2 failed"
1986 get_mirror_ids $tf-1
1987 $mirror_cmd --only ${mirror_array[0]} $tf-1 &> /dev/null &&
1988 error "at least 2 mirror ids needed with '--only' option"
1989 $mirror_cmd --only ${mirror_array[0]},${mirror_array[1]} $tf-1 $tf-2 \
1991 error "'--only' option cannot be used upon multiple files"
1992 $mirror_cmd --only 65534,${mirror_array[0]},65535 $tf-1 &&
1993 error "invalid specified mirror ids"
1995 # change the content of $tf and merge it into $tf-1
1997 echo a | dd of=$tf bs=1M seek=$i conv=notrunc ||
1998 error "change $tf with seek=$i failed"
1999 echo b | dd of=$tf-1 bs=1M seek=$i conv=notrunc ||
2000 error "change $tf-1 with seek=$i failed"
2003 $LFS mirror resync $tf-1 || error "resync $tf-1 failed"
2004 $LFS mirror extend --no-verify -N -f $tf $tf-1 ||
2005 error "merge $tf into $tf-1 failed"
2007 # verify the mirrored files
2008 echo "Verify $tf-1 without -v option:"
2009 $mirror_cmd $tf-1 &&
2010 error "verify $tf-1 should fail" || echo "PASS"
2012 echo "Verify $tf-1 with -v option:"
2013 $mirror_cmd -v $tf-1 &&
2014 error "verify $tf-1 should fail"
2016 get_mirror_ids $tf-1
2017 echo "Verify $tf-1 with --only option:"
2018 $mirror_cmd -v --only ${mirror_array[1]},${mirror_array[-1]} $tf-1 &&
2019 error "verify $tf-1 with mirror ${mirror_array[1]} and" \
2020 "${mirror_array[-1]} should fail"
2022 $mirror_cmd --only ${mirror_array[0]},${mirror_array[1]} $tf-1 ||
2023 error "verify $tf-1 with mirror ${mirror_array[0]} and" \
2024 "${mirror_array[1]} should succeed"
2026 # set stale components in $tf-1
2027 for i in 0x40002 0x40003; do
2028 $LFS setstripe --comp-set -I$i --comp-flags=stale $tf-1 ||
2029 error "set stale flag on component $i failed"
2032 # verify the mirrored file
2033 echo "Verify $tf-1 with stale components:"
2034 $mirror_cmd -vvv $tf-1 ||
2035 error "verify $tf-1 with stale components should succeed"
2037 echo "Verify $tf-1 with stale components and --only option:"
2038 $mirror_cmd -vvv --only ${mirror_array[1]},${mirror_array[-1]} $tf-1 ||
2039 error "verify $tf-1 with mirror ${mirror_array[1]} and" \
2040 "${mirror_array[-1]} should succeed"
2042 run_test 42 "lfs mirror verify"
2044 # inactivate one OST && write && restore the OST
2048 local PARAM="osp.${FSNAME}-OST000${ost}-osc-M*.active"
2051 wait=$(do_facet $SINGLEMDS \
2052 "$LCTL get_param -n lod.*MDT0000-*.qos_maxage")
2053 wait=${wait%%[^0-9]*}
2055 echo " **deactivate OST$ost, waiting for $((wait*2+2)) seconds"
2056 $(do_facet $SINGLEMDS "$LCTL set_param -n $PARAM 0")
2057 # lod_qos_statfs_update needs 2*$wait seconds to refresh targets statfs
2058 sleep $(($wait * 2 + 2))
2059 echo " **write $file"
2060 dd if=/dev/zero of=$file bs=1M count=1 || error "write $file failed"
2061 echo " **restore activating OST$ost, waiting for $((wait*2+2)) seconds"
2062 $(do_facet $SINGLEMDS "$LCTL set_param -n $PARAM 1")
2063 sleep $((wait * 2 + 2))
2065 local flags=$($LFS getstripe -v $file | awk '/lcm_flags:/ { print $2 }')
2066 [ $flags = wp ] || error "file mirror state $flags != wp"
2070 [ $OSTCOUNT -lt 3 ] && skip "needs >= 3 OSTs" && return
2072 local tf=$DIR/$tfile
2076 ## mirror 0 ost (0, 1)
2077 ## mirror 1 ost (1, 2)
2078 ## mirror 2 ost (2, 0)
2079 $LFS mirror create -N -Eeof -c2 -o0,1 -N -Eeof -c2 -o1,2 \
2080 -N -Eeof -c2 -o2,0 $tf ||
2081 error "create 3 mirrors file $tf failed"
2083 ################## OST0 ###########################################
2085 echo " **verify components"
2086 verify_comp_attr lcme_flags $tf 0x10001 init,stale
2087 verify_comp_attr lcme_flags $tf 0x20002 init
2088 verify_comp_attr lcme_flags $tf 0x30003 init,stale
2091 echo " **resync $tf"
2092 $LFS mirror resync $tf
2093 flags=$($LFS getstripe -v $tf | awk '/lcm_flags:/ { print $2 }')
2094 [ $flags = ro ] || error "file mirror state $flags != ro"
2096 ################## OST1 ###########################################
2098 echo " **verify components"
2099 verify_comp_attr lcme_flags $tf 0x10001 init,stale
2100 verify_comp_attr lcme_flags $tf 0x20002 init,stale
2101 verify_comp_attr lcme_flags $tf 0x30003 init
2104 echo " **resync $tf"
2105 $LFS mirror resync $tf
2106 flags=$($LFS getstripe -v $tf | awk '/lcm_flags:/ { print $2 }')
2107 [ $flags = ro ] || error "file mirror state $flags != ro"
2109 ################## OST2 ###########################################
2111 echo " **verify components"
2112 verify_comp_attr lcme_flags $tf 0x10001 init
2113 verify_comp_attr lcme_flags $tf 0x20002 init,stale
2114 verify_comp_attr lcme_flags $tf 0x30003 init,stale
2116 run_test 43a "mirror pick on write"
2119 local tf=$DIR/$tdir/$tfile
2121 test_mkdir $DIR/$tdir
2123 stack_trap "rm -rf $tf"
2125 # create 3 mirrors FLR file, the first 2 mirrors are preferred
2126 $LFS setstripe -N -Eeof --flags=prefer -N -Eeof --flags=prefer \
2127 -N -Eeof $tf || error "create 3 mirrors file $tf failed"
2128 verify_flr_state $tf "ro"
2130 echo " ** write to $tf"
2131 dd if=/dev/zero of=$tf bs=1M count=1 || error "write $tf failed"
2132 verify_flr_state $tf "wp"
2134 echo " ** resync $tf"
2135 $LFS mirror resync $tf || error "resync $tf failed"
2136 verify_flr_state $tf "ro"
2138 run_test 43b "allow writing to multiple preferred mirror file"
2141 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
2144 local tf=$DIR/$tdir/$tfile
2145 local tf1=$DIR/$tdir-1/$tfile-1
2147 stack_trap "rm -rf $tf $tf1"
2149 $LFS setdirstripe -i 0 -c 1 $DIR/$tdir ||
2150 error "create directory failed"
2151 $LFS setdirstripe -i 1 -c 1 $DIR/$tdir-1 ||
2152 error "create remote directory failed"
2153 rm -f $tf $tf1 $tf.mirror~2
2154 # create file with 4 mirrors
2155 $LFS mirror create -N -E 2M -S 1M -E 4M -E -1 \
2156 -N -E 1M -E 2M -E 3M -E -1 -N2 $tf ||
2157 error "create PFLR file $tf failed"
2159 # file should be in ro status
2160 verify_flr_state $tf "ro"
2162 # write data in [0, 3M)
2163 dd if=/dev/urandom of=$tf bs=1M count=3 conv=notrunc ||
2164 error "writing $tf failed"
2166 verify_flr_state $tf "wp"
2168 # disallow destroying the last non-stale mirror
2169 ! $LFS mirror delete --mirror-id 1 $tf > /dev/null 2>&1 ||
2170 error "destroying mirror 1 should fail"
2172 # synchronize all mirrors of the file
2173 $LFS mirror resync $tf || error "mirror resync $tf failed"
2175 verify_flr_state $tf "ro"
2178 $LFS mirror split --mirror-id 1 -f $tf1 $tf ||
2179 error "split to $tf1 failed"
2181 local idx0=$($LFS getstripe -m $tf)
2182 local idx1=$($LFS getstripe -m $tf1)
2184 [[ x$idx0 == x0 ]] || error "$tf is not on MDT0"
2185 [[ x$idx1 == x1 ]] || error "$tf1 is not on MDT1"
2187 # verify mirror count
2188 verify_mirror_count $tf 3
2189 verify_mirror_count $tf1 1
2191 $LFS mirror split --mirror-id 2 $tf ||
2192 error "split mirror 2 failed"
2194 verify_mirror_count $tf 2
2195 verify_mirror_count $tf.mirror~2 1
2197 $LFS setstripe --comp-set -I 0x30008 --comp-flags=stale $tf ||
2198 error "setting stale flag on component 0x30008 failed"
2200 # disallow destroying the last non-stale mirror
2201 ! $LFS mirror split --mirror-id 4 -d $tf > /dev/null 2>&1 ||
2202 error "destroying mirror 4 should fail"
2204 $LFS mirror resync $tf || error "resynchronizing $tf failed"
2206 $LFS mirror split --mirror-id 3 -d $tf ||
2207 error "destroying mirror 3 failed"
2208 verify_mirror_count $tf 1
2210 # verify splitted file contains the same content as the orig file does
2211 diff $tf $tf1 || error "splited file $tf1 diffs from $tf"
2212 diff $tf $tf.mirror~2 ||
2213 error "splited file $tf.mirror~2 diffs from $tf"
2215 run_test 44a "lfs mirror split check"
2218 (( $MDS1_VERSION >= $(version_code 2.14.56) )) ||
2219 skip "Need MDS version at least 2.14.56"
2222 local tf=$DIR/$tdir/$tfile
2224 mkdir -p $DIR/$tdir || error "create directory failed"
2228 # create 2 mirrors file
2229 $LFS mirror extend -N -c1 $tf
2233 verify_flr_state $tf "wp"
2235 local str=$(cat $tf)
2237 [[ $str == "YYY" ]] || error "$tf content is not YYY"
2239 # get the non-stale mirror id
2240 local ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' |
2242 local mirror_ids=($($LFS getstripe $tf |
2243 awk '/lcme_mirror_id/{print $2}' | tr '\n' ' '))
2244 for ((i = 0; i < 2; i++)); do
2245 $LFS getstripe -I${ids[$i]} --component-flags $tf |
2246 grep stale > /dev/null || break
2249 [[ $i -ge 2 ]] && ( $LFS getstripe $tf; error "no stale mirror" )
2253 # split the updated mirror, should fail
2254 echo "split mirror_id ${mirror_ids[$i]} id ${ids[$i]}, should fail"
2255 $LFS mirror split --mirror-id=${mirror_ids[$i]} $tf &> /dev/null &&
2256 error "split --mirror-id=${mirror_ids[$i]} $tf should fail"
2259 # split the stale mirror
2260 echo "split mirror_id ${mirror_ids[$i]} id ${ids[$i]}"
2261 $LFS mirror split --mirror-id=${mirror_ids[$i]} -d $tf ||
2262 error "mirror split --mirror-id=${mirror_ids[$i]} $tf failed"
2264 echo "make sure there's no stale comp in the file"
2265 # make sure there's no stale comp in the file
2266 $LFS getstripe $tf | awk '/lcme_flags/{print $2}' | grep stale &&
2267 ( $LFS getstripe $tf; error "stale mirror file" )
2270 [[ $str == "YYY" ]] ||
2271 ( cat $tf; error "$tf content is not YYY after split" )
2273 run_test 44b "mirror split does not create stale file"
2276 local tf=$DIR/$tdir/$tfile
2278 stack_trap "rm -f $tf"
2280 [ $MDS1_VERSION -ge $(version_code 2.14.52) ] ||
2281 skip "Need MDS version at least 2.14.52"
2283 [ "$FSTYPE" != "zfs" ] || skip "ZFS file's block number is not accurate"
2285 mkdir -p $DIR/$tdir || error "create directroy failed"
2287 dd if=/dev/zero of=$tf bs=1M count=10 || error "dd write $tfile failed"
2289 block1=$(( $(stat -c "%b*%B" $tf) ))
2290 echo " ** before mirror ops, file blocks=$((block1/1024)) KiB"
2292 $LFS mirror extend -N2 -c1 $tf || error "mirror extend $tfile failed"
2294 block2=$(( $(stat -c "%b*%B" $tf) ))
2295 echo " ** after mirror extend, file blocks=$((block2/1024)) KiB"
2297 $LFS mirror split -d --mirror-id=2 $tf ||
2298 error "mirror split $tfile failed"
2299 $LFS mirror split -d --mirror-id=3 $tf ||
2300 error "mirror split $tfile failed"
2303 block3=$(( $(stat -c "%b*%B" $tf) ))
2304 echo " ** after mirror split, file blocks=$((block3/1024)) KiB"
2306 [[ $block1 -eq $block3 ]] ||
2307 error "mirror split does not reduce block# $block3 != $block1"
2309 run_test 44c "lfs mirror split reduces block size of a file"
2312 local tf=$DIR/$tdir/$tfile
2318 stack_trap "rm -f $tf"
2320 mkdir -p $DIR/$tdir || error "create directroy failed"
2322 dd if=/dev/zero of=$tf bs=1M count=10 || error "dd write $tfile failed"
2324 size1=$(stat -c "%s" $tf)
2325 echo " ** before mirror ops, file size=$size1"
2327 $LFS mirror extend -N2 -c1 $tf || error "mirror extend $tfile failed"
2329 size2=$(stat -c "%s" $tf)
2330 echo " ** after mirror extend, file size=$size2"
2332 (($size1 == $size2)) ||
2333 error "mirror extend should not change size, before: $size1, after $size2"
2335 $LFS mirror split -d --mirror-id=2 $tf ||
2336 error "mirror split $tfile failed"
2338 size2=$(stat -c "%s" $tf)
2339 echo " ** after mirror split, file size=$size2"
2340 (($size1 == $size2)) ||
2341 error "mirror split should not change size, before: $size1, after $size2"
2343 # Remount client to clear cached size information
2344 remount_client $MOUNT
2345 size2=$(stat -c "%s" $tf)
2346 echo " ** after mirror split & remount, file size=$size2"
2347 (($size1 == $size2)) ||
2348 error "mirror extend should not change size, before: $size1, after $size2"
2351 run_test 44d "lfs mirror split does not break size"
2354 local tf=$DIR/$tdir/$tfile
2355 local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
2359 test_mkdir $DIR/$tdir
2360 [ $MDS1_VERSION -ge $(version_code 2.14.52) ] ||
2361 skip "Need MDS version at least 2.14.52"
2363 $LFS mirror create -N2 $tf || error "create mirrored file $tf failed"
2365 # Disable xattr caching so we can repeatedly check SOM with lfs getsom
2366 $LCTL set_param llite.*.xattr_cache=0
2367 stack_trap "$LCTL set_param llite.*.xattr_cache=1"
2369 dd if=/dev/zero of=$tf bs=1M count=10 || error "dd write $tfile failed"
2371 size1=$(stat -c "%s" $tf)
2372 echo " ** before mirror resync, file size=$size1"
2374 $LFS mirror resync $tf || error "mirror resync file $tf failed"
2375 size1=$(stat -c "%s" $tf)
2376 size2=$($LFS getsom -s $tf)
2380 ((size1 == size2)) ||
2381 error "mirrored file with strict SOM $size1 != disabled SOM $size2"
2383 # Remount client to clear cached size information
2384 remount_client $MOUNT
2386 save_lustre_params $(get_facets MDS) mdt.*MDT*.enable_strict_som > $p
2387 stack_trap "restore_lustre_params < $p; rm -f $p"
2388 local mds_facet=mds$(($($LFS getstripe -m $tf) + 1))
2390 do_facet $mds_facet $LCTL set_param mdt.*MDT*.enable_strict_som=0
2392 size2=$(stat -c "%s" $tf)
2393 # 'getsom' here is just for debugging
2396 ((size2 == size1)) ||
2397 error "mirrored file in sync with som disabled, size with som disabled ($size2) and without som disabled ($size1) should agree"
2399 run_test 44e "basic FLR SOM tests + disable SOM"
2402 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
2404 local file=$DIR/$tdir/$tfile
2405 local dir=$DIR/$tdir/$dir
2406 local temp=$DIR/$tdir/template
2408 test_mkdir $DIR/$tdir
2410 $LFS setstripe -N -E1m -S1m -c2 -o0,1 -E2m -Eeof -N -E4m -Eeof \
2411 -N -E3m -S1m -Eeof -N -E8m -Eeof $file ||
2412 error "Create $file failed"
2414 verify_yaml_layout $file $file.copy $temp "1. FLR file"
2415 rm -f $file $file.copy
2417 $LFS setstripe -N -E1m -S1m -c2 -o0,1 -E2m -Eeof -N -E4m -Eeof \
2418 -N -E3m -S1m -Eeof -N -E8m --flags=prefer -Eeof $file ||
2419 error "Create $file failed"
2421 verify_yaml_layout $file $file.copy $temp "2. FLR file with flags"
2423 run_test 45 "Verify setstripe/getstripe with YAML with FLR file"
2430 $LFS setstripe --copy=$src $dst || error "setstripe $dst failed"
2432 local layout1=$(get_layout_param $src)
2433 local layout2=$(get_layout_param $dst)
2434 # compare their layout info
2435 [ "$layout1" == "$layout2" ] ||
2436 error "$msg_prefix $src <=> $dst layouts are not equal"
2440 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" && return
2442 local file=$DIR/$tdir/$tfile
2443 test_mkdir $DIR/$tdir
2445 ########################### 1. PFL file #############################
2446 echo " ** 1. PFL file"
2448 $LFS setstripe -E1m -S 1M -c2 -o0,1 -E2m -c2 -E3m -o1,0 -E4m -c1 -E-1 \
2449 $file || error "1. Create PFL $file failed"
2452 verify_46 $file $file.copy "1. PFL file"
2454 ########################### 2. plain file ###########################
2455 echo " ** 2. plain file"
2457 $LFS setstripe -c2 -o0,1 -i1 $file ||
2458 error "2. Create plain $file failed"
2461 verify_46 $file $file.copy "2. plain file"
2463 ########################### 3. FLR file #############################
2464 echo " ** 3. FLR file"
2466 $LFS setstripe -N -E1m -S 1M -c2 -o0,1 -E4m -c1 -Eeof -N -E16m -Eeof \
2467 $file || error "3. Create FLR $file failed"
2470 verify_46 $file $file.copy "3. FLR file"
2472 local dir=$DIR/$tdir/dir
2473 ########################### 4. PFL dir ##############################
2474 echo " ** 4. PFL dir"
2476 $LFS setstripe -E1m -S 1M -c2 -E2m -c1 -E-1 $dir ||
2477 error "4. setstripe PFL $dir failed"
2479 test_mkdir $dir.copy
2480 verify_46 $dir $dir.copy "4. PFL dir"
2482 ########################### 5. plain dir ############################
2483 echo " ** 5. plain dir"
2484 $LFS setstripe -c2 -i-1 $dir || error "5. setstripe plain $dir failed"
2486 verify_46 $dir $dir.copy "5. plain dir"
2488 ########################### 6. FLR dir ##############################
2489 echo " ** 6. FLR dir"
2490 $LFS setstripe -N -E1m -S 1M -c2 -E2m -c1 -Eeof -N -E4m -Eeof $dir ||
2491 error "6. setstripe FLR $dir failed"
2493 verify_46 $dir $dir.copy "6. FLR dir"
2495 (( $MDS1_VERSION >= $(version_code 2.13.53.205) )) ||
2496 echo "server version $MDS1_VERSION does not support SEL" &&
2499 ########################### 7. SEL file ##############################
2500 echo " ** 7. SEL file"
2502 $LFS setstripe -E256M -S 1M -c2 -o0,1 -z 64M -E-1 -o1,0 -z 128M \
2503 $file || error "Create $file failed"
2506 verify_46 $file $file.copy "7. SEL file"
2508 ########################### 8. SEL dir ##############################
2509 echo " ** 8. SEL dir"
2510 $LFS setstripe -E256M -S 1M -c2 -z 64M -E-1 -z 128M \
2511 $dir || error "setstripe $dir failed"
2513 verify_46 $dir $dir.copy "8. SEL dir"
2515 ########################### 9. FLR SEL file ##########################
2516 echo " ** 9. FLR SEL file"
2518 $LFS setstripe -N -E256M -c2 -z 64M -E-1 -z 128M \
2519 -N -E1G -c4 -z128M -E-1 -z256M $file || error "Create $file failed"
2522 verify_46 $file $file.copy "9. SEL file"
2524 ########################### 10. FLR SEL dir #########################
2525 echo " ** 10. FLR SEL dir"
2526 $LFS setstripe -N -E256M -c2 -z 64M -E-1 -z 128M \
2527 -N -E1G -c4 -z128M -E-1 -z256M $dir || error "Create $file failed"
2529 verify_46 $dir $dir.copy "10. SEL dir"
2531 run_test 46 "Verify setstripe --copy option"
2534 [ $OSTCOUNT -lt 3 ] && skip "needs >= 3 OSTs" && return
2536 local file=$DIR/$tdir/$tfile
2541 test_mkdir $DIR/$tdir
2545 # mirror1: [comp0]ost0, [comp1]ost1 and ost2
2546 # mirror2: [comp2] , [comp3] should not use ost1 or ost2
2547 $LFS mirror create -N -E2m -c1 -o0 --flags=prefer -Eeof -c2 -o1,2 \
2548 -N -E2m -c1 -Eeof -c1 $file || error "create FLR $file failed"
2549 ids=($($LFS getstripe $file | awk '/lcme_id/{print $2}' | tr '\n' ' '))
2551 dd if=/dev/zero of=$file bs=1M count=3 || error "dd $file failed"
2552 $LFS mirror resync $file || error "resync $file failed"
2554 ost=$($LFS getstripe -I${ids[2]} $file | awk '/l_ost_idx/{print $5}')
2555 if [[ x$ost == "x0," ]]; then
2556 $LFS getstripe $file
2557 error "component ${ids[2]} objects allocated on $ost " \
2561 ost=$($LFS getstripe -I${ids[3]} $file | awk '/l_ost_idx/{print $5}')
2562 if [[ x$ost == "x1," || x$ost == "x2," ]]; then
2563 $LFS getstripe $file
2564 error "component ${ids[3]} objects allocated on $ost " \
2565 "shouldn't on OST1 or on OST2"
2570 # mirror1: [comp0] [comp1]
2571 # mirror2: [comp2] [comp3]
2572 # mirror3: [comp4] [comp5]
2573 # mirror4: [comp6] [comp7]
2574 $LFS mirror create -N4 -E1m -c1 -Eeof -c1 $file ||
2575 error "create FLR $file failed"
2576 ids=($($LFS getstripe $file | awk '/lcme_id/{print $2}' | tr '\n' ' '))
2578 dd if=/dev/zero of=$file bs=1M count=3 || error "dd $file failed"
2579 $LFS mirror resync $file || error "resync $file failed"
2581 for ((i = 0; i < 6; i++)); do
2582 osts[$i]=$($LFS getstripe -I${ids[$i]} $file |
2583 awk '/l_ost_idx/{print $5}')
2585 # comp[0],comp[2],comp[4] should use different osts
2586 if [[ ${osts[0]} == ${osts[2]} || ${osts[0]} == ${osts[4]} ||
2587 ${osts[2]} == ${osts[4]} ]]; then
2588 $LFS getstripe $file
2589 error "component ${ids[0]}, ${ids[2]}, ${ids[4]} have objects "\
2590 "allocated on duplicated OSTs"
2592 # comp[1],comp[3],comp[5] should use different osts
2593 if [[ ${osts[1]} == ${osts[3]} || ${osts[1]} == ${osts[5]} ||
2594 ${osts[3]} == ${osts[5]} ]]; then
2595 $LFS getstripe $file
2596 error "component ${ids[1]}, ${ids[3]}, ${ids[5]} have objects "\
2597 "allocated on duplicated OSTs"
2602 run_test 47 "Verify mirror obj alloc"
2605 [ $MDS1_VERSION -lt $(version_code 2.11.55) ] &&
2606 skip "Need MDS version at least 2.11.55"
2608 local tf=$DIR/$tfile
2611 echo " ** create 2 mirrors FLR file $tf"
2612 $LFS mirror create -N -E2M -Eeof --flags prefer \
2613 -N -E1M -Eeof $tf ||
2614 error "create FLR file $tf failed"
2617 dd if=/dev/urandom of=$tf bs=1M count=3 || error "write $tf failed"
2618 verify_flr_state $tf "wp"
2620 local sum0=$(md5sum < $tf)
2622 echo " ** resync the file"
2623 $LFS mirror resync $tf
2625 echo " ** snapshot mirror 2"
2626 $LFS setstripe --comp-set -I 0x20003 --comp-flags=nosync $tf
2628 echo " ** write it again"
2629 dd if=/dev/urandom of=$tf bs=1M count=3 || error "write $tf failed"
2630 echo " ** resync it again"
2631 $LFS mirror resync $tf
2633 verify_flr_state $tf "wp"
2634 verify_comp_attr lcme_flags $tf 0x20003 nosync,stale
2636 local sum1=$($LFS mirror read -N1 $tf | md5sum)
2637 local sum2=$($LFS mirror read -N2 $tf | md5sum)
2639 echo " ** verify mirror 2 doesn't change"
2640 echo "original checksum: $sum0"
2641 echo "mirror 1 checksum: $sum1"
2642 echo "mirror 2 checksum: $sum2"
2643 [[ $sum0 = $sum2 ]] ||
2644 error "original checksum: $sum0, mirror 2 checksum: $sum2"
2645 echo " ** mirror 2 stripe info"
2646 $LFS getstripe -v --mirror-index=2 $tf
2648 echo " ** resync mirror 2"
2649 $LFS mirror resync --only 2 $tf
2651 verify_flr_state $tf "ro"
2652 verify_comp_attr lcme_flags $tf 0x20003 nosync,^stale
2654 sum1=$($LFS mirror read -N1 $tf | md5sum)
2655 sum2=$($LFS mirror read -N2 $tf | md5sum)
2657 echo " ** verify mirror 2 resync-ed"
2658 echo "original checksum: $sum0"
2659 echo "mirror 1 checksum: $sum1"
2660 echo "mirror 2 checksum: $sum2"
2661 [[ $sum1 = $sum2 ]] ||
2662 error "mirror 1 checksum: $sum1, mirror 2 checksum: $sum2"
2663 echo " ** mirror 2 stripe info"
2664 $LFS getstripe -v --mirror-index=2 $tf
2666 run_test 48 "Verify snapshot mirror"
2675 [ "$OSTCOUNT" -lt "2" ] && skip_env "needs >= 2 OSTs"
2677 trap cleanup_49 EXIT RETURN
2679 local file=$DIR/$tfile
2681 $LFS setstripe -N -E eof -c1 -o1 -N -E eof -c1 -o0 $file ||
2682 error "setstripe on $file"
2684 dd if=/dev/zero of=$file bs=1M count=1 || error "dd failed for $file"
2685 $LFS mirror resync $file
2687 filefrag -ves $file || error "filefrag $file failed"
2688 filefrag_op=$(filefrag -ve -k $file |
2689 sed -n '/ext:/,/found/{/ext:/d; /found/d; p}')
2691 #Filesystem type is: bd00bd0
2692 #File size of /mnt/lustre/f49a.sanity-flr is 1048576 (1024 blocks of 1024 bytes)
2693 # ext: device_logical: physical_offset: length: dev: flags:
2694 # 0: 0.. 1023: 1572864.. 1573887: 1024: 0001: net,eof
2695 # 1: 0.. 1023: 1572864.. 1573887: 1024: 0000: last,net,eof
2696 #/mnt/lustre/f49a.sanity-flr: 2 extents found
2698 last_lun=$(echo $filefrag_op | cut -d: -f5)
2702 for line in $filefrag_op; do
2703 frag_lun=$(echo $line | cut -d: -f5)
2704 ext_len=$(echo $line | cut -d: -f4)
2705 if [[ "$frag_lun" != "$last_lun" ]]; then
2706 if (( tot_len != 1024 )); then
2708 error "$file: OST$last_lun $tot_len != 1024"
2714 (( tot_len += ext_len ))
2717 if (( num_luns != 2 || tot_len != 1024 )); then
2719 error "$file: $num_luns != 2, $tot_len != 1024 on OST$last_lun"
2722 echo "FIEMAP on $file succeeded"
2724 run_test 49a "FIEMAP upon FLR file"
2726 test_50A() { # EX-2179
2729 local file=$DIR/$tdir/$tfile
2731 $LFS setstripe -c1 -i0 $file || error "setstripe $file failed"
2733 $LFS mirror extend -N -c1 -i1 $file ||
2734 error "extending mirror for $file failed"
2736 local olv=$($LFS getstripe $file | awk '/lcm_layout_gen/{print $2}')
2740 $LFS mirror split -d --mirror-id=1 $file || error "split $file failed"
2742 local flv=$($LFS getstripe $file | awk '/lcm_layout_gen/{print $2}')
2744 echo "$file layout generation from $olv to $flv"
2745 (( $flv != ($olv + 1) )) &&
2746 error "split does not increase layout gen from $olv to $flv"
2748 dd if=/dev/zero of=$file bs=1M count=1 || error "write $file failed"
2750 $LFS getstripe -v $file || error "getstripe $file failed"
2752 run_test 50A "mirror split update layout generation"
2755 $LCTL get_param osc.*.import | grep -q 'connect_flags:.*seek' ||
2756 skip "OST does not support SEEK_HOLE"
2757 [ "$FSTYPE" != "zfs" ] ||
2758 skip "lseek for ZFS is not accurate if obj is not committed"
2760 local file=$DIR/$tdir/$tfile
2768 echo " ** create striped file $file"
2769 $LFS setstripe -E 1M -c1 -S 1M -E eof -c2 -S1M $file ||
2770 error "cannot create file with PFL layout"
2771 echo " ** write 1st data chunk at 1M boundary"
2772 dd if=/dev/urandom of=$file bs=1k count=20 seek=1021 ||
2773 error "cannot write data at 1M boundary"
2774 echo " ** write 2nd data chunk at 2M boundary"
2775 dd if=/dev/urandom of=$file bs=1k count=20 seek=2041 ||
2776 error "cannot write data at 2M boundary"
2777 echo " ** create hole at the file end"
2778 $TRUNCATE $file 3700000 || error "truncate fails"
2780 echo " ** verify sparseness"
2781 offset=$(lseek_test -d 1000 $file)
2782 echo " first data offset: $offset"
2783 [[ $offset == 1000 ]] &&
2784 error "src: data is not expected at offset $offset"
2785 offset=$(lseek_test -l 3500000 $file)
2786 echo " hole at the end: $offset"
2787 [[ $offset == 3500000 ]] ||
2788 error "src: hole is expected at offset $offset"
2790 echo " ** extend the file with new mirror"
2791 # migrate_copy_data() is used
2792 $LFS mirror extend -N -E 2M -S 1M -E 1G -S 2M -E eof $file ||
2793 error "cannot create mirror"
2794 $LFS getstripe $file | grep lcme_flags | grep stale > /dev/null &&
2795 error "$file still has stale component"
2797 # check migrate_data_copy() was correct
2798 sum_1=$($LFS mirror read -N 1 $file | md5sum)
2799 sum_2=$($LFS mirror read -N 2 $file | md5sum)
2800 [[ $sum_1 == $sum_2 ]] ||
2801 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
2803 # stale first mirror
2804 $LFS setstripe --comp-set -I0x10001 --comp-flags=stale $file
2805 $LFS setstripe --comp-set -I0x10002 --comp-flags=stale $file
2807 echo " ** verify mirror #2 sparseness"
2808 offset=$(lseek_test -d 1000 $file)
2809 echo " first data offset: $offset"
2810 [[ $offset == 1000 ]] &&
2811 error "dst: data is not expected at offset $offset"
2812 offset=$(lseek_test -l 3500000 $file)
2813 echo " hole at the end: $offset"
2814 [[ $offset == 3500000 ]] ||
2815 error "dst: hole is expected at offset $offset"
2817 echo " ** copy mirror #2 to mirror #1"
2818 $LFS mirror copy -i 2 -o 1 $file || error "mirror copy fails"
2819 $LFS getstripe $file | grep lcme_flags | grep stale > /dev/null &&
2820 error "$file still has stale component"
2822 # check llapi_mirror_copy_many correctness
2823 sum_1=$($LFS mirror read -N 1 $file | md5sum)
2824 sum_2=$($LFS mirror read -N 2 $file | md5sum)
2825 [[ $sum_1 == $sum_2 ]] ||
2826 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
2828 # stale 1st component of mirror #2 before lseek call
2829 $LFS setstripe --comp-set -I0x20001 --comp-flags=stale $file
2831 echo " ** verify mirror #1 sparseness again"
2832 offset=$(lseek_test -d 1000 $file)
2833 echo " first data offset: $offset"
2834 [[ $offset == 1000 ]] &&
2835 error "dst: data is not expected at offset $offset"
2836 offset=$(lseek_test -l 3500000 $file)
2837 echo " hole at the end: $offset"
2838 [[ $offset == 3500000 ]] ||
2839 error "dst: hole is expected at offset $offset"
2841 cancel_lru_locks osc
2843 blocks=$(stat -c%b $file)
2844 echo " ** final consumed blocks: $blocks"
2845 # for 3.5Mb file consumes ~6000 blocks, use 1000 to check
2846 # that file is still sparse
2847 (( blocks < 1000 )) ||
2848 error "Mirrored file consumes $blocks blocks"
2852 run_test 50a "mirror extend/copy preserves sparseness"
2855 $LCTL get_param osc.*.import | grep -q 'connect_flags:.*seek' ||
2856 skip "OST does not support SEEK_HOLE"
2857 [ "$FSTYPE" != "zfs" ] ||
2858 skip "lseek for ZFS is not accurate if obj is not committed"
2860 local file=$DIR/$tdir/$tfile
2868 echo " ** create mirrored file $file"
2869 $LFS mirror create -N -E1M -c1 -S1M -E eof \
2870 -N -E2M -S1M -E eof -S2M $file ||
2871 error "cannot create mirrored file"
2872 echo " ** write data chunk at 1M boundary"
2873 dd if=/dev/urandom of=$file bs=1k count=20 seek=1021 ||
2874 error "cannot write data at 1M boundary"
2875 echo " ** create hole at the file end"
2876 $TRUNCATE $file 3700000 || error "truncate fails"
2878 echo " ** verify sparseness"
2879 offset=$(lseek_test -d 1000 $file)
2880 echo " first data offset: $offset"
2881 [[ $offset == 1000 ]] &&
2882 error "src: data is not expected at offset $offset"
2883 offset=$(lseek_test -l 3500000 $file)
2884 echo " hole at the end: $offset"
2885 [[ $offset == 3500000 ]] ||
2886 error "src: hole is expected at 3500000"
2888 echo " ** resync mirror #2 to mirror #1"
2889 $LFS mirror resync $file
2891 # check llapi_mirror_copy_many correctness
2892 sum_1=$($LFS mirror read -N 1 $file | md5sum)
2893 sum_2=$($LFS mirror read -N 2 $file | md5sum)
2894 [[ $sum_1 == $sum_2 ]] ||
2895 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
2897 cancel_lru_locks osc
2899 blocks=$(stat -c%b $file)
2900 echo " ** consumed blocks: $blocks"
2901 # without full punch() support the first component can be not sparse
2902 # but the last one should be, so file should use far fewer blocks
2903 (( blocks < 5000 )) ||
2904 error "Mirrored file consumes $blocks blocks"
2906 # stale first component in mirror #1
2907 $LFS setstripe --comp-set -I0x10001 --comp-flags=stale,nosync $file
2908 echo " ** truncate file down"
2910 echo " ** write data chunk at 2M boundary"
2911 dd if=/dev/urandom of=$file bs=1k count=20 seek=2041 conv=notrunc ||
2912 error "cannot write data at 2M boundary"
2913 echo " ** resync mirror #2 to mirror #1 with nosync 1st component"
2914 $LFS mirror resync $file || error "mirror rsync fails"
2915 # first component is still stale
2916 $LFS getstripe $file | grep 'lcme_flags:.*stale' > /dev/null ||
2917 error "$file still has no stale component"
2918 echo " ** resync mirror #2 to mirror #1 again"
2919 $LFS setstripe --comp-set -I0x10001 --comp-flags=stale,^nosync $file
2920 $LFS mirror resync $file || error "mirror rsync fails"
2921 $LFS getstripe $file | grep 'lcme_flags:.*stale' > /dev/null &&
2922 error "$file still has stale component"
2924 # check llapi_mirror_copy_many correctness
2925 sum_1=$($LFS mirror read -N 1 $file | md5sum)
2926 sum_2=$($LFS mirror read -N 2 $file | md5sum)
2927 [[ $sum_1 == $sum_2 ]] ||
2928 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
2930 cancel_lru_locks osc
2932 blocks=$(stat -c%b $file)
2933 echo " ** final consumed blocks: $blocks"
2934 # while the first component can lose sparseness, the last one should
2935 # not, so whole file should still use far fewer blocks in total
2936 (( blocks < 3000 )) ||
2937 error "Mirrored file consumes $blocks blocks"
2940 run_test 50b "mirror rsync handles sparseness"
2943 local tf=$DIR/$tdir/$tfile
2945 test_mkdir $DIR/$tdir
2947 $LFS setstripe -N2 -c-1 $tf || error "create FLR $tf failed"
2948 verify_flr_state $tf "ro"
2950 if [[ "$FSTYPE" == "ldiskfs" ]]; then
2951 # ZFS does not support fallocate for now
2952 out=$(fallocate -p -o 1MiB -l 1MiB $tf 2>&1) ||
2953 skip_eopnotsupp "$out|punch hole in $tf failed"
2954 verify_flr_state $tf "wp"
2957 dd if=/dev/zero of=$tf bs=4096 count=4 || error "write $tf failed"
2958 $LFS mirror resync $tf || error "mirror resync $tf failed"
2959 verify_flr_state $tf "ro"
2961 $MULTIOP $tf OSMWUc || error "$MULTIOP $tf failed"
2962 verify_flr_state $tf "wp"
2964 run_test 50c "punch_hole/mmap_write stale other mirrors"
2967 $LCTL get_param osc.*.import | grep -q 'connect_flags:.*seek' ||
2968 skip "OST does not support SEEK_HOLE"
2969 (( $LINUX_VERSION_CODE > $(version_code 3.0.0) )) ||
2970 skip "client kernel does not support SEEK_HOLE"
2971 [ "$FSTYPE" != "zfs" ] ||
2972 skip "lseek for ZFS is not accurate if obj is not committed"
2974 local file=$DIR/$tdir/$tfile
2981 echo " ** create mirrored file $file"
2982 $LFS mirror create -N -E1M -c1 -S1M -E eof \
2983 -N -E2M -S1M -E eof -S2M $file ||
2984 error "cannot create mirrored file"
2985 echo " ** write data chunk at 1M boundary"
2986 dd if=/dev/urandom of=$file bs=1k count=20 seek=1021 ||
2987 error "cannot write data at 1M boundary"
2988 echo " ** create hole at the file start"
2989 prt=$(fallocate -p -o 0 -l 1M $file 2>&1)
2992 if [[ $rc -eq 0 ]]; then
2993 verify_flr_state $file "wp"
2994 elif [[ ! $prt =~ unsupported ]]; then
2995 error "punch hole in $file failed: $prt"
2997 skip "Fallocate punch is not supported: $prt"
3000 echo " ** verify sparseness"
3001 offset=$(lseek_test -d 1000 $file)
3002 echo " first data offset: $offset"
3003 (( $offset >= 1024 * 1024 )) ||
3004 error "src: data is not expected at offset $offset"
3006 echo " ** resync mirror #2"
3007 $LFS mirror resync $file
3009 # check llapi_mirror_copy_many correctness
3010 sum_1=$($LFS mirror read -N 1 $file | md5sum)
3011 sum_2=$($LFS mirror read -N 2 $file | md5sum)
3012 [[ $sum_1 == $sum_2 ]] ||
3013 error "data mismatch: \'$sum_1\' vs. \'$sum_2\'"
3015 cancel_lru_locks osc
3017 # stale first component in mirror #1
3018 $LFS setstripe --comp-set -I0x10001 --comp-flags=stale,nosync $file
3019 echo " ** verify sparseness of mirror #2"
3020 offset=$(lseek_test -d 1000 $file)
3021 echo " first data offset: $offset"
3022 (( $offset >= 1024 * 1024 )) ||
3023 error "src: data is not expected at offset $offset"
3025 run_test 50d "mirror rsync keep holes"
3028 $LCTL get_param osc.*.import | grep -q 'connect_flags:.*seek' ||
3029 skip "OST does not support SEEK_HOLE"
3031 local file=$DIR/$tdir/$tfile
3032 local old_size=2147483648 # 2GiB
3036 dd if=/dev/urandom of=$file bs=4096 count=1 seek=$((134217728 / 4096))
3037 $TRUNCATE $file $old_size
3039 $LFS mirror extend -N -c 1 $file
3040 dd if=/dev/urandom of=$file bs=4096 count=1 seek=$((134217728 / 4096)) conv=notrunc
3041 $LFS mirror resync $file
3043 new_size=$(stat --format='%s' $file)
3044 if ((new_size != old_size)); then
3045 error "new_size ($new_size) is not equal to old_size ($old_size)"
3050 run_test 60a "mirror extend sets correct size on sparse file"
3052 get_flr_layout_gen() {
3053 getfattr -n lustre.lov --only-values $tf 2>/dev/null |
3054 od -tx4 | awk '/000000/ { print "0x"$4; exit; }'
3057 check_layout_gen() {
3060 local v1=$(get_flr_layout_gen $tf)
3061 local v2=$($LFS getstripe -v $tf | awk '/lcm_layout_gen/ { print $2 }')
3063 [[ $v1 -eq $v2 ]] ||
3064 error "$tf in-memory layout gen $v1 != $v2 after $2"
3068 local tf=$DIR/$tdir/$tfile
3070 test_mkdir $DIR/$tdir
3072 $LFS setstripe -Eeof $tf || error "setstripe $tf failed"
3074 for ((i = 0; i < 20; i++)); do
3075 $LFS mirror extend -N $tf ||
3076 error "extending mirror for $tf failed"
3077 check_layout_gen $tf "extend"
3079 $LFS mirror split -d --mirror-id=$((i+1)) $tf ||
3080 error "split $tf failed"
3081 check_layout_gen $tf "split"
3084 run_test 60b "mirror merge/split cancel client's in-memory layout gen"
3087 stat --format='%X %Y %Z' $file || error "$file: cannot get times"
3092 local -a old=( $2 $3 $4 )
3095 new=( $(get_times_61 $file) )
3096 ((${old[0]} == ${new[0]})) ||
3097 error "$file: atime: old '${old[0]}' != new '${new[0]}'"
3099 ((${old[1]} == ${new[1]})) ||
3100 error "$file: mtime: old '${old[1]}' != new '${new[1]}'"
3103 test_61a() { # LU-14508
3104 local file=$DIR/$tdir/$tfile
3110 $LFS setstripe -E1M -Eeof $file || error "setstripe $file failed"
3111 echo "create $file-2"
3112 $LFS setstripe -E2M -Eeof $file-2 || error "setstripe $file-2 failed"
3114 echo XXX > $file || error "write $file failed"
3115 chown $RUNAS_ID $DIR/$tdir $file || error "chown $file failed"
3117 echo "sleep $nap seconds, then cat $tfile"
3119 cat $file || error "cat $file failed"
3121 echo "sleep $nap seconds, then re-write $tfile"
3123 echo XXXX > $file || error "write $file failed"
3124 cp -p $file $file-2 || error "copy $file-2 failed"
3126 echo "sleep $nap seconds"
3129 tim=( $(get_times_61 $file) )
3131 echo "mirror merge $tfile-2 to $tfile and test timestamps"
3132 $LFS mirror extend -N -f $file-2 $file ||
3133 error "cannot mirror merge $file-2 to $file"
3134 check_times_61 $file "${tim[@]}"
3136 echo "mirror extend $tfile and test timestamps"
3137 $LFS mirror extend -N -c1 -i1 $file ||
3138 error "cannot extend mirror $file"
3139 check_times_61 $file "${tim[@]}"
3141 echo "migrate $tfile and test timestamps"
3142 $LFS migrate -n $file || error "cannot migrate $file"
3143 check_times_61 $file "${tim[@]}"
3145 echo "normal user migrate $tfile and test timestamps"
3146 $RUNAS $LFS migrate -n $file || error "cannot migrate $file"
3147 check_times_61 $file "${tim[@]}"
3149 run_test 61a "mirror extend and migrate preserve timestamps"
3151 test_61b() { # LU-14508
3152 local file=$DIR/$tdir/$tfile
3158 echo XXX > $file || error "create $file failed"
3159 chown $RUNAS_ID $DIR/$tdir $file || error "chown $file failed"
3161 echo "sleep $nap seconds, then cat $tfile"
3163 cat $file || error "cat $file failed"
3165 echo "sleep $nap seconds, then re-write $tfile"
3167 echo XXXX > $file || error "write $file failed"
3169 echo "sleep $nap seconds, then test timestamps"
3172 tim=( $(get_times_61 $file) )
3174 echo "mirror extend $tfile and test timestamps"
3175 $LFS mirror extend -N -c1 -i1 $file ||
3176 error "cannot extend mirror $file"
3177 check_times_61 $file "${tim[@]}"
3179 echo "mirror split $tfile and test timestamps"
3180 $LFS mirror split -d --mirror-id=1 $file ||
3181 error "cannot split mirror 1 off $file"
3182 check_times_61 $file "${tim[@]}"
3184 echo "normal user mirror extend $tfile and test timestamps"
3185 $RUNAS $LFS mirror extend -N -c1 -i1 $file ||
3186 error "cannot extend mirror $file"
3187 check_times_61 $file "${tim[@]}"
3189 run_test 61b "mirror extend and split preserve timestamps"
3191 test_61c() { # LU-14508
3192 local file=$DIR/$tdir/$tfile
3198 echo XXX > $file || error "create $file failed"
3199 chown $RUNAS_ID $DIR/$tdir $file || error "chown $file failed"
3201 echo "sleep $nap seconds, then cat $tfile"
3203 cat $file || error "cat $file failed"
3205 echo "sleep $nap seconds, then mirror extend $tfile and write it"
3207 $LFS mirror extend -N -c1 -i1 $file ||
3208 error "cannot extend mirror $file"
3209 echo XXXX > $file || error "write $file failed"
3211 echo "sleep $nap seconds, then resync $tfile and test timestamps"
3212 tim=( $(get_times_61 $file) )
3214 $LFS mirror resync $file || error "cannot resync mirror $file"
3215 check_times_61 $file "${tim[@]}"
3217 echo XXXXXX > $file || error "write $tfile failed"
3219 echo "normal user resync $tfile and test timestamps"
3220 tim=( $(get_times_61 $file) )
3221 $RUNAS $LFS mirror resync $file || error "cannot resync mirror $file"
3222 check_times_61 $file "${tim[@]}"
3224 run_test 61c "mirror resync preserves timestamps"
3227 local tf=$DIR/$tdir/$tfile
3229 test_mkdir $DIR/$tdir
3233 $LFS mirror create -N -E 1M -c -1 -E eof -N $tf
3237 echo "mirror create pid $c_pid"
3240 $LFS mirror split -d --mirror-id=1 $tf &> /dev/null
3243 echo "mirror split pid $s_pid"
3245 echo "mirror create and split race for 60 seconds, should not crash"
3247 kill -9 $c_pid &> /dev/null
3248 kill -9 $s_pid &> /dev/null
3253 run_test 70 "mirror create and split race"
3256 local tf=$DIR/$tdir/$tfile
3258 (( $OST1_VERSION >= $(version_code 2.14.51) )) ||
3259 skip "Need OST version at least 2.14.51"
3262 test_mkdir $DIR/$tdir
3264 $LFS setstripe -N -E1M -c-1 -Eeof -c-1 $tf ||
3265 error "setstripe $tf failed"
3267 FSXNUM=${FSXNUM:-1000}
3268 $FSX -p 5 -N $FSXNUM -S 0 -M $tf || error "fsx FLR file $tf failed"
3270 run_test 100 "flr mode fsx test"
3272 ctrl_file=$(mktemp /tmp/CTRL.XXXXXX)
3273 lock_file=$(mktemp /var/lock/FLR.XXXXXX)
3278 local fsize=$(stat --printf=%s $tf)
3280 while [ -f $ctrl_file ]; do
3281 local off=$((RANDOM << 8))
3282 local len=$((RANDOM << 5 + 131072))
3284 [ $((off + len)) -gt $fsize ] && {
3285 fsize=$((off + len))
3286 echo "Extending file size to $fsize .."
3289 flock -s $lock_file -c \
3290 "$MULTIOP $tf oO_WRONLY:z${off}w${len}c" ||
3292 error "failed writing to $off:$len"; }
3293 sleep 0.$((RANDOM % 2 + 1))
3300 while [ -f $ctrl_file ]; do
3301 flock -s $lock_file -c "cat $tf &> /dev/null" ||
3302 { rm -f $ctrl_file; error "read failed"; }
3303 sleep 0.$((RANDOM % 2 + 1))
3310 options=("" "-e resync_start" "-e delay_before_copy -d 1" "" "")
3312 exec 200<>$lock_file
3313 while [ -f $ctrl_file ]; do
3314 local lock_taken=false
3315 local index=$((RANDOM % ${#options[@]}))
3316 local cmd="mirror_io resync ${options[$index]}"
3318 [ "${options[$index]}" = "" ] && cmd="$LFS mirror resync"
3320 [ $((RANDOM % 4)) -eq 0 ] && {
3326 echo -n "resync file $tf with '$cmd' .."
3328 if [[ $lock_taken = "true" ]]; then
3330 $cmd $tf &> /dev/null && echo "done" || echo "failed"
3333 $cmd $tf &> /dev/null && echo "done" || echo "failed"
3336 sleep 0.$((RANDOM % 8 + 1))
3341 local tf=$DIR/$tfile
3342 local tf2=$DIR2/$tfile
3343 local tf3=$DIR3/$tfile
3345 $LFS setstripe -E 1M -S 1M -E 2M -c 2 -E 4M -E 16M -E eof $tf
3346 $LFS setstripe -E 2M -S 1M -E 6M -c 2 -E 8M -E 32M -E eof $tf-2
3347 $LFS setstripe -E 4M -c 2 -E 8M -E 64M -E eof $tf-3
3349 $LFS mirror extend -N -f $tf-2 $tf ||
3350 error "merging $tf-2 into $tf failed"
3351 $LFS mirror extend -N -f $tf-3 $tf ||
3352 error "merging $tf-3 into $tf failed"
3354 mkdir -p $MOUNT2 && mount_client $MOUNT2
3356 mkdir -p $MOUNT3 && mount_client $MOUNT3
3358 verify_flr_state $tf3 "ro"
3360 #define OBD_FAIL_FLR_RANDOM_PICK_MIRROR 0x1A03
3361 $LCTL set_param fail_loc=0x1A03
3363 local mds_idx=mds$(($($LFS getstripe -m $tf) + 1))
3364 do_facet $mds_idx $LCTL set_param fail_loc=0x1A03
3368 write_file_200 $tf &
3374 write_file_200 $tf2 &
3377 read_file_200 $tf2 &
3380 resync_file_200 $tf3 &
3384 [ "$SLOW" = "yes" ] && sleep_time=360
3385 while [ $sleep_time -gt 0 -a -f $ctrl_file ]; do
3392 echo "Waiting ${pids[*]}"
3395 umount_client $MOUNT2
3396 umount_client $MOUNT3
3400 # resync and verify mirrors
3401 $LFS mirror resync $tf || error "final resync failed"
3404 local csum=$($LFS mirror read -N "${mirror_array[0]}" $tf | md5sum)
3406 for id in "${mirror_array[@]:1}"; do
3407 [ "$($LFS mirror read -N $id $tf | md5sum)" = "$csum" ] ||
3408 error "checksum error for mirror $id"
3413 run_test 200 "stress test"
3415 cleanup_test_201() {
3417 do_facet $SINGLEMDS $LCTL --device $MDT0 changelog_deregister $CL_USER
3419 umount_client $MOUNT2
3423 local delay=${RESYNC_DELAY:-5}
3425 MDT0=$($LCTL get_param -n mdc.*.mds_server_uuid |
3426 awk '{ gsub(/_UUID/,""); print $1 }' | head -n1)
3428 trap cleanup_test_201 EXIT
3430 CL_USER=$(do_facet $SINGLEMDS $LCTL --device $MDT0 \
3431 changelog_register -n)
3433 mkdir -p $MOUNT2 && mount_client $MOUNT2
3437 local log=$($LFS changelog $MDT0 $index | grep FLRW)
3438 [ -z "$log" ] && { sleep 1; continue; }
3440 index=$(echo $log | awk '{print $1}')
3441 local ts=$(date -d "$(echo $log | awk '{print $3}')" "+%s" -u)
3442 local fid=$(echo $log | awk '{print $6}' | sed -e 's/t=//')
3443 local file=$($LFS fid2path $MOUNT2 $fid 2> /dev/null)
3446 [ -z "$file" ] && continue
3448 local now=$(date +%s)
3450 echo "file: $file $fid was modified at $ts, now: $now, " \
3451 "will be resynced at $((ts+delay))"
3453 [ $now -lt $((ts + delay)) ] && sleep $((ts + delay - now))
3455 $LFS mirror resync $file
3456 echo "$file resync done"
3461 run_test 201 "FLR data mover"
3464 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
3466 local tf=$DIR/$tfile
3469 $LFS setstripe -E 1M -S 1M -c 1 $tf
3470 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
3471 verify_comp_attr stripe-count $tf ${ids[0]} 1
3473 $LFS setstripe --component-add -E 2M -c -1 $tf
3474 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
3475 verify_comp_attr stripe-count $tf ${ids[0]} 1
3476 verify_comp_attr stripe-count $tf ${ids[1]} -1
3478 dd if=/dev/zero of=$tf bs=1M count=2
3479 ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' | tr '\n' ' '))
3480 verify_comp_attr stripe-count $tf ${ids[0]} 1
3481 verify_comp_attr stripe-count $tf ${ids[1]} $OSTCOUNT
3483 run_test 202 "lfs setstripe --add-component wide striping"
3486 [ $MDS1_VERSION -lt $(version_code 2.11.55) ] &&
3487 skip "Need MDS version at least 2.11.55"
3488 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs"
3490 local tf=$DIR/$tfile
3493 $LFS mirror create -N2 -c1 $tf || error "create FLR file $tf"
3494 #delete first mirror
3495 $LFS mirror delete --mirror-id=1 $tf || error "delete first mirror"
3498 local old_id=$($LFS getstripe --mirror-id=2 -I $tf)
3499 local count=$($LFS getstripe --mirror-id=2 -c $tf) ||
3500 error "getstripe count of mirror 2"
3501 [[ x$count = x1 ]] || error "mirror 2 stripe count $count is not 1"
3503 #extend a mirror with 2 OSTs
3504 $LFS mirror extend -N -c2 $tf || error "extend mirror"
3507 local new_id=$($LFS getstripe --mirror-id=2 -I $tf)
3508 count=$($LFS getstripe --mirror-id=2 -c $tf) ||
3509 error "getstripe count of mirror 2"
3510 [[ x$old_id = x$new_id ]] ||
3511 error "mirror 2 changed ID from $old_id to $new_id"
3512 [[ x$count = x1 ]] || error "mirror 2 stripe count $count is not 1"
3514 count=$($LFS getstripe --mirror-id=3 -c $tf) ||
3515 error "getstripe count of mirror 3"
3516 [[ x$count = x2 ]] || error "mirror 3 stripe count $count is not 2"
3518 run_test 203 "mirror file preserve mirror ID"
3520 # Simple test of FLR + self-extending layout, SEL in non-primary mirror
3522 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3523 skip "skipped for lustre < $SEL_VER"
3525 local comp_file=$DIR/$tdir/$tfile
3529 test_mkdir $DIR/$tdir
3531 # first mirror is 0-10M, then 10M-(-1), second mirror is 1M followed
3532 # by extension space to -1
3533 $LFS setstripe -N -E 10M -E-1 -N -E 1M -E-1 -z64M $comp_file ||
3534 error "Create $comp_file failed"
3536 # Write to first component, extending & staling second mirror
3537 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc ||
3538 error "dd to extend + stale failed"
3540 $LFS getstripe $comp_file
3542 flg_opts="--component-flags init,stale"
3543 found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
3544 [ $found -eq 1 ] || error "write: Second comp end incorrect"
3546 flg_opts="--component-flags extension"
3547 found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
3548 [ $found -eq 1 ] || error "write: Third comp start incorrect"
3550 # mirror resync should not change the extents
3551 $LFS mirror resync $comp_file
3553 flg_opts="--component-flags init"
3554 found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
3555 [ $found -eq 1 ] || error "resync: Second comp end incorrect"
3557 flg_opts="--component-flags extension"
3558 found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
3559 [ $found -eq 1 ] || error "resync: Third comp start incorrect"
3561 sel_layout_sanity $comp_file 5
3565 run_test 204a "FLR write/stale/resync tests with self-extending mirror"
3567 # Simple test of FLR + self-extending layout, SEL in primary mirror
3569 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3570 skip "skipped for lustre < $SEL_VER"
3572 local comp_file=$DIR/$tdir/$tfile
3576 test_mkdir $DIR/$tdir
3578 # first mirror is 1M followed by extension space to -1, second mirror
3579 # is 0-10M, then 10M-(-1),
3580 $LFS setstripe -N -E 1M -E-1 -z64M -N -E 10M -E-1 $comp_file ||
3581 error "Create $comp_file failed"
3583 # Write to first component, extending first component & staling
3585 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc ||
3586 error "dd to extend + stale failed"
3588 $LFS getstripe $comp_file
3590 flg_opts="--component-flags init"
3591 found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
3592 [ $found -eq 1 ] || error "write: First comp end incorrect"
3594 flg_opts="--component-flags extension"
3595 found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
3596 [ $found -eq 1 ] || error "write: Second comp start incorrect"
3598 flg_opts="--component-flags init,stale"
3599 found=$($LFS find --component-end 10M $flg_opts $comp_file | wc -l)
3600 [ $found -eq 1 ] || error "write: First mirror comp flags incorrect"
3602 # This component is staled because it overlaps the extended first
3603 # component of the primary mirror, even though it doesn't overlap
3604 # the actual write - thus not inited.
3605 flg_opts="--component-flags stale"
3606 found=$($LFS find --component-start 10M $flg_opts $comp_file | wc -l)
3607 [ $found -eq 1 ] || error "write: Second mirror comp flags incorrect"
3609 # mirror resync should not change the extents
3610 $LFS mirror resync $comp_file
3612 $LFS getstripe $comp_file
3614 flg_opts="--component-flags init"
3615 found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
3616 [ $found -eq 1 ] || error "resync: First comp end incorrect"
3618 flg_opts="--component-flags extension"
3619 found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
3620 [ $found -eq 1 ] || error "resync: Second comp start incorrect"
3622 flg_opts="--component-flags init"
3623 found=$($LFS find --component-end 10M $flg_opts $comp_file | wc -l)
3624 [ $found -eq 1 ] || error "resync: First mirror comp flags incorrect"
3626 flg_opts="--component-flags init"
3627 found=$($LFS find --component-start 10M $flg_opts $comp_file | wc -l)
3628 [ $found -eq 1 ] || error "resync: Second mirror comp flags incorrect"
3630 sel_layout_sanity $comp_file 5
3634 run_test 204b "FLR write/stale/resync tests with self-extending primary"
3636 # FLR + SEL failed extension & component removal
3637 # extension space in second mirror
3639 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
3640 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3641 skip "skipped for lustre < $SEL_VER"
3643 local comp_file=$DIR/$tdir/$tfile
3646 local ost_name=$(ostname_from_index $ost_idx1)
3648 test_mkdir $DIR/$tdir
3650 # first mirror is is 0-10M, then 10M-(-1), second mirror is 0-1M, then
3651 # extension space from 1M to 1G, then normal space to -1
3652 $LFS setstripe -N -E 10M -E-1 -N -E 1M -E 1G -i $ost_idx1 -z 64M \
3653 -E -1 $comp_file || error "Create $comp_file failed"
3655 do_facet ost1 $LCTL set_param -n obdfilter.$ost_name.degraded=1
3658 # write to first comp (0 - 10M) of mirror 1, extending + staling
3659 # first + second comp of mirror 2
3660 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc
3663 do_facet ost1 $LCTL set_param -n obdfilter.$ost_name.degraded=0
3666 [ $RC -eq 0 ] || error "dd to extend + stale failed"
3668 $LFS getstripe $comp_file
3670 found=$($LFS find --component-start 0m --component-end 1m \
3671 --comp-flags init,stale $comp_file | wc -l)
3672 [ $found -eq 1 ] || error "write: First mirror comp incorrect"
3674 found=$($LFS find --component-start 1m --component-end EOF \
3675 --comp-flags stale,^init $comp_file | wc -l)
3676 [ $found -eq 1 ] || error "write: Second mirror comp incorrect"
3678 local mirror_id=$($LFS getstripe --component-start=1m \
3679 --component-end=EOF $comp_file | \
3680 grep lcme_mirror_id | awk '{ print $2 }')
3682 [[ $mirror_id -eq 2 ]] ||
3683 error "component not in correct mirror? $mirror_id"
3685 $LFS mirror resync $comp_file
3687 $LFS getstripe $comp_file
3689 # component dimensions should not change from resync
3690 found=$($LFS find --component-start 1m --component-end EOF \
3691 --component-flags init $comp_file | wc -l)
3692 [ $found -eq 1 ] || error "resync: Second mirror comp incorrect"
3694 sel_layout_sanity $comp_file 4
3698 run_test 204c "FLR write/stale/resync test with component removal"
3700 # Successful repeated component in primary mirror
3702 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
3703 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3704 skip "skipped for lustre < $SEL_VER"
3706 local comp_file=$DIR/$tdir/$tfile
3709 wait_delete_completed
3711 test_mkdir $DIR/$tdir
3713 # first mirror is 64M followed by extension space to -1, second mirror
3714 # is 0-10M, then 10M-(-1)
3715 $LFS setstripe -N -E-1 -z64M -N -E 10M -E-1 $comp_file ||
3716 error "Create $comp_file failed"
3718 local ost_idx1=$($LFS getstripe -I65537 -i $comp_file)
3719 local ost_name=$(ostname_from_index $ost_idx1)
3720 # degrade OST for first comp so we won't extend there
3721 do_facet ost$((ost_idx1+1)) $LCTL set_param -n \
3722 obdfilter.$ost_name.degraded=1
3725 # Write beyond first component, causing repeat & stale second mirror
3726 dd if=/dev/zero bs=1M count=1 seek=66 of=$comp_file conv=notrunc
3729 do_facet ost$((ost_idx1+1)) $LCTL set_param -n \
3730 obdfilter.$ost_name.degraded=0
3733 [ $RC -eq 0 ] || error "dd to repeat & stale failed"
3735 $LFS getstripe $comp_file
3737 found=$($LFS find --component-start 64m --component-end 128m \
3738 --component-flags init $comp_file | wc -l)
3739 [ $found -eq 1 ] || error "write: Repeat comp incorrect"
3741 local ost_idx2=$($LFS getstripe --component-start=64m \
3742 --component-end=128m --component-flags=init \
3744 [[ $ost_idx1 -eq $ost_idx2 ]] && error "$ost_idx1 == $ost_idx2"
3745 local mirror_id=$($LFS getstripe --component-start=64m \
3746 --component-end=128m --component-flags=init \
3747 $comp_file | grep lcme_mirror_id | awk '{ print $2 }')
3748 [[ $mirror_id -eq 1 ]] ||
3749 error "component not in correct mirror: $mirror_id, not 1"
3751 $LFS mirror resync $comp_file
3753 $LFS getstripe $comp_file
3755 # component dimensions should not change from resync
3756 found=$($LFS find --component-start 0m --component-end 64m \
3757 --component-flags init $comp_file | wc -l)
3758 [ $found -eq 1 ] || error "resync: first comp incorrect"
3759 found=$($LFS find --component-start 64m --component-end 128m \
3760 --component-flags init $comp_file | wc -l)
3761 [ $found -eq 1 ] || error "resync: repeat comp incorrect"
3763 sel_layout_sanity $comp_file 5
3767 run_test 204d "FLR write/stale/resync sel test with repeated comp"
3769 # Successful repeated component, SEL in non-primary mirror
3771 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
3772 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3773 skip "skipped for lustre < $SEL_VER"
3775 local comp_file=$DIR/$tdir/$tfile
3778 wait_delete_completed
3781 test_mkdir $DIR/$tdir
3783 # first mirror is is 0-100M, then 100M-(-1), second mirror is extension
3784 # space to -1 (-z 64M, so first comp is 0-64M)
3785 # Note: we have to place both 1st components on OST0, otherwise 2 OSTs
3786 # will be not enough - one will be degraded, the other is used on
3787 # an overlapping mirror.
3788 $LFS setstripe -N -E 100M -i 0 -E-1 -N -E-1 -i 0 -z 64M $comp_file ||
3789 error "Create $comp_file failed"
3791 local ost_idx1=$($LFS getstripe --component-start=0 \
3792 --component-end=64m -i $comp_file)
3793 local ost_name=$(ostname_from_index $ost_idx1)
3794 # degrade OST for first comp of 2nd mirror so we won't extend there
3795 do_facet ost$((ost_idx1+1)) $LCTL set_param -n \
3796 obdfilter.$ost_name.degraded=1
3799 $LFS getstripe $comp_file
3801 # Write to first component, stale & instantiate second mirror components
3802 # overlapping with the written component (0-100M);
3803 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc
3806 do_facet ost$((ost_idx1+1)) $LCTL set_param -n \
3807 obdfilter.$ost_name.degraded=0
3809 $LFS getstripe $comp_file
3811 [ $RC -eq 0 ] || error "dd to repeat & stale failed"
3813 found=$($LFS find --component-start 0m --component-end 64m \
3814 --component-flags init,stale $comp_file | wc -l)
3815 [ $found -eq 1 ] || error "write: first comp incorrect"
3817 # was repeated due to degraded ost
3818 found=$($LFS find --component-start 64m --component-end 128m \
3819 --component-flags init,stale $comp_file | wc -l)
3820 [ $found -eq 1 ] || error "write: repeated comp incorrect"
3822 local ost_idx2=$($LFS getstripe --component-start=64m \
3823 --component-end=128m --component-flags=init \
3825 [[ $ost_idx1 -eq $ost_idx2 ]] && error "$ost_idx1 == $ost_idx2"
3826 local mirror_id=$($LFS getstripe --component-start=0m \
3827 --component-end=64m --component-flags=init \
3828 $comp_file | grep lcme_mirror_id | awk '{ print $2 }')
3829 [[ $mirror_id -eq 2 ]] ||
3830 error "component not in correct mirror? $mirror_id"
3832 $LFS mirror resync $comp_file
3834 $LFS getstripe $comp_file
3836 # component dimensions should not change from resync
3837 found=$($LFS find --component-start 0m --component-end 64m \
3838 --component-flags init,^stale $comp_file | wc -l)
3839 [ $found -eq 1 ] || error "resync: first comp incorrect"
3840 found=$($LFS find --component-start 64m --component-end 128m \
3841 --component-flags init,^stale $comp_file | wc -l)
3842 [ $found -eq 1 ] || error "resync: repeated comp incorrect"
3844 sel_layout_sanity $comp_file 5
3848 run_test 204e "FLR write/stale/resync sel test with repeated comp"
3850 # FLR + SEL: failed repeated component, SEL in non-primary mirror
3852 [ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs"
3853 [ "$MDS1_VERSION" -lt $(version_code $SEL_VER) ] &&
3854 skip "skipped for lustre < $SEL_VER"
3856 local comp_file=$DIR/$tdir/$tfile
3859 wait_delete_completed
3861 test_mkdir $DIR/$tdir
3863 pool_add $TESTNAME || error "Pool creation failed"
3864 pool_add_targets $TESTNAME 0 1 || error "Pool add targets failed"
3866 # first mirror is is 0-100M, then 100M-(-1), second mirror is extension
3867 # space to -1 (-z 64M, so first comp is 0-64M)
3868 $LFS setstripe -N -E 100M -E-1 -N --pool="$TESTNAME" \
3869 -E-1 -c 1 -z 64M $comp_file || error "Create $comp_file failed"
3871 local ost_name0=$(ostname_from_index 0)
3872 local ost_name1=$(ostname_from_index 1)
3874 # degrade both OSTs in pool, so we'll try to repeat, then fail and
3875 # extend original comp
3876 do_facet ost1 $LCTL set_param -n obdfilter.$ost_name0.degraded=1
3877 do_facet ost2 $LCTL set_param -n obdfilter.$ost_name1.degraded=1
3880 # a write to the 1st component, 100M length, which will try to stale
3881 # the first 100M of mirror 2, attempting to extend its 0-64M component
3882 dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc
3885 do_facet ost1 $LCTL set_param -n obdfilter.$ost_name0.degraded=0
3886 do_facet ost2 $LCTL set_param -n obdfilter.$ost_name1.degraded=0
3889 [ $RC -eq 0 ] || error "dd to extend mirror comp failed"
3891 $LFS getstripe $comp_file
3893 found=$($LFS find --component-start 0m --component-end 128m \
3894 --component-flags init,stale $comp_file | wc -l)
3895 [ $found -eq 1 ] || error "write: First mirror comp incorrect"
3897 local mirror_id=$($LFS getstripe --component-start=0m \
3898 --component-end=128m --component-flags=init \
3899 $comp_file | grep lcme_mirror_id | awk '{ print $2 }')
3901 [[ $mirror_id -eq 2 ]] ||
3902 error "component not in correct mirror? $mirror_id, not 2"
3904 $LFS mirror resync $comp_file
3906 $LFS getstripe $comp_file
3908 # component dimensions should not change from resync
3909 found=$($LFS find --component-start 0m --component-end 128m \
3910 --component-flags init,^stale $comp_file | wc -l)
3911 [ $found -eq 1 ] || error "resync: First mirror comp incorrect"
3913 sel_layout_sanity $comp_file 4
3917 run_test 204f "FLR write/stale/resync sel w/forced extension"
3919 function test_205() {
3920 local tf=$DIR/$tfile
3923 $LFS setstripe -c1 $tf
3924 $LFS mirror extend -N $tf
3925 mirrors=$($LFS getstripe $tf | grep lcme_mirror_id | wc -l )
3926 (( $mirrors == 2 )) || error "no new mirror was created?"
3928 $LFS mirror extend -N --flags=prefer $tf
3929 mirrors=$($LFS getstripe $tf | grep lcme_mirror_id | wc -l )
3930 (( $mirrors == 3 )) || error "no new mirror was created?"
3932 $($LFS getstripe $tf | grep lcme_flags: | tail -1 | grep -q prefer) ||
3933 error "prefer flag was not set on the new mirror"
3935 run_test 205 "lfs mirror extend to set prefer flag"
3937 function test_206() {
3938 # create a new OST pool
3939 local pool_name=$TESTNAME
3941 create_pool $FSNAME.$pool_name ||
3942 error "create OST pool $pool_name failed"
3943 # add OSTs into the pool
3944 pool_add_targets $pool_name 0 1 ||
3945 error "add OSTs into pool $pool_name failed"
3947 $LFS setstripe -c1 --pool=$pool_name $DIR/$tfile ||
3948 error "can't setstripe"
3949 $LFS mirror extend -N $DIR/$tfile ||
3950 error "can't create replica"
3951 if $LFS getstripe $DIR/$tfile | grep -q prefer ; then
3952 $LFS getstripe $DIR/$tfile
3953 error "prefer found"
3955 $LFS setstripe --comp-set --comp-flags=prefer -p $pool_name $DIR/$tfile || {
3956 $LFS getstripe $DIR/$tfile
3957 error "can't setstripe prefer"
3960 if ! $LFS getstripe $DIR/$tfile | grep -q prefer ; then
3961 $LFS getstripe $DIR/$tfile
3962 error "no prefer found"
3968 run_test 206 "lfs setstripe -pool .. --comp-flags=.. "
3971 local file=$DIR/$tfile
3972 local tmpfile=$DIR/$tfile-tt
3974 [ $MDS1_VERSION -lt $(version_code 2.14.50) ] &&
3975 skip "Need MDS version at least 2.14.50"
3977 stack_trap "rm -f $tmpfile $file"
3979 # generate data for verification
3980 dd if=/dev/urandom of=$tmpfile bs=1M count=1 ||
3981 error "can't generate file with random data"
3983 # create a mirrored file with one stale replica
3984 $LFS mirror create -N -S 4M -c 2 -N -S 1M -c -1 $file ||
3985 error "create mirrored file $file failed"
3986 get_mirror_ids $file
3987 echo "mirror IDs: ${mirror_array[*]}"
3989 dd if=$tmpfile of=$file bs=1M || error "can't copy"
3990 get_mirror_ids $file
3991 echo "mirror IDs: ${mirror_array[*]}"
3994 cmp $tmpfile $file || error "files don't match"
3995 get_mirror_ids $file
3996 echo "mirror IDs: ${mirror_array[*]}"
3998 # mirror creation should work fine
3999 $LFS mirror extend -N -S 8M -c -1 $file ||
4000 error "mirror extend $file failed"
4002 get_mirror_ids $file
4003 echo "mirror IDs: ${mirror_array[*]}"
4006 $LFS mirror verify -v $file || error "verification failed"
4007 cmp $tmpfile $file || error "files don't match"
4009 run_test 207 "create another replica with existing out-of-sync one"
4011 function check_ost_used() {
4020 cancel_lru_locks osc # to drop pages
4021 cancel_lru_locks mdc # to refresh layout
4022 # XXX: cancel_lru_locks mdc doesn't work
4023 # XXX: need a better way to reload the layout
4024 umount_client $MOUNT || error "umount failed"
4025 mount_client $MOUNT || error "mount failed"
4027 # refresh non-rotation status on MDTs
4029 touch $DIR/$tfile-temp
4030 rm -f $DIR/$tfile-temp
4031 # refresh non-rotational status on the client
4035 $LCTL set_param osc.*.stats=clear >/dev/null
4036 if [[ $io == "read" ]]; then
4037 ddarg="if=$file of=/dev/null"
4038 elif [[ $io == "write" ]]; then
4039 ddarg="if=/dev/zero of=$file"
4041 error "unknown type $io"
4043 dd $ddarg bs=2M count=1 || error "can't $io $file"
4044 cancel_lru_locks osc
4046 # check only specified OSTs got reads
4047 for ((ost = 0; ost < $OSTCOUNT; ost++)); do
4048 local nr=$($LCTL get_param -n \
4049 osc.$FSNAME-OST000$ost-osc-[-0-9a-f]*.stats |
4050 awk "/ost_$io/{print \$2}")
4052 if [[ " $* " =~ $ost ]]; then
4053 (( nr > 0 )) || error "expected reads on $ost"
4055 (( nr == 0 )) || error "unexpected $nr reads on $ost"
4061 local tf=$DIR/$tfile
4062 local osts=$(comma_list $(osts_nodes))
4064 (( $OSTCOUNT >= 4 )) || skip "needs >= 4 OSTs"
4065 (( $MDS1_VERSION >= $(version_code 2.14.55) )) ||
4066 skip "Need MDS version at least 2.14.55"
4068 local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
4070 save_lustre_params $(get_facets OST) osd*.*OST*.nonrotational > $p
4071 stack_trap "restore_lustre_params < $p; rm -f $p"
4073 stack_trap "rm -f $tf"
4074 $LFS setstripe -i0 -c1 $tf || error "can't setstripe"
4075 dd if=/dev/zero of=$tf bs=2M count=1 || error "can't dd (1)"
4076 $LFS mirror extend -N -c1 -o1 $tf || error "can't create mirror"
4077 $LFS mirror extend -N -c2 -o 2,3 $tf || error "can't create mirror"
4078 $LFS mirror resync $tf || error "can't resync"
4081 log "set OST0000 non-rotational"
4083 $LCTL set_param osd*.*OST*.nonrotational=0
4085 $LCTL set_param osd*.*OST0000*.nonrotational=1
4086 check_ost_used $tf read 0
4088 log "set OST0002 and OST0003 non-rotational, two fast OSTs is better"
4090 $LCTL set_param osd*.*OST*.nonrotational=0
4092 $LCTL set_param osd*.*OST0002*.nonrotational=1 \
4093 osd*.*OST0003*.nonrotational=1
4094 check_ost_used $tf read 2 3
4096 log "set mirror 1 on OST0001 preferred"
4097 $LFS setstripe --comp-set -I 0x20001 --comp-flags=prefer $tf ||
4098 error "can't set prefer"
4099 check_ost_used $tf read 1
4101 run_test 208a "mirror selection to prefer non-rotational devices for reads"
4104 local tf=$DIR/$tfile
4105 local osts=$(comma_list $(osts_nodes))
4107 (( $OSTCOUNT >= 4 )) || skip "needs >= 4 OSTs"
4108 (( $MDS1_VERSION >= $(version_code 2.14.55) )) ||
4109 skip "Need MDS version at least 2.14.55"
4111 local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
4113 save_lustre_params $(get_facets OST) osd*.*OST*.nonrotational > $p
4114 stack_trap "restore_lustre_params < $p; rm -f $p"
4116 stack_trap "rm -f $tf"
4117 $LFS setstripe -i0 -c1 $tf || error "can't setstripe"
4118 dd if=/dev/zero of=$tf bs=2M count=1 || error "can't dd (1)"
4119 $LFS mirror extend -N -c1 -o1 $tf || error "can't create mirror"
4120 $LFS mirror extend -N -c2 -o 2,3 $tf || error "can't create mirror"
4121 $LFS mirror resync $tf || error "can't resync"
4122 $LFS getstripe $tf | grep -q flags.*stale && error "still stale"
4124 log "set OST0000 non-rotational"
4126 $LCTL set_param osd*.*OST*.nonrotational=0
4128 $LCTL set_param osd*.*OST0000*.nonrotational=1
4129 check_ost_used $tf write 0
4130 $LFS mirror resync $tf || error "can't resync"
4132 log "set OST0002 and OST0003 non-rotational, two fast OSTs is better"
4134 $LCTL set_param osd*.*OST*.nonrotational=0
4136 $LCTL set_param osd*.*OST0002*.nonrotational=1 \
4137 osd*.*OST0003*.nonrotational=1
4138 check_ost_used $tf write 2 3
4139 $LFS mirror resync $tf || error "can't resync"
4141 log "set mirror 1 on OST0001 preferred"
4142 $LFS setstripe --comp-set -I 0x20001 --comp-flags=prefer $tf ||
4143 error "can't set prefer"
4144 check_ost_used $tf write 1
4146 run_test 208b "mirror selection to prefer non-rotational devices for writes"
4149 check_and_cleanup_lustre