3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
9 export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin
12 # Bug number for skipped test:
13 ALWAYS_EXCEPT="$SANITY_FLR_EXCEPT 201"
14 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
16 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
17 . $LUSTRE/tests/test-framework.sh
19 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
22 if [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.53) ]]; then
23 skip_env "Need MDS version at least 2.10.53" && exit
26 [ $UID -eq 0 -a $RUNAS_ID -eq 0 ] &&
27 error "\$RUNAS_ID set to 0, but \$UID is also 0!"
28 check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS
30 check_and_setup_lustre
36 # global array to store mirror IDs
37 declare -a mirror_array
44 for id in $($LFS getstripe $tf | awk '/lcme_id/{print $2}'); do
45 array[${#array[@]}]=$((id >> 16))
48 mirror_array=($(printf "%s\n" "${array[@]}" | sort -u))
50 echo ${#mirror_array[@]}
54 echo 3 > /proc/sys/vm/drop_caches
65 wait_osc_import_state client ost$idx DISCONN
73 start ost$idx $(ostdevname $idx) $OST_MOUNT_OPTS ||
74 error "start ost$idx failed"
78 wait_osc_import_state client ost$idx FULL
82 # command line test cases
85 local mirror_count=16 # LUSTRE_MIRROR_COUNT_MAX
86 local mirror_create_cmd="$LFS mirror create"
87 local stripes[0]=$OSTCOUNT
89 mirror_create_cmd+=" -N -c ${stripes[0]}"
90 for ((i = 1; i < $mirror_count; i++)); do
91 # add mirrors with different stripes to the file
92 stripes[$i]=$((RANDOM % OSTCOUNT))
93 [ ${stripes[$i]} -eq 0 ] && stripes[$i]=1
95 mirror_create_cmd+=" -N -c ${stripes[$i]}"
98 eval $mirror_create_cmd $tf || error "creating mirrored file $tf failed"
100 [ $(get_mirror_ids $tf) -ne $mirror_count ] &&
101 error "mirror count error"
103 # can't create mirrors exceeding LUSTRE_MIRROR_COUNT_MAX
104 $LFS mirror extend -N $tf &&
105 error "Creating the $((mirror_count+1))th mirror succeeded"
107 local ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' |
110 # verify the range of components and stripe counts
111 for ((i = 0; i < $mirror_count; i++)); do
112 local sc=$($LFS getstripe -I${ids[$i]} -c $tf)
113 local start=$($LFS getstripe -I${ids[$i]} --component-start $tf)
114 local end=$($LFS getstripe -I${ids[$i]} --component-end $tf)
116 [[ ${stripes[$i]} = $sc ]] || {
117 $LFS getstripe -v $tf;
118 error "$i: sc error: id: ${ids[$i]}, ${stripes[$i]}";
120 [ $start -eq 0 ] || {
121 $LFS getstripe -v $tf;
122 error "$i: start error id: ${ids[$i]}";
124 [ $end = "EOF" ] || {
125 $LFS getstripe -v $tf;
126 error "$i: end error id: ${ids[$i]}";
130 run_test 1 "create components with setstripe options"
134 local tf2=$DIR/$tfile-2
136 $LFS setstripe -E 1M -E EOF -c 1 $tf
137 $LFS setstripe -E 2M -E EOF -c -1 $tf2
139 local layout=$($LFS getstripe $tf2 | grep -A 4 lmm_objects)
141 $LFS mirror extend -N -f $tf2 $tf ||
142 error "merging $tf2 into $tf failed"
144 [ $(get_mirror_ids $tf) -ne 2 ] && error "mirror count should be 2"
145 [[ ! -e $tf2 ]] || error "$tf2 was not unlinked"
147 run_test 2 "create components from existing files"
150 [[ $MDSCOUNT -lt 2 ]] && skip "need >= 2 MDTs" && return
152 for ((i = 0; i < 2; i++)); do
153 $LFS mkdir -i $i $DIR/$tdir-$i
154 $LFS setstripe -E -1 $DIR/$tdir-$i/$tfile
157 $LFS mirror extend -N -f $DIR/$tdir-1/$tfile \
158 $DIR/$tdir-0/$tfile || error "creating mirrors"
160 # mdt doesn't support to cancel layout lock for remote objects, do
164 # make sure the mirrorted file was created successfully
165 [[ $($LFS getstripe --component-count $DIR/$tdir-0/$tfile) -eq 2 ]] ||
166 { $LFS getstripe $DIR/$tdir-0/$tfile;
167 error "expected 2 components"; }
172 run_test 3 "create components from files located on different MDTs"
176 local tf2=$DIR/$tfile-2
178 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
180 $LFS setstripe -E EOF -o 0 $tf
181 $LFS setstripe -E EOF -o 1 $tf2
183 local dd_count=$((RANDOM % 20 + 1))
184 dd if=/dev/zero of=$tf bs=1M count=$dd_count
185 dd if=/dev/zero of=$tf2 bs=1M count=1 seek=$((dd_count - 1))
188 local blocks=$(du -kc $tf $tf2 | awk '/total/{print $1}')
191 $LFS mirror extend -N -f $tf2 $tf ||
192 error "merging $tf2 into $tf failed"
197 local new_blocks=$(du -k $tf | awk '{print $1}')
198 [ $new_blocks -eq $blocks ] ||
199 error "i_blocks error expected: $blocks, actual: $new_blocks"
201 run_test 21 "glimpse should report accurate i_blocks"
203 get_osc_lock_count() {
210 osc_name=${FSNAME}-OST$(printf "%04x" $((idx-1)))-osc-'ffff*'
211 count=$($LCTL get_param -n ldlm.namespaces.$osc_name.lock_count)
212 lock_count=$((lock_count + count))
220 $LFS setstripe -E EOF -o 0 $tf
221 dd if=/dev/zero of=$tf bs=1M count=$((RANDOM % 20 + 1))
223 # add component, two mirrors located on the same OST ;-)
224 $LFS mirror extend -N -o 0 $tf ||
225 error "extending mirrored file $tf failed"
227 size_blocks=$(stat --format="%b %s" $tf)
232 local new_size_blocks=$(stat --format="%b %s" $tf)
234 # make sure there is no lock cached
235 [ $(get_osc_lock_count 1) -eq 0 ] || error "glimpse requests were sent"
237 [ "$new_size_blocks" = "$size_blocks" ] ||
238 echo "size expected: $size_blocks, actual: $new_size_blocks"
242 run_test 22 "no glimpse to OSTs for READ_ONLY files"
247 $LFS mirror create -N -o 0 -N -o 1 $tf ||
248 error "creating mirrored file $tf failed"
250 #define OBD_FAIL_GLIMPSE_IMMUTABLE 0x1A00
251 $LCTL set_param fail_loc=0x1A00
254 for ((ost_idx = 1; ost_idx <= 2; ost_idx++)); do
258 local tmpfile=$(mktemp)
259 stat --format="%b %s" $tf > $tmpfile &
263 while [ $cnt -le 5 ]; do
264 kill -0 $pid > /dev/null 2>&1 || break
268 kill -0 $pid > /dev/null 2>&1 &&
269 error "stat process stuck due to unavailable OSTs"
271 # make sure glimpse request has been sent
272 [ $(get_osc_lock_count 1 2) -ne 0 ] ||
273 error "OST $ost_idx: no glimpse request was sent"
278 run_test 31 "make sure glimpse request can be retried"
281 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
282 rm -f $DIR/$tfile $DIR/$tfile-2
284 $LFS setstripe -E EOF -o 0 $DIR/$tfile
285 dd if=/dev/urandom of=$DIR/$tfile bs=1M count=$((RANDOM % 10 + 2))
287 local fsize=$(stat -c %s $DIR/$tfile)
288 [[ $fsize -ne 0 ]] || error "file size is (wrongly) zero"
290 local cksum=$(md5sum $DIR/$tfile)
292 # create a new mirror in sync mode
293 $LFS mirror extend -N -o 1 $DIR/$tfile ||
294 error "extending mirrored file $DIR/$tfile failed"
296 # make sure the mirrored file was created successfully
297 [ $(get_mirror_ids $DIR/$tfile) -eq 2 ] ||
298 { $LFS getstripe $DIR/$tfile; error "expected 2 mirrors"; }
303 # check size is correct, glimpse request should go to the 2nd mirror
304 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
305 error "file size error $fsize vs. $(stat -c %s $DIR/$tfile)"
307 echo "reading file from the 2nd mirror and verify checksum"
308 [[ "$cksum" == "$(md5sum $DIR/$tfile)" ]] ||
309 error "checksum error: expected $cksum"
313 run_test 32 "data should be mirrored to newly created mirror"
316 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
318 rm -f $DIR/$tfile $DIR/$tfile-2
320 # create a file with two mirrors
321 $LFS setstripe -E EOF -o 0 $DIR/$tfile
324 while [ $count -lt $max_count ]; do
325 echo "ost1" >> $DIR/$tfile
326 count=$((count + 1));
329 # tmp file that will be used as mirror
330 $LFS setstripe -E EOF -o 1 $DIR/$tfile-2
332 while [ $count -lt $max_count ]; do
333 echo "ost2" >> $DIR/$tfile-2
334 count=$((count + 1));
337 # create a mirrored file
338 $LFS mirror extend -N -f $DIR/$tfile-2 $DIR/$tfile &&
339 error "merging $DIR/$tfile-2 into $DIR/$tfile" \
340 "with verification should fail"
341 $LFS mirror extend --no-verify -N -f $DIR/$tfile-2 $DIR/$tfile ||
342 error "merging $DIR/$tfile-2 into $DIR/$tfile" \
343 "without verification failed"
345 # make sure that $tfile has two mirrors and $tfile-2 does not exist
346 [ $(get_mirror_ids $DIR/$tfile) -eq 2 ] ||
347 { $LFS getstripe $DIR/$tfile; error "expected count 2"; }
349 [[ ! -e $DIR/$tfile-2 ]] || error "$DIR/$tfile-2 was not unlinked"
352 local fsize=$((5 * max_count))
353 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
354 error "mirrored file size is not $fsize"
356 # read file - all OSTs are available
357 echo "reading file (data should be provided by ost1)... "
358 local rs=$(cat $DIR/$tfile | head -1)
359 [[ "$rs" == "ost1" ]] ||
360 error "file content error: expected: \"ost1\", actual: \"$rs\""
362 # read file again with ost1 failed
366 echo "reading file (data should be provided by ost2)..."
367 local rs=$(cat $DIR/$tfile | head -1)
368 [[ "$rs" == "ost2" ]] ||
369 error "file content error: expected: \"ost2\", actual: \"$rs\""
374 # read file again with ost2 failed
375 $LCTL set_param ldlm.namespaces.lustre-*-osc-ffff*.lru_size=clear
380 # check size, glimpse should work
381 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
382 error "mirrored file size is not $fsize"
384 echo "reading file (data should be provided by ost1)..."
385 local rs=$(cat $DIR/$tfile | head -1)
386 [[ "$rs" == "ost1" ]] ||
387 error "file content error: expected: \"ost1\", actual: \"$rs\""
389 wait_osc_import_state client ost2 FULL
391 run_test 33 "read can choose available mirror to read"
394 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
396 rm -f $DIR/$tfile $DIR/$tfile-2 $DIR/$tfile-ref
399 $LFS setstripe -o 0 $DIR/$tfile-ref
400 dd if=/dev/urandom of=$DIR/$tfile-ref bs=1M count=3
402 # create a file with two mirrors
403 $LFS setstripe -E -1 -o 0,1 -S 1M $DIR/$tfile
404 dd if=$DIR/$tfile-ref of=$DIR/$tfile bs=1M
406 $LFS setstripe -E -1 -o 2,3 -S 1M $DIR/$tfile-2
407 dd if=$DIR/$tfile-ref of=$DIR/$tfile-2 bs=1M
409 $CHECKSTAT -t file -s $((3 * 1024 * 1024)) $DIR/$tfile ||
410 error "mirrored file size is not 3M"
412 # merge a mirrored file
413 $LFS mirror extend -N -f $DIR/$tfile-2 $DIR/$tfile ||
414 error "merging $DIR/$tfile-2 into $DIR/$tfile failed"
418 # stop two OSTs, so the 2nd stripe of the 1st mirror and
419 # the 1st stripe of the 2nd mirror will be inaccessible, ...
422 echo "comparing files ... "
424 # however, read can still return the correct data. It should return
425 # the 1st stripe from mirror 1 and 2st stripe from mirror 2.
426 cmp -n 2097152 <(rwv -f $DIR/$tfile -r -o -n 1 2097152) \
427 $DIR/$tfile-ref || error "file reading error"
431 run_test 34a "read mirrored file with multiple stripes"
434 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
436 rm -f $DIR/$tfile $DIR/$tfile-2 $DIR/$tfile-ref
439 $LFS setstripe -o 0 $DIR/$tfile-ref
440 dd if=/dev/urandom of=$DIR/$tfile-ref bs=1M count=3
442 $LFS setstripe -E 1M -S 1M -o 0 -E eof -o 1 $DIR/$tfile
443 dd if=$DIR/$tfile-ref of=$DIR/$tfile bs=1M
445 $LFS setstripe -E 1M -S 1M -o 2 -E eof -o 3 $DIR/$tfile-2
446 dd if=$DIR/$tfile-ref of=$DIR/$tfile-2 bs=1M
448 $CHECKSTAT -t file -s $((3 * 1024 * 1024)) $DIR/$tfile ||
449 error "mirrored file size is not 3M"
451 # merge a mirrored file
452 $LFS mirror extend -N -f $DIR/$tfile-2 $DIR/$tfile ||
453 error "merging $DIR/$tfile-2 into $DIR/$tfile failed"
457 # stop two OSTs, so the 2nd component of the 1st mirror and
458 # the 1st component of the 2nd mirror will be inaccessible, ...
461 echo "comparing files ... "
463 # however, read can still return the correct data. It should return
464 # the 1st stripe from mirror 1 and 2st stripe from mirror 2.
465 cmp -n 2097152 <(rwv -f $DIR/$tfile -r -o -n 1 2097152) \
466 $DIR/$tfile-ref || error "file reading error"
470 run_test 34b "read mirrored file with multiple components"
475 $LFS setstripe -E eof $tf
477 # add an out-of-sync mirror to the file
478 $LFS mirror extend -N -c 2 $tf ||
479 error "extending mirrored file $tf failed"
481 $MULTIOP $tf oO_WRONLY:c ||
482 error "write open a mirrored file failed"
484 # truncate file should return error
485 $TRUNCATE $tf 100 || error "error truncating a mirrored file"
487 run_test 35 "allow to write to mirrored files"
489 verify_ost_layout_version() {
492 # get file layout version
493 local flv=$($LFS getstripe $tf | awk '/lcm_layout_gen/{print $2}')
495 # layout version from OST objects
496 local olv=$($MULTIOP $tf oXc | awk '/ostlayoutversion/{print $2}')
498 [ $flv -eq $olv ] || error "layout version mismatch: $flv vs. $olv"
505 $LFS setstripe -E 1M -E 2M -E 4M -E eof -c -1 $tf
506 $LFS setstripe -E 3M -E 6M -E eof -c -1 $tf-tmp
508 $LFS mirror extend -N -f $tf-tmp $tf ||
509 error "merging $tf-tmp into $tf failed"
516 create_file_36 $tf $tf-2 $tf-3
518 [ $(get_mirror_ids $tf) -gt 1 ] || error "wrong mirror count"
520 # test case 1 - check file write and verify layout version
521 $MULTIOP $tf oO_WRONLY:c ||
522 error "write open a mirrored file failed"
524 # write open file should not return error
525 $MULTIOP $tf oO_WRONLY:w1024Yc || error "write mirrored file error"
527 # instantiate components should work
528 dd if=/dev/zero of=$tf bs=1M count=12 || error "write file error"
530 # verify OST layout version
531 verify_ost_layout_version $tf
534 local mds_idx=mds$(($($LFS getstripe -M $tf-2) + 1))
537 do_facet $mds_idx $LCTL set_param fail_val=$delay_sec
539 #define OBD_FAIL_FLR_LV_DELAY 0x1A01
540 do_facet $mds_idx $LCTL set_param fail_loc=0x1A01
542 # write should take at least $fail_loc seconds and succeed
544 $MULTIOP $tf-2 oO_WRONLY:w1024Yc || error "write mirrored file error"
546 [ $(date +%s) -ge $((st+delay_sec)) ] ||
547 error "write finished before layout version is transmitted"
549 # verify OST layout version
550 verify_ost_layout_version $tf
552 do_facet $mds_idx $LCTL set_param fail_loc=0
555 mds_idx=mds$(($($LFS getstripe -M $tf-3) + 1))
557 #define OBD_FAIL_FLR_LV_INC 0x1A02
558 do_facet $mds_idx $LCTL set_param fail_loc=0x1A02
560 # write open file should return error
561 $MULTIOP $tf-3 oO_WRONLY:O_SYNC:w1024c &&
562 error "write a mirrored file succeeded" || true
564 do_facet $mds_idx $LCTL set_param fail_loc=0
566 run_test 36 "write to mirrored files"
572 echo "create test files with size $fsize .."
576 $LFS setstripe -E 1M -c 1 -E eof -c -1 $tf
578 dd if=/dev/urandom of=$tf bs=1M count=16 &> /dev/null
586 local tf2=$DIR/$tfile-2
587 local tf3=$DIR/$tfile-3
589 create_files_37 $((RANDOM + 15 * 1048576)) $tf $tf2 $tf3
591 # assume the mirror id will be 1, 2, and 3
593 checksums[1]=$(md5sum $tf | cut -f 1 -d' ')
594 checksums[2]=$(md5sum $tf2 | cut -f 1 -d' ')
595 checksums[3]=$(md5sum $tf3 | cut -f 1 -d' ')
597 printf '%s\n' "${checksums[@]}"
599 # merge these files into a mirrored file
600 $LFS mirror extend --no-verify -N -f $tf2 $tf ||
601 error "merging $tf2 into $tf failed"
602 $LFS mirror extend --no-verify -N -f $tf3 $tf ||
603 error "merging $tf3 into $tf failed"
607 # verify mirror read, checksums should equal to the original files'
608 echo "Verifying mirror read .."
611 for i in ${mirror_array[@]}; do
612 sum=$(mirror_io dump -i $i $tf | md5sum | cut -f 1 -d' ')
613 [ "$sum" = "${checksums[$i]}" ] ||
614 error "$i: mismatch: \'${checksums[$i]}\' vs. \'$sum\'"
617 # verify mirror copy, write to this mirrored file will invalidate
618 # the other two mirrors
619 echo "Verifying mirror copy .."
621 local osts=$(comma_list $(osts_nodes))
623 # define OBD_FAIL_OST_SKIP_LV_CHECK 0x241
624 do_nodes $osts lctl set_param fail_loc=0x241
626 mirror_io copy -i ${mirror_array[0]} \
627 -t $(echo ${mirror_array[@]:1} | tr ' ' ',') $tf ||
628 error "mirror copy error"
630 do_nodes $osts lctl set_param fail_loc=0
632 # verify copying is successful by checking checksums
633 remount_client $MOUNT
634 for i in ${mirror_array[@]}; do
635 sum=$(mirror_io dump -i $i $tf | md5sum | cut -f 1 -d' ')
636 [ "$sum" = "${checksums[1]}" ] ||
637 error "$i: mismatch checksum after copy"
642 run_test 37 "mirror I/O API verification"
647 local expected_state=$2
648 local state_strings=("not_flr" "read_only" "write_pending" \
651 local state=$($LFS getstripe -v $tf | awk '/lcm_flags/{ print $2 }')
652 [ $expected_state = ${state_strings[$state]} ] ||
653 error "expected: $expected_state, " \
654 "actual ${state_strings[$state]}($state)"
659 local ref=$DIR/${tfile}-ref
661 $LFS setstripe -E 1M -c 1 -E 4M -c 2 -E eof -c -1 $tf
662 $LFS setstripe -E 2M -c 1 -E 6M -c 2 -E 8M -c -1 -E eof -c -1 $tf-2
663 $LFS setstripe -E 4M -c 1 -E 8M -c 2 -E eof -c -1 $tf-3
665 # instantiate all components
666 $LFS mirror extend -N -f $tf-2 $tf ||
667 error "merging $tf-2 into $tf failed"
668 $LFS mirror extend -N -f $tf-3 $tf ||
669 error "merging $tf-3 into $tf failed"
670 $LFS mirror extend -N -c 1 $tf ||
671 error "extending mirrored file $tf failed"
673 verify_flr_state $tf "read_only"
675 dd if=/dev/urandom of=$ref bs=1M count=16 &> /dev/null
677 local fsize=$((RANDOM << 8 + 1048576))
678 $TRUNCATE $ref $fsize
680 local ref_cksum=$(md5sum $ref | cut -f 1 -d' ')
682 # case 1: verify write to mirrored file & resync work
683 cp $ref $tf || error "copy from $ref to $f error"
684 verify_flr_state $tf "write_pending"
686 local file_cksum=$(md5sum $tf | cut -f 1 -d' ')
687 [ "$file_cksum" = "$ref_cksum" ] || error "write failed, cksum mismatch"
690 echo "mirror IDs: ${mirror_array[@]}"
692 local valid_mirror stale_mirror id mirror_cksum
693 for id in "${mirror_array[@]}"; do
694 mirror_cksum=$(mirror_io dump -i $id $tf |
695 md5sum | cut -f 1 -d' ')
696 [ "$ref_cksum" == "$mirror_cksum" ] &&
697 { valid_mirror=$id; continue; }
702 [ -z "$stale_mirror" ] && error "stale mirror doesn't exist"
703 [ -z "$valid_mirror" ] && error "valid mirror doesn't exist"
705 mirror_io resync $tf || error "resync failed"
706 verify_flr_state $tf "read_only"
708 mirror_cksum=$(mirror_io dump -i $stale_mirror $tf |
709 md5sum | cut -f 1 -d' ')
710 [ "$file_cksum" = "$ref_cksum" ] || error "resync failed"
712 # case 2: inject an error to make mirror_io exit after changing
713 # the file state to sync_pending so that we can start a concurrent
715 $MULTIOP $tf oO_WRONLY:w$((RANDOM % 1048576 + 1024))c
716 verify_flr_state $tf "write_pending"
718 mirror_io resync -e resync_start $tf && error "resync succeeded"
719 verify_flr_state $tf "sync_pending"
721 # from sync_pending to write_pending
722 $MULTIOP $tf oO_WRONLY:w$((RANDOM % 1048576 + 1024))c
723 verify_flr_state $tf "write_pending"
725 mirror_io resync -e resync_start $tf && error "resync succeeded"
726 verify_flr_state $tf "sync_pending"
728 # from sync_pending to read_only
729 mirror_io resync $tf || error "resync failed"
730 verify_flr_state $tf "read_only"
734 ctrl_file=$(mktemp /tmp/CTRL.XXXXXX)
735 lock_file=$(mktemp /var/lock/FLR.XXXXXX)
740 local fsize=$(stat --printf=%s $tf)
742 while [ -f $ctrl_file ]; do
743 local off=$((RANDOM << 8))
744 local len=$((RANDOM << 5 + 131072))
746 [ $((off + len)) -gt $fsize ] && {
748 echo "Extending file size to $fsize .."
751 flock -s $lock_file -c \
752 "$MULTIOP $tf oO_WRONLY:z${off}w${len}c" ||
754 error "failed writing to $off:$len"; }
755 sleep 0.$((RANDOM % 2 + 1))
762 while [ -f $ctrl_file ]; do
763 flock -s $lock_file -c "cat $tf &> /dev/null" ||
764 { rm -f $ctrl_file; error "read failed"; }
765 sleep 0.$((RANDOM % 2 + 1))
772 options=("" "-e resync_start" "-e delay_before_copy -d 1" "" "")
775 while [ -f $ctrl_file ]; do
776 local index=$((RANDOM % ${#options[@]}))
777 local lock_taken=false
779 [ $((RANDOM % 4)) -eq 0 ] && {
785 echo -n "resync file $tf with '${options[$index]}' .."
787 $lock_taken && flock -x 200
788 mirror_io resync ${options[$index]} $tf &> /dev/null &&
789 echo "done" || echo "failed"
791 $lock_taken && flock -u 200
793 sleep 0.$((RANDOM % 8 + 1))
799 local tf2=$DIR2/$tfile
800 local tf3=$DIR3/$tfile
802 $LFS setstripe -E 1M -E 2M -c 2 -E 4M -E 16M -E eof $tf
803 $LFS setstripe -E 2M -E 6M -c 2 -E 8M -E 32M -E eof $tf-2
804 $LFS setstripe -E 4M -c 2 -E 8M -E 64M -E eof $tf-3
806 $LFS mirror extend -N -f $tf-2 $tf ||
807 error "merging $tf-2 into $tf failed"
808 $LFS mirror extend -N -f $tf-3 $tf ||
809 error "merging $tf-3 into $tf failed"
811 mkdir -p $MOUNT2 && mount_client $MOUNT2
813 mkdir -p $MOUNT3 && mount_client $MOUNT3
815 verify_flr_state $tf3 "read_only"
817 #define OBD_FAIL_FLR_RANDOM_PICK_MIRROR 0x1A03
818 $LCTL set_param fail_loc=0x1A03
820 local mds_idx=mds$(($($LFS getstripe -M $tf) + 1))
821 do_facet $mds_idx $LCTL set_param fail_loc=0x1A03
831 write_file_200 $tf2 &
837 resync_file_200 $tf3 &
841 [ "$SLOW" = "yes" ] && sleep_time=360
842 while [ $sleep_time -gt 0 -a -f $ctrl_file ]; do
849 echo "Waiting ${pids[@]}"
852 umount_client $MOUNT2
853 umount_client $MOUNT3
857 # resync and verify mirrors
861 local csum=$(mirror_io dump -i ${mirror_array[0]} $tf | md5sum)
862 for id in ${mirror_array[@]:1}; do
863 [ "$(mirror_io dump -i $id $tf | md5sum)" = "$csum" ] ||
864 error "checksum error for mirror $id"
869 run_test 200 "stress test"
873 do_facet $SINGLEMDS $LCTL --device $MDT0 changelog_deregister $CL_USER
875 umount_client $MOUNT2
879 local delay=${RESYNC_DELAY:-5}
881 MDT0=$($LCTL get_param -n mdc.*.mds_server_uuid |
882 awk '{ gsub(/_UUID/,""); print $1 }' | head -n1)
884 trap cleanup_test_201 EXIT
886 CL_USER=$(do_facet $SINGLEMDS $LCTL --device $MDT0 \
887 changelog_register -n)
889 mkdir -p $MOUNT2 && mount_client $MOUNT2
893 local log=$($LFS changelog $MDT0 $index | grep FLRW)
894 [ -z "$log" ] && { sleep 1; continue; }
896 index=$(echo $log | awk '{print $1}')
897 local ts=$(date -d "$(echo $log | awk '{print $3}')" "+%s" -u)
898 local fid=$(echo $log | awk '{print $6}' | sed -e 's/t=//')
899 local file=$($LFS fid2path $MOUNT2 $fid 2> /dev/null)
902 [ -z "$file" ] && continue
904 local now=$(date +%s)
906 echo "file: $file $fid was modified at $ts, now: $now, " \
907 "will be resynced at $((ts+delay))"
909 [ $now -lt $((ts + delay)) ] && sleep $((ts + delay - now))
911 mirror_io resync $file
912 echo "$file resync done"
917 run_test 201 "FLR data mover"
920 check_and_cleanup_lustre