3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
6 # Run test by setting NOSETUP=true when ltest has setup env for us
11 export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin
14 # Bug number for skipped test:
15 ALWAYS_EXCEPT="$SANITY_FLR_EXCEPT"
16 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
18 [ "$ALWAYS_EXCEPT$EXCEPT" ] &&
19 echo "Skipping tests: $ALWAYS_EXCEPT $EXCEPT"
22 CHECKSTAT=${CHECKSTAT:-"checkstat -v"}
25 MULTIOP=${MULTIOP:-multiop}
27 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
28 . $LUSTRE/tests/test-framework.sh
30 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
33 check_and_setup_lustre
37 if [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.64) ]]; then
38 skip_env "Need MDS version at least 2.7.64" && exit
43 [ $UID -eq 0 -a $RUNAS_ID -eq 0 ] &&
44 error "\$RUNAS_ID set to 0, but \$UID is also 0!"
45 check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS
47 # global array to store mirror IDs
48 declare -a mirror_array
55 for id in $($LFS getstripe $tf | awk '/lcme_id/{print $2}'); do
56 array[${#array[@]}]=$((id >> 16))
59 mirror_array=($(printf "%s\n" "${array[@]}" | sort -u))
61 echo ${#mirror_array[@]}
65 echo 3 > /proc/sys/vm/drop_caches
76 wait_osc_import_state client ost$idx DISCONN
84 start ost$idx $(ostdevname $idx) $OST_MOUNT_OPTS ||
85 error "start ost$idx failed"
89 wait_osc_import_state client ost$idx FULL
93 # command line test cases
96 local mirror_count=16 # LUSTRE_MIRROR_COUNT_MAX
98 $LFS setstripe -E EOF -c -1 $tf
100 local stripes[0]=$OSTCOUNT
102 for ((i = 1; i < $mirror_count; i++)); do
103 # add mirrors with different stripes to the file
104 stripes[$i]=$((RANDOM % OSTCOUNT))
105 [ ${stripes[$i]} -eq 0 ] && stripes[$i]=1
107 $LFS setstripe --component-add --mirror -c ${stripes[$i]} $tf
110 [ $(get_mirror_ids $tf) -ne $mirror_count ] &&
111 error "mirror count error"
113 # can't create mirrors exceeding LUSTRE_MIRROR_COUNT_MAX
114 $LFS setstripe --component-add --mirror $tf &&
115 error "Creating the $((mirror_count+1))th mirror succeeded"
117 local ids=($($LFS getstripe $tf | awk '/lcme_id/{print $2}' |
120 # verify the range of components and stripe counts
121 for ((i = 0; i < $mirror_count; i++)); do
122 local sc=$($LFS getstripe -I${ids[$i]} -c $tf)
123 local start=$($LFS getstripe -I${ids[$i]} --component-start $tf)
124 local end=$($LFS getstripe -I${ids[$i]} --component-end $tf)
126 [[ ${stripes[$i]} = $sc ]] || {
127 $LFS getstripe -v $tf;
128 error "$i: sc error: id: ${ids[$i]}, ${stripes[$i]}";
130 [ $start -eq 0 ] || {
131 $LFS getstripe -v $tf;
132 error "$i: start error id: ${ids[$i]}";
134 [ $end = "EOF" ] || {
135 $LFS getstripe -v $tf;
136 error "$i: end error id: ${ids[$i]}";
140 run_test 1 "create components with setstripe options"
144 local tf2=$DIR/$tfile-2
146 $LFS setstripe -E 1M -E EOF -c 1 $tf
147 $LFS setstripe -E 2M -E EOF -c -1 $tf2
149 local layout=$($LFS getstripe $tf2 | grep -A 4 lmm_objects)
151 $LFS setstripe --component-add --mirror=$tf2 $tf
153 [ $(get_mirror_ids $tf) -ne 2 ] && error "mirror count should be 2"
154 $LFS getstripe $tf2 | grep -q 'no stripe info' ||
155 error "$tf2 still has stripe info"
157 run_test 2 "create components from existing files"
160 [[ $MDSCOUNT -lt 2 ]] && skip "need >= 2 MDTs" && return
162 for ((i = 0; i < 2; i++)); do
163 $LFS mkdir -i $i $DIR/$tdir-$i
164 $LFS setstripe -E -1 $DIR/$tdir-$i/$tfile
167 $LFS setstripe --component-add --mirror=$DIR/$tdir-1/$tfile \
168 $DIR/$tdir-0/$tfile || error "creating mirrors"
170 # mdt doesn't support to cancel layout lock for remote objects, do
174 # make sure the mirrorted file was created successfully
175 [[ $($LFS getstripe --component-count $DIR/$tdir-0/$tfile) -eq 2 ]] ||
176 { $LFS getstripe $DIR/$tdir-0/$tfile;
177 error "expected 2 components"; }
182 run_test 3 "create components from files located on different MDTs"
186 local tf2=$DIR/$tfile-2
188 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
190 $LFS setstripe -E EOF -o 0 $tf
191 $LFS setstripe -E EOF -o 1 $tf2
193 local dd_count=$((RANDOM % 20 + 1))
194 dd if=/dev/zero of=$tf bs=1M count=$dd_count
195 dd if=/dev/zero of=$tf2 bs=1M count=1 seek=$((dd_count - 1))
198 local blocks=$(du -kc $tf $tf2 | awk '/total/{print $1}')
201 $LFS setstripe --component-add --mirror=$tf2 $tf
206 local new_blocks=$(du -k $tf | awk '{print $1}')
207 [ $new_blocks -eq $blocks ] ||
208 error "i_blocks error expected: $blocks, actual: $new_blocks"
210 run_test 21 "glimpse should report accurate i_blocks"
212 get_osc_lock_count() {
219 osc_name=${FSNAME}-OST$(printf "%04x" $((idx-1)))-osc-'ffff*'
220 count=$($LCTL get_param -n ldlm.namespaces.$osc_name.lock_count)
221 lock_count=$((lock_count + count))
229 $LFS setstripe -E EOF -o 0 $tf
230 dd if=/dev/zero of=$tf bs=1M count=$((RANDOM % 20 + 1))
232 # add component, two mirrors located on the same OST ;-)
233 $LFS setstripe --component-add --mirror -o 0 $tf
235 size_blocks=$(stat --format="%b %s" $tf)
240 local new_size_blocks=$(stat --format="%b %s" $tf)
242 # make sure there is no lock cached
243 [ $(get_osc_lock_count 1) -eq 0 ] || error "glimpse requests were sent"
245 [ "$new_size_blocks" = "$size_blocks" ] ||
246 echo "size expected: $size_blocks, actual: $new_size_blocks"
250 run_test 22 "no glimpse to OSTs for READ_ONLY files"
255 $LFS setstripe -E EOF -o 0 $tf
256 $LFS setstripe --component-add --mirror -o 1 $tf
258 #define OBD_FAIL_GLIMPSE_IMMUTABLE 0x1A00
259 $LCTL set_param fail_loc=0x1A00
262 for ((ost_idx = 1; ost_idx <= 2; ost_idx++)); do
266 local tmpfile=$(mktemp)
267 stat --format="%b %s" $tf > $tmpfile &
271 while [ $cnt -le 5 ]; do
272 kill -0 $pid > /dev/null 2>&1 || break
276 kill -0 $pid > /dev/null 2>&1 &&
277 error "stat process stuck due to unavailable OSTs"
279 # make sure glimpse request has been sent
280 [ $(get_osc_lock_count 1 2) -ne 0 ] ||
281 error "OST $ost_idx: no glimpse request was sent"
286 run_test 31 "make sure glimpse request can be retried"
289 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
290 rm -f $DIR/$tfile $DIR/$tfile-2
292 $LFS setstripe -E EOF -o 0 $DIR/$tfile
293 dd if=/dev/urandom of=$DIR/$tfile bs=1M count=$((RANDOM % 10 + 2))
295 local fsize=$(stat -c %s $DIR/$tfile)
296 [[ $fsize -ne 0 ]] || error "file size is (wrongly) zero"
298 local cksum=$(md5sum $DIR/$tfile)
300 # create a new mirror in sync mode
301 $LFS setstripe --component-add --mirror -o 1 $DIR/$tfile
303 # make sure the mirrored file was created successfully
304 [ $(get_mirror_ids $DIR/$tfile) -eq 2 ] ||
305 { $LFS getstripe $DIR/$tfile; error "expected 2 mirrors"; }
310 # check size is correct, glimpse request should go to the 2nd mirror
311 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
312 error "file size error $fsize vs. $(stat -c %s $DIR/$tfile)"
314 echo "reading file from the 2nd mirror and verify checksum"
315 [[ "$cksum" == "$(md5sum $DIR/$tfile)" ]] ||
316 error "checksum error: expected $cksum"
320 run_test 32 "data should be mirrored to newly created mirror"
323 [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
325 rm -f $DIR/$tfile $DIR/$tfile-2
327 # create a file with two mirrors
328 $LFS setstripe -E EOF -o 0 $DIR/$tfile
331 while [ $count -lt $max_count ]; do
332 echo "ost1" >> $DIR/$tfile
333 count=$((count + 1));
336 # tmp file that will be used as mirror
337 $LFS setstripe -E EOF -o 1 $DIR/$tfile-2
339 while [ $count -lt $max_count ]; do
340 echo "ost2" >> $DIR/$tfile-2
341 count=$((count + 1));
344 # create a mirrored file
345 $LFS setstripe --component-add --mirror=$DIR/$tfile-2 $DIR/$tfile
347 # make sure that $tfile has two mirrors and $tfile-2 has no stripe
348 [ $(get_mirror_ids $DIR/$tfile) -eq 2 ] ||
349 { $LFS getstripe $DIR/$tfile; error "expected count 2"; }
350 $LFS getstripe $DIR/$tfile-2 | grep -q "no stripe info" ||
351 { $LFS getstripe $DIR/$tfile; error "expected no stripe"; }
354 local fsize=$((5 * max_count))
355 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
356 error "mirrored file size is not $fsize"
358 # read file - all OSTs are available
359 echo "reading file (data should be provided by ost1)... "
360 local rs=$(cat $DIR/$tfile | head -1)
361 [[ "$rs" == "ost1" ]] ||
362 error "file content error: expected: \"ost1\", actual: \"$rs\""
364 # read file again with ost1 failed
368 echo "reading file (data should be provided by ost2)..."
369 local rs=$(cat $DIR/$tfile | head -1)
370 [[ "$rs" == "ost2" ]] ||
371 error "file content error: expected: \"ost2\", actual: \"$rs\""
376 # read file again with ost2 failed
377 $LCTL set_param ldlm.namespaces.lustre-*-osc-ffff*.lru_size=clear
382 # check size, glimpse should work
383 $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
384 error "mirrored file size is not $fsize"
386 echo "reading file (data should be provided by ost1)..."
387 local rs=$(cat $DIR/$tfile | head -1)
388 [[ "$rs" == "ost1" ]] ||
389 error "file content error: expected: \"ost1\", actual: \"$rs\""
391 wait_osc_import_state client ost2 FULL
393 run_test 33 "read can choose available mirror to read"
396 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
398 rm -f $DIR/$tfile $DIR/$tfile-2 $DIR/$tfile-ref
401 $LFS setstripe -o 0 $DIR/$tfile-ref
402 dd if=/dev/urandom of=$DIR/$tfile-ref bs=1M count=3
404 # create a file with two mirrors
405 $LFS setstripe -E -1 -o 0,1 -S 1M $DIR/$tfile
406 dd if=$DIR/$tfile-ref of=$DIR/$tfile bs=1M
408 $LFS setstripe -E -1 -o 2,3 -S 1M $DIR/$tfile-2
409 dd if=$DIR/$tfile-ref of=$DIR/$tfile-2 bs=1M
411 $CHECKSTAT -t file -s $((3 * 1024 * 1024)) $DIR/$tfile ||
412 error "mirrored file size is not 3M"
414 # merge a mirrored file
415 $LFS setstripe --component-add --mirror=$DIR/$tfile-2 $DIR/$tfile
419 # stop two OSTs, so the 2nd stripe of the 1st mirror and
420 # the 1st stripe of the 2nd mirror will be inaccessible, ...
423 echo "comparing files ... "
425 # however, read can still return the correct data. It should return
426 # the 1st stripe from mirror 1 and 2st stripe from mirror 2.
427 cmp -n 2097152 <(rwv -f $DIR/$tfile -r -o -n 1 2097152) \
428 $DIR/$tfile-ref || error "file reading error"
432 run_test 34a "read mirrored file with multiple stripes"
435 [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
437 rm -f $DIR/$tfile $DIR/$tfile-2 $DIR/$tfile-ref
440 $LFS setstripe -o 0 $DIR/$tfile-ref
441 dd if=/dev/urandom of=$DIR/$tfile-ref bs=1M count=3
443 $LFS setstripe -E 1M -S 1M -o 0 -E eof -o 1 $DIR/$tfile
444 dd if=$DIR/$tfile-ref of=$DIR/$tfile bs=1M
446 $LFS setstripe -E 1M -S 1M -o 2 -E eof -o 3 $DIR/$tfile-2
447 dd if=$DIR/$tfile-ref of=$DIR/$tfile-2 bs=1M
449 $CHECKSTAT -t file -s $((3 * 1024 * 1024)) $DIR/$tfile ||
450 error "mirrored file size is not 3M"
452 # merge a mirrored file
453 $LFS setstripe --component-add --mirror=$DIR/$tfile-2 $DIR/$tfile
457 # stop two OSTs, so the 2nd component of the 1st mirror and
458 # the 1st component of the 2nd mirror will be inaccessible, ...
461 echo "comparing files ... "
463 # however, read can still return the correct data. It should return
464 # the 1st stripe from mirror 1 and 2st stripe from mirror 2.
465 cmp -n 2097152 <(rwv -f $DIR/$tfile -r -o -n 1 2097152) \
466 $DIR/$tfile-ref || error "file reading error"
470 run_test 34b "read mirrored file with multiple components"
475 $LFS setstripe -E eof $tf
477 # add an out-of-sync mirror to the file
478 $LFS setstripe --component-add --mirror -c 2 $tf
480 $MULTIOP $tf oO_WRONLY:c ||
481 error "write open a mirrored file failed"
483 # truncate file should return error
484 $TRUNCATE $tf 100 || error "error truncating a mirrored file"
486 run_test 35 "allow to write to mirrored files"
488 verify_ost_layout_version() {
491 # get file layout version
492 local flv=$($LFS getstripe $tf | awk '/lcm_layout_gen/{print $2}')
494 # layout version from OST objects
495 local olv=$($MULTIOP $tf oXc | awk '/ostlayoutversion/{print $2}')
497 [ $flv -eq $olv ] || error "layout version mismatch: $flv vs. $olv"
504 $LFS setstripe -E 1M -E 2M -E 4M -E eof -c -1 $tf
505 $LFS setstripe -E 3M -E 6M -E eof -c -1 $tf-tmp
507 $LFS setstripe --component-add --mirror=$tf-tmp $tf
515 create_file_36 $tf $tf-2 $tf-3
517 [ $(get_mirror_ids $tf) -gt 1 ] || error "wrong mirror count"
519 # test case 1 - check file write and verify layout version
520 $MULTIOP $tf oO_WRONLY:c ||
521 error "write open a mirrored file failed"
523 # write open file should not return error
524 $MULTIOP $tf oO_WRONLY:w1024Yc || error "write mirrored file error"
526 # instantiate components should work
527 dd if=/dev/zero of=$tf bs=1M count=12 || error "write file error"
529 # verify OST layout version
530 verify_ost_layout_version $tf
533 local mds_idx=mds$(($($LFS getstripe -M $tf-2) + 1))
536 do_facet $mds_idx $LCTL set_param fail_val=$delay_sec
538 #define OBD_FAIL_FLR_LV_DELAY 0x1A01
539 do_facet $mds_idx $LCTL set_param fail_loc=0x1A01
541 # write should take at least $fail_loc seconds and succeed
543 $MULTIOP $tf-2 oO_WRONLY:w1024Yc || error "write mirrored file error"
545 [ $(date +%s) -ge $((st+delay_sec)) ] ||
546 error "write finished before layout version is transmitted"
548 # verify OST layout version
549 verify_ost_layout_version $tf
551 do_facet $mds_idx $LCTL set_param fail_loc=0
554 mds_idx=mds$(($($LFS getstripe -M $tf-3) + 1))
556 #define OBD_FAIL_FLR_LV_INC 0x1A02
557 do_facet $mds_idx $LCTL set_param fail_loc=0x1A02
559 # write open file should return error
560 $MULTIOP $tf-3 oO_WRONLY:O_SYNC:w1024c &&
561 error "write a mirrored file succeeded" || true
563 do_facet $mds_idx $LCTL set_param fail_loc=0
565 run_test 36 "write to mirrored files"
571 echo "create test files with size $fsize .."
575 $LFS setstripe -E 1M -c 1 -E eof -c -1 $tf
577 dd if=/dev/urandom of=$tf bs=1M count=16 &> /dev/null
585 local tf2=$DIR/$tfile-2
586 local tf3=$DIR/$tfile-3
588 create_files_37 $((RANDOM + 15 * 1048576)) $tf $tf2 $tf3
590 # assume the mirror id will be 1, 2, and 3
592 checksums[1]=$(md5sum $tf | cut -f 1 -d' ')
593 checksums[2]=$(md5sum $tf2 | cut -f 1 -d' ')
594 checksums[3]=$(md5sum $tf3 | cut -f 1 -d' ')
596 printf '%s\n' "${checksums[@]}"
598 # merge these files into a mirrored file
599 $LFS setstripe --component-add --mirror=$tf2 $tf
600 $LFS setstripe --component-add --mirror=$tf3 $tf
604 # verify mirror read, checksums should equal to the original files'
605 echo "Verifying mirror read .."
608 for i in ${mirror_array[@]}; do
609 sum=$(mirror_io dump -i $i $tf | md5sum | cut -f 1 -d' ')
610 [ "$sum" = "${checksums[$i]}" ] ||
611 error "$i: mismatch: \'${checksums[$i]}\' vs. \'$sum\'"
614 # verify mirror copy, write to this mirrored file will invalidate
615 # the other two mirrors
616 echo "Verifying mirror copy .."
618 local osts=$(comma_list $(osts_nodes))
620 # define OBD_FAIL_OST_SKIP_LV_CHECK 0x241
621 do_nodes $osts lctl set_param fail_loc=0x241
623 mirror_io copy -i ${mirror_array[0]} \
624 -t $(echo ${mirror_array[@]:1} | tr ' ' ',') $tf ||
625 error "mirror copy error"
627 do_nodes $osts lctl set_param fail_loc=0
629 # verify copying is successful by checking checksums
630 remount_client $MOUNT
631 for i in ${mirror_array[@]}; do
632 sum=$(mirror_io dump -i $i $tf | md5sum | cut -f 1 -d' ')
633 [ "$sum" = "${checksums[1]}" ] ||
634 error "$i: mismatch checksum after copy"
639 run_test 37 "mirror I/O API verification"
642 check_and_cleanup_lustre