+test_31() {
+ local tf=$DIR/$tfile
+
+ $LFS setstripe -E EOF -o 0 $tf
+ $LFS setstripe --component-add --mirror -o 1 $tf
+
+ #define OBD_FAIL_GLIMPSE_IMMUTABLE 0x1A00
+ $LCTL set_param fail_loc=0x1A00
+
+ local ost_idx
+ for ((ost_idx = 1; ost_idx <= 2; ost_idx++)); do
+ cancel_lru_locks osc
+ stop_osts $ost_idx
+
+ local tmpfile=$(mktemp)
+ stat --format="%b %s" $tf > $tmpfile &
+ local pid=$!
+
+ local cnt=0
+ while [ $cnt -le 5 ]; do
+ kill -0 $pid > /dev/null 2>&1 || break
+ sleep 1
+ ((cnt += 1))
+ done
+ kill -0 $pid > /dev/null 2>&1 &&
+ error "stat process stuck due to unavailable OSTs"
+
+ # make sure glimpse request has been sent
+ [ $(get_osc_lock_count 1 2) -ne 0 ] ||
+ error "OST $ost_idx: no glimpse request was sent"
+
+ start_osts $ost_idx
+ done
+}
+run_test 31 "make sure glimpse request can be retried"
+
+test_32() {
+ [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
+ rm -f $DIR/$tfile $DIR/$tfile-2
+
+ $LFS setstripe -E EOF -o 0 $DIR/$tfile
+ dd if=/dev/urandom of=$DIR/$tfile bs=1M count=$((RANDOM % 10 + 2))
+
+ local fsize=$(stat -c %s $DIR/$tfile)
+ [[ $fsize -ne 0 ]] || error "file size is (wrongly) zero"
+
+ local cksum=$(md5sum $DIR/$tfile)
+
+ # create a new mirror in sync mode
+ $LFS setstripe --component-add --mirror -o 1 $DIR/$tfile
+
+ # make sure the mirrored file was created successfully
+ [ $(get_mirror_ids $DIR/$tfile) -eq 2 ] ||
+ { $LFS getstripe $DIR/$tfile; error "expected 2 mirrors"; }
+
+ drop_client_cache
+ stop_osts 1
+
+ # check size is correct, glimpse request should go to the 2nd mirror
+ $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
+ error "file size error $fsize vs. $(stat -c %s $DIR/$tfile)"
+
+ echo "reading file from the 2nd mirror and verify checksum"
+ [[ "$cksum" == "$(md5sum $DIR/$tfile)" ]] ||
+ error "checksum error: expected $cksum"
+
+ start_osts 1
+}
+run_test 32 "data should be mirrored to newly created mirror"
+
+test_33() {
+ [[ $OSTCOUNT -lt 2 ]] && skip "need >= 2 OSTs" && return
+
+ rm -f $DIR/$tfile $DIR/$tfile-2
+
+ # create a file with two mirrors
+ $LFS setstripe -E EOF -o 0 $DIR/$tfile
+ local max_count=100
+ local count=0
+ while [ $count -lt $max_count ]; do
+ echo "ost1" >> $DIR/$tfile
+ count=$((count + 1));
+ done
+
+ # tmp file that will be used as mirror
+ $LFS setstripe -E EOF -o 1 $DIR/$tfile-2
+ count=0
+ while [ $count -lt $max_count ]; do
+ echo "ost2" >> $DIR/$tfile-2
+ count=$((count + 1));
+ done
+
+ # create a mirrored file
+ $LFS setstripe --component-add --mirror=$DIR/$tfile-2 $DIR/$tfile
+
+ # make sure that $tfile has two mirrors and $tfile-2 has no stripe
+ [ $(get_mirror_ids $DIR/$tfile) -eq 2 ] ||
+ { $LFS getstripe $DIR/$tfile; error "expected count 2"; }
+ $LFS getstripe $DIR/$tfile-2 | grep -q "no stripe info" ||
+ { $LFS getstripe $DIR/$tfile; error "expected no stripe"; }
+
+ # execpted file size
+ local fsize=$((5 * max_count))
+ $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
+ error "mirrored file size is not $fsize"
+
+ # read file - all OSTs are available
+ echo "reading file (data should be provided by ost1)... "
+ local rs=$(cat $DIR/$tfile | head -1)
+ [[ "$rs" == "ost1" ]] ||
+ error "file content error: expected: \"ost1\", actual: \"$rs\""
+
+ # read file again with ost1 failed
+ stop_osts 1
+ drop_client_cache
+
+ echo "reading file (data should be provided by ost2)..."
+ local rs=$(cat $DIR/$tfile | head -1)
+ [[ "$rs" == "ost2" ]] ||
+ error "file content error: expected: \"ost2\", actual: \"$rs\""
+
+ # remount ost1
+ start_osts 1
+
+ # read file again with ost2 failed
+ $LCTL set_param ldlm.namespaces.lustre-*-osc-ffff*.lru_size=clear
+
+ fail ost2 &
+ sleep 1
+
+ # check size, glimpse should work
+ $CHECKSTAT -t file -s $fsize $DIR/$tfile ||
+ error "mirrored file size is not $fsize"
+
+ echo "reading file (data should be provided by ost1)..."
+ local rs=$(cat $DIR/$tfile | head -1)
+ [[ "$rs" == "ost1" ]] ||
+ error "file content error: expected: \"ost1\", actual: \"$rs\""
+
+ wait_osc_import_state client ost2 FULL
+}
+run_test 33 "read can choose available mirror to read"
+
+test_34a() {
+ [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
+
+ rm -f $DIR/$tfile $DIR/$tfile-2 $DIR/$tfile-ref
+
+ # reference file
+ $LFS setstripe -o 0 $DIR/$tfile-ref
+ dd if=/dev/urandom of=$DIR/$tfile-ref bs=1M count=3
+
+ # create a file with two mirrors
+ $LFS setstripe -E -1 -o 0,1 -S 1M $DIR/$tfile
+ dd if=$DIR/$tfile-ref of=$DIR/$tfile bs=1M
+
+ $LFS setstripe -E -1 -o 2,3 -S 1M $DIR/$tfile-2
+ dd if=$DIR/$tfile-ref of=$DIR/$tfile-2 bs=1M
+
+ $CHECKSTAT -t file -s $((3 * 1024 * 1024)) $DIR/$tfile ||
+ error "mirrored file size is not 3M"
+
+ # merge a mirrored file
+ $LFS setstripe --component-add --mirror=$DIR/$tfile-2 $DIR/$tfile
+
+ cancel_lru_locks osc
+
+ # stop two OSTs, so the 2nd stripe of the 1st mirror and
+ # the 1st stripe of the 2nd mirror will be inaccessible, ...
+ stop_osts 2 3
+
+ echo "comparing files ... "
+
+ # however, read can still return the correct data. It should return
+ # the 1st stripe from mirror 1 and 2st stripe from mirror 2.
+ cmp -n 2097152 <(rwv -f $DIR/$tfile -r -o -n 1 2097152) \
+ $DIR/$tfile-ref || error "file reading error"
+
+ start_osts 2 3
+}
+run_test 34a "read mirrored file with multiple stripes"
+
+test_34b() {
+ [[ $OSTCOUNT -lt 4 ]] && skip "need >= 4 OSTs" && return
+
+ rm -f $DIR/$tfile $DIR/$tfile-2 $DIR/$tfile-ref
+
+ # reference file
+ $LFS setstripe -o 0 $DIR/$tfile-ref
+ dd if=/dev/urandom of=$DIR/$tfile-ref bs=1M count=3
+
+ $LFS setstripe -E 1M -S 1M -o 0 -E eof -o 1 $DIR/$tfile
+ dd if=$DIR/$tfile-ref of=$DIR/$tfile bs=1M
+
+ $LFS setstripe -E 1M -S 1M -o 2 -E eof -o 3 $DIR/$tfile-2
+ dd if=$DIR/$tfile-ref of=$DIR/$tfile-2 bs=1M
+
+ $CHECKSTAT -t file -s $((3 * 1024 * 1024)) $DIR/$tfile ||
+ error "mirrored file size is not 3M"
+
+ # merge a mirrored file
+ $LFS setstripe --component-add --mirror=$DIR/$tfile-2 $DIR/$tfile
+
+ cancel_lru_locks osc
+
+ # stop two OSTs, so the 2nd component of the 1st mirror and
+ # the 1st component of the 2nd mirror will be inaccessible, ...
+ stop_osts 2 3
+
+ echo "comparing files ... "
+
+ # however, read can still return the correct data. It should return
+ # the 1st stripe from mirror 1 and 2st stripe from mirror 2.
+ cmp -n 2097152 <(rwv -f $DIR/$tfile -r -o -n 1 2097152) \
+ $DIR/$tfile-ref || error "file reading error"
+
+ start_osts 2 3
+}
+run_test 34b "read mirrored file with multiple components"
+