LU-13805 tests: Add racing tests of BIO, DIO

We're a bit short on racing tests for buffered IO and
direct IO.  This patch adds a number of tests.  These
were originally part of the unaligned DIO patch, but
some of them have shown issues without unaligned IO.

So this patch puts in these tests with only aligned IO so
we can see which failures are specific to the unaligned IO
changes and which are not.

This patch should be landable like this.

Signed-off-by: Patrick Farrell <pfarrell@whamcloud.com>
Change-Id: I861bcaec785936cb9c3752f8148dcab4054f6078
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/50529
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Qian Yingjin <qian@ddn.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index b659904..71aeecc 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -13450,6 +13450,168 @@ test_119c() # bug 13099
 }
 run_test 119c "Testing for direct read hitting hole"
 
+# Note: test 119d was removed, skipping 119d for new tests to avoid polluting
+# Maloo test history
+
+test_119e()
+{
+       (( $OSTCOUNT >= 2 )) || skip "needs >= 2 OSTs"
+
+       local stripe_size=$((1024 * 1024)) #1 MiB
+       # Max i/o below is ~ 4 * stripe_size, so this gives ~5 i/os
+       local file_size=$((25 * stripe_size))
+       local bsizes
+
+       $LFS setstripe -c 2 -S $stripe_size $DIR/$tfile.1
+       stack_trap "rm -f $DIR/$tfile*"
+
+       # Just a bit bigger than the largest size in the test set below
+       dd if=/dev/urandom bs=$file_size count=1 of=$DIR/$tfile.1 ||
+               error "buffered i/o to create file failed"
+
+       if zfs_or_rotational; then
+               # DIO on ZFS can take up to 2 seconds per IO
+               # rotational is better, but still slow.
+               # Limit testing on those media to larger sizes
+               bsizes="$((stripe_size - PAGE_SIZE)) $stripe_size"
+       else
+               bsizes="$PAGE_SIZE $((PAGE_SIZE * 4)) $stripe_size \
+                       $((stripe_size * 4))"
+       fi
+
+       for bs in $bsizes; do
+               $LFS setstripe -c 2 -S $stripe_size $DIR/$tfile.2
+               echo "Read/write with DIO at size $bs"
+               # Read and write with DIO from source to dest
+               dd if=$DIR/$tfile.1 bs=$bs of=$DIR/$tfile.2 \
+                       iflag=direct oflag=direct ||
+                       error "dio failed"
+
+               ls -la $DIR/$tfile.1 $DIR/$tfile.2
+               $CHECKSTAT -t file -s $file_size $DIR/$tfile.2 ||
+                       error "size incorrect, file copy read/write bsize: $bs"
+               cmp --verbose $DIR/$tfile.1 $DIR/$tfile.2 ||
+                       error "files differ, bsize $bs"
+               rm -f $DIR/$tfile.2
+       done
+}
+run_test 119e "Basic tests of dio read and write at various sizes"
+
+test_119f()
+{
+       (( $OSTCOUNT >= 2 )) || skip "needs >= 2 OSTs"
+
+       local stripe_size=$((1024 * 1024)) #1 MiB
+       # Max i/o below is ~ 4 * stripe_size, so this gives ~5 i/os
+       local file_size=$((25 * stripe_size))
+       local bsizes
+
+       $LFS setstripe -c 2 -S $stripe_size $DIR/$tfile.1
+       stack_trap "rm -f $DIR/$tfile*"
+
+       # Just a bit bigger than the largest size in the test set below
+       dd if=/dev/urandom bs=$file_size count=1 of=$DIR/$tfile.1 ||
+               error "buffered i/o to create file failed"
+
+       if zfs_or_rotational; then
+               # DIO on ZFS can take up to 2 seconds per IO
+               # rotational is better, but still slow.
+               # Limit testing on those media to larger sizes
+               bsizes="$((stripe_size - PAGE_SIZE)) $stripe_size"
+       else
+               bsizes="$PAGE_SIZE $((PAGE_SIZE * 4)) $stripe_size \
+                       $((stripe_size * 4))"
+       fi
+
+       for bs in $bsizes; do
+               $LFS setstripe -c 2 -S $stripe_size $DIR/$tfile.2
+               # Read and write with DIO from source to dest in two
+               # threads - should give correct copy of file
+
+               echo "bs: $bs"
+               dd if=$DIR/$tfile.1 bs=$bs of=$DIR/$tfile.2 iflag=direct \
+                       oflag=direct conv=notrunc &
+               pid_dio1=$!
+               # Note block size is different here for a more interesting race
+               dd if=$DIR/$tfile.1 bs=$((bs * 2)) of=$DIR/$tfile.2 \
+                       iflag=direct oflag=direct conv=notrunc &
+               pid_dio2=$!
+               wait $pid_dio1
+               rc1=$?
+               wait $pid_dio2
+               rc2=$?
+               if (( rc1 != 0 )); then
+                       error "dio copy 1 w/bsize $bs failed: $rc1"
+               fi
+               if (( rc2 != 0 )); then
+                       error "dio copy 2 w/bsize $bs failed: $rc2"
+               fi
+
+
+               $CHECKSTAT -t file -s $file_size $DIR/$tfile.2 ||
+                       error "size incorrect, file copy read/write bsize: $bs"
+               cmp --verbose $DIR/$tfile.1 $DIR/$tfile.2 ||
+                       error "files differ, bsize $bs"
+               rm -f $DIR/$tfile.2
+       done
+}
+run_test 119f "dio vs dio race"
+
+test_119g()
+{
+       (( $OSTCOUNT >= 2 )) || skip "needs >= 2 OSTs"
+
+       local stripe_size=$((1024 * 1024)) #1 MiB
+       # Max i/o below is ~ 4 * stripe_size, so this gives ~5 i/os
+       local file_size=$((25 * stripe_size))
+       local bsizes
+
+       $LFS setstripe -c 2 -S $stripe_size $DIR/$tfile.1
+       stack_trap "rm -f $DIR/$tfile*"
+
+       # Just a bit bigger than the largest size in the test set below
+       dd if=/dev/urandom bs=$file_size count=1 of=$DIR/$tfile.1 ||
+               error "buffered i/o to create file failed"
+
+       if zfs_or_rotational; then
+               # DIO on ZFS can take up to 2 seconds per IO
+               # rotational is better, but still slow.
+               # Limit testing on those media to larger sizes
+               bsizes="$((stripe_size - PAGE_SIZE)) $stripe_size"
+       else
+               bsizes="$PAGE_SIZE $((PAGE_SIZE * 4)) $stripe_size \
+                       $((stripe_size * 4))"
+       fi
+
+       for bs in $bsizes; do
+               $LFS setstripe -c 2 -S $stripe_size $DIR/$tfile.2
+               echo "bs: $bs"
+               dd if=$DIR/$tfile.1 bs=$bs of=$DIR/$tfile.2 iflag=direct \
+                       oflag=direct conv=notrunc &
+               pid_dio1=$!
+               # Buffered I/O with similar but not the same block size
+               dd if=$DIR/$tfile.1 bs=$((bs * 2)) of=$DIR/$tfile.2 &
+               pid_bio2=$!
+               wait $pid_dio1
+               rc1=$?
+               wait $pid_bio2
+               rc2=$?
+               if (( rc1 != 0 )); then
+                       error "dio copy 1 w/bsize $bs failed: $rc1"
+               fi
+               if (( rc2 != 0 )); then
+                       error "buffered copy 2 w/bsize $bs failed: $rc2"
+               fi
+
+               $CHECKSTAT -t file -s $file_size $DIR/$tfile.2 ||
+                       error "size incorrect"
+               cmp --verbose $DIR/$tfile.1 $DIR/$tfile.2 ||
+                       error "files differ, bsize $bs"
+               rm -f $DIR/$tfile.2
+       done
+}
+run_test 119g "dio vs buffered I/O race"
+
 test_120a() {
        [ $PARALLEL == "yes" ] && skip "skip parallel run"
        remote_mds_nodsh && skip "remote MDS with nodsh"
@@ -26531,6 +26693,53 @@ test_398p()
 }
 run_test 398p "race aio with buffered i/o"
 
+test_398q()
+{
+       (( $OSTCOUNT >= 2 )) || skip "needs >= 2 OSTs"
+
+       local stripe_size=$((1024 * 1024)) #1 MiB
+       # Max i/o below is ~ 4 * stripe_size, so this gives ~5 i/os
+       local file_size=$((25 * stripe_size))
+
+       $LFS setstripe -c 2 -S $stripe_size $DIR/$tfile.1
+       $LFS setstripe -c 2 -S $stripe_size $DIR/$tfile.2
+
+       # Just a bit bigger than the largest size in the test set below
+       dd if=/dev/urandom bs=$file_size count=1 of=$DIR/$tfile.1 ||
+               error "buffered i/o to create file failed"
+
+       for bs in $PAGE_SIZE $((PAGE_SIZE * 4)) $stripe_size \
+               $((stripe_size * 4)); do
+
+               echo "bs: $bs, file_size $file_size"
+               dd if=$DIR/$tfile.1 bs=$((bs *2 )) of=$DIR/tfile.2 \
+                       conv=notrunc oflag=direct iflag=direct &
+               pid_dio1=$!
+               # Buffered I/O with similar but not the same block size
+               dd if=$DIR/$tfile.1 bs=$((bs * 2)) of=$DIR/$tfile.2 \
+                       conv=notrunc &
+               pid_bio2=$!
+               wait $pid_dio1
+               rc1=$?
+               wait $pid_bio2
+               rc2=$?
+               if (( rc1 != 0 )); then
+                       error "dio copy 1 w/bsize $bs failed: $rc1"
+               fi
+               if (( rc2 != 0 )); then
+                       error "buffered copy 2 w/bsize $bs failed: $rc2"
+               fi
+
+               $CHECKSTAT -t file -s $file_size $DIR/$tfile.2 ||
+                       error "size incorrect"
+               diff $DIR/$tfile.1 $DIR/$tfile.2 ||
+                       error "files differ, bsize $bs"
+       done
+
+       rm -f $DIR/$tfile*
+}
+run_test 398q "race dio with buffered i/o"
+
 test_fake_rw() {
        local read_write=$1
        if [ "$read_write" = "write" ]; then

diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh
index a283b15..6dbe10b 100755 (executable)
--- a/lustre/tests/sanityn.sh
+++ b/lustre/tests/sanityn.sh
@@ -708,6 +708,120 @@ test_16i() {
 }
 run_test 16i "read after truncate file"
 
+test_16j()
+{
+       (( $OSTCOUNT >= 2 )) || skip "needs >= 2 OSTs"
+
+       local stripe_size=$((1024 * 1024)) #1 MiB
+       # Max i/o below is ~ 4 * stripe_size, so this gives ~5 i/os
+       local file_size=$((25 * stripe_size))
+       local bsizes
+
+       $LFS setstripe -c 2 -S $stripe_size $DIR/$tfile.1
+       stack_trap "rm -f $DIR/$tfile.1"
+
+       # Just a bit bigger than the largest size in the test set below
+       dd if=/dev/urandom bs=$file_size count=1 of=$DIR/$tfile.1 ||
+               error "buffered i/o to create file failed"
+
+       if zfs_or_rotational; then
+               # DIO on ZFS can take up to 2 seconds per IO
+               # rotational is better, but still slow.
+               # Limit testing on those media to larger sizes
+               bsizes="$((stripe_size - PAGE_SIZE)) $stripe_size"
+       else
+               bsizes="$PAGE_SIZE $((PAGE_SIZE * 4)) $stripe_size \
+                       $((stripe_size * 4))"
+       fi
+
+       # 1 process (BIO or DIO) on each client
+       for bs in $bsizes; do
+               $LFS setstripe -c 2 -S $stripe_size $DIR/$tfile.2
+               echo "bs: $bs, file_size $file_size"
+               dd if=$DIR/$tfile.1 bs=$((bs *2 )) of=$DIR/tfile.2 \
+                       conv=notrunc oflag=direct iflag=direct &
+               pid_dio1=$!
+               # Buffered I/O with similar but not the same block size
+               dd if=$DIR2/$tfile.1 bs=$((bs * 2)) of=$DIR2/$tfile.2 \
+                       conv=notrunc &
+               pid_bio2=$!
+               wait $pid_dio1
+               rc1=$?
+               wait $pid_bio2
+               rc2=$?
+               if (( rc1 != 0 )); then
+                       error "dio copy 1 w/bsize $bs failed: $rc1"
+               fi
+               if (( rc2 != 0 )); then
+                       error "buffered copy 2 w/bsize $bs failed: $rc2"
+               fi
+
+               $CHECKSTAT -t file -s $file_size $DIR/$tfile.2 ||
+                       error "size incorrect"
+               $CHECKSTAT -t file -s $file_size $DIR2/$tfile.2 ||
+                       error "size incorrect - mount 2"
+               cmp --verbose $DIR/$tfile.1 $DIR/$tfile.2 ||
+                       error "files differ, bsize $bs"
+               cmp --verbose $DIR2/$tfile.1 $DIR2/$tfile.2 ||
+                       error "files differ, bsize $bs - mount 2"
+               rm -f $DIR/$tfile.2
+       done
+
+       # 2 processes - both DIO and BIO - on each client
+       for bs in $bsizes; do
+               $LFS setstripe -c 2 -S $stripe_size $DIR/$tfile.2
+               echo "bs: $bs, file_size $file_size"
+               # Client 1
+               dd if=$DIR/$tfile.1 bs=$((bs *2 )) of=$DIR/tfile.2 \
+                       conv=notrunc oflag=direct iflag=direct &
+               pid_dio1=$!
+               # Buffered I/O with similar but not the same block size
+               dd if=$DIR/$tfile.1 bs=$((bs * 2)) of=$DIR/$tfile.2 \
+                       conv=notrunc &
+               pid_bio2=$!
+               # Client 2
+               dd if=$DIR2/$tfile.1 bs=$((bs *2 )) of=$DIR2/tfile.2 \
+                       conv=notrunc oflag=direct iflag=direct &
+               pid_dio3=$!
+               # Buffered I/O with similar but not the same block size
+               dd if=$DIR2/$tfile.1 bs=$((bs * 2)) of=$DIR2/$tfile.2 \
+                       conv=notrunc &
+               pid_bio4=$!
+               wait $pid_dio1
+               rc1=$?
+               wait $pid_bio2
+               rc2=$?
+               wait $pid_dio3
+               rc3=$?
+               wait $pid_bio4
+               rc4=$?
+
+               if (( rc1 != 0 )); then
+                       error "dio copy 1 w/bsize $bs failed: $rc1"
+               fi
+               if (( rc2 != 0 )); then
+                       error "buffered copy 2 w/bsize $bs failed: $rc2"
+               fi
+               if (( rc3 != 0 )); then
+                       error "dio copy 3 w/bsize $bs failed: $rc1"
+               fi
+               if (( rc4 != 0 )); then
+                       error "buffered copy 4 w/bsize $bs failed: $rc2"
+               fi
+
+               $CHECKSTAT -t file -s $file_size $DIR/$tfile.2 ||
+                       error "size incorrect"
+               $CHECKSTAT -t file -s $file_size $DIR2/$tfile.2 ||
+                       error "size incorrect - mount 2"
+               cmp --verbose $DIR/$tfile.1 $DIR/$tfile.2 ||
+                       error "files differ, bsize $bs"
+               cmp --verbose $DIR2/$tfile.1 $DIR2/$tfile.2 ||
+                       error "files differ, bsize $bs - mount 2"
+               rm -f $DIR/$tfile.2
+       done
+}
+run_test 16j "race dio with buffered i/o"
+
 test_17() { # bug 3513, 3667
        remote_ost_nodsh && skip "remote OST with nodsh" && return
 

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 313bfe6..0ea069d 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -11315,3 +11315,15 @@ verify_yaml() {
 verify_compare_yaml() {
        python3 -c "import sys, yaml; f=open(\"$1\", \"r\"); obj1 = yaml.safe_load(f); f=open(\"$2\", \"r\"); obj2 = yaml.safe_load(f); sys.exit(obj1 != obj2)"
 }
+
+zfs_or_rotational() {
+       local ost_idx=0
+       local ost_name=$(ostname_from_index $ost_idx)
+       local nonrotat=$(do_facet ost1 $LCTL get_param -n osd-*.$ost_name.nonrotational)
+
+       if [ "$ost1_FSTYPE" == "zfs" ] || [ "$nonrotat" -eq 0 ]; then
+               return 0
+       else
+               return 1
+       fi
+}