X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Fsanityn.sh;h=1dd08386ad93385c0488a4523fff6e89786d01bc;hb=622e4c6e04baa4f9d19cc33f4636679014fc1d2f;hp=c251532868b32bfdb7a3e639c87480b1d17b1a98;hpb=7c99f67d9d39e8a037e830cf08a9df305e6d8da2;p=fs%2Flustre-release.git diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh index c251532..1dd0838 100755 --- a/lustre/tests/sanityn.sh +++ b/lustre/tests/sanityn.sh @@ -19,10 +19,16 @@ init_logging ALWAYS_EXCEPT="$SANITYN_EXCEPT " # bug number for skipped test: LU-7105 -ALWAYS_EXCEPT+=" 28" +ALWAYS_EXCEPT+=" 28 " # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! -if [ $(facet_fstype $SINGLEMDS) = "zfs" ]; then +# skip tests for PPC until they are fixed +if [[ $(uname -m) = ppc64 ]]; then + # bug number: LU-11597 LU-11787 + ALWAYS_EXCEPT+=" 16a 71a" +fi + +if [ $mds1_FSTYPE = "zfs" ]; then # LU-2829 / LU-2887 - make allowances for ZFS slowness TEST33_NFILES=${TEST33_NFILES:-1000} fi @@ -366,7 +372,7 @@ COUNT=${COUNT:-2500} # The FSXNUM reduction for ZFS is needed until ORI-487 is fixed. # We don't want to skip it entirely, but ZFS is VERY slow and cannot # pass a 2500 operation dual-mount run within the time limit. -if [ "$(facet_fstype ost1)" = "zfs" ]; then +if [ "$ost1_FSTYPE" = "zfs" ]; then FSXNUM=$((COUNT / 5)) FSXP=1 elif [ "$SLOW" = "yes" ]; then @@ -383,6 +389,8 @@ test_16a() { local stripe_size=$(do_facet $SINGLEMDS \ "$LCTL get_param -n lod.$(facet_svc $SINGLEMDS)*.stripesize") + check_set_fallocate + # to allocate grant because it may run out due to test_15. $LFS setstripe -c -1 $file1 dd if=/dev/zero of=$file1 bs=$stripe_size count=$OSTCOUNT oflag=sync @@ -390,12 +398,12 @@ test_16a() { rm -f $file1 $LFS setstripe -c -1 $file1 # b=10919 - fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 $file1 $file2 \ - || error "fsx failed" + $FSX -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 $file1 $file2 || + error "fsx failed" rm -f $file1 # O_DIRECT reads and writes must be aligned to the device block size. - fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 -Z -r 4096 \ + $FSX -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 -Z -r 4096 \ -w 4096 $file1 $file2 || error "fsx with O_DIRECT failed." } run_test 16a "$FSXNUM iterations of dual-mount fsx" @@ -406,6 +414,8 @@ test_16b() { local file2=$DIR2/$tfile local stripe_size=($($LFS getstripe -S $DIR)) + check_set_fallocate + # to allocate grant because it may run out due to test_15. lfs setstripe -c -1 $file1 dd if=/dev/zero of=$file1 bs=$stripe_size count=$OSTCOUNT oflag=sync || @@ -417,8 +427,8 @@ test_16b() { lfs setstripe -c -1 $file1 # b=10919 # -o is set to 8192 because writes < 1 page and between 1 and 2 pages # create a mix of tiny writes & normal writes - fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -o 8192 -S 0 $file1 \ - $file2 || error "fsx with tiny write failed." + $FSX -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -o 8192 -S 0 \ + $file1 $file2 || error "fsx with tiny write failed." } run_test 16b "$FSXNUM iterations of dual-mount fsx at small size" @@ -428,7 +438,9 @@ test_16c() { local stripe_size=$(do_facet $SINGLEMDS \ "$LCTL get_param -n lod.$(facet_svc $SINGLEMDS)*.stripesize") - [ $(facet_fstype ost1) != ldiskfs ] && skip "dio on ldiskfs only" + [ "$ost1_FSTYPE" != ldiskfs ] && skip "dio on ldiskfs only" + + check_set_fallocate # to allocate grant because it may run out due to test_15. $LFS setstripe -c -1 $file1 @@ -446,8 +458,8 @@ test_16c() { set_osd_param $list '' writethrough_cache_enable 0 $LFS setstripe -c -1 $file1 # b=10919 - fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 $file1 $file2 \ - || error "fsx failed" + $FSX -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 $file1 $file2 || + error "fsx failed" rm -f $file1 set_osd_param $list '' read_cache_enable 1 @@ -457,6 +469,69 @@ test_16c() { } run_test 16c "verify data consistency on ldiskfs with cache disabled (b=17397)" +test_16d() { + local file1=$DIR1/$tfile + local file2=$DIR2/$tfile + local file3=$DIR1/file + local tmpfile=$(mktemp) + local stripe_size=$(do_facet $SINGLEMDS \ + "$LCTL get_param -n lod.$(facet_svc $SINGLEMDS)*.stripesize") + + # to allocate grant because it may run out due to test_15. + $LFS setstripe -c -1 $file1 + stack_trap "rm -f $file1 $file2 $file3 $tmpfile" + dd if=/dev/zero of=$file1 bs=$stripe_size count=$OSTCOUNT oflag=sync + dd if=/dev/zero of=$file2 bs=$stripe_size count=$OSTCOUNT oflag=sync + rm -f $file1 + + $LFS setstripe -c -1 $file1 # b=10919 + $LCTL set_param ldlm.namespaces.*.lru_size=clear + + # direct write on one client and direct read from another + dd if=/dev/urandom of=$file1 bs=1M count=100 oflag=direct + dd if=$file2 of=$tmpfile iflag=direct bs=1M + diff $file1 $tmpfile || error "file different(1)" + rm -f $file1 + + # buffer write on one client, but direct read from another + dd if=$tmpfile of=$file1 bs=1M count=100 + dd if=$file2 of=$file3 bs=1M iflag=direct count=100 + diff $file3 $tmpfile || error "file different(2)" + + rm -f $file3 $file2 $file1 + # direct write on one client + dd if=$tmpfile of=$file1 bs=1M count=100 oflag=direct + # buffer read from another client + dd if=$file2 of=$file3 bs=1M count=100 + diff $file3 $tmpfile || error "file different(3)" +} +run_test 16d "Verify DIO and buffer IO with two clients" + +test_16e() { # LU-13227 + # issue: LU-14314 + + (( "$MDS1_VERSION" >= $(version_code 2.13.53) )) || + skip "Need MDS version at least 2.13.53" + + local file1=$DIR1/$tfile + local file2=$DIR2/$tfile + + # client1 write 10M data + dd if=/dev/zero of=$file1 bs=1M count=10 + # drop locks + cancel_lru_locks osc > /dev/null + # use lockahead to generate one PW lock to keep LVB loaded. + $LFS ladvise -a lockahead --start 0 --length 1M \ + --mode WRITE $file1 + # direct write to extend file size on client2 + dd if=/dev/zero of=$file2 bs=1M seek=20 count=1 \ + oflag=direct conv=notrunc + local filesize=$(stat -c %s $file2) + [ "$filesize" -eq 22020096 ] || + error "expected filesize 22020096 got $filesize" + rm -f $file1 +} +run_test 16e "Verify size consistency for O_DIRECT write" test_17() { # bug 3513, 3667 remote_ost_nodsh && skip "remote OST with nodsh" && return @@ -484,7 +559,9 @@ test_18() { excepts="$excepts -e $(($(printf %d \'$idx)-96))" done - $LUSTRE/tests/mmap_sanity -d $MOUNT1 -m $MOUNT2 $excepts + excepts="$excepts -e 7 -e 8 -e 9" + $LUSTRE/tests/mmap_sanity -d $MOUNT1 -m $MOUNT2 $excepts || + error "mmap_sanity test failed" sync; sleep 1; sync } run_test 18 "mmap sanity check =================================" @@ -520,16 +597,16 @@ test_19() { # bug3811 run_test 19 "test concurrent uncached read races ===============" test_20() { - test_mkdir $DIR1/d20 - cancel_lru_locks $OSC - CNT=$((`lctl get_param -n llite.*.dump_page_cache | wc -l`)) - $MULTIOP $DIR1/f20 Ow8190c - $MULTIOP $DIR2/f20 Oz8194w8190c - $MULTIOP $DIR1/f20 Oz0r8190c - cancel_lru_locks $OSC - CNTD=$((`lctl get_param -n llite.*.dump_page_cache | wc -l` - $CNT)) - [ $CNTD -gt 0 ] && \ - error $CNTD" page left in cache after lock cancel" || true + test_mkdir $DIR1/$tdir + cancel_lru_locks + CNT=$($LCTL get_param -n llite.*.dump_page_cache | wc -l) + $MULTIOP $DIR1/$tdir/$tfile Ow8190c + $MULTIOP $DIR2/$tdir/$tfile Oz8194w8190c + $MULTIOP $DIR1/$tdir/$tfile Oz0r8190c + cancel_lru_locks + CNT2=$($LCTL get_param -n llite.*.dump_page_cache | wc -l) + [[ $CNT2 == $CNT ]] || + error $((CNT2 - CNT))" page left in cache after lock cancel" } run_test 20 "test extra readahead page left in cache ====" @@ -685,7 +762,7 @@ test_27() { lctl clear dd if=/dev/zero of=$DIR2/$tfile bs=$((4096+4))k conv=notrunc count=4 seek=3 & DD2_PID=$! - usleep 50 + sleep 0.5 log "dd 1 started" dd if=/dev/zero of=$DIR1/$tfile bs=$((16384-1024))k conv=notrunc count=1 seek=4 & @@ -809,13 +886,15 @@ test_32a() { # bug 11270 log "checking cached lockless truncate" $TRUNCATE $DIR1/$tfile 8000000 - $CHECKSTAT -s 8000000 $DIR2/$tfile || error "wrong file size" + $CHECKSTAT -s 8000000 $DIR2/$tfile || + error "cached truncate - wrong file size" [ $(calc_stats $OSC.*.${OSC}_stats lockless_truncate) -ne 0 ] || error "cached truncate isn't lockless" log "checking not cached lockless truncate" $TRUNCATE $DIR2/$tfile 5000000 - $CHECKSTAT -s 5000000 $DIR1/$tfile || error "wrong file size" + $CHECKSTAT -s 5000000 $DIR1/$tfile || + error "not cached truncate - wrong file size" [ $(calc_stats $OSC.*.${OSC}_stats lockless_truncate) -ne 0 ] || error "not cached truncate isn't lockless" @@ -823,7 +902,8 @@ test_32a() { # bug 11270 enable_lockless_truncate 0 clear_stats $OSC.*.${OSC}_stats $TRUNCATE $DIR2/$tfile 3000000 - $CHECKSTAT -s 3000000 $DIR1/$tfile || error "wrong file size" + $CHECKSTAT -s 3000000 $DIR1/$tfile || + error "lockless truncate disabled - wrong file size" [ $(calc_stats $OSC.*.${OSC}_stats lockless_truncate) -eq 0 ] || error "lockless truncate disabling failed" rm -f $DIR1/$tfile @@ -904,52 +984,52 @@ print_jbd_stat () { # commit on sharing tests test_33a() { - remote_mds_nodsh && skip "remote MDS with nodsh" && return + remote_mds_nodsh && skip "remote MDS with nodsh" && return - [ -z "$CLIENTS" ] && skip "Need two or more clients, have $CLIENTS" && return 0 - [ $CLIENTCOUNT -lt 2 ] && - skip "Need two or more clients, have $CLIENTCOUNT" && return 0 + [ -z "$CLIENTS" ] && skip "Need two or more clients, have $CLIENTS" + [ $CLIENTCOUNT -lt 2 ] && + skip "Need two or more clients, have $CLIENTCOUNT" - local nfiles=${TEST33_NFILES:-10000} - local param_file=$TMP/$tfile-params - local fstype=$(facet_fstype $SINGLEMDS) + local nfiles=${TEST33_NFILES:-10000} + local param_file=$TMP/$tfile-params + local COS + local jbdold="N/A" + local jbdnew="N/A" + local jbd save_lustre_params $(get_facets MDS) \ "mdt.*.commit_on_sharing" > $param_file - local COS - local jbdold="N/A" - local jbdnew="N/A" - local jbd - - for COS in 0 1; do - do_facet $SINGLEMDS lctl set_param mdt.*.commit_on_sharing=$COS - avgjbd=0 - avgtime=0 - for i in 1 2 3; do - do_nodes $CLIENT1,$CLIENT2 "mkdir -p $DIR1/$tdir-\\\$(hostname)-$i" - - [ $fstype = ldiskfs ] && jbdold=$(print_jbd_stat) - echo "=== START createmany old: $jbdold transaction" - local elapsed=$(do_and_time "do_nodes $CLIENT1,$CLIENT2 createmany -o $DIR1/$tdir-\\\$(hostname)-$i/f- -r$DIR2/$tdir-\\\$(hostname)-$i/f- $nfiles > /dev/null 2>&1") - [ $fstype = ldiskfs ] && jbdnew=$(print_jbd_stat) - [ $fstype = ldiskfs ] && jbd=$(( jbdnew - jbdold )) - echo "=== END createmany new: $jbdnew transaction : $jbd transactions nfiles $nfiles time $elapsed COS=$COS" - [ $fstype = ldiskfs ] && avgjbd=$(( avgjbd + jbd )) - avgtime=$(( avgtime + elapsed )) - done - eval cos${COS}_jbd=$((avgjbd / 3)) - eval cos${COS}_time=$((avgtime / 3)) - done + for COS in 0 1; do + do_facet $SINGLEMDS lctl set_param mdt.*.commit_on_sharing=$COS + avgjbd=0 + avgtime=0 + for i in 1 2 3; do + do_nodes $CLIENT1,$CLIENT2 "mkdir -p $DIR1/$tdir-\\\$(hostname)-$i" + + [ "$mds1_FSTYPE" = ldiskfs ] && jbdold=$(print_jbd_stat) + echo "=== START createmany old: $jbdold transaction" + local elapsed=$(do_and_time "do_nodes $CLIENT1,$CLIENT2 createmany -o $DIR1/$tdir-\\\$(hostname)-$i/f- -r$DIR2/$tdir-\\\$(hostname)-$i/f- $nfiles > /dev/null 2>&1") + [ "$mds1_FSTYPE" = ldiskfs ] && jbdnew=$(print_jbd_stat) + [ "$mds1_FSTYPE" = ldiskfs ] && jbd=$(( jbdnew - jbdold )) + echo "=== END createmany new: $jbdnew transaction : $jbd transactions nfiles $nfiles time $elapsed COS=$COS" + [ "$mds1_FSTYPE" = ldiskfs ] && avgjbd=$(( avgjbd + jbd )) + avgtime=$(( avgtime + elapsed )) + done + eval cos${COS}_jbd=$((avgjbd / 3)) + eval cos${COS}_time=$((avgtime / 3)) + done - echo "COS=0 transactions (avg): $cos0_jbd time (avg): $cos0_time" - echo "COS=1 transactions (avg): $cos1_jbd time (avg): $cos1_time" - [ "$cos0_jbd" != 0 ] && echo "COS=1 vs COS=0 jbd: $((((cos1_jbd/cos0_jbd - 1)) * 100 )) %" - [ "$cos0_time" != 0 ] && echo "COS=1 vs COS=0 time: $((((cos1_time/cos0_time - 1)) * 100 )) %" + echo "COS=0 transactions (avg): $cos0_jbd time (avg): $cos0_time" + echo "COS=1 transactions (avg): $cos1_jbd time (avg): $cos1_time" + [ "$cos0_jbd" != 0 ] && + echo "COS=1 vs COS=0 jbd: $((((cos1_jbd/cos0_jbd - 1)) * 100 )) %" + [ "$cos0_time" != 0 ] && + echo "COS=1 vs COS=0 time: $((((cos1_time/cos0_time - 1)) * 100 )) %" - restore_lustre_params < $param_file - rm -f $param_file - return 0 + restore_lustre_params < $param_file + rm -f $param_file + return 0 } run_test 33a "commit on sharing, cross crete/delete, 2 clients, benchmark" @@ -1014,9 +1094,13 @@ test_33b() { run_test 33b "COS: cross create/delete, 2 clients, benchmark under remote dir" test_33c() { - [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.63) ] && - skip "DNE CoS not supported" && return + [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" + [ "$MDS1_VERSION" -lt $(version_code 2.7.63) ] && + skip "DNE CoS not supported" + + # LU-13522 + stop mds1 + start mds1 $(mdsdevname 1) $MDS_MOUNT_OPTS || error "start mds1 failed" local sync_count @@ -1072,9 +1156,9 @@ op_trigger_cos() { } test_33d() { - [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.63) ] && - skip "DNE CoS not supported" && return + [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" + [ "$MDS1_VERSION" -lt $(version_code 2.7.63) ] && + skip "DNE CoS not supported" # remote directory create op_trigger_cos "mkdir $DIR/$tdir" "$LFS mkdir -i 1 $DIR/$tdir/subdir" @@ -1109,13 +1193,12 @@ test_33d() { run_test 33d "DNE distributed operation should trigger COS" test_33e() { - [ -n "$CLIENTS" ] || { skip "Need two or more clients" && return 0; } + [ -n "$CLIENTS" ] || skip "Need two or more clients" [ $CLIENTCOUNT -ge 2 ] || - { skip "Need two or more clients, have $CLIENTCOUNT" && - return 0; } - [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.7.63) ] && - skip "DNE CoS not supported" && return + skip "Need two or more clients, have $CLIENTCOUNT" + [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" + [ "$MDS1_VERSION" -lt $(version_code 2.7.63) ] && + skip "DNE CoS not supported" local client2=${CLIENT2:-$(hostname)} @@ -1491,7 +1574,8 @@ test_40a() { remote_mds_nodsh && skip "remote MDS with nodsh" && return pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" touch $DIR2 mkdir $DIR1/$tfile & PID1=$!; pdo_sched @@ -1511,7 +1595,8 @@ test_40a() { # all operations above shouldn't wait the first one check_pdo_conflict $PID1 || error "parallel operation is blocked" - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" wait $PID1 rm -rf $DIR/$tfile* return 0 @@ -1522,7 +1607,8 @@ test_40b() { remote_mds_nodsh && skip "remote MDS with nodsh" && return pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" touch $DIR1/$tfile & PID1=$!; pdo_sched # open|create @@ -1542,7 +1628,8 @@ test_40b() { # all operations above shouldn't wait the first one check_pdo_conflict $PID1 || error "parallel operation is blocked" - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" wait $PID1 rm -rf $DIR/$tfile* return 0 @@ -1554,7 +1641,8 @@ test_40c() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" link $DIR1/$tfile $DIR1/$tfile-0 & PID1=$!; pdo_sched # open|create @@ -1574,7 +1662,8 @@ test_40c() { # all operations above shouldn't wait the first one check_pdo_conflict $PID1 || error "parallel operation is blocked" - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" wait $PID1 rm -rf $DIR/$tfile* return 0 @@ -1586,7 +1675,8 @@ test_40d() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" rm $DIR1/$tfile & PID1=$!; pdo_sched # open|create @@ -1606,7 +1696,8 @@ test_40d() { # all operations above shouldn't wait the first one check_pdo_conflict $PID1 || error "parallel operation is blocked" - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" wait $PID1 return 0 } @@ -1617,7 +1708,8 @@ test_40e() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mv $DIR1/$tfile $DIR1/$tfile-0 & PID1=$!; pdo_sched # open|create @@ -1635,7 +1727,8 @@ test_40e() { # all operations above shouldn't wait the first one check_pdo_conflict $PID1 || error "parallel operation is blocked" - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" wait $PID1 rm -rf $DIR/$tfile* return 0 @@ -1646,12 +1739,14 @@ run_test 40e "pdirops: rename and others ==============" test_41a() { pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c & PID1=$! ; pdo_sched mkdir $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; echo "mkdir isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail" rm -rf $DIR/$tfile* @@ -1662,12 +1757,14 @@ run_test 41a "pdirops: create vs mkdir ==============" test_41b() { pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c & PID1=$! ; pdo_sched $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "create must fail" rm -rf $DIR/$tfile* @@ -1679,12 +1776,14 @@ test_41c() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c & PID1=$! ; pdo_sched link $DIR2/$tfile-2 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "link must fail" rm -rf $DIR/$tfile* @@ -1695,12 +1794,14 @@ run_test 41c "pdirops: create vs link ==============" test_41d() { pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c & PID1=$! ; pdo_sched rm $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "unlink must succeed" rm -rf $DIR/$tfile* @@ -1712,12 +1813,14 @@ test_41e() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c & PID1=$! ; pdo_sched mv $DIR2/$tfile-2 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed" rm -rf $DIR/$tfile* @@ -1728,12 +1831,14 @@ run_test 41e "pdirops: create and rename (tgt) ==============" test_41f() { pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c & PID1=$! ; pdo_sched mv $DIR2/$tfile $DIR2/$tfile-2 & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed" rm -rf $DIR/$tfile* @@ -1744,12 +1849,14 @@ run_test 41f "pdirops: create and rename (src) ==============" test_41g() { pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c & PID1=$! ; pdo_sched stat $DIR2/$tfile > /dev/null & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed" rm -rf $DIR/$tfile* @@ -1760,12 +1867,14 @@ run_test 41g "pdirops: create vs getattr ==============" test_41h() { pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c & PID1=$! ; pdo_sched ls -lia $DIR2/ > /dev/null & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; } wait $PID2 rm -rf $DIR/$tfile* @@ -1773,16 +1882,69 @@ test_41h() { } run_test 41h "pdirops: create vs readdir ==============" +sub_test_41i() { + local PID1 PID2 + local fail_loc="$1" + local ret=0 + + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=${fail_loc} || true" &>/dev/null + + $MULTIOP $DIR1/$tfile oO_CREAT:O_EXCL:c 2>/dev/null & + PID1=$! + sleep 0.2 + $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c 2>/dev/null & + PID2=$! + + if ! wait $PID1 && ! wait $PID2; then + echo "Both creates failed (1 should fail, 1 should succeed)" + ret=1 + elif wait $PID1 && wait $PID2; then + echo "Both creates succeeded (1 should fail, 1 should succeed)" + ret=2 + fi + + #Clean + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x0 || true" &>/dev/null + rm -f $DIR/$tfile + + return $ret +} + +test_41i() { + [[ $MDS1_VERSION -le $(version_code 2.13.56) ]] || + skip "Need MDS version newer than 2.13.56" + local msg fail_loc + +#define OBD_FAIL_ONCE|OBD_FAIL_MDS_REINT_OPEN 0x169 +#define OBD_FAIL_ONCE|OBD_FAIL_MDS_REINT_OPEN2 0x16a + for fail_loc in "0x80000169" "0x8000016a"; do + echo "Begin 100 tests with fail_loc=$fail_loc" + printf "Progress: " + for i in {1..100}; do + printf "*" + msg=$(sub_test_41i "$fail_loc") || + { echo; error "iter=$i : $msg"; } + done + echo + done +} +run_test 41i "reint_open: create vs create" + + # test 42: unlink and blocking operations test_42a() { pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mkdir $DIR1/$tfile & PID1=$! ; pdo_sched mkdir $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "mkdir isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail" rm -rf $DIR/$tfile* @@ -1793,12 +1955,14 @@ run_test 42a "pdirops: mkdir vs mkdir ==============" test_42b() { pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mkdir $DIR1/$tfile & PID1=$! ; pdo_sched $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "create must fail" rm -rf $DIR/$tfile* @@ -1810,12 +1974,14 @@ test_42c() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mkdir $DIR1/$tfile & PID1=$! ; pdo_sched link $DIR2/$tfile-2 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "link must fail" rm -rf $DIR/$tfile* @@ -1826,12 +1992,14 @@ run_test 42c "pdirops: mkdir vs link ==============" test_42d() { pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mkdir $DIR1/$tfile & PID1=$! ; pdo_sched rmdir $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "unlink must succeed" rm -rf $DIR/$tfile* @@ -1843,12 +2011,14 @@ test_42e() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mkdir $DIR1/$tfile & PID1=$! ; pdo_sched mv -T $DIR2/$tfile-2 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "rename must fail" rm -rf $DIR/$tfile* @@ -1859,12 +2029,14 @@ run_test 42e "pdirops: mkdir and rename (tgt) ==============" test_42f() { pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mkdir $DIR1/$tfile & PID1=$! ; pdo_sched mv $DIR2/$tfile $DIR2/$tfile-2 & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed" rm -rf $DIR/$tfile* @@ -1875,12 +2047,14 @@ run_test 42f "pdirops: mkdir and rename (src) ==============" test_42g() { pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mkdir $DIR1/$tfile & PID1=$! ; pdo_sched stat $DIR2/$tfile > /dev/null & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed" rm -rf $DIR/$tfile* @@ -1891,12 +2065,14 @@ run_test 42g "pdirops: mkdir vs getattr ==============" test_42h() { pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mkdir $DIR1/$tfile & PID1=$! ; pdo_sched ls -lia $DIR2/ > /dev/null & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; } wait $PID2 rm -rf $DIR/$tfile* @@ -1904,34 +2080,28 @@ test_42h() { } run_test 42h "pdirops: mkdir vs readdir ==============" -# test 43: unlink and blocking operations +# test 43: rmdir,mkdir won't return -EEXIST test_43a() { - pdo_lru_clear - touch $DIR1/$tfile -#define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 - rm $DIR1/$tfile & - PID1=$! ; pdo_sched - mkdir $DIR2/$tfile & - PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 - check_pdo_conflict $PID1 && { wait $PID1; error "mkdir isn't blocked"; } - wait $PID2 ; [ $? -eq 0 ] || error "mkdir must succeed" - rm -rf $DIR/$tfile* + for i in {1..1000}; do + mkdir $DIR1/$tdir || error "mkdir $tdir failed" + rmdir $DIR2/$tdir || error "rmdir $tdir failed" + done return 0 } -run_test 43a "pdirops: unlink vs mkdir ==============" +run_test 43a "rmdir,mkdir doesn't return -EEXIST ==============" test_43b() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" rm $DIR1/$tfile & PID1=$! ; pdo_sched $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "create must succeed" rm -rf $DIR/$tfile* @@ -1944,12 +2114,14 @@ test_43c() { touch $DIR1/$tfile touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" rm $DIR1/$tfile & PID1=$! ; pdo_sched link $DIR2/$tfile-2 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "link must succeed" rm -rf $DIR/$tfile* @@ -1961,12 +2133,14 @@ test_43d() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" rm $DIR1/$tfile & PID1=$! ; pdo_sched rm $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "unlink must fail" rm -rf $DIR/$tfile* @@ -1979,12 +2153,14 @@ test_43e() { touch $DIR1/$tfile touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" rm $DIR1/$tfile & PID1=$! ; pdo_sched mv -u $DIR2/$tfile-2 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed" rm -rf $DIR/$tfile* @@ -1996,12 +2172,14 @@ test_43f() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" rm $DIR1/$tfile & PID1=$! ; pdo_sched mv $DIR2/$tfile $DIR2/$tfile-2 & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "rename must fail" rm -rf $DIR/$tfile* @@ -2013,12 +2191,14 @@ test_43g() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" rm $DIR1/$tfile & PID1=$! ; pdo_sched stat $DIR2/$tfile > /dev/null & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "stat must fail" rm -rf $DIR/$tfile* @@ -2030,12 +2210,14 @@ test_43h() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" rm $DIR1/$tfile & PID1=$! ; pdo_sched ls -lia $DIR2/ > /dev/null & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; } wait $PID2 rm -rf $DIR/$tfile* @@ -2048,12 +2230,14 @@ test_43i() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" rm $DIR1/$tfile & PID1=$! ; pdo_sched $LFS mkdir -i 1 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "remote mkdir isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "remote mkdir must succeed" @@ -2062,17 +2246,97 @@ test_43i() { } run_test 43i "pdirops: unlink vs remote mkdir" +test_43j() { + [[ $MDS1_VERSION -lt $(version_code 2.13.52) ]] && + skip "Need MDS version newer than 2.13.52" + + for i in {1..100}; do +#define OBD_FAIL_ONCE|OBD_FAIL_MDS_CREATE_RACE 0x167 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000167 2>/dev/null || + true" + OK=0 + mkdir $DIR1/$tdir & + PID1=$! + mkdir $DIR2/$tdir && ((OK++)) + wait $PID1 && ((OK++)) + (( OK == 1 )) || error "exactly one mkdir should succeed" + + rmdir $DIR1/$tdir || error "rmdir failed" + done + return 0 +} +run_test 43j "racy mkdir return EEXIST ==============" + +sub_test_43k() { + local PID1 PID2 + local fail_loc="$1" + local ret=0 + + # We test in a separate directory to be able to unblock server thread in + # cfs_race() if LCK_PW is taken on the parent by mdt_reint_unlink. + test_mkdir $DIR2/$tdir + touch $DIR2/$tdir/$tfile + + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=${fail_loc} || true" &>/dev/null + echo content > $DIR1/$tdir/$tfile & PID1=$! + pdo_sched + multiop $DIR2/$tdir/$tfile u & PID2=$! + + wait $PID1 || + { ret=$?; \ + echo -n "overwriting $tfile should succeed (err=$ret); "; } + wait $PID2 || + { ret=$?; \ + echo -n "unlinking $tfile should succeed (err=$ret);"; } + + #Clean + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x0 || true" &>/dev/null + rm -rf $DIR/$tdir + + return $ret +} + +test_43k() { + [[ $MDS1_VERSION -le $(version_code 2.13.56) ]] || + skip "Need MDS version newer than 2.13.56" + local msg fail_loc + +#define OBD_FAIL_ONCE|OBD_FAIL_MDS_REINT_OPEN 0x169 +#define OBD_FAIL_ONCE|OBD_FAIL_MDS_REINT_OPEN2 0x16a + for fail_loc in "0x80000169" "0x8000016a"; do + echo "Begin 100 tests with fail_loc=$fail_loc" + printf "Progress: " + for i in {1..100}; do + printf "*" + msg=$(sub_test_43k "$fail_loc") || + { echo; error "iter=$i : $msg"; } + done + echo + done + + #Clean + reset_fail_loc + + return 0 +} +run_test 43k "unlink vs create" + # test 44: rename tgt and blocking operations test_44a() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2 0x146 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true" mv $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched mkdir $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; date;error "mkdir isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail" date @@ -2085,12 +2349,14 @@ test_44b() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2 0x146 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true" mv $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "create must fail" rm -rf $DIR/$tfile* @@ -2103,12 +2369,14 @@ test_44c() { touch $DIR1/$tfile-2 touch $DIR1/$tfile-3 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2 0x146 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true" mv $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched link $DIR2/$tfile-3 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "link must fail" rm -rf $DIR/$tfile* @@ -2120,12 +2388,14 @@ test_44d() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2 0x146 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true" mv $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched rm $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "unlink must succeed" rm -rf $DIR/$tfile* @@ -2139,12 +2409,14 @@ test_44e() { touch $DIR1/$tfile-2 touch $DIR1/$tfile-3 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2 0x146 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true" mv $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched mv $DIR2/$tfile-3 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed" rm -rf $DIR/$tfile* @@ -2157,12 +2429,14 @@ test_44f() { touch $DIR1/$tfile-2 touch $DIR1/$tfile-3 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2 0x146 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true" mv $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched mv $DIR2/$tfile $DIR2/$tfile-3 & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed" rm -rf $DIR/$tfile* @@ -2174,12 +2448,14 @@ test_44g() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2 0x146 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true" mv $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched stat $DIR2/$tfile > /dev/null & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed" rm -rf $DIR/$tfile* @@ -2191,12 +2467,14 @@ test_44h() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2 0x146 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true" mv $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched ls -lia $DIR2/ > /dev/null & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; } wait $PID2 rm -rf $DIR/$tfile* @@ -2210,12 +2488,14 @@ test_44i() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2 0x146 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true" mv $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched $LFS mkdir -i 1 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "remote mkdir isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "remote mkdir must fail" @@ -2224,34 +2504,30 @@ test_44i() { } run_test 44i "pdirops: rename tgt vs remote mkdir" -# test 45: rename src and blocking operations +# test 45: rename,mkdir doesn't fail with -EEXIST test_45a() { - pdo_lru_clear - touch $DIR1/$tfile -#define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 - mv $DIR1/$tfile $DIR1/$tfile-2 & - PID1=$! ; pdo_sched - mkdir $DIR2/$tfile & - PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 - check_pdo_conflict $PID1 && { wait $PID1; error "mkdir isn't blocked"; } - wait $PID2 ; [ $? -eq 0 ] || error "mkdir must succeed" - rm -rf $DIR/$tfile* + for i in {1..1000}; do + mkdir $DIR1/$tdir || error "mkdir $tdir failed" + mrename $DIR2/$tdir $DIR2/$tdir.$i > /dev/null || + error "mrename to $tdir.$i failed" + done + rm -rf $DIR/$tdir* return 0 } -run_test 45a "pdirops: rename src vs mkdir ==============" +run_test 45a "rename,mkdir doesn't return -EEXIST ==============" test_45b() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mv $DIR1/$tfile $DIR1/$tfile-2 & PID1=$! ; pdo_sched $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "create must succeed" rm -rf $DIR/$tfile* @@ -2264,12 +2540,14 @@ test_45c() { touch $DIR1/$tfile touch $DIR1/$tfile-3 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mv $DIR1/$tfile $DIR1/$tfile-2 & PID1=$! ; pdo_sched link $DIR2/$tfile-3 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "link must succeed" rm -rf $DIR/$tfile* @@ -2281,12 +2559,14 @@ test_45d() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mv $DIR1/$tfile $DIR1/$tfile-2 & PID1=$! ; pdo_sched rm $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "unlink must fail" rm -rf $DIR/$tfile* @@ -2299,12 +2579,14 @@ test_45e() { touch $DIR1/$tfile touch $DIR1/$tfile-3 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mv $DIR1/$tfile $DIR1/$tfile-2 & PID1=$! ; pdo_sched mv $DIR2/$tfile-3 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed" rm -rf $DIR/$tfile* @@ -2316,12 +2598,14 @@ test_45f() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mv $DIR1/$tfile $DIR1/$tfile-2 & PID1=$! ; pdo_sched mv $DIR2/$tfile $DIR2/$tfile-3 & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "rename must fail" rm -rf $DIR/$tfile* @@ -2333,12 +2617,14 @@ test_45g() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mv $DIR1/$tfile $DIR1/$tfile-2 & PID1=$! ; pdo_sched stat $DIR2/$tfile > /dev/null & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "stat must fail" rm -rf $DIR/$tfile* @@ -2350,11 +2636,13 @@ test_45h() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mv $DIR1/$tfile $DIR1/$tfile-2 & PID1=$! ; pdo_sched ls -lia $DIR2/ > /dev/null & - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; } wait $PID2 rm -rf $DIR/$tfile* @@ -2367,12 +2655,14 @@ test_45i() { pdo_lru_clear touch $DIR1/$tfile #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" mv $DIR1/$tfile $DIR1/$tfile-2 & PID1=$! ; pdo_sched $LFS mkdir -i 1 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "create remote dir isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "create remote dir must succeed" @@ -2381,17 +2671,73 @@ test_45i() { } run_test 45i "pdirops: rename src vs remote mkdir" +sub_test_45j() { + local PID1 PID2 + local fail_loc="$1" + local ret=0 + + # We test in a sparate directory to be able to unblock server thread in + # cfs_race if LCK_PW is taken on the parent by mdt_reint_rename. + test_mkdir $DIR2/$tdir + echo file1 > $DIR2/$tdir/$tfile + echo file2 > $DIR2/$tdir/$tfile-2 + + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=${fail_loc} || true" &>/dev/null + + cat $DIR1/$tdir/$tfile >/dev/null & + PID1=$! + pdo_sched + mrename $DIR2/$tdir/$tfile-2 $DIR2/$tdir/$tfile > /dev/null & + PID2=$! + + wait $PID1 || + { ret=$?; echo -n "cat $tfile should succeed (err=$ret); "; } + wait $PID2 || + { ret=$?; \ + echo -n "mrename $tfile-2 to $tfile failed (err=$ret);"; } + + #Clean + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x0 || true" &>/dev/null + rm -rf $DIR/$tdir + + return $ret +} + +test_45j() { + [[ $MDS1_VERSION -le $(version_code 2.13.56) ]] || + skip "Need MDS version newer than 2.13.56" + local msg fail_loc + +#define OBD_FAIL_ONCE|OBD_FAIL_MDS_REINT_OPEN 0x169 +#define OBD_FAIL_ONCE|OBD_FAIL_MDS_REINT_OPEN2 0x16a + for fail_loc in "0x80000169" "0x8000016a"; do + echo "Begin 100 tests with fail_loc=$fail_loc" + printf "Progress: " + for i in {1..100}; do + printf "*" + msg=$(sub_test_45j "$fail_loc") || + { echo; error "iter=$i : $msg"; } + done + echo + done +} +run_test 45j "read vs rename ==============" + # test 46: link and blocking operations test_46a() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" link $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched mkdir $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "mkdir isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail" rm -rf $DIR/$tfile* @@ -2403,12 +2749,14 @@ test_46b() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" link $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "create must fail" rm -rf $DIR/$tfile* @@ -2420,12 +2768,14 @@ test_46c() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" link $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched link $DIR2/$tfile $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "link must fail" rm -rf $DIR/$tfile* @@ -2437,12 +2787,14 @@ test_46d() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" link $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched rm $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "unlink must succeed" rm -rf $DIR/$tfile* @@ -2455,12 +2807,14 @@ test_46e() { touch $DIR1/$tfile-2 touch $DIR1/$tfile-3 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" link $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched mv $DIR2/$tfile-3 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed" rm -rf $DIR/$tfile* @@ -2473,12 +2827,14 @@ test_46f() { touch $DIR1/$tfile-2 touch $DIR1/$tfile-3 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" link $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched mv $DIR2/$tfile $DIR2/$tfile-3 & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed" rm -rf $DIR/$tfile* @@ -2490,12 +2846,14 @@ test_46g() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" link $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched stat $DIR2/$tfile > /dev/null & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed" rm -rf $DIR/$tfile* @@ -2507,12 +2865,14 @@ test_46h() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" link $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched ls -lia $DIR2/ > /dev/null & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; } wait $PID2 rm -rf $DIR/$tfile* @@ -2525,12 +2885,14 @@ test_46i() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" link $DIR1/$tfile-2 $DIR1/$tfile & PID1=$! ; pdo_sched $LFS mkdir -i 1 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "remote mkdir isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "remote mkdir must fail" @@ -2544,12 +2906,14 @@ test_47a() { #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return pdo_lru_clear - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $LFS mkdir -i 1 $DIR1/$tfile & PID1=$! ; pdo_sched mkdir $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "mkdir isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail" rm -rf $DIR/$tfile* @@ -2561,13 +2925,15 @@ test_47b() { #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return pdo_lru_clear - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $LFS mkdir -i 1 $DIR1/$tfile & PID1=$! ; pdo_sched sleep 1 # please do not remove this sleep, see LU-10754 multiop $DIR2/$tfile oO_CREAT:O_EXCL:c & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "create must fail" @@ -2581,12 +2947,14 @@ test_47c() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $LFS mkdir -i 1 $DIR1/$tfile & PID1=$! ; pdo_sched link $DIR2/$tfile-2 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "link must fail" rm -rf $DIR/$tfile* @@ -2598,12 +2966,14 @@ test_47d() { [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $LFS mkdir -i 1 $DIR1/$tfile & PID1=$! ; pdo_sched rmdir $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "rmdir must succeed" @@ -2617,12 +2987,14 @@ test_47e() { pdo_lru_clear touch $DIR1/$tfile-2 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $LFS mkdir -i 1 $DIR1/$tfile & PID1=$! ; pdo_sched mv -T $DIR2/$tfile-2 $DIR2/$tfile & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -ne 0 ] || error "rename must fail" @@ -2635,12 +3007,14 @@ test_47f() { [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $LFS mkdir -i 1 $DIR1/$tfile & PID1=$! ; pdo_sched mv $DIR2/$tfile $DIR2/$tfile-2 & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed" @@ -2655,12 +3029,14 @@ test_47g() { sync_all_data pdo_lru_clear #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK 0x145 - do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true" $LFS mkdir -i 1 $DIR1/$tfile & PID1=$! ; pdo_sched stat $DIR2/$tfile > /dev/null & PID2=$! ; pdo_sched - do_facet $SINGLEMDS lctl set_param fail_loc=0 + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n fail_loc=0 2>/dev/null || true" check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; } wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed" @@ -2716,29 +3092,32 @@ test_51a() { run_test 51a "layout lock: refresh layout should work" test_51b() { - [[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.3.59) ]] || - { skip "Need MDS version at least 2.3.59"; return 0; } + [[ "$MDS1_VERSION" -ge $(version_code 2.3.59) ]] || + skip "Need MDS version at least 2.3.59" local tmpfile=`mktemp` - # create an empty file - $MCREATE $DIR1/$tfile || error "mcreate $DIR1/$tfile failed" + $LFS setstripe -E 1M -c 1 -E -1 --extension-size 64M $DIR1/$tfile || + error "Create $DIR1/$tfile failed" + + dd if=/dev/zero of=$DIR1/$tfile bs=1k count=1 conv=notrunc || + error "dd $DIR1/$tfile failed" # delay glimpse so that layout has changed when glimpse finish #define OBD_FAIL_GLIMPSE_DELAY 0x1404 - $LCTL set_param fail_loc=0x1404 + $LCTL set_param fail_loc=0x1404 fail_val=4 stat -c %s $DIR2/$tfile |tee $tmpfile & local pid=$! - sleep 1 + sleep 0.2 - # create layout of testing file - dd if=/dev/zero of=$DIR1/$tfile bs=1k count=1 conv=notrunc >/dev/null || + # extend layout of testing file + dd if=/dev/zero of=$DIR1/$tfile bs=1M count=1 seek=2 conv=notrunc || error "dd $DIR1/$tfile failed" wait $pid local fsize=$(cat $tmpfile) - [ x$fsize = x1024 ] || error "file size is $fsize, should be 1024" + [ x$fsize = x3145728 ] || error "file size is $fsize, should be 3145728" rm -f $DIR1/$tfile $tmpfile } @@ -2805,6 +3184,27 @@ test_51d() { } run_test 51d "layout lock: losing layout lock should clean up memory map region" +test_51e() { + local pid + + $MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:eW_E+eUc & + pid=$! + sleep 1 + + $LFS getstripe $DIR2/$tfile + kill -USR1 $pid + wait $pid || error "multiop failed" + + $MULTIOP $DIR/$tfile oO_RDONLY:eR_E+eUc & + pid=$! + sleep 1 + + $LFS getstripe $DIR2/$tfile + kill -USR1 $pid + wait $pid || error "multiop failed" +} +run_test 51e "lfs getstripe does not break leases, part 2" + test_54_part1() { echo "==> rename vs getattr vs setxattr should not deadlock" @@ -2958,9 +3358,8 @@ test_55d() run_test 55d "rename file vs link" test_60() { - local MDSVER=$(lustre_build_version $SINGLEMDS) - [ $(version_code $MDSVER) -lt $(version_code 2.3.0) ] && - skip "MDS version $MDSVER must be >= 2.3.0" && return 0 + [ $MDS1_VERSION -lt $(version_code 2.3.0) ] && + skip "MDS version must be >= 2.3.0" # Create a file test_mkdir $DIR1/$tdir @@ -3046,19 +3445,16 @@ test_70b() { # LU-2781 run_test 70b "remove files after calling rm_entry" test_71a() { - local server_version=$(lustre_version_code $SINGLEMDS) - - [[ $server_version -lt $(version_code 2.1.6) ]] && - skip "Need MDS version at least 2.1.6" && return + [[ "$MDS1_VERSION" -lt $(version_code 2.1.6) ]] && + skip "Need MDS version at least 2.1.6" # Patch not applied to 2.2 and 2.3 branches - [[ $server_version -ge $(version_code 2.2.0) ]] && - [[ $server_version -lt $(version_code 2.4.0) ]] && - skip "Need MDS version earlier than 2.2.0 or at least 2.4.0" && - return + [[ "$MDS1_VERSION" -ge $(version_code 2.2.0) ]] && + [[ "$MDS1_VERSION" -lt $(version_code 2.4.0) ]] && + skip "Need MDS version earlier than 2.2.0 or at least 2.4.0" checkfiemap --test || - { skip "checkfiemap not runnable: $?" && return; } + skip "checkfiemap not runnable: $?" # write data this way: hole - data - hole - data dd if=/dev/urandom of=$DIR1/$tfile bs=40K seek=1 count=1 [ "$(facet_fstype ost$(($($LFS getstripe -i $DIR1/$tfile) + 1)))" = \ @@ -3090,20 +3486,17 @@ test_71a() { run_test 71a "correct file map just after write operation is finished" test_71b() { - local server_version=$(lustre_version_code $SINGLEMDS) - - [[ $server_version -lt $(version_code 2.1.6) ]] && - skip "Need MDS version at least 2.1.6" && return + [[ "$MDS1_VERSION" -lt $(version_code 2.1.6) ]] && + skip "Need MDS version at least 2.1.6" # Patch not applied to 2.2 and 2.3 branches - [[ $server_version -ge $(version_code 2.2.0) ]] && - [[ $server_version -lt $(version_code 2.4.0) ]] && - skip "Need MDS version earlier than 2.2.0 or at least 2.4.0" && - return - [[ $OSTCOUNT -ge 2 ]] || { skip "needs >= 2 OSTs"; return; } + [[ "$MDS1_VERSION" -ge $(version_code 2.2.0) ]] && + [[ "$MDS1_VERSION" -lt $(version_code 2.4.0) ]] && + skip "Need MDS version earlier than 2.2.0 or at least 2.4.0" + [[ $OSTCOUNT -ge 2 ]] || skip "needs >= 2 OSTs" checkfiemap --test || - { skip "error $?: checkfiemap failed" && return; } + skip "error $?: checkfiemap failed" mkdir -p $DIR1/$tdir @@ -3177,8 +3570,8 @@ test_73() { run_test 73 "getxattr should not cause xattr lock cancellation" test_74() { - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.4.93) ] && - skip "Need MDS version at least 2.4.93" && return + [ "$MDS1_VERSION" -lt $(version_code 2.4.93) ] && + skip "Need MDS version at least 2.4.93" dd if=/dev/zero of=$DIR1/$tfile-1 bs=1K count=1 dd if=/dev/zero of=$DIR1/$tfile-2 bs=1K count=1 @@ -3209,10 +3602,10 @@ test_75() { run_test 75 "osc: upcall after unuse lock===================" test_76() { #LU-946 - [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.5.53) ]] && - skip "Need MDS version at least 2.5.53" && return + [[ "$MDS1_VERSION" -lt $(version_code 2.5.53) ]] && + skip "Need MDS version at least 2.5.53" - remote_mds_nodsh && skip "remote MDS with nodsh" && return + remote_mds_nodsh && skip "remote MDS with nodsh" local fcount=$((MDSCOUNT * 256)) declare -a fd_list declare -a fid_list @@ -3519,7 +3912,7 @@ test_77e() { local idis local rateis - if [ $(lustre_version_code ost1) -ge $(version_code 2.8.54) ]; then + if [ "$OST1_VERSION" -ge $(version_code 2.8.54) ]; then idis="nid=" rateis="rate=" fi @@ -3575,7 +3968,7 @@ test_77f() { local idis local rateis - if [ $(lustre_version_code ost1) -ge $(version_code 2.8.54) ]; then + if [ "$OST1_VERSION" -ge $(version_code 2.8.54) ]; then idis="jobid=" rateis="rate=" fi @@ -3630,7 +4023,7 @@ test_77g() { local idis local rateis - if [ $(lustre_version_code ost1) -ge $(version_code 2.8.54) ]; then + if [ "$OST1_VERSION" -ge $(version_code 2.8.54) ]; then idis="jobid=" rateis="rate=" fi @@ -3647,8 +4040,8 @@ test_77g() { run_test 77g "Change TBF type directly" test_77h() { - [ $(lustre_version_code ost1) -ge $(version_code 2.8.55) ] || - { skip "Need OST version at least 2.8.55"; return 0; } + [ "$OST1_VERSION" -ge $(version_code 2.8.55) ] || + skip "Need OST version at least 2.8.55" local old_policy=$(do_facet ost1 \ lctl get_param ost.OSS.ost_io.nrs_policies) @@ -3702,8 +4095,8 @@ tbf_rule_check() } test_77i() { - [ $(lustre_version_code ost1) -ge $(version_code 2.8.55) ] || - { skip "Need OST version at least 2.8.55"; return 0; } + [ "$OST1_VERSION" -ge $(version_code 2.8.55) ] || + skip "Need OST version at least 2.8.55" for i in $(seq 1 $OSTCOUNT) do @@ -3757,11 +4150,10 @@ run_test 77i "Change rank of TBF rule" test_77j() { local idis local rateis - local ost_version=$(lustre_version_code ost1) - [ $ost_version -ge $(version_code 2.9.53) ] || - { skip "Need OST version at least 2.9.53"; return 0; } - if [ $ost_version -ge $(version_code 2.8.60) ]; then + [ "$OST1_VERSION" -ge $(version_code 2.9.53) ] || + skip "Need OST version at least 2.9.53" + if [ "$OST1_VERSION" -ge $(version_code 2.8.60) ]; then idis="opcode=" rateis="rate=" fi @@ -3813,10 +4205,10 @@ test_id() { } test_77ja(){ - if [ $(lustre_version_code ost1) -lt $(version_code 2.11.50) ]; then + if [ "$OST1_VERSION" -lt $(version_code 2.11.50) ]; then skip "Need OST version at least 2.11.50" - return 0 fi + test_id "u" "500" "5" "-u 500" test_id "g" "500" "5" "-u 500 -g 500" } @@ -3840,8 +4232,8 @@ cleanup_77k() } test_77k() { - [[ $(lustre_version_code ost1) -ge $(version_code 2.9.53) ]] || - { skip "Need OST version at least 2.9.53"; return 0; } + [[ "$OST1_VERSION" -ge $(version_code 2.9.53) ]] || + skip "Need OST version at least 2.9.53" do_nodes $(comma_list $(osts_nodes)) \ lctl set_param ost.OSS.ost_io.nrs_policies="tbf" \ @@ -3883,8 +4275,8 @@ test_77k() { trap "cleanup_77k \"ext_a ext_b\" \"fifo\"" EXIT - [[ $(lustre_version_code ost1) -ge $(version_code 2.10.58) ]] || - { skip "Need OST version at least 2.10.58"; return 0; } + [[ "$OST1_VERSION" -ge $(version_code 2.10.58) ]] || + skip "Need OST version at least 2.10.58" do_nodes $(comma_list $(osts_nodes)) \ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_a" \ @@ -3914,8 +4306,8 @@ test_77k() { run_test 77k "check TBF policy with NID/JobID/OPCode expression" test_77l() { - [[ $(lustre_version_code ost1) -ge $(version_code 2.10.56) ]] || - { skip "Need OST version at least 2.10.56"; return 0; } + [[ "$OST1_VERSION" -ge $(version_code 2.10.56) ]] || + skip "Need OST version at least 2.10.56" do_facet ost1 lctl set_param ost.OSS.ost_io.nrs_policies="tbf\ nid" do_facet ost1 lctl set_param ost.OSS.ost_io.nrs_policies="tbf" @@ -3934,9 +4326,8 @@ test_77l() { run_test 77l "check the output of NRS policies for generic TBF" test_77m() { - if [ $(lustre_version_code ost1) -lt $(version_code 2.9.54) ]; then + if [ "$OST1_VERSION" -lt $(version_code 2.9.54) ]; then skip "Need OST version at least 2.9.54" - return 0 fi local dir=$DIR/$tdir @@ -3986,9 +4377,8 @@ test_77m() { run_test 77m "check NRS Delay slows write RPC processing" test_77n() { #LU-10802 - if [ $(lustre_version_code ost1) -lt $(version_code 2.10.58) ]; then + if [ "$OST1_VERSION" -lt $(version_code 2.10.58) ]; then skip "Need OST version at least 2.10.58" - return 0 fi # Configure jobid_var @@ -4298,9 +4688,55 @@ test_81b() { } run_test 81b "rename under striped directory doesn't deadlock" +test_81c() { + [ $MDSCOUNT -lt 4 ] && skip_env "needs >= 4 MDTs" + [ $MDS1_VERSION -lt $(version_code 2.13.52) ] && + skip "Need MDS version at least 2.13.52" + + # source is local, source parent is remote + $LFS mkdir -i 0 $DIR1/${tdir}_src || error "mkdir ${tdir}_src" + $LFS mkdir -i 1 $DIR1/${tdir}_tgt || error "mkdir ${tdir}_tgt" + $LFS mkdir -i 3 $DIR1/${tdir}_src/sub || error "mkdir sub" + $LFS mkdir -i 3 $DIR1/${tdir}_tgt/sub || error "mkdir sub" + stat $DIR2/${tdir}_src/sub || error "stat sub failed" + mv $DIR1/${tdir}_src/sub $DIR1/${tdir}_tgt/ || error "mv failed" + [ -f $DIR2/${tdir}_src/sub ] && error "sub should be gone" + rm -rf $DIR1/${tdir}_src $DIR1/${tdir}_tgt + + # source is remote, source parent is local + $LFS mkdir -i 3 $DIR1/${tdir}_src || error "mkdir ${tdir}_src" + $LFS mkdir -i 1 $DIR1/${tdir}_tgt || error "mkdir ${tdir}_tgt" + $LFS mkdir -i 0 $DIR1/${tdir}_src/sub || error "mkdir sub" + $LFS mkdir -i 3 $DIR1/${tdir}_tgt/sub || error "mkdir sub" + stat $DIR2/${tdir}_src/sub || error "stat sub failed" + mv $DIR1/${tdir}_src/sub $DIR1/${tdir}_tgt/ || error "mv failed" + [ -f $DIR2/${tdir}_src/sub ] && error "sub should be gone" + rm -rf $DIR1/${tdir}_src $DIR1/${tdir}_tgt + + # source and source parent are remote + $LFS mkdir -i 0 $DIR1/${tdir}_src || error "mkdir ${tdir}_src" + $LFS mkdir -i 1 $DIR1/${tdir}_tgt || error "mkdir ${tdir}_tgt" + mkdir $DIR1/${tdir}_src/sub || error "mkdir sub" + $LFS mkdir -i 3 $DIR1/${tdir}_tgt/sub || error "mkdir sub" + stat $DIR2/${tdir}_src/sub || error "stat sub failed" + mv $DIR1/${tdir}_src/sub $DIR1/${tdir}_tgt/ || error "mv failed" + [ -f $DIR2/${tdir}_src/sub ] && error "sub should be gone" + rm -rf $DIR1/${tdir}_src $DIR1/${tdir}_tgt + + # source and source parent are remote, and source is remote object + $LFS mkdir -i 0 $DIR1/${tdir}_src || error "mkdir ${tdir}_src" + $LFS mkdir -i 1 $DIR1/${tdir}_tgt || error "mkdir ${tdir}_tgt" + $LFS mkdir -i 2 $DIR1/${tdir}_src/sub || error "mkdir sub" + $LFS mkdir -i 3 $DIR1/${tdir}_tgt/sub || error "mkdir sub" + stat $DIR2/${tdir}_src/sub || error "stat sub failed" + mv $DIR1/${tdir}_src/sub $DIR1/${tdir}_tgt/ || error "mv failed" + [ -f $DIR2/${tdir}_src/sub ] && error "sub should be gone" || true +} +run_test 81c "rename revoke LOOKUP lock for remote object" + test_82() { - [[ $(lustre_version_code $SINGLEMDS) -gt $(version_code 2.6.91) ]] || - { skip "Need MDS version at least 2.6.92"; return 0; } + [[ "$MDS1_VERSION" -gt $(version_code 2.6.91) ]] || + skip "Need MDS version at least 2.6.92" # Client 1 creates a file. multiop_bg_pause $DIR1/$tfile O_ac || error "multiop_bg_pause 1" @@ -4553,7 +4989,7 @@ run_test 94 "signal vs CP callback race" # Data-on-MDT tests test_100a() { skip "Reserved for glimpse-ahead" && return - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] && + [ "$MDS1_VERSION" -lt $(version_code 2.10.55) ] && skip "Need MDS version at least 2.10.55" mkdir -p $DIR/$tdir @@ -4577,7 +5013,7 @@ test_100a() { run_test 100a "DoM: glimpse RPCs for stat without IO lock (DoM only file)" test_100b() { - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] && + [ "$MDS1_VERSION" -lt $(version_code 2.10.55) ] && skip "Need MDS version at least 2.10.55" mkdir -p $DIR/$tdir @@ -4600,7 +5036,7 @@ test_100b() { run_test 100b "DoM: no glimpse RPC for stat with IO lock (DoM only file)" test_100c() { - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] && + [ "$MDS1_VERSION" -lt $(version_code 2.10.55) ] && skip "Need MDS version at least 2.10.55" mkdir -p $DIR/$tdir @@ -4623,7 +5059,7 @@ test_100c() { run_test 100c "DoM: write vs stat without IO lock (combined file)" test_100d() { - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] && + [ "$MDS1_VERSION" -lt $(version_code 2.10.55) ] && skip "Need MDS version at least 2.10.55" mkdir -p $DIR/$tdir @@ -4648,7 +5084,7 @@ test_100d() { run_test 100d "DoM: write+truncate vs stat without IO lock (combined file)" test_100e() { - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.11.50) ] && + [ "$MDS1_VERSION" -lt $(version_code 2.11.50) ] && skip "Need MDS version at least 2.11.50" local dom=$DIR/$tdir/dom @@ -4670,8 +5106,8 @@ test_100e() { run_test 100e "DoM: read on open and file size" test_101a() { - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] && - skip "Need MDS version at least 2.10.55" && return + [ "$MDS1_VERSION" -lt $(version_code 2.10.55) ] && + skip "Need MDS version at least 2.10.55" $LFS setstripe -E 1024K -L mdt -E EOF $DIR1/$tfile # to get layout @@ -4695,8 +5131,8 @@ test_101a() { run_test 101a "Discard DoM data on unlink" test_101b() { - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] && - skip "Need MDS version at least 2.10.55" && return + [ "$MDS1_VERSION" -lt $(version_code 2.10.55) ] && + skip "Need MDS version at least 2.10.55" $LFS setstripe -E 1024K -L mdt -E EOF $DIR1/$tfile touch $DIR1/${tfile}_2 @@ -4720,8 +5156,8 @@ test_101b() { run_test 101b "Discard DoM data on rename" test_101c() { - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.10.55) ] && - skip "Need MDS version at least 2.10.55" && return + [ "$MDS1_VERSION" -lt $(version_code 2.10.55) ] && + skip "Need MDS version at least 2.10.55" $LFS setstripe -E 1024K -L mdt -E EOF $DIR1/$tfile # to get layout @@ -4754,7 +5190,7 @@ run_test 101c "Discard DoM data on close-unlink" # This test opens the file normally on $DIR1, which is on one mount, and then # opens it by handle on $DIR2, which is on a different mount. test_102() { - [ $MDS1_VERSION -lt $(version_code 2.11.57) ] && + [ "$MDS1_VERSION" -lt $(version_code 2.11.57) ] && skip "Needs MDS version 2.11.57 or later" echo "Test file_handle syscalls" > $DIR/$tfile || @@ -4768,10 +5204,10 @@ run_test 102 "Test open by handle of unlinked file" # Compare file size between first & second mount, ensuring the client correctly # glimpses even with unused speculative locks - LU-11670 test_103() { - [ $(lustre_version_code $ost1) -lt $(version_code 2.10.50) ] && + [ $OST1_VERSION -lt $(version_code 2.10.50) ] && skip "Lockahead needs OST version at least 2.10.50" - local testnum=23 + local locktest=23 test_mkdir -p $DIR/$tdir @@ -4788,7 +5224,7 @@ test_103() { do_facet ost1 $LCTL set_param fail_loc=0x415 fail_val=2 echo "Incorrect size expected (no glimpse fix):" - lockahead_test -d $DIR/$tdir -D $DIR2/$tdir -t $testnum -f $tfile + lockahead_test -d $DIR/$tdir -D $DIR2/$tdir -t $locktest -f $tfile rc=$? if [ $rc -eq 0 ]; then echo "This doesn't work 100%, but this is just reproducing the bug, not testing the fix, so OK to not fail test." @@ -4805,9 +5241,9 @@ test_103() { do_facet ost1 $LCTL set_param fail_loc=0x214 fail_val=2 # Write commit is still delayed by 2 seconds - lockahead_test -d $DIR/$tdir -D $DIR2/$tdir -t $testnum -f $tfile + lockahead_test -d $DIR/$tdir -D $DIR2/$tdir -t $locktest -f $tfile rc=$? - [ $rc -eq 0 ] || error "Lockahead test${testnum} failed, ${rc}" + [ $rc -eq 0 ] || error "Lockahead test$locktest failed, $rc" # guarantee write commit timeout has expired sleep 2 @@ -4846,7 +5282,7 @@ check_mdt_xtimes() local mdt_xtimes=($(get_mdt_xtimes $mdtdev)) echo "STAT a|m|ctime ${xtimes[*]}" - echo "MDT a|m|ctime ${xtimes[*]}" + echo "MDT a|m|ctime ${mdt_xtimes[*]}" [[ ${xtimes[0]} == ${mdt_xtimes[0]} ]] || error "$DIR/$tfile atime (${xtimes[0]}:${mdt_xtimes[0]}) diff" [[ ${xtimes[1]} == ${mdt_xtimes[1]} ]] || @@ -4924,6 +5360,252 @@ test_105() { } run_test 105 "Glimpse and lock cancel race" +test_106a() { + [ "$mds1_FSTYPE" == "ldiskfs" ] && statx_supported || + skip_env "Test only for ldiskfs and statx() supported" + + local btime + local mdt_btime + local output + local mdtdev=$(mdsdevname ${SINGLEMDS//mds/}) + + dd if=/dev/zero of=$DIR/$tfile bs=1k count=1 conv=notrunc + btime=$($STATX -c %W $DIR/$tfile) + output=$(do_facet mds1 "$DEBUGFS -c -R 'stat ROOT/$tfile' $mdtdev") + echo $output + ((mdt_btime=$(awk -F ':' /crtime/'{ print $2 }' <<< "$output"))) + [[ $btime == $mdt_btime ]] || + error "$DIR/$tfile btime ($btime:$mdt_btime) diff" + +} +run_test 106a "Verify the btime via statx()" + +test_106b() { + statx_supported || skip_env "statx() only test" + + local rpcs_before + local rpcs_after + + $LFS setstripe -c 1 $DIR/$tfile || error "$DIR/$tfile setstripe failed" + dd if=/dev/zero of=$DIR/$tfile bs=1k count=1 conv=notrunc + cancel_lru_locks $OSC + rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue) + $STATX $DIR/$tfile + rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue) + [ $rpcs_after -eq $((rpcs_before + 1)) ] || + error "$STATX should send 1 glimpse RPC to $OSC" + + cancel_lru_locks $OSC + rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue) + # %n: FILENAME; %i: STATX_INO; %A STATX_MODE; %h STATX_NLINK; + # %u: STATX_UID; %g: STATX_GID; %W STATX_BTIME; %X STATX_ATIME; + # %Z: STATX_CTIME + $STATX -c "%n %i %A %h %u %g %W %X %Z" $DIR/$tfile + rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue) + [ $rpcs_after -eq $rpcs_before ] || + error "$STATX should not send glimpse RPCs to $OSC" + + cancel_lru_locks $OSC + rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue) + $STATX --cached=always $DIR/$tfile + rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue) + [ $rpcs_after -eq $rpcs_before ] || + error "$STATX should not send glimpse RPCs to $OSC" + + cancel_lru_locks $OSC + rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue) + $STATX -c %Y $DIR/$tfile + rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue) + [ $rpcs_after -eq $((rpcs_before + 1)) ] || + error "$STATX -c %Y should send 1 glimpse RPC to $OSC" + + cancel_lru_locks $OSC + rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue) + $STATX -c %s $DIR/$tfile + rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue) + [ $rpcs_after -eq $((rpcs_before + 1)) ] || + error "$STATX -c %s should send 1 glimpse RPC to $OSC" + + cancel_lru_locks $OSC + rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue) + $STATX -c %b $DIR/$tfile + rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue) + [ $rpcs_after -eq $((rpcs_before + 1)) ] || + error "$STATX -c %b should send 1 glimpse RPC to $OSC" +} +run_test 106b "Glimpse RPCs test for statx" + +test_106c() { + statx_supported || skip_env "statx() only test" + + local mask + + touch $DIR/$tfile + # Mask supported in stx_attributes by Lustre is + # STATX_ATTR_IMMUTABLE(0x10) | STATX_ATTR_APPEND(0x20) : (0x30). + mask=$($STATX -c %p $DIR/$tfile) + [[ $mask == "30" ]] || + error "supported stx_attributes: got '$mask', expected '30'" + chattr +i $DIR/$tfile || error "chattr +i $DIR/$tfile failed" + mask=$($STATX -c %r $DIR/$tfile) + [[ $mask == "10" ]] || + error "got immutable flags '$mask', expected '10'" + chattr -i $DIR/$tfile || error "chattr -i $DIR/$tfile failed" + mask=$($STATX -c %r $DIR/$tfile) + [[ $mask == "0" ]] || error "got flags '$mask', expected '0'" + chattr +a $DIR/$tfile || error "chattr +a $DIR/$tfile failed" + mask=$($STATX -c %r $DIR/$tfile) + [[ $mask == "20" ]] || error "got flags '$mask', expected '20'" + chattr -a $DIR/$tfile || error "chattr -a $DIR/$tfile failed" + mask=$($STATX -c %r $DIR/$tfile) + [[ $mask == "0" ]] || error "got flags '$mask', expected '0'" + chattr +ia $DIR/$tfile || error "chattr +ia $DIR/$tfile failed" + mask=$($STATX -c %r $DIR/$tfile) + [[ $mask == "30" ]] || error "got flags '$mask', expected '30'" + chattr -ia $DIR/$tfile || error "chattr -ia $DIR/$tfile failed" + mask=$($STATX -c %r $DIR/$tfile) + [[ $mask == "0" ]] || error "got flags '$mask', expected '0'" +} +run_test 106c "Verify statx attributes mask" + +test_107a() { # LU-1031 + dd if=/dev/zero of=$DIR1/$tfile bs=1M count=10 + local gid1=14091995 + local gid2=16022000 + + $LFS getstripe $DIR1/$tfile + + multiop_bg_pause $DIR1/$tfile OG${gid1}_g${gid1}c || return 1 + local MULTIPID1=$! + multiop_bg_pause $DIR2/$tfile O_G${gid2}r10g${gid2}c || return 2 + local MULTIPID2=$! + kill -USR1 $MULTIPID2 + sleep 2 + if [[ $(ps h -o comm -p $MULTIPID2) == "" ]]; then + error "First grouplock does not block second one" + else + echo "First grouplock blocks second one" + fi + kill -USR1 $MULTIPID1 + wait $MULTIPID1 + wait $MULTIPID2 +} +run_test 107a "Basic grouplock conflict" + +test_107b() { + dd if=/dev/zero of=$DIR1/$tfile bs=1M count=10 + local gid1=14091995 + local gid2=16022000 + + $LFS getstripe $DIR1/$tfile + + multiop_bg_pause $DIR1/$tfile OG${gid1}_g${gid1}c || return 1 + local MULTIPID1=$! + multiop $DIR2/$tfile Or10c & + local MULTIPID2=$! + sleep 2 + + if [[ $(ps h -o comm -p $MULTIPID2) == "" ]]; then + error "Grouplock does not block IO" + else + echo "Grouplock blocks IO" + fi + + multiop $DIR2/$tfile OG${gid2}_g${gid2}c & + local MULTIPID3=$! + sleep 2 + if [[ $(ps h -o comm -p $MULTIPID3) == "" ]]; then + error "First grouplock does not block second one" + else + echo "First grouplock blocks second one" + fi + + kill -USR1 $MULTIPID1 + sleep 2 + + if [[ $(ps h -o comm -p $MULTIPID3) == "" ]]; then + error "Second grouplock thread disappeared" + fi + + if [[ $(ps h -o comm -p $MULTIPID2) == "" ]]; then + error "Second grouplock does not block IO" + else + echo "Second grouplock blocks IO" + fi + + kill -USR1 $MULTIPID3 + wait $MULTIPID1 + wait $MULTIPID2 + wait $MULTIPID3 +} +run_test 107b "Grouplock is added to the head of waiting list" + +test_108a() { + local offset + + $LFS setstripe -E 1M -c 1 -E -1 $DIR1/$tfile || + error "Create $DIR1/$tfile failed" + + dd if=/dev/zero of=$DIR1/$tfile bs=10000 count=1 || + error "dd $DIR1/$tfile failed" + offset=$(lseek_test -d 5000 $DIR2/$tfile) + [[ $offset == 5000 ]] || error "offset $offset != 5000" + + $TRUNCATE $DIR1/$tfile 2000 + offset=$(lseek_test -l 1000 $DIR2/$tfile) + [[ $offset == 2000 ]] || error "offset $offset != 2000" + + #define OBD_FAIL_OSC_DELAY_IO 0x414 + $LCTL set_param fail_val=4 fail_loc=0x80000414 + dd if=/dev/zero of=$DIR1/$tfile count=1 bs=8M conv=notrunc oflag=dsync & + local pid=$! + sleep 2 + + offset=$(lseek_test -l 8000 $DIR2/$tfile) + wait $pid + [[ $offset == 8388608 ]] || error "offset $offset != 8388608" +} +run_test 108a "lseek: parallel updates" + +# LU-14110 +test_109() { + local i + local pid1 pid2 + + ! local_mode || + skip "Clients need to be on different nodes than the servers" + + umount_client $MOUNT + umount_client $MOUNT2 + + echo "Starting race between client mount instances (50 iterations):" + for i in {1..50}; do + log "Iteration $i" + +#define OBD_FAIL_ONCE|OBD_FAIL_LLITE_RACE_MOUNT 0x80001417 + $LCTL set_param -n fail_loc=0x80001417 + + mount_client $MOUNT & pid1=$! + mount_client $MOUNT2 & pid2=$! + wait $pid1 || error "Mount $MOUNT fails with $?" + wait $pid2 || error "Mount $MOUNT2 fails with $?" + + umount_client $MOUNT & pid1=$! + umount_client $MOUNT2 & pid2=$! + wait $pid1 || error "Umount $MOUNT fails with $?" + wait $pid2 || error "Umount $MOUNT2 fails with $?" + + $LUSTRE_RMMOD || error "Fail to remove lustre modules" + load_modules + echo + done + + mount_client $MOUNT + mount_client $MOUNT2 +} + +run_test 109 "Race with several mount instances on 1 node" + log "cleanup: ======================================================" # kill and wait in each test only guarentee script finish, but command in script