LU-14547 test: skip sanityn 109 for local setup

[fs/lustre-release.git] / lustre / tests / sanityn.sh

diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh
index fe85b4a..1dd0838 100755 (executable)
--- a/lustre/tests/sanityn.sh
+++ b/lustre/tests/sanityn.sh
@@ -389,6 +389,8 @@ test_16a() {
        local stripe_size=$(do_facet $SINGLEMDS \
                "$LCTL get_param -n lod.$(facet_svc $SINGLEMDS)*.stripesize")
 
+       check_set_fallocate
+
        # to allocate grant because it may run out due to test_15.
        $LFS setstripe -c -1 $file1
        dd if=/dev/zero of=$file1 bs=$stripe_size count=$OSTCOUNT oflag=sync
@@ -396,12 +398,12 @@ test_16a() {
        rm -f $file1
 
        $LFS setstripe -c -1 $file1 # b=10919
-       fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 $file1 $file2 \
-               || error "fsx failed"
+       $FSX -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 $file1 $file2 ||
+               error "fsx failed"
        rm -f $file1
 
        # O_DIRECT reads and writes must be aligned to the device block size.
-       fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 -Z -r 4096 \
+       $FSX -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 -Z -r 4096 \
                -w 4096 $file1 $file2 || error "fsx with O_DIRECT failed."
 }
 run_test 16a "$FSXNUM iterations of dual-mount fsx"
@@ -412,6 +414,8 @@ test_16b() {
        local file2=$DIR2/$tfile
        local stripe_size=($($LFS getstripe -S $DIR))
 
+       check_set_fallocate
+
        # to allocate grant because it may run out due to test_15.
        lfs setstripe -c -1 $file1
        dd if=/dev/zero of=$file1 bs=$stripe_size count=$OSTCOUNT oflag=sync ||
@@ -423,8 +427,8 @@ test_16b() {
        lfs setstripe -c -1 $file1 # b=10919
        # -o is set to 8192 because writes < 1 page and between 1 and 2 pages
        # create a mix of tiny writes & normal writes
-       fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -o 8192 -S 0 $file1 \
-       $file2 || error "fsx with tiny write failed."
+       $FSX -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -o 8192 -S 0 \
+               $file1 $file2 || error "fsx with tiny write failed."
 }
 run_test 16b "$FSXNUM iterations of dual-mount fsx at small size"
 
@@ -436,6 +440,8 @@ test_16c() {
 
        [ "$ost1_FSTYPE" != ldiskfs ] && skip "dio on ldiskfs only"
 
+       check_set_fallocate
+
        # to allocate grant because it may run out due to test_15.
        $LFS setstripe -c -1 $file1
        dd if=/dev/zero of=$file1 bs=$stripe_size count=$OSTCOUNT oflag=sync
@@ -452,8 +458,8 @@ test_16c() {
        set_osd_param $list '' writethrough_cache_enable 0
 
        $LFS setstripe -c -1 $file1 # b=10919
-       fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 $file1 $file2 \
-               || error "fsx failed"
+       $FSX -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 $file1 $file2 ||
+               error "fsx failed"
        rm -f $file1
 
        set_osd_param $list '' read_cache_enable 1
@@ -467,16 +473,17 @@ test_16d() {
        local file1=$DIR1/$tfile
        local file2=$DIR2/$tfile
        local file3=$DIR1/file
+       local tmpfile=$(mktemp)
        local stripe_size=$(do_facet $SINGLEMDS \
                "$LCTL get_param -n lod.$(facet_svc $SINGLEMDS)*.stripesize")
 
        # to allocate grant because it may run out due to test_15.
        $LFS setstripe -c -1 $file1
+       stack_trap "rm -f $file1 $file2 $file3 $tmpfile"
        dd if=/dev/zero of=$file1 bs=$stripe_size count=$OSTCOUNT oflag=sync
        dd if=/dev/zero of=$file2 bs=$stripe_size count=$OSTCOUNT oflag=sync
        rm -f $file1
 
-       local tmpfile=`mktemp`
        $LFS setstripe -c -1 $file1 # b=10919
        $LCTL set_param ldlm.namespaces.*.lru_size=clear
        
@@ -497,12 +504,34 @@ test_16d() {
        # buffer read from another client
        dd if=$file2 of=$file3 bs=1M count=100
        diff $file3 $tmpfile || error "file different(3)"
-
-       rm -f $file1 $file2 $file3 $tmpfile
-
 }
 run_test 16d "Verify DIO and buffer IO with two clients"
 
+test_16e() { # LU-13227
+       # issue:        LU-14314
+
+       (( "$MDS1_VERSION" >= $(version_code 2.13.53) )) ||
+               skip "Need MDS version at least 2.13.53"
+
+       local file1=$DIR1/$tfile
+       local file2=$DIR2/$tfile
+
+       # client1 write 10M data
+       dd if=/dev/zero of=$file1 bs=1M count=10
+       # drop locks
+       cancel_lru_locks osc > /dev/null
+       # use lockahead to generate one PW lock to keep LVB loaded.
+       $LFS ladvise -a lockahead --start 0 --length 1M \
+               --mode WRITE $file1
+       # direct write to extend file size on client2
+       dd if=/dev/zero of=$file2 bs=1M seek=20 count=1 \
+               oflag=direct conv=notrunc
+       local filesize=$(stat -c %s $file2)
+       [ "$filesize" -eq 22020096 ] ||
+               error "expected filesize 22020096 got $filesize"
+       rm -f $file1
+}
+run_test 16e "Verify size consistency for O_DIRECT write"
 
 test_17() { # bug 3513, 3667
        remote_ost_nodsh && skip "remote OST with nodsh" && return
@@ -530,7 +559,9 @@ test_18() {
                 excepts="$excepts -e $(($(printf %d \'$idx)-96))"
         done
 
-       $LUSTRE/tests/mmap_sanity -d $MOUNT1 -m $MOUNT2 $excepts
+       excepts="$excepts -e 7 -e 8 -e 9"
+       $LUSTRE/tests/mmap_sanity -d $MOUNT1 -m $MOUNT2 $excepts ||
+               error "mmap_sanity test failed"
        sync; sleep 1; sync
 }
 run_test 18 "mmap sanity check ================================="
@@ -566,16 +597,16 @@ test_19() { # bug3811
 run_test 19 "test concurrent uncached read races ==============="
 
 test_20() {
-       test_mkdir $DIR1/d20
-       cancel_lru_locks $OSC
-       CNT=$((`lctl get_param -n llite.*.dump_page_cache | wc -l`))
-       $MULTIOP $DIR1/f20 Ow8190c
-       $MULTIOP $DIR2/f20 Oz8194w8190c
-       $MULTIOP $DIR1/f20 Oz0r8190c
-       cancel_lru_locks $OSC
-       CNTD=$((`lctl get_param -n llite.*.dump_page_cache | wc -l` - $CNT))
-       [ $CNTD -gt 0 ] && \
-           error $CNTD" page left in cache after lock cancel" || true
+       test_mkdir $DIR1/$tdir
+       cancel_lru_locks
+       CNT=$($LCTL get_param -n llite.*.dump_page_cache | wc -l)
+       $MULTIOP $DIR1/$tdir/$tfile Ow8190c
+       $MULTIOP $DIR2/$tdir/$tfile Oz8194w8190c
+       $MULTIOP $DIR1/$tdir/$tfile Oz0r8190c
+       cancel_lru_locks
+       CNT2=$($LCTL get_param -n llite.*.dump_page_cache | wc -l)
+       [[ $CNT2 == $CNT ]] ||
+               error $((CNT2 - CNT))" page left in cache after lock cancel"
 }
 run_test 20 "test extra readahead page left in cache ===="
 
@@ -731,7 +762,7 @@ test_27() {
        lctl clear
        dd if=/dev/zero of=$DIR2/$tfile bs=$((4096+4))k conv=notrunc count=4 seek=3 &
        DD2_PID=$!
-       usleep 50
+       sleep 0.5
        log "dd 1 started"
 
        dd if=/dev/zero of=$DIR1/$tfile bs=$((16384-1024))k conv=notrunc count=1 seek=4 &
@@ -855,13 +886,15 @@ test_32a() { # bug 11270
 
        log "checking cached lockless truncate"
        $TRUNCATE $DIR1/$tfile 8000000
-       $CHECKSTAT -s 8000000 $DIR2/$tfile || error "wrong file size"
+       $CHECKSTAT -s 8000000 $DIR2/$tfile ||
+               error "cached truncate - wrong file size"
        [ $(calc_stats $OSC.*.${OSC}_stats lockless_truncate) -ne 0 ] ||
                error "cached truncate isn't lockless"
 
        log "checking not cached lockless truncate"
        $TRUNCATE $DIR2/$tfile 5000000
-       $CHECKSTAT -s 5000000 $DIR1/$tfile || error "wrong file size"
+       $CHECKSTAT -s 5000000 $DIR1/$tfile ||
+               error "not cached truncate - wrong file size"
        [ $(calc_stats $OSC.*.${OSC}_stats lockless_truncate) -ne 0 ] ||
                error "not cached truncate isn't lockless"
 
@@ -869,7 +902,8 @@ test_32a() { # bug 11270
        enable_lockless_truncate 0
        clear_stats $OSC.*.${OSC}_stats
        $TRUNCATE $DIR2/$tfile 3000000
-       $CHECKSTAT -s 3000000 $DIR1/$tfile || error "wrong file size"
+       $CHECKSTAT -s 3000000 $DIR1/$tfile ||
+               error "lockless truncate disabled - wrong file size"
        [ $(calc_stats $OSC.*.${OSC}_stats lockless_truncate) -eq 0 ] ||
                error "lockless truncate disabling failed"
        rm -f $DIR1/$tfile
@@ -1064,6 +1098,10 @@ test_33c() {
        [ "$MDS1_VERSION" -lt $(version_code 2.7.63) ] &&
                skip "DNE CoS not supported"
 
+       # LU-13522
+       stop mds1
+       start mds1 $(mdsdevname 1) $MDS_MOUNT_OPTS || error "start mds1 failed"
+
        local sync_count
 
        mkdir $DIR/$tdir
@@ -1536,7 +1574,8 @@ test_40a() {
        remote_mds_nodsh && skip "remote MDS with nodsh" && return
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        touch $DIR2
        mkdir $DIR1/$tfile &
        PID1=$!; pdo_sched
@@ -1556,7 +1595,8 @@ test_40a() {
 
        #  all operations above shouldn't wait the first one
        check_pdo_conflict $PID1 || error "parallel operation is blocked"
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        wait $PID1
        rm -rf $DIR/$tfile*
        return 0
@@ -1567,7 +1607,8 @@ test_40b() {
        remote_mds_nodsh && skip "remote MDS with nodsh" && return
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        touch $DIR1/$tfile &
        PID1=$!; pdo_sched
        # open|create
@@ -1587,7 +1628,8 @@ test_40b() {
        # all operations above shouldn't wait the first one
 
         check_pdo_conflict $PID1 || error "parallel operation is blocked"
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        wait $PID1
        rm -rf $DIR/$tfile*
        return 0
@@ -1599,7 +1641,8 @@ test_40c() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile $DIR1/$tfile-0 &
        PID1=$!; pdo_sched
        # open|create
@@ -1619,7 +1662,8 @@ test_40c() {
 
         # all operations above shouldn't wait the first one
        check_pdo_conflict $PID1 || error "parallel operation is blocked"
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        wait $PID1
        rm -rf $DIR/$tfile*
        return 0
@@ -1631,7 +1675,8 @@ test_40d() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$!; pdo_sched
        # open|create
@@ -1651,7 +1696,8 @@ test_40d() {
 
        # all operations above shouldn't wait the first one
        check_pdo_conflict $PID1 || error "parallel operation is blocked"
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        wait $PID1
        return 0
 }
@@ -1662,7 +1708,8 @@ test_40e() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-0 &
        PID1=$!; pdo_sched
        # open|create
@@ -1680,7 +1727,8 @@ test_40e() {
 
        # all operations above shouldn't wait the first one
        check_pdo_conflict $PID1 || error "parallel operation is blocked"
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        wait $PID1
        rm -rf $DIR/$tfile*
        return 0
@@ -1691,12 +1739,14 @@ run_test 40e "pdirops: rename and others =============="
 test_41a() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        mkdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; echo "mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail"
        rm -rf $DIR/$tfile*
@@ -1707,12 +1757,14 @@ run_test 41a "pdirops: create vs mkdir =============="
 test_41b() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "create must fail"
        rm -rf $DIR/$tfile*
@@ -1724,12 +1776,14 @@ test_41c() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "link must fail"
        rm -rf $DIR/$tfile*
@@ -1740,12 +1794,14 @@ run_test 41c "pdirops: create vs link =============="
 test_41d() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        rm $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "unlink must succeed"
        rm -rf $DIR/$tfile*
@@ -1757,12 +1813,14 @@ test_41e() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -1773,12 +1831,14 @@ run_test 41e "pdirops: create and rename (tgt) =============="
 test_41f() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-2 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -1789,12 +1849,14 @@ run_test 41f "pdirops: create and rename (src) =============="
 test_41g() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed"
        rm -rf $DIR/$tfile*
@@ -1805,12 +1867,14 @@ run_test 41g "pdirops: create vs getattr =============="
 test_41h() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        ls -lia $DIR2/ > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; }
        wait $PID2
        rm -rf $DIR/$tfile*
@@ -1818,16 +1882,69 @@ test_41h() {
 }
 run_test 41h "pdirops: create vs readdir =============="
 
+sub_test_41i() {
+       local PID1 PID2
+       local fail_loc="$1"
+       local ret=0
+
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=${fail_loc} || true" &>/dev/null
+
+       $MULTIOP $DIR1/$tfile oO_CREAT:O_EXCL:c 2>/dev/null &
+       PID1=$!
+       sleep 0.2
+       $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c 2>/dev/null &
+       PID2=$!
+
+       if ! wait $PID1 && ! wait $PID2; then
+               echo "Both creates failed (1 should fail, 1 should succeed)"
+               ret=1
+       elif wait $PID1 && wait $PID2; then
+               echo "Both creates succeeded (1 should fail, 1 should succeed)"
+               ret=2
+       fi
+
+       #Clean
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x0 || true" &>/dev/null
+       rm -f $DIR/$tfile
+
+       return $ret
+}
+
+test_41i() {
+       [[ $MDS1_VERSION -le $(version_code 2.13.56) ]] ||
+               skip "Need MDS version newer than 2.13.56"
+       local msg fail_loc
+
+#define OBD_FAIL_ONCE|OBD_FAIL_MDS_REINT_OPEN         0x169
+#define OBD_FAIL_ONCE|OBD_FAIL_MDS_REINT_OPEN2        0x16a
+       for fail_loc in "0x80000169" "0x8000016a"; do
+               echo "Begin 100 tests with fail_loc=$fail_loc"
+               printf "Progress: "
+               for i in {1..100}; do
+                       printf "*"
+                       msg=$(sub_test_41i "$fail_loc") ||
+                               { echo; error "iter=$i : $msg"; }
+               done
+               echo
+       done
+}
+run_test 41i "reint_open: create vs create"
+
+
 # test 42: unlink and blocking operations
 test_42a() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mkdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail"
        rm -rf $DIR/$tfile*
@@ -1838,12 +1955,14 @@ run_test 42a "pdirops: mkdir vs mkdir =============="
 test_42b() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "create must fail"
        rm -rf $DIR/$tfile*
@@ -1855,12 +1974,14 @@ test_42c() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "link must fail"
        rm -rf $DIR/$tfile*
@@ -1871,12 +1992,14 @@ run_test 42c "pdirops: mkdir vs link =============="
 test_42d() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        rmdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "unlink must succeed"
        rm -rf $DIR/$tfile*
@@ -1888,12 +2011,14 @@ test_42e() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv -T $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "rename must fail"
        rm -rf $DIR/$tfile*
@@ -1904,12 +2029,14 @@ run_test 42e "pdirops: mkdir and rename (tgt) =============="
 test_42f() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-2 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -1920,12 +2047,14 @@ run_test 42f "pdirops: mkdir and rename (src) =============="
 test_42g() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed"
        rm -rf $DIR/$tfile*
@@ -1936,12 +2065,14 @@ run_test 42g "pdirops: mkdir vs getattr =============="
 test_42h() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        ls -lia $DIR2/ > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; }
        wait $PID2
        rm -rf $DIR/$tfile*
@@ -1963,12 +2094,14 @@ test_43b() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "create must succeed"
        rm -rf $DIR/$tfile*
@@ -1981,12 +2114,14 @@ test_43c() {
        touch $DIR1/$tfile
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "link must succeed"
        rm -rf $DIR/$tfile*
@@ -1998,12 +2133,14 @@ test_43d() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        rm $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "unlink must fail"
        rm -rf $DIR/$tfile*
@@ -2016,12 +2153,14 @@ test_43e() {
        touch $DIR1/$tfile
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv -u $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -2033,12 +2172,14 @@ test_43f() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-2 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "rename must fail"
        rm -rf $DIR/$tfile*
@@ -2050,12 +2191,14 @@ test_43g() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "stat must fail"
        rm -rf $DIR/$tfile*
@@ -2067,12 +2210,14 @@ test_43h() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        ls -lia $DIR2/ > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; }
        wait $PID2
        rm -rf $DIR/$tfile*
@@ -2085,12 +2230,14 @@ test_43i() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $LFS mkdir -i 1 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 &&
                { wait $PID1; error "remote mkdir isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "remote mkdir must succeed"
@@ -2105,7 +2252,9 @@ test_43j() {
 
        for i in {1..100}; do
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_CREATE_RACE         0x167
-               do_facet $SINGLEMDS lctl set_param fail_loc=0x80000167
+               do_nodes $(comma_list $(mdts_nodes)) \
+                       "lctl set_param -n fail_loc=0x80000167 2>/dev/null ||
+                               true"
                OK=0
                mkdir $DIR1/$tdir &
                PID1=$!
@@ -2119,17 +2268,75 @@ test_43j() {
 }
 run_test 43j "racy mkdir return EEXIST =============="
 
+sub_test_43k() {
+       local PID1 PID2
+       local fail_loc="$1"
+       local ret=0
+
+       # We test in a separate directory to be able to unblock server thread in
+       # cfs_race() if LCK_PW is taken on the parent by mdt_reint_unlink.
+       test_mkdir $DIR2/$tdir
+       touch $DIR2/$tdir/$tfile
+
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=${fail_loc} || true" &>/dev/null
+       echo content > $DIR1/$tdir/$tfile & PID1=$!
+       pdo_sched
+       multiop $DIR2/$tdir/$tfile u & PID2=$!
+
+       wait $PID1 ||
+               { ret=$?; \
+               echo -n "overwriting $tfile should succeed (err=$ret); "; }
+       wait $PID2 ||
+               { ret=$?; \
+               echo -n "unlinking $tfile should succeed (err=$ret);"; }
+
+       #Clean
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x0 || true" &>/dev/null
+       rm -rf $DIR/$tdir
+
+       return $ret
+}
+
+test_43k() {
+       [[ $MDS1_VERSION -le $(version_code 2.13.56) ]] ||
+               skip "Need MDS version newer than 2.13.56"
+       local msg fail_loc
+
+#define OBD_FAIL_ONCE|OBD_FAIL_MDS_REINT_OPEN         0x169
+#define OBD_FAIL_ONCE|OBD_FAIL_MDS_REINT_OPEN2        0x16a
+       for fail_loc in "0x80000169" "0x8000016a"; do
+               echo "Begin 100 tests with fail_loc=$fail_loc"
+               printf "Progress: "
+               for i in {1..100}; do
+                       printf "*"
+                       msg=$(sub_test_43k "$fail_loc") ||
+                               { echo; error "iter=$i : $msg"; }
+               done
+               echo
+       done
+
+       #Clean
+       reset_fail_loc
+
+       return 0
+}
+run_test 43k "unlink vs create"
+
 # test 44: rename tgt and blocking operations
 test_44a() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2   0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mkdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; date;error "mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail"
        date
@@ -2142,12 +2349,14 @@ test_44b() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "create must fail"
        rm -rf $DIR/$tfile*
@@ -2160,12 +2369,14 @@ test_44c() {
        touch $DIR1/$tfile-2
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile-3 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "link must fail"
        rm -rf $DIR/$tfile*
@@ -2177,12 +2388,14 @@ test_44d() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        rm $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "unlink must succeed"
        rm -rf $DIR/$tfile*
@@ -2196,12 +2409,14 @@ test_44e() {
        touch $DIR1/$tfile-2
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile-3 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -2214,12 +2429,14 @@ test_44f() {
        touch $DIR1/$tfile-2
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-3 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -2231,12 +2448,14 @@ test_44g() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed"
        rm -rf $DIR/$tfile*
@@ -2248,12 +2467,14 @@ test_44h() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        ls -lia $DIR2/ > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; }
        wait $PID2
        rm -rf $DIR/$tfile*
@@ -2267,12 +2488,14 @@ test_44i() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2   0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $LFS mkdir -i 1 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                error "remote mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "remote mkdir must fail"
@@ -2297,12 +2520,14 @@ test_45b() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "create must succeed"
        rm -rf $DIR/$tfile*
@@ -2315,12 +2540,14 @@ test_45c() {
        touch $DIR1/$tfile
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile-3 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "link must succeed"
        rm -rf $DIR/$tfile*
@@ -2332,12 +2559,14 @@ test_45d() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        rm $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "unlink must fail"
        rm -rf $DIR/$tfile*
@@ -2350,12 +2579,14 @@ test_45e() {
        touch $DIR1/$tfile
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile-3 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -2367,12 +2598,14 @@ test_45f() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-3 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "rename must fail"
        rm -rf $DIR/$tfile*
@@ -2384,12 +2617,14 @@ test_45g() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "stat must fail"
        rm -rf $DIR/$tfile*
@@ -2401,11 +2636,13 @@ test_45h() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        ls -lia $DIR2/ > /dev/null &
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; }
        wait $PID2
        rm -rf $DIR/$tfile*
@@ -2418,12 +2655,14 @@ test_45i() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        $LFS mkdir -i 1 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                error "create remote dir isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "create remote dir must succeed"
@@ -2432,17 +2671,73 @@ test_45i() {
 }
 run_test 45i "pdirops: rename src vs remote mkdir"
 
+sub_test_45j() {
+       local PID1 PID2
+       local fail_loc="$1"
+       local ret=0
+
+       # We test in a sparate directory to be able to unblock server thread in
+       # cfs_race if LCK_PW is taken on the parent by mdt_reint_rename.
+       test_mkdir $DIR2/$tdir
+       echo file1 > $DIR2/$tdir/$tfile
+       echo file2 > $DIR2/$tdir/$tfile-2
+
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=${fail_loc} || true" &>/dev/null
+
+       cat $DIR1/$tdir/$tfile >/dev/null &
+       PID1=$!
+       pdo_sched
+       mrename $DIR2/$tdir/$tfile-2 $DIR2/$tdir/$tfile > /dev/null &
+       PID2=$!
+
+       wait $PID1 ||
+               { ret=$?; echo -n "cat $tfile should succeed (err=$ret); "; }
+       wait $PID2 ||
+               { ret=$?; \
+               echo -n "mrename $tfile-2 to $tfile failed (err=$ret);"; }
+
+       #Clean
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x0 || true" &>/dev/null
+       rm -rf $DIR/$tdir
+
+       return $ret
+}
+
+test_45j() {
+       [[ $MDS1_VERSION -le $(version_code 2.13.56) ]] ||
+               skip "Need MDS version newer than 2.13.56"
+       local msg fail_loc
+
+#define OBD_FAIL_ONCE|OBD_FAIL_MDS_REINT_OPEN         0x169
+#define OBD_FAIL_ONCE|OBD_FAIL_MDS_REINT_OPEN2        0x16a
+       for fail_loc in "0x80000169" "0x8000016a"; do
+               echo "Begin 100 tests with fail_loc=$fail_loc"
+               printf "Progress: "
+               for i in {1..100}; do
+                       printf "*"
+                       msg=$(sub_test_45j "$fail_loc") ||
+                               { echo; error "iter=$i : $msg"; }
+               done
+               echo
+       done
+}
+run_test 45j "read vs rename =============="
+
 # test 46: link and blocking operations
 test_46a() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mkdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail"
        rm -rf $DIR/$tfile*
@@ -2454,12 +2749,14 @@ test_46b() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "create must fail"
        rm -rf $DIR/$tfile*
@@ -2471,12 +2768,14 @@ test_46c() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "link must fail"
        rm -rf $DIR/$tfile*
@@ -2488,12 +2787,14 @@ test_46d() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        rm $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "unlink must succeed"
        rm -rf $DIR/$tfile*
@@ -2506,12 +2807,14 @@ test_46e() {
        touch $DIR1/$tfile-2
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile-3 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -2524,12 +2827,14 @@ test_46f() {
        touch $DIR1/$tfile-2
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-3 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -2541,12 +2846,14 @@ test_46g() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed"
        rm -rf $DIR/$tfile*
@@ -2558,12 +2865,14 @@ test_46h() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        ls -lia $DIR2/ > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; }
        wait $PID2
        rm -rf $DIR/$tfile*
@@ -2576,12 +2885,14 @@ test_46i() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $LFS mkdir -i 1 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                error "remote mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "remote mkdir must fail"
@@ -2595,12 +2906,14 @@ test_47a() {
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
        [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
        pdo_lru_clear
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mkdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail"
        rm -rf $DIR/$tfile*
@@ -2612,13 +2925,15 @@ test_47b() {
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
        [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
        pdo_lru_clear
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        sleep 1 # please do not remove this sleep, see LU-10754
        multiop $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                        error "create isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "create must fail"
@@ -2632,12 +2947,14 @@ test_47c() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "link must fail"
        rm -rf $DIR/$tfile*
@@ -2649,12 +2966,14 @@ test_47d() {
        [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        rmdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                        error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rmdir must succeed"
@@ -2668,12 +2987,14 @@ test_47e() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv -T $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                        error "rename isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "rename must fail"
@@ -2686,12 +3007,14 @@ test_47f() {
        [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-2 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                        error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
@@ -2706,12 +3029,14 @@ test_47g() {
        sync_all_data
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                        error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed"
@@ -2772,24 +3097,27 @@ test_51b() {
 
        local tmpfile=`mktemp`
 
-       # create an empty file
-       $MCREATE $DIR1/$tfile || error "mcreate $DIR1/$tfile failed"
+       $LFS setstripe -E 1M -c 1 -E -1 --extension-size 64M $DIR1/$tfile ||
+               error "Create $DIR1/$tfile failed"
+
+       dd if=/dev/zero of=$DIR1/$tfile bs=1k count=1 conv=notrunc ||
+               error "dd $DIR1/$tfile failed"
 
        # delay glimpse so that layout has changed when glimpse finish
 #define OBD_FAIL_GLIMPSE_DELAY 0x1404
-       $LCTL set_param fail_loc=0x1404
+       $LCTL set_param fail_loc=0x1404 fail_val=4
        stat -c %s $DIR2/$tfile |tee $tmpfile &
        local pid=$!
-       sleep 1
+       sleep 0.2
 
-       # create layout of testing file
-       dd if=/dev/zero of=$DIR1/$tfile bs=1k count=1 conv=notrunc >/dev/null ||
+       # extend layout of testing file
+       dd if=/dev/zero of=$DIR1/$tfile bs=1M count=1 seek=2 conv=notrunc ||
                error "dd $DIR1/$tfile failed"
 
        wait $pid
        local fsize=$(cat $tmpfile)
 
-       [ x$fsize = x1024 ] || error "file size is $fsize, should be 1024"
+       [ x$fsize = x3145728 ] || error "file size is $fsize, should be 3145728"
 
        rm -f $DIR1/$tfile $tmpfile
 }
@@ -2856,6 +3184,27 @@ test_51d() {
 }
 run_test 51d "layout lock: losing layout lock should clean up memory map region"
 
+test_51e() {
+       local pid
+
+       $MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:eW_E+eUc &
+       pid=$!
+       sleep 1
+
+       $LFS getstripe $DIR2/$tfile
+       kill -USR1 $pid
+       wait $pid || error "multiop failed"
+
+       $MULTIOP $DIR/$tfile oO_RDONLY:eR_E+eUc &
+       pid=$!
+       sleep 1
+
+       $LFS getstripe $DIR2/$tfile
+       kill -USR1 $pid
+       wait $pid || error "multiop failed"
+}
+run_test 51e "lfs getstripe does not break leases, part 2"
+
 test_54_part1()
 {
        echo "==> rename vs getattr vs setxattr should not deadlock"
@@ -4339,6 +4688,52 @@ test_81b() {
 }
 run_test 81b "rename under striped directory doesn't deadlock"
 
+test_81c() {
+       [ $MDSCOUNT -lt 4 ] && skip_env "needs >= 4 MDTs"
+       [ $MDS1_VERSION -lt $(version_code 2.13.52) ] &&
+               skip "Need MDS version at least 2.13.52"
+
+       # source is local, source parent is remote
+       $LFS mkdir -i 0 $DIR1/${tdir}_src || error "mkdir ${tdir}_src"
+       $LFS mkdir -i 1 $DIR1/${tdir}_tgt || error "mkdir ${tdir}_tgt"
+       $LFS mkdir -i 3 $DIR1/${tdir}_src/sub || error "mkdir sub"
+       $LFS mkdir -i 3 $DIR1/${tdir}_tgt/sub || error "mkdir sub"
+       stat $DIR2/${tdir}_src/sub || error "stat sub failed"
+       mv $DIR1/${tdir}_src/sub $DIR1/${tdir}_tgt/ || error "mv failed"
+       [ -f $DIR2/${tdir}_src/sub ] && error "sub should be gone"
+       rm -rf $DIR1/${tdir}_src $DIR1/${tdir}_tgt
+
+       # source is remote, source parent is local
+       $LFS mkdir -i 3 $DIR1/${tdir}_src || error "mkdir ${tdir}_src"
+       $LFS mkdir -i 1 $DIR1/${tdir}_tgt || error "mkdir ${tdir}_tgt"
+       $LFS mkdir -i 0 $DIR1/${tdir}_src/sub || error "mkdir sub"
+       $LFS mkdir -i 3 $DIR1/${tdir}_tgt/sub || error "mkdir sub"
+       stat $DIR2/${tdir}_src/sub || error "stat sub failed"
+       mv $DIR1/${tdir}_src/sub $DIR1/${tdir}_tgt/ || error "mv failed"
+       [ -f $DIR2/${tdir}_src/sub ] && error "sub should be gone"
+       rm -rf $DIR1/${tdir}_src $DIR1/${tdir}_tgt
+
+       # source and source parent are remote
+       $LFS mkdir -i 0 $DIR1/${tdir}_src || error "mkdir ${tdir}_src"
+       $LFS mkdir -i 1 $DIR1/${tdir}_tgt || error "mkdir ${tdir}_tgt"
+       mkdir $DIR1/${tdir}_src/sub || error "mkdir sub"
+       $LFS mkdir -i 3 $DIR1/${tdir}_tgt/sub || error "mkdir sub"
+       stat $DIR2/${tdir}_src/sub || error "stat sub failed"
+       mv $DIR1/${tdir}_src/sub $DIR1/${tdir}_tgt/ || error "mv failed"
+       [ -f $DIR2/${tdir}_src/sub ] && error "sub should be gone"
+       rm -rf $DIR1/${tdir}_src $DIR1/${tdir}_tgt
+
+       # source and source parent are remote, and source is remote object
+       $LFS mkdir -i 0 $DIR1/${tdir}_src || error "mkdir ${tdir}_src"
+       $LFS mkdir -i 1 $DIR1/${tdir}_tgt || error "mkdir ${tdir}_tgt"
+       $LFS mkdir -i 2 $DIR1/${tdir}_src/sub || error "mkdir sub"
+       $LFS mkdir -i 3 $DIR1/${tdir}_tgt/sub || error "mkdir sub"
+       stat $DIR2/${tdir}_src/sub || error "stat sub failed"
+       mv $DIR1/${tdir}_src/sub $DIR1/${tdir}_tgt/ || error "mv failed"
+       [ -f $DIR2/${tdir}_src/sub ] && error "sub should be gone" || true
+}
+run_test 81c "rename revoke LOOKUP lock for remote object"
+
 test_82() {
        [[ "$MDS1_VERSION" -gt $(version_code 2.6.91) ]] ||
                skip "Need MDS version at least 2.6.92"
@@ -4887,7 +5282,7 @@ check_mdt_xtimes()
        local mdt_xtimes=($(get_mdt_xtimes $mdtdev))
 
        echo "STAT a|m|ctime ${xtimes[*]}"
-       echo "MDT a|m|ctime ${xtimes[*]}"
+       echo "MDT a|m|ctime ${mdt_xtimes[*]}"
        [[ ${xtimes[0]} == ${mdt_xtimes[0]} ]] ||
                error "$DIR/$tfile atime (${xtimes[0]}:${mdt_xtimes[0]}) diff"
        [[ ${xtimes[1]} == ${mdt_xtimes[1]} ]] ||
@@ -4965,6 +5360,252 @@ test_105() {
 }
 run_test 105 "Glimpse and lock cancel race"
 
+test_106a() {
+       [ "$mds1_FSTYPE" == "ldiskfs" ] && statx_supported ||
+               skip_env "Test only for ldiskfs and statx() supported"
+
+       local btime
+       local mdt_btime
+       local output
+       local mdtdev=$(mdsdevname ${SINGLEMDS//mds/})
+
+       dd if=/dev/zero of=$DIR/$tfile bs=1k count=1 conv=notrunc
+       btime=$($STATX -c %W $DIR/$tfile)
+       output=$(do_facet mds1 "$DEBUGFS -c -R 'stat ROOT/$tfile' $mdtdev")
+       echo $output
+       ((mdt_btime=$(awk -F ':' /crtime/'{ print $2 }' <<< "$output")))
+       [[ $btime == $mdt_btime ]] ||
+               error "$DIR/$tfile btime ($btime:$mdt_btime) diff"
+
+}
+run_test 106a "Verify the btime via statx()"
+
+test_106b() {
+       statx_supported || skip_env "statx() only test"
+
+       local rpcs_before
+       local rpcs_after
+
+       $LFS setstripe -c 1 $DIR/$tfile || error "$DIR/$tfile setstripe failed"
+       dd if=/dev/zero of=$DIR/$tfile bs=1k count=1 conv=notrunc
+       cancel_lru_locks $OSC
+       rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       $STATX $DIR/$tfile
+       rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       [ $rpcs_after -eq $((rpcs_before + 1)) ] ||
+               error "$STATX should send 1 glimpse RPC to $OSC"
+
+       cancel_lru_locks $OSC
+       rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       # %n: FILENAME; %i: STATX_INO; %A STATX_MODE; %h STATX_NLINK;
+       # %u: STATX_UID; %g: STATX_GID; %W STATX_BTIME; %X STATX_ATIME;
+       # %Z: STATX_CTIME
+       $STATX -c "%n %i %A %h %u %g %W %X %Z" $DIR/$tfile
+       rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       [ $rpcs_after -eq $rpcs_before ] ||
+               error "$STATX should not send glimpse RPCs to $OSC"
+
+       cancel_lru_locks $OSC
+       rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       $STATX --cached=always $DIR/$tfile
+       rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       [ $rpcs_after -eq $rpcs_before ] ||
+               error "$STATX should not send glimpse RPCs to $OSC"
+
+       cancel_lru_locks $OSC
+       rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       $STATX -c %Y $DIR/$tfile
+       rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       [ $rpcs_after -eq $((rpcs_before + 1)) ] ||
+               error "$STATX -c %Y should send 1 glimpse RPC to $OSC"
+
+       cancel_lru_locks $OSC
+       rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       $STATX -c %s $DIR/$tfile
+       rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       [ $rpcs_after -eq $((rpcs_before + 1)) ] ||
+               error "$STATX -c %s should send 1 glimpse RPC to $OSC"
+
+       cancel_lru_locks $OSC
+       rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       $STATX -c %b $DIR/$tfile
+       rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       [ $rpcs_after -eq $((rpcs_before + 1)) ] ||
+               error "$STATX -c %b should send 1 glimpse RPC to $OSC"
+}
+run_test 106b "Glimpse RPCs test for statx"
+
+test_106c() {
+       statx_supported || skip_env "statx() only test"
+
+       local mask
+
+       touch $DIR/$tfile
+       # Mask supported in stx_attributes by Lustre is
+       # STATX_ATTR_IMMUTABLE(0x10) | STATX_ATTR_APPEND(0x20) : (0x30).
+       mask=$($STATX -c %p $DIR/$tfile)
+       [[ $mask == "30" ]] ||
+               error "supported stx_attributes: got '$mask', expected '30'"
+       chattr +i $DIR/$tfile || error "chattr +i $DIR/$tfile failed"
+       mask=$($STATX -c %r $DIR/$tfile)
+       [[ $mask == "10" ]] ||
+               error "got immutable flags '$mask', expected '10'"
+       chattr -i $DIR/$tfile || error "chattr -i $DIR/$tfile failed"
+       mask=$($STATX -c %r $DIR/$tfile)
+       [[ $mask == "0" ]] || error "got flags '$mask', expected '0'"
+       chattr +a $DIR/$tfile || error "chattr +a $DIR/$tfile failed"
+       mask=$($STATX -c %r $DIR/$tfile)
+       [[ $mask == "20" ]] || error "got flags '$mask', expected '20'"
+       chattr -a $DIR/$tfile || error "chattr -a $DIR/$tfile failed"
+       mask=$($STATX -c %r $DIR/$tfile)
+       [[ $mask == "0" ]] || error "got flags '$mask', expected '0'"
+       chattr +ia $DIR/$tfile || error "chattr +ia $DIR/$tfile failed"
+       mask=$($STATX -c %r $DIR/$tfile)
+       [[ $mask == "30" ]] || error "got flags '$mask', expected '30'"
+       chattr -ia $DIR/$tfile || error "chattr -ia $DIR/$tfile failed"
+       mask=$($STATX -c %r $DIR/$tfile)
+       [[ $mask == "0" ]] || error "got flags '$mask', expected '0'"
+}
+run_test 106c "Verify statx attributes mask"
+
+test_107a() { # LU-1031
+       dd if=/dev/zero of=$DIR1/$tfile bs=1M count=10
+       local gid1=14091995
+       local gid2=16022000
+
+       $LFS getstripe $DIR1/$tfile
+
+       multiop_bg_pause $DIR1/$tfile OG${gid1}_g${gid1}c || return 1
+       local MULTIPID1=$!
+       multiop_bg_pause $DIR2/$tfile O_G${gid2}r10g${gid2}c || return 2
+       local MULTIPID2=$!
+       kill -USR1 $MULTIPID2
+       sleep 2
+       if [[ $(ps h -o comm -p $MULTIPID2) == "" ]]; then
+               error "First grouplock does not block second one"
+       else
+               echo "First grouplock blocks second one"
+       fi
+       kill -USR1 $MULTIPID1
+       wait $MULTIPID1
+       wait $MULTIPID2
+}
+run_test 107a "Basic grouplock conflict"
+
+test_107b() {
+       dd if=/dev/zero of=$DIR1/$tfile bs=1M count=10
+       local gid1=14091995
+       local gid2=16022000
+
+       $LFS getstripe $DIR1/$tfile
+
+       multiop_bg_pause $DIR1/$tfile OG${gid1}_g${gid1}c || return 1
+       local MULTIPID1=$!
+       multiop $DIR2/$tfile Or10c &
+       local MULTIPID2=$!
+       sleep 2
+
+       if [[ $(ps h -o comm -p $MULTIPID2) == "" ]]; then
+               error "Grouplock does not block IO"
+       else
+               echo "Grouplock blocks IO"
+       fi
+
+       multiop $DIR2/$tfile OG${gid2}_g${gid2}c &
+       local MULTIPID3=$!
+       sleep 2
+       if [[ $(ps h -o comm -p $MULTIPID3) == "" ]]; then
+               error "First grouplock does not block second one"
+       else
+               echo "First grouplock blocks second one"
+       fi
+
+       kill -USR1 $MULTIPID1
+       sleep 2
+
+       if [[ $(ps h -o comm -p $MULTIPID3) == "" ]]; then
+               error "Second grouplock thread disappeared"
+       fi
+
+       if [[ $(ps h -o comm -p $MULTIPID2) == "" ]]; then
+               error "Second grouplock does not block IO"
+       else
+               echo "Second grouplock blocks IO"
+       fi
+
+       kill -USR1 $MULTIPID3
+       wait $MULTIPID1
+       wait $MULTIPID2
+       wait $MULTIPID3
+}
+run_test 107b "Grouplock is added to the head of waiting list"
+
+test_108a() {
+       local offset
+
+       $LFS setstripe -E 1M -c 1 -E -1 $DIR1/$tfile ||
+               error "Create $DIR1/$tfile failed"
+
+       dd if=/dev/zero of=$DIR1/$tfile bs=10000 count=1 ||
+               error "dd $DIR1/$tfile failed"
+       offset=$(lseek_test -d 5000 $DIR2/$tfile)
+       [[ $offset == 5000 ]] || error "offset $offset != 5000"
+
+       $TRUNCATE $DIR1/$tfile 2000
+       offset=$(lseek_test -l 1000 $DIR2/$tfile)
+       [[ $offset == 2000 ]] || error "offset $offset != 2000"
+
+       #define OBD_FAIL_OSC_DELAY_IO 0x414
+       $LCTL set_param fail_val=4 fail_loc=0x80000414
+       dd if=/dev/zero of=$DIR1/$tfile count=1 bs=8M conv=notrunc oflag=dsync &
+       local pid=$!
+       sleep 2
+
+       offset=$(lseek_test -l 8000 $DIR2/$tfile)
+       wait $pid
+       [[ $offset == 8388608 ]] || error "offset $offset != 8388608"
+}
+run_test 108a "lseek: parallel updates"
+
+# LU-14110
+test_109() {
+       local i
+       local pid1 pid2
+
+       ! local_mode ||
+               skip "Clients need to be on different nodes than the servers"
+
+       umount_client $MOUNT
+       umount_client $MOUNT2
+
+       echo "Starting race between client mount instances (50 iterations):"
+       for i in {1..50}; do
+               log "Iteration $i"
+
+#define OBD_FAIL_ONCE|OBD_FAIL_LLITE_RACE_MOUNT        0x80001417
+               $LCTL set_param -n fail_loc=0x80001417
+
+               mount_client $MOUNT  & pid1=$!
+               mount_client $MOUNT2 & pid2=$!
+               wait $pid1 || error "Mount $MOUNT fails with $?"
+               wait $pid2 || error "Mount $MOUNT2 fails with $?"
+
+               umount_client $MOUNT  & pid1=$!
+               umount_client $MOUNT2 & pid2=$!
+               wait $pid1 || error "Umount $MOUNT fails with $?"
+               wait $pid2 || error "Umount $MOUNT2 fails with $?"
+
+               $LUSTRE_RMMOD || error "Fail to remove lustre modules"
+               load_modules
+               echo
+       done
+
+       mount_client $MOUNT
+       mount_client $MOUNT2
+}
+
+run_test 109 "Race with several mount instances on 1 node"
+
 log "cleanup: ======================================================"
 
 # kill and wait in each test only guarentee script finish, but command in script