LU-10059 tests: sanityn 32a error messages

[fs/lustre-release.git] / lustre / tests / sanityn.sh

diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh
index fe85b4a..d09a62f 100755 (executable)
--- a/lustre/tests/sanityn.sh
+++ b/lustre/tests/sanityn.sh
@@ -396,8 +396,8 @@ test_16a() {
        rm -f $file1
 
        $LFS setstripe -c -1 $file1 # b=10919
-       fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 $file1 $file2 \
-               || error "fsx failed"
+       fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 $file1 $file2 ||
+               error "fsx failed"
        rm -f $file1
 
        # O_DIRECT reads and writes must be aligned to the device block size.
@@ -423,8 +423,8 @@ test_16b() {
        lfs setstripe -c -1 $file1 # b=10919
        # -o is set to 8192 because writes < 1 page and between 1 and 2 pages
        # create a mix of tiny writes & normal writes
-       fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -o 8192 -S 0 $file1 \
-       $file2 || error "fsx with tiny write failed."
+       fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -o 8192 -S 0 \
+               $file1 $file2 || error "fsx with tiny write failed."
 }
 run_test 16b "$FSXNUM iterations of dual-mount fsx at small size"
 
@@ -452,8 +452,8 @@ test_16c() {
        set_osd_param $list '' writethrough_cache_enable 0
 
        $LFS setstripe -c -1 $file1 # b=10919
-       fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 $file1 $file2 \
-               || error "fsx failed"
+       fsx -c 50 -p $FSXP -N $FSXNUM -l $((SIZE * 256)) -S 0 $file1 $file2 ||
+               error "fsx failed"
        rm -f $file1
 
        set_osd_param $list '' read_cache_enable 1
@@ -503,6 +503,26 @@ test_16d() {
 }
 run_test 16d "Verify DIO and buffer IO with two clients"
 
+test_16e() { # LU-13227
+       local file1=$DIR1/$tfile
+       local file2=$DIR2/$tfile
+
+       # client1 write 10M data
+       dd if=/dev/zero of=$file1 bs=1M count=10
+       # drop locks
+       cancel_lru_locks osc > /dev/null
+       # use lockahead to generate one PW lock to keep LVB loaded.
+       $LFS ladvise -a lockahead --start 0 --length 1M \
+               --mode WRITE $file1
+       # direct write to extend file size on client2
+       dd if=/dev/zero of=$file2 bs=1M seek=20 count=1 \
+               oflag=direct conv=notrunc
+       local filesize=$(stat -c %s $file2)
+       [ "$filesize" -eq 22020096 ] ||
+               error "expected filesize 22020096 got $filesize"
+       rm -f $file1
+}
+run_test 16e "Verify size consistency for O_DIRECT write"
 
 test_17() { # bug 3513, 3667
        remote_ost_nodsh && skip "remote OST with nodsh" && return
@@ -530,7 +550,9 @@ test_18() {
                 excepts="$excepts -e $(($(printf %d \'$idx)-96))"
         done
 
-       $LUSTRE/tests/mmap_sanity -d $MOUNT1 -m $MOUNT2 $excepts
+       excepts="$excepts -e 7 -e 8 -e 9"
+       $LUSTRE/tests/mmap_sanity -d $MOUNT1 -m $MOUNT2 $excepts ||
+               error "mmap_sanity test failed"
        sync; sleep 1; sync
 }
 run_test 18 "mmap sanity check ================================="
@@ -566,16 +588,16 @@ test_19() { # bug3811
 run_test 19 "test concurrent uncached read races ==============="
 
 test_20() {
-       test_mkdir $DIR1/d20
-       cancel_lru_locks $OSC
-       CNT=$((`lctl get_param -n llite.*.dump_page_cache | wc -l`))
-       $MULTIOP $DIR1/f20 Ow8190c
-       $MULTIOP $DIR2/f20 Oz8194w8190c
-       $MULTIOP $DIR1/f20 Oz0r8190c
-       cancel_lru_locks $OSC
-       CNTD=$((`lctl get_param -n llite.*.dump_page_cache | wc -l` - $CNT))
-       [ $CNTD -gt 0 ] && \
-           error $CNTD" page left in cache after lock cancel" || true
+       test_mkdir $DIR1/$tdir
+       cancel_lru_locks
+       CNT=$($LCTL get_param -n llite.*.dump_page_cache | wc -l)
+       $MULTIOP $DIR1/$tdir/$tfile Ow8190c
+       $MULTIOP $DIR2/$tdir/$tfile Oz8194w8190c
+       $MULTIOP $DIR1/$tdir/$tfile Oz0r8190c
+       cancel_lru_locks
+       CNT2=$($LCTL get_param -n llite.*.dump_page_cache | wc -l)
+       [[ $CNT2 == $CNT ]] ||
+               error $((CNT2 - CNT))" page left in cache after lock cancel"
 }
 run_test 20 "test extra readahead page left in cache ===="
 
@@ -731,7 +753,7 @@ test_27() {
        lctl clear
        dd if=/dev/zero of=$DIR2/$tfile bs=$((4096+4))k conv=notrunc count=4 seek=3 &
        DD2_PID=$!
-       usleep 50
+       sleep 0.5
        log "dd 1 started"
 
        dd if=/dev/zero of=$DIR1/$tfile bs=$((16384-1024))k conv=notrunc count=1 seek=4 &
@@ -855,13 +877,15 @@ test_32a() { # bug 11270
 
        log "checking cached lockless truncate"
        $TRUNCATE $DIR1/$tfile 8000000
-       $CHECKSTAT -s 8000000 $DIR2/$tfile || error "wrong file size"
+       $CHECKSTAT -s 8000000 $DIR2/$tfile ||
+               error "cached truncate - wrong file size"
        [ $(calc_stats $OSC.*.${OSC}_stats lockless_truncate) -ne 0 ] ||
                error "cached truncate isn't lockless"
 
        log "checking not cached lockless truncate"
        $TRUNCATE $DIR2/$tfile 5000000
-       $CHECKSTAT -s 5000000 $DIR1/$tfile || error "wrong file size"
+       $CHECKSTAT -s 5000000 $DIR1/$tfile ||
+               error "not cached truncate - wrong file size"
        [ $(calc_stats $OSC.*.${OSC}_stats lockless_truncate) -ne 0 ] ||
                error "not cached truncate isn't lockless"
 
@@ -869,7 +893,8 @@ test_32a() { # bug 11270
        enable_lockless_truncate 0
        clear_stats $OSC.*.${OSC}_stats
        $TRUNCATE $DIR2/$tfile 3000000
-       $CHECKSTAT -s 3000000 $DIR1/$tfile || error "wrong file size"
+       $CHECKSTAT -s 3000000 $DIR1/$tfile ||
+               error "lockless truncate disabled - wrong file size"
        [ $(calc_stats $OSC.*.${OSC}_stats lockless_truncate) -eq 0 ] ||
                error "lockless truncate disabling failed"
        rm -f $DIR1/$tfile
@@ -1064,6 +1089,10 @@ test_33c() {
        [ "$MDS1_VERSION" -lt $(version_code 2.7.63) ] &&
                skip "DNE CoS not supported"
 
+       # LU-13522
+       stop mds1
+       start mds1 $(mdsdevname 1) $MDS_MOUNT_OPTS || error "start mds1 failed"
+
        local sync_count
 
        mkdir $DIR/$tdir
@@ -1536,7 +1565,8 @@ test_40a() {
        remote_mds_nodsh && skip "remote MDS with nodsh" && return
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        touch $DIR2
        mkdir $DIR1/$tfile &
        PID1=$!; pdo_sched
@@ -1556,7 +1586,8 @@ test_40a() {
 
        #  all operations above shouldn't wait the first one
        check_pdo_conflict $PID1 || error "parallel operation is blocked"
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        wait $PID1
        rm -rf $DIR/$tfile*
        return 0
@@ -1567,7 +1598,8 @@ test_40b() {
        remote_mds_nodsh && skip "remote MDS with nodsh" && return
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        touch $DIR1/$tfile &
        PID1=$!; pdo_sched
        # open|create
@@ -1587,7 +1619,8 @@ test_40b() {
        # all operations above shouldn't wait the first one
 
         check_pdo_conflict $PID1 || error "parallel operation is blocked"
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        wait $PID1
        rm -rf $DIR/$tfile*
        return 0
@@ -1599,7 +1632,8 @@ test_40c() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile $DIR1/$tfile-0 &
        PID1=$!; pdo_sched
        # open|create
@@ -1619,7 +1653,8 @@ test_40c() {
 
         # all operations above shouldn't wait the first one
        check_pdo_conflict $PID1 || error "parallel operation is blocked"
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        wait $PID1
        rm -rf $DIR/$tfile*
        return 0
@@ -1631,7 +1666,8 @@ test_40d() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$!; pdo_sched
        # open|create
@@ -1651,7 +1687,8 @@ test_40d() {
 
        # all operations above shouldn't wait the first one
        check_pdo_conflict $PID1 || error "parallel operation is blocked"
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        wait $PID1
        return 0
 }
@@ -1662,7 +1699,8 @@ test_40e() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-0 &
        PID1=$!; pdo_sched
        # open|create
@@ -1680,7 +1718,8 @@ test_40e() {
 
        # all operations above shouldn't wait the first one
        check_pdo_conflict $PID1 || error "parallel operation is blocked"
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        wait $PID1
        rm -rf $DIR/$tfile*
        return 0
@@ -1691,12 +1730,14 @@ run_test 40e "pdirops: rename and others =============="
 test_41a() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        mkdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; echo "mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail"
        rm -rf $DIR/$tfile*
@@ -1707,12 +1748,14 @@ run_test 41a "pdirops: create vs mkdir =============="
 test_41b() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "create must fail"
        rm -rf $DIR/$tfile*
@@ -1724,12 +1767,14 @@ test_41c() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "link must fail"
        rm -rf $DIR/$tfile*
@@ -1740,12 +1785,14 @@ run_test 41c "pdirops: create vs link =============="
 test_41d() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        rm $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "unlink must succeed"
        rm -rf $DIR/$tfile*
@@ -1757,12 +1804,14 @@ test_41e() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -1773,12 +1822,14 @@ run_test 41e "pdirops: create and rename (tgt) =============="
 test_41f() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-2 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -1789,12 +1840,14 @@ run_test 41f "pdirops: create and rename (src) =============="
 test_41g() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed"
        rm -rf $DIR/$tfile*
@@ -1805,12 +1858,14 @@ run_test 41g "pdirops: create vs getattr =============="
 test_41h() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
        PID1=$! ; pdo_sched
        ls -lia $DIR2/ > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; }
        wait $PID2
        rm -rf $DIR/$tfile*
@@ -1822,12 +1877,14 @@ run_test 41h "pdirops: create vs readdir =============="
 test_42a() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mkdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail"
        rm -rf $DIR/$tfile*
@@ -1838,12 +1895,14 @@ run_test 42a "pdirops: mkdir vs mkdir =============="
 test_42b() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "create must fail"
        rm -rf $DIR/$tfile*
@@ -1855,12 +1914,14 @@ test_42c() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "link must fail"
        rm -rf $DIR/$tfile*
@@ -1871,12 +1932,14 @@ run_test 42c "pdirops: mkdir vs link =============="
 test_42d() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        rmdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "unlink must succeed"
        rm -rf $DIR/$tfile*
@@ -1888,12 +1951,14 @@ test_42e() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv -T $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "rename must fail"
        rm -rf $DIR/$tfile*
@@ -1904,12 +1969,14 @@ run_test 42e "pdirops: mkdir and rename (tgt) =============="
 test_42f() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-2 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -1920,12 +1987,14 @@ run_test 42f "pdirops: mkdir and rename (src) =============="
 test_42g() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed"
        rm -rf $DIR/$tfile*
@@ -1936,12 +2005,14 @@ run_test 42g "pdirops: mkdir vs getattr =============="
 test_42h() {
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mkdir $DIR1/$tfile &
        PID1=$! ; pdo_sched
        ls -lia $DIR2/ > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; }
        wait $PID2
        rm -rf $DIR/$tfile*
@@ -1963,12 +2034,14 @@ test_43b() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "create must succeed"
        rm -rf $DIR/$tfile*
@@ -1981,12 +2054,14 @@ test_43c() {
        touch $DIR1/$tfile
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "link must succeed"
        rm -rf $DIR/$tfile*
@@ -1998,12 +2073,14 @@ test_43d() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        rm $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "unlink must fail"
        rm -rf $DIR/$tfile*
@@ -2016,12 +2093,14 @@ test_43e() {
        touch $DIR1/$tfile
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv -u $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -2033,12 +2112,14 @@ test_43f() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-2 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "rename must fail"
        rm -rf $DIR/$tfile*
@@ -2050,12 +2131,14 @@ test_43g() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "stat must fail"
        rm -rf $DIR/$tfile*
@@ -2067,12 +2150,14 @@ test_43h() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        ls -lia $DIR2/ > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; }
        wait $PID2
        rm -rf $DIR/$tfile*
@@ -2085,12 +2170,14 @@ test_43i() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        rm $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $LFS mkdir -i 1 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 &&
                { wait $PID1; error "remote mkdir isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "remote mkdir must succeed"
@@ -2105,7 +2192,9 @@ test_43j() {
 
        for i in {1..100}; do
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_CREATE_RACE         0x167
-               do_facet $SINGLEMDS lctl set_param fail_loc=0x80000167
+               do_nodes $(comma_list $(mdts_nodes)) \
+                       "lctl set_param -n fail_loc=0x80000167 2>/dev/null ||
+                               true"
                OK=0
                mkdir $DIR1/$tdir &
                PID1=$!
@@ -2124,12 +2213,14 @@ test_44a() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2   0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mkdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; date;error "mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail"
        date
@@ -2142,12 +2233,14 @@ test_44b() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "create must fail"
        rm -rf $DIR/$tfile*
@@ -2160,12 +2253,14 @@ test_44c() {
        touch $DIR1/$tfile-2
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile-3 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "link must fail"
        rm -rf $DIR/$tfile*
@@ -2177,12 +2272,14 @@ test_44d() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        rm $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "unlink must succeed"
        rm -rf $DIR/$tfile*
@@ -2196,12 +2293,14 @@ test_44e() {
        touch $DIR1/$tfile-2
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile-3 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -2214,12 +2313,14 @@ test_44f() {
        touch $DIR1/$tfile-2
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-3 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -2231,12 +2332,14 @@ test_44g() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed"
        rm -rf $DIR/$tfile*
@@ -2248,12 +2351,14 @@ test_44h() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2    0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        ls -lia $DIR2/ > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; }
        wait $PID2
        rm -rf $DIR/$tfile*
@@ -2267,12 +2372,14 @@ test_44i() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK2   0x146
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000146
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000146 2>/dev/null || true"
        mv $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $LFS mkdir -i 1 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                error "remote mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "remote mkdir must fail"
@@ -2297,12 +2404,14 @@ test_45b() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "create must succeed"
        rm -rf $DIR/$tfile*
@@ -2315,12 +2424,14 @@ test_45c() {
        touch $DIR1/$tfile
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile-3 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "link must succeed"
        rm -rf $DIR/$tfile*
@@ -2332,12 +2443,14 @@ test_45d() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        rm $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "unlink must fail"
        rm -rf $DIR/$tfile*
@@ -2350,12 +2463,14 @@ test_45e() {
        touch $DIR1/$tfile
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile-3 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -2367,12 +2482,14 @@ test_45f() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-3 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "rename must fail"
        rm -rf $DIR/$tfile*
@@ -2384,12 +2501,14 @@ test_45g() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "stat must fail"
        rm -rf $DIR/$tfile*
@@ -2401,11 +2520,13 @@ test_45h() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        ls -lia $DIR2/ > /dev/null &
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; }
        wait $PID2
        rm -rf $DIR/$tfile*
@@ -2418,12 +2539,14 @@ test_45i() {
        pdo_lru_clear
        touch $DIR1/$tfile
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        mv $DIR1/$tfile $DIR1/$tfile-2 &
        PID1=$! ; pdo_sched
        $LFS mkdir -i 1 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                error "create remote dir isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "create remote dir must succeed"
@@ -2437,12 +2560,14 @@ test_46a() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mkdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail"
        rm -rf $DIR/$tfile*
@@ -2454,12 +2579,14 @@ test_46b() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "create must fail"
        rm -rf $DIR/$tfile*
@@ -2471,12 +2598,14 @@ test_46c() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "link must fail"
        rm -rf $DIR/$tfile*
@@ -2488,12 +2617,14 @@ test_46d() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        rm $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "unlink must succeed"
        rm -rf $DIR/$tfile*
@@ -2506,12 +2637,14 @@ test_46e() {
        touch $DIR1/$tfile-2
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile-3 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -2524,12 +2657,14 @@ test_46f() {
        touch $DIR1/$tfile-2
        touch $DIR1/$tfile-3
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-3 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
        rm -rf $DIR/$tfile*
@@ -2541,12 +2676,14 @@ test_46g() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed"
        rm -rf $DIR/$tfile*
@@ -2558,12 +2695,14 @@ test_46h() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        ls -lia $DIR2/ > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "readdir isn't blocked"; }
        wait $PID2
        rm -rf $DIR/$tfile*
@@ -2576,12 +2715,14 @@ test_46i() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        link $DIR1/$tfile-2 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        $LFS mkdir -i 1 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                error "remote mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "remote mkdir must fail"
@@ -2595,12 +2736,14 @@ test_47a() {
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
        [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
        pdo_lru_clear
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mkdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "mkdir isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "mkdir must fail"
        rm -rf $DIR/$tfile*
@@ -2612,13 +2755,15 @@ test_47b() {
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
        [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
        pdo_lru_clear
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        sleep 1 # please do not remove this sleep, see LU-10754
        multiop $DIR2/$tfile oO_CREAT:O_EXCL:c &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                        error "create isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "create must fail"
@@ -2632,12 +2777,14 @@ test_47c() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        link $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1; error "link isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "link must fail"
        rm -rf $DIR/$tfile*
@@ -2649,12 +2796,14 @@ test_47d() {
        [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        rmdir $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                        error "unlink isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rmdir must succeed"
@@ -2668,12 +2817,14 @@ test_47e() {
        pdo_lru_clear
        touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv -T $DIR2/$tfile-2 $DIR2/$tfile &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                        error "rename isn't blocked"; }
        wait $PID2 ; [ $? -ne 0 ] || error "rename must fail"
@@ -2686,12 +2837,14 @@ test_47f() {
        [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        mv $DIR2/$tfile $DIR2/$tfile-2 &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                        error "rename isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "rename must succeed"
@@ -2706,12 +2859,14 @@ test_47g() {
        sync_all_data
        pdo_lru_clear
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
-       do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0x80000145 2>/dev/null || true"
        $LFS mkdir -i 1 $DIR1/$tfile &
        PID1=$! ; pdo_sched
        stat $DIR2/$tfile > /dev/null &
        PID2=$! ; pdo_sched
-       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "lctl set_param -n fail_loc=0 2>/dev/null || true"
        check_pdo_conflict $PID1 && { wait $PID1;
                                        error "getattr isn't blocked"; }
        wait $PID2 ; [ $? -eq 0 ] || error "stat must succeed"
@@ -2772,24 +2927,27 @@ test_51b() {
 
        local tmpfile=`mktemp`
 
-       # create an empty file
-       $MCREATE $DIR1/$tfile || error "mcreate $DIR1/$tfile failed"
+       $LFS setstripe -E 1M -c 1 -E -1 --extension-size 64M $DIR1/$tfile ||
+               error "Create $DIR1/$tfile failed"
+
+       dd if=/dev/zero of=$DIR1/$tfile bs=1k count=1 conv=notrunc ||
+               error "dd $DIR1/$tfile failed"
 
        # delay glimpse so that layout has changed when glimpse finish
 #define OBD_FAIL_GLIMPSE_DELAY 0x1404
-       $LCTL set_param fail_loc=0x1404
+       $LCTL set_param fail_loc=0x1404 fail_val=4
        stat -c %s $DIR2/$tfile |tee $tmpfile &
        local pid=$!
-       sleep 1
+       sleep 0.2
 
-       # create layout of testing file
-       dd if=/dev/zero of=$DIR1/$tfile bs=1k count=1 conv=notrunc >/dev/null ||
+       # extend layout of testing file
+       dd if=/dev/zero of=$DIR1/$tfile bs=1M count=1 seek=2 conv=notrunc ||
                error "dd $DIR1/$tfile failed"
 
        wait $pid
        local fsize=$(cat $tmpfile)
 
-       [ x$fsize = x1024 ] || error "file size is $fsize, should be 1024"
+       [ x$fsize = x3145728 ] || error "file size is $fsize, should be 3145728"
 
        rm -f $DIR1/$tfile $tmpfile
 }
@@ -2856,6 +3014,27 @@ test_51d() {
 }
 run_test 51d "layout lock: losing layout lock should clean up memory map region"
 
+test_51e() {
+       local pid
+
+       $MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:eW_E+eUc &
+       pid=$!
+       sleep 1
+
+       $LFS getstripe $DIR2/$tfile
+       kill -USR1 $pid
+       wait $pid || error "multiop failed"
+
+       $MULTIOP $DIR/$tfile oO_RDONLY:eR_E+eUc &
+       pid=$!
+       sleep 1
+
+       $LFS getstripe $DIR2/$tfile
+       kill -USR1 $pid
+       wait $pid || error "multiop failed"
+}
+run_test 51e "lfs getstripe does not break leases, part 2"
+
 test_54_part1()
 {
        echo "==> rename vs getattr vs setxattr should not deadlock"
@@ -4339,6 +4518,52 @@ test_81b() {
 }
 run_test 81b "rename under striped directory doesn't deadlock"
 
+test_81c() {
+       [ $MDSCOUNT -lt 4 ] && skip_env "needs >= 4 MDTs"
+       [ $MDS1_VERSION -lt $(version_code 2.13.52) ] &&
+               skip "Need MDS version at least 2.13.52"
+
+       # source is local, source parent is remote
+       $LFS mkdir -i 0 $DIR1/${tdir}_src || error "mkdir ${tdir}_src"
+       $LFS mkdir -i 1 $DIR1/${tdir}_tgt || error "mkdir ${tdir}_tgt"
+       $LFS mkdir -i 3 $DIR1/${tdir}_src/sub || error "mkdir sub"
+       $LFS mkdir -i 3 $DIR1/${tdir}_tgt/sub || error "mkdir sub"
+       stat $DIR2/${tdir}_src/sub || error "stat sub failed"
+       mv $DIR1/${tdir}_src/sub $DIR1/${tdir}_tgt/ || error "mv failed"
+       [ -f $DIR2/${tdir}_src/sub ] && error "sub should be gone"
+       rm -rf $DIR1/${tdir}_src $DIR1/${tdir}_tgt
+
+       # source is remote, source parent is local
+       $LFS mkdir -i 3 $DIR1/${tdir}_src || error "mkdir ${tdir}_src"
+       $LFS mkdir -i 1 $DIR1/${tdir}_tgt || error "mkdir ${tdir}_tgt"
+       $LFS mkdir -i 0 $DIR1/${tdir}_src/sub || error "mkdir sub"
+       $LFS mkdir -i 3 $DIR1/${tdir}_tgt/sub || error "mkdir sub"
+       stat $DIR2/${tdir}_src/sub || error "stat sub failed"
+       mv $DIR1/${tdir}_src/sub $DIR1/${tdir}_tgt/ || error "mv failed"
+       [ -f $DIR2/${tdir}_src/sub ] && error "sub should be gone"
+       rm -rf $DIR1/${tdir}_src $DIR1/${tdir}_tgt
+
+       # source and source parent are remote
+       $LFS mkdir -i 0 $DIR1/${tdir}_src || error "mkdir ${tdir}_src"
+       $LFS mkdir -i 1 $DIR1/${tdir}_tgt || error "mkdir ${tdir}_tgt"
+       mkdir $DIR1/${tdir}_src/sub || error "mkdir sub"
+       $LFS mkdir -i 3 $DIR1/${tdir}_tgt/sub || error "mkdir sub"
+       stat $DIR2/${tdir}_src/sub || error "stat sub failed"
+       mv $DIR1/${tdir}_src/sub $DIR1/${tdir}_tgt/ || error "mv failed"
+       [ -f $DIR2/${tdir}_src/sub ] && error "sub should be gone"
+       rm -rf $DIR1/${tdir}_src $DIR1/${tdir}_tgt
+
+       # source and source parent are remote, and source is remote object
+       $LFS mkdir -i 0 $DIR1/${tdir}_src || error "mkdir ${tdir}_src"
+       $LFS mkdir -i 1 $DIR1/${tdir}_tgt || error "mkdir ${tdir}_tgt"
+       $LFS mkdir -i 2 $DIR1/${tdir}_src/sub || error "mkdir sub"
+       $LFS mkdir -i 3 $DIR1/${tdir}_tgt/sub || error "mkdir sub"
+       stat $DIR2/${tdir}_src/sub || error "stat sub failed"
+       mv $DIR1/${tdir}_src/sub $DIR1/${tdir}_tgt/ || error "mv failed"
+       [ -f $DIR2/${tdir}_src/sub ] && error "sub should be gone" || true
+}
+run_test 81c "rename revoke LOOKUP lock for remote object"
+
 test_82() {
        [[ "$MDS1_VERSION" -gt $(version_code 2.6.91) ]] ||
                skip "Need MDS version at least 2.6.92"
@@ -4887,7 +5112,7 @@ check_mdt_xtimes()
        local mdt_xtimes=($(get_mdt_xtimes $mdtdev))
 
        echo "STAT a|m|ctime ${xtimes[*]}"
-       echo "MDT a|m|ctime ${xtimes[*]}"
+       echo "MDT a|m|ctime ${mdt_xtimes[*]}"
        [[ ${xtimes[0]} == ${mdt_xtimes[0]} ]] ||
                error "$DIR/$tfile atime (${xtimes[0]}:${mdt_xtimes[0]}) diff"
        [[ ${xtimes[1]} == ${mdt_xtimes[1]} ]] ||
@@ -4965,6 +5190,213 @@ test_105() {
 }
 run_test 105 "Glimpse and lock cancel race"
 
+test_106a() {
+       [ "$mds1_FSTYPE" == "ldiskfs" ] && statx_supported ||
+               skip_env "Test only for ldiskfs and statx() supported"
+
+       local btime
+       local mdt_btime
+       local output
+       local mdtdev=$(mdsdevname ${SINGLEMDS//mds/})
+
+       dd if=/dev/zero of=$DIR/$tfile bs=1k count=1 conv=notrunc
+       btime=$($STATX -c %W $DIR/$tfile)
+       output=$(do_facet mds1 "$DEBUGFS -c -R 'stat ROOT/$tfile' $mdtdev")
+       echo $output
+       ((mdt_btime=$(awk -F ':' /crtime/'{ print $2 }' <<< "$output")))
+       [[ $btime == $mdt_btime ]] ||
+               error "$DIR/$tfile btime ($btime:$mdt_btime) diff"
+
+}
+run_test 106a "Verify the btime via statx()"
+
+test_106b() {
+       statx_supported || skip_env "statx() only test"
+
+       local rpcs_before
+       local rpcs_after
+
+       $LFS setstripe -c 1 $DIR/$tfile || error "$DIR/$tfile setstripe failed"
+       dd if=/dev/zero of=$DIR/$tfile bs=1k count=1 conv=notrunc
+       cancel_lru_locks $OSC
+       rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       $STATX $DIR/$tfile
+       rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       [ $rpcs_after -eq $((rpcs_before + 1)) ] ||
+               error "$STATX should send 1 glimpse RPC to $OSC"
+
+       cancel_lru_locks $OSC
+       rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       # %n: FILENAME; %i: STATX_INO; %A STATX_MODE; %h STATX_NLINK;
+       # %u: STATX_UID; %g: STATX_GID; %W STATX_BTIME; %X STATX_ATIME;
+       # %Z: STATX_CTIME
+       $STATX -c "%n %i %A %h %u %g %W %X %Z" $DIR/$tfile
+       rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       [ $rpcs_after -eq $rpcs_before ] ||
+               error "$STATX should not send glimpse RPCs to $OSC"
+
+       cancel_lru_locks $OSC
+       rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       $STATX --cached=always $DIR/$tfile
+       rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       [ $rpcs_after -eq $rpcs_before ] ||
+               error "$STATX should not send glimpse RPCs to $OSC"
+
+       cancel_lru_locks $OSC
+       rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       $STATX -c %Y $DIR/$tfile
+       rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       [ $rpcs_after -eq $((rpcs_before + 1)) ] ||
+               error "$STATX -c %Y should send 1 glimpse RPC to $OSC"
+
+       cancel_lru_locks $OSC
+       rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       $STATX -c %s $DIR/$tfile
+       rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       [ $rpcs_after -eq $((rpcs_before + 1)) ] ||
+               error "$STATX -c %s should send 1 glimpse RPC to $OSC"
+
+       cancel_lru_locks $OSC
+       rpcs_before=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       $STATX -c %b $DIR/$tfile
+       rpcs_after=$(calc_stats $OSC.*$OSC*.stats ldlm_glimpse_enqueue)
+       [ $rpcs_after -eq $((rpcs_before + 1)) ] ||
+               error "$STATX -c %b should send 1 glimpse RPC to $OSC"
+}
+run_test 106b "Glimpse RPCs test for statx"
+
+test_106c() {
+       statx_supported || skip_env "statx() only test"
+
+       local mask
+
+       touch $DIR/$tfile
+       # Mask supported in stx_attributes by Lustre is
+       # STATX_ATTR_IMMUTABLE(0x10) | STATX_ATTR_APPEND(0x20) : (0x30).
+       mask=$($STATX -c %p $DIR/$tfile)
+       [[ $mask == "30" ]] ||
+               error "supported stx_attributes: got '$mask', expected '30'"
+       chattr +i $DIR/$tfile || error "chattr +i $DIR/$tfile failed"
+       mask=$($STATX -c %r $DIR/$tfile)
+       [[ $mask == "10" ]] ||
+               error "got immutable flags '$mask', expected '10'"
+       chattr -i $DIR/$tfile || error "chattr -i $DIR/$tfile failed"
+       mask=$($STATX -c %r $DIR/$tfile)
+       [[ $mask == "0" ]] || error "got flags '$mask', expected '0'"
+       chattr +a $DIR/$tfile || error "chattr +a $DIR/$tfile failed"
+       mask=$($STATX -c %r $DIR/$tfile)
+       [[ $mask == "20" ]] || error "got flags '$mask', expected '20'"
+       chattr -a $DIR/$tfile || error "chattr -a $DIR/$tfile failed"
+       mask=$($STATX -c %r $DIR/$tfile)
+       [[ $mask == "0" ]] || error "got flags '$mask', expected '0'"
+       chattr +ia $DIR/$tfile || error "chattr +ia $DIR/$tfile failed"
+       mask=$($STATX -c %r $DIR/$tfile)
+       [[ $mask == "30" ]] || error "got flags '$mask', expected '30'"
+       chattr -ia $DIR/$tfile || error "chattr -ia $DIR/$tfile failed"
+       mask=$($STATX -c %r $DIR/$tfile)
+       [[ $mask == "0" ]] || error "got flags '$mask', expected '0'"
+}
+run_test 106c "Verify statx attributes mask"
+
+test_107a() { # LU-1031
+       dd if=/dev/zero of=$DIR1/$tfile bs=1M count=10
+       local gid1=14091995
+       local gid2=16022000
+
+       $LFS getstripe $DIR1/$tfile
+
+       multiop_bg_pause $DIR1/$tfile OG${gid1}_g${gid1}c || return 1
+       local MULTIPID1=$!
+       multiop_bg_pause $DIR2/$tfile O_G${gid2}r10g${gid2}c || return 2
+       local MULTIPID2=$!
+       kill -USR1 $MULTIPID2
+       sleep 2
+       if [[ $(ps h -o comm -p $MULTIPID2) == "" ]]; then
+               error "First grouplock does not block second one"
+       else
+               echo "First grouplock blocks second one"
+       fi
+       kill -USR1 $MULTIPID1
+       wait $MULTIPID1
+       wait $MULTIPID2
+}
+run_test 107a "Basic grouplock conflict"
+
+test_107b() {
+       dd if=/dev/zero of=$DIR1/$tfile bs=1M count=10
+       local gid1=14091995
+       local gid2=16022000
+
+       $LFS getstripe $DIR1/$tfile
+
+       multiop_bg_pause $DIR1/$tfile OG${gid1}_g${gid1}c || return 1
+       local MULTIPID1=$!
+       multiop $DIR2/$tfile Or10c &
+       local MULTIPID2=$!
+       sleep 2
+
+       if [[ $(ps h -o comm -p $MULTIPID2) == "" ]]; then
+               error "Grouplock does not block IO"
+       else
+               echo "Grouplock blocks IO"
+       fi
+
+       multiop $DIR2/$tfile OG${gid2}_g${gid2}c &
+       local MULTIPID3=$!
+       sleep 2
+       if [[ $(ps h -o comm -p $MULTIPID3) == "" ]]; then
+               error "First grouplock does not block second one"
+       else
+               echo "First grouplock blocks second one"
+       fi
+
+       kill -USR1 $MULTIPID1
+       sleep 2
+
+       if [[ $(ps h -o comm -p $MULTIPID3) == "" ]]; then
+               error "Second grouplock thread disappeared"
+       fi
+
+       if [[ $(ps h -o comm -p $MULTIPID2) == "" ]]; then
+               error "Second grouplock does not block IO"
+       else
+               echo "Second grouplock blocks IO"
+       fi
+
+       kill -USR1 $MULTIPID3
+       wait $MULTIPID1
+       wait $MULTIPID2
+       wait $MULTIPID3
+}
+run_test 107b "Grouplock is added to the head of waiting list"
+
+test_108a() {
+       local offset
+
+       $LFS setstripe -E 1M -c 1 -E -1 $DIR1/$tfile ||
+               error "Create $DIR1/$tfile failed"
+
+       dd if=/dev/zero of=$DIR1/$tfile bs=10000 count=1 ||
+               error "dd $DIR1/$tfile failed"
+       offset=$(lseek_test -d 5000 $DIR2/$tfile)
+       [[ $offset == 5000 ]] || error "offset $offset != 5000"
+
+       $TRUNCATE $DIR1/$tfile 2000
+       offset=$(lseek_test -l 1000 $DIR2/$tfile)
+       [[ $offset == 2000 ]] || error "offset $offset != 2000"
+
+       #define OBD_FAIL_OSC_DELAY_IO 0x414
+       $LCTL set_param fail_val=4 fail_loc=0x80000414
+       dd if=/dev/zero of=$DIR1/$tfile count=1 bs=8M conv=notrunc oflag=dsync &
+       local pid=$!
+       sleep 2
+
+       offset=$(lseek_test -l 8000 $DIR2/$tfile)
+       wait $pid
+       [[ $offset == 8388608 ]] || error "offset $offset != 8388608"
+}
+run_test 108a "lseek: parallel updates"
+
 log "cleanup: ======================================================"
 
 # kill and wait in each test only guarentee script finish, but command in script