From: jcl <jacques-charles.lafoucriere@cea.fr>
Date: Sun, 5 Feb 2012 21:37:06 +0000 (+0100)
Subject: LU-1031: add grouplock test to sanity.sh
X-Git-Tag: 2.2.56~1
X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=54320d7f52e6dd2b32e64997e2f7ae1ff1060f6f;hp=ef326407d56ad9a0af8cf60e4544ba5a88a37cfd

LU-1031: add grouplock test to sanity.sh

add a simple grouplock test to sanity.sh (test_82)
replace hardcoded use of multiop by a $MULTIOP in
test scripts.

Change-Id: I94ba2248b7efafe258873c468e6d0cabdfcb7d09
Signed-off-by: jcl <jacques-charles.lafoucriere@cea.fr>
Reviewed-on: http://review.whamcloud.com/2098
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
---

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index b3e2667..b387f3b 100644
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -27,6 +27,7 @@ PTLDEBUG=${PTLDEBUG:--1}
 SAVE_PWD=$PWD
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 RLUSTRE=${RLUSTRE:-$LUSTRE}
+export MULTIOP=${MULTIOP:-multiop}
 
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 7ef48f4..5973990 100755
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -5,6 +5,7 @@ set -e
 #         bug  5494 5493
 ALWAYS_EXCEPT="24   52 $RECOVERY_SMALL_EXCEPT"
 
+export MULTIOP=${MULTIOP:-multiop}
 PTLDEBUG=${PTLDEBUG:--1}
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
@@ -120,13 +121,13 @@ run_test 10 "finish request on server after client eviction (bug 1521)"
 #bug 2460
 # wake up a thread waiting for completion after eviction
 test_11(){
-    do_facet client multiop $DIR/$tfile Ow  || return 1
-    do_facet client multiop $DIR/$tfile or  || return 2
+    do_facet client $MULTIOP $DIR/$tfile Ow  || return 1
+    do_facet client $MULTIOP $DIR/$tfile or  || return 2
 
     cancel_lru_locks osc
 
-    do_facet client multiop $DIR/$tfile or  || return 3
-    drop_bl_callback multiop $DIR/$tfile Ow || echo "evicted as expected"
+    do_facet client $MULTIOP $DIR/$tfile or  || return 3
+    drop_bl_callback $MULTIOP $DIR/$tfile Ow || echo "evicted as expected"
 
     do_facet client munlink $DIR/$tfile  || return 4
 }
@@ -134,7 +135,7 @@ run_test 11 "wake up a thread waiting for completion after eviction (b=2460)"
 
 #b=2494
 test_12(){
-    $LCTL mark multiop $DIR/$tfile OS_c 
+    $LCTL mark $MULTIOP $DIR/$tfile OS_c
     do_facet $SINGLEMDS "lctl set_param fail_loc=0x115"
     clear_failloc $SINGLEMDS $((TIMEOUT * 2)) &
     multiop_bg_pause $DIR/$tfile OS_c || return 1
@@ -349,13 +350,13 @@ run_test 19a "test expired_lock_main on mds (2867)"
 
 test_19b() {
     f=$DIR/$tfile
-    do_facet client multiop $f Ow  || return 1
-    do_facet client multiop $f or  || return 2
+    do_facet client $MULTIOP $f Ow  || return 1
+    do_facet client $MULTIOP $f or  || return 2
 
     cancel_lru_locks osc
 
-    do_facet client multiop $f or  || return 3
-    drop_ldlm_cancel multiop $f Ow  || echo "client evicted, as expected"
+    do_facet client $MULTIOP $f or  || return 3
+    drop_ldlm_cancel $MULTIOP $f Ow  || echo "client evicted, as expected"
 
     do_facet client munlink $f  || return 4
 }
@@ -398,7 +399,7 @@ test_21a() {
        close_pid=$!
 
        do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000129"
-       multiop $DIR/$tdir-2/f Oc &
+       $MULTIOP $DIR/$tdir-2/f Oc &
        open_pid=$!
        sleep 1
        do_facet $SINGLEMDS "lctl set_param fail_loc=0"
@@ -473,7 +474,7 @@ test_21d() {
        pid=$!
 
        do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000129"
-       multiop $DIR/$tdir-2/f Oc &
+       $MULTIOP $DIR/$tdir-2/f Oc &
        sleep 1
        do_facet $SINGLEMDS "lctl set_param fail_loc=0"
 
@@ -590,11 +591,11 @@ test_22() {
     f2=$DIR/${tfile}-2
     
     do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000115"
-    multiop $f2 Oc &
+    $MULTIOP $f2 Oc &
     close_pid=$!
 
     sleep 1
-    multiop $f1 msu || return 1
+    $MULTIOP $f1 msu || return 1
 
     cancel_lru_locks mdc
     do_facet $SINGLEMDS "lctl set_param fail_loc=0"
diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh
index 9afcb36..c8abcfd 100755
--- a/lustre/tests/replay-dual.sh
+++ b/lustre/tests/replay-dual.sh
@@ -13,6 +13,7 @@ LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 SETUP=${SETUP:-""}
 CLEANUP=${CLEANUP:-""}
 MOUNT_2=${MOUNT_2:-"yes"}
+export MULTIOP=${MULTIOP:-multiop}
 . $LUSTRE/tests/test-framework.sh
 
 init_test_env $@
diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 1d09159..a062321 100755
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -7,6 +7,7 @@ set -e
 # This test needs to be run on the client
 #
 SAVE_PWD=$PWD
+export MULTIOP=${MULTIOP:-multiop}
 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 SETUP=${SETUP:-}
 CLEANUP=${CLEANUP:-}
diff --git a/lustre/tests/replay-vbr.sh b/lustre/tests/replay-vbr.sh
index 0ebf389..ae7e5c8 100644
--- a/lustre/tests/replay-vbr.sh
+++ b/lustre/tests/replay-vbr.sh
@@ -12,6 +12,7 @@ LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
 SETUP=${SETUP:-}
 CLEANUP=${CLEANUP:-}
 MOUNT_2=${MOUNT_2:-"yes"}
+export MULTIOP=${MULTIOP:-multiop}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
diff --git a/lustre/tests/runmultiop_bg_pause b/lustre/tests/runmultiop_bg_pause
index 640cc29..f309ab3 100644
--- a/lustre/tests/runmultiop_bg_pause
+++ b/lustre/tests/runmultiop_bg_pause
@@ -5,6 +5,7 @@
 
 PTLDEBUG=${PTLDEBUG:--1}
 LUSTRE=${LUSTRE:-`dirname $0`/..}
+MULTIOP=${MULTIOP:-multiop}
 . $LUSTRE/tests/test-framework.sh
 
 TMP=${TMP:-/tmp}
diff --git a/lustre/tests/sanity-gss.sh b/lustre/tests/sanity-gss.sh
index 0c5c624..60104e5 100644
--- a/lustre/tests/sanity-gss.sh
+++ b/lustre/tests/sanity-gss.sh
@@ -26,6 +26,8 @@ export PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$SRCDIR/../utils/gss:$PATH:/sb
 export NAME=${NAME:-local}
 SAVE_PWD=$PWD
 
+export MULTIOP=${MULTIOP:-multiop}
+
 CLEANUP=${CLEANUP:-""}
 SETUP=${SETUP:-""}
 
@@ -241,7 +243,7 @@ test_3() {
     $RUNAS cat $file > /dev/null || error "$RUNAS_ID cat error"
 
     # start multiop
-    $RUNAS multiop $file o_r &
+    $RUNAS $MULTIOP $file o_r &
     OPPID=$!
     # wait multiop finish its open()
     sleep 1
@@ -638,7 +640,7 @@ switch_sec_test()
     do_facet $SINGLEMDS lctl set_param fail_val=36
     do_facet $SINGLEMDS lctl set_param fail_loc=0x513
     log "starting multiop"
-    multiop $filename m &
+    $MULTIOP $filename m &
     multiop_pid=$!
     echo "multiop pid=$multiop_pid"
     sleep 1
diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh
index 507768a..96d1ab0 100644
--- a/lustre/tests/sanity-quota.sh
+++ b/lustre/tests/sanity-quota.sh
@@ -50,6 +50,7 @@ SANITY_QUOTA_USERS="quota15_1 quota15_2 quota15_3 quota15_4 quota15_5 quota15_6
                     quota15_19 quota15_20 quota15_21 quota15_22 quota15_23 quota15_24 \
                     quota15_25 quota15_26 quota15_27 quota15_28 quota15_29 quota15_30"
 
+export MULTIOP=${MULTIOP:-multiop}
 TRACE=${TRACE:-""}
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
@@ -1194,10 +1195,10 @@ test_14a() {	# was test_14 b=12223 -- setting quota on root
 	$LFS setquota -u root -b 10 -B 10 -i 10 -I 10 $DIR
 	createmany -m ${TESTFILE} 20 || \
 	    quota_error u root "unexpected: user(root) create files failly!"
-        multiop ${TESTFILE} oO_CREAT:O_WRONLY:O_DIRECT:w$((4096 * 4096))c || \
+        $MULTIOP ${TESTFILE} oO_CREAT:O_WRONLY:O_DIRECT:w$((4096 * 4096))c || \
 	    quota_error u root "unexpected: user(root) write files failly!"
 	chmod 666 $TESTFILE
-        $RUNAS multiop ${TESTFILE} oO_WRONLY:O_APPEND:O_DIRECT:w$((4096 * 4096))c && \
+        $RUNAS $MULTIOP ${TESTFILE} oO_WRONLY:O_APPEND:O_DIRECT:w$((4096 * 4096))c && \
 	    quota_error u root "unexpected: user(quota_usr) write a file successfully!"
 
 	# trigger the llog
diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 612f4b0..0a667b7 100644
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -36,6 +36,7 @@ LCTL=${LCTL:-lctl}
 MCREATE=${MCREATE:-mcreate}
 OPENFILE=${OPENFILE:-openfile}
 OPENUNLINK=${OPENUNLINK:-openunlink}
+export MULTIOP=${MULTIOP:-multiop}
 READS=${READS:-"reads"}
 MUNLINK=${MUNLINK:-munlink}
 SOCKETSERVER=${SOCKETSERVER:-socketserver}
@@ -691,13 +692,13 @@ run_test 24k "touch .../R11a/f; mv .../R11a/f .../R11a/d ======="
 # bug 2429 - rename foo foo foo creates invalid file
 test_24l() {
 	f="$DIR/f24l"
-	multiop $f OcNs || error
+	$MULTIOP $f OcNs || error
 }
 run_test 24l "Renaming a file to itself ========================"
 
 test_24m() {
 	f="$DIR/f24m"
-	multiop $f OcLN ${f}2 ${f}2 || error "link ${f}2 ${f}2 failed"
+	$MULTIOP $f OcLN ${f}2 ${f}2 || error "link ${f}2 ${f}2 failed"
 	# on ext3 this does not remove either the source or target files
 	# though the "expected" operation would be to remove the source
 	$CHECKSTAT -t file ${f} || error "${f} missing"
@@ -774,7 +775,7 @@ test_24t() {
 run_test 24t "mkdir .../R16a/b/c; rename .../R16a/b/c .../R16a ="
 
 test_24u() { # bug12192
-        multiop $DIR/$tfile C2w$((2048 * 1024))c || error
+        $MULTIOP $DIR/$tfile C2w$((2048 * 1024))c || error
         $CHECKSTAT -s $((2048 * 1024)) $DIR/$tfile || error "wrong file size"
 }
 run_test 24u "create stripe file"
@@ -1543,7 +1544,7 @@ run_test 31a "open-unlink file =================================="
 test_31b() {
 	touch $DIR/f31 || error
 	ln $DIR/f31 $DIR/f31b || error
-	multiop $DIR/f31b Ouc || error
+	$MULTIOP $DIR/f31b Ouc || error
 	$CHECKSTAT -t file $DIR/f31 || error
 }
 run_test 31b "unlink file with multiple links while open ======="
@@ -1553,7 +1554,7 @@ test_31c() {
 	ln $DIR/f31 $DIR/f31c || error
 	multiop_bg_pause $DIR/f31 O_uc || return 1
 	MULTIPID=$!
-	multiop $DIR/f31c Ouc
+	$MULTIOP $DIR/f31c Ouc
 	kill -USR1 $MULTIPID
 	wait $MULTIPID
 }
@@ -2064,7 +2065,7 @@ test_34h() {
 	local sz=1000
 
 	dd if=/dev/zero of=$DIR/$tfile bs=1M count=10 || error
-	multiop $DIR/$tfile OG${gid}T${sz}g${gid}c &
+	$MULTIOP $DIR/$tfile OG${gid}T${sz}g${gid}c &
 	MULTIPID=$!
 	sleep 2
 
@@ -2782,7 +2783,7 @@ run_test 42e "verify sub-RPC writes are not done synchronously"
 test_43() {
 	mkdir -p $DIR/$tdir
 	cp -p /bin/ls $DIR/$tdir/$tfile
-	multiop $DIR/$tdir/$tfile Ow_c &
+	$MULTIOP $DIR/$tdir/$tfile Ow_c &
 	pid=$!
 	# give multiop a chance to open
 	sleep 1
@@ -2794,10 +2795,10 @@ run_test 43 "execution of file opened for write should return -ETXTBSY"
 
 test_43a() {
         mkdir -p $DIR/d43
-	cp -p `which multiop` $DIR/d43/multiop || cp -p multiop $DIR/d43/multiop
+	cp -p `which $MULTIOP` $DIR/d43/multiop || cp -p multiop $DIR/d43/multiop
         MULTIOP_PROG=$DIR/d43/multiop multiop_bg_pause $TMP/test43.junk O_c || return 1
         MULTIOP_PID=$!
-        multiop $DIR/d43/multiop Oc && error "expected error, got success"
+        $MULTIOP $DIR/d43/multiop Oc && error "expected error, got success"
         kill -USR1 $MULTIOP_PID || return 2
         wait $MULTIOP_PID || return 3
         rm $TMP/test43.junk
@@ -2806,7 +2807,7 @@ run_test 43a "open(RDWR) of file being executed should return -ETXTBSY"
 
 test_43b() {
         mkdir -p $DIR/d43
-	cp -p `which multiop` $DIR/d43/multiop || cp -p multiop $DIR/d43/multiop
+	cp -p `which $MULTIOP` $DIR/d43/multiop || cp -p multiop $DIR/d43/multiop
         MULTIOP_PROG=$DIR/d43/multiop multiop_bg_pause $TMP/test43.junk O_c || return 1
         MULTIOP_PID=$!
         $TRUNCATE $DIR/d43/multiop 0 && error "expected error, got success"
@@ -4021,7 +4022,7 @@ test_61() {
 	f="$DIR/f61"
 	dd if=/dev/zero of=$f bs=`page_size` count=1
 	cancel_lru_locks osc
-	multiop $f OSMWUc || error
+	$MULTIOP $f OSMWUc || error
 	sync
 }
 run_test 61 "mmap() writes don't make sync hang ================"
@@ -4067,7 +4068,7 @@ test_63b() {
 
 	#define OBD_FAIL_OSC_BRW_PREP_REQ        0x406
 	lctl set_param fail_loc=0x80000406
-	multiop $DIR/$tfile Owy && \
+	$MULTIOP $DIR/$tfile Owy && \
 		error "sync didn't return ENOMEM"
 	sync; sleep 2; sync	# do a real sync this time to flush page
 	lctl get_param -n llite.*.dump_page_cache | grep locked && \
@@ -4437,11 +4438,11 @@ test_73() {
 	pid1=$!
 
 	lctl set_param fail_loc=0x80000129
-	multiop $DIR/d73-1/f73-2 Oc &
+	$MULTIOP $DIR/d73-1/f73-2 Oc &
 	sleep 1
 	lctl set_param fail_loc=0
 
-	multiop $DIR/d73-2/f73-3 Oc &
+	$MULTIOP $DIR/d73-2/f73-3 Oc &
 	pid3=$!
 
 	kill -USR1 $pid1
@@ -4817,7 +4818,7 @@ test_81a() { # LU-456
 
         # write should trigger a retry and success
         $SETSTRIPE -i 0 -c 1 $DIR/$tfile
-        multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+        $MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
         RC=$?
         if [ $RC -ne 0 ] ; then
                 error "write should success, but failed for $RC"
@@ -4833,7 +4834,7 @@ test_81b() { # LU-456
 
         # write should retry several times and return -ENOSPC finally
         $SETSTRIPE -i 0 -c 1 $DIR/$tfile
-        multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+        $MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
         RC=$?
         ENOSPC=28
         if [ $RC -ne $ENOSPC ] ; then
@@ -4842,6 +4843,27 @@ test_81b() { # LU-456
 }
 run_test 81b "OST should return -ENOSPC when retry still fails ======="
 
+test_82() { # LU-1031
+	dd if=/dev/zero of=$DIR/$tfile bs=1M count=10
+	local gid1=14091995
+	local gid2=16022000
+
+	multiop_bg_pause $DIR/$tfile OG${gid1}_g${gid1}c || return 1
+	local MULTIPID1=$!
+	multiop_bg_pause $DIR/$tfile O_G${gid2}r10g${gid2}c || return 2
+	local MULTIPID2=$!
+	kill -USR1 $MULTIPID2
+	sleep 2
+	if [[ `ps h -o comm -p $MULTIPID2` == "" ]]; then
+		error "First grouplock does not block second one"
+	else
+		echo "Second grouplock blocks first one"
+	fi
+	kill -USR1 $MULTIPID1
+	wait $MULTIPID1
+	wait $MULTIPID2
+}
+run_test 82 "Basic grouplock test ==============================="
 
 test_99a() {
         [ -z "$(which cvs 2>/dev/null)" ] && skip_env "could not find cvs" && \
@@ -5903,7 +5925,7 @@ test_118a() #bug 11710
 {
 	reset_async
 
- 	multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+	$MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
 	DIRTY=$(lctl get_param -n llite.*.dump_page_cache | grep -c dirty)
         WRITEBACK=$(lctl get_param -n llite.*.dump_page_cache | grep -c writeback)
 
@@ -5923,7 +5945,7 @@ test_118b()
 
 	#define OBD_FAIL_OST_ENOENT 0x217
 	set_nodes_failloc "$(osts_nodes)" 0x217
-	multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+	$MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
 	RC=$?
 	set_nodes_failloc "$(osts_nodes)" 0
         DIRTY=$(lctl get_param -n llite.*.dump_page_cache | grep -c dirty)
@@ -5944,7 +5966,7 @@ test_118b()
 
 	# Due to the above error the OSC will issue all RPCs syncronously
 	# until a subsequent RPC completes successfully without error.
-	multiop $DIR/$tfile Ow4096yc
+	$MULTIOP $DIR/$tfile Ow4096yc
 	rm -f $DIR/$tfile
 
 	return 0
@@ -5961,7 +5983,7 @@ test_118c()
 	set_nodes_failloc "$(osts_nodes)" 0x216
 
 	# multiop should block due to fsync until pages are written
-	multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c &
+	$MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c &
 	MULTIPID=$!
 	sleep 1
 
@@ -6004,7 +6026,7 @@ test_118d()
 	#define OBD_FAIL_OST_BRW_PAUSE_BULK
 	set_nodes_failloc "$(osts_nodes)" 0x214
 	# multiop should block due to fsync until pages are written
-	multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c &
+	$MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c &
 	MULTIPID=$!
 	sleep 1
 
@@ -6041,7 +6063,7 @@ test_118f() {
         lctl set_param fail_loc=0x8000040a
 
 	# Should simulate EINVAL error which is fatal
-        multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+        $MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
         RC=$?
 	if [[ $RC -eq 0 ]]; then
 		error "Must return error due to dropped pages, rc=$RC"
@@ -6076,7 +6098,7 @@ test_118g() {
 	lctl set_param fail_loc=0x406
 
 	# simulate local -ENOMEM
-	multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+	$MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
 	RC=$?
 
 	lctl set_param fail_loc=0
@@ -6112,7 +6134,7 @@ test_118h() {
 	#define OBD_FAIL_OST_BRW_WRITE_BULK      0x20e
         set_nodes_failloc "$(osts_nodes)" 0x20e
 	# Should simulate ENOMEM error which is recoverable and should be handled by timeout
-        multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+        $MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
         RC=$?
 
         set_nodes_failloc "$(osts_nodes)" 0
@@ -6148,7 +6170,7 @@ test_118i() {
         set_nodes_failloc "$(osts_nodes)" 0x20e
 
 	# Should simulate ENOMEM error which is recoverable and should be handled by timeout
-        multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c &
+        $MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c &
 	PID=$!
 	sleep 5
 	set_nodes_failloc "$(osts_nodes)" 0
@@ -6186,7 +6208,7 @@ test_118j() {
         set_nodes_failloc "$(osts_nodes)" 0x220
 
 	# return -EIO from OST
-        multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
+        $MULTIOP $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c
         RC=$?
         set_nodes_failloc "$(osts_nodes)" 0x0
 	if [[ $RC -eq 0 ]]; then
@@ -6238,7 +6260,7 @@ test_118l()
 {
 	# LU-646
 	mkdir -p $DIR/$tdir
-	multiop $DIR/$tdir Dy || error "fsync dir failed"
+	$MULTIOP $DIR/$tdir Dy || error "fsync dir failed"
 	rm -rf $DIR/$tdir
 }
 run_test 118l "fsync dir ========="
@@ -6268,7 +6290,7 @@ test_119b() # bug 11737
         $SETSTRIPE -c 2 $DIR/$tfile || error "setstripe failed"
         dd if=/dev/zero of=$DIR/$tfile bs=1M count=1 seek=1 || error "dd failed"
         sync
-        multiop $DIR/$tfile oO_RDONLY:O_DIRECT:r$((2048 * 1024)) || \
+        $MULTIOP $DIR/$tfile oO_RDONLY:O_DIRECT:r$((2048 * 1024)) || \
                 error "direct read failed"
         rm -f $DIR/$tfile
 }
@@ -6906,7 +6928,7 @@ test_129() {
 	I=0
 	J=0
 	while [ ! $I -gt $((MAX * MDSCOUNT)) ]; do
-		multiop $DIR/$tdir/$J Oc
+		$MULTIOP $DIR/$tdir/$J Oc
 		rc=$?
 		if [ $rc -eq $EFBIG ]; then
 			set_dir_limits 0
@@ -7673,7 +7695,7 @@ test_152() {
 run_test 152 "test read/write with enomem ============================"
 
 test_153() {
-        multiop $DIR/$tfile Ow4096Ycu || error "multiop failed"
+        $MULTIOP $DIR/$tfile Ow4096Ycu || error "multiop failed"
 }
 run_test 153 "test if fdatasync does not crash ======================="
 
@@ -8226,11 +8248,11 @@ run_test 163 "kernel <-> userspace comms"
 test_169() {
 	# do directio so as not to populate the page cache
 	log "creating a 10 Mb file"
-	multiop $DIR/$tfile oO_CREAT:O_DIRECT:O_RDWR:w$((10*1048576))c || error "multiop failed while creating a file"
+	$MULTIOP $DIR/$tfile oO_CREAT:O_DIRECT:O_RDWR:w$((10*1048576))c || error "multiop failed while creating a file"
 	log "starting reads"
 	dd if=$DIR/$tfile of=/dev/null bs=4096 &
 	log "truncating the file"
-	multiop $DIR/$tfile oO_TRUNC:c || error "multiop failed while truncating the file"
+	$MULTIOP $DIR/$tfile oO_TRUNC:c || error "multiop failed while truncating the file"
 	log "killing dd"
 	kill %+ || true # reads might have finished
 	echo "wait until dd is finished"
@@ -9058,11 +9080,11 @@ run_test 217 "check lctl ping for hostnames with hiphen ('-')"
 test_218() {
        # do directio so as not to populate the page cache
        log "creating a 10 Mb file"
-       multiop $DIR/$tfile oO_CREAT:O_DIRECT:O_RDWR:w$((10*1048576))c || error "multiop failed while creating a file"
+       $MULTIOP $DIR/$tfile oO_CREAT:O_DIRECT:O_RDWR:w$((10*1048576))c || error "multiop failed while creating a file"
        log "starting reads"
        dd if=$DIR/$tfile of=/dev/null bs=4096 &
        log "truncating the file"
-       multiop $DIR/$tfile oO_TRUNC:c || error "multiop failed while truncating the file"
+       $MULTIOP $DIR/$tfile oO_TRUNC:c || error "multiop failed while truncating the file"
        log "killing dd"
        kill %+ || true # reads might have finished
        echo "wait until dd is finished"
diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh
index 53e5b669f..4897ed7 100644
--- a/lustre/tests/sanityn.sh
+++ b/lustre/tests/sanityn.sh
@@ -25,6 +25,7 @@ CHECKSTAT=${CHECKSTAT:-"checkstat -v"}
 MCREATE=${MCREATE:-mcreate}
 OPENFILE=${OPENFILE:-openfile}
 OPENUNLINK=${OPENUNLINK:-openunlink}
+export MULTIOP=${MULTIOP:-multiop}
 export TMP=${TMP:-/tmp}
 MOUNT_2=${MOUNT_2:-"yes"}
 CHECK_GRANT=${CHECK_GRANT:-"yes"}
@@ -270,7 +271,7 @@ test_14a() {
 	cp -p `which multiop` $DIR1/d14/multiop || error "cp failed"
         MULTIOP_PROG=$DIR1/d14/multiop multiop_bg_pause $TMP/test14.junk O_c || return 1
         MULTIOP_PID=$!
-        multiop $DIR2/d14/multiop Oc && error "expected error, got success"
+        $MULTIOP $DIR2/d14/multiop Oc && error "expected error, got success"
         kill -USR1 $MULTIOP_PID || return 2
         wait $MULTIOP_PID || return 3
         rm $TMP/test14.junk $DIR1/d14/multiop || error "removing multiop"
@@ -392,9 +393,9 @@ test_20() {
 	mkdir $DIR1/d20
 	cancel_lru_locks osc
 	CNT=$((`lctl get_param -n llite.*.dump_page_cache | wc -l`))
-	multiop $DIR1/f20 Ow8190c
-	multiop $DIR2/f20 Oz8194w8190c
-	multiop $DIR1/f20 Oz0r8190c
+	$MULTIOP $DIR1/f20 Ow8190c
+	$MULTIOP $DIR2/f20 Oz8194w8190c
+	$MULTIOP $DIR1/f20 Oz0r8190c
 	cancel_lru_locks osc
 	CNTD=$((`lctl get_param -n llite.*.dump_page_cache | wc -l` - $CNT))
 	[ $CNTD -gt 0 ] && \
@@ -1200,7 +1201,7 @@ run_test 40e "pdirops: rename and others =============="
 test_41a() {
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
 	do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
-	multiop $DIR1/$tfile oO_CREAT:O_RDWR:c &
+	$MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
 	PID1=$!
 	sleep 1
 	mkdir $DIR2/$tfile && error "mkdir must fail"
@@ -1213,10 +1214,10 @@ run_test 41a "pdirops: create vs mkdir =============="
 test_41b() {
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
 	do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
-	multiop $DIR1/$tfile oO_CREAT:O_RDWR:c &
+	$MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
 	PID1=$!
 	sleep 1
-	multiop $DIR2/$tfile oO_CREAT:O_EXCL:c && error "create must fail"
+	$MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c && error "create must fail"
 	check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
 	rm -r $DIR1/*
 	return 0
@@ -1227,7 +1228,7 @@ test_41c() {
 	touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
 	do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
-	multiop $DIR1/$tfile oO_CREAT:O_RDWR:c &
+	$MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
 	PID1=$!
 	sleep 1
 	link $DIR2/$tfile-2 $DIR2/$tfile && error "link must fail"
@@ -1240,7 +1241,7 @@ run_test 41c "pdirops: create vs link =============="
 test_41d() {
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
 	do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
-	multiop $DIR1/$tfile oO_CREAT:O_RDWR:c &
+	$MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
 	PID1=$!
 	sleep 1
 	rm $DIR2/$tfile || error "unlink must succeed"
@@ -1254,7 +1255,7 @@ test_41e() {
 	touch $DIR1/$tfile-2
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
 	do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
-	multiop $DIR1/$tfile oO_CREAT:O_RDWR:c &
+	$MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
 	PID1=$!
 	sleep 1
 	mv $DIR2/$tfile-2 $DIR2/$tfile || error "rename must succeed"
@@ -1267,7 +1268,7 @@ run_test 41e "pdirops: create and rename (tgt) =============="
 test_41f() {
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
 	do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
-	multiop $DIR1/$tfile oO_CREAT:O_RDWR:c &
+	$MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
 	PID1=$!
 	sleep 1
 	mv $DIR2/$tfile $DIR2/$tfile-2 || error "rename must succeed"
@@ -1280,7 +1281,7 @@ run_test 41f "pdirops: create and rename (src) =============="
 test_41g() {
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
 	do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
-	multiop $DIR1/$tfile oO_CREAT:O_RDWR:c &
+	$MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
 	PID1=$!
 	sleep 1
 	stat $DIR2/$tfile > /dev/null || error "stat must succeed"
@@ -1293,7 +1294,7 @@ run_test 41g "pdirops: create vs getattr =============="
 test_41h() {
 #define OBD_FAIL_ONCE|OBD_FAIL_MDS_PDO_LOCK    0x145
 	do_facet $SINGLEMDS lctl set_param fail_loc=0x80000145
-	multiop $DIR1/$tfile oO_CREAT:O_RDWR:c &
+	$MULTIOP $DIR1/$tfile oO_CREAT:O_RDWR:c &
 	PID1=$!
 	sleep 1
 	ls -lia $DIR2/ > /dev/null
@@ -1323,7 +1324,7 @@ test_42b() {
 	mkdir $DIR1/$tfile &
 	PID1=$!
 	sleep 1
-	multiop $DIR2/$tfile oO_CREAT:O_EXCL:c && error "create must fail"
+	$MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c && error "create must fail"
 	check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
 	rm -r $DIR1/*
 	return 0
@@ -1432,7 +1433,7 @@ test_43b() {
 	rm $DIR1/$tfile &
 	PID1=$!
 	sleep 1
-	multiop $DIR2/$tfile oO_CREAT:O_EXCL:c || error "create must succeed"
+	$MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c || error "create must succeed"
 	check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
 	rm -r $DIR1/*
 	return 0
@@ -1547,7 +1548,7 @@ test_44b() {
 	mv $DIR1/$tfile-2 $DIR1/$tfile &
 	PID1=$!
 	sleep 1
-	multiop $DIR2/$tfile oO_CREAT:O_EXCL:c && error "create must fail"
+	$MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c && error "create must fail"
 	check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
 	rm -r $DIR1/*
 	return 0
@@ -1664,7 +1665,7 @@ test_45b() {
 	mv $DIR1/$tfile $DIR1/$tfile-2 &
 	PID1=$!
 	sleep 1
-	multiop $DIR2/$tfile oO_CREAT:O_EXCL:c || error "create must succeed"
+	$MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c || error "create must succeed"
 	check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
 	rm -r $DIR1/*
 	return 0
@@ -1779,7 +1780,7 @@ test_46b() {
 	link $DIR1/$tfile-2 $DIR1/$tfile &
 	PID1=$!
 	sleep 1
-	multiop $DIR2/$tfile oO_CREAT:O_EXCL:c && error "create must fail"
+	$MULTIOP $DIR2/$tfile oO_CREAT:O_EXCL:c && error "create must fail"
 	check_pdo_conflict $PID1 && { wait $PID1; error "create isn't blocked"; }
 	rm -r $DIR1/*
 	return 0
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 70435d6..6980a99 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -3835,7 +3835,7 @@ fi"
 # Run multiop in the background, but wait for it to print
 # "PAUSING" to its stdout before returning from this function.
 multiop_bg_pause() {
-    MULTIOP_PROG=${MULTIOP_PROG:-multiop}
+    MULTIOP_PROG=${MULTIOP_PROG:-$MULTIOP}
     FILE=$1
     ARGS=$2