From: Bobi Jam <bobijam.xu@intel.com>
Date: Thu, 29 Aug 2013 00:00:08 +0000 (+0800)
Subject: LU-3853 tests: mpiexec requires strict global/local args order
X-Git-Tag: 2.5.52~83
X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=39b944ec54655bb70bcb79fe8223c724f7d08dbf

LU-3853 tests: mpiexec requires strict global/local args order

Put -machinefile option ahead of other mpiexec options, since some
version of mpiexec (at least 1.90) requests global options be put
before its local options, and -machinefile is a global options.

Signed-off-by: Bobi Jam <bobijam.xu@intel.com>
Change-Id: Ib188881a78a2b345552e8c6070f7dfa7a086ce5d
Reviewed-on: http://review.whamcloud.com/7493
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Jian Yu <jian.yu@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
---

diff --git a/lustre/tests/functions.sh b/lustre/tests/functions.sh
index c039746..a3a70e0 100644
--- a/lustre/tests/functions.sh
+++ b/lustre/tests/functions.sh
@@ -348,8 +348,8 @@ run_metabench() {
 			-n $((num_clients * mbench_THREADS)) \
 			-p $SRUN_PARTITION -- $cmd
 	else
-		mpi_run -np $((num_clients * $mbench_THREADS)) \
-			${MACHINEFILE_OPTION} ${MACHINEFILE} $cmd
+		mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+			-np $((num_clients * $mbench_THREADS)) $cmd
 	fi
 
     local rc=$?
@@ -396,8 +396,8 @@ run_simul() {
 			-n $((num_clients * simul_THREADS)) -p $SRUN_PARTITION \
 			-- $cmd
 	else
-		mpi_run -np $((num_clients * simul_THREADS)) \
-			${MACHINEFILE_OPTION} ${MACHINEFILE} $cmd
+		mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+			-np $((num_clients * simul_THREADS)) $cmd
 	fi
 
     local rc=$?
@@ -452,8 +452,8 @@ run_mdtest() {
 			-n $((num_clients * mdtest_THREADS)) \
 			-p $SRUN_PARTITION -- $cmd
 	else
-		mpi_run -np $((num_clients * mdtest_THREADS)) \
-			${MACHINEFILE_OPTION} ${MACHINEFILE} $cmd
+		mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+			-np $((num_clients * mdtest_THREADS)) $cmd
 	fi
 
     local rc=$?
@@ -586,8 +586,8 @@ run_ior() {
 			-n $((num_clients * ior_THREADS)) -p $SRUN_PARTITION \
 			-- $cmd
 	else
-		mpi_run -np $((num_clients * $ior_THREADS)) \
-			${MACHINEFILE_OPTION} ${MACHINEFILE} $cmd
+		mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+			-np $((num_clients * $ior_THREADS)) $cmd
 	fi
 
     local rc=$?
@@ -640,8 +640,8 @@ run_mib() {
 			-n $((num_clients * mib_THREADS)) -p $SRUN_PARTITION \
 			-- $cmd
 	else
-		mpi_run -np $((num_clients * mib_THREADS)) \
-			${MACHINEFILE_OPTION} ${MACHINEFILE} $cmd
+		mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+			-np $((num_clients * mib_THREADS)) $cmd
 	fi
 
     local rc=$?
@@ -682,8 +682,8 @@ run_cascading_rw() {
     local cmd="$CASC_RW -g -d $testdir -n $casc_REP"
 
 	echo "+ $cmd"
-	mpi_run -np $((num_clients * $casc_THREADS)) ${MACHINEFILE_OPTION} \
-		${MACHINEFILE} $cmd
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+		-np $((num_clients * $casc_THREADS)) $cmd
 
     local rc=$?
     if [ $rc != 0 ] ; then
@@ -724,8 +724,8 @@ run_write_append_truncate() {
     local cmd="write_append_truncate -n $write_REP $file"
 
 	echo "+ $cmd"
-	mpi_run -np $((num_clients * $write_THREADS)) ${MACHINEFILE_OPTION} \
-		${MACHINEFILE} $cmd
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+		-np $((num_clients * $write_THREADS)) $cmd
 
     local rc=$?
     if [ $rc != 0 ] ; then
@@ -764,8 +764,8 @@ run_write_disjoint() {
     local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP"
 
 	echo "+ $cmd"
-	mpi_run -np $((num_clients * $wdisjoint_THREADS)) \
-		${MACHINEFILE_OPTION} ${MACHINEFILE} $cmd
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+		-np $((num_clients * $wdisjoint_THREADS)) $cmd
 
     local rc=$?
     if [ $rc != 0 ] ; then
@@ -806,8 +806,8 @@ run_parallel_grouplock() {
 		local cmd="$PARALLEL_GROUPLOCK -g -v -d $testdir $subtest"
 		echo "+ $cmd"
 
-		mpi_run -np $parallel_grouplock_MINTASKS ${MACHINEFILE_OPTION} \
-			${MACHINEFILE} $cmd
+		mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+			-np $parallel_grouplock_MINTASKS $cmd
 		local rc=$?
 		if [ $rc != 0 ] ; then
 			error_noexit "parallel_grouplock subtests $subtest " \
@@ -883,8 +883,8 @@ run_statahead () {
     local cmd="$cmd1 $cmd2"
     echo "+ $cmd"
 
-	mpi_run -np $((num_clients * 32)) ${MACHINEFILE_OPTION} ${MACHINEFILE} \
-		$cmd
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+		-np $((num_clients * 32)) $cmd
 
     local rc=$?
     if [ $rc != 0 ] ; then
diff --git a/lustre/tests/ha.sh b/lustre/tests/ha.sh
index e4f623d..260b81e 100755
--- a/lustre/tests/ha.sh
+++ b/lustre/tests/ha.sh
@@ -280,8 +280,8 @@ ha_repeat_mpi_load()
 	while [ ! -e "$ha_stop_file" ] && ((rc == 0)); do
 		{
 			ha_on ${ha_clients[0]} mkdir -p "$dir" &&	   \
-			mpirun -np ${#ha_clients[@]} ${MACHINEFILE_OPTION} \
-				"$ha_machine_file" $cmd &&		   \
+			mpirun ${MACHINEFILE_OPTION} "$ha_machine_file"    \
+				-np ${#ha_clients[@]} $cmd &&	           \
 			ha_on ${ha_clients[0]} rm -rf "$dir"
 		} >>"$log" 2>&1 || rc=$?
 
diff --git a/lustre/tests/large-lun.sh b/lustre/tests/large-lun.sh
index d65bfb9..2dbc03e 100644
--- a/lustre/tests/large-lun.sh
+++ b/lustre/tests/large-lun.sh
@@ -93,8 +93,8 @@ run_mdsrate() {
 		--nfiles $num_files --filefmt 'file%%d'"
 
 	echo "# $command"
-	mpi_run -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) -machinefile \
-		$MACHINEFILE $command
+	mpi_run -machinefile $MACHINEFILE \
+		-np $((NUM_CLIENTS * THREADS_PER_CLIENT)) $command
 
 	if [ ${PIPESTATUS[0]} != 0 ]; then
 		error "mdsrate create failed"
diff --git a/lustre/tests/large-scale.sh b/lustre/tests/large-scale.sh
index 195f93b..ccfa0c1 100644
--- a/lustre/tests/large-scale.sh
+++ b/lustre/tests/large-scale.sh
@@ -83,8 +83,8 @@ test_3a() {
 
 			COMMAND="${MDSRATE} --create --nfiles $nfiles --dir
 				 $dir --filefmt 'f%%d'"
-			mpi_run -np $((num * nthreads)) ${MACHINEFILE_OPTION} \
-				$machinefile ${COMMAND} | tee ${LOG} &
+			mpi_run ${MACHINEFILE_OPTION} $machinefile \
+				-np $((num * nthreads)) ${COMMAND} | tee ${LOG}&
 
 			pid=$!
 			echo "pid=$pid"
diff --git a/lustre/tests/mdsrate-create-large.sh b/lustre/tests/mdsrate-create-large.sh
index 5b7414a..fd10e02 100644
--- a/lustre/tests/mdsrate-create-large.sh
+++ b/lustre/tests/mdsrate-create-large.sh
@@ -13,7 +13,7 @@ MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
 BASEDIR=$MOUNT/mdsrate
 
 # Requirements
-# set NUM_FILES=0 to force TIME_PERIOD work  
+# set NUM_FILES=0 to force TIME_PERIOD work
 NUM_FILES=${NUM_FILES:-1000000}
 TIME_PERIOD=${TIME_PERIOD:-600}                        # seconds
 
@@ -28,7 +28,7 @@ NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })
 
 [ ! -x ${MDSRATE} ] && error "${MDSRATE} not built."
 
-log "===== $0 ====== " 
+log "===== $0 ====== "
 
 check_and_setup_lustre
 
@@ -58,7 +58,7 @@ else
 	COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
 		--nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
 	echo "+ ${COMMAND}"
-	mpi_run -np 1 ${MACHINEFILE_OPTION} ${MACHINEFILE} ${COMMAND} |
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
 		tee ${LOG}
 
     if [ ${PIPESTATUS[0]} != 0 ]; then
@@ -71,7 +71,7 @@ else
     COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                 --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
     echo "+ ${COMMAND}"
-    mpi_run -np 1 ${MACHINEFILE_OPTION} ${MACHINEFILE} ${COMMAND} | tee ${LOG}
+    mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} | tee ${LOG}
 
     if [ ${PIPESTATUS[0]} != 0 ]; then
 	[ -f $LOG ] && sed -e "s/^/log: /" $LOG
@@ -97,7 +97,7 @@ else
 	COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
 		--nfiles $NUM_FILES --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
 	echo "+ ${COMMAND}"
-	mpi_run -np ${NUM_CLIENTS} ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \
 		${COMMAND} | tee ${LOG}
 
     if [ ${PIPESTATUS[0]} != 0 ]; then
@@ -110,7 +110,7 @@ else
 	COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
 		--nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
 	echo "+ ${COMMAND}"
-	mpi_run -np ${NUM_CLIENTS} ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \
 		${COMMAND} | tee ${LOG}
 
     if [ ${PIPESTATUS[0]} != 0 ]; then
diff --git a/lustre/tests/mdsrate-create-small.sh b/lustre/tests/mdsrate-create-small.sh
index dcb8d56..ce4bd06 100644
--- a/lustre/tests/mdsrate-create-small.sh
+++ b/lustre/tests/mdsrate-create-small.sh
@@ -1,6 +1,6 @@
  #!/bin/bash
 #
-# This test was used in a set of CMD3 tests (cmd3-3 test). 
+# This test was used in a set of CMD3 tests (cmd3-3 test).
 
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
@@ -36,7 +36,7 @@ fi
 # Make sure we start with a clean slate
 rm -f ${LOG}
 
-log "===== $0 ====== " 
+log "===== $0 ====== "
 
 check_and_setup_lustre
 
@@ -49,7 +49,7 @@ IFree=$(mdsrate_inodes_available)
 if [ $IFree -lt $NUM_FILES ]; then
     NUM_FILES=$IFree
 fi
-  
+
 generate_machine_file $NODES_TO_USE $MACHINEFILE || error "can not generate machinefile"
 
 if [ -n "$NOSINGLE" ]; then
@@ -66,7 +66,7 @@ else
 	COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
 		--nfiles $NUM_FILES --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
 	echo "+ ${COMMAND}"
-	mpi_run -np 1 ${MACHINEFILE_OPTION} ${MACHINEFILE} ${COMMAND} |
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
 		tee ${LOG}
 
         if [ ${PIPESTATUS[0]} != 0 ]; then
@@ -83,7 +83,7 @@ else
         COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                      --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
         echo "+ ${COMMAND}"
-        mpi_run -np 1 ${MACHINEFILE_OPTION} ${MACHINEFILE} ${COMMAND} |
+        mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
 		tee ${LOG}
 
         if [ ${PIPESTATUS[0]} != 0 ]; then
@@ -113,8 +113,9 @@ else
 	COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
 		--nfiles $NUM_FILES --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
 	echo "+ ${COMMAND}"
-	mpi_run -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) \
-		${MACHINEFILE_OPTION} ${MACHINEFILE} ${COMMAND} | tee ${LOG}
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+		-np $((NUM_CLIENTS * THREADS_PER_CLIENT)) ${COMMAND} |
+		tee ${LOG}
         if [ ${PIPESTATUS[0]} != 0 ]; then
             [ -f $LOG ] && sed -e "s/^/log: /" $LOG
             error "mdsrate create on multiple nodes failed, aborting"
@@ -129,8 +130,9 @@ else
         COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                       --nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
         echo "+ ${COMMAND}"
-        mpi_run -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) \
-		${MACHINEFILE_OPTION} ${MACHINEFILE} ${COMMAND} | tee ${LOG}
+        mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+		-np $((NUM_CLIENTS * THREADS_PER_CLIENT)) ${COMMAND} |
+		tee ${LOG}
         if [ ${PIPESTATUS[0]} != 0 ]; then
             [ -f $LOG ] && sed -e "s/^/log: /" $LOG
             error "mdsrate unlinks multiple nodes failed, aborting"
@@ -142,7 +144,7 @@ fi
 
 complete $SECONDS
 rmdir $BASEDIR || true
-rm -f $MACHINEFILE 
+rm -f $MACHINEFILE
 check_and_cleanup_lustre
 #rm -f $LOG
 
diff --git a/lustre/tests/mdsrate-lookup-10dirs.sh b/lustre/tests/mdsrate-lookup-10dirs.sh
index 6dd5b11..8f772e0 100644
--- a/lustre/tests/mdsrate-lookup-10dirs.sh
+++ b/lustre/tests/mdsrate-lookup-10dirs.sh
@@ -4,7 +4,7 @@
 
 # Directory lookup retrieval rate 10 directories 1 million files each
 # 6000 random lookups/sec per client node 62,000 random lookups/sec aggregate
-# 
+#
 # In 10 dirs containing 1 million files each the mdsrate Test Program will
 # perform lookups for 10 minutes. This test is run from a single node for
 # #1 and from all nodes for #2 aggregate test to measure lookup performance.
@@ -34,7 +34,7 @@ rm -f $LOG
 
 [ ! -x ${MDSRATE} ] && error "${MDSRATE} not built."
 
-log "===== $0 ====== " 
+log "===== $0 ====== "
 
 check_and_setup_lustre
 
@@ -71,7 +71,7 @@ else
 	# This is just a test preparation, does not matter how many threads we
 	# use for files creation; we just should be aware that NUM_DIRS is less
 	# than or equal to the number of threads np
-	mpi_run -np ${NUM_DIRS} ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_DIRS} \
 		${COMMAND} 2>&1
 
     # No lookup if error occurs on file creation, abort.
@@ -88,7 +88,7 @@ if [ -n "$NOSINGLE" ]; then
 else
 	log "===== $0 ### 1 NODE LOOKUPS ###"
 	echo "+" ${COMMAND}
-	mpi_run -np 1 ${MACHINEFILE_OPTION} ${MACHINEFILE} ${COMMAND} |
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
 		tee ${LOG}
 
     if [ ${PIPESTATUS[0]} != 0 ]; then
@@ -104,7 +104,7 @@ if [ -n "$NOMULTI" ]; then
 else
 	log "===== $0 ### ${NUM_CLIENTS} NODES LOOKUPS ###"
 	echo "+" ${COMMAND}
-	mpi_run -np ${NUM_CLIENTS} ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \
 		${COMMAND} | tee ${LOG}
 
     if [ ${PIPESTATUS[0]} != 0 ]; then
diff --git a/lustre/tests/mdsrate-lookup-1dir.sh b/lustre/tests/mdsrate-lookup-1dir.sh
index ac25e72..12f36ce 100644
--- a/lustre/tests/mdsrate-lookup-1dir.sh
+++ b/lustre/tests/mdsrate-lookup-1dir.sh
@@ -4,7 +4,7 @@
 
 # Directory lookup retrieval rate single directory 10 million files
 # 5900 random lookups/sec per client node 62,000 random lookups/sec aggregate
-# 
+#
 # In a dir containing 10 million non-striped files the mdsrate Test Program will
 # perform lookups for 10 minutes. This test can be run from a single node for
 # #1 and from all nodes for #2 aggregate test to measure lookup performance.
@@ -33,7 +33,7 @@ rm -f $LOG
 
 [ ! -x ${MDSRATE} ] && error "${MDSRATE} not built."
 
-log "===== $0 ====== " 
+log "===== $0 ====== "
 
 check_and_setup_lustre
 
@@ -64,7 +64,7 @@ else
     COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir ${TESTDIR}
                         --nfiles ${NUM_FILES} --filefmt 'f%%d'"
 	echo "+" ${COMMAND}
-	mpi_run -np ${NUM_THREADS} ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_THREADS} \
 		${COMMAND} 2>&1
 
     # No lockup if error occurs on file creation, abort.
@@ -80,7 +80,7 @@ if [ -n "$NOSINGLE" ]; then
 else
 	log "===== $0 ### 1 NODE LOOKUPS ###"
 	echo "+" ${COMMAND}
-	mpi_run -np 1 ${MACHINEFILE_OPTION} ${MACHINEFILE} ${COMMAND} |
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
 		tee ${LOG}
 
     if [ ${PIPESTATUS[0]} != 0 ]; then
@@ -96,7 +96,7 @@ if [ -n "$NOMULTI" ]; then
 else
 	log "===== $0 ### ${NUM_CLIENTS} NODES LOOKUPS ###"
 	echo "+" ${COMMAND}
-	mpi_run -np ${NUM_CLIENTS} ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \
 		${COMMAND} | tee ${LOG}
 
     if [ ${PIPESTATUS[0]} != 0 ]; then
diff --git a/lustre/tests/mdsrate-stat-large.sh b/lustre/tests/mdsrate-stat-large.sh
index 19c4375..1b779f4 100644
--- a/lustre/tests/mdsrate-stat-large.sh
+++ b/lustre/tests/mdsrate-stat-large.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # This test was used in a set of CMD3 tests (cmd3-8 test).
 
@@ -8,7 +8,7 @@
 # In a dir containing 10 million striped files, the mdsrate Test Program will
 # perform directory ordered stat's (readdir) for 10 minutes. This test will be
 # run from a single node for #1 and from all nodes for #2 aggregate test to
-# measure stat performance.  
+# measure stat performance.
 
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
@@ -35,7 +35,7 @@ rm -f $LOG
 
 [ ! -x ${MDSRATE} ] && error "${MDSRATE} not built."
 
-log "===== $0 ====== " 
+log "===== $0 ====== "
 
 check_and_setup_lustre
 
@@ -68,7 +68,7 @@ else
         NUM_THREADS=$NUM_CLIENTS
     fi
 
-	mpi_run -np ${NUM_THREADS} ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_THREADS} \
 		${COMMAND} 2>&1
 	[ ${PIPESTATUS[0]} != 0 ] &&
 		error "mdsrate file creation failed, aborting"
@@ -84,7 +84,7 @@ else
     log "===== $0 ### 1 NODE STAT ###"
     echo "+" ${COMMAND}
 
-	mpi_run -np 1 ${MACHINEFILE_OPTION} ${MACHINEFILE} ${COMMAND} |
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
 		tee ${LOG}
 
     if [ ${PIPESTATUS[0]} != 0 ]; then
@@ -101,7 +101,7 @@ else
     log "===== $0 ### ${NUM_CLIENTS} NODES STAT ###"
     echo "+" ${COMMAND}
 
-	mpi_run -np ${NUM_CLIENTS} ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \
 		${COMMAND} | tee ${LOG}
 
     if [ ${PIPESTATUS[0]} != 0 ]; then
diff --git a/lustre/tests/mdsrate-stat-small.sh b/lustre/tests/mdsrate-stat-small.sh
index 0d75135..8617469 100644
--- a/lustre/tests/mdsrate-stat-small.sh
+++ b/lustre/tests/mdsrate-stat-small.sh
@@ -35,7 +35,7 @@ rm -f $LOG
 
 [ ! -x ${MDSRATE} ] && error "${MDSRATE} not built."
 
-log "===== $0 ====== " 
+log "===== $0 ====== "
 
 check_and_setup_lustre
 
@@ -68,7 +68,7 @@ else
         NUM_THREADS=$NUM_CLIENTS
     fi
 
-	mpi_run -np ${NUM_THREADS} ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_THREADS} \
 		${COMMAND} 2>&1
 	[ ${PIPESTATUS[0]} != 0 ] &&
 		error "mdsrate file creation failed, aborting"
@@ -85,7 +85,7 @@ else
     log "===== $0 ### 1 NODE STAT ###"
     echo "+" ${COMMAND}
 
-	mpi_run -np 1 ${MACHINEFILE_OPTION} ${MACHINEFILE} ${COMMAND} |
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
 		tee ${LOG}
 
     if [ ${PIPESTATUS[0]} != 0 ]; then
@@ -102,7 +102,7 @@ else
     log "===== $0 ### ${NUM_CLIENTS} NODES STAT ###"
     echo "+" ${COMMAND}
 
-	mpi_run -np ${NUM_CLIENTS} ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \
 		${COMMAND} | tee ${LOG}
 
     if [ ${PIPESTATUS[0]} != 0 ]; then
diff --git a/lustre/tests/metadata-updates.sh b/lustre/tests/metadata-updates.sh
index cad4473..eab2800 100755
--- a/lustre/tests/metadata-updates.sh
+++ b/lustre/tests/metadata-updates.sh
@@ -254,8 +254,8 @@ if [ -f "$WRITE_DISJOINT" ]; then
 	set $TRACE
 	MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
 	generate_machine_file $NODES_TO_USE $MACHINEFILE
-	mpi_run -np $(get_node_count ${NODES_TO_USE//,/ }) \
-		${MACHINEFILE_OPTION} $MACHINEFILE $WRITE_DISJOINT \
+	mpi_run ${MACHINEFILE_OPTION} $MACHINEFILE \
+		-np $(get_node_count ${NODES_TO_USE//,/ }) $WRITE_DISJOINT \
 		-f $WRITE_DISJOINT_FILE -n $NUMLOOPS || STATUS=1
 else
     skip_env "$0 : write_disjoint not found "
diff --git a/lustre/tests/parallel-scale.sh b/lustre/tests/parallel-scale.sh
index ab526c8..bb366a9 100644
--- a/lustre/tests/parallel-scale.sh
+++ b/lustre/tests/parallel-scale.sh
@@ -25,7 +25,7 @@ fi
 # common setup
 MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
 clients=${CLIENTS:-$HOSTNAME}
-generate_machine_file $clients $MACHINEFILE || \
+generate_machine_file $clients $MACHINEFILE ||
     error "Failed to generate machine file"
 num_clients=$(get_node_count ${clients//,/ })
 
diff --git a/lustre/tests/run_IOR.sh b/lustre/tests/run_IOR.sh
index dd0c831..228926d 100755
--- a/lustre/tests/run_IOR.sh
+++ b/lustre/tests/run_IOR.sh
@@ -44,8 +44,8 @@ while [ ! -e "$END_RUN_FILE" ] && $CONTINUE; do
     # need this only if TESTDIR is not default
     chmod -R 777 $TESTDIR
 
-	mpi_run -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) \
-		${MACHINEFILE_OPTION} ${MACHINEFILE} $IOR -a POSIX -b 1g \
+	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
+		-np $((NUM_CLIENTS * THREADS_PER_CLIENT)) $IOR -a POSIX -b 1g \
 		-o $TESTDIR/IOR-file -s 1 -t 1m -v -w -r 1>$LOG &
     load_pid=$!
     wait $load_pid
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 58a88a4..a7dcc28 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -5376,7 +5376,7 @@ get_mds_dir () {
 
 mdsrate_cleanup () {
 	if [ -d $4 ]; then
-		mpi_run -np $1 ${MACHINEFILE_OPTION} $2 ${MDSRATE} --unlink \
+		mpi_run ${MACHINEFILE_OPTION} $2 -np $1 ${MDSRATE} --unlink \
 			--nfiles $3 --dir $4 --filefmt $5 $6
 		rmdir $4
 	fi
@@ -5389,7 +5389,7 @@ delayed_recovery_enabled () {
 
 ########################
 
-convert_facet2label() { 
+convert_facet2label() {
     local facet=$1
 
     if [ x$facet = xost ]; then
@@ -5400,7 +5400,7 @@ convert_facet2label() {
 
     if [ -n ${!varsvc} ]; then
         echo ${!varsvc}
-    else  
+    else
         error "No lablel for $facet!"
     fi
 }