X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre-iokit%2Fior-survey%2Fior-survey;h=c632d31d9f08f338b06050723c42704ba3ddf495;hb=455f30ac2817d342125d886e54c67abd07133a73;hp=03595a9fa0470c87060e9397390f5261d440a1f8;hpb=56819ed59edf3e904e3eda7d72ab5be8abb49e91;p=fs%2Flustre-release.git

diff --git a/lustre-iokit/ior-survey/ior-survey b/lustre-iokit/ior-survey/ior-survey
index 03595a9..c632d31 100644
--- a/lustre-iokit/ior-survey/ior-survey
+++ b/lustre-iokit/ior-survey/ior-survey
@@ -1,73 +1,63 @@
 #!/bin/bash
 
+# the value of these can be set in the environment.
+# This test assumes a typical pdsh naming scheme, where
+# node names can be expressed as a string
+# forllowed by a number 
 # cluster name (all node names are this followed by the node number)
-cluster=nid000
+cluster=${cluster:-""}
+if [ -z "$cluster" ];then  echo  "cluster not defined"; exit 1; fi
 
 # client node numbers (individual numbers or inclusive ranges)
-clients=(1-64)
+clients=${clients:-""}
+if [ -z "$clients" ]; then echo "clients not defined"; exit 1; fi
 
 # numbers of clients to survey
-clients_lo=1
-clients_hi=64
-clients_iterator="*=2"
+clients_lo=${clients_lo:-1}
+clients_hi=${clients_hi:-3}
+clients_iterator=${clients_iterator:-"+=1"}
 
 # numbers of tasks per client to survey
-tasks_per_client_lo=1
-tasks_per_client_hi=1
-tasks_per_client_iterator="*=2"
+tasks_per_client_lo=${task_per_client_lo:-1}
+tasks_per_client_hi=${tasks_per_client_hi:-8}
+tasks_per_client_iterator=${tasks_per_client_iterator:-"*=2"}
 
 # record sizes to survey
-rsize_lo=1M
-rsize_hi=1M
-rsize_iterator="*=2"
+rsize_lo=${rsize_lo:-1M}
+rsize_hi=${rsize_hi:-1M}
+rsize_iterator=${rsize_iterator:-"*=2"}
 
-# This line contains all of the possible tests.
-# IMPORTANT:::if you want to remount, put it in your tests array
-# in the order you want to remount to clear the cache.  For ex:
-# tests=(write rewrite remount read reread rewrite_again) has a
-# remount between the rewrite and remount tests
-tests=(write read)
+## which tests to run (first must be write)
+# clear_cache)   not really a test; just uncache everything
+# *write*)   write
+# *)         read
+tests=(write rewrite clear_cache read reread)
 
 # total # bytes written/read by any client node
-min_per_client_size=1G
-min_total_size=1G
+min_per_client_size=${min_per_client_size:-"1G"}
+min_total_size=${min_total_size:-"2G"}
 
 # should each task do I/O to its own file?
-file_per_task=1
+file_per_task=${file_per_task:-1}
 
 # the binaries
-IOR="/spin/home/henken/IOR-2.8.4/src/C/IOR"
-llmount=llmount
-
-#Command to run IOR (pdsh,mpirun,yod)
-runior="yod"
-#Path to binary for program specified in runior
-pathtobin="$(which yod)"
-#location of machines file for mpirun, this file
-#will be built from the cluster and client ranges
-#above
-machines=machines
+IOR=${IOR:-"/usr/local/sbin/IOR"}
+llmount=${llmount:-"llmount"}
+# Select mpirun, yod, or pdsh
+fanout_cmd=${fanout_cmd:-"pdsh"}
+# mpirun still uses pdsh for cleanup
+pdsh=${pdsh:-"pdsh"}
+pdsh_args="-R ssh -S -b -w "
 
 # the result file prefix (date/time + hostname makes unique)
-rslt=/spin/home/henken/ior_survey/ior_survey_`date +%F@%R`_`uname -n`
-#rslt=/home/larry/ior_survey
+rslt_loc=${rslt_loc:-"/tmp"}
+rslt=${rslt:-"$rslt_loc/ior_survey_`date +%F@%R`_`uname -n`"}
 
 # where lustre is mounted on the clients
-lustre=/lustre/fs1/nic/
+lustre=${lustre:-"/mnt/lustre"}
 
 # basename of the test file(s)
-testfile=${lustre}/ior_survey_testfile
-
-
-# how to unmount and remount the F/S on a client (to clear the cache)
-# change this depending on lustre config (network type, MDS etc)
-unmount="umount $lustre"
-remount="llmount pegasus:/mds1/client $lustre"
-
-# pdsh args required to instantiate all instances of IOR in parallel
-# the chosen module must support '-n <procs-per-node>'
-# -R<module>, -f<fanout> etc
-pdsh_mpiargs="-Rmqsh"
+testfile=${testfile:-"${lustre}/ior/ior_survey_testfile"}
 
 #don't spin for MPI completions
 export LIBELAN_WAITTYPE=0
@@ -75,6 +65,53 @@ export LIBELAN_WAITTYPE=0
 ################################################################################
 # dont change stuff below here unless you know what you're doing...
 
+# This is to allow use of yod, pdsh, etc.
+fanout() {
+	local clients=$1; shift
+	local tmpfile=$1; shift
+	local result
+	case $fanout_cmd in
+	'pdsh')
+		$fanout_cmd $pdsh_args "$clients" "$@" >> $tmpfile 2>&1
+		echo $?
+		return
+		;;
+	'mpirun')
+		# horrible misuse of globals
+		$fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1
+		echo $?
+		return
+		;;
+	'yod')
+		# and another
+		$fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1
+		echo $?
+		return
+		;;
+		
+	*)
+		echo "255"
+		return
+		;;
+	esac
+}
+
+dump_cache() {
+	# we are assuming mpi uses will also have pdsh
+	local clients=$1;shift
+	local tmpfile=$1;shift
+	clear_cache='lctl set_param ldlm.namespaces.*.lru_size=clear'
+        echo "=> $clear_cache" >> $tmpfile
+        $pdsh $pdsh_args "$test_clients" "$clear_cache" >> $tmpfile 2>&1 
+        status=$?
+        echo "Completion Status: $status" >> $tmpfile
+
+        if ((status)); then
+              echo "ERROR"
+        else
+              echo "OK"
+	fi
+}
 count_range() {
     echo $1 | awk '{ nvals=split($1, vals, "-");\
 	             if (nvals == 1) print 1;\
@@ -93,7 +130,7 @@ idx2nodenum() {
 	    return
 	fi
 	chunk=`count_range $range`
-	if ((chunk > n)); then
+	if ((chunk > $n)); then
 	    base=`base_range $range`
 	    echo $((base + n))
 	    return
@@ -113,7 +150,7 @@ n2noderange() {
 	fi
 	local base=`base_range $range`
 	local chunk=`count_range $range`
-	if ((chunk > n)); then chunk=n; fi
+	if ((chunk > $n)); then chunk=$n; fi
 	local nodes="${nodes}${sep}${base}"; sep=","
 	if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
 	n=$((n-chunk))
@@ -149,11 +186,11 @@ pp_number() {
     local G=$((1024*1024*1024))
     local M=$((1024*1024))
     local K=$((1024))
-    if ((n%G == 0 && n >= G)); then
+    if ((n%G == 0 && n >= $G)); then
 	echo "$((n/G))G"
-    elif ((n%M == 0 && n >= M)); then
+    elif ((n%M == 0 && n >= $M)); then
 	echo "$((n/M))M"
-    elif ((n%K == 0 && n >= K)); then
+    elif ((n%K == 0 && n >= $K)); then
 	echo "$((n/K))K"
     else
 	echo $n
@@ -180,37 +217,21 @@ print_summary () {
     echo $minusn "$*"
 }
 
-mpi_client_file() {
-	echo -n > $machines
-	local base=`base_range $1`
-	echo $base
-	local chunk=`count_range $1`
-	echo $chunk
-	local high=$((base+chunk-1))
-	echo $high
-	for ((nmpi=$base; nmpi<=$high; nmpi++)); do
-		echo $cluster$nmpi >> $machines
-	done
-}
+check_mount() {
+    local lustre=$1; shift
+    local tmpb=$1; shift
+    local clients=$1; shift
+    local tmpfile=${tmpb}_tmp
+    # check lustre is mounted everywhere it's needed
+   cmd="grep $lustre /proc/mounts"
+   $pdsh $pdsh_args "$clients" "$cmd" >> $tmpfile
+   status=$?
+   	if (($status)); then
+		print_summary "Lustre NOT mounted on $lustre somewhere"
+	exit 1
+   	fi
 
-parse_cmdline() {
-    case $runior in
-		'mpirun') 
-		#echo "this"
-		$pathtobin -np $((ntask*nclnt)) -machinefile $machines >> $tmpf 2>1 \
-			"${cmdline[@]}";;
-		'pdsh')
-		$pathtobin -S -b $pdsh_mpiargs -w "$test_clients" -n $ntask \
-			>> $tmpf 2>&1 "${cmdline[@]}";;
-		'yod')
-			$pathtobin -np $((ntask*nclnt)) >> $tmpf 2>&1 "${cmdline[@]}";;
-	esac	
 }
-
-if [ $runior = "mpirun" ]; then
-	mpi_client_file ${clients[@]}
-fi
-
 # convert params to actual numbers
 min_per_client_size=`parse_number $min_per_client_size`
 min_total_size=`parse_number $min_total_size`
@@ -220,20 +241,30 @@ rsize_hi=`parse_number $rsize_hi`
 
 # check on actual numbers of client nodes
 nclients=`countnodes ${clients[@]}`
-if ((clients_hi > nclients)); then clients_hi=$nclients; fi
+if ((clients_hi > $nclients)); then clients_hi=$nclients; fi
+
+cur_date=`date`
+machine=`uname -n`
+script_name=`echo $0 | cut -d "/" -f2`
 
-for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
+echo "$cur_date $script_name on $lustre from $machine" >> $workf
+
+for ((rsize=$rsize_lo; rsize<=$rsize_hi; rsize$rsize_iterator)); do
     pp_rsize=`pp_number $rsize`
 
-    for ((nclnt=clients_lo; nclnt<=clients_hi; nclnt$clients_iterator)); do
+    for ((nclnt=$clients_lo; nclnt<=$clients_hi; nclnt$clients_iterator)); do
 	test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`"
-	 
+	echo $test_clients
+	if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ];then
+		check_mount $lustre $workf $test_clients
+	fi
 	per_client_size=$((min_total_size/nclnt))
-	if ((per_client_size < min_per_client_size)); then
+	if ((per_client_size < $min_per_client_size)); then
 	    per_client_size=$min_per_client_size
 	fi
 
-	for ((ntask=tasks_per_client_lo; ntask <= tasks_per_client_hi; ntask$tasks_per_client_iterator)); do
+	for ((ntask=$tasks_per_client_lo; ntask <= $tasks_per_client_hi; \
+		ntask$tasks_per_client_iterator)); do
 	    per_task_size=$((per_client_size/ntask))
 	    if ((per_task_size%rsize != 0)); then
 		per_task_size=$(((per_task_size/rsize + 1)*rsize))
@@ -252,41 +283,14 @@ for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
 		tmpf=${workf}_tmp
 		echo -n > $tmpf
 
-		if [ "$test" = "remount" ]; then
-		    echo "=> $remount" >> $tmpf
-		    if [ "$runior" = "pdsh" ]; then
-				$pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
-					"$unmount"
-				$pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
-					"$remount"
-		    else
-				$unmount
-				$remount
-		    fi
-		    status=$?
-		    echo "Completion Status: $status" >> $tmpf
-
-		    if ((status)); then
-			result="ERROR"
-		    else
-			result="OK"
-		    fi
+		if [ "$test" = "clear_cache" ]; then
+		    if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ]; then
+				result=`dump_cache $test_clients $tmpf`
+		     else
+				echo "Haven't figured out how to clear cache" >> $tmpf
+				result="N/A"
+		     fi
 		else
-		    # check lustre is mounted everywhere it's needed
-		    cmd="(mount -t lustre; mount -t lustre_lite) | grep $lustre"
-		    echo "=> Mount Check: $cmd" >> $tmpf
-		    if [ "$runior" = "pdsh" ]; then
-				$pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
-					"$cmd"
-		    fi
-		    status=$?
-		    echo "Completion Status: $status" >> $tmpf
-		    if ((status)); then
-			cat $tmpf >> $workf
-			rm $tmpf
-			print_summary "Lustre NOT mounted on $lustre somewhere"
-			exit 1
-		    fi
 
 		    cmdline=(
 		    $IOR                     # the command
@@ -300,13 +304,14 @@ for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
 		    idx=${#cmdline[@]}
 
                     # keep the test file(s) unless this is the last test
-		    ((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
+		    #((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
+		    cmdline[$((idx++))]="-k"
 
 		    # use the existing test file(s) unless this is the first test
 		    ((test_idx > 0)) && cmdline[$((idx++))]="-E"
 
 		    # file-per-task
-		    ((file_per_task)) && cmdline[$((idx++))]="-F"
+		    (($file_per_task)) && cmdline[$((idx++))]="-F"
 
 		    case "$test" in
 		    *write*) cmdline[$((idx++))]="-w"
@@ -316,17 +321,22 @@ for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
                     esac
 
 		    echo "=> ${cmdline[@]}" >> $tmpf
-
-		    parse_cmdline 	
-		    status=$?
+	
+		    status=`fanout $test_clients $tmpf ${cmdline[@]}`
 
 		    echo "Completion Status: $status" >> $tmpf
 	       
-		    if ((status)); then
+		    if (($status)); then
 			result="ERROR"
 		    else
-			result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
+			# pdsh adds an extra field
+			if [ "$fanout_cmd" = "pdsh" ]; then 
+				result=`awk < $tmpf "/$awkstr/ {print $ 4; found=1; exit}\
+			                     END       {if (!found) print \"ERROR\"}"`
+			else
+				result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
 			                     END       {if (!found) print \"ERROR\"}"`
+			fi
 		    fi
 		fi