3 ######################################################################
6 # CHOOSE EITHER scsidevs or rawdevs
7 # the SCSI devices to measure - WARNING: will be erased.
8 # The raw devices to use
9 # rawdevs=${rawdevs:-"/dev/raw/raw1"}
10 # scsidevs=`ls /dev/sd[a-z] /dev/sd[a-z][a-z]` # all devices, if you use udev
13 # NB ensure the path exists on all servers if it includes subdirs
14 rslt_loc=${rslt_loc:-"/tmp"}
15 rslt=${rslt:-"$rslt_loc/sgpdd_survey_`date +%F@%R`"}
17 # what to do (read or write)
18 actions=${actions:-"write read"}
20 # total size per device (MBytes)
21 # NB bigger than device cache is good
24 # record size (KBytes)
28 # Concurrent regions per device
32 # boundary blocks between concurrent regions per device
33 boundary=${boundary:-1024}
35 # threads to share between concurrent regions per device
36 # multiple threads per region simulates a deeper request queue
37 # NB survey skips over #thr < #regions and #thr/#regions > SG_MAX_QUEUE
41 #####################################################################
42 # leave the rest of this alone unless you know what you're doing...
44 # and max # threads one instance will spawn
49 echo "$@" | xargs -n1 echo | sort -u
55 *:*) host=`echo $name | sed 's/:.*$//'`
56 name=`echo $name | sed 's/[^:]*://'`
72 command="export PATH=/sbin:/usr/sbin:\$PATH; $command"
76 if [ -n "$user" ]; then
79 $DSH $user$node "$command"
82 if [ -n "$user" ]; then
85 $DSH $user $node "$command"
90 # how to run commands on other nodes
95 if [ "$host" = "localhost" -o "$host" = `uname -n` ]; then
98 # split $host into $host and $user
100 if [[ $host == *@* ]]; then
104 dsh $host "$user" "$cmds"
109 # check either scsidevs or rawdevs is specified
110 # but only one of them
111 if [ -n "$scsidevs" -a -n "$rawdevs" -o -z "$scsidevs$rawdevs" ]; then
112 echo "Must either specify scsidevs or rawdevs"
116 # retrieve host and device if specified as "hostname:device"
119 for d in $scsidevs $rawdevs; do
120 str=(`split_hostname $d`)
121 hosts[$ndevs]=${str[0]}
122 devs[$ndevs]=${str[1]}
125 unique_hosts=(`unique ${hosts[@]}`)
127 # map given device names into SG device names
128 if [ "$scsidevs" ]; then
129 # make sure sg kernel module is loaded
130 for host in ${unique_hosts[@]}; do
131 sg_is_loaded=$(remote_shell $host grep -q "^sg " /proc/modules \
132 && echo true || echo false)
133 if ! $sg_is_loaded; then
134 echo "loading the sg kernel module on $host"
135 remote_shell $host modprobe sg
136 sg_was_loaded_on="$sg_was_loaded_on $host"
140 for ((i=0; i < $ndevs; i++)); do
141 # resolv symbolic link if any
142 devs[$i]=$(remote_shell ${hosts[$i]} readlink -f ${devs[$i]})
144 # retrieve associated sg device
145 # we will test for a LUN, the test for a partition
146 # if the partition number is > 9 this will fail
147 devs[$i]=$(remote_shell ${hosts[$i]} sg_map | \
148 awk -v dev=${devs[$i]} '{if ($2 == dev) print $1}')
149 if [ -z "${devs[i]}" ]; then
150 echo "Can't find SG device for ${hosts[$i]}:${devs[$i]}, " \
151 "testing for partition"
152 pt=`echo ${devs[$i]} | sed 's/[0-9]*$//'`
154 devs[$i]=$(remote_shell ${hosts[$i]} sg_map | \
155 awk -v dev=$pt '{if ($2 == dev) print $1}')
156 if [ -z "${devs[i]}" ]; then
157 echo -e "Can't find SG device ${hosts[$i]}:$pt.\n" \
158 "Do you have the sg module configured for your kernel?"
163 elif [ "$rawdevs" ]; then
164 for ((i=0; i < $ndevs; i++)); do
165 RES=$(remote_shell ${hosts[$i]} raw -q ${devs[$i]})
167 echo "Raw device ${hosts[$i]}:${devs[$i]} not set up"
173 # determine block size of each device. This should also work for raw devices
174 # If it fails, set to 512
175 for ((i=0; i < $ndevs; i++)); do
176 # retrieve device size (in kbytes) and block size (in bytes)
177 tmp=( `remote_shell ${hosts[$i]} sg_readcap -lb ${devs[$i]}` )
179 if [ ${bs[$i]} == 0 ]; then
180 echo "sg_readcap on device ${hosts[$i]}:${devs[$i]} failed, " \
181 "setting block size to 512"
184 devsize=$((tmp[0]*bs[$i]/1024))
186 # check record size is a multiple of block size
187 if [ $((rszlo*1024%bs[$i])) -ne 0 ]; then
188 echo "Record size is not a multiple of block size (${bs[$i]} bytes) " \
189 "for device ${hosts[$i]}:${devs[$i]}"
194 if [ $devsize -lt $((size*1024)) ]; then
195 echo -e "device ${hosts[$i]}:${devs[$i]} not big enough: " \
196 "$devsize < $((size*1024)).\nConsider reducing \$size"
201 rsltf=${rslt}.summary
208 if [ "$1" = "-n" ]; then
213 echo $minusn "$*" >> $rsltf
217 print_summary "$(date) sgpdd-survey on $rawdevs$scsidevs from $(hostname)"
219 for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do
220 for ((crg=$crglo;crg<=$crghi;crg*=2)); do
221 for ((thr=$thrlo;thr<=$thrhi;thr*=2)); do
222 if ((thr < crg || thr/crg > SG_MAX_QUEUE)); then
225 # compute total size (in kbytes)
227 for ((i=0; i < $ndevs; i++)); do
228 tsize=$((size*1024*1024/bs[$i]/crg*crg*bs[$i]/1024))
229 total_size=$((total_size+tsize))
231 # show test parameters
232 str=`printf 'dev %2d sz %8dK rsz %4dK crg %5d thr %5d ' \
233 $ndevs $total_size $rsz $((crg*ndevs)) $((thr*ndevs))`
234 echo "==============> $str" >> $workf
235 print_summary -n "$str"
237 # check memory for each host
238 for host in ${unique_hosts[@]}; do
240 for ((i=0; i < $ndevs; i++)); do
241 if [ ${hosts[$i]} == $host ]; then
242 numdevs=$((numdevs+1))
245 freemem=$(remote_shell $host cat /proc/meminfo | \
246 awk '/^MemTotal:/ {printf "%d\n", $2}')
247 if (((rsz*thr/crg + 64)*crg*numdevs > freemem)); then
248 echo "ENOMEM on $host" >> $workf
249 print_summary "ENOMEM"
255 for action in $actions; do
257 print_summary -n "$action "
258 echo "=====> $action" >> $workf
261 # create per-host script files
262 for host in ${unique_hosts[@]}; do
263 echo -n > ${cmdsf}_${host}
265 for ((i=0; i < $ndevs; i++)); do
266 bpt=$((rsz*1024/bs[$i]))
267 blocks=$((size*((1024*1024)/bs[$i])/crg))
271 if [ $action = read ]; then
280 for ((j=0;j<crg;j++)); do
281 echo >> ${cmdsf}_${host} \
282 "sgp_dd 2> ${tmpf}_${i}_${j} $inf $outf " \
283 "${skip}=$((boundary+j*blocks)) " \
284 "thr=$((thr/crg)) count=$count bs=${bs[$i]} " \
288 for host in ${unique_hosts[@]}; do
289 echo "wait" >> ${cmdsf}_${host}
292 # run of all the per-host script files
295 for host in ${unique_hosts[@]}; do
296 remote_shell $host bash < ${cmdsf}_${host} &
297 pidarray[$pidcount]=$!
298 pidcount=$((pidcount+1))
301 for host in ${unique_hosts[@]}; do
302 wait ${pidarray[$pidcount]}
303 pidcount=$((pidcount+1))
307 # clean up per-host script files
308 for host in ${unique_hosts[@]}; do
312 # collect/check individual stats
315 for ((i=0;i<ndevs;i++)); do
316 for ((j=0;j<crg;j++)); do
317 rtmp=${tmpf}_${i}_${j}_local
318 remote_shell ${hosts[$i]} cat ${tmpf}_${i}_${j} > $rtmp
319 if grep 'error' $rtmp > /dev/null 2>&1; then
320 echo "Error found in $rtmp"
321 elif grep 'time to transfer data' $rtmp > /dev/null 2>&1; then
325 cat ${rtmp} >> $workf
327 remote_shell ${hosts[$i]} rm ${tmpf}_${i}_${j}
330 if ((ok != ndevs*crg)); then
331 print_summary -n "$((ndevs*crg - ok)) failed "
333 # compute MB/sec from elapsed
334 bw=`awk "BEGIN {printf \"%7.2f MB/s\", \
335 $total_size / (( $t1 - $t0 ) * 1024); exit}"`
336 # compute MB/sec from nregions*slowest
338 '/time to transfer data/ {mb=$8/1.048576; \
339 if (n == 0 || mb < min) min = mb; n++}\
340 END {printf "%5d x %6.2f = %7.2f MB/s", n, min, min * n}'`
341 print_summary -n "$bw $check "
350 for host in $sg_was_loaded_on; do
351 echo "unloading sg module on $host"
352 remote_shell $host rmmod sg