3 ######################################################################
6 # CHOOSE EITHER scsidevs or rawdevs
7 # the SCSI devices to measure - WARNING: will be erased.
8 # The raw devices to use
9 # rawdevs=${rawdevs:-"/dev/raw/raw1"}
10 # scsidevs=`ls /dev/sd[a-z] /dev/sd[a-z][a-z]` # all devices, if you use udev
13 # NB ensure the path exists on all servers if it includes subdirs
14 rslt_loc=${rslt_loc:-"/tmp"}
15 rslt=${rslt:-"$rslt_loc/sgpdd_survey_`date +%F@%R`"}
17 # what to do (read or write)
18 actions=${actions:-"write read"}
20 # total size per device (MBytes)
21 # NB bigger than device cache is good
24 # record size (KBytes)
28 # Concurrent regions per device
32 # boundary blocks between concurrent regions per device
33 boundary=${boundary:-1024}
35 # threads to share between concurrent regions per device
36 # multiple threads per region simulates a deeper request queue
37 # NB survey skips over #thr < #regions and #thr/#regions > SG_MAX_QUEUE
42 # User provided script that returns a cpu list from a specified device.
43 # Implementation depends on the type of device (scsi/raw, with/without
44 # multipath, technology fc/sas/ib)
49 # pci=$(readlink -f /sys/class/block/$dev | cut -d/ -f1-5)
50 # cat ${pci}/local_cpulist
51 dev2cpus=${dev2cpus:-""}
53 #####################################################################
54 # leave the rest of this alone unless you know what you're doing...
56 # and max # threads one instance will spawn
60 NUMACTL=${NUMACTL:-"/usr/bin/numactl"}
63 echo "$@" | xargs -n1 echo | sort -u
69 *:*) host=`echo $name | sed 's/:.*$//'`
70 name=`echo $name | sed 's/[^:]*://'`
86 command="export PATH=/sbin:/usr/sbin:\$PATH; $command"
90 if [ -n "$user" ]; then
93 $DSH $user$node "$command"
96 if [ -n "$user" ]; then
99 $DSH $user $node "$command"
104 # how to run commands on other nodes
109 if [ "$host" = "localhost" -o "$host" = `uname -n` ]; then
112 # split $host into $host and $user
114 if [[ $host == *@* ]]; then
118 dsh $host "$user" "$cmds"
123 # check either scsidevs or rawdevs is specified
124 # but only one of them
125 if [ -n "$scsidevs" -a -n "$rawdevs" -o -z "$scsidevs$rawdevs" ]; then
126 echo "Must either specify scsidevs or rawdevs"
130 # retrieve host and device if specified as "hostname:device"
133 for d in $scsidevs $rawdevs; do
134 str=(`split_hostname $d`)
135 hosts[$ndevs]=${str[0]}
136 devs[$ndevs]=${str[1]}
139 unique_hosts=(`unique ${hosts[@]}`)
141 # get device cpu list
143 if [ -n "$dev2cpus" ]; then
144 for ((i=0; i < $ndevs; i++)); do
145 devcpus[$i]=$(remote_shell ${hosts[$i]} $dev2cpus ${devs[$i]})
149 # map given device names into SG device names
150 if [ "$scsidevs" ]; then
151 # make sure sg kernel module is loaded
152 for host in ${unique_hosts[@]}; do
153 sg_is_loaded=$(remote_shell $host grep -q "^sg " /proc/modules \
154 && echo true || echo false)
155 if ! $sg_is_loaded; then
156 echo "loading the sg kernel module on $host"
157 remote_shell $host modprobe sg
158 sg_was_loaded_on="$sg_was_loaded_on $host"
162 for ((i=0; i < $ndevs; i++)); do
163 # resolve symbolic link if any
164 devs[$i]=$(remote_shell ${hosts[$i]} readlink -f ${devs[$i]})
166 # retrieve associated sg device
167 # we will test for a LUN, the test for a partition
168 # if the partition number is > 9 this will fail
169 tmp=$(remote_shell ${hosts[$i]} sg_map | \
170 awk -v dev=${devs[$i]} '{if ($2 == dev) print $1}')
171 if [ -z "$tmp" ]; then
172 echo "Can't find SG device for ${hosts[$i]}:${devs[$i]}, " \
173 "testing for partition"
174 pt=`echo ${devs[$i]} | sed 's/[0-9]*$//'`
176 tmp=$(remote_shell ${hosts[$i]} sg_map | \
177 awk -v dev=$pt '{if ($2 == dev) print $1}')
178 if [ -z "$tmp" ]; then
179 echo -e "Can't find SG device ${hosts[$i]}:$pt.\n" \
180 "Do you have the sg module configured for your kernel?"
186 elif [ "$rawdevs" ]; then
187 for ((i=0; i < $ndevs; i++)); do
188 RES=$(remote_shell ${hosts[$i]} raw -q ${devs[$i]})
190 echo "Raw device ${hosts[$i]}:${devs[$i]} not set up"
196 # determine block size of each device. This should also work for raw devices
197 # If it fails, set to 512
198 for ((i=0; i < $ndevs; i++)); do
199 # retrieve device size (in kbytes) and block size (in bytes)
200 tmp=( `remote_shell ${hosts[$i]} sg_readcap -lb ${devs[$i]}` )
202 if [ ${bs[$i]} == 0 ]; then
203 echo "sg_readcap on device ${hosts[$i]}:${devs[$i]} failed, " \
204 "setting block size to 512"
207 devsize=$((tmp[0]*bs[$i]/1024))
209 # check record size is a multiple of block size
210 if [ $((rszlo*1024%bs[$i])) -ne 0 ]; then
211 echo "Record size is not a multiple of block size (${bs[$i]} bytes) " \
212 "for device ${hosts[$i]}:${devs[$i]}"
217 if [ $devsize -lt $((size*1024)) ]; then
218 echo -e "device ${hosts[$i]}:${devs[$i]} not big enough: " \
219 "$devsize < $((size*1024)).\nConsider reducing \$size"
224 rsltf=${rslt}.summary
231 if [ "$1" = "-n" ]; then
236 echo $minusn "$*" >> $rsltf
240 print_summary "$(date) sgpdd-survey on $rawdevs$scsidevs from $(hostname)"
242 for ((rsz=$rszlo;rsz<=$rszhi;rsz*=2)); do
243 for ((crg=$crglo;crg<=$crghi;crg*=2)); do
244 for ((thr=$thrlo;thr<=$thrhi;thr*=2)); do
245 if ((thr < crg || thr/crg > SG_MAX_QUEUE)); then
248 # compute total size (in kbytes)
250 for ((i=0; i < $ndevs; i++)); do
251 tsize=$((size*1024*1024/bs[$i]/crg*crg*bs[$i]/1024))
252 total_size=$((total_size+tsize))
254 # show test parameters
255 str=`printf 'dev %2d sz %8dK rsz %4dK crg %5d thr %5d ' \
256 $ndevs $total_size $rsz $((crg*ndevs)) $((thr*ndevs))`
257 echo "==============> $str" >> $workf
258 print_summary -n "$str"
260 # check memory for each host
261 for host in ${unique_hosts[@]}; do
263 for ((i=0; i < $ndevs; i++)); do
264 if [ ${hosts[$i]} == $host ]; then
265 numdevs=$((numdevs+1))
268 freemem=$(remote_shell $host cat /proc/meminfo | \
269 awk '/^MemTotal:/ {printf "%d\n", $2}')
270 if (((rsz*thr/crg + 64)*crg*numdevs > freemem)); then
271 echo "ENOMEM on $host" >> $workf
272 print_summary "ENOMEM"
278 for action in $actions; do
280 print_summary -n "$action "
281 echo "=====> $action" >> $workf
284 # create per-host script files
285 for host in ${unique_hosts[@]}; do
286 echo -n > ${cmdsf}_${host}
288 for ((i=0; i < $ndevs; i++)); do
289 bpt=$((rsz*1024/bs[$i]))
290 blocks=$((size*((1024*1024)/bs[$i])/crg))
294 if [ $action = read ]; then
303 if [ -n "${devcpus[$i]}" -a -x "$NUMACTL" ]; then
304 numacmd="$NUMACTL --physcpubind=${devcpus[$i]} --localalloc"
308 for ((j=0;j<crg;j++)); do
309 echo >> ${cmdsf}_${host} \
311 "sgp_dd 2> ${tmpf}_${i}_${j} $inf $outf " \
312 "${skip}=$((boundary+j*blocks)) " \
313 "thr=$((thr/crg)) count=$count bs=${bs[$i]} " \
317 for host in ${unique_hosts[@]}; do
318 echo "wait" >> ${cmdsf}_${host}
321 # run of all the per-host script files
324 for host in ${unique_hosts[@]}; do
325 remote_shell $host bash < ${cmdsf}_${host} &
326 pidarray[$pidcount]=$!
327 pidcount=$((pidcount+1))
330 for host in ${unique_hosts[@]}; do
331 wait ${pidarray[$pidcount]}
332 pidcount=$((pidcount+1))
336 # clean up per-host script files
337 for host in ${unique_hosts[@]}; do
341 # collect/check individual stats
344 for ((i=0;i<ndevs;i++)); do
345 for ((j=0;j<crg;j++)); do
346 rtmp=${tmpf}_${i}_${j}_local
347 remote_shell ${hosts[$i]} cat ${tmpf}_${i}_${j} > $rtmp
348 if grep 'error' $rtmp > /dev/null 2>&1; then
349 echo "Error found in $rtmp"
350 elif grep 'time to transfer data' $rtmp > /dev/null 2>&1; then
354 cat ${rtmp} >> $workf
356 remote_shell ${hosts[$i]} rm ${tmpf}_${i}_${j}
359 if ((ok != ndevs*crg)); then
360 print_summary -n "$((ndevs*crg - ok)) failed "
362 # compute bandwidth in MiB/s from total data / elapsed time
363 bw=`awk "BEGIN {printf \"%7.2f \", \
364 $total_size / (( $t1 - $t0 ) * 1024); exit}"`
365 # compute global min/max stats
366 minmax=`awk < $tmpf \
367 '/time to transfer data/ {mb=$8/1.048576; \
368 if (n == 0 || mb < min) min = mb; \
369 if (n == 0 || mb > max) max = mb; \
371 END {printf "[ %7.2f, %7.2f] ",min,max;}'`
372 print_summary -n "$bw $minmax "
381 for host in $sg_was_loaded_on; do
382 echo "unloading sg module on $host"
383 remote_shell $host rmmod sg