From ea3381f9ce751d78fc7a68119eab22ac15520791 Mon Sep 17 00:00:00 2001 From: nathan Date: Mon, 30 Oct 2006 22:29:58 +0000 Subject: [PATCH] b=10959 r=nathan Adding perl scripts to do simple "dd" test on each ost, and plot the results. (Originally by Linsyssoft) --- lustre-iokit/ost-survey/README | 53 +++----- lustre-iokit/ost-survey/ost-survey.pl | 244 ++++++++++++++++++++++++++++++++++ lustre-iokit/ost-survey/ost-survey.sh | 193 --------------------------- lustre-iokit/ost-survey/plot-ost.pl | 79 +++++++++++ 4 files changed, 340 insertions(+), 229 deletions(-) create mode 100755 lustre-iokit/ost-survey/ost-survey.pl delete mode 100755 lustre-iokit/ost-survey/ost-survey.sh create mode 100755 lustre-iokit/ost-survey/plot-ost.pl diff --git a/lustre-iokit/ost-survey/README b/lustre-iokit/ost-survey/README index 7fa66db..1541939 100644 --- a/lustre-iokit/ost-survey/README +++ b/lustre-iokit/ost-survey/README @@ -1,45 +1,26 @@ -This script is to be used to test the performance of the -individual OSTs in a Lustre filesystem. - -Goal: - - Generate a fixed amount of IO per OST - - Identify the fastest and slowest OST +ost-survey.pl (OST performance survey) +====================================== + This script is designed to test the client-to-disk performance +of the individual OSTs in a Lustre filesystem. The network transfer +time from the client is included; to get a better idea of the isolated +disk perfomance, run this script on a client running on the OST. -Parameters - - Size of per-OST IO - -Plan +Syntax: + $ ost-survey [-h] [-s ] + where -s : size in MB + -h : help + : mount point of lustre client - Identify number of OSTs in system - For each OST - - create a directory - - use lfs setstripe to restrict IO to that directory - - run some form of IO - Assumptions - Lustre filesystem is up and running - Script is being run on a client -Steps - - 0. Check pre-requisites - - sgp_dd - 1. Identify the number of OSTs in the system - - store value +plot-ost.pl (OST survey graph) +====================================== + The plot-ost.pl script can be used to plot the results from the +ost-survey script using gnuplot. - 2. Collect input parameters - - store value +Syntax: $ ost-survey /mnt/lustre > ost_log + $ plot-ost.pl ost_log - 3. Set up array to hold output - - 4. Loop over all OSTs - - Create a directory - - Set striping on that directory - - Run some IO and time - - Resolve MB/s - - Store in array - - 5. Determine max an min for the array - - diff --git a/lustre-iokit/ost-survey/ost-survey.pl b/lustre-iokit/ost-survey/ost-survey.pl new file mode 100755 index 0000000..38625a9d --- /dev/null +++ b/lustre-iokit/ost-survey/ost-survey.pl @@ -0,0 +1,244 @@ +#!/usr/bin/perl +# This script is to be run on a client machine and will test all the +# OSTs to determine which is the fastest and slowest +# The current test method is as follows: +# -Create a directory for each OST +# -Use 'lfs setstripe' to set the Lustre striping such that IO goes to +# only one OST +# -Use 'dd' to write and read a file of a specified size +# -Compute the average, and Standard deviation +# -Find the slowest OST for read and write +# -Find the Fastest OST for read and write + +# GLOBALS +$pname = $0; # to hold program name +$OSTS = 0; # Number of OSTS we will loop over +$BSIZE = 1024 * 1024; # Size of i/o block +$MNT = "/mnt/lustre"; # Location of Lustre file system +$CNT = 0; +$FSIZE = 30; # Number of i/o blocks + +# Usage +sub usage () { + print "Usage: $pname [-s ] [-h] \n"; + print "[OPTIONS]\n"; + print " -s: size of test file in MB (default $FSIZE MB)\n"; + print " -h: To display this help\n"; + print "example : $pname /mnt/lustre\n"; + exit 1; +} + +# ost_count subroutine ets globle variable $OST with Number of OST's +# Also fills 1 for active OST indexes in ACTIVEOST_INX array. +sub ost_count () { + # numobd gives number of ost's and activeobd gives number of active ost's + my $tempfile = glob ("/proc/fs/lustre/lov/lustre-clilov-*/activeobd"); + open(PTR, $tempfile) || die "Cannot open $tempfile: $!\n"; + $OSTS = ; + close PTR; + print "Number of Active OST devices : $OSTS"; + my $tempfile = glob ("/proc/fs/lustre/lov/lustre-clilov-*/numobd"); + open(PTR, $tempfile) || die "Cannot open $tempfile: $!\n"; + $numost = ; + close PTR; + if ( $numost != $OSTS ) { + printf "Number of non active ots(s): %d\n", ( $numost - $OSTS ); + $OSTS = $numost; + } + my $tempfile = glob ("/proc/fs/lustre/lov/lustre-clilov-*/target_obd"); + open(PTR, $tempfile) || die "Cannot open $tempfile: $!\n"; + my $count = 0; + my $temp; + while () { + chop; + my ($ost_num, $ost_name, $ost_status) = split(/\s+/, $_); + if ( $ost_status eq "ACTIVE" ) { + $ACTIVEOST_INX[$count] = 1; + } + $count++; + } +} + +# make_dummy subroutine creates a dummy file that will be used for read operation. +sub make_dummy () { + my $SIZE = $_[0]; + my $tempfile = $_[1]; + system ("dd of=$tempfile if=/dev/zero count=$SIZE bs=$BSIZE 2> /dev/null"); +} + +# run_test subroutine actually writes and reads data to/from dummy file +# and compute corresponding time taken for read and write operation and +# byte transfer for the both operations. +# This subroutine also fill corresponding globle arrays with above information. +sub run_test () { + my $SIZE = $_[0]; + my $INX=$_[1]; + my $ACTION=$_[2]; + my $tempfile = $_[3]; + + if ( !(-f $tempfile) && $ACTION eq "read" ) { + &make_dummy($SIZE, $tempfile); + } + my $LoadTimeHiRes = "use Time::HiRes qw(gettimeofday)"; + eval ($LoadTimeHiRes); + system("sync"); + my ($ts0, $tu0) = gettimeofday(); + $tu0 = $ts0 + ($tu0 / 1000000); + if ( $ACTION eq "write" ) { + system("dd of=$tempfile if=/dev/zero count=$SIZE bs=$BSIZE 2> /dev/null"); + } elsif ( $ACTION eq "read" ) { + system("dd if=$tempfile of=/dev/null count=$SIZE bs=$BSIZE 2> /dev/null"); + } else { + print "Action is neither read nor write\n"; + exit 1; + } + system("sync"); + my ($ts1, $tu1) = gettimeofday(); + $tu1 = $ts1 + ($tu1/1000000); + my $tdelta = $tu1 - $tu0; + my $delta = ($SIZE * $BSIZE / ( $tu1 - $tu0 )) / (1024 * 1024); + if ( $ACTION eq "write" ) { + $wTime[$INX] = $tdelta; + $wMBs[$INX] = $delta; + } else { + $rTime[$INX] = $tdelta; + $rMBs[$INX] = $delta; + } +} + +# calculate subroutine compute following things and displays them. +# - Finds worst and best OST for both read and write operations. +# - Compute average of read and write rate from all OSTS +# - Compute Standard deviation for read and write form all OST's +sub calculate () { + my ($op, $MBs); + $op = $_[0]; + @MBs = @_[1..$#_]; + my $count = 0; + my $total = 0; + my $avg = 0; + my $sd = 0; + my $best_OST = 0; + my $worst_OST = 0; + my $max_mb = 0; + my $min_mb = 999999999; + while ($count < $OSTS ) { + if ( $ACTIVEOST_INX[$count] ) { + $total = $total + $MBs[$count]; + if ($max_mb < $MBs[$count] ) { + $max_mb = $MBs[$count]; + $best_OST = $count; + } + if ($min_mb > $MBs[$count] ) { + $min_mb = $MBs[$count]; + $worst_OST = $count; + } + } + $count++; + } + $avg = $total/$OSTS; + $total = 0; + $count = 0; + while ($count < $OSTS ) { + if ( $ACTIVEOST_INX[$count] ) { + $total = $total + ($MBs[$count] - $avg) * ($MBs[$count] - $avg); + } + $count++; + } + $sd = sqrt($total/$OSTS); + printf "Worst %s OST indx: %d speed: %f\n", $op, $worst_OST, $min_mb; + printf "Best %s OST indx: %d speed: %f\n", $op, $best_OST, $max_mb; + printf "%s Avrage: %f +/- %f MB/s\n", $op, $avg, $sd; +} + +# output_all_data subroutine displays speed and time information +# for all OST's for both read and write operations. +sub output_all_data () { + my $count = 0; + print "Ost# Read(MB/s) Write(MB/s) Read-time Write-time\n"; + print "----------------------------------------------------\n"; + while ( $count < $OSTS ) { + if ( $ACTIVEOST_INX[$count] ) { + printf "%d %.3f %.3f %.3f %.3f\n",$count, + $rMBs[$count], $wMBs[$count], $rTime[$count], $wTime[$count]; + } else { + printf "%d Inactive ost\n",$count; + } + $count = $count + 1; + } +} + +@rTime = (); +@wTime = (); +@rMBs = (); +@wMBs = (); +@ACTIVEOST_INX; + +# Locals +my $filename = ""; +my $dirpath = ""; +my $flag = 0; + +# Command line parameter parsing +use Getopt::Std; +getopts('s:h') or usage(); +usage() if $opt_h; +$FSIZE = $opt_s if $opt_s; + +my $i = 0; +foreach (@ARGV) { + $MNT = $_; + $i++; + if ($i > 1) { + print "ERROR: extra argument $_\n"; + usage(); + } +} +#Check for Time::HiRes module +my $CheckTimeHiRes = "require Time::HiRes"; +eval ($CheckTimeHiRes) or die "You need to install the perl-Time-HiRes package to use this script\n"; + +use POSIX qw(strftime); +my $time_v = time(); +my $hostname = `lctl list_nids | head -1` or die "You need to install lctl to use this script\n"; +chop($hostname); +print "$pname: ", strftime("%D", localtime($time_v)); +print " OST speed survey on $MNT from $hostname\n"; + +# get OST count +ost_count (); + +use File::Path; +while ($CNT < $OSTS) { + $dirpath = "$MNT/tmpdir$CNT"; + eval { mkpath($dirpath) }; + if ($@) { + print "Couldn't create $dirpath: $@"; + exit 1; + } + $filename = "$dirpath/file$CNT"; + if ( $ACTIVEOST_INX[$CNT] ) { + # set stripe for OST number $CNT + system ("lfs setstripe $filename 0 $CNT 1"); + # Perform write for OST number $CNT + &run_test($FSIZE,$CNT,"write",$filename); + # Perform read for OST number $CNT + &run_test($FSIZE,$CNT,"read",$filename); + $flag++; + } + eval { rmtree($dirpath) }; + if ($@) { + print "Warning: Couldn't $dirpath: $@"; + } + $CNT = $CNT + 1; +} +# if read or write performed on any OST then display information. +if ( $flag ) { + if ( $flag > 1 ) { + &calculate("Read",@rMBs); + &calculate("Write",@wMBs); + } + output_all_data (); +} else { + print "There is no active OST's found\n"; +} diff --git a/lustre-iokit/ost-survey/ost-survey.sh b/lustre-iokit/ost-survey/ost-survey.sh deleted file mode 100755 index ba438a4..0000000 --- a/lustre-iokit/ost-survey/ost-survey.sh +++ /dev/null @@ -1,193 +0,0 @@ -#!/bin/bash - -# This script is to be run on a client machine and will test all the -# OSTs to determine which is the fastest and slowest -# The current test method -# Create a directory for each OST -# Use 'lfs setstripe' to set the Lustre striping such that IO goes to -# only one OST -# Use 'dd' to write a file of a specified size -# Use 'dd' to read a file of a specified size -# Compute the average -# Find the slowest OST - - -declare -a rTime=() # Time to read some data -declare -a wTime=() # Time to write some data -declare -a rMBs=() # Read speed -declare -a wMBs=() # Write speed - -# GLOBALS -OSTS=0 # Number of OSTS we will loop over -OFILE=testdummy # File name to use -BSIZE=1024 # size of blocks to be written -MNT='' # Location of Lustre file system -DIR="tmpdir" # Name used to create a series of tmp directories -VERBOSE=1 # Set this to get verbose output ( TODO - use getopts? ) - -# Usage -if [ $# -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - - -test_preq () { - # check for a mounted Lustre filesystem - MNT=`grep ":/lustre" /proc/mounts | awk '{print $2}'` - if [ -z "$MNT" ]; then - echo "Mounted Lustre filesystem not found" - exit 1 - fi - - # Check for Lustre utilites in PATH - # Check for dd -} - -ost_count () { - OSTS=$(cat /proc/fs/lustre/lov/lustre-clilov-*/numobd | head -1) -} - -make_dummy () { -# Create a file full of zeros - echo "make dummy" - local DIR=$1 - local SIZE=$2 - mkdir -p $MNT/$DIR - dd if=/dev/zero of=$MNT/$DIR/$OFILE count=$SIZE bs=$BSIZE 2> /dev/null - -} - -output_all_data () { - echo "$OSTS OST devices found" - local CNT=0 - while [ $CNT -lt $OSTS ]; do - echo "Ost index $CNT Read speed ${rMBs[$CNT]} Write speed ${wMBs[$CNT]}" - echo "Ost index $CNT Read time ${rTime[$CNT]} Write time ${wTime[$CNT]}" - CNT=$(( $CNT + 1 )) - done -} -run_test () { - local DIR=$1 - local SIZE=$2 - local INX=$3 - local ACTION=$4 - - if [ ! -f $MNT/$DIR/$OFILE ] && [ $ACTION == 'read' ]; then - make_dummy $DIR $SIZE - fi - - t0=`date +%s.%N` - if [ $ACTION == 'read' ]; then - OUTS=`dd if=$MNT/$DIR/$OFILE of=/dev/null count=$SIZE bs=$BSIZE 2> /dev/null` - elif [ $ACTION == 'write' ]; then - OUTS=`dd of=$MNT/$DIR/$OFILE if=/dev/zero count=$SIZE bs=$BSIZE 2> /dev/null` - else - echo "Action not read||write" - exit 1 - fi - t1=`date +%s.%N` - - tdelta=`awk "BEGIN {printf \"%7.2f\", $t1 - $t0; exit}"` - sdelta=$(( $SIZE * $BSIZE )) - delta=`awk "BEGIN {printf \"%7.2f\", ($SIZE * $BSIZE / ( $t1 - $t0 )) / ( 1024 * 1024 ) ; exit}"` - - if [ $ACTION == 'read' ]; then - rTime[$INX]=$tdelta - rMBs[$INX]=$delta - else - wTime[$INX]=$tdelta - wMBs[$INX]=$delta - fi -} - -display_average () { - local CNT=0 - local OP=$1 - while [ $CNT -lt $OSTS ]; do - if [ $OP == "read" ]; then - echo "${rMBs[$CNT]} $OP" - elif [ $OP == "write" ]; then - echo "${wMBs[$CNT]} $OP" - else - echo "Bad param" - exit 1 - fi - CNT=$(( $CNT + 1 )) - done | awk '{ c++; t+= $1; op = $2 }; END { printf "Average %s Speed: %7.2f\n", op, t/c }' - -} - -find_min () { - local CNT=0 - local OP=$1 - while [ $CNT -lt $OSTS ]; do - if [ $OP == "read" ]; then - echo "${rMBs[$CNT]} $CNT $OP" - elif [ $OP == "write" ]; then - echo "${wMBs[$CNT]} $CNT $OP" - else - echo "Bad param" - exit 1 - fi - CNT=$(( $CNT + 1 )) - done | awk '{ - if (NR == 1) { min = $1; indx = $2; op = $3 } - else if (min > $1){ min = $1; indx = $ 2; op = $3} - } - END {printf "%s - Worst OST indx %d %7.2f MB/s\n", op, indx, min}' -} - -find_max () { - local CNT=0 - local OP=$1 - while [ $CNT -lt $OSTS ]; do - if [ $OP == "read" ]; then - echo "${rMBs[$CNT]} $CNT $OP" - elif [ $OP == "write" ]; then - echo "${wMBs[$CNT]} $CNT $OP" - else - echo "Bad param" - exit 1 - fi - CNT=$(( $CNT + 1 )) - done | awk '{ - if (NR == 1) { max = $1; indx = $2; op = $3 } - else if (max < $1){ max = $1; indx = $ 2; op = $3 } - } - END {printf "%s - Best OST indx %d %7.2f MB/s\n", op, indx, max}' -} -# Temp cleanup - -CNT=0 -MYSIZE=1024 - -test_preq -ost_count - -while [ $CNT -lt $OSTS ]; do - rm -rf $MNT/${DIR}${CNT} - mkdir -p $MNT/${DIR}${CNT} - lfs setstripe $MNT/${DIR}${CNT} 0 $CNT 1 - run_test ${DIR}${CNT} $MYSIZE $CNT write - run_test ${DIR}${CNT} $MYSIZE $CNT read - CNT=$(( $CNT + 1 )) -done - -MAX_MB=0 -MIN_T=999999999 - -display_average read -display_average write -find_min read -find_min write -find_max read -find_max write - -CNT=0 - - -if [ $VERBOSE ]; then - output_all_data -fi - diff --git a/lustre-iokit/ost-survey/plot-ost.pl b/lustre-iokit/ost-survey/plot-ost.pl new file mode 100755 index 0000000..f5bf792 --- /dev/null +++ b/lustre-iokit/ost-survey/plot-ost.pl @@ -0,0 +1,79 @@ +#!/usr/bin/perl -w +# Report generation for ost-survey.pl +# =================================== +# The plot-ost.pl script is used to generate csv file and +# instructions files for gnuplot from the output of ost-survey.pl script. +# +# The plot-ost.pl also creates .scr file that contains instructions +# for gnuplot to plot the graph. After generating .dat and .scr files this +# script invokes gnuplot to display graph. +# +# Syntax: +# $ plot-ost.pl +# [Note: This script may need modifications whenever there will be +# modifications in output format of ost-survey.pl script.] + +# arg 0 is filename +sub usages_msg(){ + print "Usage: $0 \n"; + print " $0 produces graphs from the output of ost-survey.pl\n"; + print " using gnuplot.\n"; + print "e.g.# perl ost-survey /mnt/lustre > ost-log; perl $0 ost-log\n"; + exit 1; +} + +my $count = 0; # count for number of rows in csv(.dat) file. +my @line; # To store recently read line from log file +my $flag = 0; +my @GraphTitle; +if ( !$ARGV[0] ) { + usages_msg(); +} + +$file = $ARGV[0]; +# Open log file for reading +open ( PFILE, "$file") or die "Can't open results log file"; +# Open .csv file for writting required columns from log file. +open ( DATAFILE, "> $file.dat" ) or die "Can't open csv file for writting"; +LABLE:while ( ) { + chomp; + @line = split( /\s+/ ); # splits line into tokens + # This comparison may be changed if there will be changes log file. + if ( $line[0] eq "Ost#" ) { + print DATAFILE "$line[0] $line[1] $line[2]\n"; + $flag = 1; + ; # skip the "---------" line from result file. + last LABLE; + } + if ($line[2] eq "OST" && $line[3] eq "speed") { + @GraphTitle = @line; + @GraphTitle = split( /:/ ); + } +} +if ( !$flag) { + print "Invalid logfile format\n"; + exit 1; +} +while ( ) { + chomp; + @line = split( /\s+/ ); # splits line into tokens + if ( $line[1] ne "Inactive" ) { + print DATAFILE "$count $line[1] $line[2]\n"; + } + $count = $count + 1; +} +close PFILE; +close DATAFILE; +# Open .scr file for writting instructions for gnuplot. +open ( SCRFILE, "> $file.scr" ) or die "Can't open scr file for writting"; +# generate instructions for gnuplot. decide axes depends on ranges in @columnvalues +print SCRFILE "set title \"$GraphTitle[1]\"\n"; +print SCRFILE "set xlabel \"OST index\"\n"; +print SCRFILE "set ylabel \"MB/s\"\n"; +print SCRFILE "set boxwidth 0.3\n"; +print SCRFILE "plot \"$file.dat\" using 1:2 axes x1y1 title \"Read(MB/s)\" with boxes\n"; +print SCRFILE "replot \"$file.dat\" using 1:3 axes x1y1 title \"Write(MB/s)\" with boxes\n"; +print SCRFILE "pause -1\n"; +close SCRFILE; +# invoke gnuplot to display graph. +system ("gnuplot $file.scr") == 0 or die "ERROR: while ploting graph.\nMake sure that gnuplot is working properly"; -- 1.8.3.1