2 # This script is to be run on a client machine and will test all the
3 # OSTs to determine which is the fastest and slowest
4 # The current test method is as follows:
5 # -Create a directory for each OST
6 # -Use 'lfs setstripe' to set the Lustre striping such that IO goes to
8 # -Use 'dd' to write and read a file of a specified size
9 # -Compute the average, and Standard deviation
10 # -Find the slowest OST for read and write
11 # -Find the Fastest OST for read and write
14 $pname = $0; # to hold program name
15 $OSTS = 0; # Number of OSTS we will loop over
16 $BSIZE = 1024 * 1024; # Size of i/o block
17 $MNT = "/mnt/lustre"; # Location of Lustre file system
18 $FSIZE = 30; # Number of i/o blocks
22 print "Usage: $pname [-s <size>] [-h] <Lustre_Path>\n";
24 print " -s: size of test file in MB (default $FSIZE MB)\n";
25 print " -h: To display this help\n";
26 print "example : $pname /mnt/lustre\n";
30 # ost_count subroutine ets globle variable $OST with Number of OST's
31 # Also fills 1 for active OST indexes in ACTIVEOST_INX array.
33 # numobd gives number of ost's and activeobd gives number of active ost's
34 my $tempfile = glob ("/proc/fs/lustre/lov/*-clilov-*/activeobd");
35 open(PTR, $tempfile) || die "Cannot open $tempfile: $!\n";
38 print "Number of Active OST devices : $OSTS";
39 my $tempfile = glob ("/proc/fs/lustre/lov/*-clilov-*/numobd");
40 open(PTR, $tempfile) || die "Cannot open $tempfile: $!\n";
43 if ( $numost != $OSTS ) {
44 printf "Number of non active ots(s): %d\n", ( $numost - $OSTS );
47 my $tempfile = glob ("/proc/fs/lustre/lov/*-clilov-*/target_obd");
48 open(PTR, $tempfile) || die "Cannot open $tempfile: $!\n";
53 my ($ost_num, $ost_name, $ost_status) = split(/\s+/, $_);
54 if ( $ost_status eq "ACTIVE" ) {
55 $ACTIVEOST_INX[$count] = 1;
62 $CACHEFILE = glob ("/proc/fs/lustre/llite/*/max_cached_mb");
63 open(PTR, $CACHEFILE) || die "Cannot open $tempfile: $!\n";
66 system("echo 0 >> $CACHEFILE");
70 system("echo $CACHESZ >> $CACHEFILE");
73 # make_dummy subroutine creates a dummy file that will be used for read operation.
77 system ("dd of=$tempfile if=/dev/zero count=$SIZE bs=$BSIZE 2> /dev/null");
80 # run_test subroutine actually writes and reads data to/from dummy file
81 # and compute corresponding time taken for read and write operation and
82 # byte transfer for the both operations.
83 # This subroutine also fill corresponding globle arrays with above information.
90 if ( !(-f $tempfile) && $ACTION eq "read" ) {
91 &make_dummy($SIZE, $tempfile);
94 my ($ts0, $tu0) = gettimeofday();
95 $tu0 = $ts0 + ($tu0 / 1000000);
96 if ( $ACTION eq "write" ) {
97 system("dd of=$tempfile if=/dev/zero count=$SIZE bs=$BSIZE 2> /dev/null");
98 } elsif ( $ACTION eq "read" ) {
99 system("dd if=$tempfile of=/dev/null count=$SIZE bs=$BSIZE 2> /dev/null");
101 print "Action is neither read nor write\n";
105 my ($ts1, $tu1) = gettimeofday();
106 $tu1 = $ts1 + ($tu1/1000000);
107 my $tdelta = $tu1 - $tu0;
108 my $delta = ($SIZE * $BSIZE / ( $tu1 - $tu0 )) / (1024 * 1024);
109 if ( $ACTION eq "write" ) {
110 $wTime[$INX] = $tdelta;
111 $wMBs[$INX] = $delta;
113 $rTime[$INX] = $tdelta;
114 $rMBs[$INX] = $delta;
118 # calculate subroutine compute following things and displays them.
119 # - Finds worst and best OST for both read and write operations.
120 # - Compute average of read and write rate from all OSTS
121 # - Compute Standard deviation for read and write form all OST's
133 my $min_mb = 999999999;
134 while ($count < $OSTS ) {
135 if ( $ACTIVEOST_INX[$count] ) {
136 $total = $total + $MBs[$count];
137 if ($max_mb < $MBs[$count] ) {
138 $max_mb = $MBs[$count];
141 if ($min_mb > $MBs[$count] ) {
142 $min_mb = $MBs[$count];
151 while ($count < $OSTS ) {
152 if ( $ACTIVEOST_INX[$count] ) {
153 $total = $total + ($MBs[$count] - $avg) * ($MBs[$count] - $avg);
157 $sd = sqrt($total/$OSTS);
158 printf "Worst %s OST indx: %d speed: %f\n", $op, $worst_OST, $min_mb;
159 printf "Best %s OST indx: %d speed: %f\n", $op, $best_OST, $max_mb;
160 printf "%s Average: %f +/- %f MB/s\n", $op, $avg, $sd;
163 # output_all_data subroutine displays speed and time information
164 # for all OST's for both read and write operations.
165 sub output_all_data () {
167 print "Ost# Read(MB/s) Write(MB/s) Read-time Write-time\n";
168 print "----------------------------------------------------\n";
169 while ( $count < $OSTS ) {
170 if ( $ACTIVEOST_INX[$count] ) {
171 printf "%d %.3f %.3f %.3f %.3f\n",$count,
172 $rMBs[$count], $wMBs[$count], $rTime[$count], $wTime[$count];
174 printf "%d Inactive ost\n",$count;
191 # Command line parameter parsing
193 getopts('s:h') or usage();
195 $FSIZE = $opt_s if $opt_s;
202 print "ERROR: extra argument $_\n";
206 #Check for Time::HiRes module
207 my $CheckTimeHiRes = "require Time::HiRes";
208 eval ($CheckTimeHiRes) or die "You need to install the perl-Time-HiRes package to use this script\n";
209 my $LoadTimeHiRes = "use Time::HiRes qw(gettimeofday)";
210 eval ($LoadTimeHiRes);
212 use POSIX qw(strftime);
214 my $hostname = `lctl list_nids | head -1` or die "You need to install lctl to use this script\n";
216 print "$pname: ", strftime("%D", localtime($time_v));
217 print " OST speed survey on $MNT from $hostname\n";
221 # turn off local cache
224 $dirpath = "$MNT/ost_survey_tmp";
225 eval { mkpath($dirpath) };
227 print "Couldn't create $dirpath: $@";
233 while ($CNT < $OSTS) {
234 $filename = "$dirpath/file$CNT";
235 if ( $ACTIVEOST_INX[$CNT] ) {
236 # set stripe for OST number $CNT
237 system ("lfs setstripe $filename 0 $CNT 1");
238 # Perform write for OST number $CNT
239 &run_test($FSIZE,$CNT,"write",$filename);
245 while ($CNT < $OSTS) {
246 $filename = "$dirpath/file$CNT";
247 if ( $ACTIVEOST_INX[$CNT] ) {
248 # Perform read for OST number $CNT
249 &run_test($FSIZE,$CNT,"read",$filename);
255 # if read or write performed on any OST then display information.
258 &calculate("Read",@rMBs);
259 &calculate("Write",@wMBs);
263 print "There is no active OST's found\n";
268 eval { rmtree($dirpath) };
270 print "Warning: Couldn't remove $dirpath: $@";