2 # This script is to be run on a client machine and will test all the
3 # OSTs to determine which is the fastest and slowest
4 # The current test method is as follows:
5 # -Create a directory for each OST
6 # -Use 'lfs setstripe' to set the Lustre striping such that IO goes to
8 # -Use 'dd' to write and read a file of a specified size
9 # -Compute the average, and Standard deviation
10 # -Find the slowest OST for read and write
11 # -Find the Fastest OST for read and write
14 $pname = $0; # to hold program name
15 $OSTS = 0; # Number of OSTS we will loop over
16 $BSIZE = 1024 * 1024; # Size of i/o block
17 $MNT = "/mnt/lustre"; # Location of Lustre file system
18 $FSIZE = 30; # Number of i/o blocks
22 print "Usage: $pname [-s <size>] [-h] <Lustre_Path>\n";
24 print " -s: size of test file in MB (default $FSIZE MB)\n";
25 print " -h: To display this help\n";
26 print "example : $pname /mnt/lustre\n";
30 # ost_count subroutine ets globle variable $OST with Number of OST's
31 # Also fills 1 for active OST indexes in ACTIVEOST_INX array.
33 $OSTS = `lctl get_param -n lov.*-clilov-*.activeobd`;
35 die "Read lov.*-clilov-*.activeobd error: $?\n";
37 print "Number of Active OST devices : $OSTS";
39 $numost = `lctl get_param -n lov.*-clilov-*.numobd`;
41 die "Read lov.*-clilov-*.numobd error: $?\n";
44 if ( $numost != $OSTS ) {
45 printf "Number of non active ots(s): %d\n", ( $numost - $OSTS );
49 $targets = `lctl get_param -n lov.*-clilov-*.target_obd`;
51 die "Read lov.*-clilov-*.target_obd error: $?\n";
55 foreach $line (split /\n/, $targets) {
56 my ($ost_num, $ost_name, $ost_status) = split(/\s+/, $line);
57 if ( $ost_status eq "ACTIVE" ) {
58 $ACTIVEOST_INX[$count] = 1;
65 $CACHESZ = `lctl get_param -n llite.*.max_cached_mb`;
67 die "Read llite.*.max_cached_mb error: $?\n";
70 $CACHESZ = `echo "$CACHESZ" | grep max_cached_mb | awk '{print \$2}'`;
72 system("lctl set_param -n llite.*.max_cached_mb=0");
76 system("lctl set_param -n llite.*.max_cached_mb=$CACHESZ");
79 # make_dummy subroutine creates a dummy file that will be used for read operation.
83 system ("dd of=$tempfile if=/dev/zero count=$SIZE bs=$BSIZE 2> /dev/null");
86 # run_test subroutine actually writes and reads data to/from dummy file
87 # and compute corresponding time taken for read and write operation and
88 # byte transfer for the both operations.
89 # This subroutine also fill corresponding globle arrays with above information.
96 if ( !(-f $tempfile) && $ACTION eq "read" ) {
97 &make_dummy($SIZE, $tempfile);
100 my ($ts0, $tu0) = gettimeofday();
101 $tu0 = $ts0 + ($tu0 / 1000000);
102 if ( $ACTION eq "write" ) {
103 system("dd of=$tempfile if=/dev/zero count=$SIZE bs=$BSIZE 2> /dev/null");
104 } elsif ( $ACTION eq "read" ) {
105 system("dd if=$tempfile of=/dev/null count=$SIZE bs=$BSIZE 2> /dev/null");
107 print "Action is neither read nor write\n";
111 my ($ts1, $tu1) = gettimeofday();
112 $tu1 = $ts1 + ($tu1/1000000);
113 my $tdelta = $tu1 - $tu0;
114 my $delta = ($SIZE * $BSIZE / ( $tu1 - $tu0 )) / (1024 * 1024);
115 if ( $ACTION eq "write" ) {
116 $wTime[$INX] = $tdelta;
117 $wMBs[$INX] = $delta;
119 $rTime[$INX] = $tdelta;
120 $rMBs[$INX] = $delta;
124 # calculate subroutine compute following things and displays them.
125 # - Finds worst and best OST for both read and write operations.
126 # - Compute average of read and write rate from all OSTS
127 # - Compute Standard deviation for read and write form all OST's
139 my $min_mb = 999999999;
140 while ($count < $OSTS ) {
141 if ( $ACTIVEOST_INX[$count] ) {
142 $total = $total + $MBs[$count];
143 if ($max_mb < $MBs[$count] ) {
144 $max_mb = $MBs[$count];
147 if ($min_mb > $MBs[$count] ) {
148 $min_mb = $MBs[$count];
157 while ($count < $OSTS ) {
158 if ( $ACTIVEOST_INX[$count] ) {
159 $total = $total + ($MBs[$count] - $avg) * ($MBs[$count] - $avg);
163 $sd = sqrt($total/$OSTS);
164 printf "Worst %s OST indx: %d speed: %f\n", $op, $worst_OST, $min_mb;
165 printf "Best %s OST indx: %d speed: %f\n", $op, $best_OST, $max_mb;
166 printf "%s Average: %f +/- %f MB/s\n", $op, $avg, $sd;
169 # output_all_data subroutine displays speed and time information
170 # for all OST's for both read and write operations.
171 sub output_all_data () {
173 print "Ost# Read(MB/s) Write(MB/s) Read-time Write-time\n";
174 print "----------------------------------------------------\n";
175 while ( $count < $OSTS ) {
176 if ( $ACTIVEOST_INX[$count] ) {
177 printf "%d %.3f %.3f %.3f %.3f\n",$count,
178 $rMBs[$count], $wMBs[$count], $rTime[$count], $wTime[$count];
180 printf "%d Inactive ost\n",$count;
197 # Command line parameter parsing
199 getopts('s:h') or usage();
201 $FSIZE = $opt_s if $opt_s;
208 print "ERROR: extra argument $_\n";
212 #Check for Time::HiRes module
213 my $CheckTimeHiRes = "require Time::HiRes";
214 eval ($CheckTimeHiRes) or die "You need to install the perl-Time-HiRes package to use this script\n";
215 my $LoadTimeHiRes = "use Time::HiRes qw(gettimeofday)";
216 eval ($LoadTimeHiRes);
218 use POSIX qw(strftime);
220 my $hostname = `lctl list_nids | head -1` or die "You need to install lctl to use this script\n";
222 print "$pname: ", strftime("%D", localtime($time_v));
223 print " OST speed survey on $MNT from $hostname\n";
227 # turn off local cache
230 $dirpath = "$MNT/ost_survey_tmp";
231 eval { mkpath($dirpath) };
233 print "Couldn't create $dirpath: $@";
239 while ($CNT < $OSTS) {
240 $filename = "$dirpath/file$CNT";
241 if ( $ACTIVEOST_INX[$CNT] ) {
242 # set stripe for OST number $CNT
243 system ("lfs setstripe $filename 0 $CNT 1");
244 # Perform write for OST number $CNT
245 &run_test($FSIZE,$CNT,"write",$filename);
251 while ($CNT < $OSTS) {
252 $filename = "$dirpath/file$CNT";
253 if ( $ACTIVEOST_INX[$CNT] ) {
254 # Perform read for OST number $CNT
255 &run_test($FSIZE,$CNT,"read",$filename);
261 # if read or write performed on any OST then display information.
264 &calculate("Read",@rMBs);
265 &calculate("Write",@wMBs);
269 print "There is no active OST's found\n";
274 eval { rmtree($dirpath) };
276 print "Warning: Couldn't remove $dirpath: $@";