From 892ab143608d43590cab73366afda201cf018803 Mon Sep 17 00:00:00 2001 From: James Nunez Date: Wed, 8 Oct 2014 17:46:28 -0600 Subject: [PATCH] LU-4768 tests: Update ost-survey script Currently, ost-survey hangs due to calling 'lfs setstripe' in an old (positional) style and setting max_cached_mb to zero. The call to 'lfs setstripe' is updated to use the '-S', '-i' and '-c' flags. max_cached_mb is now set to pagesize * 256 (in MB). The patch also gets parameters for the correct file system if more than one Lustre file system is mounted, and corrects a few typos in comments. In ll_max_cached_mb_seq_write(), the number of pages requested is set to the max of pages requested or PTLRPC_MAX_BRW_PAGES to allow the client to make well formed RPCs. This is a backport to b2_5 Lustre-change: http://review.whamcloud.com/#/c/11971/4 Lustre-commit: 46bec835ac726aa2d06024b197002be7b112e192 with some changes to ost-survey from Lustre-change: http://review.whamcloud.com/#/c/10534/8 Lustre-commit: fe3ddff77475dd487873e2f873835455aab7aa38 Signed-off-by: James Nunez Change-Id: Ia9f22c19f95b85d14687bdaf70cad05ca3a3f455 Reviewed-on: http://review.whamcloud.com/12240 Tested-by: Jenkins Reviewed-by: Cliff White Reviewed-by: Jian Yu Tested-by: Maloo Reviewed-by: Andreas Dilger --- lustre-iokit/ost-survey/ost-survey | 109 ++++++++++++++++++++++++------------- lustre/llite/lproc_llite.c | 2 + 2 files changed, 72 insertions(+), 39 deletions(-) diff --git a/lustre-iokit/ost-survey/ost-survey b/lustre-iokit/ost-survey/ost-survey index fc93117..7a2bd04 100755 --- a/lustre-iokit/ost-survey/ost-survey +++ b/lustre-iokit/ost-survey/ost-survey @@ -14,7 +14,9 @@ $pname = $0; # to hold program name $OSTS = 0; # Number of OSTS we will loop over $BSIZE = 1024 * 1024; # Size of i/o block +$MBSIZE = 1024 * 1024; # One MB $MNT = "/mnt/lustre"; # Location of Lustre file system +$CACHESZ = 0; # max_cached_mb parameter $FSIZE = 30; # Number of i/o blocks # Usage @@ -27,30 +29,33 @@ sub usage () { exit 1; } -# ost_count subroutine ets globle variable $OST with Number of OST's +# ost_count subroutine sets global variable $OST with number of OSTs # Also fills 1 for active OST indexes in ACTIVEOST_INX array. sub ost_count () { - # numobd gives number of ost's and activeobd gives number of active ost's - my $tempfile = glob ("/proc/fs/lustre/lov/*-clilov-*/activeobd"); - open(PTR, $tempfile) || die "Cannot open $tempfile: $!\n"; - $OSTS = ; - close PTR; + $OSTS = `lctl get_param -n lov.$FSNAME-clilov-*.activeobd`; + if ( $? ) { + die "Read lov.*-clilov-$FSNAME.activeobd error: $?\n"; + } print "Number of Active OST devices : $OSTS"; - my $tempfile = glob ("/proc/fs/lustre/lov/*-clilov-*/numobd"); - open(PTR, $tempfile) || die "Cannot open $tempfile: $!\n"; - $numost = ; - close PTR; + + $numost = `lctl get_param -n lov.$FSNAME-clilov-*.numobd`; + if ( $? ) { + die "Read lov.$FSNAME-clilov-*.numobd error: $?\n"; + } + if ( $numost != $OSTS ) { printf "Number of non active ots(s): %d\n", ( $numost - $OSTS ); $OSTS = $numost; } - my $tempfile = glob ("/proc/fs/lustre/lov/*-clilov-*/target_obd"); - open(PTR, $tempfile) || die "Cannot open $tempfile: $!\n"; + + $targets = `lctl get_param -n lov.$FSNAME-clilov-*.target_obd`; + if ( $? ) { + die "Read lov.$FSNAME-clilov-*.target_obd error: $?\n"; + } + my $count = 0; - my $temp; - while () { - chop; - my ($ost_num, $ost_name, $ost_status) = split(/\s+/, $_); + foreach $line (split /\n/, $targets) { + my ($ost_num, $ost_name, $ost_status) = split(/\s+/, $line); if ( $ost_status eq "ACTIVE" ) { $ACTIVEOST_INX[$count] = 1; } @@ -58,16 +63,29 @@ sub ost_count () { } } -sub cache_off () { - $CACHEFILE = glob ("/proc/fs/lustre/llite/*/max_cached_mb"); - open(PTR, $CACHEFILE) || die "Cannot open $tempfile: $!\n"; - $CACHESZ = 0 + ; - close PTR; - system("echo 0 >> $CACHEFILE"); +# cache_mod subroutine sets global variable $CACHESZ and sets max_cached_mb +# to 256 * system page size in MB. +sub cache_mod () { + use POSIX qw(sysconf _SC_PAGESIZE); + $CACHESZ = `lctl get_param -n llite.$FSNAME-*.max_cached_mb`; + if ( $? ) { + die "Read llite.$FSNAME-*.max_cached_mb error: $?\n"; + } + + $CACHESZ =~ /max_cached_mb\: (\d+)/; + $CACHESZ = $1; + my $pagesize = sysconf(_SC_PAGESIZE); + $req_cache_mb = ($pagesize * 256)/ $MBSIZE; + + system("lctl set_param -n llite.$FSNAME-*.max_cached_mb=$req_cache_mb"); + if ( $? ) { + die "Set llite.$FSNAME-*.max_cached_mb error: $?\n"; + } } -sub cache_on () { - system("echo $CACHESZ >> $CACHEFILE"); +# cache_return subroutine returns max_cached_mb to original value +sub cache_return () { + system("lctl set_param -n llite.$FSNAME-*.max_cached_mb=$CACHESZ"); } # make_dummy subroutine creates a dummy file that will be used for read operation. @@ -78,9 +96,9 @@ sub make_dummy () { } # run_test subroutine actually writes and reads data to/from dummy file -# and compute corresponding time taken for read and write operation and +# and computes corresponding time taken for read and write operation and # byte transfer for the both operations. -# This subroutine also fill corresponding globle arrays with above information. +# This subroutine also fill corresponding global arrays with above information. sub run_test () { my $SIZE = $_[0]; my $INX=$_[1]; @@ -122,7 +140,7 @@ sub run_test () { sub calculate () { my ($op, $MBs); $op = $_[0]; - @MBs = @_[1..$#_]; + @MBs = @_[1..$#_]; my $count = 0; my $total = 0; my $avg = 0; @@ -136,11 +154,11 @@ sub calculate () { $total = $total + $MBs[$count]; if ($max_mb < $MBs[$count] ) { $max_mb = $MBs[$count]; - $best_OST = $count; + $best_OST = $count; } if ($min_mb > $MBs[$count] ) { $min_mb = $MBs[$count]; - $worst_OST = $count; + $worst_OST = $count; } } $count++; @@ -160,18 +178,18 @@ sub calculate () { printf "%s Average: %f +/- %f MB/s\n", $op, $avg, $sd; } -# output_all_data subroutine displays speed and time information +# output_all_data subroutine displays speed and time information # for all OST's for both read and write operations. sub output_all_data () { my $count = 0; print "Ost# Read(MB/s) Write(MB/s) Read-time Write-time\n"; print "----------------------------------------------------\n"; while ( $count < $OSTS ) { - if ( $ACTIVEOST_INX[$count] ) { + if ( $ACTIVEOST_INX[$count] ) { printf "%d %.3f %.3f %.3f %.3f\n",$count, $rMBs[$count], $wMBs[$count], $rTime[$count], $wTime[$count]; } else { - printf "%d Inactive ost\n",$count; + printf "%d Inactive ost\n",$count; } $count = $count + 1; } @@ -188,6 +206,10 @@ my $filename = ""; my $dirpath = ""; my $flag = 0; +# Check number of arguments +my $numargs = $#ARGV + 1; +usage() if $numargs < 1; + # Command line parameter parsing use Getopt::Std; getopts('s:h') or usage(); @@ -201,9 +223,9 @@ foreach (@ARGV) { if ($i > 1) { print "ERROR: extra argument $_\n"; usage(); - } + } } -#Check for Time::HiRes module +#Check for Time::HiRes module my $CheckTimeHiRes = "require Time::HiRes"; eval ($CheckTimeHiRes) or die "You need to install the perl-Time-HiRes package to use this script\n"; my $LoadTimeHiRes = "use Time::HiRes qw(gettimeofday)"; @@ -216,10 +238,18 @@ chop($hostname); print "$pname: ", strftime("%D", localtime($time_v)); print " OST speed survey on $MNT from $hostname\n"; +# Get the file system name. +$FSNAME = `lfs getname $MNT`; +if ( $? ) { + die "`lfs getname $MNT` error: $?\n"; +} +$FSNAME =~ /(.*)-/; +$FSNAME = $1; + # get OST count ost_count (); -# turn off local cache -cache_off (); +# Modify local cache +cache_mod (); $dirpath = "$MNT/ost_survey_tmp"; eval { mkpath($dirpath) }; @@ -234,7 +264,7 @@ while ($CNT < $OSTS) { $filename = "$dirpath/file$CNT"; if ( $ACTIVEOST_INX[$CNT] ) { # set stripe for OST number $CNT - system ("lfs setstripe $filename 0 $CNT 1"); + system ("lfs setstripe -S 0 -i $CNT -c 1 $filename"); # Perform write for OST number $CNT &run_test($FSIZE,$CNT,"write",$filename); $flag++; @@ -252,7 +282,7 @@ while ($CNT < $OSTS) { $CNT = $CNT + 1; } -# if read or write performed on any OST then display information. +# if read or write performed on any OST then display information. if ( $flag ) { if ( $flag > 1 ) { &calculate("Read",@rMBs); @@ -263,7 +293,8 @@ if ( $flag ) { print "There is no active OST's found\n"; } -cache_on (); +# Return cache to original size +cache_return (); eval { rmtree($dirpath) }; if ($@) { diff --git a/lustre/llite/lproc_llite.c b/lustre/llite/lproc_llite.c index 18b8cf3..3c1489f 100644 --- a/lustre/llite/lproc_llite.c +++ b/lustre/llite/lproc_llite.c @@ -445,6 +445,8 @@ static int ll_wr_max_cached_mb(struct file *file, const char *buffer, totalram_pages >> (20 - PAGE_CACHE_SHIFT)); RETURN(-ERANGE); } + /* Allow enough cache so clients can make well-formed RPCs */ + pages_number = max_t(long, pages_number, PTLRPC_MAX_BRW_PAGES); if (sbi->ll_dt_exp == NULL) /* being initialized */ GOTO(out, rc = 0); -- 1.8.3.1