3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
10 ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-"$DOM_PERFORMANCE_EXCEPT"}
11 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
13 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
15 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
17 . $LUSTRE/tests/test-framework.sh
20 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
23 SAVED_FAIL_ON_ERROR=$FAIL_ON_ERROR
26 SAVED_DEBUG=$($LCTL get_param -n debug 2> /dev/null)
29 . $LUSTRE/tests/functions.sh
31 check_and_setup_lustre
33 # if MACHINEFILE set and exists -- use it
34 MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh)-$(hostname).machines}
35 clients=${CLIENTS:-$HOSTNAME}
36 generate_machine_file $clients $MACHINEFILE ||
37 error "Failed to generate machine file"
39 DP_DIO=${DP_DIO:-"no"}
41 DOM_SIZE=${DOM_SIZE:-"1M"}
48 DP_DOM_DNE=$DIR/dp_dne
49 DP_STATS=${DP_STATS:-"no"}
51 # total number of files
52 DP_FNUM=${DP_FNUM:-16384}
56 # 1 stripe for normal files
58 $LFS setstripe -c 2 $DP_NORM ||
59 error "Cannot create test directory for ordinary files"
61 if [[ $MDSCOUNT -gt 1 ]] ; then
62 $LFS setdirstripe -i 0 -c $MDSCOUNT $DP_DOM_DNE ||
63 error_noexit "Cannot create striped directory"
64 $LFS setstripe -E ${DOM_SIZE} -L mdt -E EOF $DP_DOM_DNE ||
65 error_noexit "Cannot create test directory for dom files"
69 $LFS setstripe -E ${DOM_SIZE} -L mdt -E EOF $DP_DOM ||
70 error "Cannot create test directory for dom files"
75 $LCTL set_param -n osc.*.stats=0
76 $LCTL set_param -n mdc.*.stats=0
77 $LCTL set_param -n ${cli}.*.${cli}_stats=0
78 $LCTL set_param -n ${cli}.*.rpc_stats=0
79 $LCTL set_param -n llite.*.read_ahead_stats=0
80 $LCTL set_param -n llite.*.unstable_stats=0
87 echo ----- MDC RPCs: $(calc_stats mdc.*.stats req_active)
88 echo ----- OSC RPCs: $(calc_stats osc.*.stats req_active)
90 if [ "x$DP_STATS" != "xyes" ] ; then
94 $LCTL get_param ${cli}.*.${cli}_stats
95 $LCTL get_param ${cli}.*.rpc_stats
96 # for OSC get both OSC and MDC stats
97 if [ $cli == "osc" ] ; then
98 $LCTL get_param mdc.*.stats
100 $LCTL get_param ${cli}.*.stats
101 $LCTL get_param ${cli}.*.unstable_stats
102 $LCTL get_param ${cli}.*.${cli}_cached_mb
103 $LCTL get_param llite.*.read_ahead_stats
109 cancel_lru_locks $cli
110 ### drop all debug except critical
111 $LCTL set_param -n debug="error warning console emerg"
117 local cmdlog=$TMP/dp_cmd.log
120 dp_setup_test $DP_OSC
121 if ! grep -qw "$MOUNT" /proc/mounts ; then
122 echo "!!!!! Lustre is not mounted !!!!!, aborting"
126 echo "## $cmd" | awk '{ if (NR==1) {gsub(/[ \t\r\n]+/, " "); \
127 gsub(/\|.*$/, ""); print }}'
128 echo "## $(date +'%F %H:%M:%S'): START"
129 eval $cmd 2>&1 | tee $cmdlog || true
132 if [ $rc -eq 0 ] && grep -q "p4_error:" $cmdlog ; then
136 dp_collect_stats $DP_OSC
137 remount_client $DIR > /dev/null
142 if ! which mdtest > /dev/null 2>&1 ; then
143 echo "Mdtest is not installed, skipping"
147 local mdtest=$(which mdtest)
149 local TDIR=${1:-$MOUNT}
150 local th_num=$((DP_FNUM * 2 / DP_NUM))
155 [ "$SLOW" = "yes" ] && bsizes="4096 32768"
157 for bsize in $bsizes ; do
158 dp_run_cmd "mpi_run -np $DP_NUM $mdtest -i 3 -I $th_num -F \
159 -z 1 -b 1 -L -u -w $bsize -R -d $TDIR"
160 if [ ${PIPESTATUS[0]} != 0 ]; then
161 error "MDtest failed, aborting"
170 if [ ! -f createmany ] ; then
171 echo "Createmany is not installed, skipping"
175 if [ ! -f smalliomany ] ; then
176 echo "Smalliomany is not installed, skipping"
180 local TDIR=${1:-$DIR}
183 local MIN=$((count * 16))
184 [ $MDSSIZE -le $MIN ] && count=$((MDSSIZE / 16))
186 dp_run_cmd "./createmany -o $TDIR/file- $count | grep 'total:'"
187 if [ ${PIPESTATUS[0]} != 0 ]; then
188 error "File creation failed, aborting"
191 if [ -f statmany ]; then
192 dp_run_cmd "./statmany -s $TDIR/file- $count $((count * 5)) |
194 if [ ${PIPESTATUS[0]} != 0 ]; then
195 error "File stat failed, aborting"
200 for opc in w a r ; do
201 dp_run_cmd "./smalliomany -${opc} $TDIR/file- $count 300 |
203 if [ ${PIPESTATUS[0]} != 0 ]; then
204 error "SmallIO -${opc} failed, aborting"
209 dp_run_cmd "./unlinkmany $TDIR/file- $count | grep 'total:'"
210 if [ ${PIPESTATUS[0]} != 0 ]; then
211 error "SmallIO failed, aborting"
218 if ! which IOR > /dev/null 2>&1 ; then
219 echo "IOR is not installed, skipping"
223 local IOR=$(which IOR)
224 local iter=$((DP_FNUM / DP_NUM))
227 if [ "x$DP_DIO" == "xyes" ] ; then
231 local TDIR=${1:-$MOUNT}
236 [ "$SLOW" = "yes" ] && bsizes="4 32"
238 for bsize in $bsizes ; do
239 segments=$((128 / bsize))
241 dp_run_cmd "mpi_run -np $DP_NUM $IOR \
242 -a POSIX -b ${bsize}K -t ${bsize}K -o $TDIR/ -k \
243 -s $segments -w -r -i $iter -F -E -z -m -Z $direct" |
244 awk '($1 !~ /^(write|read|access)$/) || NF>12 {print}'
245 if [ ${PIPESTATUS[0]} != 0 ]; then
246 error "IOR write test for ${bsize}K failed, aborting"
249 # check READ performance only (no cache)
250 dp_run_cmd "mpi_run -np $DP_NUM $IOR \
251 -a POSIX -b ${bsize}K -t ${bsize}K -o $TDIR/ -X 42\
252 -s $segments -r -i $iter -F -E -z -m -Z $direct" |
253 awk '($1 !~ /^(read|access|remove)$/) || NF>12 {print}'
254 if [ ${PIPESTATUS[0]} != 0 ]; then
255 error "IOR read test for ${bsize}K failed, aborting"
264 if ! which dbench > /dev/null 2>&1 ; then
265 echo "Dbench is not installed, skipping"
269 local TDIR=${1:-$MOUNT}
271 if [ "x$DP_DOM_DNE" == "x$TDIR" ] ; then
272 echo "dbench uses subdirs, skipping for DNE dir"
276 dp_run_cmd "dbench -D $TDIR $DP_NUM | egrep -v 'warmup|execute'"
277 if [ ${PIPESTATUS[0]} != 0 ]; then
278 error "Dbench failed, aborting"
286 # https://github.com/axboe/fio/archive/fio-2.8.zip
287 if ! which fio > /dev/null 2>&1 ; then
288 echo "No FIO installed, skipping"
292 local fnum=128 # per thread
293 local total=$((fnum * DP_NUM)) # files in all threads
294 local loops=$((DP_FNUM / total)) # number of loops
298 if [ $loops -eq 0 ] ; then
302 if [ "x$DP_DIO" == "xyes" ] ; then
305 direct="--buffered=1 --bs_unaligned=1"
308 if [ "x$DP_STATS" != "xyes" ] ; then
312 local TDIR=${1:-$MOUNT}
313 base_cmd="fio --name=smallio --ioengine=posixaio $output \
314 --iodepth=$((DP_NUM * 4)) --directory=$TDIR \
315 --nrfiles=$fnum --openfiles=10000 \
316 --numjobs=$DP_NUM --filesize=64k --lockfile=readwrite"
318 dp_run_cmd "$base_cmd --create_only=1" > /dev/null
319 if [ ${PIPESTATUS[0]} != 0 ]; then
320 error "FIO file creation failed, aborting"
324 [ "$SLOW" = "yes" ] && bsizes="4 32"
326 for bsize in $bsizes ; do
327 local write_cmd="$base_cmd --bs=${bsize}k --rw=randwrite \
328 $direct --file_service_type=random --randrepeat=1 \
329 --norandommap --group_reporting=1 --loops=$loops"
330 if [ "x$DP_STATS" != "xyes" ] ; then
331 dp_run_cmd "$write_cmd | awk -F\; '{printf \"WRITE: \
332 BW %dKiB/sec, IOPS %d, lat (%d/%d/%d)usec\n\", \
333 \$48, \$49, \$53, \$57, \$81}'"
335 dp_run_cmd "$write_cmd"
337 if [ ${PIPESTATUS[0]} != 0 ]; then
338 error "FIO write test with ${bsize}k failed, aborting"
341 local read_cmd="$base_cmd --bs=${bsize}k --rw=randread \
342 $direct --file_service_type=random --randrepeat=1 \
343 --norandommap --group_reporting=1 --loops=$loops"
344 if [ "x$DP_STATS" != "xyes" ] ; then
345 dp_run_cmd "$read_cmd | awk -F\; '{printf \"READ : \
346 BW %dKiB/sec, IOPS %d, lat (%d/%d/%d)usec\n\", \
347 \$7, \$8, \$12, \$16, \$40}'"
349 dp_run_cmd "$read_cmd"
351 if [ ${PIPESTATUS[0]} != 0 ]; then
352 error "FIO read test with ${bsize}k failed, aborting"
360 if ! which compilebench > /dev/null 2>&1 ; then
361 echo "Compilebench is not installed, skipping"
365 local TDIR=${1:-$MOUNT}
367 dp_run_cmd "compilebench -D $TDIR -i 2 -r 2 --makej"
368 if [ ${PIPESTATUS[0]} != 0 ]; then
369 error "Compilebench failed, aborting"
377 local facets=$(get_facets MDS)
378 local nodes=$(comma_list $(mdts_nodes))
379 local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
381 save_lustre_params $facets "mdt.*.dom_lock" >> $p
383 printf "\n##### $test: DoM files\n"
384 do_nodes $nodes "lctl set_param -n mdt.*.dom_lock=1"
388 if [ -d $DP_DOM_DNE ] ; then
389 printf "\n##### $test: DoM files + DNE\n"
391 run_${test} $DP_DOM_DNE
394 printf "\n##### $test: OST files\n"
398 restore_lustre_params < $p
405 run_test smallio "Performance comparision: smallio"
410 run_test mdtest "Performance comparision: mdtest"
415 run_test IOR "Performance comparision: IOR"
420 run_test dbench "Performance comparision: dbench"
425 run_test fio "Performance comparision: FIO"
428 dp_test_run compbench
430 run_test compbench "Performance comparision: compilebench"
432 FAIL_ON_ERROR=$SAVED_FAIL_ON_ERROR
433 $LCTL set_param -n debug="$SAVED_DEBUG"
436 check_and_cleanup_lustre