3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
11 LUSTRE=${LUSTRE:-$(dirname $0)/..}
12 . $LUSTRE/tests/test-framework.sh
16 ALWAYS_EXCEPT="$DOM_PERFORMANCE_EXCEPT"
19 SAVED_FAIL_ON_ERROR=$FAIL_ON_ERROR
22 SAVED_DEBUG=$($LCTL get_param -n debug 2> /dev/null)
24 . $LUSTRE/tests/functions.sh
25 check_and_setup_lustre
27 clients=${CLIENTS:-$HOSTNAME}
28 generate_machine_file $clients $MACHINEFILE ||
29 error "Failed to generate machine file"
31 DP_DIO=${DP_DIO:-"no"}
33 DOM_SIZE=${DOM_SIZE:-"1M"}
38 DP_DOM_DNE=$DIR/dp_dne
39 DP_STATS=${DP_STATS:-"no"}
44 rm -rf $DP_NORM $DP_DOM $DP_DOM_DNE
47 # total number of files
48 DP_FNUM=${DP_FNUM:-16384}
52 # 1 stripe for normal files
54 $LFS setstripe -c 2 $DP_NORM ||
55 error "Cannot create test directory for ordinary files"
57 if [[ $MDSCOUNT -gt 1 ]] ; then
58 $LFS setdirstripe -i 0 -c $MDSCOUNT $DP_DOM_DNE ||
59 error_noexit "Cannot create striped directory"
60 $LFS setstripe -E ${DOM_SIZE} -L mdt -E EOF $DP_DOM_DNE ||
61 error_noexit "Cannot create test directory for dom files"
65 $LFS setstripe -E ${DOM_SIZE} -L mdt -E EOF $DP_DOM ||
66 error "Cannot create test directory for dom files"
71 $LCTL set_param -n osc.*.stats=0
72 $LCTL set_param -n mdc.*.stats=0
73 $LCTL set_param -n ${cli}.*.${cli}_stats=0
74 $LCTL set_param -n ${cli}.*.rpc_stats=0
75 $LCTL set_param -n llite.*.read_ahead_stats=0
76 $LCTL set_param -n llite.*.unstable_stats=0
83 echo ----- MDC RPCs: $(calc_stats mdc.*.stats req_active)
84 echo ----- OSC RPCs: $(calc_stats osc.*.stats req_active)
86 if [ "x$DP_STATS" != "xyes" ] ; then
90 $LCTL get_param ${cli}.*.${cli}_stats
91 $LCTL get_param ${cli}.*.rpc_stats
92 # for OSC get both OSC and MDC stats
93 if [ $cli == "osc" ] ; then
94 $LCTL get_param mdc.*.stats
96 $LCTL get_param ${cli}.*.stats
97 $LCTL get_param ${cli}.*.unstable_stats
98 $LCTL get_param ${cli}.*.${cli}_cached_mb
99 $LCTL get_param llite.*.read_ahead_stats
105 cancel_lru_locks $cli
106 ### drop all debug except critical
107 $LCTL set_param -n debug="error warning console emerg"
113 local cmdlog=$TMP/dp_cmd.log
116 dp_setup_test $DP_OSC
117 if ! grep -qw "$MOUNT" /proc/mounts ; then
118 echo "!!!!! Lustre is not mounted !!!!!, aborting"
122 echo "## $cmd" | awk '{ if (NR==1) {gsub(/[ \t\r\n]+/, " "); \
123 gsub(/\|.*$/, ""); print }}'
124 echo "## $(date +'%F %H:%M:%S'): START"
125 eval $cmd 2>&1 | tee $cmdlog || true
128 if [ $rc -eq 0 ] && grep -q "p4_error:" $cmdlog ; then
132 dp_collect_stats $DP_OSC
133 remount_client $DIR > /dev/null
138 if ! which mdtest > /dev/null 2>&1 ; then
139 skip_env "Mdtest is not installed, skipping"
142 local mdtest=$(which mdtest)
144 local TDIR=${1:-$MOUNT}
145 local th_num=$((DP_FNUM * 2 / DP_NUM))
150 [ "$SLOW" = "yes" ] && bsizes="4096 32768"
152 for bsize in $bsizes ; do
153 dp_run_cmd "mpi_run -np $DP_NUM $mdtest -i 3 -I $th_num -F \
154 -z 1 -b 1 -L -u -w $bsize -R -d $TDIR"
155 if [ ${PIPESTATUS[0]} != 0 ]; then
156 error "MDtest failed, aborting"
165 local TDIR=${1:-$DIR}
168 local MIN=$((count * 16))
169 local mdssize=$(mdssize_from_index $TDIR 0)
170 [ $mdssize -le $MIN ] && count=$((mdssize / 16))
172 dp_run_cmd "createmany -o $TDIR/file- $count | grep 'total:'"
173 if [ ${PIPESTATUS[0]} != 0 ]; then
174 error "File creation failed, aborting"
177 dp_run_cmd "statmany -s $TDIR/file- $count $((count * 5)) |
179 if [ ${PIPESTATUS[0]} != 0 ]; then
180 error "File stat failed, aborting"
183 for opc in w a r ; do
184 dp_run_cmd "smalliomany -${opc} $TDIR/file- $count 300 |
186 if [ ${PIPESTATUS[0]} != 0 ]; then
187 error "SmallIO -${opc} failed, aborting"
192 dp_run_cmd "unlinkmany $TDIR/file- $count | grep 'total:'"
193 if [ ${PIPESTATUS[0]} != 0 ]; then
194 error "SmallIO failed, aborting"
201 if ! which IOR > /dev/null 2>&1 ; then
202 skip_env "IOR is not installed, skipping"
205 # Requires at least 20GB (roughly)
206 (( MDSSIZE >= 20000000 )) || skip "Require MDS of at least 20GB"
208 local IOR=$(which IOR)
209 local iter=$((DP_FNUM / DP_NUM))
212 if [ "x$DP_DIO" == "xyes" ] ; then
216 local TDIR=${1:-$MOUNT}
220 # for DoM large files (beyond the DoM size) use
222 # bsize="4096 " - 4Mb
223 # nsegments=$((128 * 1024))
225 # bsize="1024 " - 1Mb
226 # nsegments=$((32 * 1024))
227 local bsizes=${BSIZES:-"4 32"}
228 local nsegments=${NSEGMENTS:-128}
229 [ "$SLOW" = "no" ] && bsizes="8"
231 for bsize in $bsizes ; do
232 segments=$((nsegments / bsize))
234 dp_run_cmd "mpi_run -np $DP_NUM $IOR \
235 -a POSIX -b ${bsize}K -t ${bsize}K -o $TDIR/ -k \
236 -s $segments -w -r -i $iter -F -E -z -m -Z $direct" |
237 awk '($1 !~ /^(write|read|access)$/) || NF>12 {print}'
238 if [ ${PIPESTATUS[0]} != 0 ]; then
239 error "IOR write test for ${bsize}K failed, aborting"
242 # check READ performance only (no cache)
243 dp_run_cmd "mpi_run -np $DP_NUM $IOR \
244 -a POSIX -b ${bsize}K -t ${bsize}K -o $TDIR/ -X 42\
245 -s $segments -r -i $iter -F -E -z -m -Z $direct" |
246 awk '($1 !~ /^(read|access|remove)$/) || NF>12 {print}'
247 if [ ${PIPESTATUS[0]} != 0 ]; then
248 error "IOR read test for ${bsize}K failed, aborting"
257 if ! which dbench > /dev/null 2>&1 ; then
258 skip_env "Dbench is not installed, skipping"
261 local TDIR=${1:-$MOUNT}
263 if [ "x$DP_DOM_DNE" == "x$TDIR" ] ; then
264 echo "dbench uses subdirs, skipping for DNE dir"
268 dp_run_cmd "dbench -D $TDIR $DP_NUM | egrep -v 'warmup|execute'"
269 if [ ${PIPESTATUS[0]} != 0 ]; then
270 error "Dbench failed, aborting"
278 # https://github.com/axboe/fio/archive/fio-2.8.zip
279 if ! which fio > /dev/null 2>&1 ; then
280 skip_env "No FIO installed, skipping"
283 local fnum=128 # per thread
284 local total=$((fnum * DP_NUM)) # files in all threads
285 local loops=$((DP_FNUM / total)) # number of loops
289 if [ $loops -eq 0 ] ; then
293 if [ "x$DP_DIO" == "xyes" ] ; then
296 direct="--buffered=1 --bs_unaligned=1"
299 if [ "x$DP_STATS" != "xyes" ] ; then
303 local TDIR=${1:-$MOUNT}
304 base_cmd="fio --name=smallio --ioengine=posixaio $output \
305 --iodepth=$((DP_NUM * 4)) --directory=$TDIR \
306 --nrfiles=$fnum --openfiles=10000 \
307 --numjobs=$DP_NUM --filesize=64k --lockfile=readwrite"
309 dp_run_cmd "$base_cmd --create_only=1" > /dev/null
310 if [ ${PIPESTATUS[0]} != 0 ]; then
311 error "FIO file creation failed, aborting"
315 [ "$SLOW" = "yes" ] && bsizes="4 32"
317 for bsize in $bsizes ; do
318 local write_cmd="$base_cmd --bs=${bsize}k --rw=randwrite \
319 $direct --file_service_type=random --randrepeat=1 \
320 --norandommap --group_reporting=1 --loops=$loops"
321 if [ "x$DP_STATS" != "xyes" ] ; then
322 dp_run_cmd "$write_cmd | awk -F\; '{printf \"WRITE: \
323 BW %dKiB/sec, IOPS %d, lat (%d/%d/%d)usec\n\", \
324 \$48, \$49, \$53, \$57, \$81}'"
326 dp_run_cmd "$write_cmd"
328 if [ ${PIPESTATUS[0]} != 0 ]; then
329 error "FIO write test with ${bsize}k failed, aborting"
332 local read_cmd="$base_cmd --bs=${bsize}k --rw=randread \
333 $direct --file_service_type=random --randrepeat=1 \
334 --norandommap --group_reporting=1 --loops=$loops"
335 if [ "x$DP_STATS" != "xyes" ] ; then
336 dp_run_cmd "$read_cmd | awk -F\; '{printf \"READ : \
337 BW %dKiB/sec, IOPS %d, lat (%d/%d/%d)usec\n\", \
338 \$7, \$8, \$12, \$16, \$40}'"
340 dp_run_cmd "$read_cmd"
342 if [ ${PIPESTATUS[0]} != 0 ]; then
343 error "FIO read test with ${bsize}k failed, aborting"
352 if [ x$cbench_DIR = x ]; then
353 compilebench=$(which compilebench 2> /dev/null)
356 [ -x compilebench ] ||
357 skip_env "compilebench is missing in $cbench_DIR"
358 compilebench=compilebench
361 [ x$compilebench != x ] ||
362 skip_env "Compilebench is not installed, skipping"
364 local TDIR=${1:-$MOUNT}
366 dp_run_cmd "$compilebench -D $TDIR -i 2 -r 2 --makej"
367 if [ ${PIPESTATUS[0]} != 0 ]; then
368 error "Compilebench failed, aborting"
376 local facets=$(get_facets MDS)
377 local nodes=$(comma_list $(mdts_nodes))
378 local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
380 save_lustre_params $facets "mdt.*.dom_lock" >> $p
382 printf "\n##### $test: DoM files\n"
383 do_nodes $nodes "lctl set_param -n mdt.*.dom_lock=1"
387 if [ -d $DP_DOM_DNE ] ; then
388 printf "\n##### $test: DoM files + DNE\n"
390 run_${test} $DP_DOM_DNE
393 printf "\n##### $test: OST files\n"
397 restore_lustre_params < $p
404 run_test smallio "Performance comparision: smallio"
409 run_test mdtest "Performance comparision: mdtest"
414 run_test IOR "Performance comparision: IOR"
419 run_test dbench "Performance comparision: dbench"
424 run_test fio "Performance comparision: FIO"
427 dp_test_run compbench
429 run_test compbench "Performance comparision: compilebench"
431 FAIL_ON_ERROR=$SAVED_FAIL_ON_ERROR
432 $LCTL set_param -n debug="$SAVED_DEBUG"
434 complete_test $SECONDS
435 check_and_cleanup_lustre