lustre/tests/dom-performance.sh

   1 #!/bin/bash
   2 #
   3 # Run select tests by setting ONLY, or as arguments to the script.
   4 # Skip specific tests by setting EXCEPT.
   5 #
   6
   7 set -e
   8
   9 ONLY=${ONLY:-"$*"}
  10 ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-"$DOM_PERFORMANCE_EXCEPT"}
  11 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
  12
  13 [ "$SLOW" = "no" ] && EXCEPT_SLOW=""
  14
  15 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
  16
  17 . $LUSTRE/tests/test-framework.sh
  18
  19 init_test_env $@
  20 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
  21 init_logging
  22
  23 SAVED_FAIL_ON_ERROR=$FAIL_ON_ERROR
  24 FAIL_ON_ERROR=false
  25
  26 SAVED_DEBUG=$($LCTL get_param -n debug 2> /dev/null)
  27
  28 check_and_setup_lustre
  29
  30 build_test_filter
  31
  32 DP_DIO=${DP_DIO:-"no"}
  33
  34 DOM_SIZE=${DOM_SIZE:-"1M"}
  35 DP_OSC="mdc"
  36
  37 rm -rf $DIR/*
  38
  39 DP_NORM=$DIR/dp_norm
  40 DP_DOM=$DIR/dp_dom
  41 DP_DOM_DNE=$DIR/dp_dne
  42 DP_STATS=${DP_STATS:-"no"}
  43
  44 # total number of files
  45 DP_FNUM=${DP_FNUM:-16384}
  46 # number of threads
  47 DP_NUM=${DP_NUM:-4}
  48
  49 # 1 stripe for normal files
  50 mkdir -p $DP_NORM
  51 $LFS setstripe -c 2 $DP_NORM ||
  52         error "Cannot create test directory for ordinary files"
  53
  54 if [[ $MDSCOUNT -gt 1 ]] ; then
  55         $LFS setdirstripe -i 0 -c $MDSCOUNT $DP_DOM_DNE ||
  56                 error_noexit "Cannot create striped directory"
  57         $LFS setstripe -E ${DOM_SIZE} -L mdt -E EOF $DP_DOM_DNE ||
  58                 error_noexit "Cannot create test directory for dom files"
  59 fi
  60
  61 mkdir -p $DP_DOM
  62 $LFS setstripe -E ${DOM_SIZE} -L mdt -E EOF $DP_DOM ||
  63         error "Cannot create test directory for dom files"
  64
  65 dp_clear_stats() {
  66         local cli=$1
  67
  68         $LCTL set_param -n osc.*.stats=0
  69         $LCTL set_param -n mdc.*.stats=0
  70         $LCTL set_param -n ${cli}.*.${cli}_stats=0
  71         $LCTL set_param -n ${cli}.*.rpc_stats=0
  72         $LCTL set_param -n llite.*.read_ahead_stats=0
  73         $LCTL set_param -n llite.*.unstable_stats=0
  74 }
  75
  76 dp_collect_stats() {
  77         local cli=$1
  78
  79         sync;sync
  80         echo ----- MDC RPCs: $(calc_stats mdc.*.stats req_active)
  81         echo ----- OSC RPCs: $(calc_stats osc.*.stats req_active)
  82
  83         if [ "x$DP_STATS" != "xyes" ] ; then
  84                 return 0
  85         fi
  86
  87         $LCTL get_param ${cli}.*.${cli}_stats
  88         $LCTL get_param ${cli}.*.rpc_stats
  89         # for OSC get both OSC and MDC stats
  90         if [ $cli == "osc" ] ; then
  91                 $LCTL get_param mdc.*.stats
  92         fi
  93         $LCTL get_param ${cli}.*.stats
  94         $LCTL get_param ${cli}.*.unstable_stats
  95         $LCTL get_param ${cli}.*.${cli}_cached_mb
  96         $LCTL get_param llite.*.read_ahead_stats
  97 }
  98
  99 dp_setup_test() {
 100         local cli=$1
 101
 102         cancel_lru_locks $cli
 103         ### drop all debug except critical
 104         $LCTL set_param -n debug="error warning console emerg"
 105         dp_clear_stats $cli
 106 }
 107
 108 dp_run_cmd() {
 109         local cmd=$1
 110         local cmdlog=$TMP/dp_cmd.log
 111         local rc
 112
 113         dp_setup_test $DP_OSC
 114         if ! grep -qw "$MOUNT" /proc/mounts ; then
 115                 echo "!!!!! Lustre is not mounted !!!!!, aborting"
 116                 return 0
 117         fi
 118
 119         echo "## $cmd" | awk '{ if (NR==1) {gsub(/[ \t\r\n]+/, " "); \
 120                                 gsub(/\|.*$/, ""); print }}'
 121         echo "## $(date +'%F %H:%M:%S'): START"
 122         eval $cmd 2>&1 | tee $cmdlog || true
 123
 124         rc=${PIPESTATUS[0]}
 125         if [ $rc -eq 0 ] && grep -q "p4_error:" $cmdlog ; then
 126                 rc=1
 127         fi
 128
 129         dp_collect_stats $DP_OSC
 130         remount_client $DIR > /dev/null
 131         return $rc
 132 }
 133
 134 run_MDtest() {
 135         if ! which mdtest > /dev/null 2>&1 ; then
 136                 echo "Mdtest is not installed, skipping"
 137                 return 0
 138         fi
 139
 140         local mdtest=$(which mdtest)
 141
 142         local TDIR=${1:-$MOUNT}
 143         local th_num=$((DP_FNUM * 2 / DP_NUM))
 144         local bsizes="8192"
 145
 146         [ "$SLOW" = "yes" ] && bsizes="4096 32768"
 147
 148         for bsize in $bsizes ; do
 149                 dp_run_cmd "mpirun -np $DP_NUM $mdtest -i 3 -I $th_num -F \
 150                         -z 1 -b 1 -L -u -w $bsize -R -d $TDIR"
 151                 if [ ${PIPESTATUS[0]} != 0 ]; then
 152                         error "MDtest failed, aborting"
 153                 fi
 154         done
 155
 156         rm -rf $TDIR/*
 157         return 0
 158 }
 159
 160 run_SmallIO() {
 161         if [ ! -f createmany ] ; then
 162                 echo "Createmany is not installed, skipping"
 163                 return 0
 164         fi
 165
 166         if [ ! -f smalliomany ] ; then
 167                 echo "Smalliomany is not installed, skipping"
 168                 return 0
 169         fi
 170
 171         local TDIR=${1:-$DIR}
 172         local count=$DP_FNUM
 173
 174         local MIN=$((count * 16))
 175         [ $MDSSIZE -le $MIN ] && count=$((MDSSIZE / 16))
 176
 177         dp_run_cmd "./createmany -o $TDIR/file- $count | grep 'total:'"
 178         if [ ${PIPESTATUS[0]} != 0 ]; then
 179                 error "File creation failed, aborting"
 180         fi
 181
 182         if [ -f statmany ]; then
 183                 dp_run_cmd "./statmany -s $TDIR/file- $count $((count * 5)) |
 184                         grep 'total:'"
 185                 if [ ${PIPESTATUS[0]} != 0 ]; then
 186                         error "File stat failed, aborting"
 187                 fi
 188
 189         fi
 190
 191         for opc in w a r ; do
 192                 dp_run_cmd "./smalliomany -${opc} $TDIR/file- $count 300 |
 193                         grep 'total:'"
 194                 if [ ${PIPESTATUS[0]} != 0 ]; then
 195                         error "SmallIO -${opc} failed, aborting"
 196                 fi
 197
 198         done
 199
 200         dp_run_cmd "./unlinkmany $TDIR/file- $count | grep 'total:'"
 201         if [ ${PIPESTATUS[0]} != 0 ]; then
 202                 error "SmallIO failed, aborting"
 203         fi
 204
 205         return 0
 206 }
 207
 208 run_IOR() {
 209         if ! which IOR > /dev/null 2>&1 ; then
 210                 echo "IOR is not installed, skipping"
 211                 return 0
 212         fi
 213
 214         local IOR=$(which IOR)
 215         local iter=$((DP_FNUM / DP_NUM))
 216         local direct=""
 217
 218         if [ "x$DP_DIO" == "xyes" ] ; then
 219                 direct="-B"
 220         fi
 221
 222         local TDIR=${1:-$MOUNT}
 223         local bsizes="8"
 224         [ "$SLOW" = "yes" ] && bsizes="4 32"
 225
 226         for bsize in $bsizes ; do
 227                 segments=$((128 / bsize))
 228
 229                 dp_run_cmd "mpirun -np $DP_NUM $IOR \
 230                         -a POSIX -b ${bsize}K -t ${bsize}K -o $TDIR/ -k \
 231                         -s $segments -w -r -i $iter -F -E -z -m -Z $direct"
 232                 if [ ${PIPESTATUS[0]} != 0 ]; then
 233                         error "IOR write test for ${bsize}K failed, aborting"
 234                 fi
 235
 236                 # check READ performance only (no cache)
 237                 dp_run_cmd "mpirun -np $DP_NUM $IOR \
 238                         -a POSIX -b ${bsize}K -t ${bsize}K -o $TDIR/ -X 42\
 239                         -s $segments -r -i $iter -F -E -z -m -Z $direct"
 240                 if [ ${PIPESTATUS[0]} != 0 ]; then
 241                         error "IOR read test for ${bsize}K failed, aborting"
 242                 fi
 243
 244         done
 245         rm -rf $TDIR/*
 246         return 0
 247 }
 248
 249 run_Dbench() {
 250         if ! which dbench > /dev/null 2>&1 ; then
 251                 echo "Dbench is not installed, skipping"
 252                 return 0
 253         fi
 254
 255         local TDIR=${1:-$MOUNT}
 256
 257         if [ "x$DP_DOM_DNE" == "x$TDIR" ] ; then
 258                 echo "dbench uses subdirs, skipping for DNE dir"
 259                 return 0
 260         fi
 261
 262         dp_run_cmd "dbench -D $TDIR $DP_NUM | egrep -v 'warmup|execute'"
 263         if [ ${PIPESTATUS[0]} != 0 ]; then
 264                 error "Dbench failed, aborting"
 265         fi
 266
 267         rm -rf $TDIR/*
 268         return 0
 269 }
 270
 271 run_FIO() {
 272         # https://github.com/axboe/fio/archive/fio-2.8.zip
 273         if ! which fio > /dev/null 2>&1 ; then
 274                 echo "No FIO installed, skipping"
 275                 return 0
 276         fi
 277
 278         local fnum=128 # per thread
 279         local total=$((fnum * DP_NUM)) # files in all threads
 280         local loops=$((DP_FNUM / total)) # number of loops
 281         local direct=""
 282         local output=""
 283
 284         if [ $loops -eq 0 ] ; then
 285                 loops=1
 286         fi
 287
 288         if [ "x$DP_DIO" == "xyes" ] ; then
 289                 direct="--direct=1"
 290         else
 291                 direct="--buffered=1 --bs_unaligned=1"
 292         fi
 293
 294         if [ "x$DP_STATS" != "xyes" ] ; then
 295                 output="--minimal"
 296         fi
 297
 298         local TDIR=${1:-$MOUNT}
 299         base_cmd="fio --name=smallio --ioengine=posixaio $output \
 300                   --iodepth=$((DP_NUM * 4)) --directory=$TDIR \
 301                   --nrfiles=$fnum --openfiles=10000 \
 302                   --numjobs=$DP_NUM --filesize=64k --lockfile=readwrite"
 303
 304         dp_run_cmd "$base_cmd --create_only=1" > /dev/null
 305         if [ ${PIPESTATUS[0]} != 0 ]; then
 306                 error "FIO file creation failed, aborting"
 307         fi
 308
 309         local bsizes="8"
 310         [ "$SLOW" = "yes" ] && bsizes="4 32"
 311
 312         for bsize in $bsizes ; do
 313                 local write_cmd="$base_cmd --bs=${bsize}k --rw=randwrite \
 314                         $direct --file_service_type=random --randrepeat=1 \
 315                          --norandommap --group_reporting=1 --loops=$loops"
 316                 if [ "x$DP_STATS" != "xyes" ] ; then
 317                         dp_run_cmd "$write_cmd | awk -F\; '{printf \"WRITE: \
 318                                 BW %dKiB/sec, IOPS %d, lat (%d/%d/%d)usec\n\", \
 319                                 \$48, \$49, \$53, \$57, \$81}'"
 320                 else
 321                         dp_run_cmd "$write_cmd"
 322                 fi
 323                 if [ ${PIPESTATUS[0]} != 0 ]; then
 324                         error "FIO write test with ${bsize}k failed, aborting"
 325                 fi
 326
 327                 local read_cmd="$base_cmd --bs=${bsize}k --rw=randread \
 328                         $direct --file_service_type=random --randrepeat=1 \
 329                          --norandommap --group_reporting=1 --loops=$loops"
 330                 if [ "x$DP_STATS" != "xyes" ] ; then
 331                         dp_run_cmd "$read_cmd | awk -F\; '{printf \"READ : \
 332                                 BW %dKiB/sec, IOPS %d, lat (%d/%d/%d)usec\n\", \
 333                                 \$7, \$8, \$12, \$16, \$40}'"
 334                 else
 335                         dp_run_cmd "$read_cmd"
 336                 fi
 337                 if [ ${PIPESTATUS[0]} != 0 ]; then
 338                         error "FIO read test with ${bsize}k failed, aborting"
 339                 fi
 340         done
 341         rm -rf $TDIR/*
 342         return 0
 343 }
 344
 345 run_compbench() {
 346         if ! which compilebench > /dev/null 2>&1 ; then
 347                 echo "Compilebench is not installed, skipping"
 348                 return 0
 349         fi
 350
 351         local TDIR=${1:-$MOUNT}
 352
 353         dp_run_cmd "compilebench -D $TDIR -i 2 -r 2 --makej"
 354         if [ ${PIPESTATUS[0]} != 0 ]; then
 355                 error "Compilebench failed, aborting"
 356         fi
 357
 358         rm -rf $TDIR/*
 359 }
 360
 361 dp_test_run() {
 362         local test=$1
 363         local facets=$(get_facets MDS)
 364         local nodes=$(comma_list $(mdts_nodes))
 365         local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
 366
 367         save_lustre_params $facets "mdt.*.dom_lock" >> $p
 368
 369         printf "\n##### $test: DoM files\n"
 370         do_nodes $nodes "lctl set_param -n mdt.*.dom_lock=1"
 371         DP_OSC="mdc"
 372         run_${test} $DP_DOM
 373
 374         if [ -d $DP_DOM_DNE ] ; then
 375                 printf "\n##### $test: DoM files + DNE\n"
 376                 DP_OSC="mdc"
 377                 run_${test} $DP_DOM_DNE
 378         fi
 379
 380         printf "\n##### $test: OST files\n"
 381         DP_OSC="osc"
 382         run_${test} $DP_NORM
 383
 384         restore_lustre_params < $p
 385         rm -f $p
 386 }
 387
 388 test_smallio() {
 389         dp_test_run SmallIO
 390 }
 391 run_test smallio "Performance comparision: smallio"
 392
 393 test_mdtest() {
 394         dp_test_run MDtest
 395 }
 396 run_test mdtest "Performance comparision: mdtest"
 397
 398 test_IOR() {
 399         dp_test_run IOR
 400 }
 401 run_test IOR "Performance comparision: IOR"
 402
 403 test_dbench() {
 404         dp_test_run Dbench
 405 }
 406 run_test dbench "Performance comparision: dbench"
 407
 408 test_fio() {
 409         dp_test_run FIO
 410 }
 411 run_test fio "Performance comparision: FIO"
 412
 413 test_compbench() {
 414         dp_test_run compbench
 415 }
 416 run_test compbench "Performance comparision: compilebench"
 417
 418 FAIL_ON_ERROR=$SAVED_FAIL_ON_ERROR
 419 $LCTL set_param -n debug="$SAVED_DEBUG"
 420
 421 complete $SECONDS
 422 check_and_cleanup_lustre
 423 exit_status