lustre/tests/dom-performance.sh

   1 #!/bin/bash
   2 #
   3 # Run select tests by setting ONLY, or as arguments to the script.
   4 # Skip specific tests by setting EXCEPT.
   5 #
   6
   7 set -e
   8
   9 ONLY=${ONLY:-"$*"}
  10
  11 LUSTRE=${LUSTRE:-$(dirname $0)/..}
  12 . $LUSTRE/tests/test-framework.sh
  13 init_test_env $@
  14 init_logging
  15
  16 ALWAYS_EXCEPT="$DOM_PERFORMANCE_EXCEPT"
  17 build_test_filter
  18
  19 SAVED_FAIL_ON_ERROR=$FAIL_ON_ERROR
  20 FAIL_ON_ERROR=false
  21
  22 SAVED_DEBUG=$($LCTL get_param -n debug 2> /dev/null)
  23
  24 . $LUSTRE/tests/functions.sh
  25 check_and_setup_lustre
  26
  27 clients=${CLIENTS:-$HOSTNAME}
  28 generate_machine_file $clients $MACHINEFILE ||
  29         error "Failed to generate machine file"
  30
  31 DP_DIO=${DP_DIO:-"no"}
  32
  33 DOM_SIZE=${DOM_SIZE:-"1M"}
  34 DP_OSC="mdc"
  35
  36 DP_NORM=$DIR/dp_norm
  37 DP_DOM=$DIR/dp_dom
  38 DP_DOM_DNE=$DIR/dp_dne
  39 DP_STATS=${DP_STATS:-"no"}
  40
  41 if $DO_CLEANUP; then
  42         rm -rf $DIR/*
  43 else
  44         rm -rf $DP_NORM $DP_DOM $DP_DOM_DNE
  45 fi
  46
  47 # total number of files
  48 DP_FNUM=${DP_FNUM:-16384}
  49 # number of threads
  50 DP_NUM=${DP_NUM:-4}
  51
  52 # 1 stripe for normal files
  53 mkdir -p $DP_NORM
  54 $LFS setstripe -c 2 $DP_NORM ||
  55         error "Cannot create test directory for ordinary files"
  56
  57 if [[ $MDSCOUNT -gt 1 ]] ; then
  58         $LFS setdirstripe -i 0 -c $MDSCOUNT $DP_DOM_DNE ||
  59                 error_noexit "Cannot create striped directory"
  60         $LFS setstripe -E ${DOM_SIZE} -L mdt -E EOF $DP_DOM_DNE ||
  61                 error_noexit "Cannot create test directory for dom files"
  62 fi
  63
  64 mkdir -p $DP_DOM
  65 $LFS setstripe -E ${DOM_SIZE} -L mdt -E EOF $DP_DOM ||
  66         error "Cannot create test directory for dom files"
  67
  68 dp_clear_stats() {
  69         local cli=$1
  70
  71         $LCTL set_param -n osc.*.stats=0
  72         $LCTL set_param -n mdc.*.stats=0
  73         $LCTL set_param -n ${cli}.*.${cli}_stats=0
  74         $LCTL set_param -n ${cli}.*.rpc_stats=0
  75         $LCTL set_param -n llite.*.read_ahead_stats=0
  76         $LCTL set_param -n llite.*.unstable_stats=0
  77 }
  78
  79 dp_collect_stats() {
  80         local cli=$1
  81
  82         sync;sync
  83         echo ----- MDC RPCs: $(calc_stats mdc.*.stats req_active)
  84         echo ----- OSC RPCs: $(calc_stats osc.*.stats req_active)
  85
  86         if [ "x$DP_STATS" != "xyes" ] ; then
  87                 return 0
  88         fi
  89
  90         $LCTL get_param ${cli}.*.${cli}_stats
  91         $LCTL get_param ${cli}.*.rpc_stats
  92         # for OSC get both OSC and MDC stats
  93         if [ $cli == "osc" ] ; then
  94                 $LCTL get_param mdc.*.stats
  95         fi
  96         $LCTL get_param ${cli}.*.stats
  97         $LCTL get_param ${cli}.*.unstable_stats
  98         $LCTL get_param ${cli}.*.${cli}_cached_mb
  99         $LCTL get_param llite.*.read_ahead_stats
 100 }
 101
 102 dp_setup_test() {
 103         local cli=$1
 104
 105         cancel_lru_locks $cli
 106         ### drop all debug except critical
 107         $LCTL set_param -n debug="error warning console emerg"
 108         dp_clear_stats $cli
 109 }
 110
 111 dp_run_cmd() {
 112         local cmd=$1
 113         local cmdlog=$TMP/dp_cmd.log
 114         local rc
 115
 116         dp_setup_test $DP_OSC
 117         if ! grep -qw "$MOUNT" /proc/mounts ; then
 118                 echo "!!!!! Lustre is not mounted !!!!!, aborting"
 119                 return 0
 120         fi
 121
 122         echo "## $cmd" | awk '{ if (NR==1) {gsub(/[ \t\r\n]+/, " "); \
 123                                 gsub(/\|.*$/, ""); print }}'
 124         echo "## $(date +'%F %H:%M:%S'): START"
 125         eval $cmd 2>&1 | tee $cmdlog || true
 126
 127         rc=${PIPESTATUS[0]}
 128         if [ $rc -eq 0 ] && grep -q "p4_error:" $cmdlog ; then
 129                 rc=1
 130         fi
 131
 132         dp_collect_stats $DP_OSC
 133         remount_client $DIR > /dev/null
 134         return $rc
 135 }
 136
 137 run_MDtest() {
 138         if ! which mdtest > /dev/null 2>&1 ; then
 139                 skip_env "Mdtest is not installed, skipping"
 140         fi
 141
 142         local mdtest=$(which mdtest)
 143
 144         local TDIR=${1:-$MOUNT}
 145         local th_num=$((DP_FNUM * 2 / DP_NUM))
 146         local bsizes="8192"
 147
 148         chmod 0777 $TDIR
 149
 150         [ "$SLOW" = "yes" ] && bsizes="4096 32768"
 151
 152         for bsize in $bsizes ; do
 153                 dp_run_cmd "mpi_run -np $DP_NUM $mdtest -i 3 -I $th_num -F \
 154                         -z 1 -b 1 -L -u -w $bsize -R -d $TDIR"
 155                 if [ ${PIPESTATUS[0]} != 0 ]; then
 156                         error "MDtest failed, aborting"
 157                 fi
 158         done
 159
 160         rm -rf $TDIR/*
 161         return 0
 162 }
 163
 164 run_SmallIO() {
 165         local TDIR=${1:-$DIR}
 166         local count=$DP_FNUM
 167
 168         local MIN=$((count * 16))
 169         local mdssize=$(mdssize_from_index $TDIR 0)
 170         [ $mdssize -le $MIN ] && count=$((mdssize / 16))
 171
 172         dp_run_cmd "createmany -o $TDIR/file- $count | grep 'total:'"
 173         if [ ${PIPESTATUS[0]} != 0 ]; then
 174                 error "File creation failed, aborting"
 175         fi
 176
 177         dp_run_cmd "statmany -s $TDIR/file- $count $((count * 5)) |
 178                 grep 'total:'"
 179         if [ ${PIPESTATUS[0]} != 0 ]; then
 180                 error "File stat failed, aborting"
 181         fi
 182
 183         for opc in w a r ; do
 184                 dp_run_cmd "smalliomany -${opc} $TDIR/file- $count 300 |
 185                         grep 'total:'"
 186                 if [ ${PIPESTATUS[0]} != 0 ]; then
 187                         error "SmallIO -${opc} failed, aborting"
 188                 fi
 189
 190         done
 191
 192         dp_run_cmd "unlinkmany $TDIR/file- $count | grep 'total:'"
 193         if [ ${PIPESTATUS[0]} != 0 ]; then
 194                 error "SmallIO failed, aborting"
 195         fi
 196
 197         return 0
 198 }
 199
 200 run_IOR() {
 201         if ! which IOR > /dev/null 2>&1 ; then
 202                 skip_env "IOR is not installed, skipping"
 203         fi
 204
 205         # Requires at least 20GB (roughly)
 206         (( MDSSIZE >= 20000000 )) || skip "Require MDS of at least 20GB"
 207
 208         local IOR=$(which IOR)
 209         local iter=$((DP_FNUM / DP_NUM))
 210         local direct=""
 211
 212         if [ "x$DP_DIO" == "xyes" ] ; then
 213                 direct="-B"
 214         fi
 215
 216         local TDIR=${1:-$MOUNT}
 217
 218         chmod 0777 $TDIR
 219
 220         # for DoM large files (beyond the DoM size) use
 221         # DOM_SIZE=1M :
 222         #     bsize="4096 " - 4Mb
 223         #     nsegments=$((128 * 1024))
 224         # DOM_SIZE=64k :
 225         #     bsize="1024 " - 1Mb
 226         #     nsegments=$((32 * 1024))
 227         local bsizes=${BSIZES:-"4 32"}
 228         local nsegments=${NSEGMENTS:-128}
 229         [ "$SLOW" = "no" ] && bsizes="8"
 230
 231         for bsize in $bsizes ; do
 232                 segments=$((nsegments / bsize))
 233
 234                 dp_run_cmd "mpi_run -np $DP_NUM $IOR \
 235                         -a POSIX -b ${bsize}K -t ${bsize}K -o $TDIR/ -k \
 236                         -s $segments -w -r -i $iter -F -E -z -m -Z $direct" |
 237                         awk '($1 !~ /^(write|read|access)$/) || NF>12 {print}'
 238                 if [ ${PIPESTATUS[0]} != 0 ]; then
 239                         error "IOR write test for ${bsize}K failed, aborting"
 240                 fi
 241
 242                 # check READ performance only (no cache)
 243                 dp_run_cmd "mpi_run -np $DP_NUM $IOR \
 244                         -a POSIX -b ${bsize}K -t ${bsize}K -o $TDIR/ -X 42\
 245                         -s $segments -r -i $iter -F -E -z -m -Z $direct" |
 246                         awk '($1 !~ /^(read|access|remove)$/) || NF>12 {print}'
 247                 if [ ${PIPESTATUS[0]} != 0 ]; then
 248                         error "IOR read test for ${bsize}K failed, aborting"
 249                 fi
 250
 251         done
 252         rm -rf $TDIR/*
 253         return 0
 254 }
 255
 256 run_Dbench() {
 257         if ! which dbench > /dev/null 2>&1 ; then
 258                 skip_env "Dbench is not installed, skipping"
 259         fi
 260
 261         local TDIR=${1:-$MOUNT}
 262
 263         if [ "x$DP_DOM_DNE" == "x$TDIR" ] ; then
 264                 echo "dbench uses subdirs, skipping for DNE dir"
 265                 return 0
 266         fi
 267
 268         dp_run_cmd "dbench -D $TDIR $DP_NUM | egrep -v 'warmup|execute'"
 269         if [ ${PIPESTATUS[0]} != 0 ]; then
 270                 error "Dbench failed, aborting"
 271         fi
 272
 273         rm -rf $TDIR/*
 274         return 0
 275 }
 276
 277 run_FIO() {
 278         # https://github.com/axboe/fio/archive/fio-2.8.zip
 279         if ! which fio > /dev/null 2>&1 ; then
 280                 skip_env "No FIO installed, skipping"
 281         fi
 282
 283         local fnum=128 # per thread
 284         local total=$((fnum * DP_NUM)) # files in all threads
 285         local loops=$((DP_FNUM / total)) # number of loops
 286         local direct=""
 287         local output=""
 288
 289         if [ $loops -eq 0 ] ; then
 290                 loops=1
 291         fi
 292
 293         if [ "x$DP_DIO" == "xyes" ] ; then
 294                 direct="--direct=1"
 295         else
 296                 direct="--buffered=1 --bs_unaligned=1"
 297         fi
 298
 299         if [ "x$DP_STATS" != "xyes" ] ; then
 300                 output="--minimal"
 301         fi
 302
 303         local TDIR=${1:-$MOUNT}
 304         base_cmd="fio --name=smallio --ioengine=posixaio $output \
 305                   --iodepth=$((DP_NUM * 4)) --directory=$TDIR \
 306                   --nrfiles=$fnum --openfiles=10000 \
 307                   --numjobs=$DP_NUM --filesize=64k --lockfile=readwrite"
 308
 309         dp_run_cmd "$base_cmd --create_only=1" > /dev/null
 310         if [ ${PIPESTATUS[0]} != 0 ]; then
 311                 error "FIO file creation failed, aborting"
 312         fi
 313
 314         local bsizes="8"
 315         [ "$SLOW" = "yes" ] && bsizes="4 32"
 316
 317         for bsize in $bsizes ; do
 318                 local write_cmd="$base_cmd --bs=${bsize}k --rw=randwrite \
 319                         $direct --file_service_type=random --randrepeat=1 \
 320                          --norandommap --group_reporting=1 --loops=$loops"
 321                 if [ "x$DP_STATS" != "xyes" ] ; then
 322                         dp_run_cmd "$write_cmd | awk -F\; '{printf \"WRITE: \
 323                                 BW %dKiB/sec, IOPS %d, lat (%d/%d/%d)usec\n\", \
 324                                 \$48, \$49, \$53, \$57, \$81}'"
 325                 else
 326                         dp_run_cmd "$write_cmd"
 327                 fi
 328                 if [ ${PIPESTATUS[0]} != 0 ]; then
 329                         error "FIO write test with ${bsize}k failed, aborting"
 330                 fi
 331
 332                 local read_cmd="$base_cmd --bs=${bsize}k --rw=randread \
 333                         $direct --file_service_type=random --randrepeat=1 \
 334                          --norandommap --group_reporting=1 --loops=$loops"
 335                 if [ "x$DP_STATS" != "xyes" ] ; then
 336                         dp_run_cmd "$read_cmd | awk -F\; '{printf \"READ : \
 337                                 BW %dKiB/sec, IOPS %d, lat (%d/%d/%d)usec\n\", \
 338                                 \$7, \$8, \$12, \$16, \$40}'"
 339                 else
 340                         dp_run_cmd "$read_cmd"
 341                 fi
 342                 if [ ${PIPESTATUS[0]} != 0 ]; then
 343                         error "FIO read test with ${bsize}k failed, aborting"
 344                 fi
 345         done
 346         rm -rf $TDIR/*
 347         return 0
 348 }
 349
 350 run_compbench() {
 351         local compilebench
 352         if [ x$cbench_DIR = x ]; then
 353                 compilebench=$(which compilebench 2> /dev/null)
 354         else
 355                 cd $cbench_DIR
 356                 [ -x compilebench ] ||
 357                         skip_env "compilebench is missing in $cbench_DIR"
 358                 compilebench=compilebench
 359         fi
 360
 361         [ x$compilebench != x ] ||
 362                 skip_env "Compilebench is not installed, skipping"
 363
 364         local TDIR=${1:-$MOUNT}
 365
 366         dp_run_cmd "$compilebench -D $TDIR -i 2 -r 2 --makej"
 367         if [ ${PIPESTATUS[0]} != 0 ]; then
 368                 error "Compilebench failed, aborting"
 369         fi
 370
 371         rm -rf $TDIR/*
 372 }
 373
 374 dp_test_run() {
 375         local test=$1
 376         local facets=$(get_facets MDS)
 377         local nodes=$(comma_list $(mdts_nodes))
 378         local p="$TMP/$TESTSUITE-$TESTNAME.parameters"
 379
 380         save_lustre_params $facets "mdt.*.dom_lock" >> $p
 381
 382         printf "\n##### $test: DoM files\n"
 383         do_nodes $nodes "lctl set_param -n mdt.*.dom_lock=1"
 384         DP_OSC="mdc"
 385         run_${test} $DP_DOM
 386
 387         if [ -d $DP_DOM_DNE ] ; then
 388                 printf "\n##### $test: DoM files + DNE\n"
 389                 DP_OSC="mdc"
 390                 run_${test} $DP_DOM_DNE
 391         fi
 392
 393         printf "\n##### $test: OST files\n"
 394         DP_OSC="osc"
 395         run_${test} $DP_NORM
 396
 397         restore_lustre_params < $p
 398         rm -f $p
 399 }
 400
 401 test_smallio() {
 402         dp_test_run SmallIO
 403 }
 404 run_test smallio "Performance comparision: smallio"
 405
 406 test_mdtest() {
 407         dp_test_run MDtest
 408 }
 409 run_test mdtest "Performance comparision: mdtest"
 410
 411 test_IOR() {
 412         dp_test_run IOR
 413 }
 414 run_test IOR "Performance comparision: IOR"
 415
 416 test_dbench() {
 417         dp_test_run Dbench
 418 }
 419 run_test dbench "Performance comparision: dbench"
 420
 421 test_fio() {
 422         dp_test_run FIO
 423 }
 424 run_test fio "Performance comparision: FIO"
 425
 426 test_compbench() {
 427         dp_test_run compbench
 428 }
 429 run_test compbench "Performance comparision: compilebench"
 430
 431 FAIL_ON_ERROR=$SAVED_FAIL_ON_ERROR
 432 $LCTL set_param -n debug="$SAVED_DEBUG"
 433
 434 complete $SECONDS
 435 check_and_cleanup_lustre
 436 exit_status