lustre/tests/acceptance-small.sh

   1 #!/bin/bash
   2 # script which _must_ complete successfully (at minimum) before checkins to
   3 # the CVS HEAD are allowed.
   4 #set -vx
   5 set -e
   6
   7 [ -z "$CONFIG" -a "$NAME" ] && CONFIGS=$NAME
   8 [ "$CONFIGS" ] || CONFIGS="local"  #"local lov"
   9 [ "$MAX_THREADS" ] || MAX_THREADS=20
  10 RAMKB=`awk '/MemTotal:/ { print $2 }' /proc/meminfo`
  11 if [ -z "$THREADS" ]; then
  12         THREADS=$((RAMKB / 16384))
  13         [ $THREADS -gt $MAX_THREADS ] && THREADS=$MAX_THREADS
  14 fi
  15 [ "$SIZE" ] || SIZE=$((RAMKB * 2))
  16 [ "$RSIZE" ] || RSIZE=512
  17 [ "$UID" ] || UID=1000
  18 [ "$MOUNT" ] || MOUNT=/mnt/lustre
  19 [ "$MOUNT2" ] || MOUNT2=${MOUNT}2
  20 [ "$TMP" ] || TMP=/tmp
  21 [ "$COUNT" ] || COUNT=1000
  22 [ "$DEBUG_LVL" ] || DEBUG_LVL=0
  23 [ "$DEBUG_OFF" ] || DEBUG_OFF="eval lctl set_param debug=\"$DEBUG_LVL\""
  24 [ "$DEBUG_ON" ] || DEBUG_ON="eval lctl set_param debug=0x33f0484"
  25
  26 export TESTSUITE_LIST="RUNTESTS SANITY DBENCH BONNIE IOZONE FSX SANITYN LFSCK LIBLUSTRE RACER REPLAY_SINGLE CONF_SANITY RECOVERY_SMALL REPLAY_OST_SINGLE REPLAY_DUAL REPLAY_VBR INSANITY SANITY_QUOTA PERFORMANCE_SANITY LARGE_SCALE RECOVERY_MDS_SCALE RECOVERY_DOUBLE_SCALE RECOVERY_RANDOM_SCALE PARALLEL_SCALE METADATA_UPDATES OST_POOLS SANITY_BENCHMARK LNET_SELFTEST OBDFILTER_SURVEY MMP"
  27
  28 if [ "$ACC_SM_ONLY" ]; then
  29     for O in $TESTSUITE_LIST; do
  30         export ${O}="no"
  31     done
  32     for O in $ACC_SM_ONLY; do
  33         O=`echo ${O%.sh} | tr "-" "_"`
  34         O=`echo $O | tr "[:lower:]" "[:upper:]"`
  35         export ${O}="yes"
  36     done
  37 fi
  38
  39 LIBLUSTRETESTS=${LIBLUSTRETESTS:-../liblustre/tests}
  40
  41 STARTTIME=`date +%s`
  42 RANTEST=""
  43
  44 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
  45 . $LUSTRE/tests/test-framework.sh
  46 init_test_env $@
  47
  48 SETUP=${SETUP:-setupall}
  49 FORMAT=${FORMAT:-formatall}
  50 CLEANUP=${CLEANUP:-stopall}
  51
  52 setup_if_needed() {
  53     nfs_client_mode && return
  54
  55     local MOUNTED=$(mounted_lustre_filesystems)
  56     if $(echo $MOUNTED | grep -w -q $MOUNT); then
  57         check_config_clients $MOUNT
  58         init_facets_vars
  59         init_param_vars
  60         return
  61     fi
  62
  63     echo "Lustre is not mounted, trying to do setup SETUP=$SETUP ... "
  64     [ "$REFORMAT" ] && $FORMAT
  65     $SETUP
  66
  67     MOUNTED=$(mounted_lustre_filesystems)
  68     if ! $(echo $MOUNTED | grep -w -q $MOUNT); then
  69         echo "Lustre is not mounted after setup! SETUP=$SETUP"
  70         exit 1
  71     fi
  72 }
  73
  74 title() {
  75     # update titlebar if stdin is attaached to an xterm
  76     if ${UPDATE_TITLEBAR:-false}; then
  77         if tty -s; then
  78             case $TERM in
  79                 xterm*)
  80                     echo -ne "\033]2; acceptance-small: $* \007" >&0
  81                     ;;
  82             esac
  83         fi
  84     fi
  85     log "-----============= acceptance-small: "$*" ============----- `date`"
  86     RANTEST=${RANTEST}$*", "
  87 }
  88
  89 skip_remost()
  90 {
  91         remote_ost_nodsh && log "SKIP: $1: remote OST with nodsh" && return 0
  92         return 1
  93 }
  94
  95 skip_remmds()
  96 {
  97         remote_mds_nodsh && log "SKIP: $1: remote MDS with nodsh" && return 0
  98         return 1
  99 }
 100
 101 for NAME in $CONFIGS; do
 102         export NAME MOUNT START CLEAN
 103         . $LUSTRE/tests/cfg/$NAME.sh
 104
 105         assert_env mds_HOST MDS_MKFS_OPTS MDSDEV
 106         assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
 107         assert_env FSNAME MOUNT MOUNT2
 108
 109         setup_if_needed
 110
 111         MSKIPPED=0
 112         OSKIPPED=0
 113         if [ "$RUNTESTS" != "no" ]; then
 114                 title runtests
 115                 bash runtests
 116                 $CLEANUP
 117                 $SETUP
 118                 RUNTESTS="done"
 119         fi
 120
 121         if [ "$SANITY" != "no" ]; then
 122                 title sanity
 123                 MOUNT2="" bash sanity.sh
 124                 $CLEANUP
 125                 $SETUP
 126                 SANITY="done"
 127         fi
 128
 129         which dbench > /dev/null 2>&1 || DBENCH=no
 130         if [ "$DBENCH" != "no" ]; then
 131                 title dbench
 132                 DBENCHDIR=$MOUNT/$HOSTNAME
 133                 mkdir -p $DBENCHDIR
 134                 SPACE=`df -P $MOUNT | tail -n 1 | awk '{ print $4 }'`
 135                 DB_THREADS=$((SPACE / 50000))
 136                 [ $THREADS -lt $DB_THREADS ] && DB_THREADS=$THREADS
 137
 138                 $DEBUG_OFF
 139                 myUID=$RUNAS_ID
 140                 myRUNAS=$RUNAS
 141                 FAIL_ON_ERROR=false check_runas_id_ret $myUID $myUID $myRUNAS || { myRUNAS="" && myUID=$UID; }
 142                 chown $myUID:$myUID $DBENCHDIR
 143                 duration=""
 144                 [ "$SLOW" = "no" ] && duration=" -t 120"
 145                 if [ "$SLOW" != "no" -o $DB_THREADS -eq 1 ]; then
 146                         $myRUNAS bash rundbench -D $DBENCHDIR 1 $duration || error "dbench failed!"
 147                         $DEBUG_ON
 148                         $CLEANUP
 149                         $SETUP
 150                 fi
 151                 if [ $DB_THREADS -gt 1 ]; then
 152                         $DEBUG_OFF
 153                         $myRUNAS bash rundbench -D $DBENCHDIR $DB_THREADS $duration
 154                         $DEBUG_ON
 155                         $CLEANUP
 156                         $SETUP
 157                 fi
 158                 rm -rf $DBENCHDIR
 159                 DBENCH="done"
 160         fi
 161
 162         which bonnie++ > /dev/null 2>&1 || BONNIE=no
 163         if [ "$BONNIE" != "no" ]; then
 164                 title bonnie
 165                 BONDIR=$MOUNT/d0.bonnie
 166                 mkdir -p $BONDIR
 167                 $LFS setstripe -c -1 $BONDIR
 168                 sync
 169                 MIN=`lctl get_param -n osc.*.kbytesavail | sort -n | head -n1`
 170                 SPACE=$(( OSTCOUNT * MIN ))
 171                 [ $SPACE -lt $SIZE ] && SIZE=$((SPACE * 3 / 4))
 172                 log "min OST has ${MIN}kB available, using ${SIZE}kB file size"
 173                 $DEBUG_OFF
 174                 myUID=$RUNAS_ID
 175                 myRUNAS=$RUNAS
 176                 FAIL_ON_ERROR=false check_runas_id_ret $myUID $myUID $myRUNAS || { myRUNAS="" && myUID=$UID; }
 177                 chown $myUID:$myUID $BONDIR
 178                 $myRUNAS bonnie++ -f -r 0 -s$((SIZE / 1024)) -n 10 -u$myUID:$myUID -d$BONDIR
 179                 $DEBUG_ON
 180                 $CLEANUP
 181                 $SETUP
 182                 BONNIE="done"
 183         fi
 184
 185         export O_DIRECT
 186         [ "$SLOW" = "no" ] && export IOZONE=no # 5 minutes
 187
 188         which iozone > /dev/null 2>&1 || IOZONE=no
 189         if [ "$IOZONE" != "no" ]; then
 190                 title iozone
 191                 IOZDIR=$MOUNT/d0.iozone
 192                 mkdir -p $IOZDIR
 193                 $LFS setstripe -c -1 $IOZDIR
 194                 sync
 195                 MIN=`lctl get_param -n osc.*.kbytesavail | sort -n | head -n1`
 196                 SPACE=$(( OSTCOUNT * MIN ))
 197                 [ $SPACE -lt $SIZE ] && SIZE=$((SPACE * 3 / 4))
 198                 log "min OST has ${MIN}kB available, using ${SIZE}kB file size"
 199                 IOZONE_OPTS="-i 0 -i 1 -i 2 -e -+d -r $RSIZE"
 200                 IOZFILE="$IOZDIR/iozone"
 201                 IOZLOG=$TMP/iozone.log
 202                 # $SPACE was calculated with all OSTs
 203                 $DEBUG_OFF
 204                 myUID=$RUNAS_ID
 205                 myRUNAS=$RUNAS
 206                 FAIL_ON_ERROR=false check_runas_id_ret $myUID $myUID $myRUNAS || { myRUNAS="" && myUID=$UID; }
 207                 chown $myUID:$myUID $IOZDIR
 208                 $myRUNAS iozone $IOZONE_OPTS -s $SIZE -f $IOZFILE 2>&1 | tee $IOZLOG
 209                 tail -1 $IOZLOG | grep -q complete || \
 210                         { error "iozone (1) failed" && false; }
 211                 rm -f $IOZLOG
 212                 $DEBUG_ON
 213                 $CLEANUP
 214                 $SETUP
 215
 216                 # check if O_DIRECT support is implemented in kernel
 217                 if [ -z "$O_DIRECT" ]; then
 218                         touch $MOUNT/f.iozone
 219                         if ! ./directio write $MOUNT/f.iozone 0 1; then
 220                                 log "SKIP iozone DIRECT IO test"
 221                                 O_DIRECT=no
 222                         fi
 223                         rm -f $MOUNT/f.iozone
 224                 fi
 225                 if [ "$O_DIRECT" != "no" -a "$IOZONE_DIR" != "no" ]; then
 226                         $DEBUG_OFF
 227                         $myRUNAS iozone -I $IOZONE_OPTS -s $SIZE -f $IOZFILE.odir 2>&1 | tee $IOZLOG
 228                         tail -1 $IOZLOG | grep -q complete || \
 229                                 { error "iozone (2) failed" && false; }
 230                         rm -f $IOZLOG
 231                         $DEBUG_ON
 232                         $CLEANUP
 233                         $SETUP
 234                 fi
 235
 236                 SPACE=`df -P $MOUNT | tail -n 1 | awk '{ print $4 }'`
 237                 IOZ_THREADS=$((SPACE / SIZE * 2 / 3 ))
 238                 [ $THREADS -lt $IOZ_THREADS ] && IOZ_THREADS=$THREADS
 239                 IOZVER=`iozone -v | awk '/Revision:/ {print $3}' | tr -d .`
 240                 if [ "$IOZ_THREADS" -gt 1 -a "$IOZVER" -ge 3145 ]; then
 241                         $LFS setstripe -c -1 $IOZDIR
 242                         $DEBUG_OFF
 243                         THREAD=1
 244                         IOZFILE=" "
 245                         while [ $THREAD -le $IOZ_THREADS ]; do
 246                                 IOZFILE="$IOZFILE $IOZDIR/iozone.$THREAD"
 247                                 THREAD=$((THREAD + 1))
 248                         done
 249                         $myRUNAS iozone $IOZONE_OPTS -s $((SIZE / IOZ_THREADS)) -t $IOZ_THREADS -F $IOZFILE 2>&1 | tee $IOZLOG
 250                         tail -1 $IOZLOG | grep -q complete || \
 251                                 { error "iozone (3) failed" && false; }
 252                         rm -f $IOZLOG
 253                         $DEBUG_ON
 254                         $CLEANUP
 255                         $SETUP
 256                 elif [ $IOZVER -lt 3145 ]; then
 257                         VER=`iozone -v | awk '/Revision:/ { print $3 }'`
 258                         echo "iozone $VER too old for multi-thread test"
 259                 fi
 260                 IOZONE="done"
 261         fi
 262
 263         if [ "$FSX" != "no" ]; then
 264                 title fsx
 265                 FSX_SIZE=$((RAMKB / 2))
 266                 SPACE=`df -P $MOUNT | tail -n 1 | awk '{ print $4 }'`
 267                 [ $SPACE -lt $FSX_SIZE ] && FSX_SIZE=$((SPACE * 3 / 4))
 268                 $DEBUG_OFF
 269                 FSX_SEED=${FSX_SEED:-$RANDOM}
 270                 rm -f $MOUNT/fsxfile
 271                 $LFS setstripe -c -1 $MOUNT/fsxfile
 272                 echo Using FSX_SEED=$FSX_SEED FSX_SIZE=$FSX_SIZE COUNT=$COUNT
 273                 ./fsx -c 50 -p 1000 -S $FSX_SEED -P $TMP -l $FSX_SIZE \
 274                         -N $(($COUNT * 100)) $MOUNT/fsxfile
 275                 $DEBUG_ON
 276                 $CLEANUP
 277                 $SETUP
 278                 FSX="done"
 279         fi
 280
 281         if [ "$SANITYN" != "no" ]; then
 282                 title sanityN
 283                 bash sanityN.sh
 284                 $CLEANUP
 285                 $SETUP
 286                 SANITYN="done"
 287         fi
 288
 289         if [ "$LFSCK" != "no" ]; then
 290                 title lfsck
 291                 bash lfscktest.sh
 292                 LFSCK="done"
 293         fi
 294
 295         [ "$NETTYPE" = "tcp" -o "$NETTYPE" = "ptl" ] || LIBLUSTRE=no # bug 15660
 296         if [ "$LIBLUSTRE" != "no" ] && ! check_versions ; then
 297                 skip liblustre version mismatch: cli $CLIVER, mds $MDSVER, ost $OSTVER
 298                 LIBLUSTRE=no    # bug 17696
 299         fi
 300         if [ "$LIBLUSTRE" != "no" ]; then
 301                 title liblustre
 302                 assert_env MGSNID MOUNT2
 303                 export LIBLUSTRE_MOUNT_POINT=$MOUNT2
 304                 export LIBLUSTRE_MOUNT_RETRY=5
 305                 export LIBLUSTRE_MOUNT_TARGET=$MGSNID:/$FSNAME
 306                 export LIBLUSTRE_TIMEOUT=`lctl get_param -n timeout`
 307                 #export LIBLUSTRE_DEBUG_MASK=`lctl get_param -n debug`
 308                 if [ -x $LIBLUSTRETESTS/sanity ]; then
 309                         mkdir -p $MOUNT2
 310                         if [ "$LIBLUSTRE_EXCEPT" ]; then
 311                                 LIBLUSTRE_OPT="$LIBLUSTRE_OPT \
 312                                         $(echo ' '$LIBLUSTRE_EXCEPT  | sed -re 's/\s+/ -e /g')"
 313                         fi
 314                         echo $LIBLUSTRETESTS/sanity --target=$LIBLUSTRE_MOUNT_TARGET $LIBLUSTRE_OPT
 315                         $LIBLUSTRETESTS/sanity --target=$LIBLUSTRE_MOUNT_TARGET $LIBLUSTRE_OPT
 316                 fi
 317                 $CLEANUP
 318                 #$SETUP
 319                 LIBLUSTRE="done"
 320         fi
 321
 322         [ "$RACER" != "no" ] && [ -n "$CLIENTS" -a "$PDSH" = "no_dsh" ] && log "Remote client with no_dsh" && RACER=no
 323         if [ "$RACER" != "no" ]; then
 324                 title racer
 325                 setup_if_needed
 326                 DURATION=${DURATION:-900}
 327                 [ "$SLOW" = "no" ] && DURATION=300
 328                 RACERCLIENTS=$HOSTNAME
 329                 [ ! -z ${CLIENTS} ] && RACERCLIENTS=$CLIENTS
 330                 log "racer on clients: $RACERCLIENTS DURATION=$DURATION RACERDIRS=$RACERDIRS"
 331                 CLIENTS=${RACERCLIENTS} DURATION=$DURATION bash runracer $RACERDIRS
 332                 $CLEANUP
 333                 $SETUP
 334                 RACER="done"
 335         fi
 336 done
 337
 338 if [ "$SANITY_BENCHMARK" != "no" ]; then
 339         title sanity-benchmark
 340         bash sanity-benchmark.sh
 341         SANITY_BENCHMARK="done"
 342 fi
 343
 344 [ "$REPLAY_SINGLE" != "no" ] && skip_remmds replay-single && REPLAY_SINGLE=no && MSKIPPED=1
 345 if [ "$REPLAY_SINGLE" != "no" ]; then
 346         title replay-single
 347         bash replay-single.sh
 348         REPLAY_SINGLE="done"
 349 fi
 350
 351 [ "$CONF_SANITY" != "no" ] && skip_remmds conf-sanity && CONF_SANITY=no && MSKIPPED=1
 352 [ "$CONF_SANITY" != "no" ] && skip_remost conf-sanity && CONF_SANITY=no && OSKIPPED=1
 353 if [ "$CONF_SANITY" != "no" ]; then
 354         title conf-sanity
 355         bash conf-sanity.sh
 356         CONF_SANITY="done"
 357 fi
 358
 359 [ "$RECOVERY_SMALL" != "no" ] && skip_remmds recover-small && RECOVERY_SMALL=no && MSKIPPED=1
 360 if [ "$RECOVERY_SMALL" != "no" ]; then
 361         title recovery-small
 362         bash recovery-small.sh
 363         RECOVERY_SMALL="done"
 364 fi
 365
 366 [ "$REPLAY_OST_SINGLE" != "no" ] && skip_remost replay-ost-single && REPLAY_OST_SINGLE=no && OSKIPPED=1
 367 if [ "$REPLAY_OST_SINGLE" != "no" ]; then
 368         title replay-ost-single
 369         bash replay-ost-single.sh
 370         REPLAY_OST_SINGLE="done"
 371 fi
 372
 373 [ "$REPLAY_DUAL" != "no" ] && skip_remost replay-dual && REPLAY_DUAL=no && OSKIPPED=1
 374 if [ "$REPLAY_DUAL" != "no" ]; then
 375         title replay-dual
 376         bash replay-dual.sh
 377         REPLAY_DUAL="done"
 378 fi
 379
 380 [ "$REPLAY_VBR" != "no" ] && skip_remmds replay-vbr && REPLAY_VBR=no && MSKIPPED=1
 381 if [ "$REPLAY_VBR" != "no" ]; then
 382         title replay-vbr
 383         bash replay-vbr.sh
 384         REPLAY_VBR="done"
 385 fi
 386
 387 [ "$INSANITY" != "no" ] && skip_remmds insanity && INSANITY=no && MSKIPPED=1
 388 [ "$INSANITY" != "no" ] && skip_remost insanity && INSANITY=no && OSKIPPED=1
 389 if [ "$INSANITY" != "no" ]; then
 390         title insanity
 391         bash insanity.sh -r
 392         INSANITY="done"
 393 fi
 394
 395 [ "$SANITY_QUOTA" != "no" ] && skip_remmds sanity-quota && SANITY_QUOTA=no && MSKIPPED=1
 396 [ "$SANITY_QUOTA" != "no" ] && skip_remost sanity-quota && SANITY_QUOTA=no && OSKIPPED=1
 397 if [ "$SANITY_QUOTA" != "no" ]; then
 398         title sanity-quota
 399         bash sanity-quota.sh
 400         SANITY_QUOTA="done"
 401 fi
 402
 403
 404 [ "$SLOW" = no ] && PERFORMANCE_SANITY="no"
 405 [ -x "$MDSRATE" ] || PERFORMANCE_SANITY="no"
 406 which mpirun > /dev/null 2>&1 || PERFORMANCE_SANITY="no"
 407 if [ "$PERFORMANCE_SANITY" != "no" ]; then
 408         title performance-sanity
 409         bash performance-sanity.sh
 410         PERFORMANCE_SANITY="done"
 411 fi
 412
 413 [ "$LARGE_SCALE" != "no" ] && skip_remmds large-scale && LARGE_SCALE=no && MSKIPPED=1
 414 if [ "$LARGE_SCALE" != "no" ]; then
 415         title large-scale
 416         bash large-scale.sh
 417         LARGE_SCALE="done"
 418 fi
 419
 420 [ "$RECOVERY_MDS_SCALE" != "no" ] && skip_remmds recovery-mds-scale && RECOVERY_MDS_SCALE=no && MSKIPPED=1
 421 [ "$RECOVERY_MDS_SCALE" != "no" ] && skip_remost recovery-mds-scale && RECOVERY_MDS_SCALE=no && OSKIPPED=1
 422 if [ "$RECOVERY_MDS_SCALE" != "no" ]; then
 423         title recovery-mds-scale
 424         bash recovery-mds-scale.sh
 425         RECOVERY_MDS_SCALE="done"
 426 fi
 427
 428 [ "$RECOVERY_DOUBLE_SCALE" != "no" ] && skip_remmds recovery-double-scale && RECOVERY_DOUBLE_SCALE=no && MSKIPPED=1
 429 [ "$RECOVERY_DOUBLE_SCALE" != "no" ] && skip_remost recovery-double-scale && RECOVERY_DOUBLE_SCALE=no && OSKIPPED=1
 430 if [ "$RECOVERY_DOUBLE_SCALE" != "no" ]; then
 431         title recovery-double-scale
 432         bash recovery-double-scale.sh
 433         RECOVERY_DOUBLE_SCALE="done"
 434 fi
 435
 436 [ "$RECOVERY_RANDOM_SCALE" != "no" ] && skip_remmds recovery-random-scale && RECOVERY_RANDOM_SCALE=no && MSKIPPED=1
 437 if [ "$RECOVERY_RANDOM_SCALE" != "no" ]; then
 438         title recovery-random-scale
 439         bash recovery-random-scale.sh
 440         RECOVERY_RANDOM_SCALE="done"
 441 fi
 442
 443 which mpirun > /dev/null 2>&1 || PARALLEL_SCALE="no"
 444 if [ "$PARALLEL_SCALE" != "no" ]; then
 445         title parallel-scale
 446         bash parallel-scale.sh
 447         PARALLEL_SCALE="done"
 448 fi
 449
 450 if [ "$METADATA_UPDATES" != "no" ]; then
 451         title metadata-updates
 452         bash metadata-updates.sh
 453         METADATA_UPDATES="done"
 454 fi
 455
 456 [ "$OST_POOLS" != "no" ] && skip_remmds ost-pools && OST_POOLS=no && MSKIPPED=1
 457 [ "$OST_POOLS" != "no" ] && skip_remost ost-pools && OST_POOLS=no && OSKIPPED=1
 458 if [ "$OST_POOLS" != "no" ]; then
 459         title ost-pools
 460         bash ost-pools.sh
 461         OST_POOLS="done"
 462 fi
 463
 464 if [ "$LNET_SELFTEST" != "no" ]; then
 465         title lnet-selftest
 466         bash lnet-selftest.sh
 467         LNET_SELFTEST="done"
 468 fi
 469
 470 if [ "$OBDFILTER_SURVEY" != "no" ]; then
 471         title obdfilter-survey
 472         bash obdfilter-survey.sh
 473         OBDFILTER_SURVEY="done"
 474 fi
 475
 476 if [ "$MMP" != "no" ]; then
 477         title mmp
 478         bash mmp.sh
 479         MMP="done"
 480 fi
 481
 482 RC=$?
 483 title FINISHED
 484 echo "Finished at `date` in $((`date +%s` - $STARTTIME))s"
 485 echo "Tests ran: $RANTEST"
 486 print_summary
 487 [ "$MSKIPPED" = 1 ] && log "FAIL: remote MDS tests skipped" && RC=1
 488 [ "$OSKIPPED" = 1 ] && log "FAIL: remote OST tests skipped" && RC=1
 489 echo "$0: completed with rc $RC" && exit $RC