LU-11912 tests: consume precreated objects in parallel

[fs/lustre-release.git] / lustre / tests / auster

diff --git a/lustre/tests/auster b/lustre/tests/auster
index ed55576..d885bbe 100755 (executable)
--- a/lustre/tests/auster
+++ b/lustre/tests/auster
@@ -1,34 +1,49 @@
 #!/bin/bash
 #!/bin/bash

+# SPDX-License-Identifier: GPL-2.0
+
+#
+# This file is part of Lustre, http://www.lustre.org/
+#
+# lustre/tests/auster
+#
+# Drive lustre tests

 #
 #

+

 #
 #

-# auster - drive lustre tests

 # TODO
 #  1. --time-limt <seconds>  add per test time limit, kill test if it runs to long
 #  2. Read list of tests to run from a file. same syntax as cli, but one test per line
 #  3. Run test on remote node
 # TODO
 #  1. --time-limt <seconds>  add per test time limit, kill test if it runs to long
 #  2. Read list of tests to run from a file. same syntax as cli, but one test per line
 #  3. Run test on remote node

-#  4. Use long opts for auster options
+#

 
 set -e
 
 export TF_FAIL=/tmp/tf.fail
 
 set -e
 
 export TF_FAIL=/tmp/tf.fail

+export TF_SKIP=/tmp/tf.skip

 
 usage() {
 
 usage() {

-    less -F <<EOF 
-Usage ${0##*/} [options]  suite [suite optoins] [suite [suite options]]
+       less -F <<EOF
+Usage ${0##*/} [options]  suite [suite options] [suite [suite options]]

 Run Lustre regression tests suites.
 Run Lustre regression tests suites.

-      -c CONFIG Test environment config file
-      -d LOGDIR Top level directory for logs
-      -f STR    Config name (cfg/<name>.sh)
-      -i N      Repeat tests N times (default 1). A new directory
-                will be created under LOGDIR for each iteration.
-      -k        Don't stop when subtests fail
-      -R        Remount lustre between tests
-      -r        Reformat (during initial configuration if needed)
-      -s        SLOW=yes
-      -v        Verbose mode
-      -l        Send logs to the Maloo database after run 
-                  (can be done later by running maloo_upload.sh)
-      -h        This help.
+      -c, --config CONFIG   Test environment config file
+      -d, --log-dir LOGDIR            Top level directory for logs
+      -D, --full-log-dir FULLLOGDIR   Full directory for logs
+      -f, --cfg-name STR    Config name (cfg/<name>.sh)
+      -g, --group GROUP     Test group file (Overrides tests listed on command line)
+      -S, --suite TESTSUITE           First test suite to run allows for restarts
+      -H, --honor           Honor the EXCEPT and ALWAYS_EXCEPT list when --only is used
+      -i, --repeat N        Repeat tests N times (default 1). A new directory
+                            will be created under LOGDIR for each iteration.
+      -k  --no-stop         Don't stop when subtests fail
+      -R, --remount         Remount lustre between tests
+      -r, --reformat        Reformat (during initial configuration if needed)
+      -s  --slow            SLOW=yes
+      -v, --verbose         Verbose mode
+      -l, --send-logs       Send logs to the Maloo database after run
+                              (can be done later by running maloo_upload.sh)
+      -L, --lang            Script language of test suite. Default: bash
+      -N, --no-setup        No setup. Do not setup Lustre prior to executing test suite.
+      -h, --help            This help.

 
 Suite options
 These are suite specific options that can be specified after each suite on
 
 Suite options
 These are suite specific options that can be specified after each suite on


@@ -47,12 +62,12 @@ the default "local" configuration.
 
   auster -s sanity replay-single --except 70b
 
 
   auster -s sanity replay-single --except 70b
 

-Run test listed in batch-list 5 times  using large config.
+Run all tests in the regression group 5 times using large config.

 
 

-  auster -f large -r 5 -b batch-list
+  auster -f large -g test-groups/regression -i 5

 
 EOF
 
 EOF

-    exit
+       exit

 }
 
 dry_run=false
 }
 
 dry_run=false


@@ -61,43 +76,74 @@ verbose=false
 repeat_count=1
 upload_logs=false
 reformat=false
 repeat_count=1
 upload_logs=false
 reformat=false

-test_logs_dir=/tmp/test_logs
-export SLOW=no
-export ${NAME:=local}
-while getopts "c:d:nkf:i:rRslhv" opt
+script_lang=bash
+test_logs_dir=/tmp/test_logs/$(date +%Y-%m-%d)/$(date +%H%M%S)
+export HONOR_EXCEPT=
+do_setup=true
+export "${SLOW:=no}"
+export "${NAME:=local}"
+
+# Replace long option with corresponding short option
+for arg in "$@"; do
+       shift
+       case "$arg" in
+               --config) set -- "$@" '-c';;
+               --log-dir) set -- "$@" '-d';;
+               --full-log-dir) set -- "$@" '-D';;
+               --group) set -- "$@" '-g';;
+               --suite) set -- "$@" '-S';;
+               --no-stop) set -- "$@" '-k';;
+               --verbose) set -- "$@" '-v';;
+               --honor) set -- "$@" '-H';;
+               --repeat) set -- "$@" '-i';;
+               --cfg-name) set -- "$@" '-f';;
+               --remount) set -- "$@" '-R';;
+               --reformat) set -- "$@" '-r';;
+               --slow) set -- "$@" '-s';;
+               --send-logs) set -- "$@" '-l';;
+               --lang) set -- "$@" '-L';;
+               --no-setup) set -- "$@" '-N';;
+               --help) set -- "$@" '-h';;
+               *) set -- "$@" "$arg";;
+       esac
+done
+
+while getopts "c:d:D:nkf:S:g:Hi:rRslL:Nhv" opt

 do
 do

-    case "$opt" in
-       c) CONFIG=$OPTARG;;
-       d) test_logs_dir=$OPTARG;;
-       k) export FAIL_ON_ERROR=false;;
-        n) dry_run=:;;
-        v) verbose=:;;
-       i) repeat_count=$OPTARG;;
-       f) NAME=$OPTARG;;
-       R) do_reset=:;;
-       r) reformat=:;;
-       s) SLOW=yes;;
-       l) upload_logs=true;;
-        h|\?) usage;;
-    esac
+       case "$opt" in
+               c) export CONFIG=$OPTARG;;
+               d) test_logs_dir=$OPTARG/$(date +%Y-%m-%d)/$(date +%H%M%S);;
+               D) test_logs_dir=$OPTARG;;
+               g) test_group_file=$OPTARG;;
+               S) FIRST_SUITE=$OPTARG;;
+               k) export FAIL_ON_ERROR=false;;
+               n) dry_run=:;;
+               v) verbose=:;;
+               H) export HONOR_EXCEPT="y";;
+               i) repeat_count=$OPTARG;;
+               f) NAME=$OPTARG;;
+               R) do_reset=:;;
+               r) reformat=:;;
+               s) export SLOW=yes;;
+               l) upload_logs=true;;
+               L) script_lang=$OPTARG;;
+               N) do_setup=false;;
+               h|\?) usage;;
+       esac

 done
 done

-shift $((OPTIND -1))

 
 

-#
-# Various paramters for the tests scripts
-# 
-#: ${SIZE:=$((RAMKB * 2))}
-#: ${RSIZE:=512}
-#: ${UID:=1000}
-#: ${MOUNT=/mnt/lustre}
-#: ${MOUNT2:=${MOUNT}2}
-#: ${COUNT:=1000}
-#: ${TMP:=/tmp}
+# If a test_group_file is specified, then ignore rest of command line
+if [[ $test_group_file ]]; then
+    export TEST_GROUP=$(basename $test_group_file)
+    set $(sed 's/#.*$//' $test_group_file)
+else
+    shift $((OPTIND -1))
+fi

 
 reset_lustre() {
     if $do_reset; then
 
 reset_lustre() {
     if $do_reset; then

-       stopall
-       setupall
+        stopall
+        setupall

     fi
 }
 
     fi
 }
 


@@ -117,47 +163,48 @@ print_summary () {
 
 
 setup_if_needed() {
 
 
 setup_if_needed() {

-    nfs_client_mode && return
-    auster_cleanup=false
-
-    local MOUNTED=$(mounted_lustre_filesystems)
-    if $(echo $MOUNTED | grep -w -q $MOUNT); then
-        check_config_clients $MOUNT
-       # init_facets_vars
-       # init_param_vars
-        return
-    fi
+       ! ${do_setup} && return
+       nfs_client_mode && return
+       AUSTER_CLEANUP=false
+
+       local MOUNTED=$(mounted_lustre_filesystems)
+       if $(echo $MOUNTED' ' | grep -w -q $MOUNT' '); then
+               check_config_clients $MOUNT
+               # init_facets_vars
+               # init_param_vars
+               return
+       fi

 
 

-    echo "Lustre is not mounted, trying to do setup ... "
-    $reformat && formatall
-    setupall
+       echo "Lustre is not mounted, trying to do setup ... "
+       $reformat && CLEANUP_DM_DEV=true formatall
+       setupall

 
 

-    MOUNTED=$(mounted_lustre_filesystems)
-    if ! $(echo $MOUNTED | grep -w -q $MOUNT); then
-        echo "Lustre is not mounted after setup! "
-        exit 1
-    fi
-    auster_cleanup=true
+       MOUNTED=$(mounted_lustre_filesystems)
+       if ! $(echo $MOUNTED' ' | grep -w -q $MOUNT' '); then
+               echo "Lustre is not mounted after setup! "
+               exit 1
+       fi
+       AUSTER_CLEANUP=true

 }
 
 cleanup_if_needed() {
 }
 
 cleanup_if_needed() {

-    if $auster_cleanup; then
-       cleanupall
-    fi
+       if $AUSTER_CLEANUP; then
+               cleanupall
+       fi

 }
 
 find_script_in_path() {
     target=$1
     path=$2
     for dir in $(tr : " " <<< $path); do
 }
 
 find_script_in_path() {
     target=$1
     path=$2
     for dir in $(tr : " " <<< $path); do

-      if [ -e $dir/$target ]; then
-         echo $dir/$target
-          return 0
-      fi
-      if [ -e $dir/$target.sh ]; then
-         echo $dir/$target.sh
-          return 0
-      fi
+        if [ -f $dir/$target ]; then
+            echo $dir/$target
+            return 0
+        fi
+        if [ -f $dir/$target.sh ]; then
+            echo $dir/$target.sh
+            return 0
+        fi

     done
     return 1
 }
     done
     return 1
 }


@@ -172,59 +219,65 @@ doit() {
         return 0
     fi
     if $verbose; then
         return 0
     fi
     if $verbose; then

-        printf "Running: %s\n" "$*" 
+        printf "Running: %s\n" "$*"

     fi
     "$@"
 }
 
 
 run_suite() {
     fi
     "$@"
 }
 
 
 run_suite() {

-    suite_name=$1
-    suite_script=$2
-    only=$3
-    title $suite_name
-    log_test $suite_name
-
-    local start_ts=$(date +%s)
-    doit bash $suite_script $only
-    rc=$?
-    duration=$(($(date +%s) - $start_ts))
-    if [ -f $TF_FAIL -o $rc -ne 0 ]; then
-        status="FAIL"
-    else
-        status="PASS"
-    fi
-    log_test_status $duration $status
+       local suite_name=$1
+       local suite_script=$2
+       title $suite_name
+       log_test $suite_name
+
+       rm -f $TF_FAIL
+       touch $TF_SKIP
+       local start_ts=$(date +%s)
+       doit $script_lang $suite_script
+       local rc=$?
+       local duration=$(($(date +%s) - $start_ts))
+
+       local status="PASS"
+       if [[ $rc -ne 0 || -f $TF_FAIL ]]; then
+               status="FAIL"
+       elif [[ -f $TF_SKIP ]]; then
+               status="SKIP"
+       fi
+       log_test_status $duration $status
+       [[ ! -f $TF_SKIP ]] || rm -f $TF_SKIP

 
 

-    reset_lustre
+       reset_lustre
+
+       return $rc

 }
 
 run_suite_logged() {
 }
 
 run_suite_logged() {

-    local suite_name=${1%.sh}
-    local suite=$(echo ${suite_name} | tr "[:lower:]-" "[:upper:]_")
-    local suite_only=${suite}_ONLY
+       local suite_name=${1%.sh}
+       local suite=$(echo ${suite_name} | tr "[:lower:]-" "[:upper:]_")

 
 

-    suite_script=$(find_script_in_path $suite_name $PATH:$LUSTRE/tests)
+       suite_script=$(find_script_in_path $suite_name $LUSTRE/tests)

 
 

-    if [[ -z $suite_script ]]; then
-        echo "Can't find test script for $suite_name"
-        return 1
-    fi
-    
-    echo "run_suite $suite_name $suite_script ${!suite_only}"
-    local log_name=${suite_name}.suite_log.$(hostname).log
-    if $verbose; then
-       run_suite $suite_name $suite_script "${!suite_only}" 2>&1 |tee  $LOGDIR/$log_name
-    else
-       run_suite $suite_name $suite_script "${!suite_only}"  > $LOGDIR/$log_name 2>&1
-    fi
+       if [[ -z $suite_script ]]; then
+               echo "Can't find test script for $suite_name"
+               return 1
+       fi

 
 

+       echo "run_suite $suite_name $suite_script"
+       local log_name=${suite_name}.suite_log.$(hostname -s).log
+       if $verbose; then
+               run_suite $suite_name $suite_script 2>&1 |tee  $LOGDIR/$log_name
+       else
+               run_suite $suite_name $suite_script > $LOGDIR/$log_name 2>&1
+       fi
+
+       return ${PIPESTATUS[0]}

 }
 
 #
 # Add this to test-framework somewhere.
 reset_logging() {
 }
 
 #
 # Add this to test-framework somewhere.
 reset_logging() {

-    export LOGDIR=${1:-${test_logs_dir}/$(date +%Y-%m-%d)/$(date +%H%M%S)}
+    export LOGDIR=$1

     unset YAML_LOG
     init_logging
 }
     unset YAML_LOG
     init_logging
 }


@@ -236,54 +289,65 @@ split_commas() {
 run_suites() {
     local n=0
     local argv=("$@")
 run_suites() {
     local n=0
     local argv=("$@")

-    local basedir=${test_logs_dir}/$(date +%Y-%m-%d)/$(date +%H%M%S)

     while ((n < repeat_count)); do
     while ((n < repeat_count)); do

-       local RC=0
-       local logdir=$basedir
-       ((repeat_count > 1)) && logdir="$logdir/$n"
-       reset_logging $logdir
-       set -- "${argv[@]}"
-       while [[ -n $1 ]]; do
-           unset ONLY EXCEPT START_AT STOP_AT
-           local opts=""
-           local time_limit=""
-#          echo "argv: $*"
-           suite=$1
-           shift;
-           while [[ -n $1 ]]; do
-               case "$1" in
-                   --only)
-                       shift;
-                       export ONLY=$(split_commas $1)
-                       opts+="ONLY=$ONLY ";;
-                   --except)
-                       shift;
-                       export EXCEPT=$(split_commas $1)
-                       opts+="EXCEPT=$EXCEPT ";;
-                   --start-at)
-                       shift;
-                       export START_AT=$1
-                       opts+="START_AT=$START_AT ";;
-                   --stop-at)
-                       shift;
-                       export STOP_AT=$1
-                       opts+="STOP_AT=$STOP_AT ";;
-                   --time-limit)
-                       shift;
-                       time_limit=$1;;
-                   *)
-                       break;;
-               esac
-               shift
-           done
-           echo "running: $suite $opts"
-           run_suite_logged $suite || RC=$?
-           echo $suite returned $RC
-       done
-       if $upload_logs; then
-           $upload_script $LOGDIR
-       fi
-       n=$((n + 1))
+        local RC=0
+        local logdir=${test_logs_dir}
+        local first_suite=$FIRST_SUITE
+        ((repeat_count > 1)) && logdir="$logdir/$n"
+        reset_logging $logdir
+        set -- "${argv[@]}"
+        while [[ -n $1 ]]; do
+            unset ONLY EXCEPT START_AT STOP_AT
+            local opts=""
+            local time_limit=""
+            #echo "argv: $*"
+            suite=$1
+            shift;
+            while [[ -n $1 ]]; do
+                case "$1" in
+                    --only)
+                        shift;
+                        export ONLY=$(split_commas $1)
+                        opts+="ONLY=$ONLY ";;
+                    --suite)
+                        shift;
+                        export SUITE=$(split_commas $1)
+                        opts+="SUITE=$SUITE ";;
+                    --pattern)
+                        shift;
+                        export PATTERN=$(split_commas $1)
+                        opts+="PATTERN=$PATTERN ";;
+                    --except)
+                        shift;
+                        export EXCEPT=$(split_commas $1)
+                        opts+="EXCEPT=$EXCEPT ";;
+                    --start-at)
+                        shift;
+                        export START_AT=$1
+                        opts+="START_AT=$START_AT ";;
+                    --stop-at)
+                        shift;
+                        export STOP_AT=$1
+                        opts+="STOP_AT=$STOP_AT ";;
+                    --time-limit)
+                        shift;
+                        time_limit=$1;;
+                    *)
+                        break;;
+                esac
+                shift
+            done
+            if [ "x"$first_suite == "x" ] || [ $first_suite == $suite ]; then   # If first_suite not set or this is the first suite
+                echo "running: $suite $opts"
+                run_suite_logged $suite || RC=$?
+                unset first_suite
+                echo $suite returned $RC
+            fi
+        done
+        if $upload_logs; then
+            $upload_script $LOGDIR
+        fi
+        n=$((n + 1))

     done
 }
 
     done
 }
 


@@ -291,7 +355,7 @@ if [ $upload_logs = true ] ; then
     upload_script=$(find_script_in_path maloo_upload.sh $PATH:$LUSTRE/tests)
     if [[ -z $upload_script ]]; then
         echo "Can't find maloo_upload.sh script"
     upload_script=$(find_script_in_path maloo_upload.sh $PATH:$LUSTRE/tests)
     if [[ -z $upload_script ]]; then
         echo "Can't find maloo_upload.sh script"

-        exit 1 
+        exit 1

     fi
 
     if [ ! -r ~/.maloorc ] ; then
     fi
 
     if [ ! -r ~/.maloorc ] ; then


@@ -304,10 +368,11 @@ fi
 export NAME MOUNT START CLEAN
 . ${CONFIG:-$LUSTRE/tests/cfg/$NAME.sh}
 
 export NAME MOUNT START CLEAN
 . ${CONFIG:-$LUSTRE/tests/cfg/$NAME.sh}
 

-assert_env mds_HOST MDS_MKFS_OPTS
-assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT
+assert_env mds_HOST
+assert_env ost_HOST OSTCOUNT

 assert_env FSNAME MOUNT MOUNT2
 
 assert_env FSNAME MOUNT MOUNT2
 

+echo "Started at `date`"

 setup_if_needed
 
 run_suites "$@"
 setup_if_needed
 
 run_suites "$@"