From 18bba79f1600351b1de9285f6addece88919e565 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 8 Feb 2011 15:25:06 +0000 Subject: [PATCH] Updates to test scripts: * Add auster test-runner script * Add integration with maloo test DB Change-Id: Ifb4903ffad3fafc989e3261d72d41051b171a90e --- lustre/tests/Makefile.am | 2 +- lustre/tests/auster | 313 +++++++++++++++++++++++++++++++++++++++++ lustre/tests/maloo_upload.sh | 31 ++++ lustre/tests/test-framework.sh | 37 ++++- 4 files changed, 378 insertions(+), 5 deletions(-) create mode 100644 lustre/tests/auster create mode 100644 lustre/tests/maloo_upload.sh diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index a0f19d9..c6cdc50 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -25,7 +25,7 @@ noinst_SCRIPTS += run_dbench.sh run_IOR.sh recovery-double-scale.sh noinst_SCRIPTS += recovery-random-scale.sh parallel-scale.sh metadata-updates.sh noinst_SCRIPTS += lustre-rsync-test.sh ost-pools.sh rpc.sh yaml.sh liblustre.sh noinst_SCRIPTS += lnet-selftest.sh obdfilter-survey.sh mmp.sh -noinst_SCRIPTS += sgpdd-survey.sh +noinst_SCRIPTS += sgpdd-survey.sh maloo_upload.sh auster nobase_noinst_SCRIPTS = cfg/local.sh nobase_noinst_SCRIPTS += acl/make-tree acl/run cfg/ncli.sh nobase_noinst_SCRIPTS += racer/dir_create.sh racer/file_create.sh racer/file_list.sh diff --git a/lustre/tests/auster b/lustre/tests/auster new file mode 100644 index 0000000..ff29a4c --- /dev/null +++ b/lustre/tests/auster @@ -0,0 +1,313 @@ +#!/bin/bash +# +# +# auster - drive lustre tests +# TODO +# 1. --time-limt add per test time limit, kill test if it runs to long +# 2. Read list of tests to run from a file. same syntax as cli, but one test per line +# 3. Run test on remote node +# 4. Use long opts for auster options + +set -e + +export TF_FAIL=/tmp/tf.fail + +usage() { + cat <.sh) + -i N Repeat tests N times (default 1). A new directory + will be created under LOGDIR for each iteration. + -k Don't stop when subtests fail + -R Remount lustre between tests + -r Reformat (during initial configuration if needed) + -s SLOW=yes + -v Verbose mode + -l No logger to Maloo database + -h This help. + +Suite options +These are suite specific options that can be specified after each suite on +the command line. + suite-name [options] + --only LIST Run only specific list of subtests + --except LIST Skip list of subtests + --start-at SUBTEST Start testing from subtest + --stop-at SUBTEST Stop testing at subtest + --time-limit LIMIT Don't allow this suite to run longer + than LIMT seconds. [UNIMPLEMENTED] + +Example usage: +Run all of sanity and all of replay-single except for 70b with SLOW=y using +the default "local" configuration. + + auster -s sanity replay-single --except 70b + +Run test listed in batch-list 5 times using large config. + + auster -f large -r 5 -b batch-list + +EOF + exit +} + +dry_run=false +do_reset=false +verbose=false +repeat_count=1 +logging=true +reformat=false +test_logs_dir=/tmp/test_logs +export SLOW=no +export ${NAME:=local} +while getopts "c:d:nkf:i:rRslhv" opt +do + case "$opt" in + c) CONFIG=$OPTARG;; + d) test_logs_dir=$OPTARG;; + k) export FAIL_ON_ERROR=false;; + n) dry_run=:;; + v) verbose=:;; + i) repeat_count=$OPTARG;; + f) NAME=$OPTARG;; + R) do_reset=:;; + r) reformat=:;; + s) SLOW=yes;; + l) logging=false;; + h|\?) usage;; + esac +done +shift $((OPTIND -1)) + +if [ $logging = true ] ; then + if [ ! -r ~/.maloorc ] ; then + echo "A ~/.maloorc file is required in order to upload results." + echo " * Use -l to run without results upload (you can upload results later)" + echo " or " + echo " * Visit your maloo web interface to download your .maloorc file" + exit -1 + fi +fi + +# +# Various paramters for the tests scripts +# +#: ${SIZE:=$((RAMKB * 2))} +#: ${RSIZE:=512} +#: ${UID:=1000} +#: ${MOUNT=/mnt/lustre} +#: ${MOUNT2:=${MOUNT}2} +#: ${COUNT:=1000} +#: ${TMP:=/tmp} + +reset_lustre() { + if $do_reset; then + stopall + setupall + fi +} + +STARTTIME=`date +%s` + +: ${LUSTRE:=$(cd $(dirname $0)/..; echo $PWD)} +. $LUSTRE/tests/test-framework.sh +init_test_env + +print_summary () { + trap 0 + local form="%-13s %-17s %s\n" + printf "$form" "status" "script" "skipped tests E(xcluded) S(low)" + echo "------------------------------------------------------------------------------------" + echo "Done!" +} + + +setup_if_needed() { + nfs_client_mode && return + auster_cleanup=false + + local MOUNTED=$(mounted_lustre_filesystems) + if $(echo $MOUNTED | grep -w -q $MOUNT); then + check_config_clients $MOUNT + # init_facets_vars + # init_param_vars + return + fi + + echo "Lustre is not mounted, trying to do setup ... " + $reformat && formatall + setupall + + MOUNTED=$(mounted_lustre_filesystems) + if ! $(echo $MOUNTED | grep -w -q $MOUNT); then + echo "Lustre is not mounted after setup! " + exit 1 + fi + auster_cleanup=true +} + +cleanup_if_needed() { + if $auster_cleanup; then + cleanupall + fi +} + +find_script_in_path() { + target=$1 + path=$2 + for dir in $(tr : " " <<< $path); do + if [ -e $dir/$target ]; then + echo $dir/$target + return 0 + fi + if [ -e $dir/$target.sh ]; then + echo $dir/$target.sh + return 0 + fi + done + return 1 +} + +title() { + log "-----============= acceptance-small: "$*" ============----- `date`" +} + +doit() { + if $dry_run; then + printf "Would have run: %s\n" "$*" + return 0 + fi + if $verbose; then + printf "Running: %s\n" "$*" + fi + "$@" +} + + +run_suite() { + suite_name=$1 + suite_script=$2 + only=$3 + title $suite_name + log_test $suite_name + + local start_ts=$(date +%s) + doit bash $suite_script $only + rc=$? + duration=$(($(date +%s) - $start_ts)) + if [ -f $TF_FAIL -o $rc -ne 0 ]; then + status="FAIL" + else + status="PASS" + fi + log_test_status $duration $status + + reset_lustre +} + +run_suite_logged() { + local suite_name=${1%.sh} + local suite=$(echo ${suite_name} | tr "[:lower:]-" "[:upper:]_") + local suite_only=${suite}_ONLY + + suite_script=$(find_script_in_path $suite_name $PATH:$LUSTRE/tests) + + if [[ -z $suite_script ]]; then + echo "Can't find test script for $suite_name" + return 1 + fi + + echo "run_suite $suite_name $suite_script ${!suite_only}" + local log_name=${suite_name}.suite_log.$(hostname).log + run_suite $suite_name $suite_script "${!suite_only}" > $LOGDIR/$log_name 2>&1 +# run_suite $suite_name $suite_script "${!suite_only}" 2>&1 |tee $LOGDIR/$log_name + +} + +# +# Add this to test-framework somewhere. +reset_logging() { + export LOGDIR=${1:-/tmp/test_logs/$(date +%Y-%m-%d)/$(date +%H%M%S)} + unset YAML_LOG + init_logging +} + +split_commas() { + echo "${*//,/ }" +} + +run_suites() { + local n=0 + local argv=("$@") + local basedir=${test_logs_dir}/$(date +%Y-%m-%d)/$(date +%H%M%S) + while ((n < repeat_count)); do + local RC=0 + local logdir=$basedir + ((repeat_count > 1)) && logdir="$logdir/$n" + reset_logging $logdir + set -- "${argv[@]}" + while [[ -n $1 ]]; do + unset ONLY EXCEPT START_AT STOP_AT + local opts="" + local time_limit="" +# echo "argv: $*" + suite=$1 + shift; + while [[ -n $1 ]]; do + case "$1" in + --only) + shift; + export ONLY=$(split_commas $1) + opts+="ONLY=$ONLY ";; + --except) + shift; + export EXCEPT=$(split_commas $1) + opts+="EXCEPT=$EXCEPT ";; + --start-at) + shift; + export START_AT=$1 + opts+="START_AT=$START_AT ";; + --stop-at) + shift; + export STOP_AT=$1 + opts+="STOP_AT=$STOP_AT ";; + --time-limit) + shift; + time_limit=$1;; + *) + break;; + esac + shift + done + echo "running: $suite $opts" + run_suite_logged $suite || RC=$? + echo $suite returned $RC + done + if [ $logging = true ]; then + ./maloo_upload.sh $LOGDIR + fi + n=$((n + 1)) + done +} + +export NAME MOUNT START CLEAN +. ${CONFIG:-$LUSTRE/tests/cfg/$NAME.sh} + +assert_env mds_HOST MDS_MKFS_OPTS +assert_env ost_HOST OST_MKFS_OPTS OSTCOUNT +assert_env FSNAME MOUNT MOUNT2 + +setup_if_needed + +run_suites "$@" +RC=$? + +if [[ $RC -eq 0 ]]; then + cleanup_if_needed +fi + +echo "Finished at `date` in $((`date +%s` - $STARTTIME))s" +echo "$0: completed with rc $RC" && exit $RC diff --git a/lustre/tests/maloo_upload.sh b/lustre/tests/maloo_upload.sh new file mode 100644 index 0000000..dc81ed0 --- /dev/null +++ b/lustre/tests/maloo_upload.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +FILENAME=$1 + +if [ -r ~/.maloorc ] ; then + source ~/.maloorc +else + echo "Error: ~/.maloorc not found. Please obtain this file from the maloo web interface, under 'Upload results'" + exit 1 +fi + +if [ -z $FILENAME ] ; then + echo "Usage: ${0} " + exit 2 +fi + + +if [ ! -r $FILENAME ] ; then + echo "Input file '$FILENAME' not found" + exit 3 +fi + +echo Uploading $FILENAME to $MALOO_URL +if [ -d $FILENAME ] ; then + pushd $FILENAME + tar czf - * | curl -F "user_id=${MALOO_USER_ID}" -F "upload=@-" -F "user_upload_token=${MALOO_UPLOAD_TOKEN}" ${MALOO_URL} > /dev/null + popd +else + curl -F "user_id=${MALOO_USER_ID}" -F "upload=@${FILENAME}" -F "user_upload_token=${MALOO_UPLOAD_TOKEN}" ${MALOO_URL} > /dev/null +fi +echo Complete. diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 7cc95d8..dab56fe2 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -2921,6 +2921,8 @@ skip_env () { skip () { echo log " SKIP: ${TESTSUITE} ${TESTNAME} $@" + [ "$ALWAYS_SKIPPED" ] && \ + skip_logged ${TESTNAME} "$@" || true [ "$TESTSUITELOG" ] && \ echo "${TESTSUITE}: SKIP: $TESTNAME $@" >> $TESTSUITELOG || true } @@ -2936,9 +2938,12 @@ build_test_filter() { log "excepting tests: `echo $EXCEPT $ALWAYS_EXCEPT`" [ "$EXCEPT_SLOW" ] && \ log "skipping tests SLOW=no: `echo $EXCEPT_SLOW`" - for E in $EXCEPT $ALWAYS_EXCEPT; do + for E in $EXCEPT; do eval EXCEPT_${E}=true done + for E in $ALWAYS_EXCEPT; do + eval EXCEPT_ALWAYS_${E}=true + done for E in $EXCEPT_SLOW; do eval EXCEPT_SLOW_${E}=true done @@ -2957,6 +2962,7 @@ basetest() { # print a newline if the last test was skipped export LAST_SKIPPED= +export ALWAYS_SKIPPED= # # Main entry into test-framework. This is called with the name and # description of a test. The name is used to find the function to run @@ -2998,6 +3004,20 @@ run_test() { TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)" return 0 fi + testname=EXCEPT_ALWAYS_$1 + if [ ${!testname}x != x ]; then + LAST_SKIPPED="y" + ALWAYS_SKIPPED="y" + TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1" + return 0 + fi + testname=EXCEPT_ALWAYS_$base + if [ ${!testname}x != x ]; then + LAST_SKIPPED="y" + ALWAYS_SKIPPED="y" + TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1 (base $base)" + return 0 + fi testname=EXCEPT_SLOW_$1 if [ ${!testname}x != x ]; then LAST_SKIPPED="y" @@ -3012,6 +3032,7 @@ run_test() { fi LAST_SKIPPED= + ALWAYS_SKIPPED= run_one_logged $1 "$2" return $? @@ -3151,6 +3172,14 @@ run_one_logged() { return 0 } +# +# Print information of skipped tests to result.yml +# +skip_logged(){ + log_sub_test_begin $1 + log_sub_test_end "SKIP" "0" "0" "\"$2\"" +} + canonical_path() { (cd `dirname $1`; echo $PWD/`basename $1`) } @@ -4377,7 +4406,7 @@ init_logging() { echo "Logging to local directory: $LOGDIR" fi - yml_nodes_file $LOGDIR + yml_nodes_file $LOGDIR >> $YAML_LOG yml_results_file >> $YAML_LOG } @@ -4390,11 +4419,11 @@ log_test_status() { } log_sub_test_begin() { - yml_log_sub_test_begin $@ >> $YAML_LOG + yml_log_sub_test_begin "$@" >> $YAML_LOG } log_sub_test_end() { - yml_log_sub_test_end $@ >> $YAML_LOG + yml_log_sub_test_end "$@" >> $YAML_LOG } run_llverdev() -- 1.8.3.1