From 01fb7bda971ee9d5dcd3950f40668131f306b8c5 Mon Sep 17 00:00:00 2001 From: Alex Deiter Date: Wed, 9 Nov 2022 21:06:37 +0400 Subject: [PATCH] LU-15495 tests: fixed dbench test * Using awk to get list shared libraries * Fixed shellcheck warnings Test-Parameters: trivial testlist=sanity \ clientdistro=el7.9 clientarch=x86_64 \ env=SLOW=yes,ONLY=71 Test-Parameters: trivial testlist=sanity \ clientdistro=el8.6 clientarch=x86_64 \ env=SLOW=yes,ONLY=71 Test-Parameters: trivial testlist=sanity \ clientdistro=el8.6 clientarch=aarch64 \ env=SLOW=yes,ONLY=71 Test-Parameters: trivial testlist=sanity \ clientdistro=el8.6 clientarch=ppc64le \ env=SLOW=yes,ONLY=71 Test-Parameters: trivial testlist=sanity \ clientdistro=el9.0 clientarch=x86_64 \ env=SLOW=yes,ONLY=71 Test-Parameters: trivial testlist=sanity \ clientdistro=sles12sp5 clientarch=x86_64 \ env=SLOW=yes,ONLY=71 Test-Parameters: trivial testlist=sanity \ clientdistro=sles15sp4 clientarch=x86_64 \ env=SLOW=yes,ONLY=71 Test-Parameters: trivial testlist=sanity \ clientdistro=ubuntu2004 clientarch=x86_64 \ env=SLOW=yes,ONLY=71 Test-Parameters: trivial testlist=sanity \ clientdistro=ubuntu2204 clientarch=x86_64 \ env=SLOW=yes,ONLY=71 Test-Parameters: trivial env=SLOW=yes,ONLY=26 testlist=replay-dual Test-Parameters: trivial env=SLOW=yes,ONLY=70b testlist=replay-single Test-Parameters: trivial env=SLOW=yes,ONLY=8 testlist=sanity-pfl Test-Parameters: trivial env=SLOW=yes,ENABLE_QUOTA=yes,ONLY=8 \ testlist=sanity-quota Signed-off-by: Alex Deiter Change-Id: Ic28bd67dcfb5ff24e65e33278ac867409a2c1cc6 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49088 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Jian Yu Reviewed-by: Oleg Drokin --- lustre/tests/replay-dual.sh | 101 +++++++++++++++++----- lustre/tests/rundbench | 203 ++++++++++++++++++++++++++++---------------- 2 files changed, 209 insertions(+), 95 deletions(-) diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh index 45c28a4..81a0c12 100755 --- a/lustre/tests/replay-dual.sh +++ b/lustre/tests/replay-dual.sh @@ -936,11 +936,24 @@ test_25() { } run_test 25 "replay|resend" +pids_26=() +dirs_26=() + cleanup_26() { trap 0 - kill -9 $tar_26_pid - kill -9 $dbench_26_pid - killall -9 dbench + + for pid_26 in "${pids_26[@]}"; do + if [[ -n "$pid_26" ]]; then + kill -0 "$pid_26" && kill "$pid_26" && \ + wait "$pid_26" || true + fi + done + + for dir_26 in "${dirs_26[@]}"; do + if [[ -n "$dir_26" && -d "$dir_26" ]]; then + rm -rf "$dir_26" + fi + done } test_26() { @@ -949,51 +962,93 @@ test_26() { zconf_mount_clients $clients $MOUNT local duration=600 - [ "$SLOW" = "no" ] && duration=200 + [[ "$SLOW" == "no" ]] && duration=200 # set duration to 900 because it takes some time to boot node - [ "$FAILURE_MODE" = HARD ] && duration=900 + [[ "$FAILURE_MODE" == HARD ]] && duration=900 local start_ts=$SECONDS local rc=0 - trap cleanup_26 EXIT + stack_trap cleanup_26 + + local tar_dir=$DIR/$tdir/run_tar + + dirs_26+=($tar_dir) ( - local tar_dir=$DIR/$tdir/run_tar + stack_trap 'set +e; jobs -p | xargs -r kill; wait; exit' \ + SIGTERM + while true; do test_mkdir -p -c$MDSCOUNT $tar_dir || break - if [ $MDSCOUNT -ge 2 ]; then + + if (( MDSCOUNT >= 2 )); then $LFS setdirstripe -D -c$MDSCOUNT $tar_dir || error "set default dirstripe failed" fi + cd $tar_dir || break - tar cf - /etc | tar xf - || error "tar failed" + tar -C / -cf - etc | tar -xf - & + + local tar_pid=$! + local tar_rc=0 + + wait $tar_pid || tar_rc=$? + + if (( tar_rc > 0 && tar_rc <= 128 )); then + error "tar failed with rc $tar_rc" + fi + cd $DIR/$tdir || break rm -rf $tar_dir || break done - )& - tar_26_pid=$! - echo "Started tar $tar_26_pid" + ) & + + local tar_26_pid=$! + + echo "Started tar loop with pid $tar_26_pid" + pids_26+=($tar_26_pid) + local dbench_dir=$DIR2/$tdir/run_dbench + + dirs_26+=($dbench_dir) ( - local dbench_dir=$DIR2/$tdir/run_dbench + stack_trap 'set +e; jobs -p | xargs -r kill; wait; exit' \ + SIGTERM + while true; do test_mkdir -p -c$MDSCOUNT $dbench_dir || break - if [ $MDSCOUNT -ge 2 ]; then + + if (( MDSCOUNT >= 2 )); then $LFS setdirstripe -D -c$MDSCOUNT $dbench_dir || error "set default dirstripe failed" fi + cd $dbench_dir || break - rundbench 1 -D $dbench_dir -t 100 &>/dev/null || break + bash rundbench 1 -D $dbench_dir -t 100 & + + local dbench_pid=$! + local dbench_rc=0 + + wait $dbench_pid || dbench_rc=$? + + if (( dbench_rc > 0 && dbench_rc <= 128 )); then + error "dbench failed with rc $dbench_rc" + fi + cd $DIR/$tdir || break rm -rf $dbench_dir || break done - )& - dbench_26_pid=$! - echo "Started dbench $dbench_26_pid" + ) & + + local dbench_26_pid=$! + + echo "Started dbench loop with $dbench_26_pid" + pids_26+=($dbench_26_pid) local num_failovers=0 local fail_index=1 - while [ $((SECONDS - start_ts)) -lt $duration ]; do + + while (( (SECONDS - start_ts) < duration )); do kill -0 $tar_26_pid || error "tar $tar_26_pid missing" kill -0 $dbench_26_pid || error "dbench $dbench_26_pid missing" sleep 2 @@ -1003,15 +1058,17 @@ test_26() { num_failovers=$((num_failovers + 1)) log "$TESTNAME fail mds$fail_index $num_failovers times" fail mds$fail_index - if [ $fail_index -ge $MDSCOUNT ]; then - fail_index=1 - else + if (( fail_index < MDSCOUNT )); then fail_index=$((fail_index + 1)) + else + fail_index=1 fi done + # stop the client loads kill -0 $tar_26_pid || error "tar $tar_26_pid stopped" kill -0 $dbench_26_pid || error "dbench $dbench_26_pid stopped" + cleanup_26 || true } run_test 26 "dbench and tar with mds failover" diff --git a/lustre/tests/rundbench b/lustre/tests/rundbench index 11f6191..d366712 100755 --- a/lustre/tests/rundbench +++ b/lustre/tests/rundbench @@ -1,100 +1,157 @@ #!/bin/bash -LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} +set -o errexit +set -o pipefail + +MOUNT=${MOUNT:-/mnt/lustre} +DIR=${DIR:-$MOUNT/$HOSTNAME} +SCRIPT=$(realpath "$0") +LUSTRE=${LUSTRE:-$(dirname "$SCRIPT")/..} . $LUSTRE/tests/test-framework.sh +CHROOT="no" +PREFIX="on" +LIBS71=() +DBENCHPID="" + usage() { - echo "-C use chroot instead of cd" - echo "-D DIR - use 'DIR' as work directory" - echo - exit; + cat <<-EOF + Usage: $SCRIPT [-C] [-D DIR] [dbench options] + -C use chroot instead of change directory + -D DIR use DIR as work directory + EOF + + exit 1 } -PATH=${DBENCH_LIB}:${PATH} -MOUNT=${MOUNT:-/mnt/lustre} -DIR=${DIR:-$MOUNT/`hostname`} +clean_dbench() { + local rc=$? -PREFIX="on" -while getopts "CD:" opt $*; do - case $opt in - D) [ -d $OPTARG ] && DIR=$OPTARG ;; + trap '' ERR HUP INT TERM EXIT + + if (( rc > 128 )); then + local signal=$((rc - 128)) + echo "dbench killed by signal $signal" + rc=0 + fi + + echo "stopping dbench $PREFIX $DIR at $(date) with return code $rc" + + if [[ -n "$DBENCHPID" ]]; then + if ps -h -p "$DBENCHPID" && kill "$DBENCHPID"; then + echo "killed dbench main pid $DBENCHPID" + fi + fi + + if [[ -d "$DIR" ]]; then + echo "clean dbench files $PREFIX $DIR" + pushd "$DIR" + rm -frv clients client.txt dbench "${LIBS71[@]}" + popd + fi + + if (( rc == 0 )); then + echo "dbench successfully finished" + else + echo "dbench exit with error $rc" + fi + + exit $rc +} + +trap clean_dbench ERR HUP INT TERM EXIT + +if [[ -n "$DBENCH_LIB" ]]; then + PATH=${DBENCH_LIB}:${PATH} +fi + +while getopts "CD:" OPT; do + case "$OPT" in + D) test -d "$OPTARG" && DIR="$OPTARG" ;; C) CHROOT="yes" ;; \?) usage ;; esac done -mkdir -p $DIR -TGT=$DIR/client.txt -CLIENT_PREFIX="${DBENCH_LIB} /usr/share/dbench /usr/local/share /usr/lib/dbench" -CLIENT_FILE="client.txt client_plain.txt dbench_client" -if ! which dbench > /dev/null 2>&1 ; then - [ "$MISSING_DBENCH_OK" ] || { error "$0 : $(hostname) dbench is not installed !" && exit 3; } - skip_env "$0 : $(hostname) dbench is not installed" - exit 0 +shift $((OPTIND - 1)) + +echo "looking for dbench program" +if ! type -P dbench; then + if [[ -z "$MISSING_DBENCH_OK" ]]; then + error "dbench is not installed" + fi + + skip_env "dbench is not installed" + + exit 0 fi -CLIENT="" -for prefix in $CLIENT_PREFIX; do - for file in $CLIENT_FILE; do - if [ -s "${prefix}/${file}" ]; then - CLIENT="${prefix}/${file}"; - break; - fi +if [[ ! -d "$DIR" ]]; then + echo "creating output directory $DIR" + mkdir -pv "$DIR" +fi + +if ! cd "$DIR"; then + error "failed to change directory to $DIR" +fi + +if [[ -n "$DBENCH_SRC" && -s "$DBENCH_SRC" ]]; then + echo "use dbench client file $DBENCH_SRC" + cp -v "$DBENCH_SRC" client.txt +else + CLIENT_DIRS=(/usr/share/dbench /usr/local/share /usr/lib/dbench) + CLIENT_FILES=(client.txt client_plain.txt dbench_client) + + if [[ -n "$DBENCH_LIB" ]]; then + CLIENT_DIRS=("$DBENCH_LIB" "${CLIENT_DIRS[@]}") + fi + + for CLIENT_DIR in "${CLIENT_DIRS[@]}"; do + for CLIENT_FILE in "${CLIENT_FILES[@]}"; do + CLIENT_PATH="$CLIENT_DIR/$CLIENT_FILE" + if [[ -s "$CLIENT_PATH" ]]; then + echo "found dbench client file $CLIENT_PATH" + cp -v "$CLIENT_PATH" client.txt + break 2 + fi + done done - [ "x$CLIENT" != "x" ] && break; -done +fi -if [ -n "$DBENCH_SRC" -a -s "$DBENCH_SRC" ]; then - CLIENT=$DBENCH_SRC +if [[ ! -s client.txt ]]; then + skip_env "no dbench client file found" \ + "DBENCH_LIB=$DBENCH_LIB DBENCH_SRC=$DBENCH_SRC" + exit 0 fi -[ ! -s "$CLIENT" ] && - skip_env "$0: no client file found for dbench on $(hostname): "\ - "DBENCH_LIB=$DBENCH_LIB DBENCH_SRC=$DBENCH_SRC" && exit 0 - -[ ! -s "$TGT" ] && echo "copying $CLIENT to $TGT" && cp $CLIENT $TGT -[ ! -s "$TGT" ] && \ - echo "$0 : $(hostname) $TGT file doesn't exist after cp $CLIENT $TGT" && exit 1 - -if [ "x$CHROOT" == "xyes" ]; then - echo "copying necessary libs to $DIR" - cp `which dbench` $DIR - LIBS71=$(ldd $DIR/dbench | - sed -e 's/\t*//' -e 's/.*=> //' -e 's/ .*//' -e 's/^\///' | - grep -v 'linux-vdso.so.1') - tar -C / -chf - $LIBS71 | tar -C $DIR -xvf - - if [ "${PIPESTATUS[*]}" != "0 0" ]; then - echo "can't copy libs $LIBS71 to $DIR" - rm -f $DIR/dbench - exit 1 +if [[ "$CHROOT" == "yes" ]]; then + echo "copying required dbench files to chroot directory $DIR" + cp -v "$(type -P dbench)" dbench + LIBS71=($(ldd dbench | awk ' + { + if (NF == 2 && $1 ~ /^[/]/) + lib = $1 + else if (NF == 4 && $3 ~ /^[/]/) + lib = $3 + else + next + print(substr(lib, 2)) + } + ')) + if (( ${#LIBS71[@]} == 0 )); then + error "failed to get required dbench libraries" + fi + if ! tar -C / -chf - "${LIBS71[@]}" | tar -xvf -; then + error "failed to copy libs ${LIBS71[*]} to $DIR" fi RUN="chroot $DIR" PREFIX="in" PATH=.:/:$PATH fi -shift $((OPTIND - 1)) - -trap ' -echo kill dbench main pid=$DBENCHPID -kill $DBENCHPID -rm -rf dbench $LIBS71 client.txt -exit 0 -' TERM - -# turn off errexit. rundbench will return error code on failure -trap '' ERR -set +e - -cd $DIR -echo "running 'dbench $@' $PREFIX $PWD at `date`" - -$RUN dbench -c client.txt $@ & +export PATH +echo "running 'dbench $*' $PREFIX $DIR at $(date)" +$RUN dbench -c client.txt "$@" & DBENCHPID=$! -echo "dbench PID=$DBENCHPID" +echo "waiting for dbench pid $DBENCHPID" wait $DBENCHPID -RC=$? -[ $RC -ne 0 ] && killall -9 dbench - -rm -rf dbench $LIBS71 client.txt -exit $RC -- 1.8.3.1