}
run_test 25 "replay|resend"
+pids_26=()
+dirs_26=()
+
cleanup_26() {
trap 0
- kill -9 $tar_26_pid
- kill -9 $dbench_26_pid
- killall -9 dbench
+
+ for pid_26 in "${pids_26[@]}"; do
+ if [[ -n "$pid_26" ]]; then
+ kill -0 "$pid_26" && kill "$pid_26" && \
+ wait "$pid_26" || true
+ fi
+ done
+
+ for dir_26 in "${dirs_26[@]}"; do
+ if [[ -n "$dir_26" && -d "$dir_26" ]]; then
+ rm -rf "$dir_26"
+ fi
+ done
}
test_26() {
zconf_mount_clients $clients $MOUNT
local duration=600
- [ "$SLOW" = "no" ] && duration=200
+ [[ "$SLOW" == "no" ]] && duration=200
# set duration to 900 because it takes some time to boot node
- [ "$FAILURE_MODE" = HARD ] && duration=900
+ [[ "$FAILURE_MODE" == HARD ]] && duration=900
local start_ts=$SECONDS
local rc=0
- trap cleanup_26 EXIT
+ stack_trap cleanup_26
+
+ local tar_dir=$DIR/$tdir/run_tar
+
+ dirs_26+=($tar_dir)
(
- local tar_dir=$DIR/$tdir/run_tar
+ stack_trap 'set +e; jobs -p | xargs -r kill; wait; exit' \
+ SIGTERM
+
while true; do
test_mkdir -p -c$MDSCOUNT $tar_dir || break
- if [ $MDSCOUNT -ge 2 ]; then
+
+ if (( MDSCOUNT >= 2 )); then
$LFS setdirstripe -D -c$MDSCOUNT $tar_dir ||
error "set default dirstripe failed"
fi
+
cd $tar_dir || break
- tar cf - /etc | tar xf - || error "tar failed"
+ tar -C / -cf - etc | tar -xf - &
+
+ local tar_pid=$!
+ local tar_rc=0
+
+ wait $tar_pid || tar_rc=$?
+
+ if (( tar_rc > 0 && tar_rc <= 128 )); then
+ error "tar failed with rc $tar_rc"
+ fi
+
cd $DIR/$tdir || break
rm -rf $tar_dir || break
done
- )&
- tar_26_pid=$!
- echo "Started tar $tar_26_pid"
+ ) &
+
+ local tar_26_pid=$!
+
+ echo "Started tar loop with pid $tar_26_pid"
+ pids_26+=($tar_26_pid)
+ local dbench_dir=$DIR2/$tdir/run_dbench
+
+ dirs_26+=($dbench_dir)
(
- local dbench_dir=$DIR2/$tdir/run_dbench
+ stack_trap 'set +e; jobs -p | xargs -r kill; wait; exit' \
+ SIGTERM
+
while true; do
test_mkdir -p -c$MDSCOUNT $dbench_dir || break
- if [ $MDSCOUNT -ge 2 ]; then
+
+ if (( MDSCOUNT >= 2 )); then
$LFS setdirstripe -D -c$MDSCOUNT $dbench_dir ||
error "set default dirstripe failed"
fi
+
cd $dbench_dir || break
- rundbench 1 -D $dbench_dir -t 100 &>/dev/null || break
+ bash rundbench 1 -D $dbench_dir -t 100 &
+
+ local dbench_pid=$!
+ local dbench_rc=0
+
+ wait $dbench_pid || dbench_rc=$?
+
+ if (( dbench_rc > 0 && dbench_rc <= 128 )); then
+ error "dbench failed with rc $dbench_rc"
+ fi
+
cd $DIR/$tdir || break
rm -rf $dbench_dir || break
done
- )&
- dbench_26_pid=$!
- echo "Started dbench $dbench_26_pid"
+ ) &
+
+ local dbench_26_pid=$!
+
+ echo "Started dbench loop with $dbench_26_pid"
+ pids_26+=($dbench_26_pid)
local num_failovers=0
local fail_index=1
- while [ $((SECONDS - start_ts)) -lt $duration ]; do
+
+ while (( (SECONDS - start_ts) < duration )); do
kill -0 $tar_26_pid || error "tar $tar_26_pid missing"
kill -0 $dbench_26_pid || error "dbench $dbench_26_pid missing"
sleep 2
num_failovers=$((num_failovers + 1))
log "$TESTNAME fail mds$fail_index $num_failovers times"
fail mds$fail_index
- if [ $fail_index -ge $MDSCOUNT ]; then
- fail_index=1
- else
+ if (( fail_index < MDSCOUNT )); then
fail_index=$((fail_index + 1))
+ else
+ fail_index=1
fi
done
+
# stop the client loads
kill -0 $tar_26_pid || error "tar $tar_26_pid stopped"
kill -0 $dbench_26_pid || error "dbench $dbench_26_pid stopped"
+
cleanup_26 || true
}
run_test 26 "dbench and tar with mds failover"
#!/bin/bash
-LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
+set -o errexit
+set -o pipefail
+
+MOUNT=${MOUNT:-/mnt/lustre}
+DIR=${DIR:-$MOUNT/$HOSTNAME}
+SCRIPT=$(realpath "$0")
+LUSTRE=${LUSTRE:-$(dirname "$SCRIPT")/..}
. $LUSTRE/tests/test-framework.sh
+CHROOT="no"
+PREFIX="on"
+LIBS71=()
+DBENCHPID=""
+
usage() {
- echo "-C use chroot instead of cd"
- echo "-D DIR - use 'DIR' as work directory"
- echo
- exit;
+ cat <<-EOF
+ Usage: $SCRIPT [-C] [-D DIR] [dbench options]
+ -C use chroot instead of change directory
+ -D DIR use DIR as work directory
+ EOF
+
+ exit 1
}
-PATH=${DBENCH_LIB}:${PATH}
-MOUNT=${MOUNT:-/mnt/lustre}
-DIR=${DIR:-$MOUNT/`hostname`}
+clean_dbench() {
+ local rc=$?
-PREFIX="on"
-while getopts "CD:" opt $*; do
- case $opt in
- D) [ -d $OPTARG ] && DIR=$OPTARG ;;
+ trap '' ERR HUP INT TERM EXIT
+
+ if (( rc > 128 )); then
+ local signal=$((rc - 128))
+ echo "dbench killed by signal $signal"
+ rc=0
+ fi
+
+ echo "stopping dbench $PREFIX $DIR at $(date) with return code $rc"
+
+ if [[ -n "$DBENCHPID" ]]; then
+ if ps -h -p "$DBENCHPID" && kill "$DBENCHPID"; then
+ echo "killed dbench main pid $DBENCHPID"
+ fi
+ fi
+
+ if [[ -d "$DIR" ]]; then
+ echo "clean dbench files $PREFIX $DIR"
+ pushd "$DIR"
+ rm -frv clients client.txt dbench "${LIBS71[@]}"
+ popd
+ fi
+
+ if (( rc == 0 )); then
+ echo "dbench successfully finished"
+ else
+ echo "dbench exit with error $rc"
+ fi
+
+ exit $rc
+}
+
+trap clean_dbench ERR HUP INT TERM EXIT
+
+if [[ -n "$DBENCH_LIB" ]]; then
+ PATH=${DBENCH_LIB}:${PATH}
+fi
+
+while getopts "CD:" OPT; do
+ case "$OPT" in
+ D) test -d "$OPTARG" && DIR="$OPTARG" ;;
C) CHROOT="yes" ;;
\?) usage ;;
esac
done
-mkdir -p $DIR
-TGT=$DIR/client.txt
-CLIENT_PREFIX="${DBENCH_LIB} /usr/share/dbench /usr/local/share /usr/lib/dbench"
-CLIENT_FILE="client.txt client_plain.txt dbench_client"
-if ! which dbench > /dev/null 2>&1 ; then
- [ "$MISSING_DBENCH_OK" ] || { error "$0 : $(hostname) dbench is not installed !" && exit 3; }
- skip_env "$0 : $(hostname) dbench is not installed"
- exit 0
+shift $((OPTIND - 1))
+
+echo "looking for dbench program"
+if ! type -P dbench; then
+ if [[ -z "$MISSING_DBENCH_OK" ]]; then
+ error "dbench is not installed"
+ fi
+
+ skip_env "dbench is not installed"
+
+ exit 0
fi
-CLIENT=""
-for prefix in $CLIENT_PREFIX; do
- for file in $CLIENT_FILE; do
- if [ -s "${prefix}/${file}" ]; then
- CLIENT="${prefix}/${file}";
- break;
- fi
+if [[ ! -d "$DIR" ]]; then
+ echo "creating output directory $DIR"
+ mkdir -pv "$DIR"
+fi
+
+if ! cd "$DIR"; then
+ error "failed to change directory to $DIR"
+fi
+
+if [[ -n "$DBENCH_SRC" && -s "$DBENCH_SRC" ]]; then
+ echo "use dbench client file $DBENCH_SRC"
+ cp -v "$DBENCH_SRC" client.txt
+else
+ CLIENT_DIRS=(/usr/share/dbench /usr/local/share /usr/lib/dbench)
+ CLIENT_FILES=(client.txt client_plain.txt dbench_client)
+
+ if [[ -n "$DBENCH_LIB" ]]; then
+ CLIENT_DIRS=("$DBENCH_LIB" "${CLIENT_DIRS[@]}")
+ fi
+
+ for CLIENT_DIR in "${CLIENT_DIRS[@]}"; do
+ for CLIENT_FILE in "${CLIENT_FILES[@]}"; do
+ CLIENT_PATH="$CLIENT_DIR/$CLIENT_FILE"
+ if [[ -s "$CLIENT_PATH" ]]; then
+ echo "found dbench client file $CLIENT_PATH"
+ cp -v "$CLIENT_PATH" client.txt
+ break 2
+ fi
+ done
done
- [ "x$CLIENT" != "x" ] && break;
-done
+fi
-if [ -n "$DBENCH_SRC" -a -s "$DBENCH_SRC" ]; then
- CLIENT=$DBENCH_SRC
+if [[ ! -s client.txt ]]; then
+ skip_env "no dbench client file found" \
+ "DBENCH_LIB=$DBENCH_LIB DBENCH_SRC=$DBENCH_SRC"
+ exit 0
fi
-[ ! -s "$CLIENT" ] &&
- skip_env "$0: no client file found for dbench on $(hostname): "\
- "DBENCH_LIB=$DBENCH_LIB DBENCH_SRC=$DBENCH_SRC" && exit 0
-
-[ ! -s "$TGT" ] && echo "copying $CLIENT to $TGT" && cp $CLIENT $TGT
-[ ! -s "$TGT" ] && \
- echo "$0 : $(hostname) $TGT file doesn't exist after cp $CLIENT $TGT" && exit 1
-
-if [ "x$CHROOT" == "xyes" ]; then
- echo "copying necessary libs to $DIR"
- cp `which dbench` $DIR
- LIBS71=$(ldd $DIR/dbench |
- sed -e 's/\t*//' -e 's/.*=> //' -e 's/ .*//' -e 's/^\///' |
- grep -v 'linux-vdso.so.1')
- tar -C / -chf - $LIBS71 | tar -C $DIR -xvf -
- if [ "${PIPESTATUS[*]}" != "0 0" ]; then
- echo "can't copy libs $LIBS71 to $DIR"
- rm -f $DIR/dbench
- exit 1
+if [[ "$CHROOT" == "yes" ]]; then
+ echo "copying required dbench files to chroot directory $DIR"
+ cp -v "$(type -P dbench)" dbench
+ LIBS71=($(ldd dbench | awk '
+ {
+ if (NF == 2 && $1 ~ /^[/]/)
+ lib = $1
+ else if (NF == 4 && $3 ~ /^[/]/)
+ lib = $3
+ else
+ next
+ print(substr(lib, 2))
+ }
+ '))
+ if (( ${#LIBS71[@]} == 0 )); then
+ error "failed to get required dbench libraries"
+ fi
+ if ! tar -C / -chf - "${LIBS71[@]}" | tar -xvf -; then
+ error "failed to copy libs ${LIBS71[*]} to $DIR"
fi
RUN="chroot $DIR"
PREFIX="in"
PATH=.:/:$PATH
fi
-shift $((OPTIND - 1))
-
-trap '
-echo kill dbench main pid=$DBENCHPID
-kill $DBENCHPID
-rm -rf dbench $LIBS71 client.txt
-exit 0
-' TERM
-
-# turn off errexit. rundbench will return error code on failure
-trap '' ERR
-set +e
-
-cd $DIR
-echo "running 'dbench $@' $PREFIX $PWD at `date`"
-
-$RUN dbench -c client.txt $@ &
+export PATH
+echo "running 'dbench $*' $PREFIX $DIR at $(date)"
+$RUN dbench -c client.txt "$@" &
DBENCHPID=$!
-echo "dbench PID=$DBENCHPID"
+echo "waiting for dbench pid $DBENCHPID"
wait $DBENCHPID
-RC=$?
-[ $RC -ne 0 ] && killall -9 dbench
-
-rm -rf dbench $LIBS71 client.txt
-exit $RC