From 9a1d68f9b8d9dc7edcbcbc9543450e046c8303b4 Mon Sep 17 00:00:00 2001 From: Andreas Dilger Date: Wed, 15 Jun 2022 23:03:45 -0600 Subject: [PATCH] LU-15913 tests: add rename stress test via racer Add a rename stress test using the racer framework. Use mrename if found, to avoid stat and allow directory rename. Sometimes create and rename files to/from subdirectories. Run e2fsck after every run to confirm filesystem structure. Allow tunable parameters via environment variables so they can be set via Test-Parameters. Parameters can be set on different nodes via variables CLIENT_LCTL_SETPARAM_PARAM, MDS_LCTL_SETPARAM_PARAM, OSS_LCTL_SETPARAM_PARAM. Test-Parameters: trivial testlist=racer env=ONLY=2 Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2 Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2 Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2 Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2 Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2 Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2 Signed-off-by: Andreas Dilger Change-Id: I2ae034b864a5ccb8a59bf7028d22cd67c643f51f Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/47643 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Jian Yu Reviewed-by: Sergey Cheremencev Reviewed-by: Alex Deiter Reviewed-by: Oleg Drokin --- lustre/tests/racer.sh | 78 ++++++++++++++++++++++++++++++++++++--- lustre/tests/racer/file_create.sh | 1 + lustre/tests/racer/file_rename.sh | 11 ++++-- lustre/tests/test-framework.sh | 54 +++++++++++++++++++-------- 4 files changed, 121 insertions(+), 23 deletions(-) diff --git a/lustre/tests/racer.sh b/lustre/tests/racer.sh index d4c39cb..a4248d6 100755 --- a/lustre/tests/racer.sh +++ b/lustre/tests/racer.sh @@ -126,11 +126,11 @@ test_1() { $LFS setstripe $d/racer $RACER_EXTRA_LAYOUT || error "setstripe $RACER_EXTRA_LAYOUT failed" fi - if [ $MDSCOUNT -ge 2 ]; then - for i in $(seq $((MDSCOUNT - 1))); do + if (( MDSCOUNT >= 2 )); then + for ((i = 0; i < MDSCOUNT; i++)); do RDIRS="$RDIRS $d/racer$i" - if [ ! -e $d/racer$i ]; then - $LFS mkdir -i $i $d/racer$i || + if [[ ! -e $d/racer$i ]]; then + $LFS mkdir -i $i $RACER_MKDIR_OPTS $d/racer$i || error "lfs mkdir $i failed" fi if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then @@ -223,6 +223,74 @@ test_1() { } run_test 1 "racer on clients: ${CLIENTS:-$(hostname)} DURATION=$DURATION" +# racer rename stress test +test_2() { + local rrc=0 + local rc=0 + local clients=$CLIENTS + local RDIRS + local i + local racer_done=$TMP/racer_done + + (( MDSCOUNT > 1 )) || skip "need at least 2 MDTs" + + rm -f $racer_done + + for d in ${RACERDIRS}; do + is_mounted $d || continue + mkdir -p $d + + for ((i = 0; i < $MDSCOUNT; i++)); do + RDIRS+=" $d/racer$i" + [[ -d "$d/racer$i" ]] && continue + $LFS mkdir $RACER_MKDIR_OPTS $d/racer$i || + error "mkdir $d/racer$i failed" + if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then + $LFS setstripe $d/racer$i $RACER_EXTRA_LAYOUT || + error "extra $RACER_EXTRA_LAYOUT failed" + fi + done + done + + local rpids="" + local progs="dir_create+dir_remote+file_rename+file_rename+file_create+file_rm" + for rdir in $RDIRS; do + echo "starting on $clients:$rdir with: ${progs//+/ }" + do_nodes $clients "DURATION=$DURATION \ + MDSCOUNT=$MDSCOUNT OSTCOUNT=$OSTCOUNT\ + RACER_MAX_MB=0 \ + RACER_ENABLE_FLR=false \ + RACER_ENABLE_DOM=false \ + RACER_ENABLE_SEL=false \ + RACER_ENABLE_MIGRATION=false \ + RACER_MAX_CLEANUP_WAIT=$RACER_MAX_CLEANUP_WAIT \ + RACER_EXTRA=\\\"$RACER_EXTRA\\\" \ + RACER_EXTRA_LAYOUT=\\\"$RACER_EXTRA_LAYOUT\\\" \ + RACER_PROGS="$progs" \ + NUM_THREADS=$NUM_THREADS \ + MAX_FILES=$MAX_FILES \ + LFS=$LFS \ + LCTL=$LCTL \ + $racer $rdir $NUM_RACER_THREADS" & + pid=$! + rpids="$rpids $pid" + done + + echo racers pids: $rpids + for pid in $rpids; do + wait $pid + rc=$? + echo "pid=$pid rc=$rc" + if [ $rc != 0 ]; then + rrc=$((rrc + 1)) + fi + done + + return $rrc +} +run_test 2 "racer rename: ${CLIENTS:-$(hostname)} DURATION=$DURATION" + complete_test $SECONDS -check_and_cleanup_lustre +FSCK_ALWAYS=${FSCK_ALWAYS:-"yes"} check_and_cleanup_lustre + exit_status diff --git a/lustre/tests/racer/file_create.sh b/lustre/tests/racer/file_create.sh index 6c91df6..9a8d857 100755 --- a/lustre/tests/racer/file_create.sh +++ b/lustre/tests/racer/file_create.sh @@ -30,6 +30,7 @@ echo "layout: ${layout[*]}" while /bin/true; do file=$((RANDOM % MAX)) + (( RANDOM % 100 < 25 )) && file+="/$((RANDOM % MAX))" # $RANDOM is between 0 and 32767, and we want $blockcount in 64kB units blockcount=$((RANDOM * MAX_MB / 32 / 64)) $RACER_ENABLE_OVERSTRIPE && diff --git a/lustre/tests/racer/file_rename.sh b/lustre/tests/racer/file_rename.sh index b5ea2d1..3024ac6 100755 --- a/lustre/tests/racer/file_rename.sh +++ b/lustre/tests/racer/file_rename.sh @@ -4,8 +4,13 @@ trap 'kill $(jobs -p)' EXIT DIR=$1 MAX=$2 +RENAME=${RENAME:-$(which mrename 2> /dev/null)} +RENAME=${RENAME:-"mv"} + while /bin/true ; do - file=$((RANDOM % MAX)) - new_file=$((RANDOM % MAX)) - mv $DIR/$file $DIR/$new_file 2> /dev/null + file=$((RANDOM % MAX)) + ((RANDOM % 100 < 10)) && file+="/$((RANDOM % MAX))" + new_file=$((RANDOM % MAX)) + ((RANDOM % 100 < 10)) && new_file+="/$((RANDOM % MAX))" + $RENAME $DIR/$file $DIR/$new_file 2> /dev/null done diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index b329c40..87db39c 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1927,22 +1927,43 @@ set_default_debug_facet () { set_default_debug_nodes $node "$debug" "$subsys" $debug_size } -set_params_nodes () { - [[ $# -ge 2 ]] || return 0 - +set_params_nodes() { local nodes=$1 shift - do_nodes $nodes $LCTL set_param "$@" + local params="$@" + + [[ -n "$params" ]] || return 0 + + do_nodes $nodes "$LCTL set_param $params" } -set_params_clients () { +set_params_clients() { + (( $# >= 2 )) || return 0 local clients=${1:-$CLIENTS} - local params=${2:-$CLIENT_LCTL_SETPARAM_PARAM} + shift + local params="${@:-$CLIENT_LCTL_SETPARAM_PARAM}" - [[ -n $params ]] || return 0 set_params_nodes $clients $params } +set_params_mdts() { + (( $# >= 2 )) || return 0 + local mdts=${1:-$(comma_list $(mdts_nodes))} + shift + local params="${@:-$MDS_LCTL_SETPARAM_PARAM}" + + set_params_nodes $mdts $params +} + +set_params_osts() { + (( $# >= 2 )) || return 0 + local osts=${1:-$(comma_list $(osts_nodes))} + shift + local params="${@:-$OSS_LCTL_SETPARAM_PARAM}" + + set_params_nodes $osts $params +} + set_hostid () { local hostid=${1:-$(hostid)} @@ -5920,12 +5941,12 @@ check_config_clients () { } check_timeout () { - local mdstimeout=$(do_facet $SINGLEMDS "lctl get_param -n timeout") - local cltimeout=$(lctl get_param -n timeout) - if [ $mdstimeout -ne $TIMEOUT ] || [ $mdstimeout -ne $cltimeout ]; then - error "timeouts are wrong! mds: $mdstimeout, client: $cltimeout, TIMEOUT=$TIMEOUT" - return 1 - fi + local mdstimeout=$(do_facet $SINGLEMDS "lctl get_param -n timeout") + local cltimeout=$(lctl get_param -n timeout) + if [ $mdstimeout -ne $TIMEOUT ] || [ $mdstimeout -ne $cltimeout ]; then + error "timeouts are wrong! mds: $mdstimeout, client: $cltimeout, TIMEOUT=$TIMEOUT" + return 1 + fi } is_mounted () { @@ -6073,8 +6094,7 @@ do_check_and_setup_lustre() { # i.e. if: # 1) remote client has mounted other Lustre fs ? # 2) it has insane env ? - # let's try umount MOUNT2 on all clients and mount it - # again: + # try to umount MOUNT2 on all clients and mount again: if ! check_config_clients $MOUNT2; then cleanup_mount $MOUNT2 restore_mount $MOUNT2 @@ -6138,6 +6158,10 @@ do_check_and_setup_lustre() { set_pools_quota fi + # set tunable parameters passed to test environment + set_params_clients + set_params_mdts + set_params_osts echo "=== $TESTSUITE: finish setup $(date +'%H:%M:%S (%s)') ===" if [[ "$ONLY" == "setup" ]]; then -- 1.8.3.1