LU-15913 tests: add rename stress test via racer

Add a rename stress test using the racer framework.  Use
mrename if found, to avoid stat and allow directory rename.
Sometimes create and rename files to/from subdirectories.

Run e2fsck after every run to confirm filesystem structure.

Allow tunable parameters via environment variables so they
can be set via Test-Parameters.  Parameters can be set on
different nodes via variables CLIENT_LCTL_SETPARAM_PARAM,
MDS_LCTL_SETPARAM_PARAM, OSS_LCTL_SETPARAM_PARAM.

Test-Parameters: trivial testlist=racer env=ONLY=2
Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2
Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2
Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2
Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2
Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2
Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I2ae034b864a5ccb8a59bf7028d22cd67c643f51f
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/47643
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Sergey Cheremencev <scherementsev@ddn.com>
Reviewed-by: Alex Deiter
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/racer.sh b/lustre/tests/racer.sh
index d4c39cb..a4248d6 100755 (executable)
--- a/lustre/tests/racer.sh
+++ b/lustre/tests/racer.sh
@@ -126,11 +126,11 @@ test_1() {
                        $LFS setstripe $d/racer $RACER_EXTRA_LAYOUT ||
                        error "setstripe $RACER_EXTRA_LAYOUT failed"
                fi
-               if [ $MDSCOUNT -ge 2 ]; then
-                       for i in $(seq $((MDSCOUNT - 1))); do
+               if (( MDSCOUNT >= 2 )); then
+                       for ((i = 0; i < MDSCOUNT; i++)); do
                                RDIRS="$RDIRS $d/racer$i"
-                               if [ ! -e $d/racer$i ]; then
-                                       $LFS mkdir -i $i $d/racer$i ||
+                               if [[ ! -e $d/racer$i ]]; then
+                                       $LFS mkdir -i $i $RACER_MKDIR_OPTS $d/racer$i ||
                                                error "lfs mkdir $i failed"
                                fi
                                if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
@@ -223,6 +223,74 @@ test_1() {
 }
 run_test 1 "racer on clients: ${CLIENTS:-$(hostname)} DURATION=$DURATION"
 
+# racer rename stress test
+test_2() {
+       local rrc=0
+       local rc=0
+       local clients=$CLIENTS
+       local RDIRS
+       local i
+       local racer_done=$TMP/racer_done
+
+       (( MDSCOUNT > 1 )) || skip "need at least 2 MDTs"
+
+       rm -f $racer_done
+
+       for d in ${RACERDIRS}; do
+               is_mounted $d || continue
+               mkdir -p $d
+
+               for ((i = 0; i < $MDSCOUNT; i++)); do
+                       RDIRS+=" $d/racer$i"
+                       [[  -d "$d/racer$i" ]] && continue
+                       $LFS mkdir $RACER_MKDIR_OPTS $d/racer$i ||
+                               error "mkdir $d/racer$i failed"
+                       if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
+                               $LFS setstripe $d/racer$i $RACER_EXTRA_LAYOUT ||
+                                       error "extra $RACER_EXTRA_LAYOUT failed"
+                       fi
+               done
+       done
+
+       local rpids=""
+       local progs="dir_create+dir_remote+file_rename+file_rename+file_create+file_rm"
+       for rdir in $RDIRS; do
+               echo "starting on $clients:$rdir with: ${progs//+/ }"
+               do_nodes $clients "DURATION=$DURATION \
+                       MDSCOUNT=$MDSCOUNT OSTCOUNT=$OSTCOUNT\
+                       RACER_MAX_MB=0 \
+                       RACER_ENABLE_FLR=false \
+                       RACER_ENABLE_DOM=false \
+                       RACER_ENABLE_SEL=false \
+                       RACER_ENABLE_MIGRATION=false \
+                       RACER_MAX_CLEANUP_WAIT=$RACER_MAX_CLEANUP_WAIT \
+                       RACER_EXTRA=\\\"$RACER_EXTRA\\\" \
+                       RACER_EXTRA_LAYOUT=\\\"$RACER_EXTRA_LAYOUT\\\" \
+                       RACER_PROGS="$progs" \
+                       NUM_THREADS=$NUM_THREADS \
+                       MAX_FILES=$MAX_FILES \
+                       LFS=$LFS \
+                       LCTL=$LCTL \
+                       $racer $rdir $NUM_RACER_THREADS" &
+               pid=$!
+               rpids="$rpids $pid"
+       done
+
+       echo racers pids: $rpids
+       for pid in $rpids; do
+               wait $pid
+               rc=$?
+               echo "pid=$pid rc=$rc"
+               if [ $rc != 0 ]; then
+                   rrc=$((rrc + 1))
+               fi
+       done
+
+       return $rrc
+}
+run_test 2 "racer rename: ${CLIENTS:-$(hostname)} DURATION=$DURATION"
+
 complete_test $SECONDS
-check_and_cleanup_lustre
+FSCK_ALWAYS=${FSCK_ALWAYS:-"yes"} check_and_cleanup_lustre
+
 exit_status

diff --git a/lustre/tests/racer/file_create.sh b/lustre/tests/racer/file_create.sh
index 6c91df6..9a8d857 100755 (executable)
--- a/lustre/tests/racer/file_create.sh
+++ b/lustre/tests/racer/file_create.sh
@@ -30,6 +30,7 @@ echo "layout: ${layout[*]}"
 
 while /bin/true; do
        file=$((RANDOM % MAX))
+       (( RANDOM % 100 < 25 )) && file+="/$((RANDOM % MAX))"
        # $RANDOM is between 0 and 32767, and we want $blockcount in 64kB units
        blockcount=$((RANDOM * MAX_MB / 32 / 64))
        $RACER_ENABLE_OVERSTRIPE &&

diff --git a/lustre/tests/racer/file_rename.sh b/lustre/tests/racer/file_rename.sh
index b5ea2d1..3024ac6 100755 (executable)
--- a/lustre/tests/racer/file_rename.sh
+++ b/lustre/tests/racer/file_rename.sh
@@ -4,8 +4,13 @@ trap 'kill $(jobs -p)' EXIT
 DIR=$1
 MAX=$2
 
+RENAME=${RENAME:-$(which mrename 2> /dev/null)}
+RENAME=${RENAME:-"mv"}
+
 while /bin/true ; do
-    file=$((RANDOM % MAX))
-    new_file=$((RANDOM % MAX))
-    mv $DIR/$file $DIR/$new_file 2> /dev/null
+       file=$((RANDOM % MAX))
+       ((RANDOM % 100 < 10)) && file+="/$((RANDOM % MAX))"
+       new_file=$((RANDOM % MAX))
+       ((RANDOM % 100 < 10)) && new_file+="/$((RANDOM % MAX))"
+       $RENAME $DIR/$file $DIR/$new_file 2> /dev/null
 done

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index b329c40..87db39c 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -1927,22 +1927,43 @@ set_default_debug_facet () {
        set_default_debug_nodes $node "$debug" "$subsys" $debug_size
 }
 
-set_params_nodes () {
-       [[ $# -ge 2 ]] || return 0
-
+set_params_nodes() {
        local nodes=$1
        shift
-       do_nodes $nodes $LCTL set_param "$@"
+       local params="$@"
+
+       [[ -n "$params" ]] || return 0
+
+       do_nodes $nodes "$LCTL set_param $params"
 }
 
-set_params_clients () {
+set_params_clients() {
+       (( $# >= 2 )) || return 0
        local clients=${1:-$CLIENTS}
-       local params=${2:-$CLIENT_LCTL_SETPARAM_PARAM}
+       shift
+       local params="${@:-$CLIENT_LCTL_SETPARAM_PARAM}"
 
-       [[ -n $params ]] || return 0
        set_params_nodes $clients $params
 }
 
+set_params_mdts() {
+       (( $# >= 2 )) || return 0
+       local mdts=${1:-$(comma_list $(mdts_nodes))}
+       shift
+       local params="${@:-$MDS_LCTL_SETPARAM_PARAM}"
+
+       set_params_nodes $mdts $params
+}
+
+set_params_osts() {
+       (( $# >= 2 )) || return 0
+       local osts=${1:-$(comma_list $(osts_nodes))}
+       shift
+       local params="${@:-$OSS_LCTL_SETPARAM_PARAM}"
+
+       set_params_nodes $osts $params
+}
+
 set_hostid () {
        local hostid=${1:-$(hostid)}
 
@@ -5920,12 +5941,12 @@ check_config_clients () {
 }
 
 check_timeout () {
-    local mdstimeout=$(do_facet $SINGLEMDS "lctl get_param -n timeout")
-    local cltimeout=$(lctl get_param -n timeout)
-    if [ $mdstimeout -ne $TIMEOUT ] || [ $mdstimeout -ne $cltimeout ]; then
-        error "timeouts are wrong! mds: $mdstimeout, client: $cltimeout, TIMEOUT=$TIMEOUT"
-        return 1
-    fi
+       local mdstimeout=$(do_facet $SINGLEMDS "lctl get_param -n timeout")
+       local cltimeout=$(lctl get_param -n timeout)
+       if [ $mdstimeout -ne $TIMEOUT ] || [ $mdstimeout -ne $cltimeout ]; then
+               error "timeouts are wrong! mds: $mdstimeout, client: $cltimeout, TIMEOUT=$TIMEOUT"
+               return 1
+       fi
 }
 
 is_mounted () {
@@ -6073,8 +6094,7 @@ do_check_and_setup_lustre() {
                        # i.e. if:
                        # 1) remote client has mounted other Lustre fs ?
                        # 2) it has insane env ?
-                       # let's try umount MOUNT2 on all clients and mount it
-                       # again:
+                       # try to umount MOUNT2 on all clients and mount again:
                        if ! check_config_clients $MOUNT2; then
                                cleanup_mount $MOUNT2
                                restore_mount $MOUNT2
@@ -6138,6 +6158,10 @@ do_check_and_setup_lustre() {
                set_pools_quota
        fi
 
+       # set tunable parameters passed to test environment
+       set_params_clients
+       set_params_mdts
+       set_params_osts
        echo "=== $TESTSUITE: finish setup $(date +'%H:%M:%S (%s)') ==="
 
        if [[ "$ONLY" == "setup" ]]; then