LU-15913 tests: add rename stress test via racer

Add a rename stress test using the racer framework.  Use
mrename if found, to avoid stat and allow directory rename.
Sometimes create and rename files to/from subdirectories.

Run e2fsck after every run to confirm filesystem structure.

Allow tunable parameters via environment variables so they
can be set via Test-Parameters.  Parameters can be set on
different nodes via variables CLIENT_LCTL_SETPARAM_PARAM,
MDS_LCTL_SETPARAM_PARAM, OSS_LCTL_SETPARAM_PARAM.

Lustre-change: https://review.whamcloud.com/47643
Lustre-commit: TBD (from 6c63c882741637a246012a81e41289fcf0e2dbbe)

Test-Parameters: trivial testlist=racer env=ONLY=2
Test-Parameters: testlist=racer env=ONLY=2 mdtcount=2
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I2ae034b864a5ccb8a59bf7028d22cd67c643f51f
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53751
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Jian Yu <yujian@whamcloud.com>

diff --git a/lustre/tests/mrename.c b/lustre/tests/mrename.c
index 6cff758..942a0ef 100644 (file)
--- a/lustre/tests/mrename.c
+++ b/lustre/tests/mrename.c
@@ -44,8 +44,8 @@ int main(int argc, char *argv[])
 
        rc = rename(argv[1], argv[2]);
        if (rc)
-               fprintf(stderr, "rename '%s' returned %d: %s\n",
-                       argv[1], rc, strerror(errno));
+               fprintf(stderr,
+                       "rename returned %d: %s\n", rc, strerror(errno));
 
        return rc;
 }

diff --git a/lustre/tests/racer.sh b/lustre/tests/racer.sh
index a4b2c21..a4458b1 100644 (file)
--- a/lustre/tests/racer.sh
+++ b/lustre/tests/racer.sh
@@ -101,14 +101,23 @@ test_1() {
 
                RDIRS="$RDIRS $d/racer"
                mkdir -p $d/racer
-       #       lfs setstripe $d/racer -c -1
-               if [ $MDSCOUNT -ge 2 ]; then
-                       for i in $(seq $((MDSCOUNT - 1))); do
+               if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
+                       $LFS setstripe $d/racer $RACER_EXTRA_LAYOUT ||
+                       error "setstripe $RACER_EXTRA_LAYOUT failed"
+               fi
+               if (( MDSCOUNT >= 2 )); then
+                       for ((i = 0; i < MDSCOUNT; i++)); do
                                RDIRS="$RDIRS $d/racer$i"
-                               if [ ! -e $d/racer$i ]; then
-                                       $LFS mkdir -i $i $d/racer$i ||
+                               if [[ ! -e $d/racer$i ]]; then
+                                       $LFS mkdir -i $i $RACER_MKDIR_OPTS $d/racer$i ||
                                                error "lfs mkdir $i failed"
                                fi
+                               if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
+                                       $LFS setstripe $d/racer$i \
+                                               $RACER_EXTRA_LAYOUT ||
+                                       error "setstripe \
+                                               $RACER_EXTRA_LAYOUT failed"
+                               fi
                        done
                fi
        done
@@ -185,7 +194,74 @@ test_1() {
 }
 run_test 1 "racer on clients: ${CLIENTS:-$(hostname)} DURATION=$DURATION"
 
+# racer rename stress test
+test_2() {
+       local rrc=0
+       local rc=0
+       local clients=$CLIENTS
+       local RDIRS
+       local i
+       local racer_done=$TMP/racer_done
+
+       (( MDSCOUNT > 1 )) || skip "need at least 2 MDTs"
+
+       rm -f $racer_done
+
+       for d in ${RACERDIRS}; do
+               is_mounted $d || continue
+               mkdir -p $d
+
+               for ((i = 0; i < $MDSCOUNT; i++)); do
+                       RDIRS+=" $d/racer$i"
+                       [[  -d "$d/racer$i" ]] && continue
+                       $LFS mkdir $RACER_MKDIR_OPTS $d/racer$i ||
+                               error "mkdir $d/racer$i failed"
+                       if [[ -n "$RACER_EXTRA_LAYOUT" ]]; then
+                               $LFS setstripe $d/racer$i $RACER_EXTRA_LAYOUT ||
+                                       error "extra $RACER_EXTRA_LAYOUT failed"
+                       fi
+               done
+       done
+
+       local rpids=""
+       local progs="dir_create+dir_remote+file_rename+file_rename+file_create+file_rm"
+       for rdir in $RDIRS; do
+               echo "starting on $clients:$rdir with: ${progs//+/ }"
+               do_nodes $clients "DURATION=$DURATION \
+                       MDSCOUNT=$MDSCOUNT OSTCOUNT=$OSTCOUNT\
+                       RACER_MAX_MB=0 \
+                       RACER_ENABLE_FLR=false \
+                       RACER_ENABLE_DOM=false \
+                       RACER_ENABLE_SEL=false \
+                       RACER_ENABLE_MIGRATION=false \
+                       RACER_MAX_CLEANUP_WAIT=$RACER_MAX_CLEANUP_WAIT \
+                       RACER_EXTRA=\\\"$RACER_EXTRA\\\" \
+                       RACER_EXTRA_LAYOUT=\\\"$RACER_EXTRA_LAYOUT\\\" \
+                       RACER_PROGS="$progs" \
+                       NUM_THREADS=$NUM_THREADS \
+                       MAX_FILES=$MAX_FILES \
+                       LFS=$LFS \
+                       LCTL=$LCTL \
+                       $racer $rdir $NUM_RACER_THREADS" &
+               pid=$!
+               rpids="$rpids $pid"
+       done
+
+       echo racers pids: $rpids
+       for pid in $rpids; do
+               wait $pid
+               rc=$?
+               echo "pid=$pid rc=$rc"
+               if [ $rc != 0 ]; then
+                   rrc=$((rrc + 1))
+               fi
+       done
+
+       return $rrc
+}
+run_test 2 "racer rename: ${CLIENTS:-$(hostname)} DURATION=$DURATION"
+
 complete_test $SECONDS
 do_facet mds1 $LCTL set_param debug="-quota"
-check_and_cleanup_lustre
+FSCK_ALWAYS=${FSCK_ALWAYS:-"yes"} check_and_cleanup_lustre
 exit_status

diff --git a/lustre/tests/racer/file_create.sh b/lustre/tests/racer/file_create.sh
index 43aad3c..c673633 100755 (executable)
--- a/lustre/tests/racer/file_create.sh
+++ b/lustre/tests/racer/file_create.sh
@@ -26,6 +26,7 @@ echo "layout: ${layout[*]}"
 
 while /bin/true; do
        file=$((RANDOM % MAX))
+       (( RANDOM % 100 < 25 )) && file+="/$((RANDOM % MAX))"
        # $RANDOM is between 0 and 32767, and we want $blockcount in 64kB units
        blockcount=$((RANDOM * MAX_MB / 32 / 64))
        stripecount=$((RANDOM % (OSTCOUNT + 1)))

diff --git a/lustre/tests/racer/file_rename.sh b/lustre/tests/racer/file_rename.sh
index b5ea2d1..3024ac6 100755 (executable)
--- a/lustre/tests/racer/file_rename.sh
+++ b/lustre/tests/racer/file_rename.sh
@@ -4,8 +4,13 @@ trap 'kill $(jobs -p)' EXIT
 DIR=$1
 MAX=$2
 
+RENAME=${RENAME:-$(which mrename 2> /dev/null)}
+RENAME=${RENAME:-"mv"}
+
 while /bin/true ; do
-    file=$((RANDOM % MAX))
-    new_file=$((RANDOM % MAX))
-    mv $DIR/$file $DIR/$new_file 2> /dev/null
+       file=$((RANDOM % MAX))
+       ((RANDOM % 100 < 10)) && file+="/$((RANDOM % MAX))"
+       new_file=$((RANDOM % MAX))
+       ((RANDOM % 100 < 10)) && new_file+="/$((RANDOM % MAX))"
+       $RENAME $DIR/$file $DIR/$new_file 2> /dev/null
 done

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 1dd8506..6bb202c 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -1724,29 +1724,50 @@ set_default_debug_facet () {
        set_default_debug_nodes $node "$debug" "$subsys" $debug_size
 }
 
-set_params_nodes () {
-       [[ $# -ge 2 ]] || return 0
-
+set_params_nodes() {
        local nodes=$1
        shift
-       do_nodes $nodes $LCTL set_param $@
+       local params="$@"
+
+       [[ -n "$params" ]] || return 0
+
+       do_nodes $nodes "$LCTL set_param $params"
 }
 
-set_params_clients () {
+set_params_clients() {
+       (( $# >= 2 )) || return 0
        local clients=${1:-$CLIENTS}
-       local params=${2:-$CLIENT_LCTL_SETPARAM_PARAM}
+       shift
+       local params="${@:-$CLIENT_LCTL_SETPARAM_PARAM}"
 
-       [[ -n $params ]] || return 0
        set_params_nodes $clients $params
 }
 
+set_params_mdts() {
+       (( $# >= 2 )) || return 0
+       local mdts=${1:-$(comma_list $(mdts_nodes))}
+       shift
+       local params="${@:-$MDS_LCTL_SETPARAM_PARAM}"
+
+       set_params_nodes $mdts $params
+}
+
+set_params_osts() {
+       (( $# >= 2 )) || return 0
+       local osts=${1:-$(comma_list $(osts_nodes))}
+       shift
+       local params="${@:-$OSS_LCTL_SETPARAM_PARAM}"
+
+       set_params_nodes $osts $params
+}
+
 set_hostid () {
-    local hostid=${1:-$(hostid)}
+       local hostid=${1:-$(hostid)}
 
-    if [ ! -s /etc/hostid ]; then
-       printf $(echo -n $hostid |
-           sed 's/\(..\)\(..\)\(..\)\(..\)/\\x\4\\x\3\\x\2\\x\1/') >/etc/hostid
-    fi
+       if [ ! -s /etc/hostid ]; then
+               printf $(echo -n $hostid |
+               sed 's/\(..\)\(..\)\(..\)\(..\)/\\x\4\\x\3\\x\2\\x\1/') >/etc/hostid
+       fi
 }
 
 # Facet functions
@@ -5646,12 +5667,12 @@ check_config_clients () {
 }
 
 check_timeout () {
-    local mdstimeout=$(do_facet $SINGLEMDS "lctl get_param -n timeout")
-    local cltimeout=$(lctl get_param -n timeout)
-    if [ $mdstimeout -ne $TIMEOUT ] || [ $mdstimeout -ne $cltimeout ]; then
-        error "timeouts are wrong! mds: $mdstimeout, client: $cltimeout, TIMEOUT=$TIMEOUT"
-        return 1
-    fi
+       local mdstimeout=$(do_facet $SINGLEMDS "lctl get_param -n timeout")
+       local cltimeout=$(lctl get_param -n timeout)
+       if [ $mdstimeout -ne $TIMEOUT ] || [ $mdstimeout -ne $cltimeout ]; then
+               error "timeouts are wrong! mds: $mdstimeout, client: $cltimeout, TIMEOUT=$TIMEOUT"
+               return 1
+       fi
 }
 
 is_mounted () {
@@ -5771,6 +5792,10 @@ do_check_and_setup_lustre() {
                set_flavor_all $SEC
        fi
 
+       # set tunable parameters passed to test environment
+       set_params_clients
+       set_params_mdts
+       set_params_osts
        echo "=== $TESTSUITE: finish setup $(date +'%H:%M:%S (%s)') ==="
 
        if [[ "$ONLY" == "setup" ]]; then