From 01d16dadab75e5015c494cfa52a783a87f8bc8bf Mon Sep 17 00:00:00 2001 From: Alex Deiter Date: Thu, 22 Jun 2023 17:28:48 +0400 Subject: [PATCH] LU-14697 tests: change performance-sanity to use mdtest Replace mdsrate by mdtest in performance-sanity.sh Test-Parameters: trivial Test-Parameters: testlist=performance-sanity clientdistro=el7.9 Test-Parameters: testlist=performance-sanity clientdistro=el8.8 Test-Parameters: testlist=performance-sanity clientdistro=el9.2 Test-Parameters: testlist=performance-sanity clientdistro=ubuntu2204 Signed-off-by: Alex Deiter Change-Id: I1a80bab4ccbe085d3ff8d8b332c8e117e14ea9cb Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/51414 Tested-by: Maloo Tested-by: jenkins Reviewed-by: Andreas Dilger Reviewed-by: James Simmons Reviewed-by: Oleg Drokin Reviewed-by: Jian Yu --- lustre/tests/Makefile.am | 5 +- lustre/tests/cfg/local.sh | 1 + lustre/tests/mdsrate-create-large.sh | 152 ----------------------------- lustre/tests/mdsrate-create-small.sh | 176 ---------------------------------- lustre/tests/mdsrate-lookup-10dirs.sh | 159 ------------------------------ lustre/tests/mdsrate-lookup-1dir.sh | 145 ---------------------------- lustre/tests/mdsrate-stat-large.sh | 149 ---------------------------- lustre/tests/mdsrate-stat-small.sh | 149 ---------------------------- lustre/tests/performance-sanity.sh | 67 ++++++------- lustre/tests/test-framework.sh | 145 ++++++++++++++++++++++++++++ 10 files changed, 174 insertions(+), 974 deletions(-) delete mode 100644 lustre/tests/mdsrate-create-large.sh delete mode 100644 lustre/tests/mdsrate-create-small.sh delete mode 100644 lustre/tests/mdsrate-lookup-10dirs.sh delete mode 100644 lustre/tests/mdsrate-lookup-1dir.sh delete mode 100644 lustre/tests/mdsrate-stat-large.sh delete mode 100644 lustre/tests/mdsrate-stat-small.sh diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index 49e2799..e99b9f2 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -20,10 +20,7 @@ noinst_SCRIPTS += insanity.sh oos.sh oos2.sh dne_sanity.sh sanity-compr.sh noinst_SCRIPTS += recovery-small.sh replay-dual.sh sanity-quota.sh noinst_SCRIPTS += replay-ost-single.sh replay-single.sh run-llog.sh sanityn.sh noinst_SCRIPTS += large-scale.sh racer.sh replay-vbr.sh -noinst_SCRIPTS += performance-sanity.sh mdsrate-create-small.sh -noinst_SCRIPTS += mdsrate-create-large.sh mdsrate-lookup-1dir.sh -noinst_SCRIPTS += mdsrate-lookup-10dirs.sh sanity-benchmark.sh -noinst_SCRIPTS += mdsrate-stat-small.sh mdsrate-stat-large.sh +noinst_SCRIPTS += performance-sanity.sh sanity-benchmark.sh noinst_SCRIPTS += lockorder.sh socketclient socketserver runmultiop_bg_pause noinst_SCRIPTS += sanity-sec.sh sanity-selinux.sh recovery-scale-lib.sh noinst_SCRIPTS += recovery-mds-scale.sh recovery-oss-scale.sh diff --git a/lustre/tests/cfg/local.sh b/lustre/tests/cfg/local.sh index 3a966c7..1de9611 100644 --- a/lustre/tests/cfg/local.sh +++ b/lustre/tests/cfg/local.sh @@ -154,6 +154,7 @@ POWER_UP=${POWER_UP:-"powerman --on"} SLOW=${SLOW:-no} FAIL_ON_ERROR=${FAIL_ON_ERROR:-true} +MDTEST=${MDTEST:-$(which mdtest 2>/dev/null || true)} MPIRUN=${MPIRUN:-$(which mpirun 2>/dev/null || true)} MPIRUN_OPTIONS=${MPIRUN_OPTIONS:-"--oversubscribe"} MPI_USER=${MPI_USER:-mpiuser} diff --git a/lustre/tests/mdsrate-create-large.sh b/lustre/tests/mdsrate-create-large.sh deleted file mode 100644 index 9b259a4..0000000 --- a/lustre/tests/mdsrate-create-large.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/bash -# -# This test was used in a set of CMD3 tests (cmd3-4 test). - -LUSTRE=${LUSTRE:-$(dirname $0)/..} -. $LUSTRE/tests/test-framework.sh -init_test_env "$@" - -assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN - -BASEDIR=$MOUNT/mdsrate - -# Requirements -# set NUM_FILES=0 to force TIME_PERIOD work -NUM_FILES=${NUM_FILES:-1000000} -TIME_PERIOD=${TIME_PERIOD:-600} # seconds - -# Local test variables -TESTDIR_SINGLE="${BASEDIR}/single" -TESTDIR_MULTI="${BASEDIR}/multi" - -LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} -CLIENT=$SINGLECLIENT -NODES_TO_USE=${NODES_TO_USE:-$CLIENTS} -NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) - -[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built." - -log "===== $0 ====== " - -check_and_setup_lustre - -MDSRATE_ENABLE_DNE=${MDSRATE_ENABLE_DNE:-false} -if $MDSRATE_ENABLE_DNE; then - test_mkdir $BASEDIR - mdtcount_opt="--mdtcount $MDSCOUNT" -else - mkdir $BASEDIR -fi -if $VERBOSE; then - debug_opt="--debug" -fi -chmod 0777 $BASEDIR -mdsrate_STRIPEPARAMS=${mdsrate_STRIPEPARAMS:-${fs_STRIPEPARAMS:-"-c -1"}} -setstripe_getstripe $BASEDIR $mdsrate_STRIPEPARAMS - -IFree=$(mdsrate_inodes_available) -if [ $IFree -lt $NUM_FILES ]; then - NUM_FILES=$IFree -fi - -generate_machine_file $NODES_TO_USE $MACHINEFILE || - error "can not generate machinefile" - -p="$TMP/$TESTSUITE-$TESTNAME.parameters" -save_lustre_params $(get_facets MDS) mdt.*.enable_remote_dir_gid > $p -do_nodes $(comma_list $(mdts_nodes)) \ - $LCTL set_param mdt.*.enable_remote_dir_gid=-1 - -# Make sure we start with a clean slate -rm -f ${LOG} - -if [ -n "$NOSINGLE" ]; then - echo "NO Test for creates for a single client." -else - # We can use np = $NUM_CLIENTS to speed up the cleanup - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_SINGLE 'f%%d' --ignore - - log "===== $0 ### 1 NODE CREATE ###" - - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD} - --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d' - $mdtcount_opt $debug_opt" - echo "+ ${COMMAND}" - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} | - tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error_noexit "mdsrate create on single client failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \ - $TESTDIR_SINGLE 'f%%d' --ignore - exit 1 - fi - - log "===== $0 ### 1 NODE UNLINK ###" - - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink - --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'" - echo "+ ${COMMAND}" - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} | tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error "mdsrate unlink on a single client failed, aborting" - fi - - rmdir $TESTDIR_SINGLE -fi - -IFree=$(mdsrate_inodes_available) -if [ $IFree -lt $NUM_FILES ]; then - NUM_FILES=$IFree -fi - -[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes -if [ -n "$NOMULTI" ]; then - echo "NO test for create on multiple nodes." -else - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_MULTI 'f%%d' --ignore - - log "===== $0 ### $NUM_CLIENTS NODES CREATE ###" - - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD} - --nfiles $NUM_FILES --dir ${TESTDIR_MULTI} --filefmt 'f%%d'" - echo "+ ${COMMAND}" - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \ - ${COMMAND} | tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error_noexit "mdsrate create on multiple nodes failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \ - $TESTDIR_MULTI 'f%%d' --ignore - exit 1 - fi - - log "===== $0 ### $NUM_CLIENTS NODES UNLINK ###" - - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink - --nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'" - echo "+ ${COMMAND}" - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \ - ${COMMAND} | tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error "mdsrate unlink on multiple nodes failed, aborting" - fi - - rmdir $TESTDIR_MULTI -fi - -restore_lustre_params < $p -rmdir $BASEDIR || true -rm -f $MACHINEFILE -check_and_cleanup_lustre -#rm -f $LOG - -exit 0 diff --git a/lustre/tests/mdsrate-create-small.sh b/lustre/tests/mdsrate-create-small.sh deleted file mode 100644 index 439b1d4..0000000 --- a/lustre/tests/mdsrate-create-small.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/bin/bash -# -# This test was used in a set of CMD3 tests (cmd3-3 test). - -LUSTRE=${LUSTRE:-$(dirname $0)/..} -. $LUSTRE/tests/test-framework.sh -init_test_env "$@" - -assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN - -BASEDIR=$MOUNT/mdsrate - -# Requirements -NUM_FILES=${NUM_FILES:-1000000} -TIME_PERIOD=${TIME_PERIOD:-600} # seconds - -# Local test variables -TESTDIR_SINGLE="${BASEDIR}/single" -TESTDIR_MULTI="${BASEDIR}/multi" - -LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} -CLIENT=$SINGLECLIENT -NODES_TO_USE=${NODES_TO_USE:-$CLIENTS} -NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) -# XXX - this needs to be determined given the number of MDTs and the number -# of clients. -THREADS_PER_CLIENT=3 # threads/client for multi client test -if [ $NUM_CLIENTS -gt 50 ]; then - THREADS_PER_CLIENT=1 -fi - -[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built." - -# Make sure we start with a clean slate -rm -f ${LOG} - -log "===== $0 ====== " - -check_and_setup_lustre - -MDSRATE_ENABLE_DNE=${MDSRATE_ENABLE_DNE:-false} -if $MDSRATE_ENABLE_DNE; then - test_mkdir $BASEDIR - mdtcount_opt="--mdtcount $MDSCOUNT" -else - mkdir $BASEDIR -fi -if $VERBOSE; then - debug_opt="--debug" -fi -chmod 0777 $BASEDIR -mdsrate_STRIPEPARAMS=${mdsrate_STRIPEPARAMS:-${fs_STRIPEPARAMS:-"-i 0 -c 1"}} -setstripe_getstripe $BASEDIR $mdsrate_STRIPEPARAMS - -IFree=$(mdsrate_inodes_available) -if [ $IFree -lt $NUM_FILES ]; then - NUM_FILES=$IFree -fi - -generate_machine_file $NODES_TO_USE $MACHINEFILE || - error "can not generate machinefile" - -p="$TMP/$TESTSUITE-$TESTNAME.parameters" -save_lustre_params $(get_facets MDS) mdt.*.enable_remote_dir_gid > $p -do_nodes $(comma_list $(mdts_nodes)) \ - $LCTL set_param mdt.*.enable_remote_dir_gid=-1 - -if [ -n "$NOSINGLE" ]; then - echo "NO Tests on single client." -else - if [ -n "$NOCREATE" ]; then - echo "NO Test for creates for a single client." - else - # We can use np = $NUM_CLIENTS to speed up the cleanup - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_SINGLE 'f%%d' --ignore - - log "===== $0 ### 1 NODE CREATE ###" - - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD} - --nfiles $NUM_FILES --dir ${TESTDIR_SINGLE} --filefmt 'f%%d' - $mdtcount_opt $debug_opt" - echo "+ ${COMMAND}" - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} | - tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error_noexit "mdsrate create on single client failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \ - $TESTDIR_SINGLE 'f%%d' --ignore - exit 1 - fi - fi - - if [ -n "$NOUNLINK" ]; then - echo "NO Test for unlinks for a single client." - else - log "===== $0 ### 1 NODE UNLINK ###" - - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink - --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'" - echo "+ ${COMMAND}" - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} | - tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error "mdsrate unlinks for a single client failed, aborting" - fi - - rmdir $TESTDIR_SINGLE - fi -fi - -IFree=$(mdsrate_inodes_available) -if [ $IFree -lt $NUM_FILES ]; then - NUM_FILES=$IFree -fi - -if [ -n "$NOMULTI" ]; then - echo "NO tests on multiple nodes." -else - if [ -n "$NOCREATE" ]; then - echo "NO test for create on multiple nodes." - else - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_MULTI 'f%%d' --ignore - - log "===== $0 ### $NUM_CLIENTS NODES CREATE with $THREADS_PER_CLIENT threads per client ###" - - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD} - --nfiles $NUM_FILES --dir ${TESTDIR_MULTI} --filefmt 'f%%d'" - echo "+ ${COMMAND}" - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \ - -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) ${COMMAND} | - tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error_noexit "mdsrate create on multiple nodes failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup $((NUM_CLIENTS * THREADS_PER_CLIENT)) \ - $MACHINEFILE $NUM_FILES \ - $TESTDIR_MULTI 'f%%d' --ignore - exit 1 - fi - fi - - if [ -n "$NOUNLINK" ]; then - echo "NO Test for unlinks multiple nodes." - else - log "===== $0 ### $NUM_CLIENTS NODES UNLINK with $THREADS_PER_CLIENT threads per client ###" - - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink - --nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'" - echo "+ ${COMMAND}" - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \ - -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) ${COMMAND} | - tee ${LOG} - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error "mdsrate unlinks multiple nodes failed, aborting" - fi - - rmdir $TESTDIR_MULTI - fi -fi - -complete_test $SECONDS -restore_lustre_params < $p -rmdir $BASEDIR || true -rm -f $MACHINEFILE -check_and_cleanup_lustre -#rm -f $LOG - -exit 0 diff --git a/lustre/tests/mdsrate-lookup-10dirs.sh b/lustre/tests/mdsrate-lookup-10dirs.sh deleted file mode 100644 index cb63c06..0000000 --- a/lustre/tests/mdsrate-lookup-10dirs.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/bin/bash -# -# This test was used in a set of CMD3 tests (cmd3-5 test). - -# Directory lookup retrieval rate 10 directories 1 million files each -# 6000 random lookups/sec per client node 62,000 random lookups/sec aggregate -# -# In 10 dirs containing 1 million files each the mdsrate Test Program will -# perform lookups for 10 minutes. This test is run from a single node for -# #1 and from all nodes for #2 aggregate test to measure lookup performance. -# TEst performs lookups across all 10 directories. - -LUSTRE=${LUSTRE:-$(dirname $0)/..} -. $LUSTRE/tests/test-framework.sh -init_test_env "$@" - -assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN - -# Do not use name [df][0-9]* to avoid cleanup by rm, bug 18045 -BASEDIR=$MOUNT/mdsrate - -# Requirements -NUM_DIRS=${NUM_DIRS:-10} -NUM_FILES=${NUM_FILES:-1000000} -TIME_PERIOD=${TIME_PERIOD:-600} # seconds - -LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} -CLIENT=$SINGLECLIENT -NODES_TO_USE=${NODES_TO_USE:-$CLIENTS} -NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) - -rm -f $LOG - -[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built." - -log "===== $0 ====== " - -check_and_setup_lustre - -MDSRATE_ENABLE_DNE=${MDSRATE_ENABLE_DNE:-false} -if $MDSRATE_ENABLE_DNE; then - test_mkdir $BASEDIR - mdtcount_opt="--mdtcount $MDSCOUNT" -else - mkdir $BASEDIR -fi -if $VERBOSE; then - debug_opt="--debug" -fi -chmod 0777 $BASEDIR -mdsrate_STRIPEPARAMS=${mdsrate_STRIPEPARAMS:-${fs_STRIPEPARAMS:-"-c 1"}} -setstripe_getstripe $BASEDIR $mdsrate_STRIPEPARAMS - -IFree=$(($(mdsrate_inodes_available) - NUM_DIRS)) -if [ $IFree -lt $((NUM_FILES * NUM_DIRS)) ]; then - NUM_FILES=$((IFree / NUM_DIRS)) -fi - -generate_machine_file $NODES_TO_USE $MACHINEFILE || - error "can not generate machinefile" - -p="$TMP/$TESTSUITE-$TESTNAME.parameters" -save_lustre_params $(get_facets MDS) mdt.*.enable_remote_dir_gid > $p -do_nodes $(comma_list $(mdts_nodes)) \ - $LCTL set_param mdt.*.enable_remote_dir_gid=-1 - -DIRfmt="${BASEDIR}/lookup-%d" - -# -# Unlink the files created in the directories under $BASEDIR. -# FIXME: does it make sense to add the possibility to unlink dirfmt to mdsrate? -# -mdsrate_cleanup_all() { - local i - for i in $(seq 0 $NUM_DIRS); do - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \ - $BASEDIR/lookup-$i 'f%%d' --ignore - done -} - -if [ -n "$NOCREATE" ]; then - echo "NOCREATE=$NOCREATE => no file creation." -else - mdsrate_cleanup_all - - log "===== $0 Test preparation: creating ${NUM_DIRS} dirs with ${NUM_FILES} files." - - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --mknod - --ndirs ${NUM_DIRS} --dirfmt '${DIRfmt}' - --nfiles ${NUM_FILES} --filefmt 'f%%d' - $mdtcount_opt $debug_opt" - - echo "+" ${COMMAND} - # For files creation we can use -np equal to NUM_DIRS - # This is just a test preparation, does not matter how many threads we - # use for files creation; we just should be aware that NUM_DIRS is less - # than or equal to the number of threads np - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_DIRS} \ - ${COMMAND} 2>&1 - - # No lookup if error occurs on file creation, abort. - if [ ${PIPESTATUS[0]} != 0 ]; then - error_noexit "mdsrate file creation failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup_all - exit 1 - fi -fi - -COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --lookup --time ${TIME_PERIOD} ${SEED_OPTION} - --ndirs ${NUM_DIRS} --dirfmt '${DIRfmt}' - --nfiles ${NUM_FILES} --filefmt 'f%%d'" - -# 1 -if [ -n "$NOSINGLE" ]; then - echo "NO Test for lookups on a single client." -else - log "===== $0 ### 1 NODE LOOKUPS ###" - echo "+" ${COMMAND} - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} | - tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error_noexit "mdsrate lookup on single client failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup_all - exit 1 - fi -fi - -# 2 -[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes -if [ -n "$NOMULTI" ]; then - echo "NO test for lookups on multiple nodes." -else - log "===== $0 ### ${NUM_CLIENTS} NODES LOOKUPS ###" - echo "+" ${COMMAND} - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \ - ${COMMAND} | tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error_noexit "mdsrate lookup on multiple nodes failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup_all - exit 1 - fi -fi - -complete_test $SECONDS -restore_lustre_params < $p -mdsrate_cleanup_all -rmdir $BASEDIR || true -rm -f $MACHINEFILE -check_and_cleanup_lustre -#rm -f $LOG - -exit 0 diff --git a/lustre/tests/mdsrate-lookup-1dir.sh b/lustre/tests/mdsrate-lookup-1dir.sh deleted file mode 100644 index 0b9a793..0000000 --- a/lustre/tests/mdsrate-lookup-1dir.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/bash -# -# This test was used in a set of CMD3 tests (cmd3-5 test). - -# Directory lookup retrieval rate single directory 10 million files -# 5900 random lookups/sec per client node 62,000 random lookups/sec aggregate -# -# In a dir containing 10 million non-striped files the mdsrate Test Program will -# perform lookups for 10 minutes. This test can be run from a single node for -# #1 and from all nodes for #2 aggregate test to measure lookup performance. - -LUSTRE=${LUSTRE:-$(dirname $0)/..} -. $LUSTRE/tests/test-framework.sh -init_test_env "$@" - -assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN - -# Do not use name [df][0-9]* to avoid cleanup by rm, bug 18045 -BASEDIR=$MOUNT/mdsrate -TESTDIR=$BASEDIR/lookup - -# Requirements -NUM_FILES=${NUM_FILES:-1000000} -TIME_PERIOD=${TIME_PERIOD:-600} # seconds - -LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} -CLIENT=$SINGLECLIENT -NODES_TO_USE=${NODES_TO_USE:-$CLIENTS} -NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) - -rm -f $LOG - -[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built." - -log "===== $0 ====== " - -check_and_setup_lustre - -MDSRATE_ENABLE_DNE=${MDSRATE_ENABLE_DNE:-false} -if $MDSRATE_ENABLE_DNE; then - test_mkdir $BASEDIR - mdtcount_opt="--mdtcount $MDSCOUNT" -else - mkdir $BASEDIR -fi -if $VERBOSE; then - debug_opt="--debug" -fi -chmod 0777 $BASEDIR -mdsrate_STRIPEPARAMS=${mdsrate_STRIPEPARAMS:-${fs_STRIPEPARAMS:-"-c 1"}} -setstripe_getstripe $BASEDIR $mdsrate_STRIPEPARAMS - -IFree=$(mdsrate_inodes_available) -if [ $IFree -lt $NUM_FILES ]; then - NUM_FILES=$IFree -fi - -generate_machine_file $NODES_TO_USE $MACHINEFILE || - error "can not generate machinefile" - -p="$TMP/$TESTSUITE-$TESTNAME.parameters" -save_lustre_params $(get_facets MDS) mdt.*.enable_remote_dir_gid > $p -do_nodes $(comma_list $(mdts_nodes)) \ - $LCTL set_param mdt.*.enable_remote_dir_gid=-1 - -if [ -n "$NOCREATE" ]; then - echo "NOCREATE=$NOCREATE => no file creation." -else - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' --ignore - - log "===== $0 Test preparation: creating ${NUM_FILES} files." - - NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) - NUM_THREADS=$((NUM_CLIENTS * MDSCOUNT)) - if [ $NUM_CLIENTS -gt 50 ]; then - NUM_THREADS=$NUM_CLIENTS - fi - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir ${TESTDIR} - --nfiles ${NUM_FILES} --filefmt 'f%%d' - $mdtcount_opt $debug_opt" - echo "+" ${COMMAND} - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_THREADS} \ - ${COMMAND} 2>&1 - - # No lockup if error occurs on file creation, abort. - if [ ${PIPESTATUS[0]} != 0 ]; then - error_noexit "mdsrate file creation failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup $NUM_THREADS $MACHINEFILE $NUM_FILES \ - $TESTDIR 'f%%d' --ignore - exit 1 - fi -fi - -COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --lookup --time ${TIME_PERIOD} ${SEED_OPTION} - --dir ${TESTDIR} --nfiles ${NUM_FILES} --filefmt 'f%%d'" - -# 1 -if [ -n "$NOSINGLE" ]; then - echo "NO Test for lookups on a single client." -else - log "===== $0 ### 1 NODE LOOKUPS ###" - echo "+" ${COMMAND} - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} | - tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error_noexit "mdsrate lookup on single client failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \ - $TESTDIR 'f%%d' --ignore - exit 1 - fi -fi - -# 2 -[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes -if [ -n "$NOMULTI" ]; then - echo "NO test for lookups on multiple nodes." -else - log "===== $0 ### ${NUM_CLIENTS} NODES LOOKUPS ###" - echo "+" ${COMMAND} - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \ - ${COMMAND} | tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error_noexit "mdsrate lookup on multiple nodes failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \ - $TESTDIR 'f%%d' --ignore - exit 1 - fi -fi - -complete_test $SECONDS -restore_lustre_params < $p -mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' -rmdir $BASEDIR || true -rm -f $MACHINEFILE -check_and_cleanup_lustre -#rm -f $LOG - -exit 0 diff --git a/lustre/tests/mdsrate-stat-large.sh b/lustre/tests/mdsrate-stat-large.sh deleted file mode 100644 index 5d4fb37..0000000 --- a/lustre/tests/mdsrate-stat-large.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash -# -# This test was used in a set of CMD3 tests (cmd3-8 test). - -# File attribute retrieval rate for large file creation -# 3300 ops/sec/OST for single node 28500 ops/sec/OST aggregate - -# In a dir containing 10 million striped files, the mdsrate Test Program will -# perform directory ordered stat's (readdir) for 10 minutes. This test will be -# run from a single node for #1 and from all nodes for #2 aggregate test to -# measure stat performance. - -LUSTRE=${LUSTRE:-$(dirname $0)/..} -. $LUSTRE/tests/test-framework.sh -init_test_env "$@" - -assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN - -# Do not use name [df][0-9]* to avoid cleanup by rm, bug 18045 -BASEDIR=$MOUNT/mdsrate -TESTDIR=$MOUNT/stat - -# Requirements -NUM_FILES=${NUM_FILES:-1000000} -TIME_PERIOD=${TIME_PERIOD:-600} # seconds - -LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} -CLIENT=$SINGLECLIENT -NODES_TO_USE=${NODES_TO_USE:-$CLIENTS} -NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) - -rm -f $LOG - -[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built." - -log "===== $0 ====== " - -check_and_setup_lustre - -MDSRATE_ENABLE_DNE=${MDSRATE_ENABLE_DNE:-false} -if $MDSRATE_ENABLE_DNE; then - test_mkdir $BASEDIR - mdtcount_opt="--mdtcount $MDSCOUNT" -else - mkdir $BASEDIR -fi -if $VERBOSE; then - debug_opt="--debug" -fi -chmod 0777 $BASEDIR -mdsrate_STRIPEPARAMS=${mdsrate_STRIPEPARAMS:-${fs_STRIPEPARAMS:-"-c -1"}} -setstripe_getstripe $BASEDIR $mdsrate_STRIPEPARAMS - -IFree=$(mdsrate_inodes_available) -if [ $IFree -lt $NUM_FILES ]; then - NUM_FILES=$IFree -fi - -generate_machine_file $NODES_TO_USE $MACHINEFILE || - error "can not generate machinefile" - -p="$TMP/$TESTSUITE-$TESTNAME.parameters" -save_lustre_params $(get_facets MDS) mdt.*.enable_remote_dir_gid > $p -do_nodes $(comma_list $(mdts_nodes)) \ - $LCTL set_param mdt.*.enable_remote_dir_gid=-1 - -if [ -n "$NOCREATE" ]; then - echo "NOCREATE=$NOCREATE => no file creation." -else - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' --ignore - - log "===== $0 Test preparation: creating ${NUM_FILES} files." - - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --dir ${TESTDIR} - --nfiles ${NUM_FILES} --filefmt 'f%%d' - $mdtcount_opt $debug_opt" - echo "+" ${COMMAND} - - NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) - NUM_THREADS=$((NUM_CLIENTS * MDSCOUNT)) - if [ $NUM_CLIENTS -gt 50 ]; then - NUM_THREADS=$NUM_CLIENTS - fi - - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_THREADS} \ - ${COMMAND} 2>&1 - - if [ ${PIPESTATUS[0]} != 0 ]; then - error_noexit "mdsrate file creation failed, aborting" - mdsrate_cleanup $NUM_THREADS $MACHINEFILE $NUM_FILES \ - $TESTDIR 'f%%d' --ignore - exit 1 - fi -fi - -COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --stat --time ${TIME_PERIOD} - --dir ${TESTDIR} --nfiles ${NUM_FILES} --filefmt 'f%%d'" - -# 1 -if [ -n "$NOSINGLE" ]; then - echo "NO Test for stats on a single client." -else - log "===== $0 ### 1 NODE STAT ###" - echo "+" ${COMMAND} - - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} | - tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error_noexit "mdsrate stat on single client failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \ - $TESTDIR 'f%%d' --ignore - exit 1 - fi -fi - -# 2 -[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes -if [ -n "$NOMULTI" ]; then - echo "NO test for stats on multiple nodes." -else - log "===== $0 ### ${NUM_CLIENTS} NODES STAT ###" - echo "+" ${COMMAND} - - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \ - ${COMMAND} | tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error_noexit "mdsrate stat on multiple nodes failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \ - $TESTDIR 'f%%d' --ignore - exit 1 - fi -fi - - -complete_test $SECONDS -restore_lustre_params < $p -mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' -rmdir $BASEDIR || true -rm -f $MACHINEFILE -check_and_cleanup_lustre -#rm -f $LOG - -exit 0 diff --git a/lustre/tests/mdsrate-stat-small.sh b/lustre/tests/mdsrate-stat-small.sh deleted file mode 100644 index 715ab3d..0000000 --- a/lustre/tests/mdsrate-stat-small.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash -# -# This test was used in a set of CMD3 tests (cmd3-7 test). - -# File attribute retrieval rate for small file creation -# 3200 ops/sec for single node 29,000 ops/sec aggregate - -# In a dir containing 10 million non-striped files, the mdsrate Test Program -# will perform directory ordered stat's (readdir) for 10 minutes. This test -# will be run from a single node for #1 and from all nodes for #2 -# aggregate test to measure stat performance. - -LUSTRE=${LUSTRE:-$(dirname $0)/..} -. $LUSTRE/tests/test-framework.sh -init_test_env "$@" - -assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN - -# Do not use name [df][0-9]* to avoid cleanup by rm, bug 18045 -BASEDIR=$MOUNT/mdsrate -TESTDIR=$BASEDIR/stat - -# Requirements -NUM_FILES=${NUM_FILES:-1000000} -TIME_PERIOD=${TIME_PERIOD:-600} # seconds - -LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} -CLIENT=$SINGLECLIENT -NODES_TO_USE=${NODES_TO_USE:-$CLIENTS} -NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) - -rm -f $LOG - -[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built." - -log "===== $0 ====== " - -check_and_setup_lustre - -MDSRATE_ENABLE_DNE=${MDSRATE_ENABLE_DNE:-false} -if $MDSRATE_ENABLE_DNE; then - test_mkdir $BASEDIR - mdtcount_opt="--mdtcount $MDSCOUNT" -else - mkdir $BASEDIR -fi -if $VERBOSE; then - debug_opt="--debug" -fi -chmod 0777 $BASEDIR -mdsrate_STRIPEPARAMS=${mdsrate_STRIPEPARAMS:-${fs_STRIPEPARAMS:-"-i 0 -c 1"}} -setstripe_getstripe $BASEDIR $mdsrate_STRIPEPARAMS - -IFree=$(mdsrate_inodes_available) -if [ $IFree -lt $NUM_FILES ]; then - NUM_FILES=$IFree -fi - -generate_machine_file $NODES_TO_USE $MACHINEFILE || - error "can not generate machinefile" - -p="$TMP/$TESTSUITE-$TESTNAME.parameters" -save_lustre_params $(get_facets MDS) mdt.*.enable_remote_dir_gid > $p -do_nodes $(comma_list $(mdts_nodes)) \ - $LCTL set_param mdt.*.enable_remote_dir_gid=-1 - -if [ -n "$NOCREATE" ]; then - echo "NOCREATE=$NOCREATE => no file creation." -else - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' --ignore - - log "===== $0 Test preparation: creating ${NUM_FILES} files." - - COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir ${TESTDIR} - --nfiles ${NUM_FILES} --filefmt 'f%%d' - $mdtcount_opt $debug_opt" - echo "+" ${COMMAND} - - NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) - NUM_THREADS=$((NUM_CLIENTS * MDSCOUNT)) - if [ $NUM_CLIENTS -gt 50 ]; then - NUM_THREADS=$NUM_CLIENTS - fi - - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_THREADS} \ - ${COMMAND} 2>&1 - - if [ ${PIPESTATUS[0]} != 0 ]; then - error_noexit "mdsrate file creation failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup $NUM_THREADS $MACHINEFILE $NUM_FILES \ - $TESTDIR 'f%%d' --ignore - exit 1 - fi -fi - -COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --stat --time ${TIME_PERIOD} - --dir ${TESTDIR} --nfiles ${NUM_FILES} --filefmt 'f%%d'" - -# 1 -if [ -n "$NOSINGLE" ]; then - echo "NO Test for stats on a single client." -else - log "===== $0 ### 1 NODE STAT ###" - echo "+" ${COMMAND} - - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} | - tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error_noexit "mdsrate stat on single client failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \ - $TESTDIR 'f%%d' --ignore - exit 1 - fi -fi - -# 2 -[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes -if [ -n "$NOMULTI" ]; then - echo "NO test for stats on multiple nodes." -else - log "===== $0 ### ${NUM_CLIENTS} NODES STAT ###" - echo "+" ${COMMAND} - - mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \ - ${COMMAND} | tee ${LOG} - - if [ ${PIPESTATUS[0]} != 0 ]; then - [ -f $LOG ] && sed -e "s/^/log: /" $LOG - error_noexit "mdsrate stat on multiple nodes failed, aborting" - restore_lustre_params < $p - mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \ - $TESTDIR 'f%%d' --ignore - exit 1 - fi -fi - -complete_test $SECONDS -restore_lustre_params < $p -mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' -rmdir $BASEDIR || true -rm -f $MACHINEFILE -check_and_cleanup_lustre -#rm -f $LOG - -exit 0 diff --git a/lustre/tests/performance-sanity.sh b/lustre/tests/performance-sanity.sh index 901c2a6..404835e 100644 --- a/lustre/tests/performance-sanity.sh +++ b/lustre/tests/performance-sanity.sh @@ -1,4 +1,5 @@ #!/bin/bash + set -e TESTNAME=$(basename $0 .sh) @@ -10,63 +11,49 @@ init_test_env "$@" init_logging ALWAYS_EXCEPT="$PERFORMANCE_SANITY_EXCEPT " -always_except LU-16658 6 build_test_filter -[ -x "$MDSRATE" ] || FAIL_ON_ERROR=true error "No mdsrate program. Aborting." -which mpirun > /dev/null 2>&1 || - FAIL_ON_ERROR=true error "No mpirun program. Aborting." +[[ -x "$MPIRUN" ]] || skip_env "no mpirun program found" +[[ -x "$MDTEST" ]] || skip_env "no mdtest program found" + +check_and_setup_lustre get_mpiuser_id $MPI_USER MPI_RUNAS=${MPI_RUNAS:-"runas -u $MPI_USER_UID -g $MPI_USER_GID"} $GSS_KRB5 && refresh_krb5_tgt $MPI_USER_UID $MPI_USER_GID $MPI_RUNAS -# mdsrate-create-small -test_3() { - echo "File creation performance tests for file objects" - bash mdsrate-create-small.sh -} -run_test 3 "small file create/open/delete ======" - -# mdsrate-create-large -test_4() { +test_1() { + echo "Small files creation performance test" # LU-2600/LU-4108 - Decrease load on zfs - [ "$SLOW" = no -a "$mds1_FSTYPE" = zfs ] && + if [[ "$SLOW" == no && "$mds1_FSTYPE" == zfs ]]; then NUM_FILES=10000 - echo "Large file creation performance" - bash mdsrate-create-large.sh -} -run_test 4 "large file create/open/delete" - -# mdsrate-lookup-1dir -test_5() { - echo "Single directory lookup retrieval rate" - bash mdsrate-lookup-1dir.sh + fi + run_mdtest create-small } -run_test 5 "lookup rate 10M file dir ======" +run_test 1 "small files create/open/delete" -# mdsrate-lookup-10dir -test_6() { - echo "Directory lookup retrieval rate 10 directories, 1 million files each" - bash mdsrate-lookup-10dirs.sh +test_2() { + echo "Large files creation performance test" + run_mdtest create-large } -run_test 6 "lookup rate 10M file 10 dir ======" +run_test 2 "large files create/open/delete" -# mdsrate-stat-small -test_7() { - echo "File attribute retrieval rate for small file creation" - bash mdsrate-stat-small.sh +test_3() { + NUM_DIRS=1 + NUM_FILES=200000 + echo "Single directory lookup rate for $NUM_FILES files" + run_mdtest lookup-single } -run_test 7 "getattr small file ======" +run_test 3 "lookup rate 200k files in single directory" -# mdsrate-stat-large -test_8() { - echo "File attribute retrieval rate for large file creation" - bash mdsrate-stat-large.sh +test_4() { + NUM_DIRS=100 + NUM_FILES=200000 + echo "Directory lookup rate $NUM_DIRS directories, $((NUM_FILES/NUM_DIRS)) files each" + run_mdtest lookup-multi } -run_test 8 "getattr large files ======" +run_test 4 "lookup rate 200k files in 100 directories" complete_test $SECONDS check_and_cleanup_lustre -[ -f "$LOG" ] && cat $LOG || true exit_status diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 0ea069d..bae1d6a 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -7742,6 +7742,16 @@ mdsrate_inodes_available () { echo $((min_inodes * 99 / 100)) } +bytes_available () { + echo $(df -P -B 1 "$MOUNT" | awk 'END {print $4}') +} + +mdsrate_bytes_available () { + local bytes=$(bytes_available) + + echo $((bytes * 99 / 100)) +} + # reset stat counters clear_stats() { local paramfile="$1" @@ -7844,6 +7854,141 @@ mdsrate_cleanup () { fi } +run_mdtest () { + local test_type="$1" + local file_size=0 + local num_files=0 + local num_cores=0 + local num_procs=0 + local num_hosts=0 + local free_space=0 + local num_inodes=0 + local num_entries=0 + local num_dirs=0 + local np=0 + local rc=0 + + local mdtest_basedir + local mdtest_actions + local mdtest_options + local stripe_options + local params_file + + case "$test_type" in + create-small) + stripe_options=(-c 1 -i 0) + mdtest_actions=(-F -R) + file_size=1024 + num_files=100000 + ;; + create-large) + stripe_options=(-c -1) + mdtest_actions=(-F -R) + file_size=$((1024 * 1024 * 1024)) + num_files=16 + ;; + lookup-single) + stripe_options=(-c 1) + mdtest_actions=(-C -D -E -k -r) + num_dirs=1 + num_files=100000 + ;; + lookup-multi) + stripe_options=(-c 1) + mdtest_actions=(-C -D -E -k -r) + num_dirs=100 + num_files=1000 + ;; + *) + stripe_options=(-c -1) + mdtest_actions=() + num_files=100000 + ;; + esac + + if [[ -n "$MDTEST_DEBUG" ]]; then + mdtest_options+=(-v -v -v) + fi + + num_dirs=${NUM_DIRS:-$num_dirs} + num_files=${NUM_FILES:-$num_files} + file_size=${FILE_SIZE:-$file_size} + free_space=$(mdsrate_bytes_available) + + if (( file_size * num_files > free_space )); then + file_size=$((free_space / num_files)) + log "change file size to $file_size due to" \ + "number of files $num_files and" \ + "free space limit in $free_space" + fi + + if (( file_size > 0 )); then + log "set file size to $file_size" + mdtest_options+=(-w=$file_size) + fi + + params_file=$TMP/$TESTSUITE-$TESTNAME.parameters + mdtest_basedir=$MOUNT/mdtest + mdtest_options+=(-d=$mdtest_basedir) + + num_cores=$(nproc) + num_hosts=$(get_node_count ${CLIENTS//,/ }) + num_procs=$((num_cores * num_hosts)) + num_inodes=$(mdsrate_inodes_available) + + if (( num_inodes < num_files )); then + log "change the number of files $num_files to the" \ + "number of available inodes $num_inodes" + num_files=$num_inodes + fi + + if (( num_dirs > 1 )); then + num_entries=$((num_files / num_dirs)) + log "split $num_files files to $num_dirs" \ + "with $num_entries files each" + mdtest_options+=(-I=$num_entries) + fi + + generate_machine_file $CLIENTS $MACHINEFILE || + error "can not generate machinefile" + + install -v -d -m 0777 $mdtest_basedir + + setstripe_getstripe $mdtest_basedir ${stripe_options[@]} + + save_lustre_params $(get_facets MDS) \ + mdt.*.enable_remote_dir_gid > $params_file + + do_nodes $(comma_list $(mdts_nodes)) \ + $LCTL set_param mdt.*.enable_remote_dir_gid=-1 + + stack_trap "restore_lustre_params < $params_file" EXIT + + for np in 1 $num_procs; do + num_entries=$((num_files / np )) + + mpi_run $MACHINEFILE_OPTION $MACHINEFILE \ + -np $np -npernode $num_cores $MDTEST \ + ${mdtest_options[@]} -n=$num_entries \ + ${mdtest_actions[@]} 2>&1 | tee -a "$LOG" + + rc=${PIPESTATUS[0]} + + if (( rc != 0 )); then + mpi_run $MACHINEFILE_OPTION $MACHINEFILE \ + -np $np -npernode $num_cores $MDTEST \ + ${mdtest_options[@]} -n=$num_entries \ + -r 2>&1 | tee -a "$LOG" + break + fi + done + + rmdir -v $mdtest_basedir + rm -v $state $MACHINEFILE + + return $rc +} + ######################## convert_facet2label() { -- 1.8.3.1