X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=515c4d180f397aab8cad35dd395d8a67d40c1ce3;hp=7cfe4fc654099aa5e96a54668b40897be1ab85ad;hb=13b269ab77daca78fdfc374986d4cc34c7e66309;hpb=9ddf386035767a96b54e21559f3ea0be126dc8cd diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 7cfe4fc..515c4d1 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -123,14 +123,61 @@ init_test_env() { export TEST_FAILED=false export FAIL_ON_SKIP_ENV=${FAIL_ON_SKIP_ENV:-false} - export MKE2FS=${MKE2FS:-mke2fs} - export DEBUGFS=${DEBUGFS:-debugfs} - export TUNE2FS=${TUNE2FS:-tune2fs} - export E2LABEL=${E2LABEL:-e2label} - export DUMPE2FS=${DUMPE2FS:-dumpe2fs} - export E2FSCK=${E2FSCK:-e2fsck} - export LFSCK_BIN=${LFSCK_BIN:-lfsck} + export MKE2FS=$MKE2FS + if [ -z "$MKE2FS" ]; then + if which mkfs.ldiskfs >/dev/null 2>&1; then + export MKE2FS=mkfs.ldiskfs + else + export MKE2FS=mke2fs + fi + fi + export DEBUGFS=$DEBUGFS + if [ -z "$DEBUGFS" ]; then + if which debugfs.ldiskfs >/dev/null 2>&1; then + export DEBUGFS=debugfs.ldiskfs + else + export DEBUGFS=debugfs + fi + fi + + export TUNE2FS=$TUNE2FS + if [ -z "$TUNE2FS" ]; then + if which tunefs.ldiskfs >/dev/null 2>&1; then + export TUNE2FS=tunefs.ldiskfs + else + export TUNE2FS=tune2fs + fi + fi + + export E2LABEL=$E2LABEL + if [ -z "$E2LABEL" ]; then + if which label.ldiskfs >/dev/null 2>&1; then + export E2LABEL=label.ldiskfs + else + export E2LABEL=e2label + fi + fi + + export DUMPE2FS=$DUMPE2FS + if [ -z "$DUMPE2FS" ]; then + if which dumpfs.ldiskfs >/dev/null 2>&1; then + export DUMPE2FS=dumpfs.ldiskfs + else + export DUMPE2FS=dumpe2fs + fi + fi + + export E2FSCK=$E2FSCK + if [ -z "$E2FSCK" ]; then + if which fsck.ldiskfs >/dev/null 2>&1; then + export E2FSCK=fsck.ldiskfs + else + export E2FSCK=e2fsck + fi + fi + + export LFSCK_BIN=${LFSCK_BIN:-lfsck} export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite export FSCK_MAX_ERR=4 # File system errors left uncorrected @@ -190,11 +237,11 @@ init_test_env() { fi fi export LL_DECODE_FILTER_FID=${LL_DECODE_FILTER_FID:-"$LUSTRE/utils/ll_decode_filter_fid"} - [ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID=$(which ll_decode_filter_fid) + [ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID="ll_decode_filter_fid" export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"} - [ ! -f "$MKFS" ] && export MKFS=$(which mkfs.lustre) + [ ! -f "$MKFS" ] && export MKFS="mkfs.lustre" export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"} - [ ! -f "$TUNEFS" ] && export TUNEFS=$(which tunefs.lustre) + [ ! -f "$TUNEFS" ] && export TUNEFS="tunefs.lustre" export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} " export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod} [ ! -f "$LUSTRE_RMMOD" ] && @@ -235,15 +282,16 @@ init_test_env() { IDENTITY_UPCALL=false ;; esac + export LOAD_MODULES_REMOTE=${LOAD_MODULES_REMOTE:-false} # Paths on remote nodes, if different export RLUSTRE=${RLUSTRE:-$LUSTRE} export RPWD=${RPWD:-$PWD} export I_MOUNTED=${I_MOUNTED:-"no"} - if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mds.ko -a \ - ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mds.ko -a \ - ! -f `dirname $0`/../mds/mds.ko ]; then + if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \ + ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \ + ! -f `dirname $0`/../mdt/mdt.ko ]; then export CLIENTMODSONLY=yes fi @@ -323,20 +371,24 @@ load_module() { optvar="MODOPTS_$(basename $module | tr a-z A-Z)" eval set -- \$$optvar if [ $# -eq 0 -a -n "$MODPROBECONF" ]; then - # Nothing in $MODOPTS_; try modprobe.conf - set -- $(grep -P "^options\\s+${BASE}" $MODPROBECONF) - # Get rid of "options $module" - (($# > 0)) && shift 2 - - # Ensure we have accept=all for lnet - if [ $(basename $module) = lnet ]; then - # OK, this is a bit wordy... - local arg accept_all_present=false - for arg in "$@"; do - [ "$arg" = accept=all ] && accept_all_present=true - done - $accept_all_present || set -- "$@" accept=all - fi + # Nothing in $MODOPTS_; try modprobe.conf + local opt + opt=$(awk -v var="^options $BASE" '$0 ~ var \ + {gsub("'"options $BASE"'",""); print}' $MODPROBECONF) + set -- $(echo -n $opt) + + # Ensure we have accept=all for lnet + if [ $(basename $module) = lnet ]; then + # OK, this is a bit wordy... + local arg accept_all_present=false + + for arg in "$@"; do + [ "$arg" = accept=all ] && \ + accept_all_present=true + done + $accept_all_present || set -- "$@" accept=all + fi + export $optvar="$*" fi fi @@ -375,22 +427,6 @@ llite_lloop_enabled() { } load_modules_local() { - if [ "$USE_OFD" == yes ]; then - if module_loaded obdfilter; then - if ! $LUSTRE_RMMOD ldiskfs; then - echo "$HOSTNAME may still be using obdfilter.ko" - return 1 - fi - fi - else - if module_loaded ofd; then - if ! $LUSTRE_RMMOD ldiskfs; then - echo "$HOSTNAME may still be using ofd.ko" - return 1 - fi - fi - fi - if [ -n "$MODPROBE" ]; then # use modprobe echo "Using modprobe to load modules" @@ -415,7 +451,6 @@ load_modules_local() { fi fi - local ncpts=0 # if there is only one CPU core, libcfs can only create one partition # if there is more than 4 CPU cores, libcfs should create multiple CPU # partitions. So we just force libcfs to create 2 partitions for @@ -423,12 +458,12 @@ load_modules_local() { if [ $ncpus -le 4 ] && [ $ncpus -gt 1 ]; then # force to enable multiple CPU partitions echo "Force libcfs to create 2 CPU partitions" - ncpts=2 + MODOPTS_LIBCFS="cpu_npartitions=2 $MODOPTS_LIBCFS" else echo "libcfs will create CPU partition based on online CPUs" fi - load_module ../libcfs/libcfs/libcfs cpu_npartitions=$ncpts + load_module ../libcfs/libcfs/libcfs [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG" [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }" @@ -454,10 +489,10 @@ load_modules_local() { grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; } [ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then + modprobe zfs load_module osd-zfs/osd_zfs fi load_module mgs/mgs - load_module mds/mds load_module mdd/mdd if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then # @@ -471,13 +506,11 @@ load_modules_local() { load_module osd-ldiskfs/osd_ldiskfs fi load_module mdt/mdt - load_module cmm/cmm load_module ost/ost - if [ "$USE_OFD" == yes ]; then - load_module ofd/ofd - else - load_module obdfilter/obdfilter - fi + load_module lod/lod + load_module osp/osp + load_module ofd/ofd + load_module osp/osp fi @@ -501,10 +534,12 @@ load_modules () { # bug 19124 # load modules on remote nodes optionally # lustre-tests have to be installed on these nodes - if $LOAD_MODULES_REMOTE ; then + if $LOAD_MODULES_REMOTE; then local list=$(comma_list $(remote_nodes_list)) - echo loading modules on $list - do_rpc_nodes $list load_modules_local + if [ -n "$list" ]; then + echo "loading modules on: '$list'" + do_rpc_nodes "$list" load_modules_local + fi fi } @@ -526,12 +561,12 @@ unload_modules() { $LUSTRE_RMMOD ldiskfs || return 2 - if $LOAD_MODULES_REMOTE ; then + if $LOAD_MODULES_REMOTE; then local list=$(comma_list $(remote_nodes_list)) - if [ ! -z $list ]; then - echo unloading modules on $list - do_rpc_nodes $list $LUSTRE_RMMOD ldiskfs - do_rpc_nodes $list check_mem_leak + if [ -n "$list" ]; then + echo "unloading modules on: '$list'" + do_rpc_nodes "$list" $LUSTRE_RMMOD ldiskfs + do_rpc_nodes "$list" check_mem_leak fi fi @@ -662,7 +697,7 @@ cleanup_gss() { facet_type() { local facet=$1 - echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9]\+//' | + echo -n $facet | sed -e 's/^fs[0-9]\+//' -e 's/[0-9_]\+//' | tr '[:lower:]' '[:upper:]' } @@ -809,15 +844,15 @@ set_default_debug () { } set_default_debug_nodes () { - local nodes=$1 + local nodes="$1" - if [[ ,$nodes, = *,$HOSTNAME,* ]]; then - nodes=$(exclude_items_from_list "$nodes" "$HOSTNAME") - set_default_debug - fi + if [[ ,$nodes, = *,$HOSTNAME,* ]]; then + nodes=$(exclude_items_from_list "$nodes" "$HOSTNAME") + set_default_debug + fi - [[ -n $nodes ]] && do_rpc_nodes $nodes set_default_debug \ - \\\"$PTLDEBUG\\\" \\\"$SUBSYSTEM\\\" $DEBUG_SIZE || true + do_rpc_nodes "$nodes" set_default_debug \ + \\\"$PTLDEBUG\\\" \\\"$SUBSYSTEM\\\" $DEBUG_SIZE || true } set_default_debug_facet () { @@ -981,7 +1016,7 @@ stop() { # save quota version (both administrative and operational quotas) # add an additional parameter if mountpoint is ever different from $MOUNT # -# XXX This function is kept for interoperability with old server (< 2.3.51), +# XXX This function is kept for interoperability with old server (< 2.3.50), # it should be removed whenever we drop the interoperability for such # server. quota_save_version() { @@ -1005,19 +1040,20 @@ quota_save_version() { # client could mount several lustre # -# XXX This function is kept for interoperability with old server (< 2.3.51), +# XXX This function is kept for interoperability with old server (< 2.3.50), # it should be removed whenever we drop the interoperability for such # server. -quota_type () { - local fsname=${1:-$FSNAME} - local rc=0 - do_facet mgs lctl get_param mdd.${fsname}-MDT*.quota_type || rc=$? - do_nodes $(comma_list $(osts_nodes)) \ - lctl get_param obdfilter.${fsname}-OST*.quota_type || rc=$? - return $rc +quota_type() { + local fsname=${1:-$FSNAME} + local rc=0 + do_facet $SINGLEMDS lctl get_param mdd.${fsname}-MDT*.quota_type || + rc=$? + do_nodes $(comma_list $(osts_nodes)) \ + lctl get_param obdfilter.${fsname}-OST*.quota_type || rc=$? + return $rc } -# XXX This function is kept for interoperability with old server (< 2.3.51), +# XXX This function is kept for interoperability with old server (< 2.3.50), # it should be removed whenever we drop the interoperability for such # server. restore_quota_old() { @@ -1030,7 +1066,7 @@ restore_quota_old() { quota_save_version $old_QUOTA_TYPE } -# XXX This function is kept for interoperability with old server (< 2.3.51), +# XXX This function is kept for interoperability with old server (< 2.3.50), # it should be removed whenever we drop the interoperability for such # server. setup_quota_old(){ @@ -1056,11 +1092,11 @@ setup_quota_old(){ local quota_usrs=$QUOTA_USERS # get_filesystem_size - local disksz=$(lfs df $mntpt | grep "filesystem summary:" | awk '{print $3}') + local disksz=$(lfs_df $mntpt | grep "summary" | awk '{print $2}') local blk_soft=$((disksz + 1024)) local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over - local Inodes=$(lfs df -i $mntpt | grep "filesystem summary:" | awk '{print $3}') + local Inodes=$(lfs_df -i $mntpt | grep "summary" | awk '{print $2}') local i_soft=$Inodes local i_hard=$((i_soft + i_soft / 20)) @@ -1085,7 +1121,7 @@ setup_quota_old(){ mdt_quota_type() { local varsvc=${SINGLEMDS}_svc do_facet $SINGLEMDS $LCTL get_param -n \ - osd-$FSTYPE.${!varsvc}.quota_slave.enabled + osd-$(facet_fstype $SINGLEMDS).${!varsvc}.quota_slave.enabled } # get ost quota type @@ -1093,12 +1129,12 @@ ost_quota_type() { # All OSTs should have same quota type local varsvc=ost1_svc do_facet ost1 $LCTL get_param -n \ - osd-$FSTYPE.${!varsvc}.quota_slave.enabled + osd-$(facet_fstype ost1).${!varsvc}.quota_slave.enabled } # restore old quota type settings restore_quota() { - if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.51) ]; then + if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then restore_quota_old return fi @@ -1113,8 +1149,15 @@ restore_quota() { fi } +# Handle the case when there is a space in the lfs df +# "filesystem summary" line the same as when there is no space. +# This will allow fixing the "lfs df" summary line in the future. +lfs_df() { + $LFS df $* | sed -e 's/filesystem /filesystem_/' +} + setup_quota(){ - if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.51) ]; then + if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then setup_quota_old $1 return fi @@ -1139,13 +1182,11 @@ setup_quota(){ local quota_usrs=$QUOTA_USERS # get_filesystem_size - local disksz=$(lfs df $mntpt | grep "filesystem summary:" | - awk '{print $3}') + local disksz=$(lfs_df $mntpt | grep "summary" | awk '{print $2}') local blk_soft=$((disksz + 1024)) local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over - local inodes=$(lfs df -i $mntpt | grep "filesystem summary:" | - awk '{print $3}') + local inodes=$(lfs_df -i $mntpt | grep "summary" | awk '{print $2}') local i_soft=$inodes local i_hard=$((i_soft + i_soft / 20)) @@ -1465,10 +1506,10 @@ _check_progs_installed () { } check_progs_installed () { - local nodes=$1 - shift + local nodes=$1 + shift - do_rpc_nodes $nodes _check_progs_installed $@ + do_rpc_nodes "$nodes" _check_progs_installed $@ } # recovery-scale functions @@ -1685,15 +1726,16 @@ cleanup_check() { check_mem_leak || exit 204 - [ "`lctl dl 2> /dev/null | wc -l`" -gt 0 ] && lctl dl && \ - echo "$0: lustre didn't clean up..." 1>&2 && return 202 || true + [ "`lctl dl 2> /dev/null | wc -l`" -gt 0 ] && lctl dl && + echo "$TESTSUITE: lustre didn't clean up..." 1>&2 && + return 202 || true - if module_loaded lnet || module_loaded libcfs; then - echo "$0: modules still loaded..." 1>&2 - /sbin/lsmod 1>&2 - return 203 - fi - return 0 + if module_loaded lnet || module_loaded libcfs; then + echo "$TESTSUITE: modules still loaded..." 1>&2 + /sbin/lsmod 1>&2 + return 203 + fi + return 0 } wait_update () { @@ -1734,23 +1776,52 @@ sync_all_data() { grep -v 'Found no match' } -wait_delete_completed () { - local TOTALPREV=`lctl get_param -n osc.*.kbytesavail | \ - awk 'BEGIN{total=0}; {total+=$1}; END{print total}'` +wait_delete_completed_mds() { + local MAX_WAIT=${1:-20} + local mds2sync="" + local stime=`date +%s` + local etime + local node + local changes + + # find MDS with pending deletions + for node in $(mdts_nodes); do + changes=$(do_node $node "lctl get_param -n osc.*MDT*.sync_*" \ + 2>/dev/null | calc_sum) + if [ -z "$changes" ] || [ $changes -eq 0 ]; then + continue + fi + mds2sync="$mds2sync $node" + done + if [ "$mds2sync" == "" ]; then + return + fi + mds2sync=$(comma_list $mds2sync) + + # sync MDS transactions + do_nodes $mds2sync "lctl set_param -n osd*.*MD*.force_sync 1" + + # wait till all changes are sent and commmitted by OSTs + # for ldiskfs space is released upon execution, but DMU + # do this upon commit + + local WAIT=0 + while [ "$WAIT" -ne "$MAX_WAIT" ]; do + changes=$(do_nodes $mds2sync "lctl get_param -n osc.*MDT*.sync_*" \ + | calc_sum) + #echo "$node: $changes changes on all" + if [ "$changes" -eq "0" ]; then + etime=`date +%s` + #echo "delete took $((etime - stime)) seconds" + return + fi + sleep 1 + WAIT=$(( WAIT + 1)) + done - local WAIT=0 - local MAX_WAIT=20 - while [ "$WAIT" -ne "$MAX_WAIT" ]; do - sleep 1 - TOTAL=`lctl get_param -n osc.*.kbytesavail | \ - awk 'BEGIN{total=0}; {total+=$1}; END{print total}'` - [ "$TOTAL" -eq "$TOTALPREV" ] && return 0 - echo "Waiting delete completed ... prev: $TOTALPREV current: $TOTAL " - TOTALPREV=$TOTAL - WAIT=$(( WAIT + 1)) - done - echo "Delete is not completed in $MAX_WAIT sec" - return 1 + etime=`date +%s` + echo "Delete is not completed in $((etime - stime)) seconds" + do_nodes $mds2sync "lctl get_param osc.*MDT*.sync_*" } wait_for_host() { @@ -1806,14 +1877,14 @@ wait_recovery_complete () { fi echo affected facets: $facets - # we can use "for" here because we are waiting the slowest - for facet in ${facets//,/ }; do - local var_svc=${facet}_svc - local param="*.${!var_svc}.recovery_status" + # we can use "for" here because we are waiting the slowest + for facet in ${facets//,/ }; do + local var_svc=${facet}_svc + local param="*.${!var_svc}.recovery_status" - local host=$(facet_active_host $facet) - do_rpc_nodes $host _wait_recovery_complete $param $MAX - done + local host=$(facet_active_host $facet) + do_rpc_nodes "$host" _wait_recovery_complete $param $MAX + done } wait_mds_ost_sync () { @@ -1844,28 +1915,35 @@ wait_mds_ost_sync () { } wait_destroy_complete () { - echo "Waiting for destroy to be done..." - # MAX value shouldn't be big as this mean server responsiveness - # never increase this just to make test pass but investigate - # why it takes so long time - local MAX=5 - local WAIT=0 - while [ $WAIT -lt $MAX ]; do - local -a RPCs=($($LCTL get_param -n osc.*.destroys_in_flight)) - local con=1 - for ((i=0; i<${#RPCs[@]}; i++)); do - [ ${RPCs[$i]} -eq 0 ] && continue - # there are still some destroy RPCs in flight - con=0 - break; - done - sleep 1 - [ ${con} -eq 1 ] && return 0 # done waiting - echo "Waiting $WAIT secs for destroys to be done." - WAIT=$((WAIT + 1)) - done - echo "Destroys weren't done in $MAX sec." - return 1 + echo "Waiting for local destroys to complete" + # MAX value shouldn't be big as this mean server responsiveness + # never increase this just to make test pass but investigate + # why it takes so long time + local MAX=5 + local WAIT=0 + while [ $WAIT -lt $MAX ]; do + local -a RPCs=($($LCTL get_param -n osc.*.destroys_in_flight)) + local con=1 + local i + + for ((i=0; i<${#RPCs[@]}; i++)); do + [ ${RPCs[$i]} -eq 0 ] && continue + # there are still some destroy RPCs in flight + con=0 + break; + done + sleep 1 + [ ${con} -eq 1 ] && return 0 # done waiting + echo "Waiting ${WAIT}s for local destroys to complete" + WAIT=$((WAIT + 1)) + done + echo "Local destroys weren't done in $MAX sec." + return 1 +} + +wait_delete_completed() { + wait_delete_completed_mds $1 || return $? + wait_destroy_complete } wait_exit_ST () { @@ -1972,32 +2050,58 @@ affected_facets () { } facet_failover() { - local facet=$1 - local sleep_time=$2 - local host=$(facet_active_host $facet) - - echo "Failing $facet on node $host" - - local affected=$(affected_facets $facet) + local facets=$1 + local sleep_time=$2 + local -a affecteds + local facet + local total=0 + local index=0 + local skip + + #Because it will only get up facets, we need get affected + #facets before shutdown + #For HARD Failure mode, it needs make sure facets on the same + #HOST will only be shutdown and reboot once + for facet in ${facets//,/ }; do + local affected_facet + skip=0 + #check whether facet has been included in other affected facets + for ((index=0; index<$total; index++)); do + [[ *,$facet,* == ,${affecteds[index]}, ]] && skip=1 + done - shutdown_facet $facet + if [ $skip -eq 0 ]; then + affecteds[$total]=$(affected_facets $facet) + total=$((total+1)) + fi + done - echo affected facets: $affected + for ((index=0; index<$total; index++)); do + facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1) + local host=$(facet_active_host $facet) + echo "Failing ${affecteds[index]} on $host" + shutdown_facet $facet + done - [ -n "$sleep_time" ] && sleep $sleep_time + for ((index=0; index<$total; index++)); do + facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1) + echo reboot facets: ${affecteds[index]} - reboot_facet $facet + reboot_facet $facet - change_active $affected + change_active ${affecteds[index]} - wait_for_facet $affected - # start mgs first if it is affected - if ! combined_mgs_mds && list_member $affected mgs; then - mount_facet mgs || error "Restart of mgs failed" - fi - # FIXME; has to be changed to mount all facets concurrently - affected=$(exclude_items_from_list $affected mgs) - mount_facets $affected + wait_for_facet ${affecteds[index]} + # start mgs first if it is affected + if ! combined_mgs_mds && + list_member ${affecteds[index]} mgs; then + mount_facet mgs || error "Restart of mgs failed" + fi + # FIXME; has to be changed to mount all facets concurrently + affected=$(exclude_items_from_list ${affecteds[index]} mgs) + echo mount facets: ${affecteds[index]} + mount_facets ${affecteds[index]} + done } obd_name() { @@ -2011,7 +2115,9 @@ replay_barrier() { # make sure there will be no seq change local clients=${CLIENTS:-$HOSTNAME} - do_nodes $clients "f=${MOUNT}/fsa-\\\$(hostname); mcreate \\\$f; rm \\\$f" + local f=fsa-\\\$\(hostname\) + do_nodes $clients "mcreate $MOUNT/$f; rm $MOUNT/$f" + do_nodes $clients "if [ -d $MOUNT2 ]; then mcreate $MOUNT2/$f; rm $MOUNT2/$f; fi" local svc=${facet}_svc do_facet $facet $LCTL --device %${!svc} notransno @@ -2219,21 +2325,26 @@ hostlist_expand() { } facet_host() { - local facet=$1 - - [ "$facet" == client ] && echo -n $HOSTNAME && return - varname=${facet}_HOST - if [ -z "${!varname}" ]; then - if [ "${facet:0:3}" == "ost" ]; then - eval ${facet}_HOST=${ost_HOST} - fi - fi - echo -n ${!varname} + local facet=$1 + local varname + + [ "$facet" == client ] && echo -n $HOSTNAME && return + varname=${facet}_HOST + if [ -z "${!varname}" ]; then + if [ "${facet:0:3}" == "ost" ]; then + eval export ${facet}_HOST=${ost_HOST} + elif [ "${facet:0:3}" == "mdt" -o \ + "${facet:0:3}" == "mds" -o \ + "${facet:0:3}" == "mgs" ]; then + eval export ${facet}_HOST=${mds_HOST} + fi + fi + echo -n ${!varname} } facet_failover_host() { local facet=$1 - local var + local varname var=${facet}failover_HOST if [ -n "${!var}" ]; then @@ -2241,12 +2352,18 @@ facet_failover_host() { return fi + if [ "${facet:0:3}" == "mdt" -o "${facet:0:3}" == "mds" -o \ + "${facet:0:3}" == "mgs" ]; then + + eval export ${facet}failover_host=${mds_HOST} + echo ${mds_HOST} + return + fi + if [[ $facet == ost* ]]; then - var=ostfailover_HOST - if [ -n "${!var}" ]; then - echo ${!var} - return - fi + eval export ${facet}failover_host=${ost_HOST} + echo ${ost_HOST} + return fi } @@ -2369,8 +2486,6 @@ get_env_vars() { echo -n " ${var}=\"$value\"" done - echo -n " USE_OFD=$USE_OFD" - for facet in ${facets//,/ }; do var=${facet}_FSTYPE if [ -n "${!var}" ]; then @@ -2553,17 +2668,16 @@ mdsvdevname() { } mgsdevname() { - DEVNAME=MGSDEV + local DEVNAME=MGSDEV + local MDSDEV1=$(mdsdevname 1) - local fstype=$(facet_fstype mds$num) + local fstype=$(facet_fstype mds1) case $fstype in - ldiskfs ) + ldiskfs|zfs ) #if $MGSDEV isn't defined, default is $MDSDEV1 + #ZFS independent mgsdev should be ${FSNAME}-mgs/mgs eval DEVPTR=${!DEVNAME:=${MDSDEV1}};; - zfs ) - #dataset name is independent of vdev device names - eval DEVPTR=${FSNAME}-mgs/mgs;; * ) error "unknown fstype!";; esac @@ -2574,7 +2688,7 @@ mgsdevname() { mgsvdevname() { DEVNAME=MGSDEV - local fstype=$(facet_fstype mds$num) + local fstype=$(facet_fstype mds1) case $fstype in ldiskfs ) @@ -2659,7 +2773,8 @@ cleanupall() { } combined_mgs_mds () { - [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]] + [[ "$(mdsdevname 1)" = "$(mgsdevname)" ]] && + [[ "$(facet_host mds1)" = "$(facet_host mgs)" ]] } lower() { @@ -2672,9 +2787,12 @@ upper() { mkfs_opts() { local facet=$1 + local dev=$2 + local fsname=${3:-"$FSNAME"} local type=$(facet_type $facet) local index=$(($(facet_number $facet) - 1)) local fstype=$(facet_fstype $facet) + local host=$(facet_host $facet) local opts local fs_mkfs_opts local var @@ -2683,14 +2801,17 @@ mkfs_opts() { return 1 fi - if [ $type == MGS ] || ( [ $type == MDS ] && combined_mgs_mds ); then + if [ $type == MGS ] || ( [ $type == MDS ] && + [ "$dev" == $(mgsdevname) ] && + [ "$host" == "$(facet_host mgs)" ] ); then opts="--mgs" else opts="--mgsnode=$MGSNID" fi if [ $type != MGS ]; then - opts+=" --fsname=$FSNAME --$(lower ${type/MDS/MDT}) --index=$index" + opts+=" --fsname=$fsname --$(lower ${type/MDS/MDT}) \ + --index=$index" fi var=${facet}failover_HOST @@ -2709,8 +2830,6 @@ mkfs_opts() { opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"} if [ $fstype == ldiskfs ]; then - opts+=${IAMDIR:+" --iam-dir"} - fs_mkfs_opts+=${MDSJOURNALSIZE:+" -J size=$MDSJOURNALSIZE"} fs_mkfs_opts+=${MDSISIZE:+" -i $MDSISIZE"} fi @@ -2762,23 +2881,24 @@ formatall() { echo Formatting mgs, mds, osts if ! combined_mgs_mds ; then echo "Format mgs: $(mgsdevname)" - add mgs $(mkfs_opts mgs) --reformat $(mgsdevname) \ - $(mgsvdevname) ${quiet:+>/dev/null} || exit 10 - fi + add mgs $(mkfs_opts mgs $(mgsdevname)) --reformat \ + $(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} || + exit 10 + fi - for num in `seq $MDSCOUNT`; do - echo "Format mds$num: $(mdsdevname $num)" - add mds$num $(mkfs_opts mds$num) --reformat \ - $(mdsdevname $num) $(mdsvdevname $num) \ + for num in $(seq $MDSCOUNT); do + echo "Format mds$num: $(mdsdevname $num)" + add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \ + --reformat $(mdsdevname $num) $(mdsvdevname $num) \ ${quiet:+>/dev/null} || exit 10 - done + done - for num in `seq $OSTCOUNT`; do - echo "Format ost$num: $(ostdevname $num)" - add ost$num $(mkfs_opts ost$num) --reformat \ - $(ostdevname $num) $(ostvdevname ${num}) \ + for num in $(seq $OSTCOUNT); do + echo "Format ost$num: $(ostdevname $num)" + add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \ + --reformat $(ostdevname $num) $(ostvdevname ${num}) \ ${quiet:+>/dev/null} || exit 10 - done + done } mount_client() { @@ -2826,23 +2946,31 @@ remount_client() zconf_mount `hostname` $1 || error "mount failed" } -writeconf_facet () { - local facet=$1 - local dev=$2 +writeconf_facet() { + local facet=$1 + local dev=$2 - do_facet $facet "$TUNEFS --writeconf $dev" + stop ${facet} -f + rm -f ${facet}active + do_facet ${facet} "$TUNEFS --quiet --writeconf $dev" || return 1 + return 0 } writeconf_all () { - for num in `seq $MDSCOUNT`; do - DEVNAME=$(mdsdevname $num) - writeconf_facet mds$num $DEVNAME - done + local mdt_count=${1:-$MDSCOUNT} + local ost_count=${2:-$OSTCOUNT} + local rc=0 - for num in `seq $OSTCOUNT`; do - DEVNAME=$(ostdevname $num) - writeconf_facet ost$num $DEVNAME - done + for num in $(seq $mdt_count); do + DEVNAME=$(mdsdevname $num) + writeconf_facet mds$num $DEVNAME || rc=$? + done + + for num in $(seq $ost_count); do + DEVNAME=$(ostdevname $num) + writeconf_facet ost$num $DEVNAME || rc=$? + done + return $rc } setupall() { @@ -2928,65 +3056,98 @@ mounted_lustre_filesystems() { } init_facet_vars () { - [ "$CLIENTONLY" ] && return 0 - local facet=$1 - shift - local device=$1 + [ "$CLIENTONLY" ] && return 0 + local facet=$1 + shift + local device=$1 - shift + shift - eval export ${facet}_dev=${device} - eval export ${facet}_opt=\"$@\" + eval export ${facet}_dev=${device} + eval export ${facet}_opt=\"$@\" - local dev=${facet}_dev - local label=$(devicelabel ${facet} ${!dev}) - [ -z "$label" ] && echo no label for ${!dev} && exit 1 + local dev=${facet}_dev - eval export ${facet}_svc=${label} + # We need to loop for the label + # in case its not initialized yet. + for wait_time in {0,1,3,5,10}; do - local varname=${facet}failover_HOST - if [ -z "${!varname}" ]; then - eval $varname=$(facet_host $facet) - fi + if [ $wait_time -gt 0 ]; then + echo "${!dev} not yet initialized,"\ + "waiting ${wait_time} seconds." + sleep $wait_time + fi - # ${facet}failover_dev is set in cfg file - varname=${facet}failover_dev - if [ -n "${!varname}" ] ; then - eval export ${facet}failover_dev=${!varname} - else - eval export ${facet}failover_dev=$device - fi + local label=$(devicelabel ${facet} ${!dev}) - # get mount point of already mounted device - # is facet_dev is already mounted then use the real - # mount point of this facet; otherwise use $(facet_mntpt $facet) - # i.e. ${facet}_MOUNT if specified by user or default - local mntpt=$(do_facet ${facet} cat /proc/mounts | \ - awk '"'${!dev}'" == $1 && $3 == "lustre" { print $2 }') - if [ -z $mntpt ]; then - mntpt=$(facet_mntpt $facet) - fi - eval export ${facet}_MOUNT=$mntpt + # Check to make sure the label does + # not include ffff at the end of the label. + # This indicates it has not been initialized yet. + + if [[ $label =~ [f|F]{4}$ ]]; then + # label is not initialized, unset the result + # and either try again or fail + unset label + else + break + fi + done + + [ -z "$label" ] && echo no label for ${!dev} && exit 1 + + eval export ${facet}_svc=${label} + + local varname=${facet}failover_HOST + if [ -z "${!varname}" ]; then + eval export $varname=$(facet_host $facet) + fi + + varname=${facet}_HOST + if [ -z "${!varname}" ]; then + eval export $varname=$(facet_host $facet) + fi + + # ${facet}failover_dev is set in cfg file + varname=${facet}failover_dev + if [ -n "${!varname}" ] ; then + eval export ${facet}failover_dev=${!varname} + else + eval export ${facet}failover_dev=$device + fi + + # get mount point of already mounted device + # is facet_dev is already mounted then use the real + # mount point of this facet; otherwise use $(facet_mntpt $facet) + # i.e. ${facet}_MOUNT if specified by user or default + local mntpt=$(do_facet ${facet} cat /proc/mounts | \ + awk '"'${!dev}'" == $1 && $3 == "lustre" { print $2 }') + if [ -z $mntpt ]; then + mntpt=$(facet_mntpt $facet) + fi + eval export ${facet}_MOUNT=$mntpt } init_facets_vars () { - local DEVNAME + local DEVNAME - if ! remote_mds_nodsh; then - for num in `seq $MDSCOUNT`; do - DEVNAME=`mdsdevname $num` - init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS - done - fi + if ! remote_mds_nodsh; then + for num in $(seq $MDSCOUNT); do + DEVNAME=`mdsdevname $num` + eval export MDSDEV${num}=$DEVNAME + init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS + done + fi + eval export MGSDEV=$(mgsdevname) combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS - remote_ost_nodsh && return - - for num in `seq $OSTCOUNT`; do - DEVNAME=`ostdevname $num` - init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS - done + if ! remote_ost_nodsh; then + for num in $(seq $OSTCOUNT); do + DEVNAME=$(ostdevname $num) + eval export OSTDEV${num}=$DEVNAME + init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS + done + fi } osc_ensure_active () { @@ -3008,6 +3169,25 @@ osc_ensure_active () { [ $period -lt $timeout ] || log "$count OST are inactive after $timeout seconds, give up" } +set_conf_param_and_check() { + local myfacet=$1 + local TEST=$2 + local PARAM=$3 + local ORIG=$(do_facet $myfacet "$TEST") + if [ $# -gt 3 ]; then + local FINAL=$4 + else + local -i FINAL + FINAL=$((ORIG + 5)) + fi + echo "Setting $PARAM from $ORIG to $FINAL" + do_facet mgs "$LCTL conf_param $PARAM='$FINAL'" || + error "conf_param $PARAM failed" + + wait_update $(facet_host $myfacet) "$TEST" "$FINAL" || + error "check $PARAM failed!" +} + init_param_vars () { remote_mds_nodsh || TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout") @@ -3112,15 +3292,14 @@ check_config_client () { } check_config_clients () { - local clients=${CLIENTS:-$HOSTNAME} - local mntpt=$1 + local clients=${CLIENTS:-$HOSTNAME} + local mntpt=$1 - nfs_client_mode && return + nfs_client_mode && return - do_rpc_nodes $clients check_config_client $mntpt + do_rpc_nodes "$clients" check_config_client $mntpt - sanity_mount_check || - error "environments are insane!" + sanity_mount_check || error "environments are insane!" } check_timeout () { @@ -3291,7 +3470,7 @@ run_e2fsck() { local extra_opts=$3 df > /dev/null # update statfs data on disk - local cmd="$E2FSCK -d -v -t -t -f -n $extra_opts $target_dev" + local cmd="$E2FSCK -d -v -t -t -f $extra_opts $target_dev" echo $cmd local rc=0 do_node $node $cmd || rc=$? @@ -3302,12 +3481,12 @@ run_e2fsck() { # verify a directory is shared among nodes. check_shared_dir() { - local dir=$1 + local dir=$1 - [ -z "$dir" ] && return 1 - do_rpc_nodes $(comma_list $(nodes_list)) check_logdir $dir - check_write_access $dir || return 1 - return 0 + [ -z "$dir" ] && return 1 + do_rpc_nodes "$(comma_list $(nodes_list))" check_logdir $dir + check_write_access $dir || return 1 + return 0 } # Run e2fsck on MDT and OST(s) to generate databases used for lfsck. @@ -3327,14 +3506,14 @@ generate_db() { [ $MDSCOUNT -eq 1 ] || error "CMD is not supported" - run_e2fsck $(mdts_nodes) $MDTDEV "--mdsdb $MDSDB" + run_e2fsck $(mdts_nodes) $MDTDEV "-n --mdsdb $MDSDB" i=0 ostidx=0 OSTDB_LIST="" for node in $(osts_nodes); do for dev in ${OSTDEVS[i]}; do - run_e2fsck $node $dev "--mdsdb $MDSDB --ostdb $OSTDB-$ostidx" + run_e2fsck $node $dev "-n --mdsdb $MDSDB --ostdb $OSTDB-$ostidx" OSTDB_LIST="$OSTDB_LIST $OSTDB-$ostidx" ostidx=$((ostidx + 1)) done @@ -3431,10 +3610,12 @@ no_dsh() { eval $@ } +# Convert a space-delimited list to a comma-delimited list. If the input is +# only whitespace, ensure the output is empty (i.e. "") so [ -n $list ] works comma_list() { - # the sed converts spaces to commas, but leaves the last space - # alone, so the line doesn't end with a comma. - echo "$*" | tr -s " " "\n" | sort -b -u | tr "\n" " " | sed 's/ \([^$]\)/,\1/g' + # echo is used to convert newlines to spaces, since it doesn't + # introduce a trailing space as using "tr '\n' ' '" does + echo $(tr -s " " "\n" <<< $* | sort -b -u) | tr ' ' ',' } list_member () { @@ -3591,6 +3772,17 @@ drop_reint_reply() { return $RC } +drop_update_reply() { +# OBD_FAIL_MDS_OBJ_UPDATE_NET + local index=$1 + shift 1 + RC=0 + do_facet mds${index} lctl set_param fail_loc=0x188 + do_facet client "$@" || RC=$? + do_facet mds${index} lctl set_param fail_loc=0 + return $RC +} + pause_bulk() { #define OBD_FAIL_OST_BRW_PAUSE_BULK 0x214 RC=0 @@ -3602,7 +3794,7 @@ pause_bulk() { } drop_ldlm_cancel() { -#define OBD_FAIL_LDLM_CANCEL 0x304 +#define OBD_FAIL_LDLM_CANCEL_NET 0x304 local RC=0 local list=$(comma_list $(mdts_nodes) $(osts_nodes)) do_nodes $list lctl set_param fail_loc=0x304 @@ -3614,12 +3806,12 @@ drop_ldlm_cancel() { } drop_bl_callback() { -#define OBD_FAIL_LDLM_BL_CALLBACK 0x305 - RC=0 - do_facet client lctl set_param fail_loc=0x305 - do_facet client "$@" || RC=$? - do_facet client lctl set_param fail_loc=0 - return $RC +#define OBD_FAIL_LDLM_BL_CALLBACK_NET 0x305 + RC=0 + do_facet client lctl set_param fail_loc=0x80000305 + do_facet client "$@" || RC=$? + do_facet client lctl set_param fail_loc=0 + return $RC } drop_ldlm_reply() { @@ -3751,9 +3943,10 @@ error_noexit() { gather_logs $(comma_list $(nodes_list)) fi - debugrestore - [ "$TESTSUITELOG" ] && echo "$0: ${TYPE}: $TESTNAME $@" >> $TESTSUITELOG - echo "$@" > $LOGDIR/err + debugrestore + [ "$TESTSUITELOG" ] && + echo "$TESTSUITE: $TYPE: $TESTNAME $@" >> $TESTSUITELOG + echo "$@" > $LOGDIR/err } exit_status () { @@ -3913,10 +4106,6 @@ run_test() { return $? } -equals_msg() { - banner "$*" -} - log() { echo "$*" module_loaded lnet || load_modules @@ -3943,9 +4132,11 @@ trace() { } complete () { - equals_msg $1 test complete, duration $2 sec + local duration=$1 + + banner test complete, duration $duration sec [ -f "$TESTSUITELOG" ] && egrep .FAIL $TESTSUITELOG || true - echo duration $2 >>$TESTSUITELOG + echo duration $duration >>$TESTSUITELOG } pass() { @@ -4266,9 +4457,7 @@ nodes_list () { } remote_nodes_list () { - local rnodes=$(nodes_list) - rnodes=$(echo " $rnodes " | sed -re "s/\s+$HOSTNAME\s+/ /g") - echo $rnodes + echo $(nodes_list) | sed -re "s/\<$HOSTNAME\>//g" } init_clients_lists () { @@ -4587,15 +4776,15 @@ restore_lustre_params() { } check_catastrophe() { - local rnodes=${1:-$(comma_list $(remote_nodes_list))} - local C=$CATASTROPHE - [ -f $C ] && [ $(cat $C) -ne 0 ] && return 1 + local rnodes=${1:-$(comma_list $(remote_nodes_list))} + local C=$CATASTROPHE + [ -f $C ] && [ $(cat $C) -ne 0 ] && return 1 + + [ -z "$rnodes" ] && return 0 - if [ $rnodes ]; then - do_nodes $rnodes "rc=\\\$([ -f $C ] && echo \\\$(< $C) || echo 0); -if [ \\\$rc -ne 0 ]; then echo \\\$(hostname): \\\$rc; fi -exit \\\$rc;" - fi + do_nodes "$rnodes" "rc=\\\$([ -f $C ] && echo \\\$(< $C) || echo 0); + if [ \\\$rc -ne 0 ]; then echo \\\$(hostname): \\\$rc; fi + exit \\\$rc;" } # CMD: determine mds index where directory inode presents @@ -4804,13 +4993,13 @@ wait_osc_import_state() { # 2. wait the deadline of client 2nd request local maxtime=$(( 2 * $(request_timeout $facet))) - if ! do_rpc_nodes $(facet_host $facet) \ - _wait_import_state $expected $param $maxtime; then - error "import is not in ${expected} state" - return 1 - fi + if ! do_rpc_nodes "$(facet_host $facet)" \ + _wait_import_state $expected $param $maxtime; then + error "import is not in ${expected} state" + return 1 + fi - return 0 + return 0 } get_clientmdc_proc_path() { @@ -4818,12 +5007,14 @@ get_clientmdc_proc_path() { } do_rpc_nodes () { - local list=$1 - shift + local list=$1 + shift + + [ -z "$list" ] && return 0 - # Add paths to lustre tests for 32 and 64 bit systems. - local RPATH="PATH=$RLUSTRE/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:$PATH" - do_nodesv $list "${RPATH} NAME=${NAME} sh rpc.sh $@ " + # Add paths to lustre tests for 32 and 64 bit systems. + local RPATH="PATH=$RLUSTRE/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:$PATH" + do_nodesv $list "${RPATH} NAME=${NAME} sh rpc.sh $@ " } wait_clients_import_state () { @@ -4848,10 +5039,10 @@ wait_clients_import_state () { local params=$(expand_list $params $proc_path) done - if ! do_rpc_nodes $list wait_import_state $expected $params; then - error "import is not in ${expected} state" - return 1 - fi + if ! do_rpc_nodes "$list" wait_import_state $expected $params; then + error "import is not in ${expected} state" + return 1 + fi } oos_full() { @@ -5440,12 +5631,13 @@ run_llverfs() { local dir=$1 local llverfs_opts=$2 + local use_partial_arg=$3 local partial_arg="" local size=$(df -B G $dir |tail -n 1 |awk '{print $2}' |sed 's/G//') #GB # Run in partial (fast) mode if the size # of a partition > 1 GB - [ $size -gt 1 ] && partial_arg="-p" + [ "x$use_partial_arg" != "xno" ] && [ $size -gt 1 ] && partial_arg="-p" llverfs $partial_arg $llverfs_opts $dir } @@ -5566,10 +5758,12 @@ get_block_size() { # Check whether the "large_xattr" feature is enabled or not. large_xattr_enabled() { - local mds_dev=$(mdsdevname ${SINGLEMDS//mds/}) + [[ $(facet_fstype $SINGLEMDS) == zfs ]] && return 0 - do_facet $SINGLEMDS "$DUMPE2FS -h $mds_dev 2>&1 | grep -q large_xattr" - return ${PIPESTATUS[0]} + local mds_dev=$(mdsdevname ${SINGLEMDS//mds/}) + + do_facet $SINGLEMDS "$DUMPE2FS -h $mds_dev 2>&1 | grep -q large_xattr" + return ${PIPESTATUS[0]} } # Get the maximum xattr size supported by the filesystem. @@ -5623,6 +5817,7 @@ mds_backup_restore() { local metaea=${TMP}/backup_restore.ea local metadata=${TMP}/backup_restore.tgz local opts=${MDS_MOUNT_OPTS} + local svc=${SINGLEMDS}_svc if ! ${rcmd} test -b ${devname}; then opts=$(csa_add "$opts" -o loop) @@ -5666,6 +5861,8 @@ mds_backup_restore() { ${rcmd} umount -d $mntpt || return 10 # step 14: cleanup tmp backup ${rcmd} rm -f $metaea $metadata + # step 15: reset device label - it's not virgin on + ${rcmd} e2label $devname ${!svc} } # remove OI files @@ -5703,3 +5900,72 @@ mds_remove_ois() { ${rcmd} umount -d $mntpt || return 2 # OI files will be recreated when mounted as lustre next time. } + +# generate maloo upload-able log file name +# \param logname specify unique part of file name +generate_logname() { + local logname=${1:-"default_logname"} + + echo "$TESTLOG_PREFIX.$TESTNAME.$logname.$(hostname -s).log" +} + +# mkdir directory on different MDTs +test_mkdir() { + local option + local parent + local child + local path + local dir + local rc=0 + + if [ $# -eq 2 ]; then + option=$1 + path=$2 + else + path=$1 + fi + + child=${path##*/} + parent=${path%/*} + + if [ "$parent" == "$child" ]; then + parent=$(pwd) + fi + + if [ "$option" == "-p" -a -d ${parent}/${child} ]; then + return $rc + fi + + # it needs to check whether there is further / in child + dir=$(echo $child | awk -F '/' '{print $2}') + if [ ! -z "$dir" ]; then + local subparent=$(echo $child | awk -F '/' '{ print $1 }') + parent=${parent}"/"${subparent} + child=$dir + fi + + if [ ! -d ${parent} ]; then + if [ "$option" == "-p" ]; then + mkdir -p ${parent} + else + return 1 + fi + fi + + if [ $MDSCOUNT -le 1 ]; then + mkdir $option ${parent}/${child} || rc=$? + else + local mdt_idx=$($LFS getstripe -M $parent) + + if [ "$mdt_idx" -ne 0 ]; then + mkdir $option ${parent}/${child} || rc=$? + return $rc + fi + + local test_num=$(echo $testnum | sed -e 's/[^0-9]*//g') + local mdt_idx=$((test_num % MDSCOUNT)) + echo "mkdir $mdt_idx for ${parent}/${child}" + $LFS setdirstripe -i $mdt_idx ${parent}/${child} || rc=$? + fi + return $rc +}