. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
get_lustre_env
+ # use localrecov to enable recovery for local clients, LU-12722
+ [[ $MDS1_VERSION -lt $(version_code 2.13.52) ]] ||
+ export MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-"-o localrecov"}
+ [[ $OST1_VERSION -lt $(version_code 2.13.52) ]] ||
+ export OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-"-o localrecov"}
}
check_cpt_number() {
# split arguments like "1.8.6-wc3" into "1", "8", "6", "wc3"
eval set -- $(tr "[:punct:]" " " <<< $*)
- echo -n "$((($1 << 16) | ($2 << 8) | $3))"
+ echo -n $(((${1:-0} << 16) | (${2:-0} << 8) | ${3:-0}))
}
export LINUX_VERSION=$(uname -r | sed -e "s/\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/")
# output: prints version string to stdout in (up to 4) dotted-decimal values
lustre_build_version() {
local facet=${1:-client}
- local ver=$(do_facet $facet "$LCTL get_param -n version 2>/dev/null ||
- $LCTL lustre_build_version 2>/dev/null ||
- $LCTL --version 2>/dev/null | cut -d' ' -f2")
+ local facet_version=${facet}_VERSION
+
+ # if the global variable is already set, then use that
+ [ -n "${!facet_version}" ] && echo ${!facet_version} && return
+
+ # this is the currently-running version of the kernel modules
+ local ver=$(do_facet $facet "$LCTL get_param -n version 2>/dev/null")
+ # we mostly test 2.10+ systems, only try others if the above fails
+ if [ -z "$ver" ]; then
+ ver=$(do_facet $facet "$LCTL lustre_build_version 2>/dev/null")
+ fi
+ if [ -z "$ver" ]; then
+ ver=$(do_facet $facet "$LCTL --version 2>/dev/null" |
+ cut -d' ' -f2)
+ fi
local lver=$(egrep -i "lustre: |version: " <<<"$ver" | head -n 1)
[ -n "$lver" ] && ver="$lver"
- sed -e 's/[^:]*: //' -e 's/^v//' -e 's/[ -].*//' -e 's/_/./g' <<<$ver |
- cut -d. -f1-4
+ lver=$(sed -e 's/[^:]*: //' -e 's/^v//' -e 's/[ -].*//' <<<$ver |
+ tr _ . | cut -d. -f1-4)
+
+ # save in global variable for the future
+ export $facet_version=$lver
+
+ echo $lver
}
# Report the Lustre numeric build version code for the supplied facet.
echo -n $mt_opts
}
+from_build_tree() {
+ local from_tree
+
+ case $LUSTRE in
+ /usr/lib/lustre/* | /usr/lib64/lustre/* | /usr/lib/lustre | \
+ /usr/lib64/lustre )
+ from_tree=false
+ ;;
+ *)
+ from_tree=true
+ ;;
+ esac
+
+ [ $from_tree = true ]
+}
+
init_gss() {
if $SHARED_KEY; then
GSS=true
return
fi
- case $LUSTRE in
- /usr/lib/lustre/* | /usr/lib64/lustre/* | /usr/lib/lustre | \
- /usr/lib64/lustre )
- from_build_tree=false
- ;;
- *)
- from_build_tree=true
- ;;
- esac
-
if ! module_loaded ptlrpc_gss; then
load_module ptlrpc/gss/ptlrpc_gss
module_loaded ptlrpc_gss ||
SK_NO_KEY=false
local lgssc_conf_file="/etc/request-key.d/lgssc.conf"
- if $from_build_tree; then
+ if from_build_tree; then
mkdir -p $SK_OM_PATH
if grep -q request-key /proc/mounts > /dev/null; then
echo "SSK: Request key already mounted."
cat $lgssc_conf_file
if ! local_mode; then
- if $from_build_tree; then
+ if from_build_tree; then
do_nodes $(comma_list $(all_nodes)) "mkdir -p \
$SK_OM_PATH"
do_nodes $(comma_list $(all_nodes)) "mount \
OST_MOUNT_OPTS=$(add_sk_mntflag $OST_MOUNT_OPTS)
MOUNT_OPTS=$(add_sk_mntflag $MOUNT_OPTS)
SEC=$SK_FLAVOR
+ if [ -z "$LGSS_KEYRING_DEBUG" ]; then
+ LGSS_KEYRING_DEBUG=4
+ fi
fi
- if [ -n "$LGSS_KEYRING_DEBUG" ]; then
+ if [ -n "$LGSS_KEYRING_DEBUG" ] && \
+ ( local_mode || from_build_tree ); then
+ lctl set_param -n \
+ sptlrpc.gss.lgss_keyring.debug_level=$LGSS_KEYRING_DEBUG
+ elif [ -n "$LGSS_KEYRING_DEBUG" ]; then
+ do_nodes $(comma_list $(all_nodes)) "modprobe ptlrpc_gss && \
lctl set_param -n \
- sptlrpc.gss.lgss_keyring.debug_level=$LGSS_KEYRING_DEBUG
+ sptlrpc.gss.lgss_keyring.debug_level=$LGSS_KEYRING_DEBUG"
fi
}
cleanup_sk() {
if $GSS_SK; then
- case $LUSTRE in
- /usr/lib/lustre/* | /usr/lib64/lustre/* | /usr/lib/lustre | \
- /usr/lib64/lustre )
- from_build_tree=false
- ;;
- *)
- from_build_tree=true
- ;;
- esac
-
if $SK_S2S; then
do_node $(mgs_node) "$LCTL nodemap_del $SK_S2SNM"
do_node $(mgs_node) "$LCTL nodemap_del $SK_S2SNMCLI"
$SK_PATH/$FSNAME*.key $SK_PATH/nodemap/$FSNAME*.key"
do_nodes $(comma_list $(all_nodes)) "keyctl show | \
awk '/lustre/ { print \\\$1 }' | xargs -IX keyctl unlink X"
- if $from_build_tree; then
+ if from_build_tree; then
# Remove the mount and clean up the files we added to
# SK_PATH
do_nodes $(comma_list $(all_nodes)) "while grep -q \
mount_mds_client() {
local mds_HOST=${SINGLEMDS}_HOST
echo $mds_HOST
- do_facet $SINGLEMDS "mkdir -p $MOUNT2"
zconf_mount $mds1_HOST $MOUNT2 $MOUNT_OPTS ||
error "unable to mount $MOUNT2 on MDS"
}
umount_mds_client() {
local mds_HOST=${SINGLEMDS}_HOST
zconf_umount $mds1_HOST $MOUNT2
- do_facet $SINGLEMDS "rm -rf $MOUNT2"
+ do_facet $SINGLEMDS "rmdir $MOUNT2"
}
# nodes is comma list
}
wait_for_host() {
- local hostlist=$1
+ local hostlist=$1
- # we can use "for" here because we are waiting the slowest
- for host in ${hostlist//,/ }; do
- check_network "$host" 900
- done
- while ! do_nodes $hostlist hostname > /dev/null; do sleep 5; done
+ # we can use "for" here because we are waiting the slowest
+ for host in ${hostlist//,/ }; do
+ check_network "$host" 900
+ done
+ while ! do_nodes $hostlist hostname > /dev/null; do sleep 5; done
}
wait_for_facet() {
# $LFS quotaoff -ug $MOUNT > /dev/null 2>&1
fi
fi
+
+ (( MDS1_VERSION <= $(version_code 2.13.52) )) ||
+ do_nodes $(comma_list $(mdts_nodes)) \
+ "$LCTL set_param lod.*.mdt_hash=crush"
return 0
}
fi
fi
+ if [ -n "$fs_STRIPEPARAMS" ]; then
+ setstripe_getstripe $MOUNT $fs_STRIPEPARAMS
+ fi
if $GSS_SK; then
set_flavor_all null
elif $GSS; then
[ "$host" = "$HOSTNAME" ] && return 0
- echo "$(date +'%H:%M:%S (%s)') waiting for $host network $max secs ..."
- if ! wait_for_function --quiet "ping -c 1 -w 3 $host" $max $sleep ; then
- echo "Network not available!"
+ if ! wait_for_function --quiet "ping -c 1 -w 3 $host" $max $sleep; then
+ echo "$(date +'%H:%M:%S (%s)') waited for $host network ${max}s"
exit 1
fi
-
- echo "$(date +'%H:%M:%S (%s)') network interface is UP"
}
no_dsh() {
fi
done
- [ "$EXCEPT$ALWAYS_EXCEPT" ] && \
- log "excepting tests: `echo $EXCEPT $ALWAYS_EXCEPT`"
- [ "$EXCEPT_SLOW" ] && \
- log "skipping tests SLOW=no: `echo $EXCEPT_SLOW`"
- for E in $EXCEPT; do
- eval EXCEPT_${E}=true
- done
- for E in $ALWAYS_EXCEPT; do
- eval EXCEPT_ALWAYS_${E}=true
- done
- for E in $EXCEPT_SLOW; do
- eval EXCEPT_SLOW_${E}=true
- done
- for G in $GRANT_CHECK_LIST; do
- eval GCHECK_ONLY_${G}=true
- done
+ [ "$EXCEPT$ALWAYS_EXCEPT" ] &&
+ log "excepting tests: `echo $EXCEPT $ALWAYS_EXCEPT`"
+ [ "$EXCEPT_SLOW" ] &&
+ log "skipping tests SLOW=no: `echo $EXCEPT_SLOW`"
+ for E in $EXCEPT; do
+ eval EXCEPT_${E}=true
+ done
+ for E in $ALWAYS_EXCEPT; do
+ eval EXCEPT_ALWAYS_${E}=true
+ done
+ for E in $EXCEPT_SLOW; do
+ eval EXCEPT_SLOW_${E}=true
+ done
+ for G in $GRANT_CHECK_LIST; do
+ eval GCHECK_ONLY_${G}=true
+ done
}
basetest() {
TEST_STATUS="PASS"
fi
- pass "$testnum" "($((SECONDS - before))s)"
+ pass "$testnum" "(${duration_sub}s)"
log_sub_test_end $TEST_STATUS $duration_sub "$rc" "$test_error"
[[ $rc != 0 ]] && break
done
(cd $(dirname $1); echo $PWD/$(basename $1))
}
+grant_from_clients() {
+ local nodes="$1"
+
+ # get client grant
+ do_nodes $nodes "$LCTL get_param -n osc.${FSNAME}-*.cur_*grant_bytes" |
+ calc_sum
+}
+
+grant_from_servers() {
+ local nodes="$1"
+
+ # get server grant
+ # which is tot_granted less grant_precreate
+ do_nodes $nodes "$LCTL get_param obdfilter.${FSNAME}-OST*.tot_granted" \
+ " obdfilter.${FSNAME}-OST*.tot_pending" \
+ " obdfilter.${FSNAME}-OST*.grant_precreate" |
+ tr '=' ' ' | awk '/tot_granted/{ total += $2 };
+ /tot_pending/{ total -= $2 };
+ /grant_precreate/{ total -= $2 };
+ END { printf("%0.0f", total) }'
+}
check_grant() {
export base=$(basetest $1)
echo -n "checking grant......"
+ local osts=$(comma_list $(osts_nodes))
local clients=$CLIENTS
[ -z "$clients" ] && clients=$(hostname)
clients_up # initiate all idling connections
# get client grant
- client_grant=$(do_nodes $clients \
- "$LCTL get_param -n osc.${FSNAME}-*.cur_*grant_bytes" |
- awk '{ total += $1 } END { printf("%0.0f", total) }')
+ cli_grant=$(grant_from_clients $clients)
# get server grant
# which is tot_granted less grant_precreate
- server_grant=$(do_nodes $(comma_list $(osts_nodes)) \
- "$LCTL get_param "\
- "obdfilter.${FSNAME}-OST*.{tot_granted,tot_pending,grant_precreate}" |
- sed 's/=/ /'| awk '/tot_granted/{ total += $2 };
- /tot_pending/{ total -= $2 };
- /grant_precreate/{ total -= $2 };
- END { printf("%0.0f", total) }')
+ srv_grant=$(grant_from_servers $osts)
+ count=0
# check whether client grant == server grant
- if [[ $client_grant -ne $server_grant ]]; then
+ while [[ $cli_grant != $srv_grant && count++ -lt 30 ]]; do
+ echo "wait for client:$cli_grant == server:$srv_grant"
+ sleep 1
+ cli_grant=$(grant_from_clients $clients)
+ srv_grant=$(grant_from_servers $osts)
+ done
+ if [[ $cli_grant -ne $srv_grant ]]; then
do_nodes $(comma_list $(osts_nodes)) \
"$LCTL get_param obdfilter.${FSNAME}-OST*.tot*" \
- "obdfilter.${FSNAME}-OST*.grant_*"
+ "obdfilter.${FSNAME}-OST*.grant_*"
do_nodes $clients "$LCTL get_param osc.${FSNAME}-*.cur_*_bytes"
- error "failed: client:${client_grant} server: ${server_grant}."
+ error "failed grant check: client:$cli_grant server:$srv_grant"
else
- echo "pass: client:${client_grant} server: ${server_grant}"
+ echo "pass grant check: client:$cli_grant server:$srv_grant"
fi
}
echo ${uuid/_UUID/}
}
+mdtname_from_index() {
+ local uuid=$(mdtuuid_from_index $1)
+ echo ${uuid/_UUID/}
+}
+
+mdssize_from_index () {
+ local mdt=$(mdtname_from_index $2)
+ $LFS df $1 | grep $mdt | awk '{ print $2 }'
+}
+
index_from_ostuuid()
{
$LFS osts $2 | sed -ne "/${1}/s/\(.*\): .* .*$/\1/p"
}
do_and_time () {
- local cmd=$1
- local rc
-
- SECONDS=0
- eval '$cmd'
+ local cmd="$1"
+ local start
+ local rc
- [ ${PIPESTATUS[0]} -eq 0 ] || rc=1
+ start=$SECONDS
+ eval '$cmd'
+ [ ${PIPESTATUS[0]} -eq 0 ] || rc=1
- echo $SECONDS
- return $rc
+ echo $((SECONDS - start))
+ return $rc
}
inodes_available () {
for node in ${nodes//,/ }; do
check_network "$node" 5
if [ $? -eq 0 ]; then
- do_node $node "rc=0;
- val=\\\$($LCTL get_param -n catastrophe 2>&1);
- if [[ \\\$? -eq 0 && \\\$val -ne 0 ]]; then
- echo \\\$(hostname -s): \\\$val;
- rc=\\\$val;
- fi;
- exit \\\$rc" || error "$node:LBUG/LASSERT detected"
+ do_node $node "$LCTL get_param catastrophe 2>&1" |
+ grep -q "catastrophe=1" &&
+ error "$node:LBUG/LASSERT detected" || true
fi
done
}
}
gather_logs () {
- local list=$1
+ local list=$1
- local ts=$(date +%s)
- local docp=true
+ local ts=$(date +%s)
+ local docp=true
- if [[ ! -f "$YAML_LOG" ]]; then
- # init_logging is not performed before gather_logs,
- # so the $LOGDIR needs to be checked here
- check_shared_dir $LOGDIR && touch $LOGDIR/shared
- fi
+ if [[ ! -f "$YAML_LOG" ]]; then
+ # init_logging is not performed before gather_logs,
+ # so the $LOGDIR needs to be checked here
+ check_shared_dir $LOGDIR && touch $LOGDIR/shared
+ fi
- [ -f $LOGDIR/shared ] && docp=false
+ [ -f $LOGDIR/shared ] && docp=false
- # dump lustre logs, dmesg
+ # dump lustre logs, dmesg, and journal if GSS_SK=true
- prefix="$TESTLOG_PREFIX.$TESTNAME"
- suffix="$ts.log"
- echo "Dumping lctl log to ${prefix}.*.${suffix}"
+ prefix="$TESTLOG_PREFIX.$TESTNAME"
+ suffix="$ts.log"
+ echo "Dumping lctl log to ${prefix}.*.${suffix}"
- if [ -n "$CLIENTONLY" -o "$PDSH" == "no_dsh" ]; then
- echo "Dumping logs only on local client."
- $LCTL dk > ${prefix}.debug_log.$(hostname -s).${suffix}
- dmesg > ${prefix}.dmesg.$(hostname -s).${suffix}
- return
- fi
+ if [ -n "$CLIENTONLY" -o "$PDSH" == "no_dsh" ]; then
+ echo "Dumping logs only on local client."
+ $LCTL dk > ${prefix}.debug_log.$(hostname -s).${suffix}
+ dmesg > ${prefix}.dmesg.$(hostname -s).${suffix}
+ [ "$SHARED_KEY" = true ] && find $SK_PATH -name '*.key' -exec \
+ lgss_sk -r {} \; &> \
+ ${prefix}.ssk_keys.$(hostname -s).${suffix}
+ [ "$SHARED_KEY" = true ] && lctl get_param 'nodemap.*.*' > \
+ ${prefix}.nodemaps.$(hostname -s).${suffix}
+ [ "$GSS_SK" = true ] && keyctl show > \
+ ${prefix}.keyring.$(hostname -s).${suffix}
+ [ "$GSS_SK" = true ] && journalctl -a > \
+ ${prefix}.journal.$(hostname -s).${suffix}
+ return
+ fi
- do_nodesv $list \
- "$LCTL dk > ${prefix}.debug_log.\\\$(hostname -s).${suffix};
- dmesg > ${prefix}.dmesg.\\\$(hostname -s).${suffix}"
+ do_nodesv $list \
+ "$LCTL dk > ${prefix}.debug_log.\\\$(hostname -s).${suffix};
+ dmesg > ${prefix}.dmesg.\\\$(hostname -s).${suffix}"
+ if [ "$SHARED_KEY" = true ]; then
+ do_nodesv $list "find $SK_PATH -name '*.key' -exec \
+ lgss_sk -r {} \; &> \
+ ${prefix}.ssk_keys.\\\$(hostname -s).${suffix}"
+ do_facet mds1 "lctl get_param 'nodemap.*.*' > \
+ ${prefix}.nodemaps.\\\$(hostname -s).${suffix}"
+ fi
+ if [ "$GSS_SK" = true ]; then
+ do_nodesv $list "keyctl show > \
+ ${prefix}.keyring.\\\$(hostname -s).${suffix}"
+ do_nodesv $list "journalctl -a > \
+ ${prefix}.journal.\\\$(hostname -s).${suffix}"
+ fi
- if [ ! -f $LOGDIR/shared ]; then
- do_nodes $list rsync -az "${prefix}.*.${suffix}" $HOSTNAME:$LOGDIR
- fi
+ if [ ! -f $LOGDIR/shared ]; then
+ do_nodes $list rsync -az "${prefix}.*.${suffix}" \
+ $HOSTNAME:$LOGDIR
+ fi
}
do_ls () {
umask $save_umask
- # If modules are not yet loaded then older "lctl lustre_build_version"
- # will fail. Use lctl build version instead.
- log "Client: $($LCTL lustre_build_version)"
- log "MDS: $(do_facet $SINGLEMDS $LCTL lustre_build_version 2>/dev/null||
- do_facet $SINGLEMDS $LCTL --version)"
- log "OSS: $(do_facet ost1 $LCTL lustre_build_version 2> /dev/null ||
- do_facet ost1 $LCTL --version)"
+ # log actual client and server versions if needed for debugging
+ log "Client: $(lustre_build_version client)"
+ log "MDS: $(lustre_build_version mds1)"
+ log "OSS: $(lustre_build_version ost1)"
}
log_test() {
test_mkdir() {
local path
local p_option
+ local hash_type
+ local hash_name=("all_char" "fnv_1a_64" "crush")
local dirstripe_count=${DIRSTRIPE_COUNT:-"2"}
local dirstripe_index=${DIRSTRIPE_INDEX:-$((base % $MDSCOUNT))}
local OPTIND=1
- while getopts "c:i:p" opt; do
+ while getopts "c:H:i:p" opt; do
case $opt in
c) dirstripe_count=$OPTARG;;
+ H) hash_type=$OPTARG;;
i) dirstripe_index=$OPTARG;;
p) p_option="-p";;
- \?) error "only support -i -c -p";;
+ \?) error "only support -c -H -i -p";;
esac
done
mdt_index=$dirstripe_index
fi
+ # randomly choose hash type
+ [ -z "$hash_type" ] &&
+ hash_type=${hash_name[$((RANDOM % ${#hash_name[@]}))]}
+
if (($MDS1_VERSION >= $(version_code 2.8.0))); then
if [ $dirstripe_count -eq -1 ]; then
dirstripe_count=$((RANDOM % MDSCOUNT + 1))
dirstripe_count=1
fi
- echo "striped dir -i$mdt_index -c$dirstripe_count $path"
- $LFS mkdir -i$mdt_index -c$dirstripe_count $path ||
- error "mkdir -i $mdt_index -c$dirstripe_count $path failed"
+ echo "striped dir -i$mdt_index -c$dirstripe_count -H $hash_type $path"
+ $LFS mkdir -i$mdt_index -c$dirstripe_count -H $hash_type $path ||
+ error "mkdir -i $mdt_index -c$dirstripe_count -H $hash_type $path failed"
fi
}
}
changelog_dump() {
+ local rc
+
for M in $(seq $MDSCOUNT); do
local facet=mds$M
local mdt="$(facet_svc $facet)"
-
- $LFS changelog $mdt | sed -e 's/^/'$mdt'./'
+ local output
+ local ret
+
+ output=$($LFS changelog $mdt)
+ ret=$?
+ if [ $ret -ne 0 ]; then
+ rc=${rc:-$ret}
+ elif [ -n "$output" ]; then
+ echo "$output" | sed -e 's/^/'$mdt'./'
+ fi
done
+
+ return ${rc:-0}
}
changelog_extract_field() {