export SK_S2S=${SK_S2S:-false}
export SK_S2SNM=${SK_S2SNM:-TestFrameNM}
export SK_S2SNMCLI=${SK_S2SNMCLI:-TestFrameNMCli}
+export SK_SKIPFIRST=${SK_SKIPFIRST:-true}
export IDENTITY_UPCALL=default
export QUOTA_AUTO=1
export FLAKEY=${FLAKEY:-true}
fi
}
+# Get information about the Lustre environment. The information collected
+# will be used in Lustre tests.
+# usage: get_lustre_env
+# input: No required or optional arguments
+# output: No return values, environment variables are exported
+
+get_lustre_env() {
+
+ export mds1_FSTYPE=${mds1_FSTYPE:-$(facet_fstype mds1)}
+ export ost1_FSTYPE=${ost1_FSTYPE:-$(facet_fstype ost1)}
+
+ export MGS_VERSION=$(lustre_version_code mgs)
+ export MDS1_VERSION=$(lustre_version_code mds1)
+ export OST1_VERSION=$(lustre_version_code ost1)
+ export CLIENT_VERSION=$(lustre_version_code client)
+}
+
init_test_env() {
export LUSTRE=$(absolute_path $LUSTRE)
export TESTSUITE=$(basename $0 .sh)
export DO_CLEANUP=${DO_CLEANUP:-true}
export KEEP_ZPOOL=${KEEP_ZPOOL:-false}
export CLEANUP_DM_DEV=false
+ export PAGE_SIZE=$(get_page_size client)
export MKE2FS=$MKE2FS
if [ -z "$MKE2FS" ]; then
SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"}
GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"}
+ export PERM_CMD=${PERM_CMD:-"$LCTL conf_param"}
+
export L_GETIDENTITY=${L_GETIDENTITY:-"$LUSTRE/utils/l_getidentity"}
if [ ! -f "$L_GETIDENTITY" ]; then
if `which l_getidentity > /dev/null 2>&1`; then
return 0
fi
+ # Create special udev test rules on every node
+ if [ -f $LUSTRE/lustre/conf/99-lustre.rules ]; then {
+ sed -e 's|/usr/sbin/lctl|$LCTL|g' $LUSTRE/lustre/conf/99-lustre.rules > /etc/udev/rules.d/99-lustre-test.rules
+ } else {
+ echo "SUBSYSTEM==\"lustre\", ACTION==\"change\", ENV{PARAM}==\"?*\", RUN+=\"$LCTL set_param '\$env{PARAM}=\$env{SETTING}'\"" > /etc/udev/rules.d/99-lustre-test.rules
+ } fi
+ udevadm control --reload-rules
+ udevadm trigger
+
echo Loading modules from $LUSTRE
local ncpus
$LUSTRE_RMMOD ldiskfs || return 2
+ [ -f /etc/udev/rules.d/99-lustre-test.rules ] &&
+ rm /etc/udev/rules.d/99-lustre-test.rules
+ udevadm control --reload-rules
+ udevadm trigger
+
if $LOAD_MODULES_REMOTE; then
local list=$(comma_list $(remote_nodes_list))
if [ -n "$list" ]; then
echo "unloading modules on: '$list'"
do_rpc_nodes "$list" $LUSTRE_RMMOD ldiskfs
do_rpc_nodes "$list" check_mem_leak
+ do_rpc_nodes "$list" "rm /etc/udev/rules.d/99-lustre-test.rules"
+ do_rpc_nodes "$list" "udevadm control --reload-rules"
+ do_rpc_nodes "$list" "udevadm trigger"
fi
fi
;;
esac
- echo -n $size
+ echo -n $((size * MDSCOUNT))
}
fs_inode_ksize() {
# security ctx config for keyring
SK_NO_KEY=false
mkdir -p $SK_OM_PATH
- mount -o bind $SK_OM_PATH /etc/request-key.d/
+ if grep -q request-key /proc/mounts > /dev/null; then
+ echo "SSK: Request key already mounted."
+ else
+ mount -o bind $SK_OM_PATH /etc/request-key.d/
+ fi
local lgssc_conf_line='create lgssc * * '
lgssc_conf_line+=$(which lgss_keyring)
lgssc_conf_line+=' %o %k %t %d %c %u %g %T %P %S'
-m $SK_PATH/$FSNAME-nmclient.key \
>/dev/null 2>&1"
fi
+ fi
+ if $GSS_SK; then
# mount options for servers and clients
MGS_MOUNT_OPTS=$(add_sk_mntflag $MGS_MOUNT_OPTS)
MDS_MOUNT_OPTS=$(add_sk_mntflag $MDS_MOUNT_OPTS)
$RPC_MODE || echo "Cleaning up Shared Key.."
do_nodes $(comma_list $(all_nodes)) "rm -f \
$SK_PATH/$FSNAME*.key $SK_PATH/nodemap/$FSNAME*.key"
+ do_nodes $(comma_list $(all_nodes)) "keyctl show | \
+ awk '/lustre/ { print \\\$1 }' | xargs -IX keyctl unlink X"
# Remove the mount and clean up the files we added to SK_PATH
- do_nodes $(comma_list $(all_nodes)) "umount \
- /etc/request-key.d/"
+ do_nodes $(comma_list $(all_nodes)) "while grep -q \
+ request-key.d /proc/mounts; do umount \
+ /etc/request-key.d/; done"
do_nodes $(comma_list $(all_nodes)) "rm -f \
$SK_OM_PATH/lgssc.conf"
do_nodes $(comma_list $(all_nodes)) "rmdir $SK_OM_PATH"
set_default_debug_facet $facet
- if [[ $facet == mds* ]]; then
- do_facet $facet \
- lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 2>/dev/null
- fi
-
if [[ $opts =~ .*nosvc.* ]]; then
echo "Start $dm_dev without service"
else
mount_facet ${facet}
RC=$?
- if [[ $facet == mds* ]]; then
- do_facet $facet \
- lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 \
- 2>/dev/null
- fi
-
return $RC
}
fi
}
-# save quota version (both administrative and operational quotas)
-# add an additional parameter if mountpoint is ever different from $MOUNT
-#
-# XXX This function is kept for interoperability with old server (< 2.3.50),
-# it should be removed whenever we drop the interoperability for such
-# server.
-quota_save_version() {
- local fsname=${2:-$FSNAME}
- local spec=$1
- local ver=$(tr -c -d "123" <<< $spec)
- local type=$(tr -c -d "ug" <<< $spec)
-
- [ -n "$ver" -a "$ver" != "3" ] && error "wrong quota version specifier"
-
- [ -n "$type" ] && { $LFS quotacheck -$type $MOUNT || error "quotacheck has failed"; }
-
- do_facet mgs "lctl conf_param ${fsname}-MDT*.mdd.quota_type=$spec"
- local varsvc
- local osts=$(get_facets OST)
- for ost in ${osts//,/ }; do
- varsvc=${ost}_svc
- do_facet mgs "lctl conf_param ${!varsvc}.ost.quota_type=$spec"
- done
-}
-
-# client could mount several lustre
-#
-# XXX This function is kept for interoperability with old server (< 2.3.50),
-# it should be removed whenever we drop the interoperability for such
-# server.
-quota_type() {
- local fsname=${1:-$FSNAME}
- local rc=0
- do_facet $SINGLEMDS lctl get_param mdd.${fsname}-MDT*.quota_type ||
- rc=$?
- do_nodes $(comma_list $(osts_nodes)) \
- lctl get_param obdfilter.${fsname}-OST*.quota_type || rc=$?
- return $rc
-}
-
# get mdt quota type
mdt_quota_type() {
local varsvc=${SINGLEMDS}_svc
# restore old quota type settings
restore_quota() {
if [ "$old_MDT_QUOTA_TYPE" ]; then
- do_facet mgs $LCTL conf_param \
- $FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE
+ if [[ $PERM_CMD == *"set_param -P"* ]]; then
+ do_facet mgs $PERM_CMD \
+ osd-*.$FSNAME-MDT*.quota_slave.enable = \
+ $old_MDT_QUOTA_TYPE
+ else
+ do_facet mgs $PERM_CMD \
+ $FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE
+ fi
fi
if [ "$old_OST_QUOTA_TYPE" ]; then
- do_facet mgs $LCTL conf_param \
- $FSNAME.quota.ost=$old_OST_QUOTA_TYPE
+ if [[ $PERM_CMD == *"set_param -P"* ]]; then
+ do_facet mgs $PERM_CMD \
+ osd-*.$FSNAME-OST*.quota_slave.enable = \
+ $old_OST_QUOTA_TYPE
+ else
+ do_facet mgs $LCTL conf_param \
+ $FSNAME.quota.ost=$old_OST_QUOTA_TYPE
+ fi
fi
}
export old_MDT_QUOTA_TYPE=$mdt_qtype
export old_OST_QUOTA_TYPE=$ost_qtype
- do_facet mgs $LCTL conf_param $FSNAME.quota.mdt=$QUOTA_TYPE ||
- error "set mdt quota type failed"
- do_facet mgs $LCTL conf_param $FSNAME.quota.ost=$QUOTA_TYPE ||
- error "set ost quota type failed"
+ if [[ $PERM_CMD == *"set_param -P"* ]]; then
+ do_facet mgs $PERM_CMD \
+ osd-*.$FSNAME-MDT*.quota_slave.enable=$QUOTA_TYPE
+ do_facet mgs $PERM_CMD \
+ osd-*.$FSNAME-OST*.quota_slave.enable=$QUOTA_TYPE
+ else
+ do_facet mgs $PERM_CMD $FSNAME.quota.mdt=$QUOTA_TYPE ||
+ error "set mdt quota type failed"
+ do_facet mgs $PERM_CMD $FSNAME.quota.ost=$QUOTA_TYPE ||
+ error "set ost quota type failed"
+ fi
local quota_usrs=$QUOTA_USERS
fi
echo "Starting client $clients: $flags $opts $device $mnt"
- if [ -n "$FILESET" -a ! -n "$SKIP_FILESET" ]; then
+ do_nodes $clients mkdir -p $mnt
+ if [ -n "$FILESET" -a -z "$SKIP_FILESET" ]; then
if $GSS_SK && ($SK_UNIQUE_NM || $SK_S2S); then
# Mount with own nodemap key
local i=0
LFS=$LFS \
LCTL=$LCTL \
FSNAME=$FSNAME \
+ MPIRUN=$MPIRUN \
+ MPIRUN_OPTIONS=\\\"$MPIRUN_OPTIONS\\\" \
+ MACHINEFILE_OPTION=\\\"$MACHINEFILE_OPTION\\\" \
+ num_clients=$(get_node_count ${CLIENTS//,/ }) \
+ ior_THREADS=$ior_THREADS ior_iteration=$ior_iteration \
+ ior_blockSize=$ior_blockSize \
+ ior_blockUnit=$ior_blockUnit \
+ ior_xferSize=$ior_xferSize ior_type=$ior_type \
+ ior_DURATION=$ior_DURATION \
+ ior_stripe_params=\\\"$ior_stripe_params\\\" \
+ ior_custom_params=\\\"$ior_custom_param\\\" \
+ mpi_ior_custom_threads=$mpi_ior_custom_threads \
run_${load}.sh" &
local ppid=$!
log "Started client load: ${load} on $client"
# Check for wide striping
if [ $OSTCOUNT -gt 160 ]; then
MDSJOURNALSIZE=${MDSJOURNALSIZE:-4096}
- fs_mkfs_opts+="-O large_xattr"
+ fs_mkfs_opts+="-O ea_inode"
fi
var=${facet}_JRN
if $GSS_SK; then
set_rule $FSNAME any cli2mdt $SK_FLAVOR
set_rule $FSNAME any cli2ost $SK_FLAVOR
- wait_flavor cli2mdt $SK_FLAVOR
- wait_flavor cli2ost $SK_FLAVOR
+ if $SK_SKIPFIRST; then
+ export SK_SKIPFIRST=false
+
+ sleep 30
+ do_nodes $CLIENTS \
+ "lctl set_param osc.*.idle_connect=1"
+ return
+ else
+ wait_flavor cli2mdt $SK_FLAVOR
+ wait_flavor cli2ost $SK_FLAVOR
+ fi
else
set_flavor_all $SEC
fi
error "check $PARAM failed!"
}
+set_persistent_param() {
+ local myfacet=$1
+ local test_param=$2
+ local param=$3
+ local orig=$(do_facet $myfacet "$LCTL get_param -n $test_param")
+
+ if [ $# -gt 3 ]; then
+ local final=$4
+ else
+ local -i final
+ final=$((orig + 5))
+ fi
+
+ if [[ $PERM_CMD == *"set_param -P"* ]]; then
+ echo "Setting $test_param from $orig to $final"
+ do_facet mgs "$PERM_CMD $test_param='$final'" ||
+ error "$PERM_CMD $test_param failed"
+ else
+ echo "Setting $param from $orig to $final"
+ do_facet mgs "$PERM_CMD $param='$final'" ||
+ error "$PERM_CMD $param failed"
+ fi
+}
+
+set_persistent_param_and_check() {
+ local myfacet=$1
+ local test_param=$2
+ local param=$3
+ local orig=$(do_facet $myfacet "$LCTL get_param -n $test_param")
+
+ if [ $# -gt 3 ]; then
+ local final=$4
+ else
+ local -i final
+ final=$((orig + 5))
+ fi
+
+ set_persistent_param $myfacet $test_param $param "$final"
+
+ wait_update_facet $myfacet "$LCTL get_param -n $test_param" "$final" ||
+ error "check $param failed!"
+}
+
init_param_vars () {
TIMEOUT=$(lctl get_param -n timeout)
TIMEOUT=${TIMEOUT:-20}
+ if [ -n $arg1 ]; then
+ [ "$arg1" = "server_only" ] && return
+ fi
+
remote_mds_nodsh && log "Using TIMEOUT=$TIMEOUT" && return 0
TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout")
osc_ensure_active $SINGLEMDS $TIMEOUT
osc_ensure_active client $TIMEOUT
+ $LCTL set_param osc.*.idle_timeout=debug
if [ -n "$(lctl get_param -n mdc.*.connect_flags|grep jobstats)" ]; then
local current_jobid_var=$($LCTL get_param -n jobid_var)
if [ $JOBID_VAR = "existing" ]; then
echo "keeping jobstats as $current_jobid_var"
elif [ $current_jobid_var != $JOBID_VAR ]; then
- echo "seting jobstats to $JOBID_VAR"
+ echo "setting jobstats to $JOBID_VAR"
- set_conf_param_and_check client \
- "$LCTL get_param -n jobid_var" \
- "$FSNAME.sys.jobid_var" $JOBID_VAR
+ set_persistent_param_and_check client \
+ "jobid_var" "$FSNAME.sys.jobid_var" $JOBID_VAR
fi
else
echo "jobstats not supported by server"
fi
fi
- init_gss
if $GSS_SK; then
set_flavor_all null
elif $GSS; then
set_flavor_all $SEC
fi
- if [ -z "$CLIENTONLY" ]; then
- # Enable remote MDT create for testing
- for num in $(seq $MDSCOUNT); do
- do_facet mds$num \
- lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 \
- 2>/dev/null
- done
- fi
-
if [ "$ONLY" == "setup" ]; then
exit 0
fi
drop_request() {
# OBD_FAIL_MDS_ALL_REQUEST_NET
RC=0
- do_facet $SINGLEMDS lctl set_param fail_loc=0x123
+ do_facet $SINGLEMDS lctl set_param fail_val=0 fail_loc=0x123
do_facet client "$1" || RC=$?
do_facet $SINGLEMDS lctl set_param fail_loc=0
return $RC
return $rc
}
-drop_ldlm_reply() {
-#define OBD_FAIL_LDLM_REPLY 0x30c
+drop_mdt_ldlm_reply() {
+#define OBD_FAIL_MDS_LDLM_REPLY_NET 0x157
RC=0
- local list=$(comma_list $(mdts_nodes) $(osts_nodes))
- do_nodes $list lctl set_param fail_loc=0x30c
+ local list=$(comma_list $(mdts_nodes))
+ do_nodes $list lctl set_param fail_loc=0x157
do_facet client "$@" || RC=$?
return $RC
}
-drop_ldlm_reply_once() {
-#define OBD_FAIL_LDLM_REPLY 0x30c
+drop_mdt_ldlm_reply_once() {
+#define OBD_FAIL_MDS_LDLM_REPLY_NET 0x157
RC=0
- local list=$(comma_list $(mdts_nodes) $(osts_nodes))
- do_nodes $list lctl set_param fail_loc=0x8000030c
+ local list=$(comma_list $(mdts_nodes))
+ do_nodes $list lctl set_param fail_loc=0x80000157
do_facet client "$@" || RC=$?
local status=0
local log=$TESTSUITELOG
- [ -f "$log" ] && grep -q FAIL $log && status=1
+ [ -f "$log" ] && grep -qw FAIL $log && status=1
exit $status
}
fi
}
+#
+# Function: skip_env()
+# Purpose: to skip a test during developer testing because some tool
+# is missing, but fail the test in release testing because the test
+# environment is not configured properly".
+#
skip_env () {
$FAIL_ON_SKIP_ENV && error false $@ || skip $@
}
all_mdts_nodes () {
local host
local failover_host
- local nodes="${mds_HOST} ${mdsfailover_HOST}"
+ local nodes
local nodes_sort
local i
nodes="$nodes ${!host} ${!failover_host}"
done
+ [ -n "$nodes" ] || nodes="${mds_HOST} ${mdsfailover_HOST}"
nodes_sort=$(for i in $nodes; do echo $i; done | sort -u)
echo -n $nodes_sort
}
all_osts_nodes () {
local host
local failover_host
- local nodes="${ost_HOST} ${ostfailover_HOST}"
+ local nodes=
local nodes_sort
local i
nodes="$nodes ${!host} ${!failover_host}"
done
+ [ -n "$nodes" ] || nodes="${ost_HOST} ${ostfailover_HOST}"
nodes_sort=$(for i in $nodes; do echo $i; done | sort -u)
echo -n $nodes_sort
}
}
get_clientosc_proc_path() {
- echo "${1}-osc-ffff*"
+ echo "${1}-osc-[-0-9a-f]*"
}
# If the 2.0 MDS was mounted on 1.8 device, then the OSC and LOV names
# wait until all MDTs are in the expected state
for ((num = 1; num <= $MDSCOUNT; num++)); do
local mdtosp=$(get_mdtosc_proc_path mds${num} ${tgt_name})
+ local wait=0
local mproc
if [ $facet = "mds" ]; then
sleep 5
local result=$(do_facet mds${num} "$LCTL get_param -n $mproc")
local max=30
- local wait=0
[ ${PIPESTATUS[0]} = 0 ] || error "Can't read $mproc"
if [ $result -eq $expected ]; then
- echo -n "target updated after"
+ echo -n "target updated after "
echo "$wait sec (got $result)"
break
fi
local clients=${CLIENTS:-$HOSTNAME}
for c in ${clients//,/ }; do
+ # reconnect if idle
+ do_node $c lctl set_param osc.*.idle_connect=1 >/dev/null 2>&1
local output=$(do_node $c lctl get_param -n \
osc.*OST*-osc-[^M][^D][^T]*.$PROC_CLI 2>/dev/null)
local tmpcnt=$(count_flvr "$output" $flavor)
do_facet $SINGLEMDS $LCTL --version)"
log "OSS: $(do_facet ost1 $LCTL lustre_build_version 2> /dev/null ||
do_facet ost1 $LCTL --version)"
+
+ get_lustre_env
}
log_test() {
#
# Get the page size (bytes) on a given facet node.
+# The local client page_size is directly available in PAGE_SIZE.
#
get_page_size() {
local facet=$1
- local size=$(getconf PAGE_SIZE 2>/dev/null)
+ local page_size=$(getconf PAGE_SIZE 2>/dev/null)
- [ -z "$CLIENTONLY" ] && size=$(do_facet $facet getconf PAGE_SIZE)
- echo -n ${size:-4096}
+ [ -z "$CLIENTONLY" -a "$facet" != "client" ] &&
+ page_size=$(do_facet $facet getconf PAGE_SIZE)
+ echo -n ${page_size:-4096}
}
#
echo -n ${size:-0}
}
-# Check whether the "large_xattr" feature is enabled or not.
+# Check whether the "ea_inode" feature is enabled or not, to allow
+# ldiskfs xattrs over one block in size. Allow both the historical
+# Lustre feature name (large_xattr) and the upstream name (ea_inode).
large_xattr_enabled() {
[[ $(facet_fstype $SINGLEMDS) == zfs ]] && return 0
test_mkdir() {
local path
local p_option
- local stripe_count=2
- local stripe_index=-1
+ local dirstripe_count=${DIRSTRIPE_COUNT:-"2"}
+ local dirstripe_index=${DIRSTRIPE_INDEX:-$((base % $MDSCOUNT))}
local OPTIND=1
while getopts "c:i:p" opt; do
case $opt in
- c) stripe_count=$OPTARG;;
- i) stripe_index=$OPTARG;;
+ c) dirstripe_count=$OPTARG;;
+ i) dirstripe_index=$OPTARG;;
p) p_option="-p";;
\?) error "only support -i -c -p";;
esac
if [ $MDSCOUNT -le 1 ]; then
mkdir $path || error "mkdir '$path' failed"
else
- local test_num=$(echo $testnum | sed -e 's/[^0-9]*//g')
local mdt_index
- if [ $stripe_index -eq -1 ]; then
- mdt_index=$((test_num % MDSCOUNT))
+ if [ $dirstripe_index -eq -1 ]; then
+ mdt_index=$((base % MDSCOUNT))
else
- mdt_index=$stripe_index
+ mdt_index=$dirstripe_index
fi
- echo "striped dir -i$mdt_index -c$stripe_count $path"
- $LFS mkdir -i$mdt_index -c$stripe_count $path ||
- error "mkdir -i $mdt_index -c$stripe_count $path failed"
+
+ if (($MDS1_VERSION >= $(version_code 2.8.0))); then
+ if [ $dirstripe_count -eq -1 ]; then
+ dirstripe_count=$((RANDOM % MDSCOUNT + 1))
+ fi
+ else
+ dirstripe_count=1
+ fi
+
+ echo "striped dir -i$mdt_index -c$dirstripe_count $path"
+ $LFS mkdir -i$mdt_index -c$dirstripe_count $path ||
+ error "mkdir -i $mdt_index -c$dirstripe_count $path failed"
fi
}
[[ -z "$file" || -z "$expected" ]] &&
error "check_stripe_count: invalid argument"
- local cmd="$GETSTRIPE -c $file"
+ local cmd="$LFS getstripe -c $file"
actual=$($cmd) || error "$cmd failed"
actual=${actual%% *}
if [[ $actual -ne $expected ]]; then
- [[ $expected -eq -1 ]] ||
- error "$cmd wrong: found $actual, expected $expected"
- [[ $actual -eq $OSTCOUNT ]] ||
- error "$cmd wrong: found $actual, expected $OSTCOUNT"
+ [[ $expected -eq -1 ]] || { $LFS getstripe $file;
+ error "$cmd not expected ($expected): found $actual"; }
+ [[ $actual -eq $OSTCOUNT ]] || { $LFS getstripe $file;
+ error "$cmd not OST count ($OSTCOUNT): found $actual"; }
fi
}
stack_trap "restore_layout $dir $layout" EXIT
}
+
+verify_yaml_layout() {
+ local src=$1
+ local dst=$2
+ local temp=$3
+ local msg_prefix=$4
+
+ echo "getstripe --yaml $src"
+ $LFS getstripe --yaml $src > $temp || error "getstripe $src failed"
+ echo "setstripe --yaml=$temp $dst"
+ $LFS setstripe --yaml=$temp $dst|| error "setstripe $dst failed"
+
+ echo "compare"
+ local layout1=$(get_layout_param $src)
+ local layout2=$(get_layout_param $dst)
+ # compare their layout info
+ [ "$layout1" == "$layout2" ] ||
+ error "$msg_prefix $src/$dst layouts are not equal"
+}
+
git://git.whamcloud.com / fs / lustre-release.git / blobdiff