X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=9caa3f7e0d84f7ded04c30d8b8ec5e814059388a;hp=834d24382c80f330333ecc4d68ddef788e00735f;hb=73ecd83a24e220b8097facf5bf3f5d93f523702c;hpb=5a6ceb664f07812c351786c1043da71ff5027f8c diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 834d243..9caa3f7 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -22,8 +22,10 @@ export SK_UNIQUE_NM=${SK_UNIQUE_NM:-false} export SK_S2S=${SK_S2S:-false} export SK_S2SNM=${SK_S2SNM:-TestFrameNM} export SK_S2SNMCLI=${SK_S2SNMCLI:-TestFrameNMCli} +export SK_SKIPFIRST=${SK_SKIPFIRST:-true} export IDENTITY_UPCALL=default export QUOTA_AUTO=1 +export FLAKEY=${FLAKEY:-true} # specify environment variable containing batch job name for server statistics export JOBID_VAR=${JOBID_VAR:-"procname_uid"} # or "existing" or "disable" @@ -44,7 +46,7 @@ LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} . $LUSTRE/tests/functions.sh . $LUSTRE/tests/yaml.sh -export LD_LIBRARY_PATH=${LUSTRE}/utils:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=${LUSTRE}/utils/.libs:${LUSTRE}/utils:${LD_LIBRARY_PATH} LUSTRE_TESTS_CFG_DIR=${LUSTRE_TESTS_CFG_DIR:-${LUSTRE}/tests/cfg} @@ -155,6 +157,7 @@ init_test_env() { export DO_CLEANUP=${DO_CLEANUP:-true} export KEEP_ZPOOL=${KEEP_ZPOOL:-false} export CLEANUP_DM_DEV=false + export PAGE_SIZE=$(get_page_size client) export MKE2FS=$MKE2FS if [ -z "$MKE2FS" ]; then @@ -273,6 +276,8 @@ init_test_env() { SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"} GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"} + export PERM_CMD=${PERM_CMD:-"$LCTL conf_param"} + export L_GETIDENTITY=${L_GETIDENTITY:-"$LUSTRE/utils/l_getidentity"} if [ ! -f "$L_GETIDENTITY" ]; then if `which l_getidentity > /dev/null 2>&1`; then @@ -303,6 +308,10 @@ init_test_env() { [ ! -f "$LR_READER" ] && export LR_READER=$(which lr_reader 2> /dev/null) [ -z "$LR_READER" ] && export LR_READER="/usr/sbin/lr_reader" + export LSOM_SYNC=${LSOM_SYNC:-"$LUSTRE/utils/llsom_sync"} + [ ! -f "$LSOM_SYNC" ] && + export LSOM_SYNC=$(which llsom_sync 2> /dev/null) + [ -z "$LSOM_SYNC" ] && export LSOM_SYNC="/usr/sbin/llsom_sync" export NAME=${NAME:-local} export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"} [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && @@ -506,62 +515,91 @@ lustre_insmod() { # /etc/modprobe.conf, from /etc/modprobe.d/Lustre, or else none will be used. # load_module() { - local optvar - EXT=".ko" - module=$1 - shift - BASE=$(basename $module $EXT) - - module_loaded ${BASE} && return - - # If no module arguments were passed, get them from $MODOPTS_, - # else from modprobe.conf - if [ $# -eq 0 ]; then - # $MODOPTS_; we could use associative arrays, but that's not in - # Bash until 4.x, so we resort to eval. - optvar="MODOPTS_$(basename $module | tr a-z A-Z)" - eval set -- \$$optvar - if [ $# -eq 0 -a -n "$MODPROBECONF" ]; then - # Nothing in $MODOPTS_; try modprobe.conf - local opt - opt=$(awk -v var="^options $BASE" '$0 ~ var \ - {gsub("'"options $BASE"'",""); print}' $MODPROBECONF) - set -- $(echo -n $opt) - - # Ensure we have accept=all for lnet - if [ $(basename $module) = lnet ]; then - # OK, this is a bit wordy... - local arg accept_all_present=false - - for arg in "$@"; do - [ "$arg" = accept=all ] && \ - accept_all_present=true - done - $accept_all_present || set -- "$@" accept=all + local module=$1 # '../libcfs/libcfs/libcfs', 'obdclass/obdclass', ... + shift + local ext=".ko" + local base=$(basename $module $ext) + local path + local -A module_is_loaded_aa + local optvar + local mod + + for mod in $(lsmod | awk '{ print $1; }'); do + module_is_loaded_aa[${mod//-/_}]=true + done + + module_is_loaded() { + ${module_is_loaded_aa[${1//-/_}]:-false} + } + + if module_is_loaded $base; then + return + fi + + if [[ -f $LUSTRE/$module$ext ]]; then + path=$LUSTRE/$module$ext + elif [[ "$base" == lnet_selftest ]] && + [[ -f $LUSTRE/../lnet/selftest/$base$ext ]]; then + path=$LUSTRE/../lnet/selftest/$base$ext + else + path='' + fi + + if [[ -n "$path" ]]; then + # Try to load any non-Lustre modules that $module depends on. + for mod in $(modinfo --field=depends $path | tr ',' ' '); do + if ! module_is_loaded $mod; then + modprobe $mod + fi + done + fi + + # If no module arguments were passed then get them from + # $MODOPTS_, otherwise from modprobe.conf. + if [ $# -eq 0 ]; then + # $MODOPTS_; we could use associative arrays, but that's + # not in Bash until 4.x, so we resort to eval. + optvar="MODOPTS_$(basename $module | tr a-z A-Z)" + eval set -- \$$optvar + if [ $# -eq 0 -a -n "$MODPROBECONF" ]; then + # Nothing in $MODOPTS_; try modprobe.conf + local opt + opt=$(awk -v var="^options $base" '$0 ~ var \ + {gsub("'"options $base"'",""); print}' \ + $MODPROBECONF) + set -- $(echo -n $opt) + + # Ensure we have accept=all for lnet + if [[ "$base" == lnet ]]; then + # OK, this is a bit wordy... + local arg accept_all_present=false + + for arg in "$@"; do + [[ "$arg" == accept=all ]] && + accept_all_present=true + done + + $accept_all_present || set -- "$@" accept=all + fi + + export $optvar="$*" fi - export $optvar="$*" - fi - fi + fi - [ $# -gt 0 ] && echo "${module} options: '$*'" + [ $# -gt 0 ] && echo "${module} options: '$*'" # Note that insmod will ignore anything in modprobe.conf, which is why - # we're passing options on the command-line. - if [[ "$BASE" == "lnet_selftest" ]] && - [[ -f ${LUSTRE}/../lnet/selftest/${module}${EXT} ]]; then - lustre_insmod ${LUSTRE}/../lnet/selftest/${module}${EXT} - elif [[ -f ${LUSTRE}/${module}${EXT} ]]; then - [[ "$BASE" != "ptlrpc_gss" ]] || modprobe sunrpc - lustre_insmod ${LUSTRE}/${module}${EXT} "$@" - else - # must be testing a "make install" or "rpm" installation - # note failed to load ptlrpc_gss is considered not fatal - if [[ "$BASE" == "ptlrpc_gss" ]]; then - modprobe $BASE "$@" 2>/dev/null || - echo "gss/krb5 is not supported" - else - modprobe $BASE "$@" + # we're passing options on the command-line. If $path does not exist + # then we must be testing a "make install" or"rpm" installation. Also + # note that failing to load ptlrpc_gss is not considered fatal. + if [[ -n "$path" ]]; then + lustre_insmod $path "$@" + elif [[ "$base" == ptlrpc_gss ]]; then + if ! modprobe $base "$@" 2>/dev/null; then + echo "gss/krb5 is not supported" fi + else + modprobe $base "$@" fi } @@ -572,6 +610,15 @@ load_modules_local() { return 0 fi + # Create special udev test rules on every node + if [ -f $LUSTRE/lustre/conf/99-lustre.rules ]; then { + sed -e 's|/usr/sbin/lctl|$LCTL|g' $LUSTRE/lustre/conf/99-lustre.rules > /etc/udev/rules.d/99-lustre-test.rules + } else { + echo "SUBSYSTEM==\"lustre\", ACTION==\"change\", ENV{PARAM}==\"?*\", RUN+=\"$LCTL set_param '\$env{PARAM}=\$env{SETTING}'\"" > /etc/udev/rules.d/99-lustre-test.rules + } fi + udevadm control --reload-rules + udevadm trigger + echo Loading modules from $LUSTRE local ncpus @@ -634,23 +681,12 @@ load_modules_local() { load_module mgc/mgc load_module obdecho/obdecho if ! client_only; then - SYMLIST=/proc/kallsyms - grep -q crc16 $SYMLIST || - { modprobe crc16 2>/dev/null || true; } - grep -q -w jbd2 $SYMLIST || - { modprobe jbd2 2>/dev/null || true; } load_module lfsck/lfsck [ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then - lsmod | grep zfs >&/dev/null || modprobe zfs load_module osd-zfs/osd_zfs - fi - if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then - grep -q exportfs_decode_fh $SYMLIST || - { modprobe exportfs 2> /dev/null || true; } - grep -q -w mbcache $SYMLIST || - { modprobe mbcache 2>/dev/null || true; } + elif [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then load_module ../ldiskfs/ldiskfs load_module osd-ldiskfs/osd_ldiskfs fi @@ -728,12 +764,20 @@ unload_modules() { $LUSTRE_RMMOD ldiskfs || return 2 + [ -f /etc/udev/rules.d/99-lustre-test.rules ] && + rm /etc/udev/rules.d/99-lustre-test.rules + udevadm control --reload-rules + udevadm trigger + if $LOAD_MODULES_REMOTE; then local list=$(comma_list $(remote_nodes_list)) if [ -n "$list" ]; then echo "unloading modules on: '$list'" do_rpc_nodes "$list" $LUSTRE_RMMOD ldiskfs do_rpc_nodes "$list" check_mem_leak + do_rpc_nodes "$list" "rm /etc/udev/rules.d/99-lustre-test.rules" + do_rpc_nodes "$list" "udevadm control --reload-rules" + do_rpc_nodes "$list" "udevadm trigger" fi fi @@ -762,7 +806,7 @@ fs_log_size() { ;; esac - echo -n $size + echo -n $((size * MDSCOUNT)) } fs_inode_ksize() { @@ -918,7 +962,11 @@ init_gss() { # security ctx config for keyring SK_NO_KEY=false mkdir -p $SK_OM_PATH - mount -o bind $SK_OM_PATH /etc/request-key.d/ + if grep -q request-key /proc/mounts > /dev/null; then + echo "SSK: Request key already mounted." + else + mount -o bind $SK_OM_PATH /etc/request-key.d/ + fi local lgssc_conf_line='create lgssc * * ' lgssc_conf_line+=$(which lgss_keyring) lgssc_conf_line+=' %o %k %t %d %c %u %g %T %P %S' @@ -994,6 +1042,8 @@ init_gss() { -m $SK_PATH/$FSNAME-nmclient.key \ >/dev/null 2>&1" fi + fi + if $GSS_SK; then # mount options for servers and clients MGS_MOUNT_OPTS=$(add_sk_mntflag $MGS_MOUNT_OPTS) MDS_MOUNT_OPTS=$(add_sk_mntflag $MDS_MOUNT_OPTS) @@ -1027,9 +1077,12 @@ cleanup_sk() { $RPC_MODE || echo "Cleaning up Shared Key.." do_nodes $(comma_list $(all_nodes)) "rm -f \ $SK_PATH/$FSNAME*.key $SK_PATH/nodemap/$FSNAME*.key" + do_nodes $(comma_list $(all_nodes)) "keyctl show | \ + awk '/lustre/ { print \\\$1 }' | xargs -IX keyctl unlink X" # Remove the mount and clean up the files we added to SK_PATH - do_nodes $(comma_list $(all_nodes)) "umount \ - /etc/request-key.d/" + do_nodes $(comma_list $(all_nodes)) "while grep -q \ + request-key.d /proc/mounts; do umount \ + /etc/request-key.d/; done" do_nodes $(comma_list $(all_nodes)) "rm -f \ $SK_OM_PATH/lgssc.conf" do_nodes $(comma_list $(all_nodes)) "rmdir $SK_OM_PATH" @@ -1582,6 +1635,7 @@ is_dm_flakey_dev() { dm_flakey_supported() { local facet=$1 + $FLAKEY || return 1 do_facet $facet "modprobe dm-flakey; $DMSETUP targets | grep -q flakey" &> /dev/null } @@ -1998,46 +2052,6 @@ stop() { fi } -# save quota version (both administrative and operational quotas) -# add an additional parameter if mountpoint is ever different from $MOUNT -# -# XXX This function is kept for interoperability with old server (< 2.3.50), -# it should be removed whenever we drop the interoperability for such -# server. -quota_save_version() { - local fsname=${2:-$FSNAME} - local spec=$1 - local ver=$(tr -c -d "123" <<< $spec) - local type=$(tr -c -d "ug" <<< $spec) - - [ -n "$ver" -a "$ver" != "3" ] && error "wrong quota version specifier" - - [ -n "$type" ] && { $LFS quotacheck -$type $MOUNT || error "quotacheck has failed"; } - - do_facet mgs "lctl conf_param ${fsname}-MDT*.mdd.quota_type=$spec" - local varsvc - local osts=$(get_facets OST) - for ost in ${osts//,/ }; do - varsvc=${ost}_svc - do_facet mgs "lctl conf_param ${!varsvc}.ost.quota_type=$spec" - done -} - -# client could mount several lustre -# -# XXX This function is kept for interoperability with old server (< 2.3.50), -# it should be removed whenever we drop the interoperability for such -# server. -quota_type() { - local fsname=${1:-$FSNAME} - local rc=0 - do_facet $SINGLEMDS lctl get_param mdd.${fsname}-MDT*.quota_type || - rc=$? - do_nodes $(comma_list $(osts_nodes)) \ - lctl get_param obdfilter.${fsname}-OST*.quota_type || rc=$? - return $rc -} - # get mdt quota type mdt_quota_type() { local varsvc=${SINGLEMDS}_svc @@ -2056,12 +2070,24 @@ ost_quota_type() { # restore old quota type settings restore_quota() { if [ "$old_MDT_QUOTA_TYPE" ]; then - do_facet mgs $LCTL conf_param \ - $FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE + if [[ $PERM_CMD = *"set_param -P"* ]]; then + do_facet mgs $PERM_CMD \ + osd-*.$FSNAME-MDT*.quota_slave.enable = \ + $old_MDT_QUOTA_TYPE + else + do_facet mgs $PERM_CMD \ + $FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE + fi fi if [ "$old_OST_QUOTA_TYPE" ]; then - do_facet mgs $LCTL conf_param \ - $FSNAME.quota.ost=$old_OST_QUOTA_TYPE + if [[ $PERM_CMD = *"set_param -P"* ]]; then + do_facet mgs $PERM_CMD \ + osd-*.$FSNAME-OST*.quota_slave.enable = \ + $old_OST_QUOTA_TYPE + else + do_facet mgs $LCTL conf_param \ + $FSNAME.quota.ost=$old_OST_QUOTA_TYPE + fi fi } @@ -2114,10 +2140,17 @@ setup_quota(){ export old_MDT_QUOTA_TYPE=$mdt_qtype export old_OST_QUOTA_TYPE=$ost_qtype - do_facet mgs $LCTL conf_param $FSNAME.quota.mdt=$QUOTA_TYPE || - error "set mdt quota type failed" - do_facet mgs $LCTL conf_param $FSNAME.quota.ost=$QUOTA_TYPE || - error "set ost quota type failed" + if [[ $PERM_CMD = *"set_param -P"* ]]; then + do_facet mgs $PERM_CMD \ + osd-*.$FSNAME-MDT*.quota_slave.enable=$QUOTA_TYPE + do_facet mgs $PERM_CMD \ + osd-*.$FSNAME-OST*.quota_slave.enable=$QUOTA_TYPE + else + do_facet mgs $PERM_CMD $FSNAME.quota.mdt=$QUOTA_TYPE || + error "set mdt quota type failed" + do_facet mgs $PERM_CMD $FSNAME.quota.ost=$QUOTA_TYPE || + error "set ost quota type failed" + fi local quota_usrs=$QUOTA_USERS @@ -4216,7 +4249,7 @@ mkfs_opts() { # Check for wide striping if [ $OSTCOUNT -gt 160 ]; then MDSJOURNALSIZE=${MDSJOURNALSIZE:-4096} - fs_mkfs_opts+="-O large_xattr" + fs_mkfs_opts+="-O ea_inode" fi var=${facet}_JRN @@ -4624,8 +4657,17 @@ setupall() { if $GSS_SK; then set_rule $FSNAME any cli2mdt $SK_FLAVOR set_rule $FSNAME any cli2ost $SK_FLAVOR - wait_flavor cli2mdt $SK_FLAVOR - wait_flavor cli2ost $SK_FLAVOR + if $SK_SKIPFIRST; then + export SK_SKIPFIRST=false + + sleep 30 + do_nodes $CLIENTS \ + "lctl set_param osc.*.idle_connect=1" + return + else + wait_flavor cli2mdt $SK_FLAVOR + wait_flavor cli2ost $SK_FLAVOR + fi else set_flavor_all $SEC fi @@ -4769,10 +4811,57 @@ set_conf_param_and_check() { error "check $PARAM failed!" } +set_persistent_param() { + local myfacet=$1 + local test_param=$2 + local param=$3 + local orig=$(do_facet $myfacet "$LCTL get_param -n $test_param") + + if [ $# -gt 3 ]; then + local final=$4 + else + local -i final + final=$((orig + 5)) + fi + + if [[ $PERM_CMD = *"set_param -P"* ]]; then + echo "Setting $test_param from $orig to $final" + do_facet mgs "$PERM_CMD $test_param='$final'" || + error "$PERM_CMD $test_param failed" + else + echo "Setting $param from $orig to $final" + do_facet mgs "$PERM_CMD $param='$final'" || + error "$PERM_CMD $param failed" + fi +} + +set_persistent_param_and_check() { + local myfacet=$1 + local test_param=$2 + local param=$3 + local orig=$(do_facet $myfacet "$LCTL get_param -n $test_param") + + if [ $# -gt 3 ]; then + local final=$4 + else + local -i final + final=$((orig + 5)) + fi + + set_persistent_param $myfacet $test_param $param "$final" + + wait_update_facet $myfacet "$LCTL get_param -n $test_param" "$final" || + error "check $param failed!" +} + init_param_vars () { TIMEOUT=$(lctl get_param -n timeout) TIMEOUT=${TIMEOUT:-20} + if [ -n $arg1 ]; then + [ "$arg1" = "server_only" ] && return + fi + remote_mds_nodsh && log "Using TIMEOUT=$TIMEOUT" && return 0 TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout") @@ -4780,6 +4869,7 @@ init_param_vars () { osc_ensure_active $SINGLEMDS $TIMEOUT osc_ensure_active client $TIMEOUT + $LCTL set_param osc.*.idle_timeout=debug if [ -n "$(lctl get_param -n mdc.*.connect_flags|grep jobstats)" ]; then local current_jobid_var=$($LCTL get_param -n jobid_var) @@ -4787,11 +4877,10 @@ init_param_vars () { if [ $JOBID_VAR = "existing" ]; then echo "keeping jobstats as $current_jobid_var" elif [ $current_jobid_var != $JOBID_VAR ]; then - echo "seting jobstats to $JOBID_VAR" + echo "setting jobstats to $JOBID_VAR" - set_conf_param_and_check client \ - "$LCTL get_param -n jobid_var" \ - "$FSNAME.sys.jobid_var" $JOBID_VAR + set_persistent_param_and_check client \ + "jobid_var" "$FSNAME.sys.jobid_var" $JOBID_VAR fi else echo "jobstats not supported by server" @@ -4989,7 +5078,6 @@ check_and_setup_lustre() { fi fi - init_gss if $GSS_SK; then set_flavor_all null elif $GSS; then @@ -7112,6 +7200,7 @@ wait_osp_active() { # wait until all MDTs are in the expected state for ((num = 1; num <= $MDSCOUNT; num++)); do local mdtosp=$(get_mdtosc_proc_path mds${num} ${tgt_name}) + local wait=0 local mproc if [ $facet = "mds" ]; then @@ -7126,7 +7215,6 @@ wait_osp_active() { sleep 5 local result=$(do_facet mds${num} "$LCTL get_param -n $mproc") local max=30 - local wait=0 [ ${PIPESTATUS[0]} = 0 ] || error "Can't read $mproc" if [ $result -eq $expected ]; then @@ -7555,6 +7643,8 @@ flvr_cnt_cli2ost() local clients=${CLIENTS:-$HOSTNAME} for c in ${clients//,/ }; do + # reconnect if idle + do_node $c lctl set_param osc.*.idle_connect=1 >/dev/null 2>&1 local output=$(do_node $c lctl get_param -n \ osc.*OST*-osc-[^M][^D][^T]*.$PROC_CLI 2>/dev/null) local tmpcnt=$(count_flvr "$output" $flavor) @@ -8039,13 +8129,15 @@ get_obd_size() { # # Get the page size (bytes) on a given facet node. +# The local client page_size is directly available in PAGE_SIZE. # get_page_size() { local facet=$1 - local size=$(getconf PAGE_SIZE 2>/dev/null) + local page_size=$(getconf PAGE_SIZE 2>/dev/null) - [ -z "$CLIENTONLY" ] && size=$(do_facet $facet getconf PAGE_SIZE) - echo -n ${size:-4096} + [ -z "$CLIENTONLY" -a "$facet" != "client" ] && + page_size=$(do_facet $facet getconf PAGE_SIZE) + echo -n ${page_size:-4096} } # @@ -8072,7 +8164,9 @@ get_block_size() { echo -n ${size:-0} } -# Check whether the "large_xattr" feature is enabled or not. +# Check whether the "ea_inode" feature is enabled or not, to allow +# ldiskfs xattrs over one block in size. Allow both the historical +# Lustre feature name (large_xattr) and the upstream name (ea_inode). large_xattr_enabled() { [[ $(facet_fstype $SINGLEMDS) == zfs ]] && return 0