+statx_supported() {
+ $STATX --quiet --version
+ return $?
+}
+
+# lfs rm_entry is disabled on native client
+is_rmentry_supported() {
+ $LFS rm_entry $DIR/dir/not/exists > /dev/null
+ # is return code ENOENT?
+ (( $? == 2 ))
+}
+
+#
+# wrappers for createmany and unlinkmany
+# to set debug=0 if number of creates is high enough
+# this is to speedup testing
+#
+function createmany() {
+ local count=${!#}
+ local rc
+
+ if (( count > 100 )); then
+ debugsave
+ do_nodes $(comma_list $(all_nodes)) $LCTL set_param -n debug=0
+ fi
+ $LUSTRE/tests/createmany $*
+ rc=$?
+ debugrestore > /dev/null
+
+ return $rc
+}
+
+function unlinkmany() {
+ local count=${!#}
+ local rc
+
+ if (( count > 100 )); then
+ debugsave
+ do_nodes $(comma_list $(all_nodes)) $LCTL set_param -n debug=0
+ fi
+ $LUSTRE/tests/unlinkmany $*
+ rc=$?
+ debugrestore > /dev/null
+
+ return $rc
+}
+
+# Check if fallocate on facet is working. Returns fallocate mode if enabled.
+# Takes optional facet name as argument, to allow separate MDS/OSS checks.
+function check_fallocate_supported()
+{
+ local facet=${1:-ost1}
+ local supported="FALLOCATE_SUPPORTED_$facet"
+ local fstype="${facet}_FSTYPE"
+
+ if [[ -n "${!supported}" ]]; then
+ echo "${!supported}"
+ return 0
+ fi
+ if [[ -z "${!fstype}" ]]; then
+ eval export $fstype=$(facet_fstype $facet)
+ fi
+ if [[ "${!fstype}" != "ldiskfs" ]]; then
+ echo "fallocate on ${!fstype} doesn't consume space" 1>&2
+ return 1
+ fi
+
+ local fa_mode="osd-ldiskfs.$(facet_svc $facet).fallocate_zero_blocks"
+ local mode=$(do_facet $facet $LCTL get_param -n $fa_mode 2>/dev/null |
+ head -n 1)
+
+ if [[ -z "$mode" ]]; then
+ echo "fallocate not supported on $facet" 1>&2
+ return 1
+ fi
+ eval export $supported="$mode"
+
+ echo ${!supported}
+ return 0
+}
+
+# Check if fallocate supported on OSTs, enable if unset, skip if unavailable.
+# Takes optional facet name as argument.
+function check_fallocate_or_skip()
+{
+ local facet=$1
+
+ check_fallocate_supported $1 || skip "fallocate not supported"
+}
+
+# Check if fallocate supported on OSTs, enable if unset, default mode=0
+# Optionally pass the OST fallocate mode (0=unwritten extents, 1=zero extents)
+function check_set_fallocate()
+{
+ local new_mode="$1"
+ local fa_mode="osd-ldiskfs.*.fallocate_zero_blocks"
+ local old_mode="$(check_fallocate_supported)"
+
+ [[ -n "$old_mode" ]] || { echo "fallocate not supported"; return 1; }
+ [[ -z "$new_mode" && "$old_mode" != "-1" ]] &&
+ { echo "keep default fallocate mode: $old_mode"; return 0; }
+ [[ "$new_mode" && "$old_mode" == "$new_mode" ]] &&
+ { echo "keep current fallocate mode: $old_mode"; return 0; }
+ local osts=$(comma_list $(osts_nodes))
+
+ stack_trap "do_nodes $osts $LCTL set_param $fa_mode=$old_mode"
+ do_nodes $osts $LCTL set_param $fa_mode=${new_mode:-0} ||
+ error "set $fa_mode=$new_mode"
+}
+
+# Check if fallocate supported on OSTs, enable if unset, skip if unavailable
+function check_set_fallocate_or_skip()
+{
+ check_set_fallocate || skip "need >= 2.13.57 and ldiskfs for fallocate"
+}
+
+function disable_opencache()
+{
+ local state=$($LCTL get_param -n "llite.*.opencache_threshold_count" |
+ head -1)
+
+ test -z "${saved_OPENCACHE_value}" &&
+ export saved_OPENCACHE_value="$state"
+
+ [[ "$state" = "off" ]] && return
+
+ $LCTL set_param -n "llite.*.opencache_threshold_count"=off
+}
+
+function set_opencache()
+{
+ local newvalue="$1"
+ local state=$($LCTL get_param -n "llite.*.opencache_threshold_count")
+
+ [[ -n "$newvalue" ]] || return
+
+ [[ -n "${saved_OPENCACHE_value}" ]] ||
+ export saved_OPENCACHE_value="$state"
+
+ $LCTL set_param -n "llite.*.opencache_threshold_count"=$newvalue
+}
+
+
+
+function restore_opencache()
+{
+ [[ -z "${saved_OPENCACHE_value}" ]] ||
+ $LCTL set_param -n "llite.*.opencache_threshold_count"=${saved_OPENCACHE_value}
+}
+
+# LU-13417: XXX lots of tests assume the directory to be created under MDT0,
+# created on MDT0, use this function to create directory on specific MDT
+# explicitly, and set default LMV to create subdirs on the same MDT too.
+mkdir_on_mdt() {
+ local mdt
+ local OPTIND=1
+
+ while getopts "i:" opt $*; do
+ case $opt in
+ i) mdt=$OPTARG;;
+ esac
+ done
+
+ shift $((OPTIND - 1))
+
+ $LFS mkdir -i $mdt -c 1 $*
+}
+
+mkdir_on_mdt0() {
+ mkdir_on_mdt -i0 $*
+}
+
+# Wait for nodemap synchronization
+wait_nm_sync() {
+ local nodemap_name=$1
+ local key=$2
+ local value=$3
+ local opt=$4
+ local proc_param
+ local is_active=$(do_facet mgs $LCTL get_param -n nodemap.active)
+ local max_retries=20
+ local is_sync
+ local out1=""
+ local out2
+ local mgs_ip=$(host_nids_address $mgs_HOST $NETTYPE | cut -d' ' -f1)
+ local i
+
+ if [ "$nodemap_name" == "active" ]; then
+ proc_param="active"
+ elif [ -z "$key" ]; then
+ proc_param=${nodemap_name}
+ else
+ proc_param="${nodemap_name}.${key}"
+ fi
+ if [ "$opt" == "inactive" ]; then
+ # check nm sync even if nodemap is not activated
+ is_active=1
+ opt=""
+ fi
+ (( is_active == 0 )) && [ "$proc_param" != "active" ] && return
+
+ if [ -z "$value" ]; then
+ out1=$(do_facet mgs $LCTL get_param $opt \
+ nodemap.${proc_param} 2>/dev/null)
+ echo "On MGS ${mgs_ip}, ${proc_param} = $out1"
+ else
+ out1=$value;
+ fi
+
+ # if servers run on the same node, it is impossible to tell if they get
+ # synced with the mgs, so just wait an arbitrary 10 seconds
+ if [ $(facet_active_host mgs) == $(facet_active_host mds) ] &&
+ [ $(facet_active_host mgs) == $(facet_active_host ost1) ]; then
+ echo "waiting 10 secs for sync"
+ sleep 10
+ return
+ fi
+
+ # wait up to 10 seconds for other servers to sync with mgs
+ for i in $(seq 1 10); do
+ for node in $(all_server_nodes); do
+ local node_ip=$(host_nids_address $node $NETTYPE |
+ cut -d' ' -f1)
+
+ is_sync=true
+ if [ -z "$value" ]; then
+ [ $node_ip == $mgs_ip ] && continue
+ fi
+
+ out2=$(do_node $node $LCTL get_param $opt \
+ nodemap.$proc_param 2>/dev/null)
+ echo "On $node ${node_ip}, ${proc_param} = $out2"
+ [ "$out1" != "$out2" ] && is_sync=false && break
+ done
+ $is_sync && break
+ sleep 1
+ done
+ if ! $is_sync; then
+ echo MGS
+ echo $out1
+ echo OTHER - IP: $node_ip
+ echo $out2
+ error "mgs and $nodemap_name ${key} mismatch, $i attempts"
+ fi
+ echo "waited $((i - 1)) seconds for sync"
+}
+
+consume_precreations() {
+ local dir=$1
+ local mfacet=$2
+ local OSTIDX=$3
+ local extra=${4:-2}
+ local OST=$(ostname_from_index $OSTIDX $dir)
+
+ mkdir_on_mdt -i $(facet_index $mfacet) $dir/${OST}
+ $LFS setstripe -i $OSTIDX -c 1 ${dir}/${OST}
+
+ # on the mdt's osc
+ local mdtosc_proc=$(get_mdtosc_proc_path $mfacet $OST)
+ local last_id=$(do_facet $mfacet $LCTL get_param -n \
+ osp.$mdtosc_proc.prealloc_last_id)
+ local next_id=$(do_facet $mfacet $LCTL get_param -n \
+ osp.$mdtosc_proc.prealloc_next_id)
+ echo "Creating to objid $last_id on ost $OST..."
+ createmany -o $dir/${OST}/f $next_id $((last_id - next_id + extra))
+}
+
+__exhaust_precreations() {
+ local OSTIDX=$1
+ local FAILLOC=$2
+ local FAILIDX=${3:-$OSTIDX}
+ local ofacet=ost$((OSTIDX + 1))
+
+ mkdir_on_mdt0 $DIR/$tdir
+ local mdtidx=$($LFS getstripe -m $DIR/$tdir)
+ local mfacet=mds$((mdtidx + 1))
+ echo OSTIDX=$OSTIDX MDTIDX=$mdtidx
+
+ local mdtosc_proc=$(get_mdtosc_proc_path $mfacet)
+ do_facet $mfacet $LCTL get_param osp.$mdtosc_proc.prealloc*
+
+#define OBD_FAIL_OST_ENOSPC 0x215
+ do_facet $ofacet $LCTL set_param fail_val=$FAILIDX fail_loc=0x215
+
+ consume_precreations $DIR/$tdir $mfacet $OSTIDX
+
+ do_facet $mfacet $LCTL get_param osp.$mdtosc_proc.prealloc*
+ do_facet $ofacet $LCTL set_param fail_loc=$FAILLOC
+}
+
+exhaust_precreations() {
+ __exhaust_precreations $1 $2 $3
+ sleep_maxage
+}
+
+exhaust_all_precreations() {
+ local i
+ for (( i=0; i < OSTCOUNT; i++ )) ; do
+ __exhaust_precreations $i $1 -1
+ done
+ sleep_maxage
+}
+
+force_new_seq_ost() {
+ local dir=$1
+ local mfacet=$2
+ local OSTIDX=$3
+ local OST=$(ostname_from_index $OSTIDX)
+ local mdtosc_proc=$(get_mdtosc_proc_path $mfacet $OST)
+
+ do_facet $mfacet $LCTL set_param \
+ osp.$mdtosc_proc.prealloc_force_new_seq=1
+ # consume preallocated objects, to wake up precreate thread
+ consume_precreations $dir $mfacet $OSTIDX
+ do_facet $mfacet $LCTL set_param \
+ osp.$mdtosc_proc.prealloc_force_new_seq=0
+}
+
+force_new_seq() {
+ local mfacet=$1
+ local MDTIDX=$(facet_index $mfacet)
+ local MDT=$(mdtname_from_index $MDTIDX $DIR)
+ local i
+
+ mkdir_on_mdt -i $MDTIDX $DIR/${MDT}
+ for (( i=0; i < OSTCOUNT; i++ )) ; do
+ force_new_seq_ost $DIR/${MDT} $mfacet $i &
+ done
+ wait
+ rm -rf $DIR/${MDT}
+}
+
+force_new_seq_all() {
+ local i
+
+ for (( i=0; i < MDSCOUNT; i++ )) ; do
+ force_new_seq mds$((i + 1)) &
+ done
+ wait
+ sleep_maxage
+}
+
+ost_set_temp_seq_width_all() {
+ local osts=$(comma_list $(osts_nodes))
+ local width=$(do_facet ost1 $LCTL get_param -n seq.*OST0000-super.width)
+
+ do_nodes $osts $LCTL set_param seq.*OST*-super.width=$1
+ stack_trap "do_nodes $osts $LCTL set_param seq.*OST*-super.width=$width"
+}
+
+verify_yaml_available() {
+ python3 -c "import yaml; yaml.safe_load('''a: b''')"
+}
+
+verify_yaml() {
+ python3 -c "import sys, yaml; obj = yaml.safe_load(sys.stdin)"
+}
+
+verify_compare_yaml() {
+ python3 -c "import sys, yaml; f=open(\"$1\", \"r\"); obj1 = yaml.safe_load(f); f=open(\"$2\", \"r\"); obj2 = yaml.safe_load(f); sys.exit(obj1 != obj2)"
+}
+
+zfs_or_rotational() {
+ local ost_idx=0
+ local ost_name=$(ostname_from_index $ost_idx $MOUNT)
+ local param="get_param -n osd-*.${ost_name}.nonrotational"
+ local nonrotat=$(do_facet ost1 $LCTL $param)
+
+ if [[ -z "$nonrotat" ]]; then
+ # At this point there is no point moving ahead.
+ # Will stop here and dump all the info
+ set -x
+ local ost_name=$(ostname_from_index $ost_idx)
+ set +x
+ error "$LCTL $input_str"
+ fi
+
+ if [[ "$ost1_FSTYPE" == "zfs" ]] || (( "$nonrotat" == 0 )); then
+ return 0
+ else
+ return 1
+ fi
+}