+
+lsnapshot_create()
+{
+ do_facet mgs "$LCTL snapshot_create -F $FSNAME $*"
+}
+
+lsnapshot_destroy()
+{
+ do_facet mgs "$LCTL snapshot_destroy -F $FSNAME $*"
+}
+
+lsnapshot_modify()
+{
+ do_facet mgs "$LCTL snapshot_modify -F $FSNAME $*"
+}
+
+lsnapshot_list()
+{
+ do_facet mgs "$LCTL snapshot_list -F $FSNAME $*"
+}
+
+lsnapshot_mount()
+{
+ do_facet mgs "$LCTL snapshot_mount -F $FSNAME $*"
+}
+
+lsnapshot_umount()
+{
+ do_facet mgs "$LCTL snapshot_umount -F $FSNAME $*"
+}
+
+lss_err()
+{
+ local msg=$1
+
+ do_facet mgs "cat $LSNAPSHOT_LOG"
+ error $msg
+}
+
+lss_cleanup()
+{
+ echo "Cleaning test environment ..."
+
+ # Every lsnapshot command takes exclusive lock with others,
+ # so can NOT destroy the snapshot during list with 'xargs'.
+ while true; do
+ local ssname=$(lsnapshot_list | grep snapshot_name |
+ grep lss_ | awk '{ print $2 }' | head -n 1)
+ [ -z "$ssname" ] && break
+
+ lsnapshot_destroy -n $ssname -f ||
+ lss_err "Fail to destroy $ssname by force"
+ done
+}
+
+lss_gen_conf_one()
+{
+ local facet=$1
+ local role=$2
+ local idx=$3
+
+ local host=$(facet_active_host $facet)
+ local dir=$(dirname $(facet_vdevice $facet))
+ local pool=$(zpool_name $facet)
+ local lfsname=$(zfs_local_fsname $facet)
+ local label=${FSNAME}-${role}$(printf '%04x' $idx)
+
+ do_facet mgs \
+ "echo '$host - $label zfs:${dir}/${pool}/${lfsname} - -' >> \
+ $LSNAPSHOT_CONF"
+}
+
+lss_gen_conf()
+{
+ do_facet mgs "rm -f $LSNAPSHOT_CONF"
+ echo "Generating $LSNAPSHOT_CONF on MGS ..."
+
+ if ! combined_mgs_mds ; then
+ [ $(facet_fstype mgs) != zfs ] &&
+ skip "Lustre snapshot 1 only works for ZFS backend"
+
+ local host=$(facet_active_host mgs)
+ local dir=$(dirname $(facet_vdevice mgs))
+ local pool=$(zpool_name mgs)
+ local lfsname=$(zfs_local_fsname mgs)
+
+ do_facet mgs \
+ "echo '$host - MGS zfs:${dir}/${pool}/${lfsname} - -' \
+ >> $LSNAPSHOT_CONF" || lss_err "generate lss conf (mgs)"
+ fi
+
+ for num in `seq $MDSCOUNT`; do
+ [ $(facet_fstype mds$num) != zfs ] &&
+ skip "Lustre snapshot 1 only works for ZFS backend"
+
+ lss_gen_conf_one mds$num MDT $((num - 1)) ||
+ lss_err "generate lss conf (mds$num)"
+ done
+
+ for num in `seq $OSTCOUNT`; do
+ [ $(facet_fstype ost$num) != zfs ] &&
+ skip "Lustre snapshot 1 only works for ZFS backend"
+
+ lss_gen_conf_one ost$num OST $((num - 1)) ||
+ lss_err "generate lss conf (ost$num)"
+ done
+
+ do_facet mgs "cat $LSNAPSHOT_CONF"
+}
+
+# Parse 'lfs getstripe -d <path_with_dir_name>' for non-composite dir
+parse_plain_dir_param()
+{
+ local invalues=($1)
+ local param=""
+
+ if [[ ${invalues[0]} =~ "stripe_count:" ]]; then
+ param="-c ${invalues[1]}"
+ fi
+ if [[ ${invalues[2]} =~ "stripe_size:" ]]; then
+ param="$param -S ${invalues[3]}"
+ fi
+ if [[ ${invalues[4]} =~ "pattern:" ]]; then
+ if [[ ${invalues[5]} =~ "stripe_offset:" ]]; then
+ param="$param -i ${invalues[6]}"
+ else
+ param="$param -L ${invalues[5]} -i ${invalues[7]}"
+ fi
+ elif [[ ${invalues[4]} =~ "stripe_offset:" ]]; then
+ param="$param -i ${invalues[5]}"
+ fi
+ echo "$param"
+}
+
+parse_plain_param()
+{
+ local line=$1
+ local val=$(awk '{print $2}' <<< $line)
+
+ if [[ $line =~ ^"lmm_stripe_count:" ]]; then
+ echo "-c $val"
+ elif [[ $line =~ ^"lmm_stripe_size:" ]]; then
+ echo "-S $val"
+ elif [[ $line =~ ^"lmm_stripe_offset:" ]]; then
+ echo "-i $val"
+ elif [[ $line =~ ^"lmm_pattern:" ]]; then
+ echo "-L $val"
+ fi
+}
+
+parse_layout_param()
+{
+ local mode=""
+ local val=""
+ local param=""
+
+ while read line; do
+ if [[ ! -z $line ]]; then
+ if [[ -z $mode ]]; then
+ if [[ $line =~ ^"stripe_count:" ]]; then
+ mode="plain_dir"
+ elif [[ $line =~ ^"lmm_stripe_count:" ]]; then
+ mode="plain_file"
+ elif [[ $line =~ ^"lcm_layout_gen:" ]]; then
+ mode="pfl"
+ fi
+ fi
+
+ if [[ $mode = "plain_dir" ]]; then
+ param=$(parse_plain_dir_param "$line")
+ elif [[ $mode = "plain_file" ]]; then
+ val=$(parse_plain_param "$line")
+ [[ ! -z $val ]] && param="$param $val"
+ elif [[ $mode = "pfl" ]]; then
+ val=$(echo $line | awk '{print $2}')
+ if [[ $line =~ ^"lcme_extent.e_end:" ]]; then
+ if [[ $val = "EOF" ]]; then
+ param="$param -E -1"
+ else
+ param="$param -E $val"
+ fi
+ elif [[ $line =~ ^"stripe_count:" ]]; then
+ # pfl dir
+ val=$(parse_plain_dir_param "$line")
+ param="$param $val"
+ else
+ #pfl file
+ val=$(parse_plain_param "$line")
+ [[ ! -z $val ]] && param="$param $val"
+ fi
+ fi
+ fi
+ done
+ echo "$param"
+}
+
+get_layout_param()
+{
+ local param=$($LFS getstripe -d $1 | parse_layout_param)
+ echo "$param"
+}
+
+lfsck_verify_pfid()
+{
+ local f
+ local rc=0
+
+ # Cancel locks before setting lfsck_verify_pfid so that errors are more
+ # controllable
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ # make sure PFID is set correctly for files
+ do_nodes $(comma_list $(osts_nodes)) \
+ "$LCTL set_param -n obdfilter.${FSNAME}-OST*.lfsck_verify_pfid=1"
+
+ for f in "$@"; do
+ cat $f &> /dev/nullA ||
+ { rc=$?; echo "verify $f failed"; break; }
+ done
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ "$LCTL set_param -n obdfilter.${FSNAME}-OST*.lfsck_verify_pfid=0"
+ return $rc
+}
+
+# check that clients "oscs" was evicted after "before"
+check_clients_evicted() {
+ local before=$1
+ shift
+ local oscs=${@}
+ local osc
+ local rc=0
+
+ for osc in $oscs; do
+ ((rc++))
+ echo "Check state for $osc"
+ local evicted=$(do_facet client $LCTL get_param osc.$osc.state |
+ tail -n 3 | awk -F"[ [,]" \
+ '/EVICTED ]$/ { if (mx<$5) {mx=$5;} } END { print mx }')
+ if (($? == 0)) && (($evicted > $before)); then
+ echo "$osc is evicted at $evicted"
+ ((rc--))
+ fi
+ done
+
+ [ $rc -eq 0 ] || error "client not evicted from OST"
+}
+
+# check that clients OSCS current_state is FULL
+check_clients_full() {
+ local timeout=$1
+ shift
+ local oscs=${@}
+
+ for osc in $oscs; do
+ wait_update_facet client \
+ "lctl get_param -n osc.$osc.state |
+ grep 'current_state: FULL'" \
+ "current_state: FULL" $timeout
+ [ $? -eq 0 ] || error "$osc state is not FULL"
+ done
+}
+
+#Changelogs
+__changelog_deregister() {
+ local facet=$1
+ local mdt="$(facet_svc $facet)"
+ local cl_user=$2
+ local rc=0
+
+ # skip cleanup if no user registered for this MDT
+ [ -z "$cl_user" ] && echo "$mdt: no changelog user" && return 0
+ # user is no longer registered, skip cleanup
+ changelog_users "$facet" | grep -q "$cl_user" ||
+ { echo "$mdt: changelog user '$cl_user' not found"; return 0; }
+
+ # From this point, if any operation fails, it is an error
+ __changelog_clear $facet $cl_user 0 ||
+ error_noexit "$mdt: changelog_clear $cl_user 0 fail: $rc"
+ do_facet $facet $LCTL --device $mdt changelog_deregister $cl_user ||
+ error_noexit "$mdt: changelog_deregister '$cl_user' fail: $rc"
+}
+
+declare -Ax CL_USERS
+changelog_register() {
+ for M in $(seq $MDSCOUNT); do
+ local facet=mds$M
+ local mdt="$(facet_svc $facet)"
+ local cl_mask
+
+ cl_mask=$(do_facet $facet $LCTL get_param \
+ mdd.${mdt}.changelog_mask -n)
+ stack_trap "do_facet $facet $LCTL \
+ set_param mdd.$mdt.changelog_mask=\'$cl_mask\' -n" EXIT
+ do_facet $facet $LCTL set_param mdd.$mdt.changelog_mask=+hsm ||
+ error "$mdt: changelog_mask=+hsm failed: $?"
+
+ local cl_user
+ cl_user=$(do_facet $facet \
+ $LCTL --device $mdt changelog_register -n) ||
+ error "$mdt: register changelog user failed: $?"
+ stack_trap "__changelog_deregister $facet $cl_user" EXIT
+
+ stack_trap "CL_USERS[$facet]='${CL_USERS[$facet]}'" EXIT
+ # Bash does not support nested arrays, but the format of a
+ # cl_user is constrained enough to use whitespaces as separators
+ CL_USERS[$facet]+="$cl_user "
+ done
+ echo "Registered $MDSCOUNT changelog users: '${CL_USERS[@]% }'"
+}
+
+changelog_deregister() {
+ local cl_user
+ # bash assoc arrays do not guarantee to list keys in created order
+ # so reorder to get same order than in changelog_register()
+ local cl_facets=$(echo "${!CL_USERS[@]}" | tr " " "\n" | sort |
+ tr "\n" " ")
+
+ for facet in $cl_facets; do
+ for cl_user in ${CL_USERS[$facet]}; do
+ __changelog_deregister $facet $cl_user || return $?
+ done
+ unset CL_USERS[$facet]
+ done
+}
+
+changelog_users() {
+ local facet=$1
+ local service=$(facet_svc $facet)
+
+ do_facet $facet $LCTL get_param -n mdd.$service.changelog_users
+}
+
+changelog_user_rec() {
+ local facet=$1
+ local cl_user=$2
+ local service=$(facet_svc $facet)
+
+ changelog_users $facet | awk '$1 == "'$cl_user'" { print $2 }'
+}
+
+changelog_chmask() {
+ local mask=$1
+
+ do_nodes $(comma_list $(mdts_nodes)) \
+ $LCTL set_param mdd.*.changelog_mask="$mask"
+}
+
+# usage: __changelog_clear FACET CL_USER [+]INDEX
+__changelog_clear()
+{
+ local facet=$1
+ local mdt="$(facet_svc $facet)"
+ local cl_user=$2
+ local -i rec
+
+ case "$3" in
+ +*)
+ # Remove the leading '+'
+ rec=${3:1}
+ rec+=$(changelog_user_rec $facet $cl_user)
+ ;;
+ *)
+ rec=$3
+ ;;
+ esac
+
+ if [ $rec -eq 0 ]; then
+ echo "$mdt: clear the changelog for $cl_user of all records"
+ else
+ echo "$mdt: clear the changelog for $cl_user to record #$rec"
+ fi
+ $LFS changelog_clear $mdt $cl_user $rec
+}
+
+# usage: changelog_clear [+]INDEX
+#
+# If INDEX is prefixed with '+', increment every changelog user's record index
+# by INDEX. Otherwise, clear the changelog up to INDEX for every changelog
+# users.
+changelog_clear() {
+ local rc
+ # bash assoc arrays do not guarantee to list keys in created order
+ # so reorder to get same order than in changelog_register()
+ local cl_facets=$(echo "${!CL_USERS[@]}" | tr " " "\n" | sort |
+ tr "\n" " ")
+
+ for facet in $cl_facets; do
+ for cl_user in ${CL_USERS[$facet]}; do
+ __changelog_clear $facet $cl_user $1 || rc=${rc:-$?}
+ done
+ done
+
+ return ${rc:-0}
+}
+
+changelog_dump() {
+ for M in $(seq $MDSCOUNT); do
+ local facet=mds$M
+ local mdt="$(facet_svc $facet)"
+
+ $LFS changelog $mdt | sed -e 's/^/'$mdt'./'
+ done
+}
+
+changelog_extract_field() {
+ local cltype=$1
+ local file=$2
+ local identifier=$3
+
+ changelog_dump | gawk "/$cltype.*$file$/ {
+ print gensub(/^.* "$identifier'(\[[^\]]*\]).*$/,"\\1",1)}' |
+ tail -1
+}
+
+# Prints a changelog record produced by "lfs changelog" as an associative array
+#
+# Example:
+# $> changelog2array 16 01CREAT 10:28:46.968438800 2018.03.09 0x0 \
+# t=[0x200000401:0x10:0x0] j=touch.501 ef=0xf u=501:501 \
+# nid=0@lo p=[0x200000007:0x1:0x0] blob
+# ([index]='16' [type]='CREAT' [time]='10:28:46.968438800'
+# [date]='2018.03.09' [flags]=0x0 ['target-fid']='[0x200000401:0x10:0x0]'
+# ['jobid']='touch.501' ['extra-flags']='0x0f' [uid]='0' ['gid']='0'
+# ['nid']='0@lo' ['parent-fid']='[0x200000007:0x1:0x0]')
+#
+# Note that the changelog record is not quoted
+# Also note that the line breaks in the output were only added for readability
+#
+# Typically, you want to eval the output of the command to fill an actual
+# associative array, like this:
+# $> eval declare -A changelog=$(changelog2array $entry)
+#
+# It can then be accessed like any bash associative array:
+# $> echo "${changelog[index]}" "${changelog[type]}" "${changelog[flags]}"
+# 16 CREAT 0x0
+# $> echo "${changelog[uid]}":"${changelog[gid]}"
+# 501:501
+#
+changelog2array()
+{
+ # Start the array
+ printf '('
+
+ # A changelog, as printed by "lfs changelog" typically looks like this:
+ # <index> <type> <time> <date> <flags> <key1=value1> <key2=value2> ...
+
+ # Parse the positional part of the changelog
+
+ # changelog_dump() prefixes records with their mdt's name
+ local index="${1##*.}"
+
+ printf "[index]='%s' [type]='%s' [time]='%s' [date]='%s' [flags]='%s'" \
+ "$index" "${2:2}" "$3" "$4" "$5"
+
+ # Parse the key/value part of the changelog
+ for arg in "${@:5}"; do
+ # Check it matches a key=value syntax
+ [[ "$arg" =~ ^[[:alpha:]]+= ]] || continue
+
+ local key="${arg%%=*}"
+ local value="${arg#*=}"
+
+ case "$key" in
+ u)
+ # u is actually for uid AND gid: u=UID:GID
+ printf " [uid]='%s'" "${value%:*}"
+ key=gid
+ value="${value#*:}"
+ ;;
+ t)
+ key=target-fid
+ value="${value#[}"
+ value="${value%]}"
+ ;;
+ j)
+ key=jobid
+ ;;
+ p)
+ key=parent-fid
+ value="${value#[}"
+ value="${value%]}"
+ ;;
+ ef)
+ key=extra-flags
+ ;;
+ m)
+ key=mode
+ ;;
+ x)
+ key=xattr
+ ;;
+ *)
+ ;;
+ esac
+
+ printf " ['%s']='%s'" "$key" "$value"
+ done
+
+ # end the array
+ printf ')'
+}
+
+# Format and print a changelog record
+#
+# Interpreted sequences are:
+# %% a single %
+# %f the "flags" attribute of a changelog record
+__changelog_printf()
+{
+ local format="$1"
+
+ local -i i
+ for ((i = 0; i < ${#format}; i++)); do
+ local char="${format:$i:1}"
+ if [ "$char" != % ]; then
+ printf '%c' "$char"
+ continue
+ fi
+
+ i+=1
+ char="${format:$i:1}"
+ case "$char" in
+ f)
+ printf '%s' "${changelog[flags]}"
+ ;;
+ %)
+ printf '%'
+ ;;
+ esac
+ done
+ printf '\n'
+}
+
+# Filter changelog records
+changelog_find()
+{
+ local -A filter
+ local action='print'
+ local format
+
+ while [ $# -gt 0 ]; do
+ case "$1" in
+ -print)
+ action='print'
+ ;;
+ -printf)
+ action='printf'
+ format="$2"
+ shift
+ ;;
+ -*)
+ filter[${1#-}]="$2"
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ local found=false
+ local record
+ changelog_dump | { while read -r record; do
+ eval local -A changelog=$(changelog2array $record)
+ for key in "${!filter[@]}"; do
+ case "$key" in
+ *)
+ [ "${changelog[$key]}" == "${filter[$key]}" ]
+ ;;
+ esac || continue 2
+ done
+
+ found=true
+
+ case "${action:-print}" in
+ print)
+ printf '%s\n' "$record"
+ ;;
+ printf)
+ __changelog_printf "$format"
+ ;;
+ esac
+ done; $found; }
+}
+
+restore_layout() {
+ local dir=$1
+ local layout=$2
+
+ [ ! -d "$dir" ] && return
+
+ [ -z "$layout" ] && {
+ $LFS setstripe -d $dir || error "error deleting stripe '$dir'"
+ return
+ }
+
+ setfattr -n trusted.lov -v $layout $dir ||
+ error "error restoring layout '$layout' to '$dir'"
+}
+
+# save the layout of a directory, the returned string will be used by
+# restore_layout() to restore the layout
+save_layout() {
+ local dir=$1
+ local str=$(getfattr -n trusted.lov --absolute-names -e hex $dir \
+ 2> /dev/null | awk -F'=' '/trusted.lov/{ print $2 }')
+ echo "$str"
+}
+
+# save layout of a directory and restore it at exit
+save_layout_restore_at_exit() {
+ local dir=$1
+ local layout=$(save_layout $dir)
+
+ stack_trap "restore_layout $dir $layout" EXIT
+}