+
+lsnapshot_create()
+{
+ do_facet mgs "$LCTL snapshot_create -F $FSNAME $*"
+}
+
+lsnapshot_destroy()
+{
+ do_facet mgs "$LCTL snapshot_destroy -F $FSNAME $*"
+}
+
+lsnapshot_modify()
+{
+ do_facet mgs "$LCTL snapshot_modify -F $FSNAME $*"
+}
+
+lsnapshot_list()
+{
+ do_facet mgs "$LCTL snapshot_list -F $FSNAME $*"
+}
+
+lsnapshot_mount()
+{
+ do_facet mgs "$LCTL snapshot_mount -F $FSNAME $*"
+}
+
+lsnapshot_umount()
+{
+ do_facet mgs "$LCTL snapshot_umount -F $FSNAME $*"
+}
+
+lss_err()
+{
+ local msg=$1
+
+ do_facet mgs "cat $LSNAPSHOT_LOG"
+ error $msg
+}
+
+lss_cleanup()
+{
+ echo "Cleaning test environment ..."
+
+ # Every lsnapshot command takes exclusive lock with others,
+ # so can NOT destroy the snapshot during list with 'xargs'.
+ while true; do
+ local ssname=$(lsnapshot_list | grep snapshot_name |
+ grep lss_ | awk '{ print $2 }' | head -n 1)
+ [ -z "$ssname" ] && break
+
+ lsnapshot_destroy -n $ssname -f ||
+ lss_err "Fail to destroy $ssname by force"
+ done
+}
+
+lss_gen_conf_one()
+{
+ local facet=$1
+ local role=$2
+ local idx=$3
+
+ local host=$(facet_active_host $facet)
+ local dir=$(dirname $(facet_vdevice $facet))
+ local pool=$(zpool_name $facet)
+ local lfsname=$(zfs_local_fsname $facet)
+ local label=${FSNAME}-${role}$(printf '%04x' $idx)
+
+ do_facet mgs \
+ "echo '$host - $label zfs:${dir}/${pool}/${lfsname} - -' >> \
+ $LSNAPSHOT_CONF"
+}
+
+lss_gen_conf()
+{
+ do_facet mgs "rm -f $LSNAPSHOT_CONF"
+ echo "Generating $LSNAPSHOT_CONF on MGS ..."
+
+ if ! combined_mgs_mds ; then
+ [ $(facet_fstype mgs) != zfs ] &&
+ skip "Lustre snapshot 1 only works for ZFS backend"
+
+ local host=$(facet_active_host mgs)
+ local dir=$(dirname $(facet_vdevice mgs))
+ local pool=$(zpool_name mgs)
+ local lfsname=$(zfs_local_fsname mgs)
+
+ do_facet mgs \
+ "echo '$host - MGS zfs:${dir}/${pool}/${lfsname} - -' \
+ >> $LSNAPSHOT_CONF" || lss_err "generate lss conf (mgs)"
+ fi
+
+ for num in `seq $MDSCOUNT`; do
+ [ $(facet_fstype mds$num) != zfs ] &&
+ skip "Lustre snapshot 1 only works for ZFS backend"
+
+ lss_gen_conf_one mds$num MDT $((num - 1)) ||
+ lss_err "generate lss conf (mds$num)"
+ done
+
+ for num in `seq $OSTCOUNT`; do
+ [ $(facet_fstype ost$num) != zfs ] &&
+ skip "Lustre snapshot 1 only works for ZFS backend"
+
+ lss_gen_conf_one ost$num OST $((num - 1)) ||
+ lss_err "generate lss conf (ost$num)"
+ done
+
+ do_facet mgs "cat $LSNAPSHOT_CONF"
+}
+
+# Parse 'lfs getstripe -d <path_with_dir_name>' for non-composite dir
+parse_plain_dir_param()
+{
+ local invalues=($1)
+ local param=""
+
+ if [[ ${invalues[0]} =~ "stripe_count:" ]]; then
+ param="-c ${invalues[1]}"
+ fi
+ if [[ ${invalues[2]} =~ "stripe_size:" ]]; then
+ param="$param -S ${invalues[3]}"
+ fi
+ if [[ ${invalues[4]} =~ "pattern:" ]]; then
+ if [[ ${invalues[5]} =~ "stripe_offset:" ]]; then
+ param="$param -i ${invalues[6]}"
+ else
+ param="$param -L ${invalues[5]} -i ${invalues[7]}"
+ fi
+ elif [[ ${invalues[4]} =~ "stripe_offset:" ]]; then
+ param="$param -i ${invalues[5]}"
+ fi
+ echo "$param"
+}
+
+parse_plain_param()
+{
+ local line=$1
+ local val=$(awk '{print $2}' <<< $line)
+
+ if [[ $line =~ ^"lmm_stripe_count:" ]]; then
+ echo "-c $val"
+ elif [[ $line =~ ^"lmm_stripe_size:" ]]; then
+ echo "-S $val"
+ elif [[ $line =~ ^"lmm_stripe_offset:" ]]; then
+ echo "-i $val"
+ elif [[ $line =~ ^"lmm_pattern:" ]]; then
+ echo "-L $val"
+ fi
+}
+
+parse_layout_param()
+{
+ local mode=""
+ local val=""
+ local param=""
+
+ while read line; do
+ if [[ ! -z $line ]]; then
+ if [[ -z $mode ]]; then
+ if [[ $line =~ ^"stripe_count:" ]]; then
+ mode="plain_dir"
+ elif [[ $line =~ ^"lmm_stripe_count:" ]]; then
+ mode="plain_file"
+ elif [[ $line =~ ^"lcm_layout_gen:" ]]; then
+ mode="pfl"
+ fi
+ fi
+
+ if [[ $mode = "plain_dir" ]]; then
+ param=$(parse_plain_dir_param "$line")
+ elif [[ $mode = "plain_file" ]]; then
+ val=$(parse_plain_param "$line")
+ [[ ! -z $val ]] && param="$param $val"
+ elif [[ $mode = "pfl" ]]; then
+ val=$(echo $line | awk '{print $2}')
+ if [[ $line =~ ^"lcme_extent.e_end:" ]]; then
+ if [[ $val = "EOF" ]]; then
+ param="$param -E -1"
+ else
+ param="$param -E $val"
+ fi
+ elif [[ $line =~ ^"stripe_count:" ]]; then
+ # pfl dir
+ val=$(parse_plain_dir_param "$line")
+ param="$param $val"
+ else
+ #pfl file
+ val=$(parse_plain_param "$line")
+ [[ ! -z $val ]] && param="$param $val"
+ fi
+ fi
+ fi
+ done
+ echo "$param"
+}
+
+get_layout_param()
+{
+ local param=$($LFS getstripe -d $1 | parse_layout_param)
+ echo "$param"
+}
+
+lfsck_verify_pfid()
+{
+ local f
+ local rc=0
+
+ # Cancel locks before setting lfsck_verify_pfid so that errors are more
+ # controllable
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ # make sure PFID is set correctly for files
+ do_nodes $(comma_list $(osts_nodes)) \
+ "$LCTL set_param -n obdfilter.${FSNAME}-OST*.lfsck_verify_pfid=1"
+
+ for f in "$@"; do
+ cat $f &> /dev/nullA ||
+ { rc=$?; echo "verify $f failed"; break; }
+ done
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ "$LCTL set_param -n obdfilter.${FSNAME}-OST*.lfsck_verify_pfid=0"
+ return $rc
+}
+
+# check that clients "oscs" was evicted after "before"
+check_clients_evicted() {
+ local before=$1
+ shift
+ local oscs=${@}
+ local osc
+ local rc=0
+
+ for osc in $oscs; do
+ ((rc++))
+ echo "Check state for $osc"
+ local evicted=$(do_facet client $LCTL get_param osc.$osc.state |
+ tail -n 3 | awk -F"[ [,]" \
+ '/EVICTED ]$/ { if (mx<$5) {mx=$5;} } END { print mx }')
+ if (($? == 0)) && (($evicted > $before)); then
+ echo "$osc is evicted at $evicted"
+ ((rc--))
+ fi
+ done
+
+ [ $rc -eq 0 ] || error "client not evicted from OST"
+}
+
+# check that clients OSCS current_state is FULL
+check_clients_full() {
+ local timeout=$1
+ shift
+ local oscs=${@}
+
+ for osc in $oscs; do
+ wait_update_facet client \
+ "lctl get_param -n osc.$osc.state |
+ grep 'current_state: FULL'" \
+ "current_state: FULL" $timeout
+ [ $? -eq 0 ] || error "$osc state is not FULL"
+ done
+}
+
+#Changelogs
+__changelog_deregister() {
+ local facet=$1
+ local mdt="$(facet_svc $facet)"
+ local cl_user=$2
+ local rc=0
+
+ # skip cleanup if no user registered for this MDT
+ [ -z "$cl_user" ] && echo "$mdt: no changelog user" && return 0
+ # user is no longer registered, skip cleanup
+ changelog_users "$facet" | grep -q "$cl_user" ||
+ { echo "$mdt: changelog user '$cl_user' not found"; return 0; }
+
+ # From this point, if any operation fails, it is an error
+ __changelog_clear $facet $cl_user 0 ||
+ error_noexit "$mdt: changelog_clear $cl_user 0 fail: $rc"
+ do_facet $facet $LCTL --device $mdt changelog_deregister $cl_user ||
+ error_noexit "$mdt: changelog_deregister '$cl_user' fail: $rc"
+}
+
+declare -Ax CL_USERS
+changelog_register() {
+ for M in $(seq $MDSCOUNT); do
+ local facet=mds$M
+ local mdt="$(facet_svc $facet)"
+ stack_trap "do_facet $facet $LCTL \
+ set_param mdd.$mdt.changelog_mask=-hsm" EXIT
+ do_facet $facet $LCTL set_param mdd.$mdt.changelog_mask=+hsm ||
+ error "$mdt: changelog_mask=+hsm failed: $?"
+
+ local cl_user
+ cl_user=$(do_facet $facet \
+ $LCTL --device $mdt changelog_register -n) ||
+ error "$mdt: register changelog user failed: $?"
+ stack_trap "__changelog_deregister $facet $cl_user" EXIT
+
+ stack_trap "CL_USERS[$facet]='${CL_USERS[$facet]}'" EXIT
+ # Bash does not support nested arrays, but the format of a
+ # cl_user is constrained enough to use whitespaces as separators
+ CL_USERS[$facet]+="$cl_user "
+ done
+ echo "Registered $MDSCOUNT changelog users: '${CL_USERS[@]% }'"
+}
+
+changelog_deregister() {
+ local cl_user
+
+ for facet in "${!CL_USERS[@]}"; do
+ for cl_user in ${CL_USERS[$facet]}; do
+ __changelog_deregister $facet $cl_user || return $?
+ done
+ unset CL_USERS[$facet]
+ done
+}
+
+changelog_users() {
+ local facet=$1
+ local service=$(facet_svc $facet)
+
+ do_facet $facet $LCTL get_param -n mdd.$service.changelog_users
+}
+
+changelog_user_rec() {
+ local facet=$1
+ local cl_user=$2
+ local service=$(facet_svc $facet)
+
+ changelog_users $facet | awk '$1 == "'$cl_user'" { print $2 }'
+}
+
+changelog_chmask() {
+ local mask=$1
+
+ do_nodes $(comma_list $(mdts_nodes)) \
+ $LCTL set_param mdd.*.changelog_mask="$mask"
+}
+
+# usage: __changelog_clear FACET CL_USER [+]INDEX
+__changelog_clear()
+{
+ local facet=$1
+ local mdt="$(facet_svc $facet)"
+ local cl_user=$2
+ local -i rec
+
+ case "$3" in
+ +*)
+ # Remove the leading '+'
+ rec=${3:1}
+ rec+=$(changelog_user_rec $facet $cl_user)
+ ;;
+ *)
+ rec=$3
+ ;;
+ esac
+
+ if [ $rec -eq 0 ]; then
+ echo "$mdt: clear the changelog for $cl_user of all records"
+ else
+ echo "$mdt: clear the changelog for $cl_user to record #$rec"
+ fi
+ $LFS changelog_clear $mdt $cl_user $rec
+}
+
+# usage: changelog_clear [+]INDEX
+#
+# If INDEX is prefixed with '+', increment every changelog user's record index
+# by INDEX. Otherwise, clear the changelog up to INDEX for every changelog
+# users.
+changelog_clear() {
+ local rc
+ for facet in ${!CL_USERS[@]}; do
+ for cl_user in ${CL_USERS[$facet]}; do
+ __changelog_clear $facet $cl_user $1 || rc=${rc:-$?}
+ done
+ done
+
+ return ${rc:-0}
+}
+
+changelog_dump() {
+ for M in $(seq $MDSCOUNT); do
+ local facet=mds$M
+ local mdt="$(facet_svc $facet)"
+
+ $LFS changelog $mdt | sed -e 's/^/'$mdt'./'
+ done
+}
+
+changelog_extract_field() {
+ local cltype=$1
+ local file=$2
+ local identifier=$3
+
+ changelog_dump | gawk "/$cltype.*$file$/ {
+ print gensub(/^.* "$identifier'(\[[^\]]*\]).*$/,"\\1",1)}' |
+ tail -1
+}
+
+restore_layout() {
+ local dir=$1
+ local layout=$2
+
+ [ ! -d "$dir" ] && return
+
+ [ -z "$layout" ] && {
+ $LFS setstripe -d $dir || error "error deleting stripe '$dir'"
+ return
+ }
+
+ setfattr -n trusted.lov -v $layout $dir ||
+ error "error restoring layout '$layout' to '$dir'"
+}
+
+# save the layout of a directory, the returned string will be used by
+# restore_layout() to restore the layout
+save_layout() {
+ local dir=$1
+ local str=$(getfattr -n trusted.lov --absolute-names -e hex $dir \
+ 2> /dev/null | awk -F'=' '/trusted.lov/{ print $2 }')
+ echo "$str"
+}
+
+# save layout of a directory and restore it at exit
+save_layout_restore_at_exit() {
+ local dir=$1
+ local layout=$(save_layout $dir)
+
+ stack_trap "restore_layout $dir $layout" EXIT
+}