lipe.spec \
lipe.spec.in \
lipe_c_check.pl \
+ lipe_find2 \
+ lipe-func.sh \
lpcc.conf \
laudit.conf.example \
pybuild/*.py \
--- /dev/null
+#!/bin/bash
+# Common functions for LiPE utilities
+
+LL_SUPER_MAGIC=0x0BD00BD0
+
+debug() {
+ if $DEBUG; then
+ echo "$PROGRAM_NAME: $@" >&2
+ fi
+}
+
+error() {
+ echo "$PROGRAM_NAME: $@" >&2
+}
+
+fatal() {
+ error "fatal:" "$@"
+ exit 1
+}
+
+# Check Lustre client mount point and device.
+check_client_mount() {
+ local mount_point="$1"
+ local device="$2"
+ local fstype
+ local label
+
+ [[ $mount_point ]] || fatal "client mount point must be specified"
+ [[ $device ]] || fatal "device must be specified"
+ [[ -b $device ]] || fatal "'$device' is not a block device"
+
+ # Check that $mount_point is a Lustre client mount.
+ fstype=$(stat --file-system --format='%t' $mount_point)
+ debug "fstype = '$fstype'"
+
+ mountpoint --quiet $mount_point &&
+ ((16#$fstype == LL_SUPER_MAGIC)) ||
+ fatal "'$mount_point' is not a Lustre client mount point"
+
+ # Check that $mount_point/.lustre/fid exists.
+ [[ -d $mount_point/.lustre/fid ]] ||
+ fatal "'$mount_point/.lustre/fid' does not exist"
+
+ # Check that $mount_point is the right Lustre client mount for $device.
+ label=$(e2label $device)
+ lfs df $mount_point | grep -q "^${label}_UUID" ||
+ fatal "'$device' is not a server device for '$mount_point'"
+
+ return 0
+}
cp \
ldsync \
lipe_find \
+ lipe_find2 \
+ lipe-func.sh \
lipe_convert_expr \
lipe_run_action \
lipe_install \
%{_bindir}/lipe_scan
%{_bindir}/lipe_scan2
%{_bindir}/lipe_find
+%{_bindir}/lipe_find2
+%{_bindir}/lipe-func.sh
+%{_bindir}/lipe_convert_expr
%{python2_sitelib}/pylipe
%config(noreplace) %{_sysconfdir}/lipe_install.conf
%config(noreplace) %{_sysconfdir}/lipe_launch.json
--- /dev/null
+#!/bin/bash
+
+set -e
+
+LIPE_PATH=${LIPE_PATH:-$(dirname $(type -p $0))}
+. $LIPE_PATH/lipe-func.sh
+
+PROGRAM_NAME=$(basename $0)
+LIPE_SCAN=${LIPE_SCAN:-"lipe_scan2"}
+LIPE_CONVERT_EXPR=${LIPE_CONVERT_EXPR:-"lipe_convert_expr"}
+LIPE_CONVERT=true
+DEBUG=false
+
+usage() {
+ echo "Usage: $PROGRAM_NAME [OPTION]... -- DEVICE [EXPRESSION]" >&2
+ echo "Try '$PROGRAM_NAME --help' for more information" >&2
+ exit 1
+}
+
+help() {
+cat <<EOF
+Usage: $PROGRAM_NAME [OPTION]... -- DEVICE [EXPRESSION]
+Scan DEVICE for files matching EXPRESSION (or all files if expression
+is omitted). Print FIDs (default), JSON object descriptions, or paths,
+of matching files to stdout.
+
+Mandatory arguments to long options are mandatory for short options too.
+ --client-mount=MOUNT use the Lustre client at MOUNT for FID to path
+ --print-fid print FID of each file matching expression
+ --print-json[=ATTRS] print a JSON object with atributes specified by ATTRS
+ describing each file matching expression. ATTRS must be
+ a comma separated list of attribute names.
+ use '$PROGRAM_NAME --list-attrs' to see available attributes names
+ --print-path[=WHICH] print path(s) of each file matching expression
+ WHICH must be 'one' (default) or 'all'
+ --absolute-paths prefix paths with MOUNT/ (for --print-path)
+ prefix FIDs with MOUNT/.lustre/fid/ (for --print-fid)
+ --delimiter=DELIM use DELIM intead of newline to delimit matches
+ --null use a NUL byte intead of newline to delimit matches
+ --threads=COUNT use COUNT scanning threads
+ -h,--help display this help text and exit
+ --debug display debug information
+ --list-attrs list available attribute names
+ --no-convert do not convert the EXPRESSION
+ --version output version information and exit
+
+If --absolute-paths or --print-path is used then --client-mount is
+required. It is also required if --print-json is used and ATTRS
+includes paths.
+
+For expression details see lipe_find(1).
+EOF
+ exit 0
+}
+
+main() {
+ local client_mount
+ local device
+ local expression
+ local threads=0 # 0 means default.
+ local -a lipe_scan2_options=()
+ local options
+ local opt
+
+ local short_options="h"
+ local long_options="client-mount:,print-fid,print-json::,print-path::"
+ long_options+="${long_options:+,}absolute-paths,delimiter:,null"
+ long_options+="${long_options:+,}threads:,help,debug,list-attrs"
+ long_options+="${long_options:+,}no-convert,version"
+ options=$(getopt --name="$PROGRAM_NAME" \
+ --options="$short_options" \
+ --longoptions="$long_options" -- "$@") ||
+ fatal "invalid options"
+
+ eval set -- "$options"
+ while true; do
+ case "$1" in
+ --client-mount)
+ shift
+ client_mount="$1"
+ lipe_scan2_options+=(--client-mount="$1")
+ ;;
+ --print-fid)
+ lipe_scan2_options+=(--print-fid)
+ ;;
+ --print-json)
+ shift
+ opt="--print-json"
+ [[ -z "$1" ]] || opt+="=$1"
+ lipe_scan2_options+=("$opt")
+ ;;
+ --print-path)
+ shift
+ opt="--print-path"
+ [[ -z "$1" ]] || opt+="=$1"
+ lipe_scan2_options+=("$opt")
+ ;;
+ --absolute-paths)
+ lipe_scan2_options+=(--absolute-paths)
+ ;;
+ --delimiter)
+ shift
+ lipe_scan2_options+=(--delimiter="$1")
+ ;;
+ --null)
+ lipe_scan2_options+=(--null)
+ ;;
+ --threads)
+ shift
+ threads="$1"
+ lipe_scan2_options+=(--threads="$1")
+ ;;
+ -h|--help)
+ help
+ ;;
+ --debug)
+ DEBUG=true
+ lipe_scan2_options+=(--debug)
+ ;;
+ --list-attrs)
+ $LIPE_SCAN --list-attrs
+ exit 0
+ ;;
+ --no-convert)
+ LIPE_CONVERT=false
+ ;;
+ --version)
+ $LIPE_SCAN --version |
+ sed -e "s/$LIPE_SCAN/$PROGRAM_NAME/"
+ exit 0
+ ;;
+ --)
+ shift
+ break
+ ;;
+ esac
+ shift
+ done
+
+ if [[ $# < 1 ]]; then
+ # Missing device.
+ usage
+ fi
+
+ device="$1"
+ shift
+ # Now $@ is just the expression.
+ debug "client_mount = '$client_mount'"
+ debug "device = '$device'"
+ debug "threads = '$threads'"
+ debug "argc = $#"
+ debug "argv = '$@'"
+
+ [[ -z "$client_mount" ]] ||
+ check_client_mount $client_mount $device
+
+ if [[ -n "$@" ]]; then
+ expression="$@"
+ ! $LIPE_CONVERT || expression=$($LIPE_CONVERT_EXPR "$@")
+ fi
+ debug "expression = '$expression'"
+
+ local start=$SECONDS
+ $LIPE_SCAN "${lipe_scan2_options[@]}" "$device" \
+ ${expression:+"$expression"}
+ echo "$PROGRAM_NAME finished in $((SECONDS - start)) seconds" >&2
+}
+
+main "$@"
which jq || skip_env "jq is not installed"
# lipe_find configuration
-LIPE_FIND="lipe_find"
+LIPE_FIND=${LIPE_FIND:-"lipe_find"}
LIPE_FIND_LOG_DIR=${LIPE_FIND_LOG_DIR:-"/var/log/lipe_find"}
LIPE_FIND_VERBOSE=${LIPE_FIND_VERBOSE:-false}
LIPE_FIND_USE_MOUNT=${LIPE_FIND_USE_MOUNT:-false}
+[[ $LIPE_FIND = *lipe_find ]] && PRINT_FID_OPT="-print-fid" ||
+ PRINT_FID_OPT="--print-fid"
+
LIPE_POOL1=${LIPE_POOL1:-"fast"}
LIPE_POOL2=${LIPE_POOL2:-"slow"}
build_test_filter
check_and_setup_lustre
+# mount Lustre clients on MDS node(s)
+LIPE_MDS_NODE=$(exclude_items_from_list $(comma_list $(all_mdts_nodes)) \
+ $HOSTNAME)
+if [[ -n $LIPE_MDS_NODE ]]; then
+ zconf_mount_clients $LIPE_MDS_NODE $MOUNT ||
+ error "failed to mount Lustre client on MDS node $LIPE_MDS_NODE"
+ stack_trap "zconf_umount_clients $LIPE_MDS_NODE $MOUNT"
+fi
+
# create OST pools
create_ost_pools() {
pool_add $LIPE_POOL1 || error "failed to create OST pool '$LIPE_POOL1'"
local facet="$1"
local action="$2"
local device="$(facet_device $facet)"
+ local device_opt
local quiet
$LIPE_FIND_VERBOSE || quiet=yes
! $LIPE_FIND_USE_MOUNT || device=$MOUNT
+ [[ $LIPE_FIND = *lipe_find ]] && device_opt="$device" ||
+ device_opt="-- $device"
+
# lipe_find fails if run twice in the same wall clock second.
VERBOSE=false do_facet $facet "sync; [[ ! -e $LIPE_FIND_LOG_DIR ]] ||" \
"find $LIPE_FIND_LOG_DIR -mindepth 1 -delete"
- do_facet $facet "$LIPE_FIND $action $device ${@:3}" \
+ do_facet $facet "$LIPE_FIND $action $device_opt ${@:3}" \
"${quiet:+2>/dev/null}"
}
expect_fid() {
local fid="$1"
local which="${2:-true}"
+ local fid_str="$(tr -d '[]' <<<$fid)"
if $which; then
- grep --fixed-strings "$fid" || error "expected FID '$fid'"
+ grep --fixed-strings "$fid_str" || error "expected FID '$fid'"
else
- ! grep --fixed-strings "$fid" || error "unexpected FID '$fid'"
+ ! grep --fixed-strings "$fid_str" || error "unexpected FID '$fid'"
fi
}
local file="$1"
local which="$2"
- do_lipe_find_facet mds1 -print-fid "${@:3}" | expect_file "$file" "$which"
+ do_lipe_find_facet mds1 $PRINT_FID_OPT "${@:3}" |
+ expect_file "$file" "$which"
}
lipe_find_expect_fid() {
local fid="$1"
local which="$2"
- do_lipe_find_facet mds1 -print-fid "${@:3}" | expect_fid "$fid" "$which"
+ do_lipe_find_facet mds1 $PRINT_FID_OPT "${@:3}" |
+ expect_fid "$fid" "$which"
}
lipe_find_max_stripe_size() {
# lipe_find test cases
test_1() {
- do_lipe_find_facet mds1 -print-fid || error "cannot run lipe_find"
+ do_lipe_find_facet mds1 $PRINT_FID_OPT || error "cannot run lipe_find"
}
run_test 1 "lipe_find works"
mkdir $td || error "mkdir $td failed"
create_lipe_find_files $tf
- # mount Lustre clients on MDS nodes
- zconf_mount_clients $(comma_list $(all_mdts_nodes)) $MOUNT ||
- error "failed to mount Lustre clients on MDS nodes"
- stack_trap "zconf_umount_clients $(comma_list $(all_mdts_nodes)) \
- $MOUNT"
-
local saved_LIPE_FIND_USE_MOUNT=$LIPE_FIND_USE_MOUNT
LIPE_FIND_USE_MOUNT=true
stack_trap "LIPE_FIND_USE_MOUNT=$saved_LIPE_FIND_USE_MOUNT"
mkdir $td-1 || error "mkdir $td-1 failed"
create_lipe_find_files $tf
- # mount Lustre clients on MDS nodes
- zconf_mount_clients $(comma_list $(all_mdts_nodes)) $MOUNT ||
- error "failed to mount Lustre clients on MDS nodes"
- stack_trap "zconf_umount_clients $(comma_list $(all_mdts_nodes)) \
- $MOUNT"
-
local saved_LIPE_FIND_USE_MOUNT=$LIPE_FIND_USE_MOUNT
LIPE_FIND_USE_MOUNT=true
stack_trap "LIPE_FIND_USE_MOUNT=$saved_LIPE_FIND_USE_MOUNT"
mkdir $td || error "mkdir $td failed"
create_lipe_find_files $tf
- # mount Lustre clients on MDS nodes
- zconf_mount_clients $(comma_list $(all_mdts_nodes)) $MOUNT ||
- error "failed to mount Lustre clients on MDS nodes"
- stack_trap "zconf_umount_clients $(comma_list $(all_mdts_nodes)) \
- $MOUNT"
-
local saved_LIPE_FIND_USE_MOUNT=$LIPE_FIND_USE_MOUNT
LIPE_FIND_USE_MOUNT=true
stack_trap "LIPE_FIND_USE_MOUNT=$saved_LIPE_FIND_USE_MOUNT"
mkdir $td || error "mkdir $td failed"
create_lipe_find_files $tf
- # mount Lustre clients on MDS nodes
- zconf_mount_clients $(comma_list $(all_mdts_nodes)) $MOUNT ||
- error "failed to mount Lustre clients on MDS nodes"
- stack_trap "zconf_umount_clients $(comma_list $(all_mdts_nodes)) \
- $MOUNT"
-
local saved_LIPE_FIND_USE_MOUNT=$LIPE_FIND_USE_MOUNT
LIPE_FIND_USE_MOUNT=true
stack_trap "LIPE_FIND_USE_MOUNT=$saved_LIPE_FIND_USE_MOUNT"
do_facet "$facet" lipe_scan2 "$device" "$@"
}
-lipe_scan2_mount_client() {
- local nodes
-
- # All MDT nodes except the current node.
- nodes=$(all_mdts_nodes | grep --fixed-strings --line-regexp --invert-match "$HOSTNAME")
- if [[ -z "$nodes" ]]; then
- return
- fi
-
- zconf_mount_clients "$(comma_list $nodes)" "$MOUNT" ||
- error "failed to mount Lustre clients on MDS nodes"
- stack_trap "zconf_umount_clients $(comma_list $nodes) $MOUNT"
-}
-
test_200() {
local file=$DIR/$tdir/$tfile
local fid
sync
cancel_lru_locks mdc
- lipe_scan2_mount_client
-
attrs=$(do_lipe_scan2_facet mds1 --client-mount="$MOUNT" --print-json=fid,paths |
jq --arg FID "$fid" 'select(.fid == $FID)')
echo "attrs = '$attrs'" >&2
git://git.whamcloud.com / fs / lustre-release.git / commitdiff