From 56e322f722b23d7070c9249dabed02b53397c82e Mon Sep 17 00:00:00 2001 From: Amir Shehata Date: Tue, 30 Apr 2013 19:01:59 -0700 Subject: [PATCH] LU-2950 lnet: Add a mechanism to configure routes from a file Created a bash script lustre_rotues_config which takes in a file with routes configured in the following format: : { gateway: , [hop:], [priority: ] } The script shall parse the file and generate: lctl --net add_route [hop [priority]] for each route. The script can be used to unconfigure routes as well by running it as follows: lustre_routes_config --cleanup In this case it will remove all routes configures via lctl del_route. Also added another script: lustre_routes_conversion, which will be used to convert from legacy syntax for configuring routes to the new syntax described above. The script can be run on a file which contains the legacy syntax and will generate a new file with the passed in name: lustre_routes_conversion Added two man pages to describe the usages of these scripts Added a test case in conf-sanity.sh to test the new scripts lustre_routes_conversion and lustre_routes_config. The test case takes in a sample routes file which has the old synatx, run the lustre_routes_conversion script to convert to the new syntax, then runs the lustre_routes_config script in --dry-run to ensure that the script configures the routes Signed-off-by: Amir Shehata Change-Id: I57f81176c2d926fedefa8ea3be34586aa1ac9d76 Reviewed-on: http://review.whamcloud.com/5757 Reviewed-by: Andreas Dilger Tested-by: Hudson Reviewed-by: Doug Oucharek Tested-by: Maloo --- lustre/doc/Makefile.am | 3 +- lustre/doc/lustre_routes_config.8 | 62 ++++++++++++++ lustre/doc/lustre_routes_conversion.8 | 47 +++++++++++ lustre/scripts/Makefile.am | 1 + lustre/scripts/lnet | 9 ++ lustre/scripts/lustre_routes_config | 142 ++++++++++++++++++++++++++++++++ lustre/scripts/lustre_routes_conversion | 110 +++++++++++++++++++++++++ lustre/tests/conf-sanity.sh | 58 +++++++++++++ 8 files changed, 431 insertions(+), 1 deletion(-) create mode 100644 lustre/doc/lustre_routes_config.8 create mode 100644 lustre/doc/lustre_routes_conversion.8 create mode 100755 lustre/scripts/lustre_routes_config create mode 100755 lustre/scripts/lustre_routes_conversion diff --git a/lustre/doc/Makefile.am b/lustre/doc/Makefile.am index 6511ea7..5cda6be 100644 --- a/lustre/doc/Makefile.am +++ b/lustre/doc/Makefile.am @@ -54,7 +54,8 @@ MANFILES = lustre.7 lfs.1 mount.lustre.8 lctl.8 \ ll_recover_lost_found_objs.8 llog_reader.8 llapi_file_open.3 \ llapi_file_create.3 llapi_file_get_stripe.3 lustreapi.7 \ lustre_rsync.8 lfs_migrate.1 lhbadm.8 ldev.8 ldev.conf.5 nids.5 \ - lfs-hsm.1 llapi_hsm_state_get.3 llapi_hsm_state_set.3 + lfs-hsm.1 llapi_hsm_state_get.3 llapi_hsm_state_set.3 \ + lustre_routes_config.8 lustre_routes_conversion.8 if SERVER MANFILES += mkfs.lustre.8 tunefs.lustre.8 diff --git a/lustre/doc/lustre_routes_config.8 b/lustre/doc/lustre_routes_config.8 new file mode 100644 index 0000000..bd7eb7b --- /dev/null +++ b/lustre/doc/lustre_routes_config.8 @@ -0,0 +1,62 @@ +.TH lustre_routes_config 8 "Apr 23, 2013" Lustre "utilities" +.SH NAME +lustre_routes_config \- Configure routes dynamically +.SH SYNOPSIS +.B "lustre_routes_config" +.SH DESCRIPTION +lustre_route_config sets or cleans up LNET routes from the specified config +file. /etc/sysconfig/lustre_routes.conf file can be used to automatically +configure routes on LNET startup. +.LP +The format of the config file is: +.br +: { gateway: @ [hop: ] + [priority: ] } +.LP +Usage: +.br +lustre_routes_config [--setup|--cleanup|--dry-run|--verbose] +.TP +.I "\-\-setup" +Configure routes listed in config_file +.TP +.I "\-\-cleanup" +Unconfigure routes listed in config_file +.TP +.I "\-\-dry-run" +Echo commands to be run, but do not execute them +.TP +.I "\-\-verbose" +Echo commands before they are executed +.LP +NOTE: An LNET router is identified when its local NID appears within the +list of routes. However, this can not be achieved by the use of this +script, since the script only adds extra routes after the role of the +router is determined. To ensure that a router is identified correctly, +make sure to add its local NID in the routes parameter in the modprobe +lustre configuration file. +.SH EXAMPLE +An example of a config file that the script expects: +.LP +tcp1: { gateway: 10.1.1.2@tcp0, priority: 3 } +.br +tcp4: { gateway: 10.3.3.4@tcp } +.br +tcp6: { gateway: 10.3.3.6@tcp, hop: 2, priority: 5 } +.br +tcp7: { gateway: 10.3.3.[6-12]@tcp, priority: 20, hop: 8 } +.LP +An example of script execution: +.LP +lustre_routes_config --setup +.br +lustre_routes_config --cleanup +.SH SEE ALSO +.BR lustre (7) +.BR nids (5) +.BR lctl (8) +.BR lustre_routes_conversion (8) +.SH FILES +/etc/sysconfig/lustre_routes.conf +.SH AUTHOR +Amir Shehata diff --git a/lustre/doc/lustre_routes_conversion.8 b/lustre/doc/lustre_routes_conversion.8 new file mode 100644 index 0000000..7a02e4e --- /dev/null +++ b/lustre/doc/lustre_routes_conversion.8 @@ -0,0 +1,47 @@ +.TH lustre_routes_conversion 8 "Apr 23, 2013" Lustre "utilities" +.SH NAME +lustre_routes_conversion \- converts a legacy routes configuration file to +the new syntax. +.SH SYNOPSIS +.B "lustre_routes_conversion" +.SH DESCRIPTION +lustre_route_conversion takes as a first parameter a file with routes +configured as follows: +.LP + [] @[:]; +.LP +Then converts it to: +.LP +: { gateway: @ [hop: ] + [priority: ] } +.LP +and appends it to the output file passed in as the second parameter to +the script. +.LP +Usage: +.br +lustre_routes_conversion +.SH EXAMPLE +An example of legacy configuration file: +.LP +tcp1 10.1.1.2@tcp0:1; +.br +tcp1 1 10.1.1.2@tcp0; +.LP + +An example of script output: +.LP +tcp1: {gateway: 10.1.1.2@tcp0 priority: 1} +.br +tcp1: {gateway: 10.1.1.2@tcp0 hop: 1} +.LP +An example of script execution: +.LP +lustre_routes_conversion +.SH SEE ALSO +.BR lustre (7) +.BR nids (5) +.BR lctl (8) +.BR lustre_routes_config (8) +.SH AUTHOR +Amir Shehata diff --git a/lustre/scripts/Makefile.am b/lustre/scripts/Makefile.am index 970d4c9..b35463f 100644 --- a/lustre/scripts/Makefile.am +++ b/lustre/scripts/Makefile.am @@ -35,6 +35,7 @@ # sbinscripts = lc_servip lustre_up14 lustre_rmmod lhbadm ldev +sbinscripts += lustre_routes_config lustre_routes_conversion # These are scripts that are generated from .in files genscripts = lustre_config lc_modprobe lc_net lc_hb lc_cluman lustre_createcsv \ diff --git a/lustre/scripts/lnet b/lustre/scripts/lnet index 7033891..62cbb39 100644 --- a/lustre/scripts/lnet +++ b/lustre/scripts/lnet @@ -168,6 +168,8 @@ status () eval $old_nullglob } +LUSTRE_ROUTES_CONFIG_FILE="/etc/sysconfig/lnet_routes.conf" + # See how we were called. case "$1" in start) @@ -175,6 +177,13 @@ case "$1" in touch /var/lock/subsys/lnet modprobe lnet || exit 1 lctl network up || exit 1 + # if a routes config file is given then use it to configure the + # routes if not then default to LUSTRE_ROUTES_CONFIG_FILE + if [ -f "$2" ]; then + lustre_routes_config $2 + elif [ -f "$LUSTRE_ROUTES_CONFIG_FILE" ]; then + lustre_routes_config $LUSTRE_ROUTES_CONFIG_FILE + fi run_postexec_check "start" ;; stop) diff --git a/lustre/scripts/lustre_routes_config b/lustre/scripts/lustre_routes_config new file mode 100755 index 0000000..f60ed11 --- /dev/null +++ b/lustre/scripts/lustre_routes_config @@ -0,0 +1,142 @@ +# !/bin/bash +# +# lustre_route_config +# This script configures lnet with the routes in the passed in file. +# The routes should be in the following format: +# : { gateway: @ [hop: ] [priority: ] } +# +# Examples: +# tcp1: { gateway: 10.1.1.2@tcp0, priority: 3 } +# tcp4: { gateway: 10.3.3.4@tcp } +# tcp6: { gateway: 10.3.3.6@tcp, hop: 2, priority: 5 } +# tcp7: { gateway: 10.3.3.[6-12]@tcp, priority: 20, hop: 8 } +# +# The purpose of this script is to circumvent the limitation on the number of +# routes which could be configured through the lustre.conf module parameters. +# +########################################################################### + + +progname=$(basename $0) + +# print usage +usage() { + cat <<- USAGE + Setup or cleanup LNET routes from specified config file" + usage: $progname [--setup|--cleanup|--dry-run|--verbose] + + --setup: configure routes listed in config_file + --cleanup: unconfigure routes listed in config_file + --dry-run: echo commands to be run, but do not execute them + --verbose: echo commands before they are executed + USAGE +} + +# Set default paramters +CMD=add_route +VERBOSE=false +EXEC=true + +# sanity check +[ -z "$1" ] && usage && exit 1 + +# check parameters +while [ ! -f "$1" ]; do + case "$1" in + -c|--cleanup) CMD=del_route; shift ;; + -h|--help) usage; exit 0 ;; + -n|--dry-run) EXEC=false; VERBOSE=true; shift ;; + -s|--setup) CMD=add_route; shift ;; + -v|--verbose) VERBOSE=true; shift ;; + *) usage; exit 1 ;; + esac +done + +# Usage: do_lctl +# execut the command and/or print if verbose is set +do_lctl() { + local RC=0 + + $VERBOSE && echo "lctl $@" + if $EXEC; then + lctl "$@" + RC=$? + fi + + return $RC +} + +# Usage: find_arg_value +find_arg_value() { + local i=0 + local value="" + local arg="$2" + + declare -a array=("${!1}") + for ((i = 0; i < ${#array[@]}; i++)); do + if [ "${array[$i]}" == "$arg" ]; then + value="${array[$((i + 1))]}" + break + fi + done + echo -n $value +} + +while read line; do + # Parse line using ':' and ',' as delimiters and ignoring all + # white space, tabs and linefeed + IFS="$IFS:," + params=($line) + + # get the mandatory parameters: network and gateway + # If either is not present skip that line + network=${params[0]} + OBR=${params[1]} + GATE=${params[2]} + gateway=${params[3]} + + if [ -z $network ] || [ -z $gateway ] || + [ $GATE != "gateway" ]; then + continue + fi + + case "$CMD" in + add_route) + baselctl="--net $network add_route $gateway" + + # walk through the optional params until you hit + # the closing brace. Build an associative db: + # option=value + i=4 + while [ $i -lt ${#params[@]} ]; do + option=${params[$i]} + if [ "$option" == "}" ]; then + break + fi + outoptions[$i]=$option + ((i++)) + value=${params[$i]} + outoptions[$i]=$value + ((i++)) + done + + # find the hop and priority + # This can be expanded later on if we add extra + # parameters + # NOTE: the order between hop and priority is not + # enforced. It's also possible to add hop without + # prio or prio without hop + priority=$(find_arg_value outoptions[@] "priority") + hop=$(find_arg_value outoptions[@] "hop") + if [ -n "$priority" ] && [ -z "$hop" ]; then + baselctl+=" 1 $priority" + else + baselctl+=" $hop $priority" + fi + ;; + del_route) + baselctl="del_route $gateway" + esac + + do_lctl $baselctl +done < "$1" diff --git a/lustre/scripts/lustre_routes_conversion b/lustre/scripts/lustre_routes_conversion new file mode 100755 index 0000000..312c452 --- /dev/null +++ b/lustre/scripts/lustre_routes_conversion @@ -0,0 +1,110 @@ +# !/bin/bash +# +# lustre_routes_conversion +# This script takes a file with routes configured as follows: +# [] @[:]; +# Ex: +# tcp1 10.1.1.2@tcp0:1 +# or +# tcp1 1 10.1.1.2@tcp0 +# +# and converts it to: +# : { gateway: @ [hop: ] [priority: +# ] } +# +# The purpose of this script is to covert legacy route configuration +# syntax to the new route configuration syntax +# +############################################################################ + +progname=$(basename $0) + +usage() { + cat <<- USAGE + convert legacy route config syntax to new route config syntax" + usage: $progname + -h|--help: display this message + USAGE +} + +while [ ! -f "$1" ]; do + case "$1" in + -h|--help) usage; exit 0 ;; + *) usage; exit 1 ;; + esac +done + +[ -z "$1" ] || [ -z "$2" ] && usage && exit 1 + +# Usage: read_and_parse +# Read a routes_config file and parse it out, then feed the proper input +# int lcl --net <> add_route <> to configure a route. +read_and_parse() +{ + local infile=$1 + local outfile=$2 + + while read line; do + # Split the input string at ';', since multiple routes on + # the same line are separated by ';' + OLDIFS="$IFS" + IFS=';' + + # It is possible that one single line can contain multiple + # route entries. + multi_routes=($line) + echo "${multi_routes[@]}" + + # Iterate over each of the routes on this line. This + # returns indicies from the routes[] array, which are + # dereferenced and split separately to avoid confusion + # between whitespaces of routes on the same line. + for index in "${!multi_routes[@]}"; do + # initialize variables. + local network="" + local gateway="" + local gatewayorhop="" + local priority="" + local hop="" + + # Split at ':' and ' ' to get the priority if it exists + # Also will split all the different tokens in the + # line. + IFS="$OLDIFS: " + tokens=(${multi_routes[$index]}) + + # Split at ' ' to separate the network from the gateway + network=${tokens[0]} + gatewayorhop=${tokens[1]} + + # since hop is an optional parameter after we get this + # position we need to check if we got the hop or gateway + # parameter. Set gateway is always of the form ip@intf, + # then we can simply check for the '@' character in the + # string. if it exists then we don't have a hop but a + # gateway. If we don't then we assume that a hop exists + # and a gateway follows it + if [[ "$gatewayorhop" == *@* ]]; then + gateway=$gatewayorhop + priority=${tokens[2]} + else + hop=$gatewayorhop + gateway=${tokens[2]} + priority=${tokens[3]} + fi + + if [ -z "$network" ] || [ -z "$gateway" ]; then + continue; + fi + + # Write the translated line into the file. + echo -n "$network: { gateway: $gateway" + [ -n "$hop" ] && echo -n ", hop: $hop" + [ -n "$priority" ] && echo -n ", priority: $priority" + echo " }" + done >> "$outfile" + done < "$infile" + echo "$progname: converted routes written to $outfile" +} + +read_and_parse $1 $2 diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 52a4031..655329f 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -3705,6 +3705,64 @@ test_66() { } run_test 66 "replace nids" +test_67() { #LU-2950 + local legacy="$TMP/legacy_lnet_config" + local new="$TMP/new_routes_test" + local out="$TMP/config_out_file" + local verify="$TMP/conv_verify" + local verify_conf="$TMP/conf_verify" + + # Create the legacy file that will be run through the + # lustre_routes_conversion script + cat <<- LEGACY_LNET_CONFIG > $legacy + tcp1 23 192.168.213.1@tcp:1; tcp5 34 193.30.4.3@tcp:4; + tcp2 54 10.1.3.2@tcp; + tcp3 10.3.4.3@tcp:3; + tcp4 10.3.3.4@tcp; + LEGACY_LNET_CONFIG + + # Create the verification file to verify the output of + # lustre_routes_conversion script against. + cat <<- VERIFY_LNET_CONFIG > $verify + tcp1: { gateway: 192.168.213.1@tcp, hop: 23, priority: 1 } + tcp5: { gateway: 193.30.4.3@tcp, hop: 34, priority: 4 } + tcp2: { gateway: 10.1.3.2@tcp, hop: 54 } + tcp3: { gateway: 10.3.4.3@tcp, priority: 3 } + tcp4: { gateway: 10.3.3.4@tcp } + VERIFY_LNET_CONFIG + + # Create the verification file to verify the output of + # lustre_routes_config script against + cat <<- VERIFY_LNET_CONFIG > $verify_conf + lctl --net tcp1 add_route 192.168.213.1@tcp 23 1 + lctl --net tcp5 add_route 193.30.4.3@tcp 34 4 + lctl --net tcp2 add_route 10.1.3.2@tcp 54 4 + lctl --net tcp3 add_route 10.3.4.3@tcp 1 3 + lctl --net tcp4 add_route 10.3.3.4@tcp 1 3 + VERIFY_LNET_CONFIG + + lustre_routes_conversion $legacy $new > /dev/null + if [ -f $new ]; then + # verify the conversion output + cmp -s $new $verify > /dev/null + if [ $? -eq 1 ]; then + error "routes conversion failed" + fi + + lustre_routes_config --dry-run --verbose $new > $out + # check that the script succeeded + cmp -s $out $verify_conf > /dev/null + if [ $? -eq 1 ]; then + error "routes config failed" + fi + else + error "routes conversion test failed" + fi + # remove generated files + rm -f $new $legacy $verify $verify_conf $out +} +run_test 67 "test routes conversion and configuration" + test_70a() { [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return local MDTIDX=1 -- 1.8.3.1