+# gss functions
+PROC_CLI="srpc_info"
+
+combination()
+{
+ local M=$1
+ local N=$2
+ local R=1
+
+ if [ $M -lt $N ]; then
+ R=0
+ else
+ N=$((N + 1))
+ while [ $N -le $M ]; do
+ R=$((R * N))
+ N=$((N + 1))
+ done
+ fi
+
+ echo $R
+ return 0
+}
+
+calc_connection_cnt() {
+ local dir=$1
+
+ # MDT->MDT = 2 * C(M, 2)
+ # MDT->OST = M * O
+ # CLI->OST = C * O
+ # CLI->MDT = C * M
+ comb_m2=$(combination $MDSCOUNT 2)
+
+ local num_clients=$(get_clients_mount_count)
+
+ local cnt_mdt2mdt=$((comb_m2 * 2))
+ local cnt_mdt2ost=$((MDSCOUNT * OSTCOUNT))
+ local cnt_cli2ost=$((num_clients * OSTCOUNT))
+ local cnt_cli2mdt=$((num_clients * MDSCOUNT))
+ local cnt_all2ost=$((cnt_mdt2ost + cnt_cli2ost))
+ local cnt_all2mdt=$((cnt_mdt2mdt + cnt_cli2mdt))
+ local cnt_all2all=$((cnt_mdt2ost + cnt_mdt2mdt + cnt_cli2ost + cnt_cli2mdt))
+
+ local var=cnt_$dir
+ local res=${!var}
+
+ echo $res
+}
+
+set_rule()
+{
+ local tgt=$1
+ local net=$2
+ local dir=$3
+ local flavor=$4
+ local cmd="$tgt.srpc.flavor"
+
+ if [ $net == "any" ]; then
+ net="default"
+ fi
+ cmd="$cmd.$net"
+
+ if [ $dir != "any" ]; then
+ cmd="$cmd.$dir"
+ fi
+
+ cmd="$cmd=$flavor"
+ log "Setting sptlrpc rule: $cmd"
+ do_facet mgs "$LCTL conf_param $cmd"
+}
+
+count_flvr()
+{
+ local output=$1
+ local flavor=$2
+ local count=0
+
+ rpc_flvr=`echo $flavor | awk -F - '{ print $1 }'`
+ bulkspec=`echo $flavor | awk -F - '{ print $2 }'`
+
+ count=`echo "$output" | grep "rpc flavor" | grep $rpc_flvr | wc -l`
+
+ if [ "x$bulkspec" != "x" ]; then
+ algs=`echo $bulkspec | awk -F : '{ print $2 }'`
+
+ if [ "x$algs" != "x" ]; then
+ bulk_count=`echo "$output" | grep "bulk flavor" | grep $algs | wc -l`
+ else
+ bulk=`echo $bulkspec | awk -F : '{ print $1 }'`
+ if [ $bulk == "bulkn" ]; then
+ bulk_count=`echo "$output" | grep "bulk flavor" \
+ | grep "null/null" | wc -l`
+ elif [ $bulk == "bulki" ]; then
+ bulk_count=`echo "$output" | grep "bulk flavor" \
+ | grep "/null" | grep -v "null/" | wc -l`
+ else
+ bulk_count=`echo "$output" | grep "bulk flavor" \
+ | grep -v "/null" | grep -v "null/" | wc -l`
+ fi
+ fi
+
+ [ $bulk_count -lt $count ] && count=$bulk_count
+ fi
+
+ echo $count
+}
+
+flvr_cnt_cli2mdt()
+{
+ local flavor=$1
+ local cnt
+
+ local clients=${CLIENTS:-`hostname`}
+
+ for c in ${clients//,/ }; do
+ output=`do_node $c lctl get_param -n mdc.*-MDT*-mdc-*.$PROC_CLI 2>/dev/null`
+ tmpcnt=`count_flvr "$output" $flavor`
+ cnt=$((cnt + tmpcnt))
+ done
+ echo $cnt
+}
+
+flvr_cnt_cli2ost()
+{
+ local flavor=$1
+ local cnt
+
+ local clients=${CLIENTS:-`hostname`}
+
+ for c in ${clients//,/ }; do
+ output=`do_node $c lctl get_param -n osc.*OST*-osc-[^M][^D][^T]*.$PROC_CLI 2>/dev/null`
+ tmpcnt=`count_flvr "$output" $flavor`
+ cnt=$((cnt + tmpcnt))
+ done
+ echo $cnt
+}
+
+flvr_cnt_mdt2mdt()
+{
+ local flavor=$1
+ local cnt=0
+
+ if [ $MDSCOUNT -le 1 ]; then
+ echo 0
+ return
+ fi
+
+ for num in `seq $MDSCOUNT`; do
+ output=`do_facet mds$num lctl get_param -n mdc.*-MDT*-mdc[0-9]*.$PROC_CLI 2>/dev/null`
+ tmpcnt=`count_flvr "$output" $flavor`
+ cnt=$((cnt + tmpcnt))
+ done
+ echo $cnt;
+}
+
+flvr_cnt_mdt2ost()
+{
+ local flavor=$1
+ local cnt=0
+
+ for num in `seq $MDSCOUNT`; do
+ output=`do_facet mds$num lctl get_param -n osc.*OST*-osc-MDT*.$PROC_CLI 2>/dev/null`
+ tmpcnt=`count_flvr "$output" $flavor`
+ cnt=$((cnt + tmpcnt))
+ done
+ echo $cnt;
+}
+
+flvr_cnt_mgc2mgs()
+{
+ local flavor=$1
+
+ output=`do_facet client lctl get_param -n mgc.*.$PROC_CLI 2>/dev/null`
+ count_flvr "$output" $flavor
+}
+
+do_check_flavor()
+{
+ local dir=$1 # from to
+ local flavor=$2 # flavor expected
+ local res=0
+
+ if [ $dir == "cli2mdt" ]; then
+ res=`flvr_cnt_cli2mdt $flavor`
+ elif [ $dir == "cli2ost" ]; then
+ res=`flvr_cnt_cli2ost $flavor`
+ elif [ $dir == "mdt2mdt" ]; then
+ res=`flvr_cnt_mdt2mdt $flavor`
+ elif [ $dir == "mdt2ost" ]; then
+ res=`flvr_cnt_mdt2ost $flavor`
+ elif [ $dir == "all2ost" ]; then
+ res1=`flvr_cnt_mdt2ost $flavor`
+ res2=`flvr_cnt_cli2ost $flavor`
+ res=$((res1 + res2))
+ elif [ $dir == "all2mdt" ]; then
+ res1=`flvr_cnt_mdt2mdt $flavor`
+ res2=`flvr_cnt_cli2mdt $flavor`
+ res=$((res1 + res2))
+ elif [ $dir == "all2all" ]; then
+ res1=`flvr_cnt_mdt2ost $flavor`
+ res2=`flvr_cnt_cli2ost $flavor`
+ res3=`flvr_cnt_mdt2mdt $flavor`
+ res4=`flvr_cnt_cli2mdt $flavor`
+ res=$((res1 + res2 + res3 + res4))
+ fi
+
+ echo $res
+}
+
+wait_flavor()
+{
+ local dir=$1 # from to
+ local flavor=$2 # flavor expected
+ local expect=${3:-$(calc_connection_cnt $dir)} # number expected
+
+ local res=0
+
+ for ((i=0;i<20;i++)); do
+ echo -n "checking..."
+ res=$(do_check_flavor $dir $flavor)
+ if [ $res -eq $expect ]; then
+ echo "found $res $flavor connections of $dir, OK"
+ return 0
+ else
+ echo "found $res $flavor connections of $dir, not ready ($expect)"
+ sleep 4
+ fi
+ done
+
+ echo "Error checking $flavor of $dir: expect $expect, actual $res"
+ return 1
+}
+
+restore_to_default_flavor()
+{
+ local proc="mgs.MGS.live.$FSNAME"
+
+ echo "restoring to default flavor..."
+
+ nrule=`do_facet mgs lctl get_param -n $proc 2>/dev/null | grep ".srpc.flavor." | wc -l`
+
+ # remove all existing rules if any
+ if [ $nrule -ne 0 ]; then
+ echo "$nrule existing rules"
+ for rule in `do_facet mgs lctl get_param -n $proc 2>/dev/null | grep ".srpc.flavor."`; do
+ echo "remove rule: $rule"
+ spec=`echo $rule | awk -F = '{print $1}'`
+ do_facet mgs "$LCTL conf_param $spec="
+ done
+ fi
+
+ # verify no rules left
+ nrule=`do_facet mgs lctl get_param -n $proc 2>/dev/null | grep ".srpc.flavor." | wc -l`
+ [ $nrule -ne 0 ] && error "still $nrule rules left"
+
+ # wait for default flavor to be applied
+ # currently default flavor for all connections are 'null'
+ wait_flavor all2all null
+ echo "now at default flavor settings"
+}
+
+set_flavor_all()
+{
+ local flavor=${1:-null}
+
+ echo "setting all flavor to $flavor"
+
+ # FIXME need parameter to this fn
+ # and remove global vars
+ local cnt_all2all=$(calc_connection_cnt all2all)
+
+ local res=$(do_check_flavor all2all $flavor)
+ if [ $res -eq $cnt_all2all ]; then
+ echo "already have total $res $flavor connections"
+ return
+ fi
+
+ echo "found $res $flavor out of total $cnt_all2all connections"
+ restore_to_default_flavor
+
+ [[ $flavor = null ]] && return 0
+
+ set_rule $FSNAME any any $flavor
+ wait_flavor all2all $flavor
+}
+
+
+check_logdir() {
+ local dir=$1
+ # Checking for shared logdir
+ if [ ! -d $dir ]; then
+ # Not found. Create local logdir
+ mkdir -p $dir
+ else
+ touch $dir/node.$(hostname).yml
+ fi
+ return 0
+}
+
+check_write_access() {
+ local dir=$1
+ for node in $(nodes_list); do
+ if [ ! -f "$dir/node.${node}.yml" ]; then
+ # Logdir not accessible/writable from this node.
+ return 1
+ fi
+ done
+ return 0
+}
+
+init_logging() {
+ if [[ -n $YAML_LOG ]]; then
+ return
+ fi
+ export YAML_LOG=${LOGDIR}/results.yml
+ mkdir -p $LOGDIR
+ init_clients_lists
+
+ do_rpc_nodes $(comma_list $(nodes_list)) check_logdir $LOGDIR
+ if check_write_access $LOGDIR; then
+ touch $LOGDIR/shared
+ echo "Logging to shared log directory: $LOGDIR"
+ else
+ echo "Logging to local directory: $LOGDIR"
+ fi
+
+ yml_nodes_file $LOGDIR
+ yml_results_file >> $YAML_LOG
+}
+
+log_test() {
+ yml_log_test $1 >> $YAML_LOG
+}
+
+log_sub_test() {
+ yml_log_sub_test $@ >> $YAML_LOG