From 9ca23c48a069b07c49cac3dfb725e94b3e634e65 Mon Sep 17 00:00:00 2001 From: Jian Yu Date: Mon, 22 Apr 2013 15:26:02 +0800 Subject: [PATCH] LU-3142 tests: gather logs on passive server nodes This patch adds all_mdts_nodes(), all_osts_nodes(), all_server_nodes() and all_nodes() into the test-framework.sh to get the active and passive server nodes. By using these functions, after recovery-*-scale tests failed in failover configuration, the logs on passive server nodes can also be gathered. This patch also fixes yml_entities() to add MGS entity and entities for passive server nodes. Test-Parameters: clientdistro=el6 serverdistro=el6 clientcount=4 \ osscount=2 mdscount=2 austeroptions=-R failover=true \ useiscsi=true testlist=recovery-mds-scale Signed-off-by: Jian Yu Change-Id: Idbba6f02f4f50b7d5097a78ea6b6954d5347919a Reviewed-on: http://review.whamcloud.com/6112 Tested-by: Hudson Reviewed-by: Li Wei Tested-by: Maloo Reviewed-by: Hongchao Zhang Reviewed-by: Oleg Drokin --- lustre/tests/recovery-double-scale.sh | 12 ++- lustre/tests/recovery-mds-scale.sh | 12 ++- lustre/tests/recovery-random-scale.sh | 12 ++- lustre/tests/test-framework.sh | 155 +++++++++++++++++++++++++--------- lustre/tests/yaml.sh | 58 ++++++++----- 5 files changed, 169 insertions(+), 80 deletions(-) diff --git a/lustre/tests/recovery-double-scale.sh b/lustre/tests/recovery-double-scale.sh index a5ef06f..2f4b81d 100644 --- a/lustre/tests/recovery-double-scale.sh +++ b/lustre/tests/recovery-double-scale.sh @@ -193,13 +193,11 @@ Status: $result: rc=$rc" # stop the client loads stop_client_loads $NODES_TO_USE $LOAD_PID_FILE - if [ $rc -ne 0 ]; then - # we are interested in only on failed clients and servers - local failedclients=$(cat $END_RUN_FILE | grep -v $0) - # FIXME: need ostfailover-s nodes also for FLAVOR=OST - gather_logs $(comma_list $(osts_nodes) $(mdts_nodes) \ - $mdsfailover_HOST $failedclients) - fi + if [ $rc -ne 0 ]; then + # we are interested in only on failed clients and servers + local failedclients=$(cat $END_RUN_FILE | grep -v $0) + gather_logs $(comma_list $(all_server_nodes) $failedclients) + fi exit $rc } diff --git a/lustre/tests/recovery-mds-scale.sh b/lustre/tests/recovery-mds-scale.sh index fb6816b..2be103c 100644 --- a/lustre/tests/recovery-mds-scale.sh +++ b/lustre/tests/recovery-mds-scale.sh @@ -96,13 +96,11 @@ Status: $result: rc=$rc" # stop the client loads stop_client_loads $NODES_TO_USE $LOAD_PID_FILE - if [ $rc -ne 0 ]; then - # we are interested in only on failed clients and servers - local failedclients=$(cat $END_RUN_FILE | grep -v $0) - # FIXME: need ostfailover-s nodes also for FLAVOR=OST - gather_logs $(comma_list $(osts_nodes) $(mdts_nodes) \ - $mdsfailover_HOST $failedclients) - fi + if [ $rc -ne 0 ]; then + # we are interested in only on failed clients and servers + local failedclients=$(cat $END_RUN_FILE | grep -v $0) + gather_logs $(comma_list $(all_server_nodes) $failedclients) + fi exit $rc } diff --git a/lustre/tests/recovery-random-scale.sh b/lustre/tests/recovery-random-scale.sh index e63523e..0b88e9d 100644 --- a/lustre/tests/recovery-random-scale.sh +++ b/lustre/tests/recovery-random-scale.sh @@ -100,13 +100,11 @@ Status: $result: rc=$rc" # stop the client loads stop_client_loads $NODES_TO_USE $LOAD_PID_FILE - if [ $rc -ne 0 ]; then - # we are interested in only on failed clients and servers - local failedclients=$(cat $END_RUN_FILE | grep -v $0) - # FIXME: need ostfailover-s nodes also for FLAVOR=OST - gather_logs $(comma_list $(osts_nodes) $(mdts_nodes) \ - $mdsfailover_HOST $failedclients) - fi + if [ $rc -ne 0 ]; then + # we are interested in only on failed clients and servers + local failedclients=$(cat $END_RUN_FILE | grep -v $0) + gather_logs $(comma_list $(all_server_nodes) $failedclients) + fi exit $rc } diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 3912ed7..b881b9e 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -2432,6 +2432,24 @@ facet_active_host() { fi } +# Get the passive failover partner host of facet. +facet_passive_host() { + local facet=$1 + [[ $facet = client ]] && return + + local host=${facet}_HOST + local failover_host=${facet}failover_HOST + local active_host=$(facet_active_host $facet) + + [[ -z ${!failover_host} || ${!failover_host} = ${!host} ]] && return + + if [[ $active_host = ${!host} ]]; then + echo -n ${!failover_host} + else + echo -n ${!host} + fi +} + change_active() { local facetlist=$1 local facet @@ -3006,7 +3024,7 @@ writeconf_facet() { local dev=$2 stop ${facet} -f - rm -f ${facet}active + rm -f $TMP/${facet}active do_facet ${facet} "$TUNEFS --quiet --writeconf $dev" || return 1 return 0 } @@ -4522,64 +4540,121 @@ local_mode () $(single_local_node $(comma_list $(nodes_list))) } -mdts_nodes () { - local MDSNODES - local NODES_sort - for num in `seq $MDSCOUNT`; do - MDSNODES="$MDSNODES $(facet_host mds$num)" - done - NODES_sort=$(for i in $MDSNODES; do echo $i; done | sort -u) - - echo $NODES_sort -} - remote_servers () { remote_ost && remote_mds } +# Get the active nodes for facets. facets_nodes () { - local facets=$1 - local nodes - local NODES_sort + local facets=$1 + local facet + local nodes + local nodes_sort + local i - for facet in ${facets//,/ }; do - if [ "$FAILURE_MODE" = HARD ]; then - nodes="$nodes $(facet_active_host $facet)" - else - nodes="$nodes $(facet_host $facet)" - fi - done - NODES_sort=$(for i in $nodes; do echo $i; done | sort -u) + for facet in ${facets//,/ }; do + nodes="$nodes $(facet_active_host $facet)" + done - echo $NODES_sort + nodes_sort=$(for i in $nodes; do echo $i; done | sort -u) + echo -n $nodes_sort } -osts_nodes () { - local facets=$(get_facets OST) - local nodes=$(facets_nodes $facets) +# Get all of the active MDS nodes. +mdts_nodes () { + echo -n $(facets_nodes $(get_facets MDS)) +} - echo $nodes +# Get all of the active OSS nodes. +osts_nodes () { + echo -n $(facets_nodes $(get_facets OST)) } +# Get all of the client nodes and active server nodes. nodes_list () { - # FIXME. We need a list of clients - local myNODES=$HOSTNAME - local myNODES_sort - - # CLIENTS (if specified) contains the local client - [ -n "$CLIENTS" ] && myNODES=${CLIENTS//,/ } + local nodes=$HOSTNAME + local nodes_sort + local i - if [ "$PDSH" -a "$PDSH" != "no_dsh" ]; then - myNODES="$myNODES $(facets_nodes $(get_facets))" - fi + # CLIENTS (if specified) contains the local client + [ -n "$CLIENTS" ] && nodes=${CLIENTS//,/ } - myNODES_sort=$(for i in $myNODES; do echo $i; done | sort -u) + if [ "$PDSH" -a "$PDSH" != "no_dsh" ]; then + nodes="$nodes $(facets_nodes $(get_facets))" + fi - echo $myNODES_sort + nodes_sort=$(for i in $nodes; do echo $i; done | sort -u) + echo -n $nodes_sort } +# Get all of the remote client nodes and remote active server nodes. remote_nodes_list () { - echo $(nodes_list) | sed -re "s/\<$HOSTNAME\>//g" + echo -n $(nodes_list) | sed -re "s/\<$HOSTNAME\>//g" +} + +# Get all of the MDS nodes, including active and passive nodes. +all_mdts_nodes () { + local host + local failover_host + local nodes + local nodes_sort + local i + + for i in $(seq $MDSCOUNT); do + host=mds${i}_HOST + failover_host=mds${i}failover_HOST + nodes="$nodes ${!host} ${!failover_host}" + done + + nodes_sort=$(for i in $nodes; do echo $i; done | sort -u) + echo -n $nodes_sort +} + +# Get all of the OSS nodes, including active and passive nodes. +all_osts_nodes () { + local host + local failover_host + local nodes + local nodes_sort + local i + + for i in $(seq $OSTCOUNT); do + host=ost${i}_HOST + failover_host=ost${i}failover_HOST + nodes="$nodes ${!host} ${!failover_host}" + done + + nodes_sort=$(for i in $nodes; do echo $i; done | sort -u) + echo -n $nodes_sort +} + +# Get all of the server nodes, including active and passive nodes. +all_server_nodes () { + local nodes + local nodes_sort + local i + + nodes="$mgs_HOST $mgsfailover_HOST $(all_mdts_nodes) $(all_osts_nodes)" + + nodes_sort=$(for i in $nodes; do echo $i; done | sort -u) + echo -n $nodes_sort +} + +# Get all of the client and server nodes, including active and passive nodes. +all_nodes () { + local nodes=$HOSTNAME + local nodes_sort + local i + + # CLIENTS (if specified) contains the local client + [ -n "$CLIENTS" ] && nodes=${CLIENTS//,/ } + + if [ "$PDSH" -a "$PDSH" != "no_dsh" ]; then + nodes="$nodes $(all_server_nodes)" + fi + + nodes_sort=$(for i in $nodes; do echo $i; done | sort -u) + echo -n $nodes_sort } init_clients_lists () { diff --git a/lustre/tests/yaml.sh b/lustre/tests/yaml.sh index e987c75..61a9a14 100644 --- a/lustre/tests/yaml.sh +++ b/lustre/tests/yaml.sh @@ -17,10 +17,10 @@ yml_nodes_file() { export logdir=$1 if [ -f $logdir/shared ]; then - do_rpc_nodes $(comma_list $(nodes_list)) \ + do_rpc_nodes $(comma_list $(all_nodes)) \ "yml_node >> $logdir/node.\\\$(hostname -s).yml" else - do_rpc_nodes $(comma_list $(nodes_list)) yml_node | split_output + do_rpc_nodes $(comma_list $(all_nodes)) yml_node | split_output fi yml_entities } @@ -129,23 +129,43 @@ EOF } yml_entities() { - local host - for num in $(seq $MDSCOUNT); do - host=$(short_hostname $(facet_active_host mds$num)) - yml_entity "MDS $num" $host >> $logdir/node.$host.yml - done - - for num in $(seq $OSTCOUNT); do - host=$(short_hostname $(facet_active_host ost$num)) - yml_entity "OST $num" $host >> $logdir/node.$host.yml - done - - i=1 - for host in ${CLIENTS//,/ }; do - host=$(short_hostname $host) - yml_entity "Client $i" $host >> $logdir/node.$host.yml - i=$((i+1)) - done + local host + local f_host + local i + + if ! combined_mgs_mds; then + host=$(short_hostname $(facet_active_host mgs)) + f_host=$(short_hostname $(facet_passive_host mgs)) + + yml_entity "MGS" $host >> $logdir/node.$host.yml + [[ -n $f_host ]] && + yml_entity "MGS" $f_host >> $logdir/node.$f_host.yml + fi + + for i in $(seq $MDSCOUNT); do + host=$(short_hostname $(facet_active_host mds$i)) + f_host=$(short_hostname $(facet_passive_host mds$i)) + + yml_entity "MDS $i" $host >> $logdir/node.$host.yml + [[ -n $f_host ]] && + yml_entity "MDS $i" $f_host >> $logdir/node.$f_host.yml + done + + for i in $(seq $OSTCOUNT); do + host=$(short_hostname $(facet_active_host ost$i)) + f_host=$(short_hostname $(facet_passive_host ost$i)) + + yml_entity "OST $i" $host >> $logdir/node.$host.yml + [[ -n $f_host ]] && + yml_entity "OST $i" $f_host >> $logdir/node.$f_host.yml + done + + i=1 + for host in ${CLIENTS//,/ }; do + host=$(short_hostname $host) + yml_entity "Client $i" $host >> $logdir/node.$host.yml + i=$((i+1)) + done } yml_log_test() { -- 1.8.3.1