X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=0165fee28aed461a7d871aec1e5294be6b2812c8;hb=8f6d85eebc312b64d8e8a35b0be3ae137a50a45c;hp=6925483a271638a031dd99c54429effb4ee35fc1;hpb=a0e86750d3fc4498f74573cb95aa2d821034bbdd;p=fs%2Flustre-release.git

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 6925483..0165fee 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -34,6 +34,15 @@ if [ -f "$EXCEPT_LIST_FILE" ]; then
     . $EXCEPT_LIST_FILE
 fi
 
+if [[ -x $LUSTRE/utils/llobdstat ]]; then
+        LLOBDSTAT=$LUSTRE/utils/llobdstat
+elif [[ -x /usr/bin/llobdstat ]]; then
+        LLOBDSTAT=/usr/bin/llobdstat
+else
+        # Good luck
+        LLOBDSTAT=llobdstat
+fi
+
 [ -z "$MODPROBECONF" -a -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf
 [ -z "$MODPROBECONF" -a -f /etc/modprobe.d/Lustre ] && MODPROBECONF=/etc/modprobe.d/Lustre
 
@@ -404,7 +413,7 @@ check_gss_daemon_nodes() {
     local list=$1
     dname=$2
 
-    do_nodes --verbose $list "num=\\\$(ps -o cmd -C $dname | grep $dname | wc -l);
+    do_nodesv $list "num=\\\$(ps -o cmd -C $dname | grep $dname | wc -l);
 if [ \\\"\\\$num\\\" -ne 1 ]; then
     echo \\\$num instance of $dname;
     exit 1;
@@ -950,6 +959,8 @@ start_client_loads () {
         testnum=$((nodenum % numloads))
         start_client_load ${clients[nodenum]} ${CLIENT_LOADS[testnum]}
     done
+    # bug 22169: wait the background threads to start
+    sleep 2
 }
 
 # only for remote client 
@@ -1535,6 +1546,10 @@ do_node() {
     return ${PIPESTATUS[0]}
 }
 
+do_nodev() {
+    do_node --verbose "$@"
+}
+
 single_local_node () {
    [ "$1" = "$HOSTNAME" ]
 }
@@ -1563,7 +1578,7 @@ do_nodes() {
 
     if single_local_node $rnodes; then
         if $verbose; then
-           do_node --verbose $rnodes "$@"
+           do_nodev $rnodes "$@"
         else
            do_node $rnodes "$@"
         fi
@@ -1597,6 +1612,10 @@ do_facet() {
     do_node $HOST "$@"
 }
 
+do_nodesv() {
+    do_nodes --verbose "$@"
+}
+
 add() {
     local facet=$1
     shift
@@ -2597,13 +2616,8 @@ run_test() {
     return $?
 }
 
-EQUALS="======================================================================"
 equals_msg() {
-    msg="$@"
-
-    local suffixlen=$((${#EQUALS} - ${#msg}))
-    [ $suffixlen -lt 5 ] && suffixlen=5
-    log `echo $(printf '===== %s %.*s\n' "$msg" $suffixlen $EQUALS)`
+    banner "$*"
 }
 
 log() {
@@ -2658,18 +2672,15 @@ reset_fail_loc () {
 # Log a message (on all nodes) padded with "=" before and after. 
 # Also appends a timestamp and prepends the testsuite name.
 # 
+
+EQUALS="===================================================================================================="
 banner() {
     msg="== ${TESTSUITE} $*"
-    # pad the message out to 70 with "="
     last=${msg: -1:1}
-    [[ $last != "=" && $last != " " ]] && msg+=" "
-    for i in $(seq $((68 - ${#msg})) ); do
-	msg+="="
-    done
+    [[ $last != "=" && $last != " " ]] && msg="$msg "
+    msg=$(printf '%s%.*s'  "$msg"  $((${#EQUALS} - ${#msg})) $EQUALS )
     # always include at least == after the message
-    msg+="=="
-
-    log "$msg $(date +"%H:%M:%S (%s)")"
+    log "$msg== $(date +"%H:%M:%S (%s)")"
 }
 
 #
@@ -2714,7 +2725,7 @@ run_one_logged() {
     rm -rf $LOGDIR/err
 
     echo
-    run_one $1 "$2" 2>&1 | tee $test_log
+    (run_one $1 "$2") 2>&1 | tee $test_log
     local RC=${PIPESTATUS[0]}
 
     [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \
@@ -2979,9 +2990,10 @@ mixed_mdt_devs () {
 generate_machine_file() {
     local nodes=${1//,/ }
     local machinefile=$2
-    rm -f $machinefile || error "can't rm $machinefile"
+    rm -f $machinefile
     for node in $nodes; do
-        echo $node >>$machinefile
+        echo $node >>$machinefile || \
+            { echo "can not generate machinefile $machinefile" && return 1; }
     done
 }
 
@@ -3000,7 +3012,7 @@ setstripe_nfsserver () {
 
     [ -z $nfsserver ] && echo "$dir is not nfs mounted" && return 1
 
-    do_node --verbose $nfsserver lfs setstripe "$@"
+    do_nodev $nfsserver lfs setstripe "$@"
 }
 
 check_runas_id_ret() {
@@ -3152,7 +3164,7 @@ calc_osc_kbytes () {
 # generate a stream of formatted strings (<node> <param name>=<param value>)
 save_lustre_params() {
         local s
-        do_nodes --verbose $1 "lctl get_param $2 | while read s; do echo \\\$s; done"
+        do_nodesv $1 "lctl get_param $2 | while read s; do echo \\\$s; done"
 }
 
 # restore lustre parameters from input stream, produces by save_lustre_params
@@ -3300,12 +3312,13 @@ get_osc_import_name() {
 }
 
 wait_import_state () {
-    local expected=$1
-    local CONN_PROC=$2
+    local facet=$1
+    local expected=$2
+    local CONN_PROC=$3
     local CONN_STATE
     local i=0
 
-    CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+    CONN_STATE=$(do_facet $facet $LCTL get_param -n $CONN_PROC | awk '/state/ {print $2}')
     while [ "${CONN_STATE}" != "${expected}" ]; do
         if [ "${expected}" == "DISCONN" ]; then
             # for disconn we can check after proc entry is removed
@@ -3318,7 +3331,7 @@ wait_import_state () {
         [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
             error "can't put import for $CONN_PROC into ${expected} state" && return 1
         sleep 1
-        CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+        CONN_STATE=$(do_facet $facet $LCTL get_param -n $CONN_PROC | awk '/state/ {print $2}')
         i=$(($i + 1))
     done
 
@@ -3331,33 +3344,26 @@ wait_osc_import_state() {
     local ost_facet=$2
     local expected=$3
     local ost=$(get_osc_import_name $facet $ost_facet)
-    local CONN_PROC
-    local CONN_STATE
-    local i=0
-
-    CONN_PROC="osc.${ost}.ost_server_uuid"
-    CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
-    while [ "${CONN_STATE}" != "${expected}" ]; do
-        if [ "${expected}" == "DISCONN" ]; then 
-            # for disconn we can check after proc entry is removed
-            [ "x${CONN_STATE}" == "x" ] && return 0
-            #  with AT we can have connect request timeout ~ reconnect timeout
-            # and test can't see real disconnect
-            [ "${CONN_STATE}" == "CONNECTING" ] && return 0
-        fi
-        # disconnect rpc should be wait not more obd_timeout
-        [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
-            error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1
-        sleep 1
-        CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
-        i=$(($i + 1))
-    done
+    local CONN_PROC="osc.${ost}.import"
 
-    log "${ost_facet} now in ${CONN_STATE} state"
+    wait_import_state $facet $expected $CONN_PROC || return 1
     return 0
 }
+
 get_clientmdc_proc_path() {
-    echo "${1}-mdc-*"
+    local mdc=$(convert_facet2label $1)
+
+    echo "${mdc}-mdc-*"
+}
+
+wait_mdc_import_state() {
+    local facet=$1
+    local expected=$2
+    local mdc=$(get_clientmdc_proc_path $facet)
+    local CONN_PROC="mdc.${mdc}.import"
+
+    wait_import_state client $expected $CONN_PROC || return 1
+    return 0
 }
 
 do_rpc_nodes () {
@@ -3366,27 +3372,32 @@ do_rpc_nodes () {
 
     # Add paths to lustre tests for 32 and 64 bit systems.
     local RPATH="$RLUSTRE/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:$PATH"
-    do_nodes --verbose $list "PATH=$RPATH sh rpc.sh $@ "
+    do_nodesv $list "PATH=$RPATH sh rpc.sh $@ "
 }
 
-wait_clients_import_state () {
-    local list=$1
-    local facet=$2
-    local expected=$3
+wait_client_import_state () {
+    local facet=$1
+    local expected=$2
     shift
 
-    local label=$(convert_facet2label $facet)
-    local proc_path
     case $facet in
-        ost* ) proc_path="osc.$(get_clientosc_proc_path $label).ost_server_uuid" ;;
-        mds* ) proc_path="mdc.$(get_clientmdc_proc_path $label).mds_server_uuid" ;;
-        *) error "unknown facet!" ;;
+        ost* ) wait_osc_import_state client $facet $expected || return 1;;
+        mds* ) wait_mdc_import_state $facet $expected || return 1 ;;
+           * ) error "unknown facet!"
+               return 1 ;;
     esac
+    return 0
+}
+
+wait_clients_import_state () {
+    local list=$1
+    shift
 
-    if ! do_rpc_nodes $list wait_import_state $expected $proc_path; then
-        error "import is not in ${expected} state"
+    if ! do_rpc_nodes $list wait_client_import_state "$@"; then
+        error "import is not in expected state"
         return 1
     fi
+    return 0
 }
 
 oos_full() {
@@ -3527,7 +3538,7 @@ gather_logs () {
         return
     fi
 
-    do_nodes --verbose $list \
+    do_nodesv $list \
         "$LCTL dk > ${prefix}.debug_log.\\\$(hostname).${suffix};
          dmesg > ${prefix}.dmesg.\\\$(hostname).${suffix}"
     if [ ! -f $LOGDIR/shared ]; then
@@ -3914,3 +3925,17 @@ log_sub_test() {
     yml_log_sub_test $@ >> $YAML_LOG
 }
 
+run_llverdev()
+{
+        local dev=$1
+        local devname=$(basename $1)
+        local size=$(grep "$devname"$ /proc/partitions | awk '{print $3}')
+        size=$(($size / 1024 / 1024)) # Gb
+
+        local partial_arg=""
+        # Run in partial (fast) mode if the size
+        # of a partition > 10 GB
+        [ $size -gt 10 ] && partial_arg="-p"
+
+        llverdev --force $partial_arg $dev
+}