X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=3686c8f44b85f5578e2abb6214e0e9d698697da9;hb=960c1ec8c030cc48b3208e892849dcf5722f28aa;hp=610c26831dc9937e4c5a75bd20b69a72a213889a;hpb=17b2232d1b3796ae65e8cc27260b6a1e2b94edc8;p=fs%2Flustre-release.git

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 610c268..3686c8f 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -2,6 +2,9 @@
 
 set -e
 
+export REFORMAT=""
+export VERBOSE=false
+
 # eg, assert_env LUSTRE MDSNODES OSTNODES CLIENTS
 assert_env() {
     local failed=""
@@ -24,16 +27,19 @@ usage() {
 init_test_env() {
     export LUSTRE=`absolute_path $LUSTRE`
     export TESTSUITE=`basename $0 .sh`
-    export XMLCONFIG="${TESTSUITE}.xml"
+    export XMLCONFIG=${XMLCONFIG:-${TESTSUITE}.xml}
     export LTESTDIR=${LTESTDIR:-$LUSTRE/../ltest}
 
     [ -d /r ] && export ROOT=/r
+    export TMP=${TMP:-$ROOT/tmp}
 
     export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests
+    export LLMOUNT=${LLMOUNT:-"llmount"}
     export LCONF=${LCONF:-"lconf"}
     export LMC=${LMC:-"lmc"}
-    export LCTL=${LCTL:-"lctl"}
+    export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
     export CHECKSTAT="${CHECKSTAT:-checkstat} "
+    export FSYTPE=${FSTYPE:-"ext3"}
 
     # Paths on remote nodes, if different 
     export RLUSTRE=${RLUSTRE:-$LUSTRE}
@@ -50,8 +56,12 @@ init_test_env() {
 	esac
     done
     
+    shift $((OPTIND - 1))
+    ONLY=${ONLY:-$*}
+    
     # save the name of the config file for the upcall
     echo "XMLCONFIG=$LUSTRE/tests/$XMLCONFIG"  > $LUSTRE/tests/XMLCONFIG
+#    echo "CONFIG=`canonical_path $CONFIG`"  > $LUSTRE/tests/CONFIG
 }
 
 # Facet functions
@@ -59,29 +69,33 @@ start() {
     facet=$1
     shift
     active=`facet_active $facet`
-    do_facet $facet $LCONF --select ${facet}_svc=${active}_facet --node ${active}_facet  --ptldebug $PTLDEBUG $@ $XMLCONFIG
+    do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
+        --node ${active}_facet  --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
+        $@ $XMLCONFIG
 }
 
 stop() {
     facet=$1
     active=`facet_active $facet`
     shift
-    do_facet $facet $LCONF --select ${facet}_svc=${active}_facet --node ${active}_facet  --ptldebug $PTLDEBUG $@ --cleanup $XMLCONFIG
+    do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
+        --node ${active}_facet  --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
+        $@ --cleanup $XMLCONFIG
 }
 
 zconf_mount() {
-    mnt=$1
+    client=$1
+    mnt=$2
+
+    do_node $client mkdir $mnt 2> /dev/null || :
 
-    [ -d $mnt ] || mkdir $mnt
-    
     if [ -x /sbin/mount.lustre ] ; then
-	mount -t lustre -o nettype=$NETTYPE \
-	    `facet_host mds`:/mds_svc/client_facet $mnt
+	do_node $client mount -t lustre -o nettype=$NETTYPE `facet_active_host mds1`:/mds1_svc/client_facet $mnt || return 1
     else
        # this is so cheating
+       do_node $client $LCONF --nosetup --node client_facet $XMLCONFIG  > /dev/null || return 2
        $LCONF --nosetup --node client_facet $XMLCONFIG
-       $LUSTRE/utils/llmount `facet_host mds`:/mds_svc/client_facet $mnt \
-            -o nettype=$NETTYPE 
+       do_node $client $LLMOUNT `facet_active_host mds1`:/mds1_svc/client_facet $mnt -o nettype=$NETTYPE|| return 4
     fi
 
     [ -d /r ] && $LCTL modules > /r/tmp/ogdb-`hostname`
@@ -89,9 +103,11 @@ zconf_mount() {
 }
 
 zconf_umount() {
-    mnt=$1
-    umount  $mnt || :
-    $LCONF --cleanup --nosetup --node client_facet $XMLCONFIG || :
+    client=$1
+    mnt=$2
+    [ "$3" ] && force=-f
+    do_node $client umount $force  $mnt || :
+    do_node $client $LCONF --cleanup --nosetup --node client_facet $XMLCONFIG > /dev/null || :
 }
 
 shutdown_facet() {
@@ -114,7 +130,7 @@ reboot_facet() {
 wait_for_host() {
    HOST=$1
    check_network  $HOST 900
-   while ! do_node $HOST "$CHECKSTAT -t dir $LUSTRE"; do sleep 5; done
+   while ! do_node $HOST "ls -d $LUSTRE " > /dev/null; do sleep 5; done
 }
 
 wait_for() {
@@ -137,9 +153,10 @@ facet_failover() {
     reboot_facet $facet
     client_df &
     DFPID=$!
+    echo "df pid is $DFPID"
     change_active $facet
     TO=`facet_active_host $facet`
-    echo "Failover MDS to $TO"
+    echo "Failover $facet to $TO"
     wait_for $facet
     start $facet
 }
@@ -156,7 +173,7 @@ replay_barrier() {
 
 mds_evict_client() {
     UUID=`cat /proc/fs/lustre/mdc/*_MNT_*/uuid`
-    do_facet mds "echo $UUID > /proc/fs/lustre/mds/mds_svc/evict_client"
+    do_facet mds "echo $UUID > /proc/fs/lustre/mds/mds1_svc/evict_client"
 }
 
 fail() {
@@ -179,6 +196,12 @@ do_lmc() {
     $LMC -m ${XMLCONFIG} $@
 }
 
+h2gm () {
+   if [ "$1" = "client" ]; then echo \'*\'; else
+       $PDSH $1 $GMNALNID -l | cut -d\  -f2
+   fi
+}
+
 h2tcp() {
    if [ "$1" = "client" ]; then echo \'*\'; else
    echo $1 
@@ -223,7 +246,11 @@ facet_active() {
 facet_active_host() {
     local facet=$1
     local active=`facet_active $facet`
-    echo `facet_host $active`
+    if [ "$facet" == client ]; then
+	hostname
+    else
+	echo `facet_host $active`
+    fi
 }
 
 change_active() {
@@ -247,22 +274,37 @@ do_node() {
     shift
 
     if $VERBOSE; then
-	echo "CMD $HOST $@"
+	echo "CMD: $HOST $@"
+	$PDSH $HOST $LCTL mark "$@" > /dev/null 2>&1 || :
     fi
     $PDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests; cd $RPWD; sh -c \"$@\")"
 }
+
+mds_list() {
+    seq -f mds%g $MDSCOUNT
+}
+
 do_facet() {
     facet=$1
     shift
-    HOST=`facet_active_host $facet`
-    do_node $HOST $@
+
+    if [ "$facet" == "mds" ]; then
+	for mds in `mds_list`; do
+           HOST=`facet_active_host $mds`
+           do_node $HOST $@
+	done
+    else
+	HOST=`facet_active_host $facet`
+	do_node $HOST $@
+    fi
 }
 
 add_facet() {
     local facet=$1
     shift
     echo "add facet $facet: `facet_host $facet`"
-    do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT
+    do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT \
+        --lustre_upcall $UPCALL --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM
     do_lmc --add net --node ${facet}_facet --nid `facet_nid $facet` \
 	--nettype $NETTYPE
 }
@@ -271,15 +313,15 @@ add_mds() {
     facet=$1
     shift
     rm -f ${facet}active
-    add_facet $facet  --lustre_upcall $UPCALL
-    do_lmc --add mds --node ${facet}_facet --mds ${facet}_svc $*
+    add_facet $facet
+    do_lmc --add mds --node ${facet}_facet --mds ${facet}_svc --fstype $FSTYPE $*
 }
 
 add_mdsfailover() {
     facet=$1
     shift
     add_facet ${facet}failover  --lustre_upcall $UPCALL
-    do_lmc --add mds  --node ${facet}failover_facet --mds ${facet}_svc $*
+    do_lmc --add mds  --node ${facet}failover_facet --mds ${facet}_svc --fstype $FSTYPE $*
 }
 
 add_ost() {
@@ -287,14 +329,26 @@ add_ost() {
     shift
     rm -f ${facet}active
     add_facet $facet
-    do_lmc --add ost --node ${facet}_facet --ost ${facet}_svc $*
+    do_lmc --add ost --node ${facet}_facet --ost ${facet}_svc --fstype $FSTYPE $*
+}
+
+del_ost() {
+    facet=$1
+    shift
+    do_lmc --delete ost --node ${facet}_facet --ost ${facet}_svc $*
+}
+
+deactivate_ost() {
+    facet=$1
+    shift
+    do_lmc --deactivate ost --node ${facet}_facet --ost ${facet}_svc $*
 }
 
 add_ostfailover() {
     facet=$1
     shift
     add_facet ${facet}failover
-    do_lmc --add ost --failover --node ${facet}failover_facet --ost ${facet}_svc $*
+    do_lmc --add ost --failover --node ${facet}failover_facet --ost ${facet}_svc --fstype $FSTYPE $*
 }
 
 add_lov() {
@@ -302,18 +356,31 @@ add_lov() {
     mds_facet=$2
     shift; shift
     do_lmc --add lov --mds ${mds_facet}_svc --lov $lov $*
-    
+}
+
+add_lov_to_lmv() {
+    lov=$1
+    lmv=$2
+    shift; shift
+    do_lmc --add lov --lmv $lmv --lov $lov $*
+}
+
+add_lmv() {
+    lmv=$1
+    shift;
+    do_lmc --add lmv --lmv $lmv $*
 }
 
 add_client() {
     facet=$1
-    mds=$2
-    shift; shift
+    shift;
     add_facet $facet --lustre_upcall $UPCALL
-    do_lmc --add mtpt --node ${facet}_facet --mds ${mds}_svc $*
-
+    do_lmc --add mtpt --node ${facet}_facet $*
 }
 
+config_commit() {
+    do_lmc --commit
+}
 
 ####### 
 # General functions
@@ -379,6 +446,15 @@ drop_reply() {
     return $RC
 }
 
+drop_reint_reply() {
+# OBD_FAIL_MDS_REINT_NET_REP
+    RC=0
+    do_facet mds "echo 0x119 > /proc/sys/lustre/fail_loc"
+    do_facet client "$@" || RC=$?
+    do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
+    return $RC
+}
+
 pause_bulk() {
 #define OBD_FAIL_OST_BRW_PAUSE_BULK      0x214
     RC=0
@@ -389,6 +465,15 @@ pause_bulk() {
     return $RC
 }
 
+drop_ldlm_cancel() {
+#define OBD_FAIL_LDLM_CANCEL             0x304
+    RC=0
+    do_facet client "echo 0x304 > /proc/sys/lustre/fail_loc"
+    do_facet client "$@" || RC=$?
+    do_facet client "echo 0 > /proc/sys/lustre/fail_loc"
+    return $RC
+}
+
 drop_bl_callback() {
 #define OBD_FAIL_LDLM_BL_CALLBACK        0x305
     RC=0
@@ -398,10 +483,41 @@ drop_bl_callback() {
     return $RC
 }
 
+clear_failloc() {
+    facet=$1
+    pause=$2
+    sleep $pause
+    echo "clearing fail_loc on $facet"
+    do_facet $facet "sysctl -w lustre.fail_loc=0"
+}
+
+cancel_lru_locks() {
+    $LCTL mark cancel_lru_locks
+    for d in /proc/fs/lustre/ldlm/namespaces/$1*; do
+	if [ -f $d/lru_size ]; then
+	    echo clear > $d/lru_size
+	    grep [0-9] $d/lock_unused_count
+	fi
+    done
+}
+
+
+pgcache_empty() {
+    for a in /proc/fs/lustre/llite/*/dump_page_cache; do
+        if [ `wc -l $a | awk '{print $1}'` -gt 1 ]; then
+                echo there is still data in page cache $a ?
+                cat $a;
+                return 1;
+        fi
+    done
+    return 0
+}
+
 ##################################
 # Test interface 
 error() {
     echo "${TESTSUITE}: **** FAIL:" $@
+    log "FAIL: $@"
     exit 1
 }
 
@@ -462,14 +578,25 @@ equals_msg() {
    printf '===== %s %.*s\n' "$msg" $suffixlen $EQUALS
 }
 
+log() {
+	echo "$*"
+	lctl mark "$*" 2> /dev/null || true
+}
+
 run_one() {
     testnum=$1
     message=$2
-    tfile=f$base
-    tdir=d$base
+    tfile=f${testnum}
+    tdir=d${base}
 
     # Pretty tests run faster.
     equals_msg $testnum: $message
 
+    log "== test $1: $2"
     test_${testnum} || error "test_$testnum failed with $?"
 }
+
+canonical_path() {
+   (cd `dirname $1`; echo $PWD/`basename $1`)
+}
+