LU-14627 tests: Create unload_modules_local

t-f allows for loading modules on single node via load_modules_local.
However, there is no corresponding unload_modules_local that can be
called to cleanup after call to load_modules_local, so we create it.
unload_modules() refactored to use unload_modules_local.

Also address a potential issue that can prevent LND modules from
unloading. Some LNet setup (particularly those in sanity-lnet) may
require that we call lnetctl lnet unconfigure (or lctl net down)
to drop a ref on the module before it can be unloaded.

HPE-bug-id: LUS-9031
Signed-off-by: Chris Horn <chris.horn@hpe.com>
Change-Id: I6458a7728f5f559f8641c5a9e29dd775c8445c38
Reviewed-on: https://review.whamcloud.com/43425
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/scripts/lustre_rmmod b/lustre/scripts/lustre_rmmod
index 9da25ff..637c6a0 100755 (executable)
--- a/lustre/scripts/lustre_rmmod
+++ b/lustre/scripts/lustre_rmmod
@@ -111,6 +111,20 @@ if $DEBUG; then
        $LCTL mark "$SCRIPT_NAME : Start debug"
 fi
 
+# LNet may have an internal ref which can prevent LND modules from
+# unloading. Try to drop it before unloading modules.
+# NB: we squelch stderr because lnetctl/lctl may complain about
+# LNet being "busy", but this is normal. We're making a best effort
+# here.
+if lsmod | grep -q lnet; then
+       # Prefer lnetctl if it is present
+       if [ -n "$(which lnetctl 2>/dev/null)" ]; then
+               lnetctl lnet unconfigure 2>/dev/null
+       elif [ -n "$(which lctl 2>/dev/null)" ]; then
+               lctl net down 2>/dev/null
+       fi
+fi
+
 for mod in ${modules[*]}; do
        unload_dep_modules_inclusive $mod || exit 1
 done

diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh
index 5909412..91b0ca7 100755 (executable)
--- a/lustre/tests/sanity-lnet.sh
+++ b/lustre/tests/sanity-lnet.sh
@@ -31,9 +31,6 @@ init_logging
 
 build_test_filter
 
-export LNETCTL=${LNETCTL:-"$LUSTRE/../lnet/utils/lnetctl"}
-[ ! -f "$LNETCTL" ] &&
-       export LNETCTL=$(which lnetctl 2> /dev/null)
 [[ -z $LNETCTL ]] && skip "Need lnetctl"
 
 restore_mounts=false

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 2385ef7..5bb3702 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -293,6 +293,8 @@ init_test_env() {
        fi
        export RSYNC_RSH=${RSYNC_RSH:-rsh}
 
+       export LNETCTL=${LNETCTL:-"$LUSTRE/../lnet/utils/lnetctl"}
+       [ ! -f "$LNETCTL" ] && export LNETCTL=$(which lnetctl 2> /dev/null)
        export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
        [ ! -f "$LCTL" ] && export LCTL=$(which lctl)
        export LFS=${LFS:-"$LUSTRE/utils/lfs"}
@@ -821,9 +823,7 @@ check_mem_leak () {
        fi
 }
 
-unload_modules() {
-       wait_exit_ST client # bug 12845
-
+unload_modules_local() {
        $LUSTRE_RMMOD ldiskfs || return 2
 
        [ -f /etc/udev/rules.d/99-lustre-test.rules ] &&
@@ -831,15 +831,23 @@ unload_modules() {
        udevadm control --reload-rules
        udevadm trigger
 
+       check_mem_leak || return 254
+
+       return 0
+}
+
+unload_modules() {
+       local rc=0
+
+       wait_exit_ST client # bug 12845
+
+       unload_modules_local || rc=$?
+
        if $LOAD_MODULES_REMOTE; then
                local list=$(comma_list $(remote_nodes_list))
                if [ -n "$list" ]; then
                        echo "unloading modules on: '$list'"
-                       do_rpc_nodes "$list" $LUSTRE_RMMOD ldiskfs
-                       do_rpc_nodes "$list" check_mem_leak
-                       do_rpc_nodes "$list" "rm -f /etc/udev/rules.d/99-lustre-test.rules"
-                       do_rpc_nodes "$list" "udevadm control --reload-rules"
-                       do_rpc_nodes "$list" "udevadm trigger"
+                       do_rpc_nodes "$list" unload_modules_local
                fi
        fi
 
@@ -850,10 +858,9 @@ unload_modules() {
                        rm -f $sbin_mount
        fi
 
-       check_mem_leak || return 254
+       [[ $rc -eq 0 ]] && echo "modules unloaded."
 
-       echo "modules unloaded."
-       return 0
+       return $rc
 }
 
 fs_log_size() {