From 16ca9e3ee2cc58c25c2002ab9f3ff4209cc11fdc Mon Sep 17 00:00:00 2001
From: Nathaniel Clark <nclark@whamcloud.com>
Date: Wed, 14 Jul 2021 08:22:46 -0400
Subject: [PATCH] EX-3476 lipe-scripts: Add --now to hp stop

The --now option will, after stopping lamigo/lpurge, kill all lfs mirror commands
and unmount the clients.  This is to aid in a full filesystem shutdown.

Change-Id: I852383b9523068a75a047e5f640f2d28057e0c64
Signed-off-by: Nathaniel Clark <nclark@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/44304
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Gaurang Tapase <gtapase@ddn.com>
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>
---
 lipe/scripts/stratagem-hp-stop.sh | 65 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 63 insertions(+), 2 deletions(-)

diff --git a/lipe/scripts/stratagem-hp-stop.sh b/lipe/scripts/stratagem-hp-stop.sh
index dba265b..d89cd75 100755
--- a/lipe/scripts/stratagem-hp-stop.sh
+++ b/lipe/scripts/stratagem-hp-stop.sh
@@ -2,16 +2,60 @@
 # Copyright DDN 2020
 # -*- indent-tabs-mode: nil -*-
 
+# Set a default 10+ minute timeout for waitfor()
+TIMEOUT=${TIMEOUT:-600}
+
+NOW=false
+
 FS=$1
 
-function usage() {
-    echo "Usage: $(basename $0) [-h] [FILESYSTEM]"
+function usage {
+    echo "Usage: $(basename $0) [-h] [--now] [FILESYSTEM]"
     echo "  Stop stratagem hotpools in HA environment"
+    echo "  --now - Stop mirroring processes and client mounts"
+}
+
+function waitfor {
+    local ids=$*
+
+    [ -z "$ids" ] && return
+
+    echo -n "Waiting for $ids"
+
+    local deadline=$((SECONDS + TIMEOUT))
+
+    local done=false
+    until $done || ((deadline < SECONDS)); do
+
+        done=true
+        for i in $ids; do
+            # This finds the host the resource is active on (empty it is stopped)
+            res=$(clush -qg ha_heads crm_resource -QW -r $i 2> /dev/null)
+            if [ -n "$res" ]; then
+                done=false
+                echo -n "."
+                sleep 1
+                break
+            fi
+        done
+    done
+
+    echo ""
+
+    if ! $done; then
+        echo "Waiting for $ids TIMED OUT!"
+        exit 1
+    fi
 }
 
 if [ "$FS" = "-h" ] || [ "$FS" = "--help" ]; then
     usage
     exit 0
+
+elif [ "$FS" = "--now" ]; then
+    shift
+    NOW=true
+    FS=$1
 fi
 
 if [ -z "$FS" ]; then
@@ -29,10 +73,27 @@ if crm_resource -QW -r $FS-hotpool > /dev/null 2>&1; then
     echo Stopping Hotpools for $FS
     clush -qS --group=ha_heads crm res stop $FS-hotpool
 
+    if $NOW; then
+        waitfor $FS-hotpool
+
+        echo Killing all lfs mirror commands for $FS
+        clush -q -g oss,mds,mgs pkill -f ^lfs.mirror
+
+        echo Stopping Client mounts for $FS
+        clush -qS -g ha_heads crm res stop cl-$FS-client
+
+        waitfor cl-$FS-client
+    fi
+
 elif clush -Ng ha_heads crm_resource -l|egrep -q "^(lamigo|lpurge)-$FS-"; then
     echo "Stopping old style Hotpools (see stratagem-hp-convert.sh)"
     clush -qS -g ha_heads crm res stop cl-$FS-client
 
+    if $NOW; then
+        echo Killing all lfs mirror commands for $FS
+        clush -q -g oss,mds,mgs pkill -f ^lfs.mirror
+    fi
+
 else
     echo "Hotpools not configured"
     echo "  Please run: stratagem-hp-config.sh"
-- 
1.8.3.1