LU-1214 ptlrpc: start minimum service threads

If the ptlrpc_min_threads parameter is changed via /proc after the
service has started, then at least the requested number of service
threads should be started.  Otherwise this parameter would only be
used at initial thread startup and ignored if changed via /proc.

Fix conf-sanity.sh test_52[ab] to verify that at least the minimum
number of threads has been started when threads_min parameter is
changed, instead of just checking the parameter itself.  Also fix
test code style for 80-column line wrapping and tabs for indents.

The head utility does not always support shortcut "-1" option. It
should be specified as "-n1".

Signed-off-by: Andreas Dilger <andreas.dilger@intel.com>
Signed-off-by: Dmitry Eremin <dmitry.eremin@intel.com>
Change-Id: I6e4bb4131d7500a93952b64102f885c76558cab0
Reviewed-on: http://review.whamcloud.com/2876
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: James Simmons <uja.ornl@gmail.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/ptlrpc/service.c b/lustre/ptlrpc/service.c
index 3bb72c5..8f8ce0b 100644 (file)
--- a/lustre/ptlrpc/service.c
+++ b/lustre/ptlrpc/service.c
@@ -2287,9 +2287,11 @@ ptlrpc_retry_rqbds(void *arg)
 static inline int
 ptlrpc_threads_enough(struct ptlrpc_service_part *svcpt)
 {
-       return svcpt->scp_nreqs_active <
-              svcpt->scp_nthrs_running - 1 -
-              (svcpt->scp_service->srv_ops.so_hpreq_handler != NULL);
+       return (svcpt->scp_nthrs_running >=
+               svcpt->scp_service->srv_nthrs_cpt_init) &&
+              (svcpt->scp_nreqs_active <
+               svcpt->scp_nthrs_running - 1 -
+               (svcpt->scp_service->srv_ops.so_hpreq_handler != NULL));
 }
 
 /**

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 60475ea..5284332 100644 (file)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -982,7 +982,7 @@ test_24a() {
        # the MDS must remain up until last MDT
        stop_mds
        MDS=$(do_facet $SINGLEMDS "$LCTL get_param -n devices" |
-             awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }' | head -1)
+             awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }' | head -n1)
        [ -z "$MDS" ] && error "No MDT"
        cleanup_fs2
        cleanup_nocli || error "cleanup_nocli failed with rc $?"
@@ -1631,7 +1631,7 @@ t32_test() {
        local img_bspace
        local img_ispace
        local fsname=t32fs
-       local nid=$($r $LCTL list_nids | head -1)
+       local nid=$($r $LCTL list_nids | head -n1)
        local mopts
        local uuid
        local nrpcs_orig
@@ -1705,7 +1705,8 @@ t32_test() {
                                "(Need MGS version at least 2.3.59)"; return 0; }
 
                        local osthost=$(facet_active_host ost1)
-                       local ostnid=$(do_node $osthost $LCTL list_nids | head -1)
+                       local ostnid=$(do_node $osthost $LCTL list_nids |
+                                      head -n1)
 
                        mopts=nosvc
                        if [ $fstype == "ldiskfs" ]; then
@@ -2215,7 +2216,7 @@ test_35a() { # bug 12459
        log "Set up a fake failnode for the MDS"
        FAKENID="127.0.0.2"
        local device=$(do_facet $SINGLEMDS "$LCTL get_param -n devices" |
-               awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }' | head -1)
+                      awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }' | head -n1)
        do_facet mgs "$LCTL conf_param \
                      ${device}.failover.node=$(h2$NETTYPE $FAKENID)" ||
                error "Setting ${device}.failover.node=\
@@ -2273,7 +2274,7 @@ test_35b() { # bug 18674
        log "Set up a fake failnode for the MDS"
        FAKENID="127.0.0.2"
        local device=$(do_facet $SINGLEMDS "$LCTL get_param -n devices" |
-               awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }' | head -1)
+                      awk '($3 ~ "mdt" && $4 ~ "MDT") { print $4 }' | head -n1)
        do_facet mgs "$LCTL conf_param \
                      ${device}.failover.node=$(h2$NETTYPE $FAKENID)" ||
                error "Set ${device}.failover.node=\
@@ -3529,49 +3530,64 @@ test_52() {
 }
 run_test 52 "check recovering objects from lost+found"
 
+thread_param_get() {
+       local facet=$1
+       local pbase=$2
+       local param=$3
+
+       do_facet $facet "lctl get_param -n $pbase.$param" || echo 0
+}
+
+cleanup_thread_sanity() {
+       trap 0
+       cleanup
+}
+
 # Checks threads_min/max/started for some service
 #
 # Arguments: service name (OST or MDT), facet (e.g., ost1, $SINGLEMDS), and a
 # parameter pattern prefix like 'ost.*.ost'.
 thread_sanity() {
-        local modname=$1
-        local facet=$2
-        local parampat=$3
-        local opts=$4
+       local modname=$1
+       local facet=$2
+       local ppat=$3
+       local opts=$4
        local basethr=$5
-        local tmin
-        local tmin2
-        local tmax
-        local tmax2
-        local tstarted
-        local paramp
-        local msg="Insane $modname thread counts"
-       local ncpts=$(check_cpt_number $facet)
+       local tmin
+       local tmin2
+       local tmax
+       local tmax2
+       local tstarted
+       local tstarted2
+       local pname
+       local pbase
+       local msg="Insane $modname thread counts"
+       local ncpts
        local nthrs
-        shift 4
+       shift 4
 
-        check_mount || return 41
+       setup
+       check_mount || { error "filesystem is not mounted"; return 40; }
+       trap cleanup_thread_sanity EXIT
 
-        # We need to expand $parampat, but it may match multiple parameters, so
-        # we'll pick the first one
-        if ! paramp=$(do_facet $facet "lctl get_param -N ${parampat}.threads_min"|head -1); then
-                error "Couldn't expand ${parampat}.threads_min parameter name"
-                return 22
-        fi
+       ncpts=$(check_cpt_number $facet)
+
+       # We need to expand $ppat, but it may match multiple parameters,
+       # so we'll pick the first one
+       pname=$(do_facet $facet "$LCTL get_param -N $ppat.threads_min" |
+               head -n1)
+       [ -n "$pname" ] || { error "Can't expand $ppat.threads_min"; return 20;}
 
        # Remove the .threads_min part
-       paramp=${paramp%.threads_min}
+       pbase=${pname%.threads_min}
 
        # Check for sanity in defaults
-       tmin=$(do_facet $facet "$LCTL get_param -n ${paramp}.threads_min" ||
-              echo 0)
-       tmax=$(do_facet $facet "$LCTL get_param -n ${paramp}.threads_max" ||
-              echo 0)
-       tstarted=$(do_facet $facet "$LCTL get_param \
-                                   -n ${paramp}.threads_started" || echo 0)
-       lassert 23 "$msg (PDSH problems?)" '(($tstarted && $tmin && $tmax))' ||
+       tmin=$(thread_param_get $facet $pbase threads_min)
+       tmax=$(thread_param_get $facet $pbase threads_max)
+       tstarted=$(thread_param_get $facet $pbase threads_started)
+       lassert 21 "$msg (PDSH problems?)" '(($tstarted && $tmin && $tmax))' ||
                return $?
-       lassert 24 "$msg" '(($tstarted >= $tmin && $tstarted <= $tmax ))' ||
+       lassert 22 "$msg" '(($tstarted >= $tmin && $tstarted <= $tmax))' ||
                return $?
        nthrs=$(expr $tmax - $tmin)
        if [ $nthrs -lt $ncpts ]; then
@@ -3582,84 +3598,63 @@ thread_sanity() {
 
        [ $tmin -eq $tmax -a $tmin -eq $tstarted ] &&
                skip_env "module parameter forced $facet thread count" &&
-               tmin=3 && tmax=$((3 * tmax))
+               return 0
 
        # Check that we can change min/max
-       do_facet $facet "$LCTL set_param \
-                        ${paramp}.threads_min=$((tmin + nthrs))"
-       do_facet $facet "$LCTL set_param \
-                        ${paramp}.threads_max=$((tmax - nthrs))"
-       tmin2=$(do_facet $facet "$LCTL get_param -n ${paramp}.threads_min" ||
-               echo 0)
-       tmax2=$(do_facet $facet "$LCTL get_param -n ${paramp}.threads_max" ||
-               echo 0)
-       lassert 25 "$msg" '(($tmin2 == ($tmin + $nthrs) &&
-                           $tmax2 == ($tmax - $nthrs)))' || return $?
+       do_facet $facet "$LCTL set_param $pbase.threads_min=$((tmin + nthrs))"
+       do_facet $facet "$LCTL set_param $pbase.threads_max=$((tmax - nthrs))"
+       tmin2=$(thread_param_get $facet $pbase threads_min)
+       tmax2=$(thread_param_get $facet $pbase threads_max)
+       lassert 23 "$msg" '(($tmin2 == ($tmin + $nthrs)))' || return $?
+       lassert 24 "$msg" '(($tmax2 == ($tmax - $nthrs)))' || return $?
+
+       sleep 3 # give threads a chance to start
+       tstarted=$(thread_param_get $facet $pbase threads_started)
+       lassert 25 "$msg" '(($tstarted >= $tmin2))' || return $?
 
        # Check that we can set min/max to the same value
-       tmin=$(do_facet $facet "$LCTL get_param -n ${paramp}.threads_min" ||
-              echo 0)
-       do_facet $facet "$LCTL set_param ${paramp}.threads_max=$tmin"
-       tmin2=$(do_facet $facet "$LCTL get_param -n ${paramp}.threads_min" ||
-               echo 0)
-       tmax2=$(do_facet $facet "$LCTL get_param -n ${paramp}.threads_max" ||
-               echo 0)
-       lassert 26 "$msg" '(($tmin2 == $tmin && $tmax2 == $tmin))' || return $?
+       do_facet $facet "$LCTL set_param $pbase.threads_max=$tmin2"
+       tmax2=$(thread_param_get $facet $pbase threads_max)
+       lassert 26 "$msg" '(($tmax2 == $tmin2))' || return $?
 
        # Check that we can't set max < min
-       do_facet $facet "$LCTL set_param ${paramp}.threads_max=$((tmin - 1))"
-       tmin2=$(do_facet $facet "$LCTL get_param -n ${paramp}.threads_min" ||
-               echo 0)
-       tmax2=$(do_facet $facet "$LCTL get_param -n ${paramp}.threads_max" ||
-               echo 0)
-       lassert 27 "$msg" '(($tmin2 <= $tmax2))' || return $?
+       do_facet $facet "$LCTL set_param $pbase.threads_max=$((tmin - 1))"
+       tmax2=$(thread_param_get $facet $pbase threads_max)
+       lassert 27 "$msg" '(($tmax2 < $tmin))' || return $?
 
        # We need to ensure that we get the module options desired; to do this
        # we set LOAD_MODULES_REMOTE=true and we call setmodopts below.
        LOAD_MODULES_REMOTE=true
        cleanup
        local oldvalue
-       local newvalue="${opts}=$(expr $basethr \* $ncpts)"
+       local newvalue="${opts}=$((basethr * ncpts))"
        setmodopts -a $modname "$newvalue" oldvalue
 
        load_modules
        setup
-       check_mount || return 41
+       check_mount || { error "filesystem failed remount"; return 41; }
 
        # Restore previous setting of MODOPTS_*
        setmodopts $modname "$oldvalue"
 
        # Check that $opts took
-       tmin=$(do_facet $facet "$LCTL get_param -n ${paramp}.threads_min")
-       tmax=$(do_facet $facet "$LCTL get_param -n ${paramp}.threads_max")
-       tstarted=$(do_facet $facet \
-                  "$LCTL get_param -n ${paramp}.threads_started")
-       lassert 28 "$msg" '(($tstarted >= $tmin && $tstarted <= $tmax ))' ||
+       tmin=$(thread_param_get $facet $pbase threads_min)
+       tmax=$(thread_param_get $facet $pbase threads_max)
+       tstarted=$(thread_param_get $facet $pbase threads_started)
+       lassert 28 "$msg" '(($tstarted == $tmin && $tstarted == $tmax ))' ||
                return $?
-       cleanup
 
+       cleanup_thread_sanity || error "cleanup failed with rc $?"
        load_modules
-       setup
 }
 
 test_53a() {
-       setup
-       thread_sanity OST ost1 'ost.*.ost' 'oss_num_threads' '16'
-       cleanup || error "cleanup failed with rc $?"
+       thread_sanity OST ost1 'ost.*.ost' oss_num_threads 16
 }
 run_test 53a "check OSS thread count params"
 
 test_53b() {
-       setup
-       local mds=$(do_facet $SINGLEMDS "$LCTL get_param \
-                                        -N mds.*.*.threads_max 2>/dev/null")
-       if [ -z "$mds" ]; then
-               #running this on an old MDT
-               thread_sanity MDT $SINGLEMDS 'mdt.*.*.' 'mdt_num_threads' 16
-       else
-               thread_sanity MDT $SINGLEMDS 'mds.*.*.' 'mds_num_threads' 16
-       fi
-       cleanup || error "cleanup failed with $?"
+       thread_sanity MDT $SINGLEMDS 'mds.*.*' mds_num_threads 16
 }
 run_test 53b "check MDS thread count params"
 
@@ -4060,8 +4055,8 @@ test_66() {
                { skip "Need MGS version at least 2.3.59"; return 0; }
 
        setup
-       local OST1_NID=$(do_facet ost1 $LCTL list_nids | head -1)
-       local MDS_NID=$(do_facet $SINGLEMDS $LCTL list_nids | head -1)
+       local OST1_NID=$(do_facet ost1 $LCTL list_nids | head -n1)
+       local MDS_NID=$(do_facet $SINGLEMDS $LCTL list_nids | head -n1)
 
        echo "replace_nids should fail if MDS, OSTs and clients are UP"
        do_facet mgs $LCTL replace_nids $FSNAME-OST0000 $OST1_NID &&
@@ -4616,15 +4611,14 @@ test_76a() {
        setup
        local MDMB_PARAM="osc.*.max_dirty_mb"
        echo "Change MGS params"
-       local MAX_DIRTY_MB=$($LCTL get_param -n $MDMB_PARAM |
-               head -1)
+       local MAX_DIRTY_MB=$($LCTL get_param -n $MDMB_PARAM | head -n1)
        echo "max_dirty_mb: $MAX_DIRTY_MB"
        local NEW_MAX_DIRTY_MB=$((MAX_DIRTY_MB + MAX_DIRTY_MB))
        echo "new_max_dirty_mb: $NEW_MAX_DIRTY_MB"
        do_facet mgs $LCTL set_param -P $MDMB_PARAM=$NEW_MAX_DIRTY_MB
        wait_update $HOSTNAME "$LCTL get_param -n $MDMB_PARAM |
-               head -1" $NEW_MAX_DIRTY_MB
-       MAX_DIRTY_MB=$($LCTL get_param -n $MDMB_PARAM | head -1)
+               head -n1" $NEW_MAX_DIRTY_MB
+       MAX_DIRTY_MB=$($LCTL get_param -n $MDMB_PARAM | head -n1)
        echo "$MAX_DIRTY_MB"
        [ $MAX_DIRTY_MB = $NEW_MAX_DIRTY_MB ] ||
                error "error while apply max_dirty_mb"
@@ -4633,8 +4627,8 @@ test_76a() {
        stopall
        setupall
        wait_update $HOSTNAME "$LCTL get_param -n $MDMB_PARAM |
-               head -1" $NEW_MAX_DIRTY_MB
-       MAX_DIRTY_MB=$($LCTL get_param -n $MDMB_PARAM | head -1)
+               head -n1" $NEW_MAX_DIRTY_MB
+       MAX_DIRTY_MB=$($LCTL get_param -n $MDMB_PARAM | head -n1)
        [ $MAX_DIRTY_MB = $NEW_MAX_DIRTY_MB ] ||
                error "max_dirty_mb is not saved after remount"
 
@@ -4642,15 +4636,15 @@ test_76a() {
        CLIENT_PARAM="obdfilter.*.client_cache_count"
        local CLIENT_CACHE_COUNT
        CLIENT_CACHE_COUNT=$(do_facet ost1 $LCTL get_param -n $CLIENT_PARAM |
-               head -1)
+                            head -n1)
        echo "client_cache_count: $CLIENT_CACHE_COUNT"
        NEW_CLIENT_CACHE_COUNT=$((CLIENT_CACHE_COUNT+CLIENT_CACHE_COUNT))
        echo "new_client_cache_count: $NEW_CLIENT_CACHE_COUNT"
        do_facet mgs $LCTL set_param -P $CLIENT_PARAM=$NEW_CLIENT_CACHE_COUNT
        wait_update $(facet_host ost1) "$LCTL get_param -n $CLIENT_PARAM |
-               head -1" $NEW_CLIENT_CACHE_COUNT
+               head -n1" $NEW_CLIENT_CACHE_COUNT
        CLIENT_CACHE_COUNT=$(do_facet ost1 $LCTL get_param -n $CLIENT_PARAM |
-               head -1)
+                            head -n1)
        echo "$CLIENT_CACHE_COUNT"
        [ $CLIENT_CACHE_COUNT = $NEW_CLIENT_CACHE_COUNT ] ||
                error "error while apply client_cache_count"
@@ -4659,9 +4653,9 @@ test_76a() {
        stopall
        setupall
        wait_update $(facet_host ost1) "$LCTL get_param -n $CLIENT_PARAM |
-               head -1" $NEW_CLIENT_CACHE_COUNT
+               head -n1" $NEW_CLIENT_CACHE_COUNT
        CLIENT_CACHE_COUNT=$(do_facet ost1 $LCTL get_param -n $CLIENT_PARAM |
-               head -1)
+               head -n1)
        echo "$CLIENT_CACHE_COUNT"
        [ $CLIENT_CACHE_COUNT = $NEW_CLIENT_CACHE_COUNT ] ||
                error "client_cache_count is not saved after remount"