LU-1607 tests: enable multiple CPU Partitions

CPU partitions number of libcfs is explicitly set to 1 which means
most SMP features are disabled by default, the reason we didn't
enable it so far is because enabling it will break some tests
because change of /proc interfaces or modparameters.

This patch covered a few things:
- use 0 as default value of cpu_npartitions
  libcfs will estimate CPU partition number based on cores
  or NUMA nodes
- a small fix for LNet
  ni_last_alive should use second instead of jiffies as time stamp
- always enable multiple CPU partitions in test-framework
- fix test_215 for sanity
- fix test_52 for conf-sanity

Signed-off-by: Liang Zhen <liang@whamcloud.com>
Change-Id: Ib1a296e254a7b2e91e8d77337a9cd9ac877321e3
Reviewed-on: http://review.whamcloud.com/3288
Tested-by: Hudson
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Bobi Jam <bobijam@whamcloud.com>
Reviewed-by: wangdi <di.wang@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>

diff --git a/libcfs/libcfs/linux/linux-cpu.c b/libcfs/libcfs/linux/linux-cpu.c
index 896fd7d..3bc7e69 100644 (file)
--- a/libcfs/libcfs/linux/linux-cpu.c
+++ b/libcfs/libcfs/linux/linux-cpu.c
@@ -46,9 +46,7 @@
  *  1 : disable multiple partitions
  * >1 : specify number of partitions
  */
-/* NB: we set it to 1 for now, multiple partitions will be enabled after
- * all smp node affinity code landed */
-static int     cpu_npartitions = 1;
+static int     cpu_npartitions;
 CFS_MODULE_PARM(cpu_npartitions, "i", int, 0444, "# of CPU partitions");
 
 /**

diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c
index c3dd0a7..327c019 100644 (file)
--- a/lnet/lnet/config.c
+++ b/lnet/lnet/config.c
@@ -173,11 +173,11 @@ lnet_ni_alloc(__u32 net, struct cfs_expr_list *el, cfs_list_t *nilist)
                ni->ni_ncpts = rc;
        }
 
-        /* LND will fill in the address part of the NID */
-        ni->ni_nid = LNET_MKNID(net, 0);
-        ni->ni_last_alive = cfs_time_current();
-        cfs_list_add_tail(&ni->ni_list, nilist);
-        return ni;
+       /* LND will fill in the address part of the NID */
+       ni->ni_nid = LNET_MKNID(net, 0);
+       ni->ni_last_alive = cfs_time_current_sec();
+       cfs_list_add_tail(&ni->ni_list, nilist);
+       return ni;
  failed:
        lnet_ni_free(ni);
        return NULL;

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index 701e58a..2a57914 100644 (file)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -2554,6 +2554,7 @@ thread_sanity() {
         local tstarted
         local paramp
         local msg="Insane $modname thread counts"
+       local ncpts=$(check_cpt_number)
         shift 4
 
         setup
@@ -2577,11 +2578,11 @@ thread_sanity() {
         lassert 24 "$msg" '(($tstarted >= $tmin && $tstarted <= $tmax ))' || return $?
 
         # Check that we can change min/max
-        do_facet $facet "lctl set_param ${paramp}.threads_min=$((tmin + 1))"
-        do_facet $facet "lctl set_param ${paramp}.threads_max=$((tmax - 1))"
-        tmin2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_min" || echo 0)
-        tmax2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_max" || echo 0)
-        lassert 25 "$msg" '(($tmin2 == ($tmin + 1) && $tmax2 == ($tmax -1)))' || return $?
+       do_facet $facet "lctl set_param ${paramp}.threads_min=$((tmin + ncpts))"
+       do_facet $facet "lctl set_param ${paramp}.threads_max=$((tmax - ncpts))"
+       tmin2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_min" || echo 0)
+       tmax2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_max" || echo 0)
+       lassert 25 "$msg" '(($tmin2 == ($tmin + $ncpts) && $tmax2 == ($tmax - $ncpts)))' || return $?
 
         # Check that we can set min/max to the same value
         tmin=$(do_facet $facet "lctl get_param -n ${paramp}.threads_min" || echo 0)
@@ -2626,12 +2627,20 @@ thread_sanity() {
 }
 
 test_53a() {
-        thread_sanity OST ost1 'ost.*.ost' 'oss_num_threads=64'
+       local ncpts=$(check_cpt_number)
+       local nthrs
+
+       nthrs=`expr 16 \* $ncpts`
+       thread_sanity OST ost1 'ost.*.ost' 'oss_num_threads='$nthrs
 }
 run_test 53a "check OSS thread count params"
 
 test_53b() {
-        thread_sanity MDT $SINGLEMDS 'mdt.*.*.' 'mdt_num_threads=64'
+       local ncpts=$(check_cpt_number)
+       local nthrs
+
+       nthrs=`expr 16 \* $ncpts`
+       thread_sanity MDT $SINGLEMDS 'mdt.*.*.' 'mdt_num_threads='$nthrs
 }
 run_test 53b "check MDT thread count params"
 

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 36e2aa0..40b4f7d 100644 (file)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -9036,10 +9036,10 @@ test_215() { # for bugs 18102, 21079, 21517
        # /proc/sys/lnet/nis should look like this:
        # nid status alive refs peer rtr max tx min
        # where nid is a string like 192.168.1.1@tcp2, status is up/down,
-       # alive is numeric (0 or >0 or <0), refs > 0, peer >= 0,
+       # alive is numeric (0 or >0 or <0), refs >= 0, peer >= 0,
        # rtr >= 0, max >=0, tx and min are numeric (0 or >0 or <0).
        L1="^nid +status +alive +refs +peer +rtr +max +tx +min$"
-       BR="^$NID +(up|down) +$I +$P +$N +$N +$N +$I +$I$"
+       BR="^$NID +(up|down) +$I +$N +$N +$N +$N +$I +$I$"
        create_lnet_proc_files "nis"
        check_lnet_proc_entry "nis.out" "/proc/sys/lnet/nis" "$BR" "$L1"
        check_lnet_proc_entry "nis.sys" "lnet.nis" "$BR" "$L1"

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index a1fe7ef..6292c8e 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -266,6 +266,17 @@ init_test_env() {
     rm -f $TMP/*active
 }
 
+check_cpt_number() {
+       local ncpts
+
+       ncpts=$(lctl get_param -n cpu_partition_table 2>/dev/null| wc -l)
+       if [ $ncpts -eq 0 ]; then
+               echo "1"
+       else
+               echo $ncpts
+       fi
+}
+
 version_code() {
     # split arguments like "1.8.6-wc3" into "1", "8", "6", "wc3"
     eval set -- $(tr "[:punct:]" " " <<< $*)
@@ -379,8 +390,39 @@ load_modules_local() {
                return 0
        fi
 
-    echo Loading modules from $LUSTRE
-    load_module ../libcfs/libcfs/libcfs
+       echo Loading modules from $LUSTRE
+
+       local ncpus
+
+       if [ -f /sys/devices/system/cpu/online ]; then
+               ncpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/online) + 1))
+               echo "detected $ncpus online CPUs by sysfs"
+       else
+               ncpus=$(getconf _NPROCESSORS_CONF 2>/dev/null)
+               local rc=$?
+               if [ $rc -eq 0 ]; then
+                       echo "detected $ncpus online CPUs by getconf"
+               else
+                       echo "Can't detect number of CPUs"
+                       ncpus=1
+               fi
+       fi
+
+       local ncpts=0
+       # if there is only one CPU core, libcfs can only create one partition
+       # if there is more than 4 CPU cores, libcfs should create multiple CPU
+       # partitions. So we just force libcfs to create 2 partitions for
+       # system with 2 or 4 cores
+       if [ $ncpus -le 4 ] && [ $ncpus -gt 1 ]; then
+               # force to enable multiple CPU partitions
+               echo "Force libcfs to create 2 CPU partitions"
+               ncpts=2
+       else
+               echo "libcfs will create CPU partition based on online CPUs"
+       fi
+
+       load_module ../libcfs/libcfs/libcfs cpu_npartitions=$ncpts
+
     [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG"
     [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }"
     load_module ../lnet/lnet/lnet