From 95d67b28076f7938a6c962a5256e9b581a439f71 Mon Sep 17 00:00:00 2001 From: Liang Zhen Date: Fri, 6 Jul 2012 12:43:11 +0800 Subject: [PATCH] LU-1607 tests: enable multiple CPU Partitions CPU partitions number of libcfs is explicitly set to 1 which means most SMP features are disabled by default, the reason we didn't enable it so far is because enabling it will break some tests because change of /proc interfaces or modparameters. This patch covered a few things: - use 0 as default value of cpu_npartitions libcfs will estimate CPU partition number based on cores or NUMA nodes - a small fix for LNet ni_last_alive should use second instead of jiffies as time stamp - always enable multiple CPU partitions in test-framework - fix test_215 for sanity - fix test_52 for conf-sanity Signed-off-by: Liang Zhen Change-Id: Ib1a296e254a7b2e91e8d77337a9cd9ac877321e3 Reviewed-on: http://review.whamcloud.com/3288 Tested-by: Hudson Reviewed-by: Andreas Dilger Reviewed-by: Bobi Jam Reviewed-by: wangdi Tested-by: Maloo --- libcfs/libcfs/linux/linux-cpu.c | 4 +--- lnet/lnet/config.c | 10 ++++----- lustre/tests/conf-sanity.sh | 23 ++++++++++++++------- lustre/tests/sanity.sh | 4 ++-- lustre/tests/test-framework.sh | 46 +++++++++++++++++++++++++++++++++++++++-- 5 files changed, 68 insertions(+), 19 deletions(-) diff --git a/libcfs/libcfs/linux/linux-cpu.c b/libcfs/libcfs/linux/linux-cpu.c index 896fd7d..3bc7e69 100644 --- a/libcfs/libcfs/linux/linux-cpu.c +++ b/libcfs/libcfs/linux/linux-cpu.c @@ -46,9 +46,7 @@ * 1 : disable multiple partitions * >1 : specify number of partitions */ -/* NB: we set it to 1 for now, multiple partitions will be enabled after - * all smp node affinity code landed */ -static int cpu_npartitions = 1; +static int cpu_npartitions; CFS_MODULE_PARM(cpu_npartitions, "i", int, 0444, "# of CPU partitions"); /** diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c index c3dd0a7..327c019 100644 --- a/lnet/lnet/config.c +++ b/lnet/lnet/config.c @@ -173,11 +173,11 @@ lnet_ni_alloc(__u32 net, struct cfs_expr_list *el, cfs_list_t *nilist) ni->ni_ncpts = rc; } - /* LND will fill in the address part of the NID */ - ni->ni_nid = LNET_MKNID(net, 0); - ni->ni_last_alive = cfs_time_current(); - cfs_list_add_tail(&ni->ni_list, nilist); - return ni; + /* LND will fill in the address part of the NID */ + ni->ni_nid = LNET_MKNID(net, 0); + ni->ni_last_alive = cfs_time_current_sec(); + cfs_list_add_tail(&ni->ni_list, nilist); + return ni; failed: lnet_ni_free(ni); return NULL; diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 701e58a..2a57914 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -2554,6 +2554,7 @@ thread_sanity() { local tstarted local paramp local msg="Insane $modname thread counts" + local ncpts=$(check_cpt_number) shift 4 setup @@ -2577,11 +2578,11 @@ thread_sanity() { lassert 24 "$msg" '(($tstarted >= $tmin && $tstarted <= $tmax ))' || return $? # Check that we can change min/max - do_facet $facet "lctl set_param ${paramp}.threads_min=$((tmin + 1))" - do_facet $facet "lctl set_param ${paramp}.threads_max=$((tmax - 1))" - tmin2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_min" || echo 0) - tmax2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_max" || echo 0) - lassert 25 "$msg" '(($tmin2 == ($tmin + 1) && $tmax2 == ($tmax -1)))' || return $? + do_facet $facet "lctl set_param ${paramp}.threads_min=$((tmin + ncpts))" + do_facet $facet "lctl set_param ${paramp}.threads_max=$((tmax - ncpts))" + tmin2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_min" || echo 0) + tmax2=$(do_facet $facet "lctl get_param -n ${paramp}.threads_max" || echo 0) + lassert 25 "$msg" '(($tmin2 == ($tmin + $ncpts) && $tmax2 == ($tmax - $ncpts)))' || return $? # Check that we can set min/max to the same value tmin=$(do_facet $facet "lctl get_param -n ${paramp}.threads_min" || echo 0) @@ -2626,12 +2627,20 @@ thread_sanity() { } test_53a() { - thread_sanity OST ost1 'ost.*.ost' 'oss_num_threads=64' + local ncpts=$(check_cpt_number) + local nthrs + + nthrs=`expr 16 \* $ncpts` + thread_sanity OST ost1 'ost.*.ost' 'oss_num_threads='$nthrs } run_test 53a "check OSS thread count params" test_53b() { - thread_sanity MDT $SINGLEMDS 'mdt.*.*.' 'mdt_num_threads=64' + local ncpts=$(check_cpt_number) + local nthrs + + nthrs=`expr 16 \* $ncpts` + thread_sanity MDT $SINGLEMDS 'mdt.*.*.' 'mdt_num_threads='$nthrs } run_test 53b "check MDT thread count params" diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 36e2aa0..40b4f7d 100644 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -9036,10 +9036,10 @@ test_215() { # for bugs 18102, 21079, 21517 # /proc/sys/lnet/nis should look like this: # nid status alive refs peer rtr max tx min # where nid is a string like 192.168.1.1@tcp2, status is up/down, - # alive is numeric (0 or >0 or <0), refs > 0, peer >= 0, + # alive is numeric (0 or >0 or <0), refs >= 0, peer >= 0, # rtr >= 0, max >=0, tx and min are numeric (0 or >0 or <0). L1="^nid +status +alive +refs +peer +rtr +max +tx +min$" - BR="^$NID +(up|down) +$I +$P +$N +$N +$N +$I +$I$" + BR="^$NID +(up|down) +$I +$N +$N +$N +$N +$I +$I$" create_lnet_proc_files "nis" check_lnet_proc_entry "nis.out" "/proc/sys/lnet/nis" "$BR" "$L1" check_lnet_proc_entry "nis.sys" "lnet.nis" "$BR" "$L1" diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index a1fe7ef..6292c8e 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -266,6 +266,17 @@ init_test_env() { rm -f $TMP/*active } +check_cpt_number() { + local ncpts + + ncpts=$(lctl get_param -n cpu_partition_table 2>/dev/null| wc -l) + if [ $ncpts -eq 0 ]; then + echo "1" + else + echo $ncpts + fi +} + version_code() { # split arguments like "1.8.6-wc3" into "1", "8", "6", "wc3" eval set -- $(tr "[:punct:]" " " <<< $*) @@ -379,8 +390,39 @@ load_modules_local() { return 0 fi - echo Loading modules from $LUSTRE - load_module ../libcfs/libcfs/libcfs + echo Loading modules from $LUSTRE + + local ncpus + + if [ -f /sys/devices/system/cpu/online ]; then + ncpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/online) + 1)) + echo "detected $ncpus online CPUs by sysfs" + else + ncpus=$(getconf _NPROCESSORS_CONF 2>/dev/null) + local rc=$? + if [ $rc -eq 0 ]; then + echo "detected $ncpus online CPUs by getconf" + else + echo "Can't detect number of CPUs" + ncpus=1 + fi + fi + + local ncpts=0 + # if there is only one CPU core, libcfs can only create one partition + # if there is more than 4 CPU cores, libcfs should create multiple CPU + # partitions. So we just force libcfs to create 2 partitions for + # system with 2 or 4 cores + if [ $ncpus -le 4 ] && [ $ncpus -gt 1 ]; then + # force to enable multiple CPU partitions + echo "Force libcfs to create 2 CPU partitions" + ncpts=2 + else + echo "libcfs will create CPU partition based on online CPUs" + fi + + load_module ../libcfs/libcfs/libcfs cpu_npartitions=$ncpts + [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG" [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }" load_module ../lnet/lnet/lnet -- 1.8.3.1