LU-6735 lnet: select appropriate optimization options for ko2iblnd

In order to support different IB cards in the best way we need to have
ability to select appropriate optimization options for ko2iblnd
according IB cards installed on the system.

The different defaults for different IB cards are selected by aliases
from /etc/modprobe.d/ko2iblnd.conf and customer can change them manually.

In this patch the general mechanism of selecting tunables is proposed.
If TrueScale IB cards are inserted a new optimal tunables are selected.

With proposed tunables the performance with TrueScale IB cards gets up
from 900 MB/sec to 2.2 GB/sec.

Change-Id: Ie296599f8b0c923100aac41994cbb9b49f99decf
Signed-off-by: Dmitry Eremin <dmitry.eremin@intel.com>
Reviewed-on: http://review.whamcloud.com/15313
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Amir Shehata <amir.shehata@intel.com>

diff --git a/lustre.spec.in b/lustre.spec.in
index bb7965b..1af072b 100644 (file)
--- a/lustre.spec.in
+++ b/lustre.spec.in
@@ -480,6 +480,7 @@ find $RPM_BUILD_ROOT%{?rootdir}/lib/modules/%{kversion}/%{kmoddir} \
 %{_datadir}/lustre
 %{_sysconfdir}/udev/rules.d/99-lustre.rules
 %config(noreplace) %{_sysconfdir}/ldev.conf
 %{_datadir}/lustre
 %{_sysconfdir}/udev/rules.d/99-lustre.rules
 %config(noreplace) %{_sysconfdir}/ldev.conf

+%config(noreplace) %{_sysconfdir}/modprobe.d/ko2iblnd.conf

 
 %if %{with lustre_modules}
 %files modules
 
 %if %{with lustre_modules}
 %files modules

diff --git a/lustre/conf/Makefile.am b/lustre/conf/Makefile.am
index 7e4f3ba..7615eaf 100644 (file)
--- a/lustre/conf/Makefile.am
+++ b/lustre/conf/Makefile.am
@@ -37,7 +37,7 @@
 #
 
 EXTRA_DIST = lustre.dtd slapd-lustre.conf lustre2ldif.xsl top.ldif \
 #
 
 EXTRA_DIST = lustre.dtd slapd-lustre.conf lustre2ldif.xsl top.ldif \

-             99-lustre.rules lustre ldev.conf
+             99-lustre.rules lustre ldev.conf ko2iblnd.conf

 ldapconfdir = $(sysconfdir)/openldap
 
 if UTILS
 ldapconfdir = $(sysconfdir)/openldap
 
 if UTILS


@@ -55,4 +55,7 @@ sysconfig_DATA = lustre
 endif
 endif
 
 endif
 endif
 

+modprobedir = $(sysconfdir)/modprobe.d
+modprobe_DATA = ko2iblnd.conf
+

 sysconf_DATA = ldev.conf
 sysconf_DATA = ldev.conf

diff --git a/lustre/conf/ko2iblnd.conf b/lustre/conf/ko2iblnd.conf
new file mode 100644 (file)
index 0000000..62d80ad
--- /dev/null
+++ b/lustre/conf/ko2iblnd.conf
@@ -0,0 +1,14 @@
+# Currently it isn't possible to auto-tune the o2iblnd parameters optimally
+# inside the kernel since the OFED API hides the details from us.
+# Unfortunately, there isn't a single set of parameters that provide optimal
+# performance on different HCA/HFI types, so this file provides optimized
+# tunables based on the installed IB interface cards.
+#
+# Card detection and tunable selection is handled via /usr/sbin/ko2iblnd-probe
+# at runtime when the ko2iblnd module is installed, either at boot or when
+# Lustre is first mounted.
+
+alias ko2iblnd-opa ko2iblnd
+options ko2iblnd-opa peer_credits=128 peer_credits_hiw=64 credits=1024 concurrent_sends=256 ntx=2048 map_on_demand=32 fmr_pool_size=2048 fmr_flush_trigger=512 fmr_cache=1
+
+install ko2iblnd /usr/sbin/ko2iblnd-probe

diff --git a/lustre/scripts/Makefile.am b/lustre/scripts/Makefile.am
index 11d8be7..b3883d2 100644 (file)
--- a/lustre/scripts/Makefile.am
+++ b/lustre/scripts/Makefile.am
@@ -39,7 +39,7 @@
 # These are scripts that are generated from .in files
 genscripts = lc_modprobe lc_net lc_hb lc_cluman lc_md lc_lvm lustre_start
 
 # These are scripts that are generated from .in files
 genscripts = lc_modprobe lc_net lc_hb lc_cluman lc_md lc_lvm lustre_start
 

-sbin_SCRIPTS = lustre_rmmod
+sbin_SCRIPTS = lustre_rmmod ko2iblnd-probe

 
 if RHEL
 initdir = $(sysconfdir)/init.d
 
 if RHEL
 initdir = $(sysconfdir)/init.d


@@ -75,7 +75,8 @@ EXTRA_DIST = license-status version_tag.pl version_tag-git.pl \
             version_tag-none.pl lustre_rmmod ldev lc_mon lhbadm \
             lc_servip lustre_routes_config lustre_routes_conversion \
             $(addsuffix .in,$(genscripts)) lfs_migrate lustre_req_history \
             version_tag-none.pl lustre_rmmod ldev lc_mon lhbadm \
             lc_servip lustre_routes_config lustre_routes_conversion \
             $(addsuffix .in,$(genscripts)) lfs_migrate lustre_req_history \

-            lustre lnet lc_common haconfig Lustre.ha_v2 dkms.mkconf zfsobj2fid
+            lustre lnet lc_common haconfig Lustre.ha_v2 dkms.mkconf \
+            zfsobj2fid ko2iblnd-probe

 
 CLEANFILES = $(genscripts)
 
 
 CLEANFILES = $(genscripts)
 

diff --git a/lustre/scripts/ko2iblnd-probe b/lustre/scripts/ko2iblnd-probe
new file mode 100755 (executable)
index 0000000..5d07886
--- /dev/null
+++ b/lustre/scripts/ko2iblnd-probe
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# flags of IB devices which present
+HFI=0
+QIB=0
+
+# no profile name by default
+PROFILE=""
+
+INFINIBAND="/sys/class/infiniband"
+
+if [ -d $INFINIBAND ]; then
+       for dev in `ls -d $INFINIBAND/* | sed -e "s#^$INFINIBAND/##" -e 's#_[0-9]*$##'`; do
+               ver=`echo $dev | sed -ne 's#[^0-9]*##p'`
+               case $dev in
+               hfi*) HFI=1 ;;
+               qib*) QIB=1 ;;
+               esac
+       done
+
+       # Set profile name according priority
+       if [ $HFI -ne 0 ]; then
+               PROFILE="-opa"
+       elif [ $QIB -ne 0 ]; then
+               PROFILE="-opa"
+       fi
+fi
+
+exec /sbin/modprobe --ignore-install ko2iblnd$PROFILE $CMDLINE_OPTS