From 70b96b22d71b83c69fbf4b5cdc292f3ce40e4e17 Mon Sep 17 00:00:00 2001 From: Dmitry Eremin Date: Tue, 16 Jun 2015 21:18:15 +0300 Subject: [PATCH] LU-6735 lnet: select appropriate optimization options for ko2iblnd In order to support different IB cards in the best way we need to have ability to select appropriate optimization options for ko2iblnd according IB cards installed on the system. The different defaults for different IB cards are selected by aliases from /etc/modprobe.d/ko2iblnd.conf and customer can change them manually. In this patch the general mechanism of selecting tunables is proposed. If TrueScale IB cards are inserted a new optimal tunables are selected. With proposed tunables the performance with TrueScale IB cards gets up from 900 MB/sec to 2.2 GB/sec. Change-Id: Ie296599f8b0c923100aac41994cbb9b49f99decf Signed-off-by: Dmitry Eremin Reviewed-on: http://review.whamcloud.com/15313 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Amir Shehata --- lustre.spec.in | 1 + lustre/conf/Makefile.am | 5 ++++- lustre/conf/ko2iblnd.conf | 14 ++++++++++++++ lustre/scripts/Makefile.am | 5 +++-- lustre/scripts/ko2iblnd-probe | 29 +++++++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 lustre/conf/ko2iblnd.conf create mode 100755 lustre/scripts/ko2iblnd-probe diff --git a/lustre.spec.in b/lustre.spec.in index bb7965b..1af072b 100644 --- a/lustre.spec.in +++ b/lustre.spec.in @@ -480,6 +480,7 @@ find $RPM_BUILD_ROOT%{?rootdir}/lib/modules/%{kversion}/%{kmoddir} \ %{_datadir}/lustre %{_sysconfdir}/udev/rules.d/99-lustre.rules %config(noreplace) %{_sysconfdir}/ldev.conf +%config(noreplace) %{_sysconfdir}/modprobe.d/ko2iblnd.conf %if %{with lustre_modules} %files modules diff --git a/lustre/conf/Makefile.am b/lustre/conf/Makefile.am index 7e4f3ba..7615eaf 100644 --- a/lustre/conf/Makefile.am +++ b/lustre/conf/Makefile.am @@ -37,7 +37,7 @@ # EXTRA_DIST = lustre.dtd slapd-lustre.conf lustre2ldif.xsl top.ldif \ - 99-lustre.rules lustre ldev.conf + 99-lustre.rules lustre ldev.conf ko2iblnd.conf ldapconfdir = $(sysconfdir)/openldap if UTILS @@ -55,4 +55,7 @@ sysconfig_DATA = lustre endif endif +modprobedir = $(sysconfdir)/modprobe.d +modprobe_DATA = ko2iblnd.conf + sysconf_DATA = ldev.conf diff --git a/lustre/conf/ko2iblnd.conf b/lustre/conf/ko2iblnd.conf new file mode 100644 index 0000000..62d80ad --- /dev/null +++ b/lustre/conf/ko2iblnd.conf @@ -0,0 +1,14 @@ +# Currently it isn't possible to auto-tune the o2iblnd parameters optimally +# inside the kernel since the OFED API hides the details from us. +# Unfortunately, there isn't a single set of parameters that provide optimal +# performance on different HCA/HFI types, so this file provides optimized +# tunables based on the installed IB interface cards. +# +# Card detection and tunable selection is handled via /usr/sbin/ko2iblnd-probe +# at runtime when the ko2iblnd module is installed, either at boot or when +# Lustre is first mounted. + +alias ko2iblnd-opa ko2iblnd +options ko2iblnd-opa peer_credits=128 peer_credits_hiw=64 credits=1024 concurrent_sends=256 ntx=2048 map_on_demand=32 fmr_pool_size=2048 fmr_flush_trigger=512 fmr_cache=1 + +install ko2iblnd /usr/sbin/ko2iblnd-probe diff --git a/lustre/scripts/Makefile.am b/lustre/scripts/Makefile.am index 11d8be7..b3883d2 100644 --- a/lustre/scripts/Makefile.am +++ b/lustre/scripts/Makefile.am @@ -39,7 +39,7 @@ # These are scripts that are generated from .in files genscripts = lc_modprobe lc_net lc_hb lc_cluman lc_md lc_lvm lustre_start -sbin_SCRIPTS = lustre_rmmod +sbin_SCRIPTS = lustre_rmmod ko2iblnd-probe if RHEL initdir = $(sysconfdir)/init.d @@ -75,7 +75,8 @@ EXTRA_DIST = license-status version_tag.pl version_tag-git.pl \ version_tag-none.pl lustre_rmmod ldev lc_mon lhbadm \ lc_servip lustre_routes_config lustre_routes_conversion \ $(addsuffix .in,$(genscripts)) lfs_migrate lustre_req_history \ - lustre lnet lc_common haconfig Lustre.ha_v2 dkms.mkconf zfsobj2fid + lustre lnet lc_common haconfig Lustre.ha_v2 dkms.mkconf \ + zfsobj2fid ko2iblnd-probe CLEANFILES = $(genscripts) diff --git a/lustre/scripts/ko2iblnd-probe b/lustre/scripts/ko2iblnd-probe new file mode 100755 index 0000000..5d07886 --- /dev/null +++ b/lustre/scripts/ko2iblnd-probe @@ -0,0 +1,29 @@ +#!/bin/sh + +# flags of IB devices which present +HFI=0 +QIB=0 + +# no profile name by default +PROFILE="" + +INFINIBAND="/sys/class/infiniband" + +if [ -d $INFINIBAND ]; then + for dev in `ls -d $INFINIBAND/* | sed -e "s#^$INFINIBAND/##" -e 's#_[0-9]*$##'`; do + ver=`echo $dev | sed -ne 's#[^0-9]*##p'` + case $dev in + hfi*) HFI=1 ;; + qib*) QIB=1 ;; + esac + done + + # Set profile name according priority + if [ $HFI -ne 0 ]; then + PROFILE="-opa" + elif [ $QIB -ne 0 ]; then + PROFILE="-opa" + fi +fi + +exec /sbin/modprobe --ignore-install ko2iblnd$PROFILE $CMDLINE_OPTS -- 1.8.3.1