From c88317218461159cae399f74b459a850ae3c84f8 Mon Sep 17 00:00:00 2001 From: Shuichi Ihara Date: Sun, 6 Jan 2013 18:40:39 +0900 Subject: [PATCH] LU-1468 o2iblnd: Support OFED-3.5 for o2ib OFED is having new structure based on linux kernel code + backports and packaging. Here is detailed information. http://lists.openfabrics.org/pipermail/ewg/2011-December/017156.html This patches are the lustre build improvements to support OFED 3.5, 3.x whatever future OFED release. Signed-off-by: Shuichi Ihara Change-Id: Id4ffc39bc7fc24cc591bf6fb47e9b0e662993bda Reviewed-on: http://review.whamcloud.com/3011 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Brian J. Murrell Reviewed-by: Liang Zhen Reviewed-by: Isaac Huang Reviewed-by: Oleg Drokin --- build/autoconf/lustre-build-linux.m4 | 49 ++++++++++++++++----- build/lbuild | 73 +++++++++++++++++++++++++------ libcfs/include/libcfs/linux/linux-tcpip.h | 4 +- libcfs/libcfs/linux/linux-tcpip.c | 6 +-- lnet/autoconf/lustre-lnet.m4 | 6 +++ lnet/klnds/o2iblnd/o2iblnd.h | 3 ++ 6 files changed, 112 insertions(+), 29 deletions(-) diff --git a/build/autoconf/lustre-build-linux.m4 b/build/autoconf/lustre-build-linux.m4 index db58704..4c0a581 100644 --- a/build/autoconf/lustre-build-linux.m4 +++ b/build/autoconf/lustre-build-linux.m4 @@ -441,17 +441,14 @@ AC_DEFUN([LB_LINUX_TRY_MAKE], ) # -# LB_CONFIG_OFED_BACKPORTS +# LB_CONFIG_COMPAT_RDMA # -# include any OFED backport headers in all compile commands -# NOTE: this does only include the backport paths, not the OFED headers -# adding the OFED headers is done in the lnet portion -AC_DEFUN([LB_CONFIG_OFED_BACKPORTS], -[AC_MSG_CHECKING([whether to use any OFED backport headers]) +AC_DEFUN([LB_CONFIG_COMPAT_RDMA], +[AC_MSG_CHECKING([whether to use Compat RDMA]) # set default AC_ARG_WITH([o2ib], AC_HELP_STRING([--with-o2ib=path], - [build o2iblnd against path]), + [build o2iblnd against path]), [ case $with_o2ib in yes) O2IBPATHS="$LINUX $LINUX/drivers/infiniband" @@ -480,6 +477,30 @@ else break fi done + compatrdma_found=false + if $o2ib_found; then + if test \( -f ${O2IBPATH}/include/linux/compat-2.6.h \); then + compatrdma_found=true + AC_MSG_RESULT([yes]) + AC_DEFINE(HAVE_COMPAT_RDMA, 1, [compat rdma found]) + else + AC_MSG_RESULT([no]) + fi + fi +fi +]) + +# +# LB_CONFIG_OFED_BACKPORTS +# +# include any OFED backport headers in all compile commands +# NOTE: this does only include the backport paths, not the OFED headers +# adding the OFED headers is done in the lnet portion +AC_DEFUN([LB_CONFIG_OFED_BACKPORTS], +[AC_MSG_CHECKING([whether to use any OFED backport headers]) +if test $ENABLEO2IB -eq 0; then + AC_MSG_RESULT([no]) +else if ! $o2ib_found; then AC_MSG_RESULT([no]) case $ENABLEO2IB in @@ -489,10 +510,12 @@ else *) AC_MSG_ERROR([internal error]);; esac else - if test -f $O2IBPATH/config.mk; then - . $O2IBPATH/config.mk - elif test -f $O2IBPATH/ofed_patch.mk; then - . $O2IBPATH/ofed_patch.mk + if ! $compatrdma_found; then + if test -f $O2IBPATH/config.mk; then + . $O2IBPATH/config.mk + elif test -f $O2IBPATH/ofed_patch.mk; then + . $O2IBPATH/ofed_patch.mk + fi fi if test -n "$BACKPORT_INCLUDES"; then OFED_BACKPORT_PATH="$O2IBPATH/${BACKPORT_INCLUDES/*\/kernel_addons/kernel_addons}/" @@ -500,7 +523,7 @@ else AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) - fi + fi fi fi ]) @@ -558,6 +581,8 @@ LB_LINUX_CONFIG([KALLSYMS],[],[ # 2.6.28 LC_MODULE_LOADING +LB_CONFIG_COMPAT_RDMA + # it's ugly to be doing anything with OFED outside of the lnet module, but # this has to be done here so that the backports path is set before all of # the LN_PROG_LINUX checks are done diff --git a/build/lbuild b/build/lbuild index 58bf239..bd5182e 100755 --- a/build/lbuild +++ b/build/lbuild @@ -389,6 +389,27 @@ check_options() { } +# compare two versions $1 and $2. if $1 < $2, return 0 otherwise return 1. +compare_version () { + [[ $1 == $2 ]] && return 1 + local IFS=. + local i val1=($1) val2=($2) + + # padding zero to val1 if it needs + for ((i=${#val1[@]}; i<${#val2[@]}; i++)); do + val1[i]=0 + done + for ((i=0; i<${#val1[@]}; i++)); do + [[ -z ${val2[i]} ]] && return 1 + + if [[ ${val1[i]} < ${val2[i]} ]]; then + return 0 + elif [[ ${val1[i]} > ${val2[i]} ]]; then + return 1 + fi + done +} + uniqify() { echo $(echo "$*" | xargs -n 1 | sort -u) @@ -542,6 +563,12 @@ download_ofed() { local location="http://www.openfabrics.org/downloads/OFED/ofed-${OFED_BASE_VERSION}/" + if [[ $OFED_VERSION = *-rc[0-9] ]]; then + local Mmv + Mmv=${OFED_VERSION%%-*} + location="http://www.openfabrics.org/downloads/OFED/ofed-${Mmv}/" + fi + if [[ $OFED_VERSION = daily-* ]]; then local Mmv daily OFED_VERSION=${OFED_VERSION/daily-/} @@ -1169,16 +1196,25 @@ unpack_linux_devel_rpm() { build_kernel_ib() { local linux="$1" + local kib_prefix="$2" + local kib_rpm="$3" - # build kernel-ib{,-devel} + # build kernel-ib{,-devel}/compat-rdma{,-devel} local K_SRC="K_SRC" # ofed 1.3 had a bug in the rpm spec if [ "$OFED_VERSION" = "1.3" ]; then K_SRC="KSRC" fi - local OFED_CORE="--with-core-mod --with-ipoib-mod --with-sdp-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod --with-rds-mod --with-madeye-mod" + local OFED_CORE="--with-core-mod --with-ipoib-mod --with-sdp-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod" local OFED_HARDWARE="--with-mthca-mod --with-mlx4-mod --with-mlx4_en-mod --with-cxgb3-mod --with-nes-mod" + + if compare_version $OFED_VERSION 3.0; then + OFED_CORE="$OFED_CORE --with-madeye-mod --with-rds-mod" + else + OFED_HARDWARE="$OFED_HARDWARE --with-cxgb4-mod" + fi + # some I/B drivers are architecture dependent and kernel-ib's configure # does not figure it out for us ~sigh~ case "$TARGET_ARCH" in @@ -1197,13 +1233,13 @@ build_kernel_ib() { # assume we are just rebuilding the SRPM local BUILD_TYPE=${BUILD_TYPE:-"--rebuild"} - local SOURCE="${TOPDIR}/OFED/SRPMS/ofa_kernel-*.src.rpm" + local SOURCE="${TOPDIR}/OFED/SRPMS/${kib_prefix}-*.src.rpm" # but switch to building from the SPEC if we need to apply patches if ls ${TOPDIR}/lustre/build/patches/ofed/* >/dev/null; then BUILD_TYPE="-bb" rpm --define "_topdir ${TOPDIR}" -ivh $SOURCE - SOURCE="${TOPDIR}/SPECS/ofa_kernel.spec" + SOURCE="${TOPDIR}/SPECS/${kib_prefix}.spec" local file ed_fragment1 ed_fragment2 n=1 for file in $(ls ${TOPDIR}/lustre/build/patches/ofed/*.patch); do ed_fragment1="$ed_fragment1 @@ -1234,6 +1270,9 @@ EOF fi local linuxrelease=$(find_linux_release "$linux") + if compare_version $OFED_VERSION 3.0; then + local OFA_KERNEL_RELEASE=$(echo -n ${linuxrelease} | sed -e 's/-/_/g') + fi if ! $RPMBUILD $BUILD_TYPE --define 'build_kernel_ib 1' --define 'build_kernel_ib_devel 1' \ ${FIND_REQUIRES:+--define "__find_requires $FIND_REQUIRES"} \ --define "_topdir ${TOPDIR}" --target ${TARGET_ARCH} \ @@ -1243,7 +1282,7 @@ EOF ${OFA_KERNEL_RELEASE:+--define "_release $OFA_KERNEL_RELEASE"} \ --define "configure_options --without-quilt $OFED_CORE $OFED_HARDWARE $OFED_ISCSI" \ ${SOURCE} 2>&1; then - fatal 1 "Error building kernel-ib" + fatal 1 "Error building ${kib_rpm}" fi } @@ -1473,6 +1512,8 @@ build_kernel_with_srpm() { build_ofed() { local linux="$1" local ofed_version="$2" + local kib_prefix + local kib_rpm # if an ofed version is given, then it means use OFED proper, # not any vendor specific "inkernel" version @@ -1495,9 +1536,15 @@ build_ofed() { else return 0 fi + elif compare_version $OFED_VERSION 3.0; then + kib_prefix="ofa_kernel" + kib_rpm="kernel-ib" + else + kib_prefix="compat-rdma" + kib_rpm="compat-rdma" fi - # build kernel-ib + # build kernel-ib/compat-rdma if $USE_BUILD_CACHE && [ -n "$REUSEBUILD" ]; then local REUSE_SIGNATURE=$({ echo "$ofed_version"; echo "$(find_linux_release ${linux})"; @@ -1529,7 +1576,7 @@ build_ofed() { create_rpmbuild_dirs fi # build it - build_kernel_ib "${linux}" + build_kernel_ib "${linux}" "${kib_prefix}" "${kib_rpm}" if [ -z "$REUSE_SIGNATURE" ]; then echo "No reuse signature was caculated so not storing the built ofed" @@ -1549,19 +1596,19 @@ build_ofed() { fi pushd "$TOPDIR" >/dev/null - rm -rf kernel-ib-devel - mkdir kernel-ib-devel - cd kernel-ib-devel + rm -rf ${kib_rpm}-devel + mkdir ${kib_rpm}-devel + cd ${kib_rpm}-devel # the actual ofed RPMs don't have the -rc$n or -$date string appened that # might be present on the file local linuxrelease=$(find_linux_release "$linux") ofed_version=$(echo $ofed_version | sed -re 's/-(20[0-9]{6,6}-[0-9]{4,4}|rc[0-9]*)$//') - local rpm=$(ls $TOPDIR/RPMS/*/kernel-ib-devel-${ofed_version}-${linuxrelease//-/_}.*.rpm) + local rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-${ofed_version}-${linuxrelease//-/_}.*.rpm) if ! rpm2cpio < $rpm | cpio -id; then - fatal 1 "could not unpack the kernel-ib-devel rpm." + fatal 1 "could not unpack the ${kib_rpm}-devel rpm." fi - CONFIGURE_FLAGS="--with-o2ib=$(pwd)/usr/src/ofa_kernel ${CONFIGURE_FLAGS}" + CONFIGURE_FLAGS="--with-o2ib=$(pwd)/usr/src/${kib_prefix} ${CONFIGURE_FLAGS}" popd >/dev/null } diff --git a/libcfs/include/libcfs/linux/linux-tcpip.h b/libcfs/include/libcfs/linux/linux-tcpip.h index 47cc539..b358773 100644 --- a/libcfs/include/libcfs/linux/linux-tcpip.h +++ b/libcfs/include/libcfs/linux/linux-tcpip.h @@ -84,10 +84,12 @@ libcfs_sock_wmem_queued(struct socket *sock) } #ifndef HAVE_SK_SLEEP -static inline wait_queue_head_t *sk_sleep(struct sock *sk) +static inline wait_queue_head_t *cfs_sk_sleep(struct sock *sk) { return sk->sk_sleep; } +#else +#define cfs_sk_sleep(sk) sk_sleep(sk) #endif #ifdef HAVE_INIT_NET diff --git a/libcfs/libcfs/linux/linux-tcpip.c b/libcfs/libcfs/linux/linux-tcpip.c index cd07648..0a31b55 100644 --- a/libcfs/libcfs/linux/linux-tcpip.c +++ b/libcfs/libcfs/linux/linux-tcpip.c @@ -592,7 +592,7 @@ libcfs_sock_accept (struct socket **newsockp, struct socket *sock) newsock->ops = sock->ops; set_current_state(TASK_INTERRUPTIBLE); - add_wait_queue(sk_sleep(sock->sk), &wait); + add_wait_queue(cfs_sk_sleep(sock->sk), &wait); rc = sock->ops->accept(sock, newsock, O_NONBLOCK); if (rc == -EAGAIN) { @@ -601,7 +601,7 @@ libcfs_sock_accept (struct socket **newsockp, struct socket *sock) rc = sock->ops->accept(sock, newsock, O_NONBLOCK); } - remove_wait_queue(sk_sleep(sock->sk), &wait); + remove_wait_queue(cfs_sk_sleep(sock->sk), &wait); set_current_state(TASK_RUNNING); if (rc != 0) @@ -620,7 +620,7 @@ EXPORT_SYMBOL(libcfs_sock_accept); void libcfs_sock_abort_accept (struct socket *sock) { - wake_up_all(sk_sleep(sock->sk)); + wake_up_all(cfs_sk_sleep(sock->sk)); } EXPORT_SYMBOL(libcfs_sock_abort_accept); diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 index bb30405..01c2794 100644 --- a/lnet/autoconf/lustre-lnet.m4 +++ b/lnet/autoconf/lustre-lnet.m4 @@ -417,6 +417,9 @@ else #include #include #include + #ifdef HAVE_COMPAT_RDMA + #include + #endif #include #include #include @@ -482,6 +485,9 @@ AC_SUBST(O2IBLND) if test $ENABLEO2IB -ne 0; then AC_MSG_CHECKING([if rdma_create_id wants four args]) LB_LINUX_TRY_COMPILE([ + #ifdef HAVE_COMPAT_RDMA + #include + #endif #include ],[ rdma_create_id(NULL, NULL, 0, 0); diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index 2c8f637..87c7cc0 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -69,6 +69,9 @@ #include #include +#ifdef HAVE_COMPAT_RDMA +#include +#endif #include #include #include -- 1.8.3.1