LU-1468 o2iblnd: Support OFED-3.5 for o2ib

OFED is having new structure based on linux kernel code + backports
and packaging. Here is detailed information.
http://lists.openfabrics.org/pipermail/ewg/2011-December/017156.html

This patches are the lustre build improvements to support OFED 3.5,
3.x whatever future OFED release.

Signed-off-by: Shuichi Ihara <sihara@ddn.com>
Change-Id: Id4ffc39bc7fc24cc591bf6fb47e9b0e662993bda
Reviewed-on: http://review.whamcloud.com/3011
Tested-by: Hudson
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Brian J. Murrell <brian.murrell@intel.com>
Reviewed-by: Liang Zhen <liang.zhen@intel.com>
Reviewed-by: Isaac Huang <he.huang@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/build/autoconf/lustre-build-linux.m4 b/build/autoconf/lustre-build-linux.m4
index db58704..4c0a581 100644 (file)
--- a/build/autoconf/lustre-build-linux.m4
+++ b/build/autoconf/lustre-build-linux.m4
@@ -441,17 +441,14 @@ AC_DEFUN([LB_LINUX_TRY_MAKE],
 )
 
 #
-# LB_CONFIG_OFED_BACKPORTS
+# LB_CONFIG_COMPAT_RDMA
 #
-# include any OFED backport headers in all compile commands
-# NOTE: this does only include the backport paths, not the OFED headers
-#       adding the OFED headers is done in the lnet portion
-AC_DEFUN([LB_CONFIG_OFED_BACKPORTS],
-[AC_MSG_CHECKING([whether to use any OFED backport headers])
+AC_DEFUN([LB_CONFIG_COMPAT_RDMA],
+[AC_MSG_CHECKING([whether to use Compat RDMA])
 # set default
 AC_ARG_WITH([o2ib],
        AC_HELP_STRING([--with-o2ib=path],
-                      [build o2iblnd against path]),
+                      [build o2iblnd against path]),
        [
                case $with_o2ib in
                yes)    O2IBPATHS="$LINUX $LINUX/drivers/infiniband"
@@ -480,6 +477,30 @@ else
                        break
                fi
        done
+       compatrdma_found=false
+       if $o2ib_found; then
+               if test \( -f ${O2IBPATH}/include/linux/compat-2.6.h \); then
+                       compatrdma_found=true
+                       AC_MSG_RESULT([yes])
+                       AC_DEFINE(HAVE_COMPAT_RDMA, 1, [compat rdma found])
+               else
+                       AC_MSG_RESULT([no])
+               fi
+       fi
+fi
+])
+
+#
+# LB_CONFIG_OFED_BACKPORTS
+#
+# include any OFED backport headers in all compile commands
+# NOTE: this does only include the backport paths, not the OFED headers
+#       adding the OFED headers is done in the lnet portion
+AC_DEFUN([LB_CONFIG_OFED_BACKPORTS],
+[AC_MSG_CHECKING([whether to use any OFED backport headers])
+if test $ENABLEO2IB -eq 0; then
+       AC_MSG_RESULT([no])
+else
        if ! $o2ib_found; then
                AC_MSG_RESULT([no])
                case $ENABLEO2IB in
@@ -489,10 +510,12 @@ else
                        *) AC_MSG_ERROR([internal error]);;
                esac
        else
-                if test -f $O2IBPATH/config.mk; then
-                       . $O2IBPATH/config.mk
-                elif test -f $O2IBPATH/ofed_patch.mk; then
-                       . $O2IBPATH/ofed_patch.mk
+               if ! $compatrdma_found; then
+                       if test -f $O2IBPATH/config.mk; then
+                               . $O2IBPATH/config.mk
+                       elif test -f $O2IBPATH/ofed_patch.mk; then
+                               . $O2IBPATH/ofed_patch.mk
+                       fi
                fi
                if test -n "$BACKPORT_INCLUDES"; then
                        OFED_BACKPORT_PATH="$O2IBPATH/${BACKPORT_INCLUDES/*\/kernel_addons/kernel_addons}/"
@@ -500,7 +523,7 @@ else
                        AC_MSG_RESULT([yes])
                else
                        AC_MSG_RESULT([no])
-                fi
+               fi
        fi
 fi
 ])
@@ -558,6 +581,8 @@ LB_LINUX_CONFIG([KALLSYMS],[],[
 # 2.6.28
 LC_MODULE_LOADING
 
+LB_CONFIG_COMPAT_RDMA
+
 # it's ugly to be doing anything with OFED outside of the lnet module, but
 # this has to be done here so that the backports path is set before all of
 # the LN_PROG_LINUX checks are done

diff --git a/build/lbuild b/build/lbuild
index 58bf239..bd5182e 100755 (executable)
--- a/build/lbuild
+++ b/build/lbuild
@@ -389,6 +389,27 @@ check_options() {
 
 }
 
+# compare two versions $1 and $2. if $1 < $2, return 0 otherwise return 1.
+compare_version () {
+    [[ $1 == $2 ]] && return 1
+    local IFS=.
+    local i val1=($1) val2=($2)
+
+    # padding zero to val1 if it needs
+    for ((i=${#val1[@]}; i<${#val2[@]}; i++)); do
+        val1[i]=0
+    done
+    for ((i=0; i<${#val1[@]}; i++)); do
+        [[ -z ${val2[i]} ]] && return 1
+
+        if [[ ${val1[i]} < ${val2[i]} ]]; then
+            return 0
+        elif [[ ${val1[i]} > ${val2[i]} ]]; then
+            return 1
+        fi
+    done
+}
+
 uniqify() {
 
     echo $(echo "$*" | xargs -n 1 | sort -u)
@@ -542,6 +563,12 @@ download_ofed() {
 
     local location="http://www.openfabrics.org/downloads/OFED/ofed-${OFED_BASE_VERSION}/"
 
+    if [[ $OFED_VERSION = *-rc[0-9] ]]; then
+        local Mmv
+        Mmv=${OFED_VERSION%%-*}
+        location="http://www.openfabrics.org/downloads/OFED/ofed-${Mmv}/"
+    fi
+
     if [[ $OFED_VERSION = daily-* ]]; then
         local Mmv daily
         OFED_VERSION=${OFED_VERSION/daily-/}
@@ -1169,16 +1196,25 @@ unpack_linux_devel_rpm() {
 
 build_kernel_ib() {
     local linux="$1"
+    local kib_prefix="$2"
+    local kib_rpm="$3"
 
-    # build kernel-ib{,-devel}
+    # build kernel-ib{,-devel}/compat-rdma{,-devel}
     local K_SRC="K_SRC"
     # ofed 1.3 had a bug in the rpm spec
     if [ "$OFED_VERSION" = "1.3" ]; then
         K_SRC="KSRC"
     fi
 
-    local OFED_CORE="--with-core-mod --with-ipoib-mod --with-sdp-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod --with-rds-mod --with-madeye-mod"
+    local OFED_CORE="--with-core-mod --with-ipoib-mod --with-sdp-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod"
     local OFED_HARDWARE="--with-mthca-mod --with-mlx4-mod --with-mlx4_en-mod --with-cxgb3-mod --with-nes-mod"
+
+    if compare_version $OFED_VERSION 3.0; then
+       OFED_CORE="$OFED_CORE --with-madeye-mod --with-rds-mod"
+    else
+       OFED_HARDWARE="$OFED_HARDWARE --with-cxgb4-mod"
+    fi
+
     # some I/B drivers are architecture dependent and kernel-ib's configure
     # does not figure it out for us ~sigh~
     case "$TARGET_ARCH" in
@@ -1197,13 +1233,13 @@ build_kernel_ib() {
 
     # assume we are just rebuilding the SRPM
     local BUILD_TYPE=${BUILD_TYPE:-"--rebuild"}
-    local SOURCE="${TOPDIR}/OFED/SRPMS/ofa_kernel-*.src.rpm"
+    local SOURCE="${TOPDIR}/OFED/SRPMS/${kib_prefix}-*.src.rpm"
 
     # but switch to building from the SPEC if we need to apply patches
     if ls ${TOPDIR}/lustre/build/patches/ofed/* >/dev/null; then
         BUILD_TYPE="-bb"
         rpm --define "_topdir ${TOPDIR}" -ivh $SOURCE
-        SOURCE="${TOPDIR}/SPECS/ofa_kernel.spec"
+        SOURCE="${TOPDIR}/SPECS/${kib_prefix}.spec"
         local file ed_fragment1 ed_fragment2 n=1
         for file in $(ls ${TOPDIR}/lustre/build/patches/ofed/*.patch); do
             ed_fragment1="$ed_fragment1
@@ -1234,6 +1270,9 @@ EOF
     fi
 
     local linuxrelease=$(find_linux_release "$linux")
+    if compare_version $OFED_VERSION 3.0; then
+        local OFA_KERNEL_RELEASE=$(echo -n ${linuxrelease} | sed -e 's/-/_/g')
+    fi
     if ! $RPMBUILD $BUILD_TYPE --define 'build_kernel_ib 1' --define 'build_kernel_ib_devel 1' \
                   ${FIND_REQUIRES:+--define "__find_requires $FIND_REQUIRES"} \
                   --define "_topdir ${TOPDIR}" --target ${TARGET_ARCH} \
@@ -1243,7 +1282,7 @@ EOF
                   ${OFA_KERNEL_RELEASE:+--define "_release $OFA_KERNEL_RELEASE"} \
                   --define "configure_options --without-quilt $OFED_CORE $OFED_HARDWARE $OFED_ISCSI" \
                   ${SOURCE} 2>&1; then
-        fatal 1 "Error building kernel-ib"
+        fatal 1 "Error building ${kib_rpm}"
     fi
 
 }
@@ -1473,6 +1512,8 @@ build_kernel_with_srpm() {
 build_ofed() {
     local linux="$1"
     local ofed_version="$2"
+    local kib_prefix
+    local kib_rpm
 
     # if an ofed version is given, then it means use OFED proper,
     # not any vendor specific "inkernel" version
@@ -1495,9 +1536,15 @@ build_ofed() {
         else
             return 0
         fi
+    elif compare_version $OFED_VERSION 3.0; then
+       kib_prefix="ofa_kernel"
+       kib_rpm="kernel-ib"
+    else
+       kib_prefix="compat-rdma"
+       kib_rpm="compat-rdma"
     fi
 
-    # build kernel-ib
+    # build kernel-ib/compat-rdma
     if $USE_BUILD_CACHE && [ -n "$REUSEBUILD" ]; then
         local REUSE_SIGNATURE=$({ echo "$ofed_version";
                                   echo "$(find_linux_release ${linux})";
@@ -1529,7 +1576,7 @@ build_ofed() {
             create_rpmbuild_dirs
         fi
         # build it
-        build_kernel_ib "${linux}"
+        build_kernel_ib "${linux}" "${kib_prefix}" "${kib_rpm}"
 
         if [ -z "$REUSE_SIGNATURE" ]; then
             echo "No reuse signature was caculated so not storing the built ofed"
@@ -1549,19 +1596,19 @@ build_ofed() {
     fi
 
     pushd "$TOPDIR" >/dev/null
-    rm -rf kernel-ib-devel
-    mkdir kernel-ib-devel
-    cd kernel-ib-devel
+    rm -rf ${kib_rpm}-devel
+    mkdir ${kib_rpm}-devel
+    cd ${kib_rpm}-devel
     # the actual ofed RPMs don't have the -rc$n or -$date string appened that
     # might be present on the file
     local linuxrelease=$(find_linux_release "$linux")
     ofed_version=$(echo $ofed_version |
                    sed -re 's/-(20[0-9]{6,6}-[0-9]{4,4}|rc[0-9]*)$//')
-    local rpm=$(ls $TOPDIR/RPMS/*/kernel-ib-devel-${ofed_version}-${linuxrelease//-/_}.*.rpm)
+    local rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-${ofed_version}-${linuxrelease//-/_}.*.rpm)
     if ! rpm2cpio < $rpm | cpio -id; then
-        fatal 1 "could not unpack the kernel-ib-devel rpm."
+        fatal 1 "could not unpack the ${kib_rpm}-devel rpm."
     fi
-    CONFIGURE_FLAGS="--with-o2ib=$(pwd)/usr/src/ofa_kernel ${CONFIGURE_FLAGS}"
+    CONFIGURE_FLAGS="--with-o2ib=$(pwd)/usr/src/${kib_prefix} ${CONFIGURE_FLAGS}"
     popd >/dev/null
 
 }

diff --git a/libcfs/include/libcfs/linux/linux-tcpip.h b/libcfs/include/libcfs/linux/linux-tcpip.h
index 47cc539..b358773 100644 (file)
--- a/libcfs/include/libcfs/linux/linux-tcpip.h
+++ b/libcfs/include/libcfs/linux/linux-tcpip.h
@@ -84,10 +84,12 @@ libcfs_sock_wmem_queued(struct socket *sock)
 }
 
 #ifndef HAVE_SK_SLEEP
-static inline wait_queue_head_t *sk_sleep(struct sock *sk)
+static inline wait_queue_head_t *cfs_sk_sleep(struct sock *sk)
 {
         return sk->sk_sleep;
 }
+#else
+#define cfs_sk_sleep(sk)       sk_sleep(sk)
 #endif
 
 #ifdef HAVE_INIT_NET

diff --git a/libcfs/libcfs/linux/linux-tcpip.c b/libcfs/libcfs/linux/linux-tcpip.c
index cd07648..0a31b55 100644 (file)
--- a/libcfs/libcfs/linux/linux-tcpip.c
+++ b/libcfs/libcfs/linux/linux-tcpip.c
@@ -592,7 +592,7 @@ libcfs_sock_accept (struct socket **newsockp, struct socket *sock)
         newsock->ops = sock->ops;
 
         set_current_state(TASK_INTERRUPTIBLE);
-        add_wait_queue(sk_sleep(sock->sk), &wait);
+       add_wait_queue(cfs_sk_sleep(sock->sk), &wait);
 
         rc = sock->ops->accept(sock, newsock, O_NONBLOCK);
         if (rc == -EAGAIN) {
@@ -601,7 +601,7 @@ libcfs_sock_accept (struct socket **newsockp, struct socket *sock)
                 rc = sock->ops->accept(sock, newsock, O_NONBLOCK);
         }
 
-        remove_wait_queue(sk_sleep(sock->sk), &wait);
+       remove_wait_queue(cfs_sk_sleep(sock->sk), &wait);
         set_current_state(TASK_RUNNING);
 
         if (rc != 0)
@@ -620,7 +620,7 @@ EXPORT_SYMBOL(libcfs_sock_accept);
 void
 libcfs_sock_abort_accept (struct socket *sock)
 {
-        wake_up_all(sk_sleep(sock->sk));
+       wake_up_all(cfs_sk_sleep(sock->sk));
 }
 
 EXPORT_SYMBOL(libcfs_sock_abort_accept);

diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4
index bb30405..01c2794 100644 (file)
--- a/lnet/autoconf/lustre-lnet.m4
+++ b/lnet/autoconf/lustre-lnet.m4
@@ -417,6 +417,9 @@ else
                        #include <linux/version.h>
                        #include <linux/pci.h>
                        #include <linux/gfp.h>
+                       #ifdef HAVE_COMPAT_RDMA
+                       #include <linux/compat-2.6.h>
+                       #endif
                        #include <rdma/rdma_cm.h>
                        #include <rdma/ib_cm.h>
                        #include <rdma/ib_verbs.h>
@@ -482,6 +485,9 @@ AC_SUBST(O2IBLND)
 if test $ENABLEO2IB -ne 0; then
        AC_MSG_CHECKING([if rdma_create_id wants four args])
        LB_LINUX_TRY_COMPILE([
+               #ifdef HAVE_COMPAT_RDMA
+               #include <linux/compat-2.6.h>
+               #endif
                #include <rdma/rdma_cm.h>
        ],[
                rdma_create_id(NULL, NULL, 0, 0);

diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h
index 2c8f637..87c7cc0 100644 (file)
--- a/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/lnet/klnds/o2iblnd/o2iblnd.h
@@ -69,6 +69,9 @@
 #include <lnet/lib-lnet.h>
 #include <lnet/lnet-sysctl.h>
 
+#ifdef HAVE_COMPAT_RDMA
+#include <linux/compat-2.6.h>
+#endif
 #include <rdma/rdma_cm.h>
 #include <rdma/ib_cm.h>
 #include <rdma/ib_verbs.h>