From 147529f46ca5b5f243d6d5cb1b636e273993be54 Mon Sep 17 00:00:00 2001 From: Minh Diep Date: Tue, 27 Sep 2016 09:03:33 -0700 Subject: [PATCH] LU-8642 build: suppport building various OFED * Remove the 01-remove-mlx4-erroneous-modprobe-config-file:rhel6.ed * differentiate each type of OFED to allow different way of downloading, unpack and build with different options * symlink SLES linux-obj after unpack the rpm Test-Parameters: trivial Change-Id: I7fcd50a6b747dbb5419bb029087967f809ef2485 Signed-off-by: Minh Diep Reviewed-on: https://review.whamcloud.com/22758 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Bob Glossman Reviewed-by: Dmitry Eremin Reviewed-by: Oleg Drokin --- contrib/lbuild/lbuild | 382 +++++++++++++++------ ...ve-mlx4-erroneous-modprobe-config-file:rhel6.ed | 13 - 2 files changed, 269 insertions(+), 126 deletions(-) delete mode 100644 contrib/patches/ofed/01-remove-mlx4-erroneous-modprobe-config-file:rhel6.ed diff --git a/contrib/lbuild/lbuild b/contrib/lbuild/lbuild index f7c1028..bc1d205 100755 --- a/contrib/lbuild/lbuild +++ b/contrib/lbuild/lbuild @@ -60,6 +60,8 @@ EXTRA_VERSION= STAGEDIR= TMPDIR=${TMPDIR:-"/var/tmp"} TIMESTAMP= +# default OFED +OFED_TYPE="inkernel" # this is the dir that should be used to store reuse products REUSEBUILD= # should cached products be used or force rebuilding? @@ -103,6 +105,8 @@ LUSTRE_TESTS=true DATE=$(date) +RPMBUILD= + export CC=${CC:-gcc} # Readlink is not present on some older distributions: emulate it. @@ -172,6 +176,24 @@ Usage: ${0##*/} [OPTION]... [-- ] Do not build a .src.rpm, a full kernel patch, or a patched kernel tarball. + --ofed-type + Type of OFED to build with lustre: inkernel, ofa, mlnx, ifs + ofa: OpenFabrics Alliance + mlnx: Mellanox + ifs: Intel True Scale Fabric + + --ofed-version + Version of external OFED to build with lustre + + --mlnx-version + Version of external Mellanox OFED to build with lustre + + --ofed-src + Tarball for either OFED. Tarball must follow below format + OFED-.tgz regardless of vendors + It's likely that you need to reconstruct the directory name + It must be placed under KERNELTREE directory + --ldiskfs Build with ldiskfs support. (Deprecated, always true) @@ -260,6 +282,23 @@ check_options() { usage 1 "Could not find Lustre source tarball '$LUSTRE'." fi + if [ -n "${OFED_SRC}" ]; then + if [ -z "${OFED_VERSION}" ]; then + usage 1 "Need to provide version for file ${OFED_SRC}." + fi + if [ "${OFED_TYPE}" = "inkernel" ]; then + usage 1 "Need to provide ofed type for file ${OFED_SRC}." + fi + else + if [ "${OFED_TYPE}" != "inkernel" -a -z "${OFED_VERSION}" ]; then + usage 1 "Need to provide version for $OFED_TYPE OFED" + fi + + if [ "${OFED_TYPE}" = "inkernel" -a -n "${OFED_VERSION}" ]; then + usage 1 "Can not specify version with inkernel OFED" + fi + fi + if [ -z "$DISTRO" ] ; then DISTRO=$(autodetect_distro) # remove separator @@ -331,7 +370,7 @@ check_options() { TIMESTAMP=$(date -d "$DATE" "+%Y%m%d%H%M%S") fi - local RPMBUILD=$(which rpmbuild 2>/dev/null | head -n 1) + RPMBUILD=$(which rpmbuild 2>/dev/null | head -n 1) RPMBUILD=${RPMBUILD:-$(which rpm 2>/dev/null | head -n 1)} if [ -z "$RPMBUILD" ]; then usage 1 "Could not find binary for making rpms (tried rpmbuild and rpm)." @@ -405,8 +444,7 @@ compare_version () { return 1 fi done - - return 0 + return 1 } uniqify() { @@ -548,39 +586,59 @@ download_file() { } download_ofed() { - local force="${1:-false}" - - if [ -z "$OFED_VERSION" -o "$OFED_VERSION" = "inkernel" ]; then - return 0 - fi - - local OFED_BASE_VERSION=$OFED_VERSION - if [[ $OFED_VERSION = *.*.*.* ]]; then - OFED_BASE_VERSION=${OFED_VERSION%.*} - fi - - local location="https://www.openfabrics.org/downloads/OFED/ofed-${OFED_BASE_VERSION}/" - - if [[ $OFED_VERSION = *-[rR][cC][0-9] ]]; then - local Mmv - Mmv=${OFED_VERSION%%-[rR][cC][0-9]} - location="https://www.openfabrics.org/downloads/OFED/ofed-${Mmv}/" - fi - - if [[ $OFED_VERSION = daily-* ]]; then - local Mmv - Mmv=${OFED_VERSION/daily-/} - daily=${OFED_VERSION##$Mmv-} - location="https://www.openfabrics.org/downloads/OFED/ofed-${Mmv}-daily/" - # find the filename for the version for the date specified - OFED_VERSION=$(curl -s "$location" | sed -nre "/${Mmv}-/s/.*href=\"OFED-(${Mmv}-[0-9]{8,8}-[0-9]{4,4}).tgz.*$/\1/p" | tail -1) - if [ -z "$OFED_VERSION" ]; then - fatal 1 "Could not determine the filename of the OFED snapshot for ${daily}" - fi - fi - - local file="OFED-${OFED_VERSION}.tgz" - download_file "$location/$file" "$KERNELTREE" "$force" + local ofed_type="$1" + local ofed_version="$2" + local force="${3:-false}" + local distro_name="${DISTRO}" + local arch="${TARGET_ARCH}" + local location + local file + + #if a src tarball has been given in the command line, we use it + #The format of the tarball must be OFED-${OFED_VERSION}.tgz + [ -n "${OFED_SRC}" ] && return 0 + + case $ofed_type in + ofa) + location="https://www.openfabrics.org/downloads/OFED/ofed-${ofed_version}/" + # version include RC + if [[ $ofed_version = *-[rR][cC][0-9] ]]; then + ofed_version_loc=${ofed_version%%-[rR][cC][0-9]} + location="https://www.openfabrics.org/downloads/OFED/ofed-${ofed_version_loc}/" + fi + # daily build + if [[ $ofed_version = *-daily ]]; then + ofed_version=${ofed_version/-daily/} + location="https://www.openfabrics.org/downloads/OFED/ofed-${ofed_version}-daily/" + # find the filename for latest version + ofed_version=$(curl -1 -s "$location" | sed -nre "s/.*href=\"OFED-(${ofed_version//./\\.}-[0-9]{8}-[0-9]{4}).tgz.*$/\1/p" | tail -1) + if [ -z "$ofed_version" ]; then + fatal 1 "Could not determine the filename of the OFED snapshot from daily " + fi + fi + + file="OFED-${ofed_version}.tgz" + download_file "$location/$file" "$KERNELTREE" "$force" + ;; + mlnx) + location="http://www.mellanox.com/downloads/ofed/MLNX_OFED-${ofed_version}" + # this is a work around for suse distro (sles11.3). what we need is + # sles11sp3. We really need to redesign how we use target and distro + [[ $distro_name =~ sles ]] && distro_name=${DISTRO/./sp} + file="MLNX_OFED_LINUX-${ofed_version}-${distro_name}-${arch}.tgz" + download_file "$location/$file" "$KERNELTREE" "$force" + ;; + ifs) + location="http://downloadmirror.intel.com/24625/eng/" + file="IntelIB-Basic.$(echo ${distro_name%%.*} | tr '[:lower:]' '[:upper:]')-${arch}.${ofed_version}.tgz" + download_file "$location/$file" "$KERNELTREE" "$force" + ;; + *) + fatal 1 "Error: unknown OFED type: $ofed_type" + + esac + # version might change due to detect daily version + OFED_VERSION=${ofed_version} } @@ -716,12 +774,63 @@ untar() { } unpack_ofed() { - - if ! untar "$KERNELTREE/OFED-${OFED_VERSION}.tgz"; then - return 1 - fi - [ -d OFED ] || ln -sf OFED-[0-9].[0-9]* OFED - + local ofed_type="$1" + local ofed_version="$2" + local distro_name="${DISTRO}" + local arch="${TARGET_ARCH}" + local file + + #if a src tarball has been given in the command line, we use it + #The format of the directory after untar MUST be in OFED-${version} + #even if it's from MLNX or IFS...or whatever + if [ -n "${OFED_SRC}" ]; then + if ! untar "$KERNELTREE/${OFED_SRC}"; then + return 1 + else + [ -d OFED ] || ln -sf OFED-[0-9].[0-9]* OFED + fi + fi + case $ofed_type in + ofa) + file="OFED-${ofed_version}" + if ! untar "$KERNELTREE/${file}.tgz"; then + return 1 + fi + [ -d OFED ] || ln -sf OFED-[0-9].[0-9]* OFED + ;; + mlnx) + # this is a work around for suse distro (sles11.3). what we need is + # sles11sp3. We really need to redesign how we use target and distro + [[ $distro_name =~ sles ]] && distro_name=${DISTRO/./sp} + file="MLNX_OFED_LINUX-${ofed_version}-${distro_name}-${arch}" + + # it's not important what distro we get the tarball since we only + # interest in the src + if ! untar "$KERNELTREE/${file}.tgz"; then + return 1 + fi + # we need to untar again to get the src since it's being + # wrapped inside the tarball + # There are cases where the source version is different + # than the tarball. + # (ie. MLNX_OFED_LINUX-2.3-1.0.1 but MLNX_OFED_SRC-2.3-1.0.0) + local src=$(ls ${file}/src/MLNX_OFED_SRC-${ofed_version%.*}*.tgz) + if ! untar "$src"; then + return 1 + fi + [ -d OFED ] || ln -sf MLNX_OFED_SRC-[0-9].[0-9]* OFED + ;; + ifs) + file="IntelIB-Basic.$(echo ${distro_name%%.*} | tr '[:lower:]' '[:upper:]')-${arch}.${ofed_version}" + if ! untar "$KERNELTREE/${file}.tgz"; then + return 1 + fi + [ -d OFED ] || ln -sf $file/IntelIB-OFED.$(echo ${distro_name%%.*} | tr '[:lower:]' '[:upper:]')-${arch}.* OFED + ofed_version="$(cat OFED/Version)" + ;; + esac + # version might change due to detect daily version + OFED_VERSION=${ofed_version} } unpack_lustre() { @@ -1102,22 +1211,25 @@ build_kernel_ib() { local linux="$1" local kib_prefix="$2" local kib_rpm="$3" + local ofed_type="${4}" + local ofed_version="${5}" # build kernel-ib{,-devel}/compat-rdma{,-devel} local K_SRC="K_SRC" - # ofed 1.3 had a bug in the rpm spec - if [ "$OFED_VERSION" = "1.3" ]; then - K_SRC="KSRC" - fi - local OFED_CORE="--with-core-mod --with-ipoib-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod" - local OFED_HARDWARE="--with-mthca-mod --with-mlx4-mod --with-mlx4_en-mod --with-cxgb3-mod --with-nes-mod --with-qib-mod" + local OFED_CORE="--with-core-mod --with-ipoib-mod --with-user_mad-mod \ + --with-user_access-mod --with-addr_trans-mod --with-madeye-mod" + local OFED_HARDWARE="--with-mthca-mod --with-mlx4-mod \ + --with-mlx4_en-mod --with-cxgb3-mod --with-mlx4_en-mod \ + --with-cxgb3-mod --with-nes-mod --with-mlx5-mod --with-cxgb4-mod \ + --with-qib-mod" - if compare_version $OFED_VERSION 3.0; then - OFED_CORE="$OFED_CORE --with-madeye-mod --with-rds-mod" - else - OFED_HARDWARE="$OFED_HARDWARE --with-mlx5-mod --with-cxgb4-mod --with-ocrdma-mod --with-qib-mod" - fi + # Removing the check for older version support + #if compare_version $OFED_VERSION 3.0; then + #OFED_CORE="$OFED_CORE --with-madeye-mod --with-rds-mod" + #else + #OFED_HARDWARE="$OFED_HARDWARE --with-mlx5-mod --with-cxgb4-mod --with-ocrdma-mod --with-qib-mod" + #fi # some I/B drivers are architecture dependent and kernel-ib's configure # does not figure it out for us ~sigh~ @@ -1184,19 +1296,35 @@ EOF fi local linuxrelease=$(find_linux_release) - if compare_version $OFED_VERSION 3.0; then - local OFA_KERNEL_RELEASE=$(echo -n ${linuxrelease} | sed -e 's/-/_/g') - fi - if ! rpmbuild $BUILD_TYPE --define 'build_kernel_ib 1' --define 'build_kernel_ib_devel 1' \ - --define "_topdir ${TOPDIR}" --target ${TARGET_ARCH} \ - --define "KVERSION ${linuxrelease}" \ - --define "$K_SRC ${linux}" \ - --define "LIB_MOD_DIR /lib/modules/${linuxrelease}/updates" \ - ${OFA_KERNEL_RELEASE:+--define "_release $OFA_KERNEL_RELEASE"} \ - --define "configure_options --without-quilt $OFED_CORE $OFED_HARDWARE $OFED_ISCSI" \ - ${SOURCE} 2>&1; then - fatal 1 "Error building ${kib_rpm}" - fi + # a place to change/add any unique config + case $ofed_type in + ofa|ifs) local K_SRC_OBJ="K_SRC_OBJ" + if ! $RPMBUILD $BUILD_TYPE --define 'build_kernel_ib 1' \ + --define 'build_kernel_ib_devel 1' \ + ${FIND_REQUIRES:+--define "__find_requires $FIND_REQUIRES"} \ + --define "_topdir ${TOPDIR}" --target ${TARGET_ARCH} \ + --define "KVERSION ${linuxrelease}" \ + --define "$K_SRC ${linux}" \ + ${K_SRC_OBJ:+--define "$K_SRC_OBJ ${linux}"} \ + ${OFA_KERNEL_RELEASE:+--define "_release $OFA_KERNEL_RELEASE"} \ + --define "configure_options --without-quilt $OFED_CORE $OFED_HARDWARE $OFED_ISCSI" \ + ${SOURCE} 2>&1; then + fatal 1 "Error building ${kib_rpm}" + fi + ;; + mlnx) + if ! $RPMBUILD $BUILD_TYPE \ + ${FIND_REQUIRES:+--define "__find_requires $FIND_REQUIRES"} \ + --define "_topdir ${TOPDIR}" --target ${TARGET_ARCH} \ + --define "KVERSION ${linuxrelease}" \ + --define "$K_SRC ${linux}" \ + ${OFA_KERNEL_RELEASE:+--define "_release $OFA_KERNEL_RELEASE"} \ + --define "configure_options --without-quilt $OFED_CORE $OFED_HARDWARE $OFED_ISCSI" \ + ${SOURCE} 2>&1; then + fatal 1 "Error building ${kib_rpm}" + fi + ;; + esac } @@ -1426,16 +1554,14 @@ build_kernel_with_srpm() { # CONFIGURE_FLAGS build_ofed() { - local linux="$1" - local ofed_version="$2" - local kib_prefix - local kib_rpm - - # if an ofed version is given, then it means use OFED proper, - # not any vendor specific "inkernel" version - if [ -z "$ofed_version" ]; then - return 0 - fi + local linux="$1" + local ofed_type="$2" + local ofed_version="$3" + local kib_prefix + local kib_rpm + local pre_prefix + local o2ib_location + local rpm if [ "$ofed_version" = "inkernel" ]; then # see if there is a distro specific override for this and use @@ -1452,13 +1578,24 @@ build_ofed() { else return 0 fi - elif compare_version $OFED_VERSION 3.0; then - kib_prefix="ofa_kernel" - kib_rpm="kernel-ib" - else - kib_prefix="compat-rdma" - kib_rpm="compat-rdma" - fi + else + case $ofed_type in + mlnx) # no compat-rdma for mlnx as of 3.1 + kib_prefix="ofa_kernel" + pre_prefix="mlnx-" + kib_rpm="${pre_prefix}${kib_prefix}" + ;; + ofa|ifs) + if compare_version $ofed_version 3.0; then + kib_prefix="ofa_kernel" + kib_rpm="${pre_prefix}${kib_prefix}" + else + kib_prefix="compat-rdma" + kib_rpm="compat-rdma" + fi + ;; + esac + fi # build kernel-ib/compat-rdma if $USE_BUILD_CACHE && [ -n "$REUSEBUILD" ]; then @@ -1495,7 +1632,7 @@ build_ofed() { create_rpmbuild_dirs fi # build it - build_kernel_ib "${linux}" "${kib_prefix}" "${kib_rpm}" + build_kernel_ib "${linux}" "${pre_prefix}${kib_prefix}" "${kib_rpm}" "${ofed_type}" if [ -z "$REUSE_SIGNATURE" ]; then echo "No reuse signature was caculated so not storing the built ofed" @@ -1518,31 +1655,42 @@ build_ofed() { rm -rf ${kib_rpm}-devel mkdir ${kib_rpm}-devel cd ${kib_rpm}-devel - # the actual ofed RPMs don't have the -rc$n or -$date string appened that - # might be present on the file - #local linuxrelease=$(find_linux_release) - #ofed_version=$(echo $ofed_version | - # sed -re 's/-(20[0-9]{6,6}-[0-9]{4,4}|rc[0-9]*)$//') - # FIXME - # OFED version will have 'hyphen' for minor release. (e.g. 3.5-1, instead - # of 3.5.1) compat-rdma and compat-rdma-devel could have same version - # number, but currectly not. Once OFED fix this in the future release, we - # can remove following filter. - #ofed_version=$(echo $ofed_version | - # sed -re 's/-([0-9]*-[rR][cC][0-9]*)$//') - #local rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-${ofed_version}-${linuxrelease//-/_}.*.rpm) - # I dont' know why we have gone through the trouble to filter out the name - # of the rpm there should only be one ${kib_rpm}-devel built - local rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-*.rpm) - if ! rpm2cpio < $rpm | cpio -id; then - fatal 1 "could not unpack the ${kib_rpm}-devel rpm." - fi - CONFIGURE_FLAGS="--with-o2ib=$(pwd)/usr/src/${kib_prefix} ${CONFIGURE_FLAGS}" - popd >/dev/null + + o2ib_location="$(pwd)/usr/src/${kib_prefix}" + case $ofed_type in + mlnx) # Prior to MOFED 3.1, we had to use build_kernel_ib=1 to + # build devel rpm. not so after 3.1 + if compare_version $ofed_version 3.0; then + rpm=$(ls $TOPDIR/RPMS/*/kernel-ib-devel-${ofed_version%%-*}-*.rpm) + else + rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-${ofed_version%%-*}-*.rpm) + fi + o2ib_location="${o2ib_location}/default" + ;; + ofa) # Prior to OFA 3.18, we had to use build_kernel_ib=1 during configure, + # not so after 3.18 + if compare_version $ofed_version 3.18; then + rpm=$(ls $TOPDIR/RPMS/*/kernel-ib-devel-${ofed_version%%-*}-*.rpm) + else + rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-${ofed_version%%-*}-*.rpm) + fi + ;; + ifs) # ifs doesn't follow any convention (if any) + rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-*.rpm) + ;; + esac + + if ! rpm2cpio < $rpm | cpio -id; then + fatal 1 "could not unpack the $rpm." + fi + CONFIGURE_FLAGS="--with-o2ib=${o2ib_location} ${CONFIGURE_FLAGS}" + popd >/dev/null } build_with_srpm() { + local ofed_type="$1" + local ofed_version="$2" if ! $PATCHLESS; then local kernel_extra_version @@ -1607,7 +1755,7 @@ g/\(\/usr\/src\/kernels\/\)/s//$tmp\/reused\1/g wq EOF - build_ofed "${LINUXOBJ:-$LINUX}" "$OFED_VERSION" || + build_ofed "${LINUXOBJ:-$LINUX}" "$ofed_type" "$ofed_version" || fatal 1 "error building OFED" # now build Lustre @@ -1768,7 +1916,7 @@ set -E [ -r ~/.lbuildrc ] && . ~/.lbuildrc -options=$(getopt -o D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ccache,norpm,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,noiokit,ofed-version:,publish,disable-zfs,release,set-value:,src,stage:,target:,target-archs:,with-linux:,xen -- "$@") +options=$(getopt -o D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ccache,norpm,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,noiokit,ofed-type:,ofed-version:,mlnx-version:,ofed-src:,publish,disable-zfs,release,set-value:,src,stage:,target:,target-archs:,with-linux:,xen -- "$@") if [ $? != 0 ]; then usage 1 @@ -1865,6 +2013,14 @@ while [ "$1" ]; do OFED_VERSION="$2" shift 2 ;; + --ofed-type) + OFED_TYPE="$2" + shift 2 + ;; + --ofed-src) + OFED_SRC="$2" + shift 2 + ;; --publish) shift ;; @@ -1922,10 +2078,10 @@ unpack_lustre load_target -if [ -n "$OFED_VERSION" -a "$OFED_VERSION" != "inkernel" ]; then - download_ofed - unpack_ofed || fatal 1 "Error unpacking OFED tarball" -fi + if [ -n "$OFED_TYPE" -a "$OFED_TYPE" != "inkernel" ]; then + download_ofed "$OFED_TYPE" "$OFED_VERSION" + unpack_ofed "$OFED_TYPE" "$OFED_VERSION" || fatal 1 "Error unpacking OFED tarball" + fi # make sure the RPM build environment is set up create_rpmbuild_dirs @@ -1936,7 +2092,7 @@ if [ -n "$LINUX" ]; then find_linux_release() { _find_linux_release $LINUX } - build_ofed "${LINUXOBJ:-$LINUX}" "$OFED_VERSION" || + build_ofed "${LINUXOBJ:-$LINUX}" "$OFED_TYPE" "$OFED_VERSION" || fatal 1 "error building OFED" build_lustre "$LINUX" "$LINUXOBJ" else @@ -1944,7 +2100,7 @@ else fatal 1 "${LBUILD_DIR}/lbuild-$DISTROMAJ not found" fi source ${LBUILD_DIR}/lbuild-$DISTROMAJ - build_with_srpm || fatal 1 "Failed to build_with_srpm" + build_with_srpm "$OFED_TYPE" "$OFED_VERSION" || fatal 1 "Failed to build_with_srpm" fi stage diff --git a/contrib/patches/ofed/01-remove-mlx4-erroneous-modprobe-config-file:rhel6.ed b/contrib/patches/ofed/01-remove-mlx4-erroneous-modprobe-config-file:rhel6.ed deleted file mode 100644 index 8404d82..0000000 --- a/contrib/patches/ofed/01-remove-mlx4-erroneous-modprobe-config-file:rhel6.ed +++ /dev/null @@ -1,13 +0,0 @@ -1 -/^%define include_udev_rules/s/%(eval.*$/0/ -/ \/sbin\/depmod.* %{KVERSION}/i -deletetohere -. -?%post -n kernel-ib -+,/deletetohere/d -+,/# END of post -n kernel-ib/d -/%preun -n kernel-ib/,/^fi$/d -/%postun -n kernel-ib/,/%postun -n kernel-ib-devel/d -g/^\/etc\/init.d\/openibd$/d -g/ofed_scripts\/mlx4_en.conf/d -g/^\/etc\/modprobe.d\/mlx4_en.conf/d -- 1.8.3.1