X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;ds=sidebyside;f=contrib%2Flbuild%2Flbuild;h=186118b311b9b7378e866c137424a002bcd7e651;hb=f625f670afbe954030ff81f0f8522137d6cdd335;hp=24c3a1ee72a9323fee833557a60c743ee7b75ef3;hpb=0041b3977ec04bd74092dc7ea0e329074c9aba27;p=fs%2Flustre-release.git diff --git a/contrib/lbuild/lbuild b/contrib/lbuild/lbuild index 24c3a1e..186118b 100755 --- a/contrib/lbuild/lbuild +++ b/contrib/lbuild/lbuild @@ -71,6 +71,7 @@ OSDLDISKFSRPM=true OSDZFSRPM=false SMPTYPES="smp bigsmp default ''" PATCHLESS=false +WITH_ZFS="" XEN=false LINUXOBJ= DISTRO= @@ -168,7 +169,7 @@ Usage: ${0##*/} [OPTION]... [-- ] Path to an existing lustre source tarball to use. --nodownload - Do not try to download a kernel from downloads.lustre.org + Do not try to download a kernel from downloads.hpdd.intel.com --nosrc Do not build a .src.rpm, a full kernel patch, or a patched kernel @@ -187,6 +188,9 @@ Usage: ${0##*/} [OPTION]... [-- ] Specifies that the files generated do not include timestamps, and that this is an official release. + --disable-zfs + Build Lustre without ZFS. + --src Build a .src.rpm, a full kernel patch, and a patched kernel tarball. @@ -284,7 +288,7 @@ check_options() { 3.10-rhel7) CANONICAL_TARGET="rhel7" ;; - 2.6-rhel6) + 2.6-rhel6*) CANONICAL_TARGET="rhel6" ;; 2.6-rhel5) @@ -431,7 +435,7 @@ download_srpm() { if $force || [ ! -r "$KERNELDIR/$srpm" ] || [ ! -s "$KERNELDIR/$srpm" ]; then if $DOWNLOAD; then - local location="http://downloads.lustre.org/public/kernels/$target/old" + local location="https://downloads.hpdd.intel.com/public/kernels/$target/old" # get the location from a distro specific method if it exists if type -p kernel_srpm_location; then location=$(kernel_srpm_location) @@ -552,17 +556,16 @@ download_ofed() { if [[ $OFED_VERSION = *-[rR][cC][0-9] ]]; then local Mmv Mmv=${OFED_VERSION%%-[rR][cC][0-9]} - location="http://www.openfabrics.org/downloads/OFED/ofed-${Mmv}/" + location="https://www.openfabrics.org/downloads/OFED/ofed-${Mmv}/" fi if [[ $OFED_VERSION = daily-* ]]; then - local Mmv daily - OFED_VERSION=${OFED_VERSION/daily-/} - Mmv=${OFED_VERSION%%-*} + local Mmv + Mmv=${OFED_VERSION/daily-/} daily=${OFED_VERSION##$Mmv-} - location="http://www.openfabrics.org/downloads/OFED/ofed-${Mmv}-daily/" + location="https://www.openfabrics.org/downloads/OFED/ofed-${Mmv}-daily/" # find the filename for the version for the date specified - OFED_VERSION=$(curl -s "$location" | sed -nre "/${daily}-/s/.*href=\"OFED-(${Mmv//./\\.}-${daily}-[0-9]{4,4}).tgz.*$/\1/p" | tail -1) + OFED_VERSION=$(curl -s "$location" | sed -nre "/${Mmv}-/s/.*href=\"OFED-(${Mmv}-[0-9]{8,8}-[0-9]{4,4}).tgz.*$/\1/p" | tail -1) if [ -z "$OFED_VERSION" ]; then fatal 1 "Could not determine the filename of the OFED snapshot for ${daily}" fi @@ -595,8 +598,12 @@ parse_mpss_info() { for url in $urls; do local ver2="" local ver3="" + # Try to match with MPSS 3.3 package + if [[ $url =~ mpss-[0-9].[0-9](.[0-9]*)?-linux.tar ]]; then + ver3=${url##*mpss-} + ver3=${ver3%%-linux.tar} # Try to match with MPSS 3.x or 3.x.x distro package - if [[ $url =~ mpss-[0-9].[0-9](.[0-9]*)?-$distro.tar ]]; then + elif [[ $url =~ mpss-[0-9].[0-9](.[0-9]*)?-$distro.tar ]]; then ver3=${url##*mpss-} ver3=${ver3%%-$distro.tar} # Try to match with MPSS 3.x or 3.x.x source package @@ -614,15 +621,10 @@ parse_mpss_info() { fi if [ -n "$ver3" ]; then - # Check for MPSS 3.x or MPSS 3.x.x - if [[ $ver3 =~ [0-9].[0-9].[0-9]* ]]; then - ver3="${ver3}-0" - else - ver3="${ver3}.0-0" - fi + ver3="${ver3}-0" if [ -z "$build" ]; then res3="$ver3" - elif [ "$build" = "$ver3" ]; then + elif [ "${build%%-*}" = "${ver3%%-*}" ]; then res3="$res3 $url" fi elif [ -n "$ver2" ]; then @@ -643,24 +645,8 @@ parse_mpss_info() { echo "$res2" fi elif [ "${build%%.*}" = "3" ]; then - local ver=${build%%-*} - local bid=${build##*-} - if [ $bid -eq 0 ]; then - # return URLs from MPSS info file - echo "$res3" - else - # Check for MPSS 3.x or MPSS 3.x.x - if [ ${ver##*.} -eq 0 ]; then - ver=${ver%.*} - fi - # return URLs from internal site with MPSS builds - # kernel sources: - echo "${MPSS_BUILDS:-"http://mic-bld.pdx.intel.com/release"}/$ver/$bid/release/knightscorner/package/mpss-src-$ver.tar" - # kernel configs: - echo "${MPSS_BUILDS:-"http://mic-bld.pdx.intel.com/release"}/$ver/$bid/release/knightscorner/package/mpss-$ver-k1om.tar" - # OFED headers: - echo "${MPSS_BUILDS:-"http://mic-bld.pdx.intel.com/release"}/$ver/$bid/release/knightscorner/package/mpss-$ver-$distro.tar" - fi + # return URLs from MPSS info file + echo "$res3" else # return URLs from MPSS info file echo "$res2" @@ -682,15 +668,12 @@ download_mpss() { local force="${2:-false}" local urls=$(parse_mpss_info "$file" ${MPSS_VERSION} ${MPSS_DISTRO}) - [ -z "$urls" ] && fatal 1 "Could not determine the URLs of MPSS $MPSS_VERSION" - - # force re-download if build number is zero - [[ $MPSS_VERSION = [0-9].[0-9].[0-9]*-0 ]] && force=true + [ -z "$urls" ] && return 0 local url local i=0 for url in $urls; do - file="$KERNELTREE/mpss_src-${MPSS_VERSION}-${MPSS_DISTRO}-part$i.tar" + file="$KERNELTREE/mpss_src-${MPSS_VER}-${MPSS_BLD}-${MPSS_DISTRO}-part$i.tar" download_file "$url" "$file" "$force" i=$((i+1)) done @@ -727,7 +710,7 @@ load_target() { # go away and the target just specify the $RPMSMPTYPE [ -z "$RPMSMPTYPE" ] && set_rpm_smp_type - # CC might have been overwriten in TARGET_FILE + # CC might have been overwritten in TARGET_FILE if [[ $CC != ccache\ * ]] && which "$CCACHE" &>/dev/null; then export CCACHE && export CC="ccache $CC" fi @@ -850,51 +833,51 @@ unpack_mpss() { [ -d mpss ] && return 0 - if [ "${MPSS_VERSION%%.*}" = "3" ]; then - local ver=${MPSS_VERSION%%-*} + if [ "$MPSS_MAJ" = "3" ]; then local file local i - # Check for MPSS 3.x or MPSS 3.x.x - if [ ${ver##*.} -eq 0 ]; then - ver=${ver%.*} - fi for i in $(seq 0 9); do - file="$KERNELTREE/mpss_src-${MPSS_VERSION}-${MPSS_DISTRO}-part$i.tar" + file="$KERNELTREE/mpss_src-${MPSS_VER}-${MPSS_BLD}-${MPSS_DISTRO}-part$i.tar" if [ -r "$file" ]; then untar "$file" \ - "mpss-$ver/*/linux-*.tar.bz2" \ - "mpss-$ver/*/kernel-dev-*.rpm" \ - "mpss-$ver/*/ofed-driver-*-devel-*.rpm" + "mpss-$MPSS_VER/*/linux-*.tar.bz2" \ + "mpss-$MPSS_VER/*/kernel-dev-*.rpm" \ + "mpss-$MPSS_VER/*/ofed-driver-devel-*.rpm" \ + "mpss-$MPSS_VER/*/ofed-driver-*-devel-*.rpm" fi done # Extract kernel configs - file=$(find mpss-$ver -type f -path "*/kernel-dev-*.rpm") + file=$(find mpss-$MPSS_VER -type f -path "*/kernel-dev-*.rpm" -print -quit) if ! rpm2cpio "$file" | cpio -idm; then echo "Error extracting MPSS kernel configs" return 1 fi # Unpack kernel sources - file=$(find mpss-$ver -type f -path "*/linux-*.tar.bz2") + file=$(find mpss-$MPSS_VER -type f -path "*/linux-*.tar.bz2" -print -quit) if ! untar "$file"; then echo "Error unpacking MPSS kernel sources" return 1 fi # Extract OFED headers - file=$(find mpss-$ver -type f -path "*/ofed-driver-*-devel-*.rpm") + file=$(find mpss-$MPSS_VER -type f -path "*/ofed-driver-*-devel-*.rpm" -print -quit) if ! rpm2cpio "$file" | cpio -idm; then - echo "Error extracting MPSS OFED headers" - return 1 + # Name was changed in MPSS 3.3 + file=$(find mpss-$MPSS_VER -type f -path "*/ofed-driver-devel-*.rpm" -print -quit) + if ! rpm2cpio "$file" | cpio -idm; then + echo "Error extracting MPSS OFED headers" + return 1 + fi fi # Remove unpacked archives to save space - rm -rf mpss-$ver + rm -rf mpss-$MPSS_VER # Make link to MPSS kernel sources ln -sf linux-* mpss else - local dir="mpss-${MPSS_VERSION}-${MPSS_DISTRO%%-*}" + local dir="mpss-${MPSS_VER}-${MPSS_BLD}-${MPSS_DISTRO%%-*}" mkdir $dir || return 255 pushd $dir >/dev/null || return 255 - if ! untar "$KERNELTREE/mpss_src-${MPSS_VERSION}-${MPSS_DISTRO}-part0.tar" "*/gpl/*full_src*"; then + if ! untar "$KERNELTREE/mpss_src-${MPSS_VER}-${MPSS_BLD}-${MPSS_DISTRO}-part0.tar" "*/gpl/*full_src*"; then popd >/dev/null rm -rf $dir echo "Error unpacking MPSS tarball 1" @@ -969,12 +952,6 @@ build_lustre() { targets="--target $arch $targets" done - local confoptions="" - - if $PATCHLESS; then - confoptions="$confoptions --disable-server" - fi - local rpmbuildopt='-tb' if $NORPM; then rpmbuildopt='-tc' @@ -985,7 +962,7 @@ build_lustre() { # These are required prior to the building of lustre server. Client does # not require spl/zfs. Use !PATCHLESS to indicate server which follows the # line above so is at least consistant. - if [ $PATCHLESS == false ] && [ "x$ZFSNOTSUPPORTED" == "x" ]; then + if [ $PATCHLESS == false ] && [ "x$WITH_ZFS" == "x" ]; then if ! build_spl_zfs; then popd >/dev/null # pushd lustre return 255 @@ -1019,7 +996,7 @@ build_lustre() { fi RPMBUILD_DEFS="$RPMBUILD_DEFS ${FIND_REQUIRES:+--define \"__find_requires $FIND_REQUIRES\"}" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"configure_args $confoptions ${CONFIGURE_FLAGS}\"" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"configure_args ${CONFIGURE_FLAGS}\"" RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"kdir $linux\"" RPMBUILD_DEFS="$RPMBUILD_DEFS ${linuxobj:+--define \"kobjdir $linuxobj\"}" RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"_tmppath $TMPDIR\"" @@ -1070,7 +1047,7 @@ build_spl_zfs() { # The spl/zfs spec files expect RPM_BUILD_ROOT to point to the root of the # destination for the rpms export RPM_BUILD_ROOT=$TOPDIR - SPLZFSVER=${SPLZFSVER:-0.6.3} + SPLZFSVER=${SPLZFSVER:-0.6.3-1.2} SPLZFSTAG=${SPLZFSTAG:-} # The files expect a kver to be set to the kernel version . @@ -1189,7 +1166,7 @@ build_spl_zfs() { prepare_mpss() { pushd mpss >/dev/null || return 255 - if [ "${MPSS_VERSION%%.*}" = "3" ]; then + if [ "$MPSS_MAJ" = "3" ]; then cp -f ../boot/config-* .config if ! make ARCH=k1om silentoldconfig ; then popd >/dev/null @@ -1324,13 +1301,13 @@ build_kernel_ib() { K_SRC="KSRC" fi - local OFED_CORE="--with-core-mod --with-ipoib-mod --with-sdp-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod" - local OFED_HARDWARE="--with-mthca-mod --with-mlx4-mod --with-mlx4_en-mod --with-cxgb3-mod --with-nes-mod" + local OFED_CORE="--with-core-mod --with-ipoib-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod" + local OFED_HARDWARE="--with-mthca-mod --with-mlx4-mod --with-mlx4_en-mod --with-cxgb3-mod --with-nes-mod --with-qib-mod" if compare_version $OFED_VERSION 3.0; then OFED_CORE="$OFED_CORE --with-madeye-mod --with-rds-mod" else - OFED_HARDWARE="$OFED_HARDWARE --with-cxgb4-mod" + OFED_HARDWARE="$OFED_HARDWARE --with-mlx5-mod --with-cxgb4-mod --with-ocrdma-mod --with-qib-mod" fi # some I/B drivers are architecture dependent and kernel-ib's configure @@ -1983,7 +1960,7 @@ set -E [ -r ~/.lbuildrc ] && . ~/.lbuildrc -options=$(getopt -o D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ccache,norpm,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,noiokit,ofed-version:,mpss-version:,publish,release,set-value:,src,stage:,target:,target-archs:,with-linux:,xen -- "$@") +options=$(getopt -o D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ccache,norpm,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,noiokit,ofed-version:,mpss-version:,publish,disable-zfs,release,set-value:,src,stage:,target:,target-archs:,with-linux:,xen -- "$@") if [ $? != 0 ]; then usage 1 @@ -2088,6 +2065,10 @@ while [ "$1" ]; do --publish) shift ;; + --disable-zfs) + WITH_ZFS="no" + shift + ;; --release) RELEASE=true shift @@ -2159,14 +2140,19 @@ if [ -n "$MPSS_VERSION" ]; then fatal 1 "Could not determine the last MPSS version" fi + MPSS_MAJ=${MPSS_VERSION%%.*} + MPSS_VER=${MPSS_VERSION%%-*} + MPSS_BLD=${MPSS_VERSION##*-} + [ "$MPSS_VER" = "$MPSS_BLD" ] && MPSS_BLD=0 + download_mpss "$MPSS_INFO" [ -r "$MPSS_INFO" ] && rm -f "$MPSS_INFO" - echo "Building with MPSS $MPSS_VERSION" + echo "Building with MPSS $MPSS_VER-$MPSS_BLD" unpack_mpss || fatal 1 "Error unpacking MPSS tarballs" prepare_mpss || fatal 1 "Error preparing MPSS for kernel modules build" if [ -z "$MPSS_OFED" ]; then - MPSS_OFED=$(find $PWD -type f -path "*/ofed-driver-*/Module.symvers") + MPSS_OFED=$(find $PWD -type f -path "*/ofed-driver*/Module.symvers" -print -quit) if [ -n "$MPSS_OFED" ]; then MPSS_OFED="--with-o2ib=$(dirname $MPSS_OFED)" else @@ -2178,18 +2164,18 @@ if [ -n "$MPSS_VERSION" ]; then PATCHLESS=true IOKITRPM=false LDISKFSRPM=false - ZFSNOTSUPPORTED="yes" + WITH_ZFS="no" # define variables for cross compilation: CROSS_SUFFIX="-mic" - if [ "${MPSS_VERSION%%.*}" = "3" ]; then + if [ "$MPSS_MAJ" = "3" ]; then CC_TARGET_ARCH=k1om-mpss-linux LINUX="$TOPDIR/mpss" RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"cross_requires /bin/sh\"" else CC_TARGET_ARCH=x86_64-k1om-linux LINUX="$TOPDIR/mpss/card/kernel" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"cross_requires intel-mic-gpl = ${MPSS_VERSION%%-*}\"" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"cross_requires intel-mic-gpl = $MPSS_VER\"" fi LUSTRE_VERSION=$(sed -n -e 's/^VERSION = //p' lustre/META) CONFIGURE_FLAGS="$CONFIGURE_FLAGS $MPSS_OFED --host=$CC_TARGET_ARCH --build=x86_64-pc-linux"