X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=contrib%2Flbuild%2Flbuild;h=09a634ca9226e22cacdd2f1dbad96cc28b92a4c4;hp=f3895054503902d040390deca542594c320583c0;hb=549c57e77b7f3a4cae5a7381d612a499c2ca3dcc;hpb=c2dc6da8e75f13b6ada5f6d941712f68cb6afcd4 diff --git a/contrib/lbuild/lbuild b/contrib/lbuild/lbuild index f389505..09a634c 100755 --- a/contrib/lbuild/lbuild +++ b/contrib/lbuild/lbuild @@ -38,7 +38,6 @@ BUILD_GEN=6 # TT-1092: don't cache the BUILD dir, to rebuild external OFED TOPDIR=$PWD -# CVSROOT is inherited from the environment KERNELDIR= LINUX= LUSTRE= @@ -48,7 +47,6 @@ RELEASE=false # simply don't account for this option DO_SRC=true DOWNLOAD=true -TAG= CANONICAL_TARGET= TARGET= TARGET_ARCH="$(uname -m)" @@ -62,9 +60,6 @@ EXTRA_VERSION= STAGEDIR= TMPDIR=${TMPDIR:-"/var/tmp"} TIMESTAMP= -# this is a dir to try reuse old kernel RPMs in (although, it seems to be -# unused in any real manner -REUSERPM= # this is the dir that should be used to store reuse products REUSEBUILD= # should cached products be used or force rebuilding? @@ -72,12 +67,11 @@ USE_BUILD_CACHE=true # what does this do exactly? does it imply no kernel build? NORPM=false IOKITRPM=true -LDISKFSRPM=true OSDLDISKFSRPM=true OSDZFSRPM=false -SKIPLDISKFSRPM="v1_4_* b1_4" SMPTYPES="smp bigsmp default ''" PATCHLESS=false +WITH_ZFS="" XEN=false LINUXOBJ= DISTRO= @@ -110,7 +104,6 @@ LUSTRE_TESTS=true DATE=$(date) -USE_DATESTAMP=1 RPMBUILD= export CC=${CC:-gcc} @@ -131,11 +124,6 @@ usage() { cat <] - -d CVSROOT - Specifies the CVS Root to use when pulling files from CVS. The - environment variable \$CVSROOT is used if this option is not - present. - --external-patches=EXTERNAL_PATCHES Directory similar to lustre/lustre/kernel_patches/ that lbuild should look for seres and config files in before looking in the lustre @@ -147,9 +135,6 @@ Usage: ${0##*/} [OPTION]... [-- ] --timestamp=TIMESTAMP Date of building lustre in format YYYYMMDDhhmmss - --reuserpm=DIR - Try to reuse old kernel RPMs from DIR - --reusebuild=DIR Try to reuse old kernel builds from DIR @@ -181,18 +166,17 @@ Usage: ${0##*/} [OPTION]... [-- ] Lustre modules and userspace are built. --lustre=LUSTRE - Path to an existing lustre source tarball to use instead of - pulling from CVS. + Path to an existing lustre source tarball to use. --nodownload - Do not try to download a kernel from downloads.lustre.org + Do not try to download a kernel from downloads.hpdd.intel.com --nosrc Do not build a .src.rpm, a full kernel patch, or a patched kernel tarball. --ldiskfs - Do ldiskfs RPM. Now true by default + Build with ldiskfs support. (Deprecated, always true) --noiokit Do not build lustre-iokit RPM. Now true by default @@ -204,6 +188,9 @@ Usage: ${0##*/} [OPTION]... [-- ] Specifies that the files generated do not include timestamps, and that this is an official release. + --disable-zfs + Build Lustre without ZFS. + --src Build a .src.rpm, a full kernel patch, and a patched kernel tarball. @@ -212,9 +199,6 @@ Usage: ${0##*/} [OPTION]... [-- ] more or less in DIR/-, and the tarball will be placed in DIR. - --tag=TAG - A CVS branch/tag name to build from when pulling from CVS. - --target=TARGET The name of the target to build. The available targets are listed below. @@ -229,16 +213,11 @@ Usage: ${0##*/} [OPTION]... [-- ] Also note that by using a non-"base" arch (eg, i386) only kernels will be built - there will be no lustre-lite-utils package. - --disable-datestamp - Prevents the datestamp flag (-D) from being passed to cvs for - checkouts. This is a workaround for a problem encountered when - using lbuild with tinderbox. - --xen Builds a Xen domX kernel. --set-value - Set's a variable to a given value. + Sets a variable to a given value. EOF @@ -280,17 +259,18 @@ canon_path() { check_options() { - if [ "$LUSTRE" ]; then - [ -r "$LUSTRE" ] || \ - usage 1 "Could not find Lustre source tarball '$LUSTRE'." - else - [ "$CVSROOT" ] || \ - usage 1 "Either specify a CVS Root with -d, or a Lustre source tarball with --lustre." - [ "$TAG" ] || \ - usage 1 "A branch/tag name must be specified with --tag when not building from a tarball." + if [ -z "$LUSTRE" -o ! -r "$LUSTRE" ]; then + usage 1 "Could not find Lustre source tarball '$LUSTRE'." fi - [ -z "$DISTRO" ] && DISTRO=$(autodetect_distro) + if [ -z "$DISTRO" ] ; then + DISTRO=$(autodetect_distro) + # remove separator + DISTRO=${DISTRO/-/} + fi + DISTROMAJ=${DISTRO%%.*} + + [ -z "$TARGET" ] && TARGET=$(autodetect_target "$DISTRO") if [ -z "$LINUX" ]; then [ "$KERNELDIR" -o "$KERNELTREE" ] || \ @@ -299,19 +279,19 @@ check_options() { [ -d "$KERNELDIR" -o -d "$KERNELTREE" ] || \ usage 1 "$KERNELDIR and $KERNELTREE are not a directory." - if ! $RELEASE; then - [ "$TAG" ] || \ - usage 1 "When building a snapshot, a tag name must be used." - fi - - [ "$TARGET" ] || TARGET=$(autodetect_target "$DISTRO") # TARGET_FILE="$TOPDIR/lustre/kernel_patches/targets/$TARGET.target" # [ -r "$TARGET_FILE" ] || \ # usage 1 "Target '$TARGET' was not found." fi case $TARGET in - 2.6-rhel6) + 3.12-sles12) + CANONICAL_TARGET="sles12" + ;; + 3.10-rhel7) + CANONICAL_TARGET="rhel7" + ;; + 2.6-rhel6*) CANONICAL_TARGET="rhel6" ;; 2.6-rhel5) @@ -425,13 +405,21 @@ fetch_url() { local url="$1" local target="$2" + if [ -z "$target" ]; then + fatal 1 "fetch_url() called without a target to fetch to" + fi + + if [ -d $target ]; then + target+="/${url##*/}" + fi + local rc=0 if which wget >/dev/null 2>&1; then if ! wget -nv "$url" -O "$target"; then rc=${PIPESTATUS[0]} fi elif which curl >/dev/null 2>&1; then - if ! curl -L -s -o "$target" "$url"; then + if ! curl -n -L -s -o "$target" "$url"; then rc=${PIPESTATUS[0]} fi else @@ -450,7 +438,7 @@ download_srpm() { if $force || [ ! -r "$KERNELDIR/$srpm" ] || [ ! -s "$KERNELDIR/$srpm" ]; then if $DOWNLOAD; then - local location="http://downloads.lustre.org/public/kernels/$target/old" + local location="https://downloads.hpdd.intel.com/public/kernels/$target/old" # get the location from a distro specific method if it exists if type -p kernel_srpm_location; then location=$(kernel_srpm_location) @@ -460,11 +448,11 @@ download_srpm() { [ ! -s "$KERNELDIR/$srpm" ]; then rm -f $KERNELDIR/$srpm # punt to a distro specific method if it exists - if ! type -p download_srpm-$DISTRO; then + if ! type -p download_srpm-$DISTROMAJ; then fatal 1 "Could not download target $target's kernel SRPM $srpm from $location." else - if ! download_srpm-$DISTRO "$target" "$srpm" "$force"; then - fatal 1 "Could not download target $target's kernel SRPM $srpm using download_srpm-$DISTRO." + if ! download_srpm-$DISTROMAJ "$target" "$srpm" "$force"; then + fatal 1 "Could not download target $target's kernel SRPM $srpm using download_srpm-$DISTROMAJ." fi fi fi @@ -566,22 +554,21 @@ download_ofed() { OFED_BASE_VERSION=${OFED_VERSION%.*} fi - local location="http://www.openfabrics.org/downloads/OFED/ofed-${OFED_BASE_VERSION}/" + local location="https://www.openfabrics.org/downloads/OFED/ofed-${OFED_BASE_VERSION}/" - if [[ $OFED_VERSION = *-rc[0-9] ]]; then + if [[ $OFED_VERSION = *-[rR][cC][0-9] ]]; then local Mmv - Mmv=${OFED_VERSION%%-*} - location="http://www.openfabrics.org/downloads/OFED/ofed-${Mmv}/" + Mmv=${OFED_VERSION%%-[rR][cC][0-9]} + location="https://www.openfabrics.org/downloads/OFED/ofed-${Mmv}/" fi if [[ $OFED_VERSION = daily-* ]]; then - local Mmv daily - OFED_VERSION=${OFED_VERSION/daily-/} - Mmv=${OFED_VERSION%%-*} + local Mmv + Mmv=${OFED_VERSION/daily-/} daily=${OFED_VERSION##$Mmv-} - location="http://www.openfabrics.org/downloads/OFED/ofed-${Mmv}-daily/" + location="https://www.openfabrics.org/downloads/OFED/ofed-${Mmv}-daily/" # find the filename for the version for the date specified - OFED_VERSION=$(curl -s "$location" | sed -nre "/${daily}-/s/.*href=\"OFED-(${Mmv//./\\.}-${daily}-[0-9]{4,4}).tgz.*$/\1/p" | tail -1) + OFED_VERSION=$(curl -s "$location" | sed -nre "/${Mmv}-/s/.*href=\"OFED-(${Mmv}-[0-9]{8,8}-[0-9]{4,4}).tgz.*$/\1/p" | tail -1) if [ -z "$OFED_VERSION" ]; then fatal 1 "Could not determine the filename of the OFED snapshot for ${daily}" fi @@ -592,6 +579,109 @@ download_ofed() { } +parse_mpss_info() { + local file="$1" + local build="$2" + local distro="$3" + + [ -r "$file" ] || fatal 1 "Could not find MPSS info file" + + # convert to MPSS distro naming + distro=${distro/sles/suse} + distro_name=${distro%%-*} + + # extract links from MPSS info file + local urls=$(cat "$file" | \ + tr -d '\t\r\n' | sed -e 's?\(\)?\1\n?g' | \ + grep '.intel.com/' | \ + sed -e 's/^.*[hH][rR][eE][fF]="\([^>"]\+\)".*$/\1/g' | \ + grep '.tar') + local res2="" + local res3="" + for url in $urls; do + local ver2="" + local ver3="" + # Try to match with MPSS 3.3 package + if [[ $url =~ mpss-[0-9].[0-9](.[0-9]*)?-linux.tar ]]; then + ver3=${url##*mpss-} + ver3=${ver3%%-linux.tar} + # Try to match with MPSS 3.x or 3.x.x distro package + elif [[ $url =~ mpss-[0-9].[0-9](.[0-9]*)?-$distro.tar ]]; then + ver3=${url##*mpss-} + ver3=${ver3%%-$distro.tar} + # Try to match with MPSS 3.x or 3.x.x source package + elif [[ $url =~ mpss-src-[0-9].[0-9](.[0-9]*)?.tar ]]; then + ver3=${url##*mpss-src-} + ver3=${ver3%%.tar} + # Try to match with MPSS 3.x or 3.x.x cross compiler package + elif [[ $url =~ mpss-[0-9].[0-9](.[0-9]*)?-k1om.tar ]]; then + ver3=${url##*mpss-} + ver3=${ver3%%-k1om.tar} + # Try to match with MPSS 2.1.x source package + elif [[ $url =~ _src-[0-9].[0-9].[0-9]*-[0-9]*_$distro_name.tar ]]; then + ver2=${url##*_src-} + ver2=${ver2%%_$distro_name.tar} + fi + + if [ -n "$ver3" ]; then + ver3="${ver3}-0" + if [ -z "$build" ]; then + res3="$ver3" + elif [ "${build%%-*}" = "${ver3%%-*}" ]; then + res3="$res3 $url" + fi + elif [ -n "$ver2" ]; then + if [ -z "$build" ]; then + res2="$ver2" + elif [ "$build" = "$ver2" ]; then + res2="$res2 $url" + fi + fi + + done + + if [ -z "$build" ]; then + # return a last version of MPSS + if [ -n "$res3" ]; then + echo "$res3" + else + echo "$res2" + fi + elif [ "${build%%.*}" = "3" ]; then + # return URLs from MPSS info file + echo "$res3" + else + # return URLs from MPSS info file + echo "$res2" + fi +} + +# Get public information about last releases of +# Intel Manycore Platform Software Stack (MPSS) +download_mpss_info() { + local file="$1" + local force="${2:-true}" + local url=${MPSS_URL:-"http://software.intel.com/en-us/articles/intel-manycore-platform-software-stack-mpss"} + + download_file "$url" "$file" "$force" +} + +download_mpss() { + local file="$1" + local force="${2:-false}" + local urls=$(parse_mpss_info "$file" ${MPSS_VERSION} ${MPSS_DISTRO}) + + [ -z "$urls" ] && return 0 + + local url + local i=0 + for url in $urls; do + file="$KERNELTREE/mpss_src-${MPSS_VER}-${MPSS_BLD}-${MPSS_DISTRO}-part$i.tar" + download_file "$url" "$file" "$force" + i=$((i+1)) + done +} + load_target() { EXTRA_VERSION_save="$EXTRA_VERSION" @@ -623,7 +713,7 @@ load_target() { # go away and the target just specify the $RPMSMPTYPE [ -z "$RPMSMPTYPE" ] && set_rpm_smp_type - # CC might have been overwriten in TARGET_FILE + # CC might have been overwritten in TARGET_FILE if [[ $CC != ccache\ * ]] && which "$CCACHE" &>/dev/null; then export CCACHE && export CC="ccache $CC" fi @@ -650,7 +740,7 @@ load_target() { fi if [ -f $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET-$TARGET_ARCH.config ]; then - CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET$XENPOSTFIX-$TARGET_ARCH${RPMSMPTYPE:+-}${RPMSMPTYPE}.config" + CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET$XENPOSTFIX-$TARGET_ARCH.config" fi local lnxrelnew=${lnxrel//-/_} @@ -672,7 +762,6 @@ load_target() { # kernel. don't make it look like one if $PATCHLESS || [ -n "$SERIES" ]; then EXTRA_VERSION=$(echo $EXTRA_VERSION | sed -e "s/\(.*_lustre\)\..*/\1/") -# EXTRA_VERSION="${EXTRA_VERSION}-${TAG}.${TIMESTAMP}" if ! $PATCHLESS && [ -n "$BUILDID" ]; then EXTRA_VERSION="${EXTRA_VERSION}.${BUILDID}" fi @@ -696,18 +785,19 @@ load_target() { tarflags() { local file="$1" + echo -n '--wildcards ' case "$file" in '') fatal 1 "tarflags(): File name argument missing." ;; *.tar.gz | *.tgz) - echo 'zxf' + echo '-zxf' ;; *.tar.bz2) - echo 'jxf' + echo '-jxf' ;; *.tar) - echo 'xf' + echo '-xf' ;; *) fatal 1 "tarflags(): Unrecognized tar extension in file: $1" @@ -737,32 +827,76 @@ unpack_ofed() { unpack_lustre() { - if [ -z "$LUSTRE" ]; then - local DATESTAMP="" - - if [ -n "$USE_DATESTAMP" ]; then - DATESTAMP="-D '$DATE'" - fi + untar "$LUSTRE" || fatal 1 "Error unpacking Lustre tarball" + [ -d lustre ] || ln -sf lustre-[0-9].[0-9]* lustre - local DIRNAME="lustre-$TAG-$TIMESTAMP" +} - cvs -d "$CVSROOT" -qz3 co $DATESTAMP -d "$DIRNAME" lustre || \ - fatal 1 "There was an error checking out toplevel Lustre from CVS." - pushd "$DIRNAME" > /dev/null - ./lustrecvs "$TAG" || \ - fatal 1 "There was an error checking out Lustre/Portals/Build from CVS." - echo "Creating lustre tarball..." - sh autogen.sh || fatal 1 "There was an error running autogen.sh." - ./configure --enable-dist || \ - fatal 1 "There was an error running ./configure to create makefiles." - make dist || fatal 1 "There was an error running 'make dist'." - LUSTRE=$PWD/lustre-*.tar.gz - popd > /dev/null +unpack_mpss() { + + [ -d mpss ] && return 0 + + if [ "$MPSS_MAJ" = "3" ]; then + local file + local i + for i in $(seq 0 9); do + file="$KERNELTREE/mpss_src-${MPSS_VER}-${MPSS_BLD}-${MPSS_DISTRO}-part$i.tar" + if [ -r "$file" ]; then + untar "$file" \ + "mpss-$MPSS_VER/*/linux-*.tar.bz2" \ + "mpss-$MPSS_VER/*/kernel-dev-*.rpm" \ + "mpss-$MPSS_VER/*/ofed-driver-devel-*.rpm" \ + "mpss-$MPSS_VER/*/ofed-driver-*-devel-*.rpm" + fi + done + # Extract kernel configs + file=$(find mpss-$MPSS_VER -type f -path "*/kernel-dev-*.rpm" -print -quit) + if ! rpm2cpio "$file" | cpio -idm; then + echo "Error extracting MPSS kernel configs" + return 1 + fi + # Unpack kernel sources + file=$(find mpss-$MPSS_VER -type f -path "*/linux-*.tar.bz2" -print -quit) + if ! untar "$file"; then + echo "Error unpacking MPSS kernel sources" + return 1 + fi + # Extract OFED headers + file=$(find mpss-$MPSS_VER -type f -path "*/ofed-driver-*-devel-*.rpm" -print -quit) + if ! rpm2cpio "$file" | cpio -idm; then + # Name was changed in MPSS 3.3 + file=$(find mpss-$MPSS_VER -type f -path "*/ofed-driver-devel-*.rpm" -print -quit) + if ! rpm2cpio "$file" | cpio -idm; then + echo "Error extracting MPSS OFED headers" + return 1 + fi + fi + # Remove unpacked archives to save space + rm -rf mpss-$MPSS_VER + # Make link to MPSS kernel sources + ln -sf linux-* mpss + else + local dir="mpss-${MPSS_VER}-${MPSS_BLD}-${MPSS_DISTRO%%-*}" + + mkdir $dir || return 255 + pushd $dir >/dev/null || return 255 + if ! untar "$KERNELTREE/mpss_src-${MPSS_VER}-${MPSS_BLD}-${MPSS_DISTRO}-part0.tar" "*/gpl/*full_src*"; then + popd >/dev/null + rm -rf $dir + echo "Error unpacking MPSS tarball 1" + return 1 + fi + local file=$(find . -type f -path "*/gpl/*full_src*") + if ! untar "$file"; then + popd >/dev/null + rm -rf $dir + echo "Error unpacking MPSS tarball 2" + return 1 + fi + popd >/dev/null + ln -sf $dir mpss fi - untar "$LUSTRE" || fatal 1 "Error unpacking Lustre tarball" - [ -d lustre ] || ln -sf lustre-[0-9].[0-9]* lustre - } do_patch_linux() { @@ -802,39 +936,6 @@ do_patch_linux() { } -build_iokit() { - local rpmbuildopt="$1" - - pushd lustre-iokit > /dev/null || return 255 - - if ! ./configure; then - echo "failed to configure in lustre-iokit" - popd >/dev/null # pushd lustre-iokit - return 255 - fi - - if ! make dist; then - echo "failed to make dist in lustre-iokit" - popd >/dev/null # pushd lustre-iokit - return 255 - fi - - if ! $RPMBUILD $rpmbuildopt lustre-iokit*.tar.gz \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - popd >/dev/null # pushd lustre-iokit - return 255 - fi - - if $DO_SRC && ! $RPMBUILD -ts lustre-iokit*.tar.gz \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - popd >/dev/null # pushd lustre-iokit - return 255 - fi - popd >/dev/null # pushd lustre-iokit -} - build_lustre() { local linux="$1" local linuxobj="$2" @@ -843,18 +944,17 @@ build_lustre() { pushd lustre >/dev/null + if ! build_lustre_dkms; then + popd >/dev/null # pushd lustre + return 255 + fi + echo "Building Lustre RPMs for: $BUILD_ARCHS..." local targets arch for arch in $BUILD_ARCHS; do targets="--target $arch $targets" done - local confoptions="" - - if $PATCHLESS; then - confoptions="$confoptions --disable-server" - fi - local rpmbuildopt='-tb' if $NORPM; then rpmbuildopt='-tc' @@ -865,86 +965,27 @@ build_lustre() { # These are required prior to the building of lustre server. Client does # not require spl/zfs. Use !PATCHLESS to indicate server which follows the # line above so is at least consistant. - if [ $PATCHLESS == false ] && [ "x$ZFSNOTSUPPORTED" == "x" ]; then + if [ $PATCHLESS == false ] && [ "x$WITH_ZFS" == "x" ]; then if ! build_spl_zfs; then popd >/dev/null # pushd lustre return 255 fi fi - ( $(skeep_ldiskfs_rpm $TAG) ) || { - - pushd ldiskfs > /dev/null || return 255 - - if ! ./configure --enable-dist; then - echo "failed to configure in ldiskfs" - popd >/dev/null # pushd ldiskfs - popd >/dev/null # pushd lustre - return 255 - fi - - if ! make dist 2>&1; then - popd >/dev/null # pushd ldiskfs - popd >/dev/null # pushd lustre - return 255 - fi - - #cp lustre-ldiskfs*.tar.gz $TOPDIR/SOURCES || \ - # fatal 1 "Could not copy lustre-ldiskfs*.tar.gz to $TOPDIR/SOURCES" - - if ! $RPMBUILD $targets $rpmbuildopt lustre-ldiskfs*.tar.gz \ - --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ - --define "kdir $linux" \ - ${linuxobj:+--define "kobjdir $linuxobj"} \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - popd >/dev/null # pushd ldiskfs - popd >/dev/null # pushd lustre - return 255 - fi - - if $DO_SRC; then - if ! $RPMBUILD -ts lustre-ldiskfs*.tar.gz \ - --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ - --define "kdir $linux" \ - ${linuxobj:+--define "kobjdir $linuxobj"} \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - popd >/dev/null # pushd ldiskfs - popd >/dev/null # pushd lustre - return 255 - fi - fi - popd >/dev/null # pushd ldiskfs - - # tell lustre where ldiskfs is - # XXX - pointing to the RPM BUILD dir is a hack. we need to flesh - # out the ldiskfs RPM build so that it builds a - # lustre-ldiskfs-devel RPM and install that and point lustre - # to that instead - confoptions="$confoptions --with-ldiskfs=$(ls -d $TOPDIR/BUILD/lustre-ldiskfs-*)" - } - - if $IOKITRPM; then - if ! build_iokit $rpmbuildopt; then - echo "failed to build lustre-iokit" - popd >/dev/null # pushd lustre - return 255 - fi - fi - - # convert the $PATCHLESS boolean to an empty/not-empty boolean - # as silly as this seems, it makes the syntax of the rpmbuild command - # simpler and not need an eval to deal with the quotes in the quotes - local is_patchless="" if $PATCHLESS; then - is_patchless="yes" + RPMBUILD_DEFS="$RPMBUILD_DEFS --without servers" + if [ -n "$CROSS_SUFFIX" ]; then + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"lustre_name lustre-client$CROSS_SUFFIX\"" + fi fi # ditto for the lustre-tests boolean - local lustre_tests="" if ! $LUSTRE_TESTS; then - lustre_tests="no" + RPMBUILD_DEFS="$RPMBUILD_DEFS --without lustre_tests" + fi + + if ! $IOKITRPM; then + RPMBUILD_DEFS="$RPMBUILD_DEFS --without lustre_iokit" fi local osd_zfs="" @@ -957,30 +998,21 @@ build_lustre() { osd_ldiskfs="yes" fi - $RPMBUILD $targets $rpmbuildopt "$LUSTRE" \ - ${is_patchless:+--define "lustre_name lustre-client"} \ - ${lustre_tests:+--define "build_lustre_tests 0"} \ - ${osd_zfs:+--define "build_lustre_osd_zfs 1"} \ - ${osd_ldiskfs:+--define "build_lustre_osd_ldiskfs 1"} \ - ${FIND_REQUIRES:+--define "__find_requires $FIND_REQUIRES"} \ - --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ - --define "kdir $linux" \ - ${linuxobj:+--define "kobjdir $linuxobj"} \ - --define "_tmppath $TMPDIR" \ - --define "_topdir $TOPDIR" 2>&1 || \ + RPMBUILD_DEFS="$RPMBUILD_DEFS ${FIND_REQUIRES:+--define \"__find_requires $FIND_REQUIRES\"}" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"configure_args ${CONFIGURE_FLAGS}\"" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"kdir $linux\"" + RPMBUILD_DEFS="$RPMBUILD_DEFS ${linuxobj:+--define \"kobjdir $linuxobj\"}" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"_tmppath $TMPDIR\"" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"_topdir $TOPDIR\"" + + eval $RPMBUILD $targets $rpmbuildopt "$LUSTRE" $RPMBUILD_DEFS \ + ${osd_zfs:+--with zfs} \ + ${osd_ldiskfs:+--with ldiskfs} 2>&1 || \ fatal 1 "Error building rpms for $BUILD_ARCHS." if $DO_SRC; then - if ! $RPMBUILD -ts "$LUSTRE" \ - ${is_patchless:+--define "lustre_name lustre-client"} \ - ${lustre_tests:+--define "build_lustre_tests 0"} \ - ${FIND_REQUIRES:+--define "__find_requires $FIND_REQUIRES"} \ - --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ - --define "kdir $linux" \ - ${linuxobj:+--define "kobjdir $linuxobj"} \ - --define "_tmppath $TMPDIR" \ - --define "_topdir $TOPDIR" 2>&1; then - popd + if ! eval $RPMBUILD -ts "$LUSTRE" $RPMBUILD_DEFS 2>&1; then + popd >/dev/null return 255 fi fi @@ -988,41 +1020,82 @@ build_lustre() { } +# Only zfs Lustre DKMS Server is supported +build_lustre_dkms() { + [ -n "$MPSS_VERSION" ] && return 0 + local ver=$(eval echo $(awk '/LUSTRE_VERSION_STRING/ {print $3}' lustre/include/lustre_ver.h)) + echo "Building Lustre DKMS RPMs for: $BUILD_ARCHS..." + ./configure --enable-dist || return 255 + + if $PATCHLESS; then + $RPMBUILD --define="_topdir $TOPDIR" --without servers -bs lustre-dkms.spec || return 255 + $RPMBUILD --rebuild --define="_topdir $TOPDIR" --without servers $TOPDIR/SRPMS/lustre-client-dkms-$ver-*.src.rpm || return 255 + else + $RPMBUILD --define="_topdir $TOPDIR" -bs lustre-dkms.spec || return 255 + $RPMBUILD --rebuild --define="_topdir $TOPDIR" $TOPDIR/SRPMS/lustre-dkms-$ver-*.src.rpm || return 255 + fi + return 0 +} + ### # build_spl_zfs # # Fetch spl/zfs from the git repo and prepare for lustre build # +# Overrides: +# SPLZFSGITREPO - URI of directory where spl.git and zfs.git are located +# SPLZFSTAG - Tag to checkout of clone repositories +# SPLZFSVER - Version to checkout of both (format zfs/spl-$SPLZFSVER) +# # return 0 if successful, else 255 build_spl_zfs() { # make sure the RPM build environment is set up + pushd $TOPDIR create_rpmbuild_dirs + popd # The spl/zfs spec files expect RPM_BUILD_ROOT to point to the root of the # destination for the rpms export RPM_BUILD_ROOT=$TOPDIR + SPLZFSVER=${SPLZFSVER:-0.6.5.2} + SPLZFSTAG=${SPLZFSTAG:-} # The files expect a kver to be set to the kernel version . - kver=$(cat ${LINUX}/include/config/kernel.release) + local kver=$(find_linux_release) # build and install the spl and zfs (and -devel) RPMs for lustre to use local pkg for pkg in spl zfs; do - local rpmmod spldir + local rpmpkg - [ "$pkg" == "zfs" ] && spldir="$(ls -d $TOPDIR/usr/src/spl-*/*)" - rpmmod=$pkg-modules + [ "$pkg" == "zfs" ] && spldir="$(ls -d $TOPDIR/usr/src/spl-*/|tail -1)" # need to fetch the repo in order to build it. # default to github but allow override git clone ${SPLZFSGITREPO:-"https://github.com/zfsonlinux"}/$pkg.git $pkg 2>&1 pushd $pkg || return 255 + if [ -n "$SPLZFSTAG" ]; then + git checkout $SPLZFSTAG + else + git checkout -b lbuild $pkg-$SPLZFSVER + fi - # XXX - _having_ to give a package a kernel just to have it configure - # enough to do a "make dist" is truely bogus! - if ! ./configure --with-linux=${LINUX} \ + # This differentiates between older zfs versions + if [ -f $pkg-modules.spec.in ]; then + rpmpkg=$pkg-modules + specdir=. + speclist="$pkg.spec $rpmpkg.spec" + else + rpmpkg=kmod-$pkg-devel + specdir=rpm/generic + speclist="$pkg.spec $pkg-kmod.spec $pkg-dkms.spec" + fi + + sh autogen.sh || return 255 + + if ! ./configure --with-linux=${LINUX} --with-linux-obj=${LINUXOBJ:-$LINUX} \ ${spldir:+--with-spl="${spldir}"} 2>&1 || ! make dist 2>&1; then popd @@ -1030,71 +1103,69 @@ build_spl_zfs() { fi popd - # XXX - dirty hack time. ~sigh~ - # because the spl tarball contains two .spec files, we cannot - # simply rebuild the tarball - ln $pkg/$pkg-*.tar.gz $TOPDIR/SOURCES || - error "failed to link $pkg/$pkg-*.tar.gz into $TOPDIR/SOURCES" - - # First build the userspace tools RPMs - if ! $RPMBUILD $targets -bb $pkg/$pkg.spec \ - --nodeps -v \ - --define "configure_args ${CONFIGURE_FLAGS}" \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - return 255 + ln -f $pkg/$pkg-*.tar.gz $TOPDIR/SOURCES || + error "failed to link $pkg/$pkg-*.tar.gz into $TOPDIR/SOURCES" + if [ -f $pkg/scripts/kmodtool ]; then + ln -f $pkg/scripts/kmodtool $TOPDIR/SOURCES/ fi + local rpmb if $DO_SRC; then - if ! $RPMBUILD -bs $pkg/$pkg.spec \ - --nodeps -v \ - --define "configure_args ${CONFIGURE_FLAGS}" \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - return 255 - fi - fi - - # Now the kernel module RPMs - # Need to provide pointers to the lustre kernel source, objects, usr/src - if ! $RPMBUILD $targets -bb $pkg/$rpmmod.spec \ - --nodeps -v \ - --define "require_kdir ${linux}" \ - ${linuxobj:+--define "require_kobj ${linuxobj}"} \ - ${spldir:+--define "require_spldir ${spldir}"} \ - --define "configure_args ${CONFIGURE_FLAGS}" \ - --define "_usrsrc $TOPDIR/usr/src" \ - --define "kver $kver" \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - return 255 + rpmb=-ba + else + rpmb=-bb fi - if $DO_SRC; then - if ! $RPMBUILD -bs $pkg/$rpmmod.spec \ + # set search dir for our own kmodtool to find correct + # directories + export KERNELSOURCE=$(dirname ${LINUX}) + # Manually build rpms + for spec in $speclist; do + echo "Building RPMs from $pkg/$specdir/$spec" + if ! $RPMBUILD $rpmb $pkg/$specdir/$spec \ --nodeps -v \ - --define "require_kdir ${linux}" \ - ${linuxobj:+--define "require_kobj ${linuxobj}"} \ + --define "require_kdir ${LINUX}" \ + ${LINUXOBJ:+--define "require_kobj ${LINUXOBJ}"} \ ${spldir:+--define "require_spldir ${spldir}"} \ - --define "configure_args ${CONFIGURE_FLAGS}" \ - --define "_usrsrc $TOPDIR/usr/src" \ --define "kver $kver" \ + --define "kernels $kver" \ --define "_tmppath /var/tmp" \ + --define "kernelbuildroot $TOPDIR/reused" \ --define "_topdir $TOPDIR" 2>&1; then return 255 fi - fi + done - # We have built the rpms for the package. No we need to extract the + # We have built the rpms for the package. Now we need to extract the # contained files so we can build further things against them - local rpm=$(ls $TOPDIR/RPMS/*/$rpmmod-devel-*.rpm) + local rpms=$(ls -1 $TOPDIR/RPMS/*/$rpmpkg-*.rpm) # cpio only extract to pwd so we need to go there. pushd $TOPDIR - rpm2cpio < $rpm | cpio -id + local rpm + for rpm in $rpms; do + rpm2cpio $rpm | cpio -id + done + + if [ "$pkg" == "zfs" ]; then + # We also need to extract both the zfs and zfs-devel rpms + # the zfs rpm is needed because it has the actual libraries in + # it and the zfs-devel rpm only has unversioned symlinks to the + # libraries in the zfs rpm + # this will all change one day when we have a libzfs rpm per + # https://github.com/zfsonlinux/zfs/issues/2329 + # and it looks like it could be one day soon: + # https://github.com/zfsonlinux/zfs/pull/2341 + local devel_rpms=$(ls -1 $TOPDIR/RPMS/*/{$pkg-devel,$pkg-$SPLZFSVER,lib*}-*.rpm) + for rpm in $devel_rpms; do + rpm2cpio $rpm | cpio -id + done + CONFIGURE_FLAGS="--with-$pkg-devel=$TOPDIR ${CONFIGURE_FLAGS}" + fi popd - CONFIGURE_FLAGS="--with-$pkg=$(ls -d $TOPDIR/usr/src/$pkg-*/*) ${CONFIGURE_FLAGS}" + CONFIGURE_FLAGS="--with-$pkg=$(ls -d $TOPDIR/usr/src/$pkg-*/|tail -1) ${CONFIGURE_FLAGS}" + CONFIGURE_FLAGS="--with-$pkg-obj=$(ls -d $TOPDIR/usr/src/$pkg-*/$kver*|tail -1) ${CONFIGURE_FLAGS}" done OSDZFSRPM=true @@ -1102,6 +1173,34 @@ build_spl_zfs() { return 0 } +prepare_mpss() { + + pushd mpss >/dev/null || return 255 + if [ "$MPSS_MAJ" = "3" ]; then + cp -f ../boot/config-* .config + if ! make ARCH=k1om silentoldconfig ; then + popd >/dev/null + return 1 + fi + if ! make ARCH=k1om modules_prepare ; then + popd >/dev/null + return 1 + fi + cp -f ../boot/Module.symvers-* Module.symvers + else + if ! make defconfig-miclinux; then + popd >/dev/null + return 1 + fi + if ! make -C card/kernel ARCH=k1om modules_prepare ; then + popd >/dev/null + return 1 + fi + fi + popd >/dev/null + +} + stage() { [ "$STAGEDIR" ] || return 0 @@ -1120,26 +1219,6 @@ stage() { } -#check if we need to build separate ldiskfs RPM -skeep_ldiskfs_rpm() { - local tag="$1" - - local skip=false - - if ! $LDISKFSRPM; then - skip=true - elif $PATCHLESS; then - skip=true - else - for skiptag in $SKIPLDISKFSRPM; do - [[ $tag == $skiptag ]] && skip=true && break - done - fi - - echo $skip - -} - set_rpm_smp_type() { local infact_arch="${TARGET_ARCH}" @@ -1166,18 +1245,25 @@ set_rpm_smp_type() { } -# This function takes a linux source pool and digs out the linux release -# from it -find_linux_release() { - local SRCDIR="$1" +# This function takes a linux include tree and digs out the linux release +# from it. It is never called directly, only called from the distro +# specific function find_linux_release() in lbuild-{rhel,sles}. +_find_linux_release() { + local SRC="$1" + local LINUXRELEASEHEADER="" - local LINUXRELEASEHEADER=$SRCDIR/include/linux/version.h - if [ -s $SRCDIR/include/linux/utsrelease.h ]; then - LINUXRELEASEHEADER=$SRCDIR/include/linux/utsrelease.h + LINUXRELEASEHEADER=$SRC/include/linux/version.h + if [ -s $SRC/include/generated/utsrelease.h ]; then + LINUXRELEASEHEADER=$SRC/include/generated/utsrelease.h + elif [ -s $SRC/include/linux/utsrelease.h ]; then + LINUXRELEASEHEADER=$SRC/include/linux/utsrelease.h fi - sed -ne 's/#define UTS_RELEASE "\(.*\)"$/\1/p' $LINUXRELEASEHEADER + if [ ! -s $LINUXRELEASEHEADER ]; then + fatal 1 "could not find UTS_RELEASE" + fi + sed -ne 's/#define UTS_RELEASE "\(.*\)"$/\1/p' $LINUXRELEASEHEADER } # unpack kernel(/source/devel) RPM @@ -1199,8 +1285,8 @@ unpack_linux_devel_rpm() { fi # call a distro specific hook, if available - if type -p unpack_linux_devel_rpm-$DISTRO; then - if ! unpack_linux_devel_rpm-$DISTRO "$kernelrpm"; then + if type -p unpack_linux_devel_rpm-$DISTROMAJ; then + if ! unpack_linux_devel_rpm-$DISTROMAJ "$kernelrpm"; then return 255 fi fi @@ -1225,13 +1311,13 @@ build_kernel_ib() { K_SRC="KSRC" fi - local OFED_CORE="--with-core-mod --with-ipoib-mod --with-sdp-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod" - local OFED_HARDWARE="--with-mthca-mod --with-mlx4-mod --with-mlx4_en-mod --with-cxgb3-mod --with-nes-mod" + local OFED_CORE="--with-core-mod --with-ipoib-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod" + local OFED_HARDWARE="--with-mthca-mod --with-mlx4-mod --with-mlx4_en-mod --with-cxgb3-mod --with-nes-mod --with-qib-mod" if compare_version $OFED_VERSION 3.0; then OFED_CORE="$OFED_CORE --with-madeye-mod --with-rds-mod" else - OFED_HARDWARE="$OFED_HARDWARE --with-cxgb4-mod" + OFED_HARDWARE="$OFED_HARDWARE --with-mlx5-mod --with-cxgb4-mod --with-ocrdma-mod --with-qib-mod" fi # some I/B drivers are architecture dependent and kernel-ib's configure @@ -1255,12 +1341,12 @@ build_kernel_ib() { local SOURCE="${TOPDIR}/OFED/SRPMS/${kib_prefix}-*.src.rpm" # but switch to building from the SPEC if we need to apply patches - if ls ${TOPDIR}/lustre/build/patches/ofed/* >/dev/null; then + if ls ${TOPDIR}/lustre/contrib/patches/ofed/* >/dev/null; then BUILD_TYPE="-bb" rpm --define "_topdir ${TOPDIR}" -ivh $SOURCE SOURCE="${TOPDIR}/SPECS/${kib_prefix}.spec" local file ed_fragment1 ed_fragment2 n=1 - for file in $(ls ${TOPDIR}/lustre/build/patches/ofed/*.patch); do + for file in $(ls ${TOPDIR}/lustre/contrib/patches/ofed/*.patch); do ed_fragment1="$ed_fragment1 Patch$n: ${file%%*/}" ed_fragment2="$ed_fragment2 @@ -1268,9 +1354,9 @@ Patch$n: ${file%%*/}" cp $file ${TOPDIR}/SOURCES let n=$n+1 done - for file in $(ls ${TOPDIR}/lustre/build/patches/ofed/*.ed); do + for file in $(ls ${TOPDIR}/lustre/contrib/patches/ofed/*.ed); do # Only apply the ed-scripts that should be used for the canonical target - # ed-files in ${TOPDIR}/lustre/build/patches/ofed/ have to follow the naming + # ed-files in ${TOPDIR}/lustre/contrib/patches/ofed/ have to follow the naming # convention # -:: ...:.ed # To apply the same change to multiple canonical target simply specify @@ -1298,7 +1384,7 @@ EOF fi fi - local linuxrelease=$(find_linux_release "$linux") + local linuxrelease=$(find_linux_release) if compare_version $OFED_VERSION 3.0; then local OFA_KERNEL_RELEASE=$(echo -n ${linuxrelease} | sed -e 's/-/_/g') fi @@ -1462,8 +1548,11 @@ build_kernel_with_srpm() { if $USE_BUILD_CACHE && [ -n "$REUSEBUILD" ]; then local REUSE_SIGNATURE=$({ echo -en $release_str; echo $BUILD_GEN; - cat $CONFIG_FILE $TARGET_FILE $FULL_PATCH; - cat ${0%/lbuild}/lbuild ${0%/lbuild}/lbuild-${DISTRO}; } | + cat "$CONFIG_FILE"; + cat "$TARGET_FILE"; + cat "$FULL_PATCH"; + cat "$LBUILD_DIR/lbuild"; + cat "$LBUILD_DIR/lbuild-$DISTROMAJ"; } | md5sum | cut -d" " -f1) # see if we can link to the reuse pool # XXX - hrm. i'm not convinced this doesn't belong in the reuse @@ -1516,7 +1605,7 @@ build_kernel_with_srpm() { # figure out the EXTRA_VERSION of the kernel we built or are re-using local KERNEL_RPM - if ! KERNEL_RPM=$(find_rpm "$TOPDIR/RPMS/$TARGET_ARCH/" provides "^kernel ="); then + if ! KERNEL_RPM=$(find_rpm "$TOPDIR/RPMS/$TARGET_ARCH/" provides "^kernel(-default)? ="); then fatal 1 "Failed to find a kernel RPM in $TOPDIR/RPMS/$TARGET_ARCH/" fi kernel_extra_version=$(rpm -q --queryformat "%{RELEASE}" -p $TOPDIR/RPMS/$TARGET_ARCH/$KERNEL_RPM) @@ -1557,9 +1646,9 @@ build_ofed() { # XXX we need to better integrate a distro specific override with # the rest of this function so that all of the reuse cache # stuff is leveraged given that 80% of this function is reuse - if type -p build_ofed-${DISTRO}; then + if type -p build_ofed-$DISTROMAJ; then local ofed_location - ofed_location=$(build_ofed-${DISTRO} ${STDOUT}) + ofed_location=$(build_ofed-$DISTROMAJ ${STDOUT}) local rc=${PIPESTATUS[0]} CONFIGURE_FLAGS="--with-o2ib=${ofed_location} ${CONFIGURE_FLAGS}" return $rc @@ -1577,11 +1666,11 @@ build_ofed() { # build kernel-ib/compat-rdma if $USE_BUILD_CACHE && [ -n "$REUSEBUILD" ]; then local REUSE_SIGNATURE=$({ echo "$ofed_version"; - echo "$(find_linux_release ${linux}; + echo "$(find_linux_release; echo "$BUILD_GEN")"; cat "${linux}/include/linux/autoconf.h"; - cat "${0%/lbuild}/lbuild" ; - cat "${0%/lbuild}/lbuild-${DISTRO}"; } | + cat "$LBUILD_DIR/lbuild"; + cat "$LBUILD_DIR/lbuild-$DISTROMAJ"; } | md5sum | cut -d" " -f1) # see if we can link to the reuse pool # XXX - hrm. i'm not convinced this doesn't belong in the reuse @@ -1634,10 +1723,20 @@ build_ofed() { cd ${kib_rpm}-devel # the actual ofed RPMs don't have the -rc$n or -$date string appened that # might be present on the file - local linuxrelease=$(find_linux_release "$linux") - ofed_version=$(echo $ofed_version | - sed -re 's/-(20[0-9]{6,6}-[0-9]{4,4}|rc[0-9]*)$//') - local rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-${ofed_version}-${linuxrelease//-/_}.*.rpm) + #local linuxrelease=$(find_linux_release) + #ofed_version=$(echo $ofed_version | + # sed -re 's/-(20[0-9]{6,6}-[0-9]{4,4}|rc[0-9]*)$//') + # FIXME + # OFED version will have 'hyphen' for minor release. (e.g. 3.5-1, instead + # of 3.5.1) compat-rdma and compat-rdma-devel could have same version + # number, but currectly not. Once OFED fix this in the future release, we + # can remove following filter. + #ofed_version=$(echo $ofed_version | + # sed -re 's/-([0-9]*-[rR][cC][0-9]*)$//') + #local rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-${ofed_version}-${linuxrelease//-/_}.*.rpm) + # I dont' know why we have gone through the trouble to filter out the name + # of the rpm there should only be one ${kib_rpm}-devel built + local rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-*.rpm) if ! rpm2cpio < $rpm | cpio -id; then fatal 1 "could not unpack the ${kib_rpm}-devel rpm." fi @@ -1671,7 +1770,7 @@ build_with_srpm() { # client build local kernelrpm if ! kernelrpm=$(find_linux_rpm "-$DEVEL_KERNEL_TYPE"); then - fatal 1 "Could not find the kernel-$DEVEL_KERNEL_TYPE RPM in ${KERNELRPMSBASE}/${lnxmaj}/${DISTRO}" + fatal 1 "Could not find the kernel-$DEVEL_KERNEL_TYPE RPM in $KERNELRPMSBASE/$lnxmaj/$DISTROMAJ" fi if ! lnxrel="$lnxrel" unpack_linux_devel_rpm "$kernelrpm" "-"; then fatal 1 "Could not find the Linux tree in $kernelrpm" @@ -1765,6 +1864,7 @@ create_rpmbuild_dirs() { fi [ -d RPMS/$arch ] || mkdir RPMS/$arch done + [ -d RPMS/noarch ] || mkdir RPMS/noarch [ -d BUILD ] || mkdir BUILD [ -d SOURCES ] || mkdir SOURCES [ -d SPECS ] || mkdir SPECS @@ -1865,15 +1965,12 @@ else seen_list=$(add_list "$seen_list" "${BASH_SOURCE[0]}:${BASH_LINENO[0]}") fi backtrace -echo -echo "Environment:" -set -) | mail -s "Untrapped error at ${BASH_SOURCE[0]##*/}:$((LINENO-15)) on $HOSTNAME" brian@whamcloud.com >&2; set $xtrace' ERR +) ; set $xtrace' ERR set -E [ -r ~/.lbuildrc ] && . ~/.lbuildrc -options=$(getopt -o d:D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ldiskfs,ccache,reuse:,norpm,disable-datestamp,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,ofed-version:,publish,release,set-value:,src,stage:,tag:,target:,target-archs:,with-linux:,xen -- "$@") +options=$(getopt -o D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ccache,norpm,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,noiokit,ofed-version:,mpss-version:,publish,disable-zfs,release,set-value:,src,stage:,target:,target-archs:,with-linux:,xen -- "$@") if [ $? != 0 ]; then usage 1 @@ -1890,10 +1987,6 @@ while [ "$1" ]; do CCACHE='ccache' shift ;; - -d) - CVSROOT=$2 - shift 2 - ;; -D) DATE=$2 shift 2 @@ -1929,10 +2022,6 @@ while [ "$1" ]; do DISTRO=$2 shift 2 ;; - --reuserpm) - REUSERPM=$2 - shift 2 - ;; --reusebuild) if ! REUSEBUILD=$(canon_path "$2"); then fatal 1 "Could not determine the canonical location of $2" @@ -1943,10 +2032,6 @@ while [ "$1" ]; do NORPM=true shift ;; - --ldiskfs) - LDISKFSRPM=true - shift - ;; --noiokit) IOKITRPM=false shift @@ -1983,9 +2068,17 @@ while [ "$1" ]; do OFED_VERSION="$2" shift 2 ;; + --mpss-version) + MPSS_VERSION="$2" + shift 2 + ;; --publish) shift ;; + --disable-zfs) + WITH_ZFS="no" + shift + ;; --release) RELEASE=true shift @@ -1998,10 +2091,6 @@ while [ "$1" ]; do STAGEDIR=$2 shift 2 ;; - --tag) - TAG=$2 - shift 2 - ;; --target) TARGET=$2 shift 2 @@ -2010,10 +2099,6 @@ while [ "$1" ]; do TARGET_ARCHS=$2 shift 2 ;; - --disable-datestamp) - USE_DATESTAMP= - shift - ;; --xen) XEN=true shift @@ -2030,7 +2115,6 @@ while [ "$1" ]; do LUSTRE_TESTS=false fi CONFIGURE_FLAGS=$@ - CONFIGURE_FLAGS="$CONFIGURE_FLAGS --enable-liblustre --enable-liblustre-tests" break ;; *) @@ -2045,7 +2129,7 @@ unpack_lustre # XXX - should we _always_ get the buildid from the META file? what are the # other (i.e. non-lustre-tarball use cases of lbuild)? -BUILDID=$(sed -ne '/^BUILDID =/s/.*= *//p' lustre/META) +BUILDID=$(sed -n -e 's/^BUILDID = //p' lustre/META) load_target @@ -2054,20 +2138,87 @@ if [ -n "$OFED_VERSION" -a "$OFED_VERSION" != "inkernel" ]; then unpack_ofed || fatal 1 "Error unpacking OFED tarball" fi +if [ -n "$MPSS_VERSION" ]; then + [ -z "$MPSS_DISTRO" ] && MPSS_DISTRO=$(autodetect_distro) + + MPSS_INFO=$(mktemp -u mpss-info-XXXXXXXXXX.html) + download_mpss_info "$MPSS_INFO" + + if [ "$MPSS_VERSION" = "last" ]; then + MPSS_VERSION=$(parse_mpss_info "$MPSS_INFO" "" ${MPSS_DISTRO}) + [ -z "$MPSS_VERSION" ] && + fatal 1 "Could not determine the last MPSS version" + fi + + MPSS_MAJ=${MPSS_VERSION%%.*} + MPSS_VER=${MPSS_VERSION%%-*} + MPSS_BLD=${MPSS_VERSION##*-} + [ "$MPSS_VER" = "$MPSS_BLD" ] && MPSS_BLD=0 + + download_mpss "$MPSS_INFO" + [ -r "$MPSS_INFO" ] && rm -f "$MPSS_INFO" + echo "Building with MPSS $MPSS_VER-$MPSS_BLD" + unpack_mpss || fatal 1 "Error unpacking MPSS tarballs" + prepare_mpss || fatal 1 "Error preparing MPSS for kernel modules build" + + if [ -z "$MPSS_OFED" ]; then + MPSS_OFED=$(find $PWD -type f -path "*/ofed-driver*/Module.symvers" -print -quit) + if [ -n "$MPSS_OFED" ]; then + MPSS_OFED="--with-o2ib=$(dirname $MPSS_OFED)" + else + MPSS_OFED="--without-o2ib" + fi + fi + + # disable unsupported parts: + PATCHLESS=true + IOKITRPM=false + LDISKFSRPM=false + WITH_ZFS="no" + + # define variables for cross compilation: + CROSS_SUFFIX="-mic" + if [ "$MPSS_MAJ" = "3" ]; then + CC_TARGET_ARCH=k1om-mpss-linux + LINUX="$TOPDIR/mpss" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"cross_requires /bin/sh\"" + else + CC_TARGET_ARCH=x86_64-k1om-linux + LINUX="$TOPDIR/mpss/card/kernel" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"cross_requires intel-mic-gpl = $MPSS_VER\"" + fi + LUSTRE_VERSION=$(sed -n -e 's/^VERSION = //p' lustre/META) + CONFIGURE_FLAGS="$CONFIGURE_FLAGS $MPSS_OFED --host=$CC_TARGET_ARCH --build=x86_64-pc-linux" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"post_script build/gen_filelist.sh\"" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"make_args ARCH=k1om CROSS_COMPILE=${CC_TARGET_ARCH}-\"" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"cross_path ${CROSS_PATH:=/opt/lustre/${LUSTRE_VERSION}/${CC_TARGET_ARCH}}\"" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"rootdir %{cross_path}\"" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"_prefix %{cross_path}/usr\"" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"_mandir %{_prefix}/share/man\"" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"_sysconfdir %{cross_path}/etc\"" + RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"kmoddir extra\"" + + # redefine CC for proper ./configure during rpmbuild + export CC=${CC_TARGET_ARCH}-gcc +fi + # make sure the RPM build environment is set up create_rpmbuild_dirs # if an unpacked kernel source tree was given on the command line # just build lustre with it (nothing distro kernel specific here) if [ -n "$LINUX" ]; then + find_linux_release() { + _find_linux_release $LINUX + } build_ofed "${LINUXOBJ:-$LINUX}" "$OFED_VERSION" || fatal 1 "error building OFED" build_lustre "$LINUX" "$LINUXOBJ" else - if [ ! -f "${LBUILD_DIR}/lbuild-$DISTRO" ]; then - fatal 1 "${LBUILD_DIR}/lbuild-$DISTRO not found" + if [ ! -f "${LBUILD_DIR}/lbuild-$DISTROMAJ" ]; then + fatal 1 "${LBUILD_DIR}/lbuild-$DISTROMAJ not found" fi - source ${LBUILD_DIR}/lbuild-$DISTRO + source ${LBUILD_DIR}/lbuild-$DISTROMAJ build_with_srpm || fatal 1 "Failed to build_with_srpm" fi