X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=contrib%2Flbuild%2Flbuild;h=725cdf193993cdc5b023e77fb78e10c6131f09a9;hp=30ae2af1fa7c882626f69fbaa88eed4d7b360db7;hb=a7d8aa2c3bac1ac0662b40ff018cc588e946cee4;hpb=fa16465704e1497146b461ebe59af420ae79fa9b diff --git a/contrib/lbuild/lbuild b/contrib/lbuild/lbuild index 30ae2af..725cdf1 100755 --- a/contrib/lbuild/lbuild +++ b/contrib/lbuild/lbuild @@ -36,7 +36,7 @@ push_exit_trap "kill -INT -$$ || true" kill_children #BUILD_GEN=5 # TT-107: don't cache the BUILD dir BUILD_GEN=6 # TT-1092: don't cache the BUILD dir, to rebuild external OFED -TOPDIR=$PWD +TOPDIR="$PWD" KERNELDIR= LINUX= @@ -60,21 +60,17 @@ EXTRA_VERSION= STAGEDIR= TMPDIR=${TMPDIR:-"/var/tmp"} TIMESTAMP= -# this is a dir to try reuse old kernel RPMs in (although, it seems to be -# unused in any real manner -REUSERPM= # this is the dir that should be used to store reuse products REUSEBUILD= # should cached products be used or force rebuilding? USE_BUILD_CACHE=true # what does this do exactly? does it imply no kernel build? -NORPM=false IOKITRPM=true -LDISKFSRPM=true OSDLDISKFSRPM=true OSDZFSRPM=false SMPTYPES="smp bigsmp default ''" PATCHLESS=false +WITH_ZFS="" XEN=false LINUXOBJ= DISTRO= @@ -107,8 +103,6 @@ LUSTRE_TESTS=true DATE=$(date) -RPMBUILD= - export CC=${CC:-gcc} # Readlink is not present on some older distributions: emulate it. @@ -138,9 +132,6 @@ Usage: ${0##*/} [OPTION]... [-- ] --timestamp=TIMESTAMP Date of building lustre in format YYYYMMDDhhmmss - --reuserpm=DIR - Try to reuse old kernel RPMs from DIR - --reusebuild=DIR Try to reuse old kernel builds from DIR @@ -151,7 +142,7 @@ Usage: ${0##*/} [OPTION]... [-- ] Use ccache --norpm - Do not build RPMs (compile only mode) + Unused. --patchless Build lustre client only @@ -175,14 +166,14 @@ Usage: ${0##*/} [OPTION]... [-- ] Path to an existing lustre source tarball to use. --nodownload - Do not try to download a kernel from downloads.lustre.org + Do not try to download a kernel from downloads.hpdd.intel.com --nosrc Do not build a .src.rpm, a full kernel patch, or a patched kernel tarball. --ldiskfs - Do ldiskfs RPM. Now true by default + Build with ldiskfs support. (Deprecated, always true) --noiokit Do not build lustre-iokit RPM. Now true by default @@ -194,6 +185,9 @@ Usage: ${0##*/} [OPTION]... [-- ] Specifies that the files generated do not include timestamps, and that this is an official release. + --disable-zfs + Build Lustre without ZFS. + --src Build a .src.rpm, a full kernel patch, and a patched kernel tarball. @@ -266,7 +260,14 @@ check_options() { usage 1 "Could not find Lustre source tarball '$LUSTRE'." fi - [ -z "$DISTRO" ] && DISTRO=$(autodetect_distro) + if [ -z "$DISTRO" ] ; then + DISTRO=$(autodetect_distro) + # remove separator + DISTRO=${DISTRO/-/} + fi + DISTROMAJ=${DISTRO%%.*} + + [ -z "$TARGET" ] && TARGET=$(autodetect_target "$DISTRO") if [ -z "$LINUX" ]; then [ "$KERNELDIR" -o "$KERNELTREE" ] || \ @@ -275,14 +276,19 @@ check_options() { [ -d "$KERNELDIR" -o -d "$KERNELTREE" ] || \ usage 1 "$KERNELDIR and $KERNELTREE are not a directory." - [ "$TARGET" ] || TARGET=$(autodetect_target "$DISTRO") # TARGET_FILE="$TOPDIR/lustre/kernel_patches/targets/$TARGET.target" # [ -r "$TARGET_FILE" ] || \ # usage 1 "Target '$TARGET' was not found." fi case $TARGET in - 2.6-rhel6) + 3.12-sles12) + CANONICAL_TARGET="sles12" + ;; + 3.10-rhel7) + CANONICAL_TARGET="rhel7" + ;; + 2.6-rhel6*) CANONICAL_TARGET="rhel6" ;; 2.6-rhel5) @@ -325,31 +331,47 @@ check_options() { TIMESTAMP=$(date -d "$DATE" "+%Y%m%d%H%M%S") fi - RPMBUILD=$(which rpmbuild 2>/dev/null | head -1) - if [ ! "$RPMBUILD" -o "$RPMBUILD" == "" ]; then - RPMBUILD=$(which rpm 2>/dev/null | head -1) - if [ ! "$RPMBUILD" -o "$RPMBUILD" == "" ]; then - usage 1 "Could not find binary for making rpms (tried rpmbuild and rpm)." - fi + local RPMBUILD=$(which rpmbuild 2>/dev/null | head -n 1) + RPMBUILD=${RPMBUILD:-$(which rpm 2>/dev/null | head -n 1)} + if [ -z "$RPMBUILD" ]; then + usage 1 "Could not find binary for making rpms (tried rpmbuild and rpm)." fi + local BINDIR="$TOPDIR/bin" + if [ -d $BINDIR ]; then + rm -rf $BINDIR >/dev/null 2>&1 || true + fi + + mkdir -p $BINDIR || fatal 1 "error trying to create $BINDIR" + export PATH=$BINDIR:$PATH + + cat >${BINDIR}/rpmbuild </dev/null && export DISTCC RPM_BUILD_NCPUS if which "$CCACHE" &>/dev/null; then - local ccache=$(which "$CCACHE") - local bindir="$TOPDIR/bin" + local ccache=$(which "$CCACHE" 2>/dev/null | head -n 1) - if [ ! -d $bindir ]; then - mkdir -p $bindir || fatal 1 "error trying to create $bindir" - else - rm ${bindir}/* > /dev/null 2>&1 || true - fi - ln -s "$ccache" ${bindir}/ccache - ln -s "$ccache" ${bindir}/cc - ln -s "$ccache" ${bindir}/$CC - export PATH=$bindir:$PATH - export CCACHE && export CC="ccache $CC" + ln -s "$ccache" ${BINDIR}/ccache + ln -s "$ccache" ${BINDIR}/cc + ln -s "$ccache" ${BINDIR}/$CC + + export CCACHE + export CC="ccache $CC" # zero the cache so we can see how effective we are being with it echo -n "ccache " ccache -z @@ -362,7 +384,6 @@ check_options() { fi return 0 - } # compare two versions $1 and $2. if $1 < $2, return 0 otherwise return 1. @@ -384,6 +405,8 @@ compare_version () { return 1 fi done + + return 0 } uniqify() { @@ -396,13 +419,21 @@ fetch_url() { local url="$1" local target="$2" + if [ -z "$target" ]; then + fatal 1 "fetch_url() called without a target to fetch to" + fi + + if [ -d $target ]; then + target+="/${url##*/}" + fi + local rc=0 if which wget >/dev/null 2>&1; then if ! wget -nv "$url" -O "$target"; then rc=${PIPESTATUS[0]} fi elif which curl >/dev/null 2>&1; then - if ! curl -L -s -o "$target" "$url"; then + if ! curl -n -L -s -o "$target" "$url"; then rc=${PIPESTATUS[0]} fi else @@ -421,7 +452,7 @@ download_srpm() { if $force || [ ! -r "$KERNELDIR/$srpm" ] || [ ! -s "$KERNELDIR/$srpm" ]; then if $DOWNLOAD; then - local location="http://downloads.lustre.org/public/kernels/$target/old" + local location="https://downloads.hpdd.intel.com/public/kernels/$target/old" # get the location from a distro specific method if it exists if type -p kernel_srpm_location; then location=$(kernel_srpm_location) @@ -431,11 +462,11 @@ download_srpm() { [ ! -s "$KERNELDIR/$srpm" ]; then rm -f $KERNELDIR/$srpm # punt to a distro specific method if it exists - if ! type -p download_srpm-$DISTRO; then + if ! type -p download_srpm-$DISTROMAJ; then fatal 1 "Could not download target $target's kernel SRPM $srpm from $location." else - if ! download_srpm-$DISTRO "$target" "$srpm" "$force"; then - fatal 1 "Could not download target $target's kernel SRPM $srpm using download_srpm-$DISTRO." + if ! download_srpm-$DISTROMAJ "$target" "$srpm" "$force"; then + fatal 1 "Could not download target $target's kernel SRPM $srpm using download_srpm-$DISTROMAJ." fi fi fi @@ -537,22 +568,21 @@ download_ofed() { OFED_BASE_VERSION=${OFED_VERSION%.*} fi - local location="http://www.openfabrics.org/downloads/OFED/ofed-${OFED_BASE_VERSION}/" + local location="https://www.openfabrics.org/downloads/OFED/ofed-${OFED_BASE_VERSION}/" if [[ $OFED_VERSION = *-[rR][cC][0-9] ]]; then local Mmv Mmv=${OFED_VERSION%%-[rR][cC][0-9]} - location="http://www.openfabrics.org/downloads/OFED/ofed-${Mmv}/" + location="https://www.openfabrics.org/downloads/OFED/ofed-${Mmv}/" fi if [[ $OFED_VERSION = daily-* ]]; then - local Mmv daily - OFED_VERSION=${OFED_VERSION/daily-/} - Mmv=${OFED_VERSION%%-*} + local Mmv + Mmv=${OFED_VERSION/daily-/} daily=${OFED_VERSION##$Mmv-} - location="http://www.openfabrics.org/downloads/OFED/ofed-${Mmv}-daily/" + location="https://www.openfabrics.org/downloads/OFED/ofed-${Mmv}-daily/" # find the filename for the version for the date specified - OFED_VERSION=$(curl -s "$location" | sed -nre "/${daily}-/s/.*href=\"OFED-(${Mmv//./\\.}-${daily}-[0-9]{4,4}).tgz.*$/\1/p" | tail -1) + OFED_VERSION=$(curl -s "$location" | sed -nre "/${Mmv}-/s/.*href=\"OFED-(${Mmv}-[0-9]{8,8}-[0-9]{4,4}).tgz.*$/\1/p" | tail -1) if [ -z "$OFED_VERSION" ]; then fatal 1 "Could not determine the filename of the OFED snapshot for ${daily}" fi @@ -594,7 +624,7 @@ load_target() { # go away and the target just specify the $RPMSMPTYPE [ -z "$RPMSMPTYPE" ] && set_rpm_smp_type - # CC might have been overwriten in TARGET_FILE + # CC might have been overwritten in TARGET_FILE if [[ $CC != ccache\ * ]] && which "$CCACHE" &>/dev/null; then export CCACHE && export CC="ccache $CC" fi @@ -621,7 +651,7 @@ load_target() { fi if [ -f $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET-$TARGET_ARCH.config ]; then - CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET$XENPOSTFIX-$TARGET_ARCH${RPMSMPTYPE:+-}${RPMSMPTYPE}.config" + CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET$XENPOSTFIX-$TARGET_ARCH.config" fi local lnxrelnew=${lnxrel//-/_} @@ -643,9 +673,6 @@ load_target() { # kernel. don't make it look like one if $PATCHLESS || [ -n "$SERIES" ]; then EXTRA_VERSION=$(echo $EXTRA_VERSION | sed -e "s/\(.*_lustre\)\..*/\1/") - if ! $PATCHLESS && [ -n "$BUILDID" ]; then - EXTRA_VERSION="${EXTRA_VERSION}.${BUILDID}" - fi fi fi # EXTRA_VERSION=${EXTRA_VERSION//-/_} @@ -666,18 +693,19 @@ load_target() { tarflags() { local file="$1" + echo -n '--wildcards ' case "$file" in '') fatal 1 "tarflags(): File name argument missing." ;; *.tar.gz | *.tgz) - echo 'zxf' + echo '-zxf' ;; *.tar.bz2) - echo 'jxf' + echo '-jxf' ;; *.tar) - echo 'xf' + echo '-xf' ;; *) fatal 1 "tarflags(): Unrecognized tar extension in file: $1" @@ -749,190 +777,97 @@ do_patch_linux() { } -build_iokit() { - local rpmbuildopt="$1" - - pushd lustre-iokit > /dev/null || return 255 - - if ! ./configure; then - echo "failed to configure in lustre-iokit" - popd >/dev/null # pushd lustre-iokit - return 255 - fi - - if ! make dist; then - echo "failed to make dist in lustre-iokit" - popd >/dev/null # pushd lustre-iokit - return 255 - fi - - if ! $RPMBUILD $rpmbuildopt lustre-iokit*.tar.gz \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - popd >/dev/null # pushd lustre-iokit - return 255 - fi - - if $DO_SRC && ! $RPMBUILD -ts lustre-iokit*.tar.gz \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - popd >/dev/null # pushd lustre-iokit - return 255 - fi - popd >/dev/null # pushd lustre-iokit -} - build_lustre() { local linux="$1" local linuxobj="$2" + local configure_args="" cp "$LUSTRE" SOURCES pushd lustre >/dev/null - echo "Building Lustre RPMs for: $BUILD_ARCHS..." - local targets arch - for arch in $BUILD_ARCHS; do - targets="--target $arch $targets" - done - - local confoptions="" - - if $PATCHLESS; then - confoptions="$confoptions --disable-server" + if ! build_lustre_dkms; then + popd >/dev/null # pushd lustre + return 255 fi - local rpmbuildopt='-tb' - if $NORPM; then - rpmbuildopt='-tc' - echo NORPM mode. Only compiling. - fi + echo "Building Lustre RPMs for: $BUILD_ARCHS..." # If server we now build the spl and zfs modules against the lustre kernel. # These are required prior to the building of lustre server. Client does # not require spl/zfs. Use !PATCHLESS to indicate server which follows the # line above so is at least consistant. - if [ $PATCHLESS == false ] && [ "x$ZFSNOTSUPPORTED" == "x" ]; then + if [ $PATCHLESS == false ] && [ "x$WITH_ZFS" == "x" ]; then if ! build_spl_zfs; then popd >/dev/null # pushd lustre return 255 fi fi - ( $(skeep_ldiskfs_rpm) ) || { - - pushd ldiskfs > /dev/null || return 255 - - if ! ./configure --enable-dist; then - echo "failed to configure in ldiskfs" - popd >/dev/null # pushd ldiskfs - popd >/dev/null # pushd lustre - return 255 - fi - - if ! make dist 2>&1; then - popd >/dev/null # pushd ldiskfs - popd >/dev/null # pushd lustre - return 255 - fi - - #cp lustre-ldiskfs*.tar.gz $TOPDIR/SOURCES || \ - # fatal 1 "Could not copy lustre-ldiskfs*.tar.gz to $TOPDIR/SOURCES" - - if ! $RPMBUILD $targets $rpmbuildopt lustre-ldiskfs*.tar.gz \ - --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ - --define "kdir $linux" \ - ${linuxobj:+--define "kobjdir $linuxobj"} \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - popd >/dev/null # pushd ldiskfs - popd >/dev/null # pushd lustre - return 255 - fi - - if $DO_SRC; then - if ! $RPMBUILD -ts lustre-ldiskfs*.tar.gz \ - --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ - --define "kdir $linux" \ - ${linuxobj:+--define "kobjdir $linuxobj"} \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - popd >/dev/null # pushd ldiskfs - popd >/dev/null # pushd lustre - return 255 - fi - fi - popd >/dev/null # pushd ldiskfs - - # tell lustre where ldiskfs is - # XXX - pointing to the RPM BUILD dir is a hack. we need to flesh - # out the ldiskfs RPM build so that it builds a - # lustre-ldiskfs-devel RPM and install that and point lustre - # to that instead - confoptions="$confoptions --with-ldiskfs=$(ls -d $TOPDIR/BUILD/lustre-ldiskfs-*)" - } - - if $IOKITRPM; then - if ! build_iokit $rpmbuildopt; then - echo "failed to build lustre-iokit" - popd >/dev/null # pushd lustre - return 255 - fi - fi - - # convert the $PATCHLESS boolean to an empty/not-empty boolean - # as silly as this seems, it makes the syntax of the rpmbuild command - # simpler and not need an eval to deal with the quotes in the quotes - local is_patchless="" if $PATCHLESS; then - is_patchless="yes" + configure_args="$configure_args --disable-server" fi # ditto for the lustre-tests boolean - local lustre_tests="" if ! $LUSTRE_TESTS; then - lustre_tests="no" + configure_args="$configure_args --disable-tests" fi - local osd_zfs="" - if $OSDZFSRPM; then - osd_zfs="yes" + if ! $IOKITRPM; then + configure_args="$configure_args --disable-iokit" fi - local osd_ldiskfs="" - if $OSDLDISKFSRPM; then - osd_ldiskfs="yes" + if ! $OSDZFSRPM; then + configure_args="$configure_args --without-zfs" fi - $RPMBUILD $targets $rpmbuildopt "$LUSTRE" \ - ${is_patchless:+--without servers} \ - ${lustre_tests:+--without lustre_tests} \ - ${osd_zfs:+--with zfs} \ - ${osd_ldiskfs:+--with ldiskfs} \ - ${FIND_REQUIRES:+--define "__find_requires $FIND_REQUIRES"} \ - --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ - --define "kdir $linux" \ - ${linuxobj:+--define "kobjdir $linuxobj"} \ - --define "_tmppath $TMPDIR" \ - --define "_topdir $TOPDIR" 2>&1 || \ + if ! $OSDLDISKFSRPM; then + configure_args="$configure_args --disable-ldiskfs" + fi + + configure_args="$configure_args --with-linux=$linux" + configure_args="$configure_args ${linuxobj:+--with-linux-obj=$linuxobj}" + + # allow environment setting to override ldiskfs series selection + [ -n "$LDISKFS_SERIES" ] && export LDISKFS_SERIES + + ./configure $configure_args $CONFIGURE_FLAGS 2>&1 || + fatal 1 "Error in configure." + make rpms 2>&1 || fatal 1 "Error building rpms for $BUILD_ARCHS." - if $DO_SRC; then - if ! $RPMBUILD -ts "$LUSTRE" \ - ${is_patchless:+--without servers} \ - ${lustre_tests:+--without lustre_tests} \ - ${FIND_REQUIRES:+--define "__find_requires $FIND_REQUIRES"} \ - --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ - --define "kdir $linux" \ - ${linuxobj:+--define "kobjdir $linuxobj"} \ - --define "_tmppath $TMPDIR" \ - --define "_topdir $TOPDIR" 2>&1; then - popd - return 255 - fi - fi + # move RPMs into place where they are expected to be + for arch in $BUILD_ARCHS; do + mv -f lustre-*.${arch}.rpm $TOPDIR/RPMS/${arch}/ + done + mv -f lustre-*.src.rpm $TOPDIR/SRPMS/ + popd >/dev/null + return 0 +} + +# Only zfs Lustre DKMS Server is supported +build_lustre_dkms() { + local build_args="" + local name_prefix="lustre" + local ver=$(sed -n -e 's/^LUSTRE_VERSION = //p' LUSTRE-VERSION-FILE) + + echo "Building Lustre DKMS RPMs for: $BUILD_ARCHS..." + ./configure --enable-dist || fatal 1 "Error in DKMS configure." + + if $PATCHLESS; then + build_args="--without servers" + name_prefix="lustre-client" + fi + + rpmbuild --define "_topdir $TOPDIR" $build_args -bs lustre-dkms.spec || + fatal 1 "Error building DKMS .src.rpm for $BUILD_ARCHS." + rpmbuild --define "_topdir $TOPDIR" $build_args \ + --rebuild $TOPDIR/SRPMS/$name_prefix-dkms-$ver-*.src.rpm || + fatal 1 "Error building DKMS .rpm for $BUILD_ARCHS." + + return 0 } ### @@ -955,8 +890,8 @@ build_spl_zfs() { # The spl/zfs spec files expect RPM_BUILD_ROOT to point to the root of the # destination for the rpms export RPM_BUILD_ROOT=$TOPDIR - SPLZFSVER=${SPLZFSVER:-0.6.1} - SPLZFSTAG=${SPLZFSTAG:-lustre-zfs} + SPLZFSVER=${SPLZFSVER:-0.6.5.7} + SPLZFSTAG=${SPLZFSTAG:-} # The files expect a kver to be set to the kernel version . local kver=$(find_linux_release) @@ -967,31 +902,31 @@ build_spl_zfs() { local rpmpkg - [ "$pkg" == "zfs" ] && spldir="$(ls -d $TOPDIR/usr/src/spl-*/|tail -1)" + [ "$pkg" == "zfs" ] && spldir="$(ls -d $TOPDIR/usr/src/spl-*|tail -1)" # need to fetch the repo in order to build it. # default to github but allow override git clone ${SPLZFSGITREPO:-"https://github.com/zfsonlinux"}/$pkg.git $pkg 2>&1 pushd $pkg || return 255 - if [ -n "$SPLZFSTAG" ]; then - git checkout $SPLZFSTAG - else - git checkout -b lbuild $pkg-$SPLZFSVER - fi - - # This differentiates between older zfs versions - if [ -f $pkg-modules.spec.in ]; then - rpmpkg=$pkg-modules - specdir=. - speclist="$pkg.spec $rpmpkg.spec" - else + if [ -n "$SPLZFSTAG" ]; then + git checkout $SPLZFSTAG + else + git checkout -b lbuild $pkg-$SPLZFSVER + fi + + # This differentiates between older zfs versions + if [ -f $pkg-modules.spec.in ]; then + rpmpkg=$pkg-modules + specdir=. + speclist="$pkg.spec $rpmpkg.spec" + else rpmpkg=kmod-$pkg-devel - specdir=rpm/generic - speclist="$pkg.spec $pkg-kmod.spec $pkg-dkms.spec" - fi + specdir=rpm/generic + speclist="$pkg.spec $pkg-kmod.spec $pkg-dkms.spec" + fi - sh autogen.sh || return 255 + sh autogen.sh || return 255 if ! ./configure --with-linux=${LINUX} --with-linux-obj=${LINUXOBJ:-$LINUX} \ ${spldir:+--with-spl="${spldir}"} 2>&1 || @@ -1003,40 +938,36 @@ build_spl_zfs() { ln -f $pkg/$pkg-*.tar.gz $TOPDIR/SOURCES || error "failed to link $pkg/$pkg-*.tar.gz into $TOPDIR/SOURCES" - if [ -f $pkg/scripts/kmodtool ]; then - ln -f $pkg/scripts/kmodtool $TOPDIR/SOURCES/ - fi - - local rpmb - if $DO_SRC; then - rpmb=-ba - else - rpmb=-bb - fi - - # set search dir for our own kmodtool to find correct - # directories - export KERNELSOURCE=$(dirname ${LINUX}) - # Manually build rpms - for spec in $speclist; do - echo "Building RPMs from $pkg/$specdir/$spec" - if ! $RPMBUILD $rpmb $pkg/$specdir/$spec \ - --nodeps -v \ - --define "require_kdir ${LINUX}" \ - ${LINUXOBJ:+--define "require_kobj ${LINUXOBJ}"} \ - ${spldir:+--define "require_spldir ${spldir}"} \ - --define "kver $kver" \ - --define "kernels $kver" \ - --define "_tmppath /var/tmp" \ - --define "kernelbuildroot $TOPDIR/reused" \ - --define "_topdir $TOPDIR" 2>&1; then - return 255 - fi - done + if [ -f $pkg/scripts/kmodtool ]; then + ln -f $pkg/scripts/kmodtool $TOPDIR/SOURCES/ + fi + + local rpmb + if $DO_SRC; then + rpmb=-ba + else + rpmb=-bb + fi -# $DO_SRC && mv $pkg*.src.rpm $TOPDIR/SRPMS/ -# mv $pkg*.noarch.rpm $TOPDIR/RPMS/noarch/ -# mv *$pkg*.$TARGET_ARCH.rpm $TOPDIR/RPMS/$(basearch $TARGET_ARCH)/ + # set search dir for our own kmodtool to find correct + # directories + export KERNELSOURCE=$(dirname ${LINUX}) + # Manually build rpms + for spec in $speclist; do + echo "Building RPMs from $pkg/$specdir/$spec" + if ! rpmbuild $rpmb $pkg/$specdir/$spec \ + --nodeps -v \ + --define "require_kdir ${LINUX}" \ + ${LINUXOBJ:+--define "require_kobj ${LINUXOBJ}"} \ + ${spldir:+--define "require_spldir ${spldir}"} \ + --define "kver $kver" \ + --define "kernels $kver" \ + --define "_tmppath /var/tmp" \ + --define "kernelbuildroot $TOPDIR/reused" \ + --define "_topdir $TOPDIR" 2>&1; then + return 255 + fi + done # We have built the rpms for the package. Now we need to extract the # contained files so we can build further things against them @@ -1044,23 +975,29 @@ build_spl_zfs() { # cpio only extract to pwd so we need to go there. pushd $TOPDIR - local rpm - for rpm in $rpms; do + local rpm + for rpm in $rpms; do rpm2cpio $rpm | cpio -id - done + done if [ "$pkg" == "zfs" ]; then - mkdir -p $TOPDIR/zfsdevel - pushd $TOPDIR/zfsdevel - # We also need to extract zfs-devel rpm - local devel_rpm=$(ls -1 $TOPDIR/RPMS/*/$pkg-devel-*.rpm) - rpm2cpio $devel_rpm | cpio -id - popd - CONFIGURE_FLAGS="--with-$pkg-devel=$TOPDIR/zfsdevel ${CONFIGURE_FLAGS}" + # We also need to extract both the zfs and zfs-devel rpms + # the zfs rpm is needed because it has the actual libraries in + # it and the zfs-devel rpm only has unversioned symlinks to the + # libraries in the zfs rpm + # this will all change one day when we have a libzfs rpm per + # https://github.com/zfsonlinux/zfs/issues/2329 + # and it looks like it could be one day soon: + # https://github.com/zfsonlinux/zfs/pull/2341 + local devel_rpms=$(ls -1 $TOPDIR/RPMS/*/{$pkg-devel,$pkg-$SPLZFSVER,lib*}-*.rpm) + for rpm in $devel_rpms; do + rpm2cpio $rpm | cpio -id + done + CONFIGURE_FLAGS="--with-$pkg-devel=$TOPDIR ${CONFIGURE_FLAGS}" fi popd - CONFIGURE_FLAGS="--with-$pkg=$(ls -d $TOPDIR/usr/src/$pkg-*/|tail -1) ${CONFIGURE_FLAGS}" + CONFIGURE_FLAGS="--with-$pkg=$(ls -d $TOPDIR/usr/src/$pkg-*|tail -1) ${CONFIGURE_FLAGS}" CONFIGURE_FLAGS="--with-$pkg-obj=$(ls -d $TOPDIR/usr/src/$pkg-*/$kver*|tail -1) ${CONFIGURE_FLAGS}" done @@ -1077,30 +1014,15 @@ stage() { rpmdir="${STAGEDIR}/${CANONICAL_TARGET}-${arch}" echo "${0##*/}: Copying RPMs into ${rpmdir}" mkdir -p "${rpmdir}" - cp -v RPMS/${arch}/*.rpm "${rpmdir}" - if [ -d RPMS/noarch ]; then - cp -v RPMS/noarch/*.rpm "${rpmdir}" - fi + for rpm in $(ls RPMS/${arch}/*.rpm RPMS/noarch/*.rpm); do + cp -v $rpm "${rpmdir}" + done done cp -v "$LUSTRE" "$STAGEDIR" } -#check if we need to build separate ldiskfs RPM -skeep_ldiskfs_rpm() { - local skip=false - - if ! $LDISKFSRPM; then - skip=true - elif $PATCHLESS; then - skip=true - fi - - echo $skip - -} - set_rpm_smp_type() { local infact_arch="${TARGET_ARCH}" @@ -1167,8 +1089,8 @@ unpack_linux_devel_rpm() { fi # call a distro specific hook, if available - if type -p unpack_linux_devel_rpm-$DISTRO; then - if ! unpack_linux_devel_rpm-$DISTRO "$kernelrpm"; then + if type -p unpack_linux_devel_rpm-$DISTROMAJ; then + if ! unpack_linux_devel_rpm-$DISTROMAJ "$kernelrpm"; then return 255 fi fi @@ -1193,13 +1115,13 @@ build_kernel_ib() { K_SRC="KSRC" fi - local OFED_CORE="--with-core-mod --with-ipoib-mod --with-sdp-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod" - local OFED_HARDWARE="--with-mthca-mod --with-mlx4-mod --with-mlx4_en-mod --with-cxgb3-mod --with-nes-mod" + local OFED_CORE="--with-core-mod --with-ipoib-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod" + local OFED_HARDWARE="--with-mthca-mod --with-mlx4-mod --with-mlx4_en-mod --with-cxgb3-mod --with-nes-mod --with-qib-mod" if compare_version $OFED_VERSION 3.0; then OFED_CORE="$OFED_CORE --with-madeye-mod --with-rds-mod" else - OFED_HARDWARE="$OFED_HARDWARE --with-cxgb4-mod" + OFED_HARDWARE="$OFED_HARDWARE --with-mlx5-mod --with-cxgb4-mod --with-ocrdma-mod --with-qib-mod" fi # some I/B drivers are architecture dependent and kernel-ib's configure @@ -1270,8 +1192,7 @@ EOF if compare_version $OFED_VERSION 3.0; then local OFA_KERNEL_RELEASE=$(echo -n ${linuxrelease} | sed -e 's/-/_/g') fi - if ! $RPMBUILD $BUILD_TYPE --define 'build_kernel_ib 1' --define 'build_kernel_ib_devel 1' \ - ${FIND_REQUIRES:+--define "__find_requires $FIND_REQUIRES"} \ + if ! rpmbuild $BUILD_TYPE --define 'build_kernel_ib 1' --define 'build_kernel_ib_devel 1' \ --define "_topdir ${TOPDIR}" --target ${TARGET_ARCH} \ --define "KVERSION ${linuxrelease}" \ --define "$K_SRC ${linux}" \ @@ -1430,8 +1351,12 @@ build_kernel_with_srpm() { if $USE_BUILD_CACHE && [ -n "$REUSEBUILD" ]; then local REUSE_SIGNATURE=$({ echo -en $release_str; echo $BUILD_GEN; - cat $CONFIG_FILE $TARGET_FILE $FULL_PATCH; - cat $LBUILD_DIR/lbuild $LBUILD_DIR/lbuild-${DISTRO}; } | + cat "$CONFIG_FILE"; + cat "$TARGET_FILE" | + sed -e '/_VERSION=/s/_[0-9]*_g.*$//g'; + cat "$FULL_PATCH"; + cat "$LBUILD_DIR/lbuild"; + cat "$LBUILD_DIR/lbuild-$DISTROMAJ"; } | md5sum | cut -d" " -f1) # see if we can link to the reuse pool # XXX - hrm. i'm not convinced this doesn't belong in the reuse @@ -1525,9 +1450,9 @@ build_ofed() { # XXX we need to better integrate a distro specific override with # the rest of this function so that all of the reuse cache # stuff is leveraged given that 80% of this function is reuse - if type -p build_ofed-${DISTRO}; then + if type -p build_ofed-$DISTROMAJ; then local ofed_location - ofed_location=$(build_ofed-${DISTRO} ${STDOUT}) + ofed_location=$(build_ofed-$DISTROMAJ ${STDOUT}) local rc=${PIPESTATUS[0]} CONFIGURE_FLAGS="--with-o2ib=${ofed_location} ${CONFIGURE_FLAGS}" return $rc @@ -1548,8 +1473,8 @@ build_ofed() { echo "$(find_linux_release; echo "$BUILD_GEN")"; cat "${linux}/include/linux/autoconf.h"; - cat "$LBUILD_DIR/lbuild" ; - cat "$LBUILD_DIR/lbuild-${DISTRO}"; } | + cat "$LBUILD_DIR/lbuild"; + cat "$LBUILD_DIR/lbuild-$DISTROMAJ"; } | md5sum | cut -d" " -f1) # see if we can link to the reuse pool # XXX - hrm. i'm not convinced this doesn't belong in the reuse @@ -1602,17 +1527,20 @@ build_ofed() { cd ${kib_rpm}-devel # the actual ofed RPMs don't have the -rc$n or -$date string appened that # might be present on the file - local linuxrelease=$(find_linux_release) - ofed_version=$(echo $ofed_version | - sed -re 's/-(20[0-9]{6,6}-[0-9]{4,4}|rc[0-9]*)$//') + #local linuxrelease=$(find_linux_release) + #ofed_version=$(echo $ofed_version | + # sed -re 's/-(20[0-9]{6,6}-[0-9]{4,4}|rc[0-9]*)$//') # FIXME # OFED version will have 'hyphen' for minor release. (e.g. 3.5-1, instead # of 3.5.1) compat-rdma and compat-rdma-devel could have same version # number, but currectly not. Once OFED fix this in the future release, we # can remove following filter. - ofed_version=$(echo $ofed_version | - sed -re 's/-([0-9]*-[rR][cC][0-9]*)$//') - local rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-${ofed_version}-${linuxrelease//-/_}.*.rpm) + #ofed_version=$(echo $ofed_version | + # sed -re 's/-([0-9]*-[rR][cC][0-9]*)$//') + #local rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-${ofed_version}-${linuxrelease//-/_}.*.rpm) + # I dont' know why we have gone through the trouble to filter out the name + # of the rpm there should only be one ${kib_rpm}-devel built + local rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-*.rpm) if ! rpm2cpio < $rpm | cpio -id; then fatal 1 "could not unpack the ${kib_rpm}-devel rpm." fi @@ -1646,7 +1574,7 @@ build_with_srpm() { # client build local kernelrpm if ! kernelrpm=$(find_linux_rpm "-$DEVEL_KERNEL_TYPE"); then - fatal 1 "Could not find the kernel-$DEVEL_KERNEL_TYPE RPM in ${KERNELRPMSBASE}/${lnxmaj}/${DISTRO}" + fatal 1 "Could not find the kernel-$DEVEL_KERNEL_TYPE RPM in $KERNELRPMSBASE/$lnxmaj/$DISTROMAJ" fi if ! lnxrel="$lnxrel" unpack_linux_devel_rpm "$kernelrpm" "-"; then fatal 1 "Could not find the Linux tree in $kernelrpm" @@ -1657,7 +1585,7 @@ build_with_srpm() { # infrastructure files so that find-requires can find our unpacked # kernel-devel artifacts cp $RPM_HELPERS_DIR/{symset-table,find-requires{,.ksyms}} . - FIND_REQUIRES="$(pwd)/find-requires" + export FIND_REQUIRES="$(pwd)/find-requires" chmod 755 {symset-table,find-requires{,.ksyms}} local tmp="$(pwd)" tmp="${tmp//\//\\/}" @@ -1846,7 +1774,7 @@ set -E [ -r ~/.lbuildrc ] && . ~/.lbuildrc -options=$(getopt -o D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ldiskfs,ccache,reuse:,norpm,disable-datestamp,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,noiokit,ofed-version:,publish,release,set-value:,src,stage:,tag:,target:,target-archs:,with-linux:,xen -- "$@") +options=$(getopt -o D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ccache,norpm,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,noiokit,ofed-version:,publish,disable-zfs,release,set-value:,src,stage:,target:,target-archs:,with-linux:,xen -- "$@") if [ $? != 0 ]; then usage 1 @@ -1898,10 +1826,6 @@ while [ "$1" ]; do DISTRO=$2 shift 2 ;; - --reuserpm) - REUSERPM=$2 - shift 2 - ;; --reusebuild) if ! REUSEBUILD=$(canon_path "$2"); then fatal 1 "Could not determine the canonical location of $2" @@ -1909,11 +1833,6 @@ while [ "$1" ]; do shift 2 ;; --norpm) - NORPM=true - shift - ;; - --ldiskfs) - LDISKFSRPM=true shift ;; --noiokit) @@ -1955,6 +1874,10 @@ while [ "$1" ]; do --publish) shift ;; + --disable-zfs) + WITH_ZFS="no" + shift + ;; --release) RELEASE=true shift @@ -1967,10 +1890,6 @@ while [ "$1" ]; do STAGEDIR=$2 shift 2 ;; - --tag) - shift 2 - echo "WARNING: \"--tag\" is deprecated" > 2 - ;; --target) TARGET=$2 shift 2 @@ -1995,7 +1914,6 @@ while [ "$1" ]; do LUSTRE_TESTS=false fi CONFIGURE_FLAGS=$@ - CONFIGURE_FLAGS="$CONFIGURE_FLAGS --enable-liblustre --enable-liblustre-tests" break ;; *) @@ -2008,10 +1926,6 @@ check_options unpack_lustre -# XXX - should we _always_ get the buildid from the META file? what are the -# other (i.e. non-lustre-tarball use cases of lbuild)? -BUILDID=$(sed -ne '/^BUILDID =/s/.*= *//p' lustre/META) - load_target if [ -n "$OFED_VERSION" -a "$OFED_VERSION" != "inkernel" ]; then @@ -2025,14 +1939,17 @@ create_rpmbuild_dirs # if an unpacked kernel source tree was given on the command line # just build lustre with it (nothing distro kernel specific here) if [ -n "$LINUX" ]; then + find_linux_release() { + _find_linux_release $LINUX + } build_ofed "${LINUXOBJ:-$LINUX}" "$OFED_VERSION" || fatal 1 "error building OFED" build_lustre "$LINUX" "$LINUXOBJ" else - if [ ! -f "${LBUILD_DIR}/lbuild-$DISTRO" ]; then - fatal 1 "${LBUILD_DIR}/lbuild-$DISTRO not found" + if [ ! -f "${LBUILD_DIR}/lbuild-$DISTROMAJ" ]; then + fatal 1 "${LBUILD_DIR}/lbuild-$DISTROMAJ not found" fi - source ${LBUILD_DIR}/lbuild-$DISTRO + source ${LBUILD_DIR}/lbuild-$DISTROMAJ build_with_srpm || fatal 1 "Failed to build_with_srpm" fi