X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=build%2Flbuild;h=15b78befb1e8d76a262ad674f436028e3e7b11fc;hp=594d7779b194a8c744245e71e426861eedbc259d;hb=a6527b28ba2082a445fbf584909a7791ce407ef3;hpb=fce84de55093827f9f45179fc2f790740fa47372 diff --git a/build/lbuild b/build/lbuild index 594d777..15b78be 100755 --- a/build/lbuild +++ b/build/lbuild @@ -2,9 +2,25 @@ # vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4: +# this is an alternative FD for stdout, to be used especially when we are +# taking stdout from a function as it's return value. i.e. foo=$(bar) +# this is a workaround until a version of bash where we can put xtrace +# on a specific FD +exec 3>&1; STDOUT=3 + #set -x +xtrace="+x" +if [[ $SHELLOPTS = *xtrace* ]]; then + xtrace="-x" +fi shopt -s extdebug +# include the exit_traps library +. ${0%/lbuild}/exit_traps.sh + +# our children should die when we do +push_exit_trap "kill -INT -$$ || true" kill_children + TOPDIR=$PWD # CVSROOT is inherited from the environment @@ -16,7 +32,7 @@ RELEASE=false # search through this file (and lbuild.old_school -- but that will # be going away soon) for "-bb" and see how many places # simply don't account for this option -DO_SRC=false +DO_SRC=true DOWNLOAD=true TAG= CANONICAL_TARGET= @@ -32,21 +48,25 @@ LUSTRE_EXTRA_VERSION= STAGEDIR= TMPDIR=${TMPDIR:-"/var/tmp"} TIMESTAMP= -# XXX - i think these two parameters/arguments/variables need to be -# cleaned up and merged. they effectively do the same thing +# this is a dir to try reuse old kernel RPMs in (although, it seems to be +# unused in any real manner REUSERPM= +# this is the dir that should be used to store reuse products REUSEBUILD= +# should cached products be used or force rebuilding? +USE_BUILD_CACHE=true # what does this do exactly? does it imply no kernel build? NORPM=false LDISKFSRPM=true SKIPLDISKFSRPM="v1_4_* b1_4" SMPTYPES="smp bigsmp default ''" -KERNCONFSMPTYPE= PATCHLESS=false XEN=false LINUXOBJ= DISTRO= KERNELTREE= +# default to not adding -lustre- into the kernel RPM package names +KERNEL_LUSTRE_NAMING=false # patchless build KERNELRPMSBASE= @@ -66,6 +86,9 @@ UP_ARCHS= # not in the target file any more CONFIG= +# build the lustre-tests rpm? +LUSTRE_TESTS=true + DATE=$(date) USE_DATESTAMP=1 @@ -95,7 +118,7 @@ cleanup() { error() { local msg="$1" - [ -n "$msg" ] && echo -e "\n${0##*/}: $msg" >&2 + [ -n "$msg" ] && echo -e "\n${0##*/}: $msg" >&3 } @@ -280,6 +303,12 @@ check_options() { 2.6-sles10) CANONICAL_TARGET="sles10-2.6" ;; + 2.6-sles11) + CANONICAL_TARGET="sles11" + ;; + 2.6-oel5) + CANONICAL_TARGET="oel5" + ;; hp_pnnl-2.4) CANONICAL_TARGET="hp-pnnl-2.4" ;; @@ -311,27 +340,37 @@ check_options() { fi if [ -n "$CCACHE" ]; then - which "$DISTCC" 2>/dev/null && export DISTCC RPM_BUILD_NCPUS + which "$DISTCC" &>/dev/null && export DISTCC RPM_BUILD_NCPUS - if which "$CCACHE" 2>/dev/null; then + if which "$CCACHE" &>/dev/null; then local ccache=$(which "$CCACHE") local bindir="$TOPDIR/bin" - [ -d $bindir ] || mkdir -p $bindir - if [ -d $bindir ]; then - rm ${bindir}/* > /dev/null 2>&1 - ln -s "$ccache" ${bindir}/ccache - ln -s "$ccache" ${bindir}/cc - ln -s "$ccache" ${bindir}/$CC - export PATH=$bindir:$PATH + if [ ! -d $bindir ]; then + mkdir -p $bindir || fatal 1 "error trying to create $bindir" + else + rm ${bindir}/* > /dev/null 2>&1 || true fi + ln -s "$ccache" ${bindir}/ccache + ln -s "$ccache" ${bindir}/cc + ln -s "$ccache" ${bindir}/$CC + export PATH=$bindir:$PATH export CCACHE && export CC="ccache $CC" # zero the cache so we can see how effective we are being with it + echo -n "ccache " ccache -z + + # get some ccache stats when we are done + push_exit_trap '[ -n "$CCACHE" ] && ccache -s' "ccache_summary" + # should remove the ccache trap if lbuild is interrupted + trap 'echo "Received an INT TERM or HUP signal, terminating."; delete_exit_trap "ccache_summary"; exit 1' INT TERM HUP fi fi [ -z "$DISTRO" ] && DISTRO=$(autodetect_distro) + + return 0 + } # autodetect used Distro @@ -350,8 +389,7 @@ autodetect_distro() { sed -e 's/^[^0-9.]*//g' | sed -e 's/[ \.].*//') fi if [ -z "$name" -o -z "$version" ]; then - fatal 1 "I don't know how to determine distro type/version.\n" \ - "Either update autodetect_distro() or use the --distro argument" + fatal 1 "I don't know how to determine distro type/version.\nEither update autodetect_distro() or use the --distro argument" fi echo ${name}${version} @@ -368,37 +406,157 @@ uniqify() { download_srpm() { local target=$1 local srpm=$2 + local force="${3:-false}" - echo "Downloading http://downloads.lustre.org/public/kernels/$target/old/$srpm..." - if ! wget -nv "http://downloads.lustre.org/public/kernels/$target/old/$srpm" \ - -O "$KERNELDIR/$srpm"; then - fatal 1 "Could not download target $target's kernel SRPM $srpm from downloads.lustre.org." + if $force || [ ! -r "$KERNELDIR/$srpm" ] || + [ ! -s "$KERNELDIR/$srpm" ]; then + if $DOWNLOAD; then + local location="http://downloads.lustre.org/public/kernels/$target/old" + echo "Downloading $location/$srpm..." + if ! wget -nv "$location/$srpm" -O "$KERNELDIR/$srpm" 2>&1 || + [ ! -s "$KERNELDIR/$srpm" ]; then + rm -f $KERNELDIR/$srpm + fatal 1 "Could not download target $target's kernel SRPM $srpm from $location." + fi + else + fatal 1 "$srpm not found in directory $KERNELDIR." + fi fi - [ -s "$KERNELDIR/$srpm" ] || { - rm -rf $KERNELDIR/$srpm - fatal 1 "Could not download target $target's kernel SRPM $srpm from downloads.lustre.org." - } } -download_ofed() { +download_file() { + local from="$1" + local to="$2" + local force="$3" + + local file=${from##*/} + + if [ -d $to ]; then + to="$to/$file" + fi + + local semaphore="$to-downloading" + + is_downloading() { + if [ ! -f $semaphore ]; then + return 1 + fi + + # make sure the download has not been aborted + local now=$(date +%s) + local file_mtime=$(stat -c %Y "$to") + local staleness=$((now - file_mtime)) + # let's assume an active download will write at least once a minute + if [ $staleness -gt 60 ]; then + return 1 + fi + + return 0 + } + + is_downloaded() { + # if the semaphore file exists, the file is either still downloading + # or a download was aborted and we cannot trust the target file + if [ -f $semaphore ]; then + return 1 + fi + + if ! is_downloading && [ -r "$to" ] && [ -s "$to" ]; then + return 0 + fi + + return 1 + } + + if $force || ! is_downloaded; then + if is_downloading; then + echo "Somebody else is downloading $from..." + while is_downloading; do + echo "Waiting for $to to finish downloading" + sleep 60 + done + if is_downloaded; then + return 0 + else + echo "The download we were waiting for seems to have been aborted" + fi + + fi - if [ -n "$OFED_VERSION" -a "$OFED_VERSION" != "inkernel" ] && \ - [ ! -r "$KERNELTREE/OFED-${OFED_VERSION}.tgz" ]; then if $DOWNLOAD; then - local location="http://downloads.lustre.org/public/OFED/" - echo "Downloading $location/OFED-${OFED_VERSION}.tgz..." - if ! wget -nv "$location/OFED-${OFED_VERSION}.tgz" \ - -O "$KERNELTREE/OFED-${OFED_VERSION}.tgz"; then - fatal 1 "Could not download OFED-${OFED_VERSION}.tgz" \ - "from downloads.lustre.org." + echo "Downloading $from..." + # flag others so they don't try to download also + push_exit_trap "rm -f $to $semaphore" "download" + touch $semaphore + if ! wget -nv "$from" -O "$to" || [ ! -s "$to" ]; then + # the trap will remove the files via the fatal below + fatal 1 "Could not download ${to##*/} from ${from%%/*}/." fi + rm -f $semaphore + delete_exit_trap "download" else - fatal 1 "OFED-${OFED_VERSION}.tgz not found in kernel" \ - "directory $KERNELTREE." + fatal 1 "${to##*/} not found in directory ${to%/*}." fi fi + return 0 + +} + +download_ofed() { + local force="${1:-false}" + + local location="http://downloads.lustre.org/public/OFED/" + + local Mmv daily + if [[ $OFED_VERSION = daily-* ]]; then + OFED_VERSION=${OFED_VERSION/daily-/} + Mmv=${OFED_VERSION%%-*} + daily=${OFED_VERSION##$Mmv-} + location="http://www.openfabrics.org/downloads/OFED/ofed-${Mmv}-daily/" + # find the filename for the version for the date specified + OFED_VERSION=$(curl -s "$location" | sed -nre "/${daily}-/s/.*href=\"OFED-(${Mmv//./\\.}-${daily}-[0-9]{4,4}).tgz.*$/\1/p" | tail -1) + if [ -z "$OFED_VERSION" ]; then + fatal 1 "Could not determine the filename of the OFED snapshot for ${daily}" + fi + fi + + if [ -z "$OFED_VERSION" -o "$OFED_VERSION" = "inkernel" ]; then + return 0 + fi + + local file="OFED-${OFED_VERSION}.tgz" + download_file "$location/$file" "$KERNELTREE" "$force" + +} + +download_rdac() { + local force="${1:-false}" + + local location="http://downloads.lustre.org/public/RDAC/" + + if [ -z "$RDAC_VERSION" -o "$RDAC_VERSION" = "inkernel" ]; then + return 0 + fi + + local file="rdac-LINUX-${RDAC_VERSION}-source.tar.gz" + download_file "$location/$file" "$KERNELTREE" "$force" + +} + +download_mptlinux() { + local force="${1:-false}" + + local location="http://downloads.lustre.org/public/MPTLINUX/" + + if [ -z "$MPTLINUX_VERSION" -o "$MPTLINUX_VERSION" = "inkernel" ]; then + return 0 + fi + + file="MPTLINUX_RHEL5_SLES10_PH15-${MPTLINUX_VERSION}.zip" + download_file "$location/$file" "$KERNELTREE" "$force" + } load_target() { @@ -413,8 +571,16 @@ load_target() { echo "Loading target config file $TARGET.target..." + # if the caller specified an OFED_VERSION it should override whatever + # the target file specifies + local env_OFED_VERSION="$OFED_VERSION" + . "$TARGET_FILE" + if [ -n "$env_OFED_VERSION" ]; then + OFED_VERSION="$env_OFED_VERSION" + fi + # doesn't make any sense to build OFED for xen domX's if $XEN; then OFED_VERSION="" @@ -425,7 +591,7 @@ load_target() { [ -z "$RPMSMPTYPE" ] && set_rpm_smp_type # CC might have been overwriten in TARGET_FILE - if [[ $CC != ccache\ * ]] && which "$CCACHE" 2>/dev/null; then + if [[ $CC != ccache\ * ]] && which "$CCACHE" &>/dev/null; then export CCACHE && export CC="ccache $CC" fi @@ -440,9 +606,7 @@ load_target() { for patchesdir in "$EXTERNAL_PATCHES" "$TOPDIR/lustre/lustre/kernel_patches"; do [ -r "$patchesdir/series/$series" ] && continue 2 done - fatal 1 "Target $TARGET's series $SERIES could not be" \ - "found.\nSearched:\n\t$EXTERNAL_PATCHES/series\n" \ - "\t$TOPDIR/lustre/lustre/kernel_patches/series." + fatal 1 "Target $TARGET's series $SERIES could not be found.\nSearched:\n\t$EXTERNAL_PATCHES/series\n\t$TOPDIR/lustre/lustre/kernel_patches/series." done fi @@ -451,20 +615,11 @@ load_target() { if $XEN; then XENPOSTFIX="-xen" fi + if [ -f $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET-$TARGET_ARCH.config ]; then - CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET$XENPOSTFIX-$TARGET_ARCH.config" + CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET$XENPOSTFIX-$TARGET_ARCH${RPMSMPTYPE:+-}${RPMSMPTYPE}.config" fi - # figure out our smp kernel type and set the .config if we have one - local smptype - for smptype in $SMPTYPES; do - [ "$smptype" = "''" ] && smptype= - if [ -f $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET-$TARGET_ARCH-${smptype}.config ]; then - CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET$XENPOSTFIX-$TARGET_ARCH-${smptype}.config" - KERNCONFSMPTYPE=$smptype - fi - done - local lnxrelnew=${lnxrel//-/_} # remember the EXTRA_VERSION before we diddle it here @@ -474,8 +629,7 @@ load_target() { PRISTINE_EXTRA_VERSION=$EXTRA_VERSION if ! $PATCHLESS && [ ! -f "$CONFIG_FILE" ]; then - fatal 1 "Config file for target $TARGET missing from" \ - "$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/." + fatal 1 "Config file for target $TARGET missing from $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/." fi if [ "$EXTRA_VERSION_save" ]; then @@ -538,11 +692,31 @@ untar() { unpack_ofed() { - untar "$KERNELTREE/OFED-${OFED_VERSION}.tgz" || fatal 1 "Error unpacking OFED tarball" + if ! untar "$KERNELTREE/OFED-${OFED_VERSION}.tgz"; then + return 1 + fi [ -d OFED ] || ln -sf OFED-[0-9].[0-9]* OFED } +unpack_rdac() { + + if ! untar "$KERNELTREE/rdac-LINUX-${RDAC_VERSION}-source.tar.gz"; then + return 1 + fi + [ -d rdac-LINUX ] || ln -sf rdac-LINUX-[0-9][0-9].* rdac-LINUX + +} + +unpack_mptlinux() { + + if ! unzip -p $KERNELTREE/MPTLINUX_RHEL5_SLES10_PH15-4.16.00.00-2.zip | tar xzvf - srpms-2/mptlinux-4.16.00.00-2.src.rpm; then + return 1 + fi + mv srpms-2/mptlinux-4.16.00.00-2.src.rpm . + +} + unpack_lustre() { if [ -z "$LUSTRE" ]; then @@ -577,7 +751,7 @@ do_patch_linux() { local do_patch=${1:-true} - FULL_PATCH="$PWD/lustre-kernel-${TARGET}${EXTRA_VERSION_DELIMITER}${EXTRA_VERSION}.patch" + FULL_PATCH="$PWD/lustre-kernel-${TARGET}-${EXTRA_VERSION}.patch" [ -f "$FULL_PATCH" ] && rm -f "$FULL_PATCH" $do_patch && pushd linux >/dev/null for series in $SERIES; do @@ -595,7 +769,7 @@ do_patch_linux() { fatal 1 "Error adding patch $patch to full patch." } if $do_patch; then - patch -s -p1 < "$PATCH_FILE" || { + patch -s -p1 < "$PATCH_FILE" 2>&1 || { rm -f $FULL_PATCH fatal 1 "Error applying patch $patch." } @@ -611,31 +785,38 @@ do_patch_linux() { } build_lustre() { + local linux="$1" + local linuxobj="$2" cp "$LUSTRE" SOURCES pushd lustre >/dev/null echo "Building Lustre RPMs for: $BUILD_ARCHS..." - targets= + local targets arch for arch in $BUILD_ARCHS; do targets="--target $arch $targets" done - local confoptions="--with-linux=${LINUX}" + local confoptions="--with-linux=${linux}" if $PATCHLESS; then - confoptions="--with-linux=${LINUX} --disable-server" + confoptions="--with-linux=${linux} --disable-server" fi - if [ ! "$LINUXOBJ" = "" ]; then - confoptions="$confoptions --with-linux-obj=${LINUXOBJ}" + if [ "$linuxobj" != "" ]; then + confoptions="$confoptions --with-linux-obj=${linuxobj}" fi - ./configure $confoptions ${CONFIGURE_FLAGS} + ./configure $confoptions ${CONFIGURE_FLAGS} 2>&1 if [ "$?" != "0" ]; then local saved_config="../config.log.$(date +%s)" cp config.log $saved_config chmod a+r $saved_config echo "Saved config.log is at $saved_config" + cat /proc/mounts + ls -l /proc/$$ + pwd + echo "config.log contents:" + cat config.log popd return 255 fi @@ -658,24 +839,33 @@ build_lustre() { echo NORPM mode. Only compiling. fi - # convert the $PATCHLESS boolean to an empty/no-empty boolean + # convert the $PATCHLESS boolean to an empty/not-empty boolean # as silly as this seems, it makes the syntax of the rpmbuild command - # simpler and not need an eval to deal with the quotes in the quotes + # simpler and not need an eval to deal with the quotes in the quotes local is_patchless="" if $PATCHLESS; then is_patchless="yes" fi + + # ditto for the lustre-tests boolean + local lustre_tests="" + if ! $LUSTRE_TESTS; then + lustre_tests="no" + fi + $RPMBUILD $targets $rpmbuildopt ../lustre.spec \ ${is_patchless:+--define "lustre_name lustre-client"} \ + ${lustre_tests:+--define "build_lustre_tests 0"} \ + --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ --define "_tmppath $TMPDIR" \ - --define "_topdir $TOPDIR" || \ + --define "_topdir $TOPDIR" 2>&1 || \ fatal 1 "Error building rpms for $BUILD_ARCHS." popd >/dev/null ( $(skeep_ldiskfs_rpm $TAG) ) && return pushd lustre/ldiskfs || return 255 - make dist + make dist 2>&1 if [ "$?" != "0" ]; then popd return 255 @@ -690,19 +880,17 @@ build_lustre() { < $ldiskfs_spec \ > ../lustre-ldiskfs.spec - $RPMBUILD $targets $rpmbuildopt ../lustre-ldiskfs.spec \ + if ! $RPMBUILD $targets $rpmbuildopt ../lustre-ldiskfs.spec \ --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" - if [ "$?" != "0" ]; then + --define "_topdir $TOPDIR" 2>&1; then popd return 255 fi if $DO_SRC; then - $RPMBUILD -bs ../lustre-ldiskfs.spec \ + if ! $RPMBUILD -bs ../lustre-ldiskfs.spec \ --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" - if [ "$?" != "0" ]; then + --define "_topdir $TOPDIR" 2>&1; then popd return 255 fi @@ -726,6 +914,7 @@ stage() { done cp -v "$LUSTRE" "$STAGEDIR" + } #check if we need to build separate ldiskfs RPM @@ -743,6 +932,11 @@ skeep_ldiskfs_rpm() { [[ $tag == $skiptag ]] && skip=true && break done fi + + pushd $TOPDIR/BUILD/lustre-[1-9]* >/dev/null + grep -q '^SERVER_TRUE[ \t]=[ \t]#$' autoMakefile && skip=true + popd >/dev/null + echo $skip } @@ -750,7 +944,7 @@ skeep_ldiskfs_rpm() { #generate LUSTRE_EXTRA_VERSION from EXTRA_VERSION gen_lustre_version() { - LUSTRE_EXTRA_VERSION="${lnxmaj}${EXTRA_VERSION_DELIMITER}${EXTRA_VERSION}${TARGET_DELIMITER}${RPMSMPTYPE:-${KERNCONFSMPTYPE:-"smp"}}" + LUSTRE_EXTRA_VERSION="${lnxmaj}-${EXTRA_VERSION}${FLAVOR_DELIMITER}${RPMSMPTYPE}" LUSTRE_EXTRA_VERSION=${LUSTRE_EXTRA_VERSION//-/_} } @@ -759,7 +953,7 @@ set_rpm_smp_type() { local infact_arch="${TARGET_ARCH}" - RPMSMPTYPE=default + RPMSMPTYPE="" [ "$infact_arch" == "i586" ] && infact_arch="i686" local smp_type @@ -771,6 +965,10 @@ set_rpm_smp_type() { [ $infact_arch == $smp_type ] && RPMSMPTYPE=bigsmp && break done + for smp_type in $DEFAULT_ARCHS; do + [ $infact_arch == $smp_type ] && RPMSMPTYPE=default && break + done + } # This function takes a linux source pool and digs out the linux release @@ -787,27 +985,29 @@ find_linux_release() { } +# XXX this needs to be re-written as a wrapper around find_rpm +# or just gotten rid of. :-) find_linux_rpm() { local prefix="$1" - local delimiter=${2:-"-"} local pathtorpms="${KERNELRPMSBASE}/${lnxmaj}/${DISTRO}" [ -d $pathtorpms ] || return 255 local kernelbinaryrpm rpmfile - local wanted_kernel="${lnxmaj}${delimiter}${lnxrel}" + local wanted_kernel="${lnxmaj}${lnxmin}-${lnxrel}" local arch ret=1 for arch in $TARGET_ARCHS_ALL; do local found_rpm="" rpm for rpm in ${pathtorpms}/${arch}/*.rpm; do - if rpm -q --provides -p "$rpm" | grep -q "kernel${prefix} = $wanted_kernel"; then + if rpm -q --provides -p "$rpm" 2>&3 | grep -q "kernel${prefix} = $wanted_kernel" 2>&3; then + found_rpm="$rpm" ret=0 break fi done - [ -f "$found_rpm" ] && TARGET_ARCH="$arch" && BUILD_ARCHS="$arch" && break + [ -f "$found_rpm" ] && break done echo "$found_rpm" @@ -815,104 +1015,108 @@ find_linux_rpm() { } -#unpack kernel(/source/devel) RPM +# unpack kernel(/source/devel) RPM # # This function and it's setting of $LINUX and $LINUXOBJ is a total hack that # needs to completely refactored. It completely ingores that $BUILD_ARCHS may # contain a list of arches for which rpmbuild commands (including the one for # lustre itself) -unpack_linux_rpm() { +unpack_linux_devel_rpm() { local kernelrpm="${1}" - local delimiter=${2:-"-"} [ -f "$kernelrpm" ] || return 255 - [ -d $TOPDIR/reused ] || mkdir $TOPDIR/reused + [ -d $TOPDIR/reused ] || mkdir $TOPDIR/reused || return 255 - pushd $TOPDIR/reused || return 255 + pushd $TOPDIR/reused &>/dev/null || return 255 - local RC=0 + if ! rpm2cpio < "$kernelrpm" | cpio -id > /dev/null 2>&1; then + return 255 + fi - rpm2cpio < "$kernelrpm" | cpio -id > /dev/null 2>&1 - if [ ${PIPESTATUS[0]} -eq 0 ]; then - # RHEL-style and SLES-style rpms - local paths="kernels/${lnxmaj}${delimiter}${lnxrel}-${TARGET_ARCH} linux-${lnxmaj}${delimiter}${lnxrel}" + # call a distro specific hook, if available + if type -p unpack_linux_devel_rpm-$DISTRO; then + if ! unpack_linux_devel_rpm-$DISTRO "$kernelrpm"; then + return 255 + fi + fi - local path - for path in $paths; do - local src='usr/src' + popd &>/dev/null - if [ -d "$src/$path/" ]; then - LINUX="$(pwd)/$src/$path" - fi - # SLES has a separate -obj tree - if [ -d "$src/${path}-obj" ]; then - local src="$src/${path}-obj" - local objects="$TARGET_ARCH/$RPMSMPTYPE" - - # Novell, are you *TRYING* to make life hard for me? - if [ -d "$src/powerpc" ]; then - objects="powerpc/$TARGET_ARCH" - elif [ $TARGET_ARCH == 'i686' ]; then - objects="i386/$RPMSMPTYPE" - fi + find_linux_devel_paths $TOPDIR/reused + + return 0 - LINUXOBJ="$(pwd)/$src/$objects" - fi - done - if [ -z "$LINUX" ]; then - RC=255 - else - # dig out the release version - LINUXRELEASE=$(find_linux_release ${LINUXOBJ:-$LINUX}) - if [ -z "$LINUXRELEASE" ]; then - echo "Failed to find linux release in ${LINUXOBJ:-$LINUX}" - RC=255 - fi - fi - else - RC=255 - fi - popd - return $RC } build_kernel_ib() { + local linux="$1" + # build kernel-ib{,-devel} - # some I/B drivers are architecture dependent and kernel-ib's configure - # does not figure it out for us ~sigh~ - local configure_options="" - case "$TARGET_ARCH" in - x86_64 | ia64) - configure_options="--with-ipath_inf-mod" - ;; - ppc64) - configure_options="--with-ipath_inf-mod --with-ehca-mod" - ;; - esac local K_SRC="K_SRC" # ofed 1.3 had a bug in the rpm spec if [ "$OFED_VERSION" = "1.3" ]; then K_SRC="KSRC" fi - $RPMBUILD --rebuild --define 'build_kernel_ib 1' --define 'build_kernel_ib_devel 1' \ - --define "_topdir ${TOPDIR}" --target ${TARGET_ARCH} \ - --define "KVERSION ${LINUXRELEASE}" \ - --define "$K_SRC ${LINUXOBJ:-${LINUX}}" \ - --define "LIB_MOD_DIR /lib/modules/${LINUXRELEASE}/updates" \ - --define "configure_options --without-quilt --with-core-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod --with-srp-target-mod --with-core-mod --with-mthca-mod --with-mlx4-mod --with-cxgb3-mod --with-nes-mod --with-ipoib-mod --with-sdp-mod --with-srp-mod --without-srp-target-mod --with-rds-mod --with-iser-mod --with-qlgc_vnic-mod --with-madeye-mod $configure_options" ${TOPDIR}/OFED/SRPMS/ofa_kernel-${OFED_VERSION}-ofed${OFED_VERSION}.src.rpm - if [ ${PIPESTATUS[0]} != 0 ]; then + local OFED_CORE="--with-core-mod --with-ipoib-mod --with-sdp-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod --with-rds-mod --with-qlgc_vnic-mod --with-madeye-mod" + local OFED_HARDWARE="--with-mthca-mod --with-mlx4-mod --with-mlx4_en-mod --with-cxgb3-mod --with-nes-mod" + # some I/B drivers are architecture dependent and kernel-ib's configure + # does not figure it out for us ~sigh~ + case "$TARGET_ARCH" in + ppc64) + OFED_HARDWARE="$OFED_HARDWARE --with-ehca-mod" + ;; + esac + # we're no longer shipping the OFED iSCSI + #OFED_ISCSI="--with-srp-mod --with-srp-target-mod" + ## ISER module has no backport support as of OFED 1.5 (i.e. only builds on + ##kernels >= 2.6.30) + #if [[ $OFED_VERSION = 1.[0-4]* ]]; then + # OFED_ISCSI="$OFED_ISCSI --with-iser-mod" + #fi + + # assume we are just rebuilding the SRPM + local BUILD_TYPE=${BUILD_TYPE:-"--rebuild"} + local SOURCE="${TOPDIR}/OFED/SRPMS/ofa_kernel-*.src.rpm" + + # but switch to building from the SPEC if we need to apply patches + if ls ${TOPDIR}/lustre/build/patches/ofed/* >/dev/null; then + BUILD_TYPE="-bb" + rpm --define "_topdir ${TOPDIR}" -ivh $SOURCE + SOURCE="${TOPDIR}/SPECS/ofa_kernel.spec" + local file ed_fragment1 ed_fragment2 n=1 + for file in ${TOPDIR}/lustre/build/patches/ofed/*; do + ed_fragment1="$ed_fragment1 +Patch$n: ${file%%*/}" + ed_fragment2="$ed_fragment2 +%patch$n -p0" + cp $file ${TOPDIR}/SOURCES + let n=$n+1 + done + + ed $SOURCE <&1; then fatal 1 "Error building kernel-ib" fi - pushd "$TOPDIR" >/dev/null - rm -rf kernel-ib-devel - mkdir kernel-ib-devel - cd kernel-ib-devel - local rpm=$(ls $TOPDIR/RPMS/*/kernel-ib-devel-${OFED_VERSION}-${LINUXRELEASE//-/_}.*.rpm) - rpm2cpio -itv < $rpm | cpio -id - CONFIGURE_FLAGS="--with-o2ib=$(pwd)/usr/src/ofa_kernel ${CONFIGURE_FLAGS}" - popd >/dev/null } store_for_reuse() { @@ -929,6 +1133,11 @@ store_for_reuse() { location="$location"/"$signature"/"$module" mkdir -p "$location" + # the cleanup script removes any directory that doesn't have a + # .lastused, so let's try to prevent that as soon as we can + # this solution still slightly racy with the cleanup script + # but the race is a lot tighter now + touch -t 197001010000 "$location/.lastused" ## use eval/echo here to make sure shell expansions are performed #if ! cp -a${linkflag} $(eval echo $articles) "$location"; then local article @@ -937,7 +1146,7 @@ store_for_reuse() { error "Failed to copy \"$article\" to \"$location\" in store_for_reuse()" # rename the cache location so that it's not cached # product, but is around for analysis - mv "$location"{,-bad-$(date +%s)} || + mv "$location"{,-bad-$(date +%s)} || error "failed to clean up a failed cache attempt" \ "in \"$location\" -- manual cleanup will be" \ "necessary" @@ -945,6 +1154,10 @@ store_for_reuse() { fi done + # flag the cache as complete (i.e. in case lbuild was previously + # interrupted while caching) + touch "$location/.lastused" + return 0 } @@ -955,9 +1168,19 @@ reuse() { local use_links="${3:-false}" local signature="$4" - if [ -n "$REUSEBUILD" ] && [ -d $REUSEBUILD/$signature/$module ]; then + if [ -n "$REUSEBUILD" ] && [ -d "$REUSEBUILD/$signature/$module" ]; then + if [ ! -f "$REUSEBUILD/$signature/$module/.lastused" ]; then + # the .lastused flag is populated at the end of the caching to + # signal that the caching was completed. if that flag is not + # there, then the cache is invalid (and should be removed in fact) + mv "$REUSEBUILD/$signature/$module"{,-bad-$(date +%s)} || + fatal 1 "failed to clean up a bad cache in location $REUSEBUILD/$signature/$module\" -- manual cleanup will be necessary" + return 1 + fi + # so that we know how stale this entry is touch $REUSEBUILD/$signature/$module/.lastused + if $use_links; then if ls $REUSEBUILD/$signature/$module/* >/dev/null 2>&1; then cp -al $REUSEBUILD/$signature/$module/* $dest/ @@ -988,7 +1211,7 @@ reuse() { find $dir -type f | xargs ln -t $dest/RPMS/${dir##*/} -s done - + fi return 0 else @@ -1007,60 +1230,120 @@ basearch() { } +# +# in a given directory, find the first rpm matching given requirements +# +find_rpm() { + local dir="$1" + local match_type="$2" + local match="$3" + + pushd "$dir" > /dev/null || \ + fatal 1 "Unable to chdir to directory \"$dir\" in find_rpm()" + + local file + for file in $(ls *.rpm); do + if [ ! -f "$file" ]; then + continue + fi + case "$match_type" in + provides) + # match is any valid ERE (i.e. given to egrep) match + if rpm -q --provides -p "$file" 2>&3 | egrep -q "$match"; then + echo "$file" + popd >/dev/null + return 0 + fi + ;; + *) + popd >/dev/null + fatal 1 "Unknown match type \"$match_type\" given to find_rpm()" + ;; + esac + done + + popd >/dev/null + return 1 +} + build_kernel_with_srpm() { + local outfd=$1 + + if [ -z "$outfd" ] || [ $outfd = 1 ]; then + fatal 1 "You must supply a file descriptor to ${FUNCNAME[0]} and it cannot be 1" + fi # need to generate the patch for this target - do_patch_linux false >&2 # sets global $FULL_PATCH (yeah, yuck) + do_patch_linux false >&${outfd} # sets global $FULL_PATCH (yeah, yuck) # get an md5sum of the kernel patch + config for reuse check # XXX really, there needs to be a signature and a CONFIG_FILE per arch # in BUILD_ARCHS - local REUSE_SIGNATURE=$(cat $CONFIG_FILE $TARGET_FILE $FULL_PATCH | md5sum | cut -d" " -f1) + local release_str + if $RELEASE; then + local release_str="RELEASE=$RELEASE\n" + fi - # see if we can link to the reuse pool - # XXX - hrm. i'm not convinced this doesn't belong in the reuse "library" - local CAN_LINK_FOR_REUSE=false - touch $REUSEBUILD/$$ - if cp -al $REUSEBUILD/$$ $TOPDIR/; then - CAN_LINK_FOR_REUSE=true + if $USE_BUILD_CACHE && [ -n "$REUSEBUILD" ]; then + local REUSE_SIGNATURE=$({ echo -en $release_str; + echo $BUILD_GEN; + cat $CONFIG_FILE $TARGET_FILE $FULL_PATCH; } | + md5sum | cut -d" " -f1) + # see if we can link to the reuse pool + # XXX - hrm. i'm not convinced this doesn't belong in the reuse + # "library" + local CAN_LINK_FOR_REUSE=false + touch $REUSEBUILD/$$ + if cp -al $REUSEBUILD/$$ $TOPDIR/ 2>/dev/null; then + CAN_LINK_FOR_REUSE=true + fi + rm $REUSEBUILD/$$ fi - rm $REUSEBUILD/$$ # the extra version string to use for the kernel (which might be a reused # kernel, remember) local kernel_extra_version="" - if $REUSERPM && reuse kernel "$TOPDIR" "$CAN_LINK_FOR_REUSE" \ - "$REUSE_SIGNATURE"; then - # figure out the EXTRA_VERSION of the kernel we are re-using - kernel_extra_version=$(ls $TOPDIR/RPMS/$TARGET_ARCH/kernel-lustre-*${lnxmaj}${EXTRA_VERSION_DELIMITER}${PRISTINE_EXTRA_VERSION}.*.$TARGET_ARCH.rpm) - kernel_extra_version=${kernel_extra_version##*kernel-lustre-*${lnxmaj}${EXTRA_VERSION_DELIMITER}} - kernel_extra_version=${kernel_extra_version%%.$TARGET_ARCH.rpm} - else + if ! $USE_BUILD_CACHE || ! reuse kernel "$TOPDIR" "$CAN_LINK_FOR_REUSE" \ + "$REUSE_SIGNATURE"; then # nothing cached, build from scratch if [ ! -r "$KERNELDIR/$KERNEL_SRPM" ]; then - download_srpm "$CANONICAL_TARGET" "$KERNEL_SRPM" + echo "Downloading kernel SRPM" + download_srpm "$CANONICAL_TARGET" "$KERNEL_SRPM" >&${outfd} fi - rpm -ivh $KERNELDIR/$KERNEL_SRPM --define "_topdir $TOPDIR" >&2 || { - # should we clean this up or leave it for analysis? + if ! rpm -ivh $KERNELDIR/$KERNEL_SRPM \ + --define "_topdir $TOPDIR" >&${outfd} 2>&1; then + # should we clean this up or leave it for analysis? #rm -rf $RPMTOPDIR fatal 1 "Error installing kernel SRPM." - } + fi # put the Lustre kernel patch into the RPM build tree cp $FULL_PATCH $TOPDIR/SOURCES/linux-${lnxmaj}-lustre.patch - prepare_and_build_srpm - - # store the resulting kernel RPM build tree for future use - if ! store_for_reuse "$TOPDIR/{SPECS,SOURCES,BUILD,SRPMS,RPMS}" \ - "kernel" "$REUSEBUILD" "$REUSE_SIGNATURE" \ - "$CAN_LINK_FOR_REUSE"; then - error "Failed to store kernel RPMS for reuse" - echo "unknown" - return 1 + prepare_and_build_srpm >&${outfd} || + fatal 1 "failed to prepare_and_build_srpm" + + if [ -z "$REUSE_SIGNATURE" ]; then + echo "No reuse signature was caculated so not storing the built kernel" >&${outfd} + else + # store the resulting kernel RPM build tree for future use + echo "Storing the built kernel for future reuse" >&${outfd} + if ! store_for_reuse "$TOPDIR/{SPECS,SOURCES,BUILD,SRPMS,RPMS}" \ + "kernel" "$REUSEBUILD" "$REUSE_SIGNATURE" \ + "$CAN_LINK_FOR_REUSE"; then + error "Failed to store kernel RPMS for reuse" + echo "unknown" + return 1 + fi fi - kernel_extra_version=$EXTRA_VERSION - fi # build reuse + fi # build reuse + + # figure out the EXTRA_VERSION of the kernel we built or are re-using + local KERNEL_RPM + if ! KERNEL_RPM=$(find_rpm "$TOPDIR/RPMS/$TARGET_ARCH/" provides "^kernel ="); then + fatal 1 "Failed to find a kernel RPM in $TOPDIR/RPMS/$TARGET_ARCH/" + fi + kernel_extra_version=$(rpm -q --queryformat "%{RELEASE}" -p $TOPDIR/RPMS/$TARGET_ARCH/$KERNEL_RPM) # should now have the following RPMs # $TOPDIR/RPMS/$arch/kernel-lustre-2.6.18-53.1.21.el5_lustre.1.6.5.1.$arch.rpm @@ -1074,59 +1357,161 @@ build_kernel_with_srpm() { } -build_with_srpm() { +build_mptlinux() { + local linux="$1" + local version="$2" - if ! $PATCHLESS; then - local kernel_extra_version - if ! kernel_extra_version=$(build_kernel_with_srpm); then - fatal 1 "Failed to build the kernel from it's SRPM" - fi -#local kernel_extra_version="60-0.33_lustre.1.8.0.50.20090311172757" + if [ -z "$version" -o "$version" = "inkernel" ]; then + return 0 + fi - for arch in $BUILD_ARCHS; do + local targets arch + for arch in $BUILD_ARCHS; do + targets="--target $arch $targets" + done - local kernel_devel_rpm="$TOPDIR/RPMS/$arch/$(devel_kernel_name true)-$lnxmaj$EXTRA_VERSION_DELIMITER$kernel_extra_version.$arch.rpm" + local rpmbuildopt='-bb' + if $NORPM; then + rpmbuildopt='-bc' + echo NORPM mode. Only compiling. + fi - # install the -devel RPM in preparation for the lustre build - if ! lnxrel="$kernel_extra_version" unpack_linux_rpm \ - $kernel_devel_rpm $EXTRA_VERSION_DELIMITER; then - fatal 1 "Could not find the Linux tree in $kernel_devel_rpm" - fi + # if only we could just rebuild the src.rpm. but the included spec + # is a real pig's breakfast. just check out the patch we need to + # apply to it to make it useful. + #$RPMBUILD --rebuild \ + # --define "_topdir ${TOPDIR}" --target ${TARGET_ARCH} \ + # ${TOPDIR}/mptlinux-*.src.rpm + rpm -ivh --define "_topdir ${TOPDIR}" ${TOPDIR}/mptlinux-*.src.rpm + + # now the big honkin' patch to the spec file + pushd ${TOPDIR}/SPECS + # to regen this patch use: + # !!cd ~/rpm/SPECS/ && diff -u mptlinux.spec{.dist,} + patch -p0 < ${TOPDIR}/lustre/build/mptlinux.spec.patch || fatal 1 "failed to patch mptlinux.spec" + popd - # XXX - superhack of all superhacks! kernel-lustre-devel doesn't - # have have sources in it, so we need to pull them out of - # the SRPM - # yeah. blech. - cp BUILD/kernel-lustre-${lnxmaj}/linux-${lnxmaj}.$arch/fs/ext3/*.[ch] \ - $TOPDIR/reused/usr/src/kernels/${lnxmaj}${EXTRA_VERSION_DELIMITER}${kernel_extra_version}-${arch}/fs/ext3 - done - else - # need to find and unpack the vendor's own kernel-devel for patchless - # client build - local kernelrpm - if ! kernelrpm=$(find_linux_rpm "-$DEVEL_KERNEL_TYPE" ${EXTRA_VERSION_DELIMITER:-"-"}); then - fatal 1 "Could not find the kernel-$DEVEL_KERNEL_TYPE RPM in ${KERNELRPMSBASE}/${lnxmaj}/${DISTRO}" + local targets arch + for arch in $BUILD_ARCHS; do + targets="--target $arch $targets" + done + + local rpmbuildopt='-bb' + if $NORPM; then + rpmbuildopt='-bc' + echo NORPM mode. Only compiling. + fi + + if ! $RPMBUILD $targets $rpmbuildopt \ + --define "_tmppath /var/tmp" \ + --define "_topdir ${TOPDIR}" \ + --define "kernel_obj $linux" \ + ${TOPDIR}/SPECS/mptlinux.spec 2>&1; then + return 1 + fi + if $DO_SRC; then + if ! $RPMBUILD -bs \ + --define "_tmppath /var/tmp" \ + --define "_topdir ${TOPDIR}" \ + --define "kernel_obj $linux" \ + ${TOPDIR}/SPECS/mptlinux.spec 2>&1; then + return 1 fi - if ! lnxrel="$lnxrel" unpack_linux_rpm "$kernelrpm" \ - "${EXTRA_VERSION_DELIMITER:--}"; then - fatal 1 "Could not find the Linux tree in $kernelrpm" + fi + + return 0 + +} + +# build RDAC +build_rdac() { + local linux="$1" + local version="$2" + + if [ -z "$version" -o "$version" = "inkernel" ]; then + return 0 + fi + + # note that we use an _, not a . before the spec on purpose. we are not + # allowed to have more than one file with a .spec trailer in a tarball + # that is supposed to be usable with rpmbuild + cp lustre/build/rdac_spec ${TOPDIR}/SPECS/rdac.spec || fatal 1 "Could not find rdac.spec in lustre/build" + + local targets arch + for arch in $BUILD_ARCHS; do + targets="--target $arch $targets" + done + + local rpmbuildopt='-bb' + if $NORPM; then + rpmbuildopt='-bc' + echo NORPM mode. Only compiling. + fi + + local distro + case $DISTRO in + rhel5) distro="REDHAT" + ;; + sles1*) distro="SUSE" + ;; + *) echo "$DISTRO not supported by RDAC, skipping" + return 0 + ;; + esac + if ! $RPMBUILD $targets $rpmbuildopt --define "dist $distro" \ + --define "_tmppath /var/tmp" \ + --define "_topdir ${TOPDIR}" \ + --define "kernel_obj $linux" \ + ${TOPDIR}/SPECS/rdac.spec 2>&1; then + return 1 + fi + if $DO_SRC; then + if ! $RPMBUILD -bs --define "dist $distro" \ + --define "_tmppath /var/tmp" \ + --define "_topdir ${TOPDIR}" \ + --define "kernel_obj $linux" \ + ${TOPDIR}/SPECS/rdac.spec 2>&1; then + return 1 fi fi + return 0 +} + +# build OFED +# globals used: +# TOPDIR +# REUSEBUILD, USE_BUILD_CACHE +# CONFIGURE_FLAGS + +build_ofed() { + local linux="$1" + local ofed_version="$2" + # before lustre, build kernel-ib - if [ -n "$OFED_VERSION" -a "$OFED_VERSION" != "inkernel" ]; then + if [ -z "$ofed_version" -o "$ofed_version" = "inkernel" ]; then + return 0 + fi + + if $USE_BUILD_CACHE && [ -n "$REUSEBUILD" ]; then + local REUSE_SIGNATURE=$({ echo "$ofed_version"; + echo "$(find_linux_release ${linux})"; + cat "${linux}/include/linux/autoconf.h"; } | + md5sum | cut -d" " -f1) # see if we can link to the reuse pool - # XXX - hrm. i'm not convinced this doesn't belong in the reuse "library" + # XXX - hrm. i'm not convinced this doesn't belong in the reuse + # "library" local CAN_LINK_FOR_REUSE=false touch $REUSEBUILD/$$ if cp -al $REUSEBUILD/$$ $TOPDIR/; then CAN_LINK_FOR_REUSE=true fi rm $REUSEBUILD/$$ + fi - local REUSE_SIGNATURE=$({ echo "$OFED_VERSION"; echo "$(find_linux_release ${LINUXOBJ:-$LINUX})"; cat "${LINUXOBJ:-${LINUX}}/include/linux/autoconf.h"; } | md5sum | cut -d" " -f1) - if ! reuse ofed "$TOPDIR" "$CAN_LINK_FOR_REUSE" \ - "$REUSE_SIGNATURE"; then + if ! $USE_BUILD_CACHE || ! reuse ofed "$TOPDIR" "$CAN_LINK_FOR_REUSE" \ + "$REUSE_SIGNATURE"; then + if [ -n "$REUSE_SIGNATURE" ]; then # stash away the existing built articles for a moment mkdir bak mv {BUILD,{S,}RPMS,S{OURCE,PEC}S} bak @@ -1137,8 +1522,15 @@ build_with_srpm() { rm -rf bak } create_rpmbuild_dirs - # build it - build_kernel_ib + fi + # build it + build_kernel_ib "${linux}" + + if [ -z "$REUSE_SIGNATURE" ]; then + echo "No reuse signature was caculated so not storing the built ofed" + else + # store the resulting RPM build tree for future use + echo "Storing the built ofed for future reuse" if ! store_for_reuse "$TOPDIR/{SPECS,SOURCES,BUILD,SRPMS,RPMS}" \ "ofed" "$REUSEBUILD" "$REUSE_SIGNATURE" \ "$CAN_LINK_FOR_REUSE"; then @@ -1151,11 +1543,71 @@ build_with_srpm() { fi fi + pushd "$TOPDIR" >/dev/null + rm -rf kernel-ib-devel + mkdir kernel-ib-devel + cd kernel-ib-devel + # the actual ofed RPMs don't have the -rc$n or -$date string appened that + # might be present on the file + local linuxrelease=$(find_linux_release "$linux") + ofed_version=$(echo $ofed_version | + sed -re 's/-(20[0-9]{6,6}-[0-9]{4,4}|rc[0-9]*)$//') + local rpm=$(ls $TOPDIR/RPMS/*/kernel-ib-devel-${ofed_version}-${linuxrelease//-/_}.*.rpm) + rpm2cpio -itv < $rpm | cpio -id + CONFIGURE_FLAGS="--with-o2ib=$(pwd)/usr/src/ofa_kernel ${CONFIGURE_FLAGS}" + popd >/dev/null + +} + +build_with_srpm() { + + if ! $PATCHLESS; then + local kernel_extra_version + if ! kernel_extra_version=$(build_kernel_with_srpm ${STDOUT}); then + fatal 1 "Failed to build the kernel from it's SRPM" + fi + + for arch in $BUILD_ARCHS; do + local kernel_devel_rpm + if ! kernel_devel_rpm=$(find_rpm "$TOPDIR/RPMS/$arch/" provides "^$(devel_kernel_name $KERNEL_LUSTRE_NAMING) ="); then + fatal 1 "Failed to find a kernel development RPM in $TOPDIR/RPMS/$arch/" + fi + + # install the -devel RPM in preparation for the lustre build + if ! lnxrel="$kernel_extra_version" unpack_linux_devel_rpm \ + "$TOPDIR/RPMS/$arch/$kernel_devel_rpm"; then + fatal 1 "Could not find the Linux tree in $TOPDIR/RPMS/$arch/$kernel_devel_rpm" + fi + done + else + # need to find and unpack the vendor's own kernel-devel for patchless + # client build + local kernelrpm + if ! kernelrpm=$(find_linux_rpm "-$DEVEL_KERNEL_TYPE"); then + fatal 1 "Could not find the kernel-$DEVEL_KERNEL_TYPE RPM in ${KERNELRPMSBASE}/${lnxmaj}/${DISTRO}" + fi + if ! lnxrel="$lnxrel" unpack_linux_devel_rpm "$kernelrpm" "-"; then + fatal 1 "Could not find the Linux tree in $kernelrpm" + fi + fi + + build_ofed "${LINUXOBJ:-$LINUX}" "$OFED_VERSION" || + fatal 1 "error building OFED" + + if ! $PATCHLESS; then + # only need RDAC for the server + build_rdac "${LINUXOBJ:-$LINUX}" "$RDAC_VERSION" || + fatal 1 "error building RDAC" + fi + + build_mptlinux "${LINUXOBJ:-$LINUX}" "$MPTLINUX_VERSION" || + fatal 1 "error building mptlinux" + # now build Lustre - if build_lustre; then + if build_lustre "$LINUX" "$LINUXOBJ"; then # the build worked. resolve any symlinked files (i.e. from reuse) # in RPMS/$arch to real files so that that that huge mess of - # complication knows as LTS can copy them yet somewhere else. + # complication known as LTS can copy them yet somewhere else. # is it any wonder this whole process is so damn so? anyone ever # heard of hardlinks? it this cool new thing that allows you save # tons of time and space by creating... well you can go read about @@ -1170,17 +1622,23 @@ build_with_srpm() { fi done popd + else + return 1 fi + } create_rpmbuild_dirs() { - if [ ! -d RPMS ]; then - mkdir -p RPMS - for arch in $BUILD_ARCHS; do - mkdir RPMS/$arch - done - fi + [ -d RPMS ] || mkdir RPMS + for arch in $BUILD_ARCHS; do + if [[ $arch = i?86 ]]; then + # some stupidity in the sles11 kernel spec requires an RPMS/i386 + # even if the target arch is i686 + [ -d RPMS/i386 ] || mkdir RPMS/i386 + fi + [ -d RPMS/$arch ] || mkdir RPMS/$arch + done [ -d BUILD ] || mkdir BUILD [ -d SOURCES ] || mkdir SOURCES [ -d SPECS ] || mkdir SPECS @@ -1226,6 +1684,9 @@ backtrace() { funcname=${FUNCNAME[$n - 1]} sourcefile=$(basename ${BASH_SOURCE[$n]}) lineno=${BASH_LINENO[$n - 1]} + if [ $n = 1 ]; then + let lineno-=11 + fi # Display function arguments if [[ ! -z "${BASH_ARGV[@]}" ]]; then local args newarg j p=0 @@ -1234,7 +1695,7 @@ backtrace() { args="${args:+${args} }'${newarg}'" done let p+=${BASH_ARGC[$n - 1]} - fi + fi echo " ${funcname} ${args:+${args} }at ${sourcefile}:${lineno}" done @@ -1246,7 +1707,11 @@ backtrace() { #echo ${FUNCNAME[*]} local i=$((${#FUNCNAME[@]} - 1)) while [ $i -ge 0 ]; do - local SOURCELINE="${BASH_SOURCE[$i + 1]}:${BASH_LINENO[$i]}" + local lineno=${BASH_LINENO[$i]} + if [ $i = 0 ]; then + let lineno-=11 + fi + local SOURCELINE="${BASH_SOURCE[$i + 1]}:${lineno}" # Can't figure out how to get function args from other frames... local FUNCTION="${FUNCNAME[$i]}()" echo "$SOURCELINE:$FUNCTION" @@ -1259,9 +1724,30 @@ backtrace() { } +seen_list=$(new_list) +trap 'set +x; +echo "An unexpected error has occurred at ${BASH_SOURCE[0]##*/}:$((LINENO-1)). +Unfortunately the above line number in the message may or may not be correct, +but details have been send to the lbuild maintainer. Attempting to continue."; (echo "Untrapped error" +echo +# have we seen this one +echo "checking seen list for ${BASH_SOURCE[0]}:${BASH_LINENO[0]}" + +if is_list_member "$seen_list" "${BASH_SOURCE[0]}:${BASH_LINENO[0]}"; then + echo "seen this one already" +else + seen_list=$(add_list "$seen_list" "${BASH_SOURCE[0]}:${BASH_LINENO[0]}") +fi +backtrace +echo +echo "Environment:" +set +) | mail -s "Untrapped error at ${BASH_SOURCE[0]##*/}:$((LINENO-15)) on $HOSTNAME" brian@sun.com >&2; set $xtrace' ERR +set -E + [ -r ~/.lbuildrc ] && . ~/.lbuildrc -options=$(getopt -o d:D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ldiskfs,ccache,reuse:,norpm,disable-datestamp,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,publish,release,src,stage:,tag:,target:,target-archs:,with-linux:,xen -- "$@") +options=$(getopt -o d:D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ldiskfs,ccache,reuse:,norpm,disable-datestamp,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,ofed-version:,publish,release,set-value:,src,stage:,tag:,target:,target-archs:,with-linux:,xen -- "$@") if [ $? != 0 ]; then usage 1 @@ -1361,6 +1847,10 @@ while [ "$1" ]; do DO_SRC=false shift 1 ;; + --ofed-version) + OFED_VERSION="$2" + shift 2 + ;; --publish) shift ;; @@ -1396,8 +1886,17 @@ while [ "$1" ]; do XEN=true shift ;; + --set-value) + eval $2 + shift 2 + ;; --) shift + # there are actually some lustre configure flags that we need to + # handle ourselves (but we still give them to configure) + if [[ \ $@\ == *\ --disable-tests\ * ]]; then + LUSTRE_TESTS=false + fi CONFIGURE_FLAGS=$@ CONFIGURE_FLAGS="$CONFIGURE_FLAGS --enable-liblustre --enable-liblustre-tests" break @@ -1413,46 +1912,47 @@ check_options unpack_lustre load_target -EXTRA_VERSION_DELIMITER=${EXTRA_VERSION_DELIMITER:-"-"} if [ -n "$OFED_VERSION" -a "$OFED_VERSION" != "inkernel" ]; then download_ofed - unpack_ofed + unpack_ofed || fatal 1 "Error unpacking OFED tarball" +fi + +if [ -n "$MPTLINUX_VERSION" -a "$MPTLINUX_VERSION" != "inkernel" ]; then + download_mptlinux + unpack_mptlinux || fatal 1 "Error unpacking MPTLINUX distribution" fi # make sure the RPM build environment is set up create_rpmbuild_dirs -trap '[ -n "$CCACHE" ] && ccache -s' EXIT +if [ -n "$RDAC_VERSION" -a "$RDAC_VERSION" != "inkernel" ]; then + download_rdac + # we don't actually need to unpack this. just put it in the SOURCES dir + #unpack_rdac || fatal 1 "Error unpacking RDAC tarball" + cp "$KERNELTREE/rdac-LINUX-${RDAC_VERSION}-source.tar.gz" ${TOPDIR}/SOURCES/ || + fatal 1 "Error copying RDAC source tarball to RPM SOURCES dir" +fi # if an unpacked kernel source tree was given on the command line # just build lustre with it (nothing distro kernel specific here) if [ -n "$LINUX" ]; then - build_lustre + build_mptlinux "${LINUXOBJ:-$LINUX}" "$MPTLINUX_VERSION" || + fatal 1 "error building mptlinux" + build_ofed "${LINUXOBJ:-$LINUX}" "$OFED_VERSION" || + fatal 1 "error building OFED" + if ! $PATCHLESS; then + build_rdac "${LINUXOBJ:-$LINUX}" "$RDAC_VERSION" || + fatal 1 "error building RDAC" + fi + build_lustre "$LINUX" "$LINUXOBJ" else if [ -f "${0%/*}/lbuild-$DISTRO" ]; then - seen_list=$(new_list) - trap '(echo "Untrapped error" -echo -# have we seen this one -echo "checking seen list for ${BASH_SOURCE[0]}:${BASH_LINENO[0]}" - -if is_list_member "$seen_list" "${BASH_SOURCE[0]}:${BASH_LINENO[0]}"; then - echo "seen this one already" -else - seen_list=$(add_list "$seen_list" "${BASH_SOURCE[0]}:${BASH_LINENO[0]}") -fi -backtrace -echo -echo "Environment:" -set -) | tee >(mail -s "Untrapped error at ${BASH_SOURCE[0]##*/}:${BASH_LINENO[0]} on $HOSTNAME" brian@sun.com) >&2' ERR - set -E - source ${0%/*}/lbuild-$DISTRO build_with_srpm || fatal 1 "Failed to build_with_srpm" else + EXTRA_VERSION_DELIMITER=${EXTRA_VERSION_DELIMITER:-"-"} source ${0%/*}/lbuild.old_school old_school_download_kernel