X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=build%2Flbuild;h=f285ea6e8c177a8242945220bf154690a6446fa9;hp=c0389a27ea2d34abeb97298ce6a34b984c9fde00;hb=e5d626bc315e76e4f6d66d57cabfaeb948bce856;hpb=269fd8247cb9f264225de46f1d8af30fc68c55be diff --git a/build/lbuild b/build/lbuild index c0389a2..f285ea6 100755 --- a/build/lbuild +++ b/build/lbuild @@ -1,4 +1,9 @@ -#!/bin/sh +#!/bin/bash + +# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4: + +#set -x +shopt -s extdebug TOPDIR=$PWD @@ -6,28 +11,48 @@ TOPDIR=$PWD KERNELDIR= LINUX= LUSTRE= -RELEASE=0 -DO_SRC=0 -DOWNLOAD=1 +RELEASE=false +# XXX - some recent hacking has pretty much neutered this option. +# search through this file (and lbuild.old_school -- but that will +# be going away soon) for "-bb" and see how many places +# simply don't account for this option +DO_SRC=false +DOWNLOAD=true TAG= +CANONICAL_TARGET= TARGET= +TARGET_ARCH=$(uname -m) TARGET_ARCHS= +TARGET_ARCHS_ALL=$TARGET_ARCH +[ "$TARGET_ARCH" = "i686" ] && TARGET_ARCHS_ALL="i686 i586 i386" CONFIGURE_FLAGS= EXTERNAL_PATCHES= EXTRA_VERSION= +LUSTRE_EXTRA_VERSION= STAGEDIR= +TMPDIR=${TMPDIR:-"/var/tmp"} +TIMESTAMP= +# XXX - i think these two parameters/arguments/variables need to be +# cleaned up and merged. they effectively do the same thing +REUSERPM= +REUSEBUILD= +# what does this do exactly? does it imply no kernel build? +NORPM=false +LDISKFSRPM=true +SKIPLDISKFSRPM="v1_4_* b1_4" +SMPTYPES="smp bigsmp default ''" +PATCHLESS=false +XEN=false +LINUXOBJ= +DISTRO= +KERNELTREE= + +# patchless build +KERNELRPMSBASE= +RPMSMPTYPE= # from target file -KERNEL= SERIES= -CONFIG= -VERSION= - -RHBUILD=0 -SUSEBUILD=0 -LINUX26=0 -SUSEBUILD=0 - BASE_ARCHS= BIGMEM_ARCHS= BOOT_ARCHS= @@ -37,11 +62,16 @@ BIGSMP_ARCHS= PSERIES64_ARCHS= UP_ARCHS= +# not in the target file any more +CONFIG= + DATE=$(date) USE_DATESTAMP=1 RPMBUILD= +OLD_SCHOOL=false + export CC=${CC:-gcc} # Readlink is not present on some older distributions: emulate it. @@ -49,47 +79,34 @@ readlink() { local path=$1 ll if [ -L "$path" ]; then - ll="$(LC_ALL=C ls -l "$path" 2> /dev/null)" && - echo "${ll/* -> }" + ll="$(LC_ALL=C ls -l "$path" 2> /dev/null)" && + echo "${ll/* -> }" else - return 1 + return 1 fi } -cleanup() -{ +cleanup() { + true } -error() -{ - [ "$1" ] && echo -e "\n${0##*/}: $1" +error() { + local msg="$1" + + [ -n "$msg" ] && echo -e "\n${0##*/}: $msg" >&2 + } -fatal() -{ +fatal() { + cleanup error "$2" exit $1 -} -is_release() -{ - (( $RELEASE )) || return 0 -} - -list_targets() -{ - echo -n "Available targets:" - for target in $TOPDIR/lustre/lustre/kernel_patches/targets/*.target ; do - target_file=${target##*/} - echo -n " ${target_file%%.target}" - done - echo } -usage() -{ +usage() { cat <] @@ -106,11 +123,39 @@ Usage: ${0##*/} [OPTION]... [-- ] --extraversion=EXTRAVERSION Text to use for the rpm release and kernel extraversion. + --timestamp=TIMESTAMP + Date of building lustre in format YYYYMMDDhhmmss + + --reuserpm=DIR + Try to reuse old kernel RPMs from DIR + + --reusebuild=DIR + Try to reuse old kernel builds from DIR + + --kernelrpm=DIR + Path to distro kernel RPM collection + + --ccache + Use ccache + + --norpm + Do not build RPMs (compile only mode) + + --patchless + Build lustre client only + + --distro=DISTRO + Which distro using. Autodetect by default + --kerneldir=KERNELDIR Directory containing Linux source tarballs referenced by target files. - --linux=LINUX + --kerneltree=KERNELTREE + Directory containing dirs with Linux source tarballs referenced by target + files. Dir names in format kernel version ('2.6.9', etc.) + + --linux=LINUX --with-linux=LINUX Directory of Linux kernel sources. When this option is used, only Lustre modules and userspace are built. @@ -119,12 +164,15 @@ Usage: ${0##*/} [OPTION]... [-- ] pulling from CVS. --nodownload - Do not try to download a kernel from ftp.lustre.org + Do not try to download a kernel from downloads.lustre.org --nosrc Do not build a .src.rpm, a full kernel patch, or a patched kernel tarball. + --ldiskfs + Do ldiskfs RPM. Now true by default + --publish Unused. @@ -158,10 +206,13 @@ Usage: ${0##*/} [OPTION]... [-- ] will be built - there will be no lustre-lite-utils package. --disable-datestamp - Prevents the datestamp flag (-D) from being passed to cvs for - checkouts. This is a workaround for a problem encountered when + Prevents the datestamp flag (-D) from being passed to cvs for + checkouts. This is a workaround for a problem encountered when using lbuild with tinderbox. + --xen + Builds a Xen domX kernel. + EOF # list_targets @@ -169,467 +220,1295 @@ EOF fatal "$1" "$2" } -check_options() -{ - if [ "$LUSTRE" ] ; then - [ -r "$LUSTRE" ] || \ - usage 1 "Could not find Lustre source tarball '$LUSTRE'." +# canonicalize a relative path +canon_path() { + local PATH="$1" + + if [ ! -d "$PATH" ]; then + return 1 + fi + + pushd "$PATH" >/dev/null || return 1 + local CANONPATH=$PWD + popd >/dev/null + + echo "$CANONPATH" + return 0 +} + +check_options() { + + if [ "$LUSTRE" ]; then + [ -r "$LUSTRE" ] || \ + usage 1 "Could not find Lustre source tarball '$LUSTRE'." else - [ "$CVSROOT" ] || \ - usage 1 "Either specify a CVS Root with -d, or a Lustre source tarball with --lustre." - [ "$TAG" ] || \ - usage 1 "A branch/tag name must be specified with --tag when not building from a tarball." + [ "$CVSROOT" ] || \ + usage 1 "Either specify a CVS Root with -d, or a Lustre source tarball with --lustre." + [ "$TAG" ] || \ + usage 1 "A branch/tag name must be specified with --tag when not building from a tarball." fi - if [ -z "$LINUX" ] ; then - [ "$KERNELDIR" ] || \ - usage 1 "A kernel directory must be specified with --kerneldir." + if [ -z "$LINUX" ]; then + [ "$KERNELDIR" -o "$KERNELTREE" ] || \ + usage 1 "A kernel directory must be specified with --kerneldir or --kerneltree." - [ -d "$KERNELDIR" ] || \ - usage 1 "$KERNELDIR is not a directory." + [ -d "$KERNELDIR" -o -d "$KERNELTREE" ] || \ + usage 1 "$KERNELDIR and $KERNELTREE are not a directory." - if ! (( $RELEASE )) ; then - [ "$TAG" ] || \ - usage 1 "When building a snapshot, a tag name must be used." - fi + if ! $RELEASE; then + [ "$TAG" ] || \ + usage 1 "When building a snapshot, a tag name must be used." + fi - [ "$TARGET" ] || usage 1 "A target must be specified with --target." + [ "$TARGET" ] || usage 1 "A target must be specified with --target." # TARGET_FILE="$TOPDIR/lustre/kernel_patches/targets/$TARGET.target" # [ -r "$TARGET_FILE" ] || \ -# usage 1 "Target '$TARGET' was not found." +# usage 1 "Target '$TARGET' was not found." fi case $TARGET in - 2.6-rhel4) - CANONICAL_TARGET="rhel-2.6" - ;; - 2.6-suse) - CANONICAL_TARGET="suse-2.6" - ;; - hp_pnnl-2.4) - CANONICAL_TARGET="hp-pnnl-2.4" - ;; - 2.6-vanilla \ - | suse-2.4.21-2 \ - | rh-2.4 \ - | rhel-2.4 \ - | sles-2.4) - CANONICAL_TARGET="$TARGET" - ;; + 2.6-rhel5) + CANONICAL_TARGET="rhel5" + ;; + 2.6-rhel4) + CANONICAL_TARGET="rhel-2.6" + ;; + 2.6-suse) + CANONICAL_TARGET="sles-2.6" + ;; + 2.6-sles10) + CANONICAL_TARGET="sles10-2.6" + ;; + 2.6-sles11) + CANONICAL_TARGET="sles11" + ;; + hp_pnnl-2.4) + CANONICAL_TARGET="hp-pnnl-2.4" + ;; + 2.6-vanilla \ + | suse-2.4.21-2 \ + | rh-2.4 \ + | rhel-2.4 \ + | sles-2.4 \ + | 2.6-patchless) + CANONICAL_TARGET="$TARGET" + ;; esac - TIMESTAMP=$(date -d "$DATE" "+%Y%m%d%H%M") + local timestampnodig=$(echo $TIMESTAMP | sed -e s/[0-9]*//g) + [ "$timestampnodig" = "" ] || TIMESTAMP=$(date -d "$DATE" "+%Y%m%d%H%M%S") + local timestamplength="${#TIMESTAMP}" + if [ $timestamplength -eq 12 ]; then + TIMESTAMP="${TIMESTAMP}00" + elif [ $timestamplength -ne 14 ]; then + TIMESTAMP=$(date -d "$DATE" "+%Y%m%d%H%M%S") + fi RPMBUILD=$(which rpmbuild 2>/dev/null | head -1) if [ ! "$RPMBUILD" -o "$RPMBUILD" == "" ]; then - RPMBUILD=$(which rpm 2>/dev/null | head -1) - if [ ! "$RPMBUILD" -o "$RPMBUILD" == "" ]; then - usage 1 "Could not find binary for making rpms (tried rpmbuild and rpm)." - fi + RPMBUILD=$(which rpm 2>/dev/null | head -1) + if [ ! "$RPMBUILD" -o "$RPMBUILD" == "" ]; then + usage 1 "Could not find binary for making rpms (tried rpmbuild and rpm)." + fi + fi + + if [ -n "$CCACHE" ]; then + which "$DISTCC" 2>/dev/null && export DISTCC RPM_BUILD_NCPUS + + if which "$CCACHE" 2>/dev/null; then + local ccache=$(which "$CCACHE") + local bindir="$TOPDIR/bin" + + [ -d $bindir ] || mkdir -p $bindir + if [ -d $bindir ]; then + rm ${bindir}/* > /dev/null 2>&1 + ln -s "$ccache" ${bindir}/ccache + ln -s "$ccache" ${bindir}/cc + ln -s "$ccache" ${bindir}/$CC + export PATH=$bindir:$PATH + fi + export CCACHE && export CC="ccache $CC" + # zero the cache so we can see how effective we are being with it + ccache -z + fi + fi + + [ -z "$DISTRO" ] && DISTRO=$(autodetect_distro) +} + +# autodetect used Distro +autodetect_distro() { + + local name + local version + + if [ -f /etc/SuSE-release ]; then + name=sles + version=$(grep ^VERSION /etc/SuSE-release) + version=${version#*= } + elif [ -f /etc/redhat-release ]; then + name=$(head -1 /etc/redhat-release) + version=$(echo "$distroname" | + sed -e 's/^[^0-9.]*//g' | sed -e 's/[ \.].*//') + fi + if [ -z "$name" -o -z "$version" ]; then + fatal 1 "I don't know how to determine distro type/version.\n" \ + "Either update autodetect_distro() or use the --distro argument" fi + + echo ${name}${version} + return 0 + } -uniqify() -{ +uniqify() { + echo $(echo "$*" | xargs -n 1 | sort -u) + } -load_target() -{ +download_srpm() { + local target=$1 + local srpm=$2 + local force="${3:-false}" + + if $force || [ ! -r "$KERNELDIR/$srpm" ] || + [ ! -s "$KERNELDIR/$srpm" ]; then + if $DOWNLOAD; then + local location="http://downloads.lustre.org/public/kernels/$target/old" + echo "Downloading $location/$srpm..." + if ! wget -nv "$location/$srpm" -O "$KERNELDIR/$srpm" || + [ ! -s "$KERNELDIR/$srpm" ]; then + rm -f $KERNELDIR/$srpm + fatal 1 "Could not download target $target's kernel SRPM" \ + "$srpm from $location." + fi + else + fatal 1 "$srpm not found in directory $KERNELDIR." + fi + fi + +} + +download_ofed() { + local force="${1:-false}" + + if [ -n "$OFED_VERSION" -a "$OFED_VERSION" != "inkernel" ] && + ( $force || [ ! -r "$KERNELTREE/OFED-${OFED_VERSION}.tgz" ] || + [ ! -s "$KERNELTREE/OFED-${OFED_VERSION}.tgz" ] ); then + if $DOWNLOAD; then + local location="http://downloads.lustre.org/public/OFED/" + echo "Downloading $location/OFED-${OFED_VERSION}.tgz..." + if ! wget -nv "$location/OFED-${OFED_VERSION}.tgz" \ + -O "$KERNELTREE/OFED-${OFED_VERSION}.tgz" || + [ ! -s "$KERNELTREE/OFED-${OFED_VERSION}.tgz" ]; then + rm -f $KERNELTREE/OFED-${OFED_VERSION}.tgz + fatal 1 "Could not download OFED-${OFED_VERSION}.tgz" \ + "from downloads.lustre.org." + fi + else + fatal 1 "OFED-${OFED_VERSION}.tgz not found in kernel" \ + "directory $KERNELTREE." + fi + fi + +} + +load_target() { + EXTRA_VERSION_save="$EXTRA_VERSION" - for patchesdir in "$EXTERNAL_PATCHES" "$TOPDIR/lustre/lustre/kernel_patches" ; do - TARGET_FILE="$patchesdir/targets/$TARGET.target" - [ -r "$TARGET_FILE" ] && break + for patchesdir in "$EXTERNAL_PATCHES" \ + "$TOPDIR/lustre/lustre/kernel_patches"; do + TARGET_FILE="$patchesdir/targets/$TARGET.target" + [ -r "$TARGET_FILE" ] && break done - [ -r "$TARGET_FILE" ] || \ - fatal 1 "Target $TARGET was not found." + [ -r "$TARGET_FILE" ] || fatal 1 "Target $TARGET was not found." - echo "Loading target config file $TARGET.target..." + echo "Loading target config file $TARGET.target..." . "$TARGET_FILE" - [ "$KERNEL" ] || fatal 1 "Target $TARGET did not specify a kernel." -# [ "$SERIES" ] || fatal 1 "Target $TARGET did not specify a kernel patch series." -# [ "$CONFIG" ] || fatal 1 "Target $TARGET did not specify a kernel config." - [ "$VERSION" ] || fatal 1 "Target $TARGET did not specify a kernel version." - - if [ "$KERNELDIR" ] ; then - KERNEL_FILE="$KERNELDIR/$KERNEL" - if [ ! -r "$KERNELDIR/$KERNEL" ] ; then - if (( $DOWNLOAD )) ; then - echo "Downloading http://ftp.lustre.org/kernels/$CANONICAL_TARGET/old/$KERNEL..." - if ! wget -nv "http://ftp.lustre.org/kernels/$CANONICAL_TARGET/old/$KERNEL" \ - -O "$KERNELDIR/$KERNEL" ; then - fatal 1 "Could not download target $TARGET's kernel file $KERNEL from ftp.lustre.org." - fi - else - fatal 1 "Target $TARGET's kernel file $KERNEL not found in kernel directory $KERNELDIR." - fi - fi + # doesn't make any sense to build OFED for xen domX's + if $XEN; then + OFED_VERSION="" + fi + + # XXX - set_rpm_smp_type is an ugly undeterministic hack. it needs to + # go away and the target just specify the $RPMSMPTYPE + [ -z "$RPMSMPTYPE" ] && set_rpm_smp_type + + # CC might have been overwriten in TARGET_FILE + if [[ $CC != ccache\ * ]] && which "$CCACHE" 2>/dev/null; then + export CCACHE && export CC="ccache $CC" fi - if [ "$SERIES" ] ; then - for series in $SERIES ; do - for patchesdir in "$EXTERNAL_PATCHES" "$TOPDIR/lustre/lustre/kernel_patches" ; do - [ -r "$patchesdir/series/$series" ] && continue 2 - done - fatal 1 "Target $TARGET's series $SERIES could not be found.\nSearched:\n\t$EXTERNAL_PATCHES/series\n\t$TOPDIR/lustre/lustre/kernel_patches/series." - done + if [ ! "$KERNELTREE" = "" ] && [ -d "$KERNELTREE" ]; then + KERNELDIR="$KERNELTREE/${lnxmaj}" + [ -d "$KERNELDIR" ] || mkdir "$KERNELDIR" fi - CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/$CONFIG" - [ -r "$CONFIG_FILE" ] || \ - fatal 1 "Target $TARGET's config file $CONFIG missing from $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/." + # verify the series is available + if [ "$SERIES" ]; then + for series in $SERIES; do + for patchesdir in "$EXTERNAL_PATCHES" "$TOPDIR/lustre/lustre/kernel_patches"; do + [ -r "$patchesdir/series/$series" ] && continue 2 + done + fatal 1 "Target $TARGET's series $SERIES could not be" \ + "found.\nSearched:\n\t$EXTERNAL_PATCHES/series\n" \ + "\t$TOPDIR/lustre/lustre/kernel_patches/series." + done + fi + + # set the location of the .config file + local XENPOSTFIX="" + if $XEN; then + XENPOSTFIX="-xen" + fi + + if [ -f $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET-$TARGET_ARCH.config ]; then + CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET$XENPOSTFIX-$TARGET_ARCH${RPMSMPTYPE:+-}${RPMSMPTYPE}.config" + fi + + local lnxrelnew=${lnxrel//-/_} + + # remember the EXTRA_VERSION before we diddle it here + # XXX - we really should not diddle with any values read in from the + # target file. if we want to modify a value, we should create + # a new variable. + PRISTINE_EXTRA_VERSION=$EXTRA_VERSION + + if ! $PATCHLESS && [ ! -f "$CONFIG_FILE" ]; then + fatal 1 "Config file for target $TARGET missing from" \ + "$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/." + fi - if [ "$EXTRA_VERSION_save" ] ; then - EXTRA_VERSION="$EXTRA_VERSION_save" - elif ! (( $RELEASE )) ; then - #remove the @VERSION@ (lustre version) - EXTRA_VERSION=$(echo $EXTRA_VERSION | sed -e "s/\(.*_lustre\)\..*/\1/") - EXTRA_VERSION="${EXTRA_VERSION}-${TAG}.${TIMESTAMP}" + if [ "$EXTRA_VERSION_save" ]; then + EXTRA_VERSION="$EXTRA_VERSION_save" + elif ! $RELEASE; then + # if there is no patch series, then this is not a lustre specific + # kernel. don't make it look like one + if [ -n "$SERIES" ]; then + #remove the @VERSION@ (lustre version) +# EXTRA_VERSION=$(echo $EXTRA_VERSION | sed -e "s/\(.*_lustre\)\..*/\1/") +# EXTRA_VERSION="${EXTRA_VERSION}-${TAG}.${TIMESTAMP}" + ! ( $PATCHLESS ) && EXTRA_VERSION="${EXTRA_VERSION}.${TIMESTAMP}" + fi fi # EXTRA_VERSION=${EXTRA_VERSION//-/_} ALL_ARCHS="$BASE_ARCHS $BIGMEM_ARCHS $BOOT_ARCHS $JENSEN_ARCHS $SMP_ARCHS $BIGSMP_ARCHS $PSERIES64_ARCHS $UP_ARCHS" BUILD_ARCHS= - for arch in $(uniqify "$ALL_ARCHS") ; do - if [ -z "$TARGET_ARCHS" ] || echo "$TARGET_ARCHS" | grep "$arch" >/dev/null 2>/dev/null ; then - BUILD_ARCHS="$BUILD_ARCHS $arch" - fi + for arch in $(uniqify "$ALL_ARCHS"); do + if [ -z "$TARGET_ARCHS" ] || + [[ \ $TARGET_ARCHS\ = *\ $arch\ * ]]; then + BUILD_ARCHS="$BUILD_ARCHS $arch" + fi done [ "$BUILD_ARCHS" ] || usage 1 "No available target archs to build." echo "Building for: $BUILD_ARCHS" } -tarflags() -{ - case "$1" in - '') - fatal 1 "tarflags(): File name argument missing." - ;; - *.tar.gz | *.tgz) - echo 'zxf' - ;; - *.tar.bz2) - echo 'jxf' - ;; - *.tar) - echo 'xf' - ;; - *) - fatal 1 "tarflags(): Unrecognized tar extension in file: $1" - ;; +tarflags() { + local file="$1" + + case "$file" in + '') + fatal 1 "tarflags(): File name argument missing." + ;; + *.tar.gz | *.tgz) + echo 'zxf' + ;; + *.tar.bz2) + echo 'jxf' + ;; + *.tar) + echo 'xf' + ;; + *) + fatal 1 "tarflags(): Unrecognized tar extension in file: $1" + ;; esac + } -untar() -{ - echo "Untarring ${1##*/}..." - tar $(tarflags "$1") "$1" +untar() { + local file="$1" + + echo "Untarring ${file##*/}..." + tar $(tarflags "$file") "$file" + } -unpack_lustre() -{ - DIRNAME="lustre-$TAG-$TIMESTAMP" - if [ "$LUSTRE" ] ; then - untar "$LUSTRE" - [ -d lustre ] || ln -sf lustre* lustre - else - if [ "$USE_DATESTAMP" ]; then - DATESTAMP="-D '$DATE'" - else - DATESTAMP="" - fi - - cvs -d "$CVSROOT" -qz3 co $DATESTAMP -d "$DIRNAME" lustre || \ - fatal 1 "There was an error checking out toplevel Lustre from CVS." - pushd "$DIRNAME" > /dev/null - ./lustrecvs "$TAG" || \ - fatal 1 "There was an error checking out Lustre/Portals/Build from CVS." - echo "Creating lustre tarball..." - sh autogen.sh || fatal 1 "There was an error running autogen.sh." - ./configure --disable-{modules,utils,liblustre,tests,doc} || \ - fatal 1 "There was an error running ./configure to create makefiles." - make dist || fatal 1 "There was an error running 'make dist'." - popd > /dev/null - fname=`basename $DIRNAME/lustre-*.tar.gz` - cp $DIRNAME/$fname . || fatal 1 "There was an error copying lustre tarball." - LUSTRE="$PWD/$fname" - ln -sf "$DIRNAME" lustre - fi -} - -unpack_linux() -{ - untar "$KERNEL_FILE" - [ -d linux ] || ln -sf linux* linux -} - -patch_linux() -{ - [ "$SERIES" ] || return 0 - FULL_PATCH="$PWD/lustre-kernel-${TARGET}-${EXTRA_VERSION}.patch" +unpack_ofed() { + + if ! untar "$KERNELTREE/OFED-${OFED_VERSION}.tgz"; then + return 1 + fi + [ -d OFED ] || ln -sf OFED-[0-9].[0-9]* OFED + +} + +unpack_lustre() { + + if [ -z "$LUSTRE" ]; then + local DATESTAMP="" + + if [ -n "$USE_DATESTAMP" ]; then + DATESTAMP="-D '$DATE'" + fi + + local DIRNAME="lustre-$TAG-$TIMESTAMP" + + cvs -d "$CVSROOT" -qz3 co $DATESTAMP -d "$DIRNAME" lustre || \ + fatal 1 "There was an error checking out toplevel Lustre from CVS." + pushd "$DIRNAME" > /dev/null + ./lustrecvs "$TAG" || \ + fatal 1 "There was an error checking out Lustre/Portals/Build from CVS." + echo "Creating lustre tarball..." + sh autogen.sh || fatal 1 "There was an error running autogen.sh." + ./configure --disable-{modules,utils,liblustre,tests,doc} || \ + fatal 1 "There was an error running ./configure to create makefiles." + make dist || fatal 1 "There was an error running 'make dist'." + LUSTRE=$PWD/lustre-*.tar.gz + popd > /dev/null + fi + + untar "$LUSTRE" || fatal 1 "Error unpacking Lustre tarball" + [ -d lustre ] || ln -sf lustre-[0-9].[0-9]* lustre + +} + +do_patch_linux() { + + local do_patch=${1:-true} + + FULL_PATCH="$PWD/lustre-kernel-${TARGET}${EXTRA_VERSION_DELIMITER}${EXTRA_VERSION}.patch" [ -f "$FULL_PATCH" ] && rm -f "$FULL_PATCH" - pushd linux >/dev/null - for series in $SERIES ; do - echo -n "Applying series $series:" - for patchesdir in "$EXTERNAL_PATCHES" "$TOPDIR/lustre/lustre/kernel_patches" ; do - [ -r "$patchesdir/series/$series" ] || continue - SERIES_FILE="$patchesdir/series/$series" - for patch in $(<"$SERIES_FILE") ; do - echo -n " $patch" - PATCH_FILE="$patchesdir/patches/$patch" - [ -r "$PATCH_FILE" ] || \ - fatal 1 "Patch $patch does not exist in Lustre tree." - cat "$PATCH_FILE" >> "$FULL_PATCH" || \ - fatal 1 "Error adding patch $patch to full patch." - patch -s -p1 < "$PATCH_FILE" || fatal 1 "Error applying patch $patch." - done - break - done - echo + $do_patch && pushd linux >/dev/null + for series in $SERIES; do + echo -n "Applying series $series:" + for patchesdir in "$EXTERNAL_PATCHES" "$TOPDIR/lustre/lustre/kernel_patches"; do + [ -r "$patchesdir/series/$series" ] || continue + SERIES_FILE="$patchesdir/series/$series" + for patch in $(<"$SERIES_FILE"); do + echo -n " $patch" + PATCH_FILE="$patchesdir/patches/$patch" + [ -r "$PATCH_FILE" ] || \ + fatal 1 "Patch $patch does not exist in Lustre tree." + cat "$PATCH_FILE" >> "$FULL_PATCH" || { + rm -f $FULL_PATCH + fatal 1 "Error adding patch $patch to full patch." + } + if $do_patch; then + patch -s -p1 < "$PATCH_FILE" || { + rm -f $FULL_PATCH + fatal 1 "Error applying patch $patch." + } + fi + done + break + done + echo done - popd >/dev/null + $do_patch && popd >/dev/null echo "Full patch has been saved in ${FULL_PATCH##*/}." - echo "Replacing .config files..." - [ -d linux/configs ] || mkdir linux/configs || \ - fatal 1 "Error creating configs directory." - rm -f linux/configs/* - copysuccess=0 - for patchesdir in "$EXTERNAL_PATCHES" "lustre/lustre/kernel_patches" ; do - [ "$patchesdir" ] && \ - cp -v $patchesdir/kernel_configs/kernel-${VERSION}-${TARGET}*.config linux/configs/ >/dev/null && copysuccess=1 + +} + +build_lustre() { + + cp "$LUSTRE" SOURCES + + pushd lustre >/dev/null + + echo "Building Lustre RPMs for: $BUILD_ARCHS..." + targets= + for arch in $BUILD_ARCHS; do + targets="--target $arch $targets" done - [ "$copysuccess" = "1" ] || \ - fatal 1 "Error copying in kernel configs." -} - -pack_linux() -{ - TARBALL="$(readlink linux)-$EXTRA_VERSION.tar.gz" - echo "Creating patched linux tarball $TARBALL..." - tar zcf "$TARBALL" "$(readlink linux)" \ - --exclude "CVS" --exclude ".cvsignore" || \ - --exclude "*.orig" --exclude "*~" --exclude "*.rej" || \ - fatal 1 "Error creating patched Linux tarball." -} - -clean_linux() -{ - [ -d linux ] || return 0 - echo "Cleaning linux..." - [ -L linux ] && rm -rf $(readlink linux) - rm -rf linux -} - -prep_kernel_build() -{ - # make .spec file - ENABLE_INIT_SCRIPTS="" - sed \ - -e "s^@BASE_ARCHS@^$BASE_ARCHS^g" \ - -e "s^@BIGMEM_ARCHS@^$BIGMEM_ARCHS^g" \ - -e "s^@BIGSMP_ARCHS@^$BIGSMP_ARCHS^g" \ - -e "s^@BOOT_ARCHS@^$BOOT_ARCHS^g" \ - -e "s^@CONFIGURE_FLAGS@^$CONFIGURE_FLAGS^g" \ - -e "s^@ENABLE_INIT_SCRIPTS@^$ENABLE_INIT_SCRIPTS^g" \ - -e "s^@JENSEN_ARCHS@^$BOOT_ARCHS^g" \ - -e "s^@KERNEL_EXTRA_VERSION@^$EXTRA_VERSION^g" \ - -e "s^@KERNEL_RELEASE@^${EXTRA_VERSION//-/_}^g" \ - -e "s^@KERNEL_SOURCE@^$KERNEL^g" \ - -e "s^@KERNEL_VERSION@^$VERSION^g" \ - -e "s^@LINUX26@^$LINUX26^g" \ - -e "s^@LUSTRE_SOURCE@^${LUSTRE##*/}^g" \ - -e "s^@LUSTRE_TARGET@^$TARGET^g" \ - -e "s^@PSERIES64_ARCHS@^$PSERIES64_ARCHS^g" \ - -e "s^@RHBUILD@^$RHBUILD^g" \ - -e "s^@SMP_ARCHS@^$SMP_ARCHS^g" \ - -e "s^@SUSEBUILD@^$SUSEBUILD^g" \ - -e "s^@SUSEBUILD@^$SUSEBUILD^g" \ - -e "s^@UP_ARCHS@^$UP_ARCHS^g" \ - < $TOPDIR/lustre/build/lustre-kernel-2.4.spec.in \ - > lustre-kernel-2.4.spec - [ -d SRPMS ] || mkdir SRPMS - [ -d RPMS ] || mkdir RPMS - [ -d BUILD ] || mkdir BUILD - [ -d SOURCES ] || mkdir SOURCES - for script in linux-{rhconfig.h,merge-config.awk,merge-modules.awk} \ - suse-{functions.sh,post.sh,postun.sh,trigger-script.sh.in} \ - sles8-{pre,post,postun,update_{INITRD_MODULES,rcfile_setting}}.sh ; do - cp $TOPDIR/lustre/build/$script SOURCES + + local confoptions="--with-linux=${LINUX}" + if $PATCHLESS; then + confoptions="--with-linux=${LINUX} --disable-server" + fi + if [ ! "$LINUXOBJ" = "" ]; then + confoptions="$confoptions --with-linux-obj=${LINUXOBJ}" + fi + + ./configure $confoptions ${CONFIGURE_FLAGS} + if [ "$?" != "0" ]; then + local saved_config="../config.log.$(date +%s)" + cp config.log $saved_config + chmod a+r $saved_config + echo "Saved config.log is at $saved_config" + cat /proc/mounts + ls -l /proc/$$ + pwd + echo "config.log contents:" + cat config.log + popd + return 255 + fi + + gen_lustre_version + + # hack. Somebody move build/lustre.spec to lustre.spec for b1_6 + local lustre_spec + [ -f lustre.spec ] && lustre_spec=lustre.spec + [ -f build/lustre.spec ] && lustre_spec=build/lustre.spec + + [ -f "$lustre_spec" ] && sed \ + -e "s^Release: .*$^Release: $LUSTRE_EXTRA_VERSION^" \ + < $lustre_spec \ + > ../lustre.spec + + local rpmbuildopt='-bb' + if $NORPM; then + rpmbuildopt='-bc' + echo NORPM mode. Only compiling. + fi + + # convert the $PATCHLESS boolean to an empty/no-empty boolean + # as silly as this seems, it makes the syntax of the rpmbuild command + # simpler and not need an eval to deal with the quotes in the quotes + local is_patchless="" + if $PATCHLESS; then + is_patchless="yes" + fi + $RPMBUILD $targets $rpmbuildopt ../lustre.spec \ + ${is_patchless:+--define "lustre_name lustre-client"} \ + --define "_tmppath $TMPDIR" \ + --define "_topdir $TOPDIR" || \ + fatal 1 "Error building rpms for $BUILD_ARCHS." + + popd >/dev/null + ( $(skeep_ldiskfs_rpm $TAG) ) && return + + pushd lustre/ldiskfs || return 255 + make dist + if [ "$?" != "0" ]; then + popd + return 255 + fi + cp lustre-ldiskfs*.tar.gz $TOPDIR/SOURCES + + gen_lustre_version + + local ldiskfs_spec=lustre-ldiskfs.spec + [ -f "$ldiskfs_spec" ] && sed \ + -e "s^Release: .*$^Release: $LUSTRE_EXTRA_VERSION^" \ + < $ldiskfs_spec \ + > ../lustre-ldiskfs.spec + + $RPMBUILD $targets $rpmbuildopt ../lustre-ldiskfs.spec \ + --define "_tmppath /var/tmp" \ + --define "_topdir $TOPDIR" + if [ "$?" != "0" ]; then + popd + return 255 + fi + + if $DO_SRC; then + $RPMBUILD -bs ../lustre-ldiskfs.spec \ + --define "_tmppath /var/tmp" \ + --define "_topdir $TOPDIR" + if [ "$?" != "0" ]; then + popd + return 255 + fi + fi + popd + +} + +stage() { + + [ "$STAGEDIR" ] || return 0 + + for arch in $BUILD_ARCHS; do + rpmdir="${STAGEDIR}/${CANONICAL_TARGET}-${arch}" + echo "${0##*/}: Copying RPMs into ${rpmdir}" + mkdir -p "${rpmdir}" + cp -v RPMS/${arch}/*.rpm "${rpmdir}" + if [ -d RPMS/noarch ]; then + cp -v RPMS/noarch/*.rpm "${rpmdir}" + fi + done + + cp -v "$LUSTRE" "$STAGEDIR" + +} + +#check if we need to build separate ldiskfs RPM +skeep_ldiskfs_rpm() { + local tag="$1" + + local skip=false + + if ! $LDISKFSRPM; then + skip=true + elif $PATCHLESS; then + skip=true + else + for skiptag in $SKIPLDISKFSRPM; do + [[ $tag == $skiptag ]] && skip=true && break + done + fi + echo $skip + +} + +#generate LUSTRE_EXTRA_VERSION from EXTRA_VERSION +gen_lustre_version() { + + LUSTRE_EXTRA_VERSION="${lnxmaj}${EXTRA_VERSION_DELIMITER}${EXTRA_VERSION}${TARGET_DELIMITER}${RPMSMPTYPE}" + LUSTRE_EXTRA_VERSION=${LUSTRE_EXTRA_VERSION//-/_} + +} + +set_rpm_smp_type() { + + local infact_arch="${TARGET_ARCH}" + + RPMSMPTYPE="" + [ "$infact_arch" == "i586" ] && infact_arch="i686" + + local smp_type + for smp_type in $SMP_ARCHS; do + [ $infact_arch == $smp_type ] && RPMSMPTYPE=smp && break + done + + for smp_type in $BIGSMP_ARCHS; do + [ $infact_arch == $smp_type ] && RPMSMPTYPE=bigsmp && break + done + + for smp_type in $DEFAULT_ARCHS; do + [ $infact_arch == $smp_type ] && RPMSMPTYPE=default && break + done + +} + +# This function takes a linux source pool and digs out the linux release +# from it +find_linux_release() { + local SRCDIR="$1" + + local LINUXRELEASEHEADER=$SRCDIR/include/linux/version.h + if [ -s $SRCDIR/include/linux/utsrelease.h ]; then + LINUXRELEASEHEADER=$SRCDIR/include/linux/utsrelease.h + fi + + sed -ne 's/#define UTS_RELEASE "\(.*\)"$/\1/p' $LINUXRELEASEHEADER + +} + +# XXX this needs to be re-written as a wrapper around find_rpm +# or just gotten rid of. :-) +find_linux_rpm() { + local prefix="$1" + local delimiter=${2:-"-"} + + local pathtorpms="${KERNELRPMSBASE}/${lnxmaj}/${DISTRO}" + [ -d $pathtorpms ] || return 255 + + local kernelbinaryrpm rpmfile + local wanted_kernel="${lnxmaj}${delimiter}${lnxrel}" + + local arch ret=1 + for arch in $TARGET_ARCHS_ALL; do + local found_rpm="" rpm + for rpm in ${pathtorpms}/${arch}/*.rpm; do + if rpm -q --provides -p "$rpm" | grep -q "kernel${prefix} = $wanted_kernel"; then + found_rpm="$rpm" + ret=0 + break + fi + done + [ -f "$found_rpm" ] && break done - cp "$LUSTRE" "$KERNEL_FILE" SOURCES - if [ "$EXTERNAL_PATCHES" -a -d "$EXTERNAL_PATCHES" ] ; then - tar zcf SOURCES/external-patches.tar.gz -C "$EXTERNAL_PATCHES" series targets patches kernel_configs + + echo "$found_rpm" + return $ret + +} + +# unpack kernel(/source/devel) RPM +# +# This function and it's setting of $LINUX and $LINUXOBJ is a total hack that +# needs to completely refactored. It completely ingores that $BUILD_ARCHS may +# contain a list of arches for which rpmbuild commands (including the one for +# lustre itself) +unpack_linux_devel_rpm() { + local kernelrpm="${1}" + # it's worth noting that neither sles10 nor rhel5 appear to use their + # extra_version delimiter for the dirname under /usr/src, so we could + # probably just get rid of this parameter + local delimiter=${2:-"-"} + + [ -f "$kernelrpm" ] || return 255 + [ -d $TOPDIR/reused ] || mkdir $TOPDIR/reused || return 255 + + pushd $TOPDIR/reused || return 255 + + if ! rpm2cpio < "$kernelrpm" | cpio -id > /dev/null 2>&1; then + return 255 + fi + + # call a distro specific hook, if available + if type -p unpack_linux_devel_rpm-$DISTRO; then + unpack_linux_devel_rpm-$DISTRO "$kernelrpm" + fi + + popd + + find_linux_devel_paths $TOPDIR/reused + + return 0 + +} + +# XXX - this rhel/sles goop needs abstracting out into the +# lbuild-{rhel5,sles10} method files +find_linux_devel_paths() { + local path="$1" + + local RC=0 + + pushd $path + # RHEL-style and SLES-style rpms + # XXX - until bug 19336 cleans this up, we need to extricate the + # ${lnxmin}- from the $lnxrel + local paths="kernels/${lnxmaj}${lnxmin}${delimiter}${lnxrel}-${TARGET_ARCH} linux-${lnxmaj}${lnxmin}${delimiter}${lnxrel##${lnxmin#.}-}" + + local path + for path in $paths; do + local src='usr/src' + + if [ -d "$src/$path/" ]; then + LINUX="$(pwd)/$src/$path" + fi + # SLES has a separate -obj tree + if [ -d "$src/${path}-obj" ]; then + local src="$src/${path}-obj" + local objects="$TARGET_ARCH/$RPMSMPTYPE" + + # Novell, are you *TRYING* to make life hard for me? + if [ -d "$src/powerpc" ]; then + objects="powerpc/$TARGET_ARCH" + elif [ $TARGET_ARCH == 'i686' ]; then + objects="i386/$RPMSMPTYPE" + fi + + LINUXOBJ="$(pwd)/$src/$objects" + fi + done + if [ -z "$LINUX" ]; then + RC=255 + else + # dig out the release version + LINUXRELEASE=$(find_linux_release ${LINUXOBJ:-$LINUX}) + if [ -z "$LINUXRELEASE" ]; then + echo "Failed to find linux release in ${LINUXOBJ:-$LINUX}" + RC=255 + fi + fi + popd + return $RC +} + +build_kernel_ib() { + # build kernel-ib{,-devel} + # some I/B drivers are architecture dependent and kernel-ib's configure + # does not figure it out for us ~sigh~ + local configure_options="" + case "$TARGET_ARCH" in + x86_64 | ia64) + configure_options="--with-ipath_inf-mod" + ;; + ppc64) + configure_options="--with-ipath_inf-mod --with-ehca-mod" + ;; + esac + local K_SRC="K_SRC" + # ofed 1.3 had a bug in the rpm spec + if [ "$OFED_VERSION" = "1.3" ]; then + K_SRC="KSRC" + fi + $RPMBUILD --rebuild --define 'build_kernel_ib 1' --define 'build_kernel_ib_devel 1' \ + --define "_topdir ${TOPDIR}" --target ${TARGET_ARCH} \ + --define "KVERSION ${LINUXRELEASE}" \ + --define "$K_SRC ${LINUXOBJ:-${LINUX}}" \ + --define "LIB_MOD_DIR /lib/modules/${LINUXRELEASE}/updates" \ + --define "configure_options --without-quilt --with-core-mod --with-user_mad-mod --with-user_access-mod --with-addr_trans-mod --with-srp-target-mod --with-core-mod --with-mthca-mod --with-mlx4-mod --with-mlx4_en-mod --with-cxgb3-mod --with-nes-mod --with-ipoib-mod --with-sdp-mod --with-srp-mod --without-srp-target-mod --with-rds-mod --with-iser-mod --with-qlgc_vnic-mod --with-madeye-mod $configure_options" ${TOPDIR}/OFED/SRPMS/ofa_kernel-*.src.rpm + + if [ ${PIPESTATUS[0]} != 0 ]; then + fatal 1 "Error building kernel-ib" + fi + +} + +store_for_reuse() { + local articles="$1" + local module="$2" + local location="$3" + local signature="$4" + local use_links="$5" + + local linkflag="" + if $use_links; then + linkflag="l" + fi + + location="$location"/"$signature"/"$module" + mkdir -p "$location" + # the cleanup script removes any directory that doesn't have a + # .lastused, so let's try to prevent that as soon as we can + # this solution still slightly racy with the cleanup script + # but the race is a lot tighter now + touch -t 197001010000 "$location/.lastused" + ## use eval/echo here to make sure shell expansions are performed + #if ! cp -a${linkflag} $(eval echo $articles) "$location"; then + local article + for article in $(eval echo $articles); do + if ! cp -a${linkflag} "$article" "$location"; then + error "Failed to copy \"$article\" to \"$location\" in store_for_reuse()" + # rename the cache location so that it's not cached + # product, but is around for analysis + mv "$location"{,-bad-$(date +%s)} || + error "failed to clean up a failed cache attempt" \ + "in \"$location\" -- manual cleanup will be" \ + "necessary" + return 1 + fi + done + + # flag the cache as complete (i.e. in case lbuild was previously + # interrupted while caching) + touch "$location/.lastused" + + return 0 + +} + +reuse() { + local module="$1" + local dest="$2" + local use_links="${3:-false}" + local signature="$4" + + if [ -n "$REUSEBUILD" ] && [ -d "$REUSEBUILD/$signature/$module" ]; then + if [ ! -f "$REUSEBUILD/$signature/$module/.lastused" ]; then + # the .lastused flag is populated at the end of the caching to + # signal that the caching was completeld. if that flag is not + # there, then the cache is invalid (and should be removed in fact) + mv "$REUSEBUILD/$signature/$module"{,-bad-$(date +%s)} || + fatal 1 "failed to clean up a bad cache in location" \ + "\"$REUSEBUILD/$signature/$module\" -- manual cleanup" \ + "will be necessary" + return 1 + fi + + # so that we know how stale this entry is + touch $REUSEBUILD/$signature/$module/.lastused + + if $use_links; then + if ls $REUSEBUILD/$signature/$module/* >/dev/null 2>&1; then + cp -al $REUSEBUILD/$signature/$module/* $dest/ + fi + else + # copying is pretty heavy + # cp -a $REUSEBUILD/$signature/$module/* $dest/ + # do some creative symlinking instead + local dir + for dir in BUILD SRPMS SPECS; do + if ls $REUSEBUILD/$signature/$module/$dir/* >/dev/null 2>&1; then + ln -s $REUSEBUILD/$signature/$module/$dir/* $dest/$dir + fi + done + # sources have to be copied by file because we need SOURCES to + # be a dir we can write into +# could overrun ls's arg list here + #ls $REUSEBUILD/$signature/$module/SOURCES/* | + find $REUSEBUILD/$signature/$module/SOURCES/ -type f | + xargs ln -t $dest/SOURCES -s + + # same for RPMS/* dirs +# could overrun ls's arg list here + #ls $REUSEBUILD/$signature/$module/RPMS/$TARGET_ARCH/* | + local dir + for dir in $REUSEBUILD/$signature/$module/RPMS/*; do + mkdir -p $dest/RPMS/${dir##*/} + find $dir -type f | + xargs ln -t $dest/RPMS/${dir##*/} -s + done + + fi + return 0 else - touch SOURCES/external-patches.tar.gz + return 1 fi } -clean_lustre() -{ - [ -d lustre ] || return 0 - echo "Cleaning Lustre..." - [ -L lustre ] && rm -rf $(readlink lustre) - rm -rf lustre +basearch() { + local arch="$1" + + if [[ $arch = i[3456]86 ]]; then + echo "i386" + else + echo "$arch" + fi + } -build_kernel() -{ - echo "Building kernel + Lustre RPMs for: $BUILD_ARCHS..." - targets= - for arch in $BUILD_ARCHS ; do - targets="--target $arch $targets" +# +# in a given directory, find the first rpm matching given requirements +# +find_rpm() { + local dir="$1" + local match_type="$2" + local match="$3" + + pushd "$dir" > /dev/null || \ + fatal 1 "Unable to chdir to directory \"$dir\" in find_rpm()" + + local file + for file in $(ls); do + if [ ! -f "$file" ]; then + continue + fi + case "$match_type" in + provides) + # match is any valid ERE (i.e. given to egrep) match + if rpm -q --provides -p "$file" | egrep -q "$match"; then + echo "$file" + popd >/dev/null + return 0 + fi + ;; + *) + popd >/dev/null + fatal 1 "Unknown match type \"$match_type\" given to find_rpm()" + ;; + esac done - $RPMBUILD $targets -bb lustre-kernel-2.4.spec \ - --define "_topdir $TOPDIR" || \ - fatal 1 "Error building rpms for $BUILD_ARCHS." + popd >/dev/null + return 1 +} + +build_kernel_with_srpm() { - if (( $DO_SRC )) ; then - $RPMBUILD -bs lustre-kernel-2.4.spec \ - --define "_topdir $TOPDIR" || \ - fatal 1 "Error building .src.rpm." + # need to generate the patch for this target + do_patch_linux false >&2 # sets global $FULL_PATCH (yeah, yuck) + + # get an md5sum of the kernel patch + config for reuse check + # XXX really, there needs to be a signature and a CONFIG_FILE per arch + # in BUILD_ARCHS + local REUSE_SIGNATURE=$({ echo $BUILD_GEN; cat $CONFIG_FILE $TARGET_FILE $FULL_PATCH; } | md5sum | cut -d" " -f1) + + # see if we can link to the reuse pool + # XXX - hrm. i'm not convinced this doesn't belong in the reuse "library" + local CAN_LINK_FOR_REUSE=false + touch $REUSEBUILD/$$ + if cp -al $REUSEBUILD/$$ $TOPDIR/; then + CAN_LINK_FOR_REUSE=true fi + rm $REUSEBUILD/$$ + + # the extra version string to use for the kernel (which might be a reused + # kernel, remember) + local kernel_extra_version="" + if $REUSERPM && reuse kernel "$TOPDIR" "$CAN_LINK_FOR_REUSE" \ + "$REUSE_SIGNATURE"; then + # figure out the EXTRA_VERSION of the kernel we are re-using + local KERNEL_RPM + if ! KERNEL_RPM=$(find_rpm "$TOPDIR/RPMS/$TARGET_ARCH/" provides "^kernel ="); then + fatal 1 "Failed to find a kernel RPM in $TOPDIR/RPMS/$TARGET_ARCH/" + fi + kernel_extra_version=$(rpm -q --queryformat "%{RELEASE}" -p $TOPDIR/RPMS/$TARGET_ARCH/$KERNEL_RPM) + else + # nothing cached, build from scratch + if [ ! -r "$KERNELDIR/$KERNEL_SRPM" ]; then + download_srpm "$CANONICAL_TARGET" "$KERNEL_SRPM" >&2 + fi + + rpm -ivh $KERNELDIR/$KERNEL_SRPM --define "_topdir $TOPDIR" >&2 || { + # should we clean this up or leave it for analysis? + #rm -rf $RPMTOPDIR + fatal 1 "Error installing kernel SRPM." + } + + # put the Lustre kernel patch into the RPM build tree + cp $FULL_PATCH $TOPDIR/SOURCES/linux-${lnxmaj}-lustre.patch + prepare_and_build_srpm >&2 + + # store the resulting kernel RPM build tree for future use + if ! store_for_reuse "$TOPDIR/{SPECS,SOURCES,BUILD,SRPMS,RPMS}" \ + "kernel" "$REUSEBUILD" "$REUSE_SIGNATURE" \ + "$CAN_LINK_FOR_REUSE"; then + error "Failed to store kernel RPMS for reuse" + echo "unknown" + return 1 + fi + kernel_extra_version=$EXTRA_VERSION + fi # build reuse + + # should now have the following RPMs + # $TOPDIR/RPMS/$arch/kernel-lustre-2.6.18-53.1.21.el5_lustre.1.6.5.1.$arch.rpm + # $TOPDIR/RPMS/$arch/kernel-lustre-devel-2.6.18-53.1.21.el5_lustre.1.6.5.1.$arch.rpm + # $TOPDIR/RPMS/$arch/kernel-lustre-headers-2.6.18-53.1.21.el5_lustre.1.6.5.1.$arch.rpm + # $TOPDIR/RPMS/$arch/kernel-lustre-debuginfo-common-2.6.18-53.1.21.el5_lustre.1.6.5.1.$arch.rpm + # $TOPDIR/RPMS/$arch/kernel-lustre-debuginfo-2.6.18-53.1.21.el5_lustre.1.6.5.1.$arch.rpm + + echo $kernel_extra_version + return 0 + } -build_lustre() -{ - [ -d SRPMS ] || mkdir SRPMS - [ -d RPMS ] || mkdir RPMS +build_with_srpm() { + + if ! $PATCHLESS; then + local kernel_extra_version + if ! kernel_extra_version=$(build_kernel_with_srpm); then + fatal 1 "Failed to build the kernel from it's SRPM" + fi + + for arch in $BUILD_ARCHS; do + + local kernel_devel_rpm + if ! kernel_devel_rpm=$(find_rpm "$TOPDIR/RPMS/$arch/" provides "^$(devel_kernel_name true) ="); then + fatal 1 "Failed to find a kernel development RPM in $TOPDIR/RPMS/$arch/" + fi + + # install the -devel RPM in preparation for the lustre build + # note that the EXTRA_VERSION_DELIMITER is *NOT* used in the + # version of the directory name under /usr/src + if ! lnxrel="$kernel_extra_version" unpack_linux_devel_rpm \ + "$TOPDIR/RPMS/$arch/$kernel_devel_rpm" "-"; then + fatal 1 "Could not find the Linux tree in $TOPDIR/RPMS/$arch/$kernel_devel_rpm" + fi + done + else + # need to find and unpack the vendor's own kernel-devel for patchless + # client build + local kernelrpm + if ! kernelrpm=$(find_linux_rpm "-$DEVEL_KERNEL_TYPE" ${EXTRA_VERSION_DELIMITER:-"-"}); then + fatal 1 "Could not find the kernel-$DEVEL_KERNEL_TYPE RPM in ${KERNELRPMSBASE}/${lnxmaj}/${DISTRO}" + fi + if ! lnxrel="$lnxrel" unpack_linux_devel_rpm "$kernelrpm" "-"; then + fatal 1 "Could not find the Linux tree in $kernelrpm" + fi + fi + + # before lustre, build kernel-ib + if [ -n "$OFED_VERSION" -a "$OFED_VERSION" != "inkernel" ]; then + # see if we can link to the reuse pool + # XXX - hrm. i'm not convinced this doesn't belong in the reuse "library" + local CAN_LINK_FOR_REUSE=false + touch $REUSEBUILD/$$ + if cp -al $REUSEBUILD/$$ $TOPDIR/; then + CAN_LINK_FOR_REUSE=true + fi + rm $REUSEBUILD/$$ + + local REUSE_SIGNATURE=$({ echo "$OFED_VERSION"; echo "$(find_linux_release ${LINUXOBJ:-$LINUX})"; cat "${LINUXOBJ:-${LINUX}}/include/linux/autoconf.h"; } | md5sum | cut -d" " -f1) + if ! reuse ofed "$TOPDIR" "$CAN_LINK_FOR_REUSE" \ + "$REUSE_SIGNATURE"; then + # stash away the existing built articles for a moment + mkdir bak + mv {BUILD,{S,}RPMS,S{OURCE,PEC}S} bak + function mv_back { + pushd bak + find . | cpio -pudlm .. + popd + rm -rf bak + } + create_rpmbuild_dirs + # build it + build_kernel_ib + if ! store_for_reuse "$TOPDIR/{SPECS,SOURCES,BUILD,SRPMS,RPMS}" \ + "ofed" "$REUSEBUILD" "$REUSE_SIGNATURE" \ + "$CAN_LINK_FOR_REUSE"; then + error "Failed to store OFED RPMS for reuse" + mv_back + return 1 + fi + # put the stuff we stashed away back + mv_back + fi + + pushd "$TOPDIR" >/dev/null + rm -rf kernel-ib-devel + mkdir kernel-ib-devel + cd kernel-ib-devel + # the actual ofed RPMs don't have the -rc$n or -$date string appened that + # might be present on the file + local ofed_version=$(echo $OFED_VERSION | + sed -re 's/-(20[0-9]{6,6}-[0-9]{4,4}|rc[0-9]*)$//') + local rpm=$(ls $TOPDIR/RPMS/*/kernel-ib-devel-${ofed_version}-${LINUXRELEASE//-/_}.*.rpm) + rpm2cpio -itv < $rpm | cpio -id + CONFIGURE_FLAGS="--with-o2ib=$(pwd)/usr/src/ofa_kernel ${CONFIGURE_FLAGS}" + popd >/dev/null + fi + + # now build Lustre + if build_lustre; then + # the build worked. resolve any symlinked files (i.e. from reuse) + # in RPMS/$arch to real files so that that that huge mess of + # complication knows as LTS can copy them yet somewhere else. + # is it any wonder this whole process is so damn so? anyone ever + # heard of hardlinks? it this cool new thing that allows you save + # tons of time and space by creating... well you can go read about + # them if you have not heard about them yet. + # can i say how much the implemenation of all of this really impedes + # RPM reuse? + pushd RPMS/$TARGET_ARCH + for file in *; do + if [ -h $file ]; then + cp $file foo + mv foo $file + fi + done + popd + else + return 1 + fi + +} + +create_rpmbuild_dirs() { + + if [ ! -d RPMS ]; then + mkdir -p RPMS + for arch in $BUILD_ARCHS; do + mkdir RPMS/$arch + done + fi [ -d BUILD ] || mkdir BUILD [ -d SOURCES ] || mkdir SOURCES + [ -d SPECS ] || mkdir SPECS + [ -d SRPMS ] || mkdir SRPMS - cp "$LUSTRE" SOURCES +} - pushd lustre >/dev/null +new_list() { - echo "Building Lustre RPMs for: $BUILD_ARCHS..." - targets= - for arch in $BUILD_ARCHS ; do - targets="--target $arch $targets" - done + echo "" - ./configure "--with-linux=${LINUX}" ${CONFIGURE_FLAGS} +} - $RPMBUILD $targets -bb build/lustre.spec \ - --define "_topdir $TOPDIR" || \ - fatal 1 "Error building rpms for $BUILD_ARCHS." +add_list() { + local list="$1" + local item="$2" + + echo "$list $item" - popd >/dev/null } -stage() -{ - [ "$STAGEDIR" ] || return +is_list_member() { + local list="$1" + local item="$2" + + [[ $list\ == *\ $item\ * ]] + +} - for arch in $BUILD_ARCHS ; do - rpmdir="${STAGEDIR}/${CANONICAL_TARGET}-${arch}" - echo "${0##*/}: Copying RPMs into ${rpmdir}" - mkdir -p "${rpmdir}" - cp -v RPMS/${arch}/*.rpm "${rpmdir}" - if [ -d RPMS/noarch ] ; then - cp -v RPMS/noarch/*.rpm "${rpmdir}" +######################################################################### +# Generate a backtrace through the call stack. +# +# Input: None +# Output: None +######################################################################### +backtrace() { + local strip=${1:-1} + + local funcname="" sourcefile="" lineno="" n + + echo "Call stack: (most recent first)" + for (( n = $strip ; n < ${#FUNCNAME[@]} ; ++n )) ; do + funcname=${FUNCNAME[$n - 1]} + sourcefile=$(basename ${BASH_SOURCE[$n]}) + lineno=${BASH_LINENO[$n - 1]} + # Display function arguments + if [[ ! -z "${BASH_ARGV[@]}" ]]; then + local args newarg j p=0 + for (( j = ${BASH_ARGC[$n - 1]}; j > 0; j-- )); do + newarg=${BASH_ARGV[$j + $p - 1]} + args="${args:+${args} }'${newarg}'" + done + let p+=${BASH_ARGC[$n - 1]} fi + echo " ${funcname} ${args:+${args} }at ${sourcefile}:${lineno}" done - cp -v "$LUSTRE" "$STAGEDIR" + echo + echo "BEGIN BACKTRACE" + + #echo ${BASH_LINENO[*]} + #echo ${BASH_SOURCE[*]} + #echo ${FUNCNAME[*]} + local i=$((${#FUNCNAME[@]} - 1)) + while [ $i -ge 0 ]; do + local SOURCELINE="${BASH_SOURCE[$i + 1]}:${BASH_LINENO[$i]}" + # Can't figure out how to get function args from other frames... + local FUNCTION="${FUNCNAME[$i]}()" + echo "$SOURCELINE:$FUNCTION" + i=$((i - 1)) + done + + echo "END BACKTRACE" + + echo $BACKTRACE + } [ -r ~/.lbuildrc ] && . ~/.lbuildrc -options=$(getopt -o d:D:h -l disable-datestamp,external-patches:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,publish,release,stage:,tag:,target:,target-archs:,with-linux: -- "$@") +options=$(getopt -o d:D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ldiskfs,ccache,reuse:,norpm,disable-datestamp,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,publish,release,src,stage:,tag:,target:,target-archs:,with-linux:,xen -- "$@") -if [ $? != 0 ] ; then +if [ $? != 0 ]; then usage 1 fi eval set -- "$options" - -while [ "$1" ] ; do + +while [ "$1" ]; do case "$1" in - '') - usage 1 - ;; - -d) - CVSROOT=$2 - shift 2 - ;; - -D) - DATE=$2 - shift 2 - ;; - --external-patches) - EXTERNAL_PATCHES=$2 - shift 2 - ;; - --extraversion) - EXTRA_VERSION=$2 - shift 2 - ;; - --help | -h) - usage 0 - ;; - --kerneldir) - KERNELDIR=$2 - shift 2 - ;; - --linux | --with-linux) - LINUX=$2 - shift 2 - ;; - --lustre) - LUSTRE=$2 - shift 2 - ;; - --nodownload) - DOWNLOAD=0 - shift 1 - ;; - --nosrc) - DO_SRC=0 - shift 1 - ;; - --publish) - shift - ;; - --release) - RELEASE=1 - shift - ;; - --src) - DO_SRC=1 - shift 1 - ;; - --stage) - STAGEDIR=$2 - shift 2 - ;; - --tag) - TAG=$2 - shift 2 - ;; - --target) - TARGET=$2 - shift 2 - ;; - --target-archs) - TARGET_ARCHS=$2 - shift 2 - ;; - --disable-datestamp) - USE_DATESTAMP= - shift - ;; - --) - shift - CONFIGURE_FLAGS=$@ - break - ;; - *) - usage 1 "Unrecognized option: $1" - ;; + '') + usage 1 + ;; + --ccache) + CCACHE='ccache' + shift + ;; + -d) + CVSROOT=$2 + shift 2 + ;; + -D) + DATE=$2 + shift 2 + ;; + --external-patches) + EXTERNAL_PATCHES=$2 + shift 2 + ;; + --extraversion) + EXTRA_VERSION=$2 + shift 2 + ;; + --help | -h) + usage 0 + ;; + --kerneldir) + KERNELDIR=$2 + shift 2 + ;; + --kerneltree) + if ! KERNELTREE=$(canon_path "$2"); then + fatal 1 "Could not determine the canonical location of $2" + fi + shift 2 + ;; + --linux | --with-linux) + if ! LINUX=$(canon_path "$2"); then + fatal 1 "Could not determine the canonical location of $2" + fi + shift 2 + ;; + --distro) + DISTRO=$2 + shift 2 + ;; + --reuserpm) + REUSERPM=$2 + shift 2 + ;; + --reusebuild) + if ! REUSEBUILD=$(canon_path "$2"); then + fatal 1 "Could not determine the canonical location of $2" + fi + shift 2 + ;; + --norpm) + NORPM=true + shift + ;; + --ldiskfs) + LDISKFSRPM=true + shift + ;; + --patchless) + PATCHLESS=true + shift + ;; + --kernelrpm) + if ! KERNELRPMSBASE=$(canon_path "$2"); then + fatal 1 "Could not determine the canonical location of $2" + fi + shift 2 + ;; + --timestamp) + TIMESTAMP=$2 + shift 2 + ;; + --lustre) + LUSTRE=$2 + shift 2 + ;; + --nodownload) + DOWNLOAD=false + shift 1 + ;; + --nosrc) + DO_SRC=false + shift 1 + ;; + --publish) + shift + ;; + --release) + RELEASE=true + shift + ;; + --src) + DO_SRC=true + shift 1 + ;; + --stage) + STAGEDIR=$2 + shift 2 + ;; + --tag) + TAG=$2 + shift 2 + ;; + --target) + TARGET=$2 + shift 2 + ;; + --target-archs) + TARGET_ARCHS=$2 + shift 2 + ;; + --disable-datestamp) + USE_DATESTAMP= + shift + ;; + --xen) + XEN=true + shift + ;; + --) + shift + CONFIGURE_FLAGS=$@ + CONFIGURE_FLAGS="$CONFIGURE_FLAGS --enable-liblustre --enable-liblustre-tests" + break + ;; + *) + usage 1 "Unrecognized option: $1" + ;; esac done @@ -637,23 +1516,68 @@ check_options unpack_lustre -# prep_build needs the .spec.in from the lustre source -if [ -z "$LINUX" ] ; then - load_target +load_target +EXTRA_VERSION_DELIMITER=${EXTRA_VERSION_DELIMITER:-"-"} - if (( $DO_SRC )) ; then - unpack_linux - patch_linux - pack_linux - clean_linux - fi +if [ -n "$OFED_VERSION" -a "$OFED_VERSION" != "inkernel" ]; then + download_ofed + unpack_ofed || fatal 1 "Error unpacking OFED tarball" +fi - prep_kernel_build - clean_lustre +# make sure the RPM build environment is set up +create_rpmbuild_dirs - build_kernel -else +trap '[ -n "$CCACHE" ] && ccache -s' EXIT + +# if an unpacked kernel source tree was given on the command line +# just build lustre with it (nothing distro kernel specific here) +if [ -n "$LINUX" ]; then build_lustre +else + if [ -f "${0%/*}/lbuild-$DISTRO" ]; then + seen_list=$(new_list) + trap '(echo "Untrapped error" +echo +# have we seen this one +echo "checking seen list for ${BASH_SOURCE[0]}:${BASH_LINENO[0]}" + +if is_list_member "$seen_list" "${BASH_SOURCE[0]}:${BASH_LINENO[0]}"; then + echo "seen this one already" +else + seen_list=$(add_list "$seen_list" "${BASH_SOURCE[0]}:${BASH_LINENO[0]}") +fi +backtrace +echo +echo "Environment:" +set +) | tee >(mail -s "Untrapped error at ${BASH_SOURCE[0]##*/}:${BASH_LINENO[0]} on $HOSTNAME" brian@sun.com) >&2' ERR + set -E + + source ${0%/*}/lbuild-$DISTRO + + build_with_srpm || fatal 1 "Failed to build_with_srpm" + else + source ${0%/*}/lbuild.old_school + + old_school_download_kernel + + build_success=false + if $PATCHLESS; then + patchless_build_sequence && build_success=true + else + [ "$DISTRO" = "sles9" ] && build_sequence_rpm_reuse && build_success=true + if ! $build_success; then + build_sequence_reuse && build_success=true + if ! $build_success; then + build_sequence && build_success=true + if $build_success; then + store_for_reuse || echo "Cannot store for future reuse" + fi + fi + fi + fi + ( $build_success ) || fatal 1 "Cannot build lustre" + fi fi stage