From 049411b6454a97711d5cd3701e4f627a1b0142ff Mon Sep 17 00:00:00 2001 From: mjmac Date: Thu, 11 Oct 2007 16:54:08 +0000 Subject: [PATCH] b=13021 i=mjmac (original work by valik) i=brian * Merge LTS buildall into lbuild --- build/lbuild | 916 +++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 868 insertions(+), 48 deletions(-) diff --git a/build/lbuild b/build/lbuild index 7e9c630..42626cc 100755 --- a/build/lbuild +++ b/build/lbuild @@ -1,24 +1,53 @@ -#!/bin/sh +#!/bin/bash # vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4: +#set -x + TOPDIR=$PWD # CVSROOT is inherited from the environment KERNELDIR= LINUX= LUSTRE= -RELEASE=0 +RELEASE=false DO_SRC=0 DOWNLOAD=1 TAG= +CANONICAL_TARGET= TARGET= +TARGET_ARCH=$(uname -m) TARGET_ARCHS= +TARGET_ARCHS_ALL=$TARGET_ARCH +[ "$TARGET_ARCH" = "i686" ] && TARGET_ARCHS_ALL="i686 i586 i386" CONFIGURE_FLAGS= EXTERNAL_PATCHES= EXTRA_VERSION= +LUSTRE_EXTRA_VERSION= STAGEDIR= TMPDIR=${TMPDIR:-"/var/tmp"} +TIMESTAMP= +REUSERPM= +REUSEBUILD= +NORPM=false +LDISKFSRPM=true +SKIPLDISKFSRPM="v1_4_* b1_4" +SMPTYPES="smp bigsmp default ''" +KERNCONFSMPTYPE= +PATCHLESS=false +LINUXOBJ= +REUSEDKERNELMASK= +DISTRO= +KERNELTREE= + +# patchless build +KERNELRPMSBASE= +KERNELRPM= +KERNELSOURCERPM= +RPMSMPTYPE= +KERNELRPMCONFIG= +KERNELRPMRELEASE= +KERNELCOMPILEDIR= # from target file KERNEL= @@ -76,11 +105,6 @@ fatal() exit $1 } -is_release() -{ - (( $RELEASE )) || return 0 -} - list_targets() { echo -n "Available targets:" @@ -113,6 +137,34 @@ Usage: ${0##*/} [OPTION]... [-- ] Directory containing Linux source tarballs referenced by target files. + --timestamp=TIMESTAMP + Date of building lustre in format YYYYMMDDhhmmss + + --reuserpm=DIR + Try to reuse old kernel RPMs from DIR + + --reusebuild=DIR + Try to reuse old kernel builds from DIR + + --kernelrpm=DIR + Path to distro kernel RPM collection + + --ccache + Use ccache + + --norpm + Do not build RPMs (compile only mode) + + --patchless + Build lustre client only + + --distro=DISTRO + Which distro using. Autodetect by default + + --kerneltree=KERNELTREE + Directory containing dirs with Linux source tarballs referenced by target + files. Dir names in format kernel version ('2.6.9', etc.) + --linux=LINUX Directory of Linux kernel sources. When this option is used, only Lustre modules and userspace are built. @@ -128,6 +180,9 @@ Usage: ${0##*/} [OPTION]... [-- ] Do not build a .src.rpm, a full kernel patch, or a patched kernel tarball. + --ldiskfs + Do ldiskfs RPM. Now true by default + --publish Unused. @@ -185,13 +240,13 @@ check_options() fi if [ -z "$LINUX" ] ; then - [ "$KERNELDIR" ] || \ - usage 1 "A kernel directory must be specified with --kerneldir." + [ "$KERNELDIR" -o "$KERNELTREE" ] || \ + usage 1 "A kernel directory must be specified with --kerneldir or --kerneltree." - [ -d "$KERNELDIR" ] || \ - usage 1 "$KERNELDIR is not a directory." + [ -d "$KERNELDIR" -o -d "$KERNELTREE" ] || \ + usage 1 "$KERNELDIR and $KERNELTREE are not a directory." - if ! (( $RELEASE )) ; then + if ! $RELEASE; then [ "$TAG" ] || \ usage 1 "When building a snapshot, a tag name must be used." fi @@ -203,6 +258,9 @@ check_options() fi case $TARGET in + 2.6-rhel5) + CANONICAL_TARGET="rhel5" + ;; 2.6-rhel4) CANONICAL_TARGET="rhel-2.6" ;; @@ -225,7 +283,14 @@ check_options() ;; esac - TIMESTAMP=$(date -d "$DATE" "+%Y%m%d%H%M") + local timestampnodig=$(echo $TIMESTAMP | sed -e s/[0-9]*//g) + [ "$timestampnodig" = "" ] || TIMESTAMP=$(date -d "$DATE" "+%Y%m%d%H%M%S") + local timestamplength="${#TIMESTAMP}" + if [ $timestamplength -eq 12 ]; then + TIMESTAMP="${TIMESTAMP}00" + elif [ $timestamplength -ne 14 ]; then + TIMESTAMP=$(date -d "$DATE" "+%Y%m%d%H%M%S") + fi RPMBUILD=$(which rpmbuild 2>/dev/null | head -1) if [ ! "$RPMBUILD" -o "$RPMBUILD" == "" ]; then @@ -234,6 +299,36 @@ check_options() usage 1 "Could not find binary for making rpms (tried rpmbuild and rpm)." fi fi + + if [ -n "$CCACHE" ]; then + which "$CCACHE" 2>/dev/null && export CCACHE && export CC="ccache gcc" + which "$DISTCC" 2>/dev/null && export DISTCC RPM_BUILD_NCPUS + local bindir="/cache/build/bin" + [ -d $bindir ] || mkdir -p $bindir + [ -d $bindir ] && rm ${bindir}/* > /dev/null 2>&1 + which "$CCACHE" 2>/dev/null && [ -d $bindir ] && ln -s `which "$CCACHE"` ${bindir}/ccache + which "$CCACHE" 2>/dev/null && [ -d $bindir ] && ln -s `which "$CCACHE"` ${bindir}/cc + which "$CCACHE" 2>/dev/null && [ -d $bindir ] && ln -s `which "$CCACHE"` ${bindir}/gcc + [ -d $bindir ] && export PATH=$bindir:$PATH + fi + + [ -z "$DISTRO" ] && autodetect_distro +} + +#autodetect used Distro +autodetect_distro() +{ + if [ -f /etc/SuSE-release ]; then + DISTRO=sles10 + elif [ -f /etc/redhat-release ]; then + local distroname=$(head -1 /etc/redhat-release | grep -e "CentOS\|Red") + if [ ! "$distroname" = "" ]; then + local version=$(echo "$distroname" | sed -e s/[^0-9.]*//g | sed -e s/\\..*// ) + distroname="rhel" + [ "$version" = "" ] || DISTRO="${distroname}${version}" + fi + fi + [ "$DISTRO" = "" ] && DISTRO="sles9" #default distro } uniqify() @@ -277,18 +372,18 @@ build_tarball() { } download_and_build_tarball() { - local TARGET=$1 - local KERNEL_FILE=$2 + local target=$1 + local kernel_file=$2 - local SRPM=kernel-${lnxmaj}-${lnxrel}.src.rpm + local srpm=kernel-${lnxmaj}-${lnxrel}.src.rpm - echo "Downloading http://ftp.lustre.org/kernels/$TARGET/old/$SRPM..." - if ! wget -nv "http://ftp.lustre.org/kernels/$TARGET/old/$SRPM" \ - -O "$KERNELDIR/$SRPM" ; then - fatal 1 "Could not download target $TARGET's kernel SRPM $SRPM from ftp.lustre.org." + echo "Downloading http://ftp.lustre.org/kernels/$target/old/$srpm..." + if ! wget -nv "http://ftp.lustre.org/kernels/$target/old/$srpm" \ + -O "$KERNELDIR/$srpm" ; then + fatal 1 "Could not download target $kernel_file's kernel SRPM $srpm from ftp.lustre.org." fi - build_tarball $TARGET $SRPM + build_tarball $target $srpm } load_target() @@ -308,9 +403,17 @@ load_target() [ "$KERNEL" ] || fatal 1 "Target $TARGET did not specify a kernel." [ "$VERSION" ] || fatal 1 "Target $TARGET did not specify a kernel version." + #CC was overwriten in TARGET_FILE + which "$CCACHE" 2>/dev/null && export CCACHE && export CC="ccache gcc" + + if [ ! "$KERNELTREE" = "" ] && [ -d "$KERNELTREE" ]; then + KERNELDIR="$KERNELTREE/${lnxmaj}" + [ -d "$KERNELDIR" ] || mkdir "$KERNELDIR" + fi + if [ "$KERNELDIR" ] ; then KERNEL_FILE="$KERNELDIR/$KERNEL" - if [ ! -r "$KERNELDIR/$KERNEL" ] ; then + if [ ! -r "$KERNEL_FILE" ] ; then # see if we have an SRPM we can build a tarball for KERNEL_SRPM=kernel-${lnxmaj}-${lnxrel}.src.rpm if [ -r "$KERNELDIR/$KERNEL_SRPM" ] ; then @@ -338,23 +441,47 @@ load_target() done fi - CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/$CONFIG" - [ -r "$CONFIG_FILE" ] || \ - fatal 1 "Target $TARGET's config file $CONFIG missing from $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/." + if [ -f $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET-$TARGET_ARCH.config ]; then + CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET-$TARGET_ARCH.config" + fi + local smptype + for smptype in $SMPTYPES; do + [ "$smptype" = "''" ] && smptype= + if [ -f $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET-$TARGET_ARCH-${smptype}.config ]; then + CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET-$TARGET_ARCH-${smptype}.config" + KERNCONFSMPTYPE=$smptype + fi + done + + local lnxrelnew=$( echo ${lnxrel} | sed s/-/_/g ) + [ -f "$CONFIG_FILE" ] || \ + fatal 1 "Config file for target $TARGET missing from $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/." if [ "$EXTRA_VERSION_save" ] ; then EXTRA_VERSION="$EXTRA_VERSION_save" - elif ! (( $RELEASE )) ; then + elif ! $RELEASE; then # if there is no patch series, then this is not a lustre specific # kernel. don't make it look like one if [ -n "$SERIES" ]; then #remove the @VERSION@ (lustre version) - EXTRA_VERSION=$(echo $EXTRA_VERSION | sed -e "s/\(.*_lustre\)\..*/\1/") - EXTRA_VERSION="${EXTRA_VERSION}-${TAG}.${TIMESTAMP}" +# EXTRA_VERSION=$(echo $EXTRA_VERSION | sed -e "s/\(.*_lustre\)\..*/\1/") +# EXTRA_VERSION="${EXTRA_VERSION}-${TAG}.${TIMESTAMP}" + ! ( $PATCHLESS ) && EXTRA_VERSION="${EXTRA_VERSION}.${TIMESTAMP}" fi fi # EXTRA_VERSION=${EXTRA_VERSION//-/_} + [ -z $REUSEDKERNELMASK ] && \ + # create mask for kernel RPM/builddir which could be reused + if $NORPM; then + REUSEDKERNELMASK=${KERNCONFSMPTYPE}-${lnxmaj}-${lnxrelnew}_lustre.${LUSTRE_VERSION}.*${TARGET_ARCH} + elif $RELEASE; then + REUSEDKERNELMASK=${KERNCONFSMPTYPE}-${lnxmaj}-${lnxrelnew}_lustre.${LUSTRE_VERSION}.${TARGET_ARCH} + else + REUSEDKERNELMASK=${KERNCONFSMPTYPE}-${lnxmaj}-${lnxrelnew}_lustre.${LUSTRE_VERSION}.${TIMESTAMP}.${TARGET_ARCH} + fi + # kernel-lustre-smp-2.6.9-55.0.2.EL_lustre.1.6.1.i686.rpm + ALL_ARCHS="$BASE_ARCHS $BIGMEM_ARCHS $BOOT_ARCHS $JENSEN_ARCHS $SMP_ARCHS $BIGSMP_ARCHS $PSERIES64_ARCHS $UP_ARCHS" BUILD_ARCHS= @@ -418,7 +545,7 @@ unpack_lustre() fatal 1 "There was an error running ./configure to create makefiles." make dist || fatal 1 "There was an error running 'make dist'." popd > /dev/null - fname=`basename $DIRNAME/lustre-*.tar.gz` + fname=$(basename $DIRNAME/lustre-*.tar.gz) cp $DIRNAME/$fname . || fatal 1 "There was an error copying lustre tarball." LUSTRE="$PWD/$fname" ln -sf "$DIRNAME" lustre @@ -511,7 +638,6 @@ prep_kernel_build() -e "s^@RHBUILD@^$RHBUILD^g" \ -e "s^@SMP_ARCHS@^$SMP_ARCHS^g" \ -e "s^@SUSEBUILD@^$SUSEBUILD^g" \ - -e "s^@SUSEBUILD@^$SUSEBUILD^g" \ -e "s^@UP_ARCHS@^$UP_ARCHS^g" \ < $TOPDIR/lustre/build/lustre-kernel-2.4.spec.in \ > lustre-kernel-2.4.spec @@ -548,7 +674,14 @@ build_kernel() targets="--target $arch $targets" done - $RPMBUILD $targets -bb lustre-kernel-2.4.spec \ + local rpmbuildopt='-bb' +# if $NORPM; then +# rpmbuildopt='-bc' +# echo NORPM mode. Only compiling. +# echo "XXX: need to fix lmake - add options to do rpmbuild -bc instead of -bb" +# fi + + $RPMBUILD $targets $rpmbuildopt lustre-kernel-2.4.spec \ --define "_tmppath $TMPDIR" \ --define "_topdir $TOPDIR" || \ fatal 1 "Error building rpms for $BUILD_ARCHS." @@ -559,6 +692,43 @@ build_kernel() --define "_topdir $TOPDIR" || \ fatal 1 "Error building .src.rpm." fi + + ( $(skeep_ldiskfs_rpm $TAG) ) && return + + pushd $TOPDIR/BUILD/lustre*/ldiskfs || return 255 + make dist + if [ "$?" != "0" ] ; then + popd + return 255 + fi + cp lustre-ldiskfs*.tar.gz $TOPDIR/SOURCES + + gen_lustre_version + + local ldiskfs_spec=lustre-ldiskfs.spec + [ -f "$ldiskfs_spec" ] && sed \ + -e "s^Release: .*$^Release: $LUSTRE_EXTRA_VERSION^" \ + < $ldiskfs_spec \ + > ../lustre-ldiskfs.spec + + $RPMBUILD $targets $rpmbuildopt ../lustre-ldiskfs.spec \ + --define "_tmppath /var/tmp" \ + --define "_topdir $TOPDIR" + if [ "$?" != "0" ] ; then + popd + return 255 + fi + + if (( $DO_SRC )) ; then + $RPMBUILD -bs ../lustre-ldiskfs.spec \ + --define "_tmppath /var/tmp" \ + --define "_topdir $TOPDIR" + if [ "$?" != "0" ] ; then + popd + return 255 + fi + fi + popd } build_lustre() @@ -578,14 +748,84 @@ build_lustre() targets="--target $arch $targets" done - ./configure "--with-linux=${LINUX}" ${CONFIGURE_FLAGS} + local confoptions="--with-linux=${LINUX}" + if $PATCHLESS; then + confoptions="--with-linux=${LINUX} --disable-server" + fi + if [ ! "$LINUXOBJ" = "" ]; then + confoptions="$confoptions --with-linux-obj=${LINUXOBJ}" + fi + + ./configure $confoptions ${CONFIGURE_FLAGS} + if [ "$?" != "0" ] ; then + local saved_config="../config.log.$(date +%s)" + cp config.log $saved_config + chmod a+r $saved_config + echo "Saved config.log is at $saved_config" + popd + return 255 + fi + + gen_lustre_version + + # hack. Somebody move build/lustre.spec to lustre.spec for b1_6 + local lustre_spec= + [ -f lustre.spec ] && lustre_spec=lustre.spec + [ -f build/lustre.spec ] && lustre_spec=build/lustre.spec + + [ -f "$lustre_spec" ] && sed \ + -e "s^Release: .*$^Release: $LUSTRE_EXTRA_VERSION^" \ + < $lustre_spec \ + > ../lustre.spec - $RPMBUILD $targets -bb lustre.spec \ + local rpmbuildopt='-bb' + if $NORPM; then + rpmbuildopt='-bc' + echo NORPM mode. Only compiling. + fi + + $RPMBUILD $targets $rpmbuildopt ../lustre.spec \ --define "_tmppath $TMPDIR" \ --define "_topdir $TOPDIR" || \ fatal 1 "Error building rpms for $BUILD_ARCHS." popd >/dev/null + ( $(skeep_ldiskfs_rpm $TAG) ) && return + + pushd lustre/ldiskfs || return 255 + make dist + if [ "$?" != "0" ] ; then + popd + return 255 + fi + cp lustre-ldiskfs*.tar.gz $TOPDIR/SOURCES + + gen_lustre_version + + local ldiskfs_spec=lustre-ldiskfs.spec + [ -f "$ldiskfs_spec" ] && sed \ + -e "s^Release: .*$^Release: $LUSTRE_EXTRA_VERSION^" \ + < $ldiskfs_spec \ + > ../lustre-ldiskfs.spec + + $RPMBUILD $targets $rpmbuildopt ../lustre-ldiskfs.spec \ + --define "_tmppath /var/tmp" \ + --define "_topdir $TOPDIR" + if [ "$?" != "0" ] ; then + popd + return 255 + fi + + if (( $DO_SRC )) ; then + $RPMBUILD -bs ../lustre-ldiskfs.spec \ + --define "_tmppath /var/tmp" \ + --define "_topdir $TOPDIR" + if [ "$?" != "0" ] ; then + popd + return 255 + fi + fi + popd } stage() @@ -605,9 +845,545 @@ stage() cp -v "$LUSTRE" "$STAGEDIR" } +#check if we need to build separate ldiskfs RPM +skeep_ldiskfs_rpm() +{ + local tag=$1 + local skip=false + if ! $LDISKFSRPM; then + skip=true + elif $PATCHLESS; then + skip=true + else + for skiptag in $SKIPLDISKFSRPM; do + [[ $tag == $skiptag ]] && skip=true && break + done + fi + echo $skip +} + +#get date of last changed target/config/series/patches +get_last_source_date() +{ + local filelist="${TOPDIR}/lustre/lustre/kernel_patches/series/${SERIES} \ + $CONFIG_FILE" + local TOPDIRnew=$(echo ${TOPDIR} | sed -e s/\\//\\\\\\//g) + filelist="$filelist $( \ + cat ${TOPDIR}/lustre/lustre/kernel_patches/series/${SERIES} | \ + sed -e s/^/${TOPDIRnew}\\/lustre\\/lustre\\/kernel_patches\\/patches\\// 2>&1)" + local sourcelastdate=$( find ${filelist} -name CVS -prune -o \ + -type f -printf "%T@\n" 2>&1 | sort | tail -1 ) + is_integer $sourcelastdate && echo $sourcelastdate +} + +#check if variable is integer +is_integer() +{ + local invariable=$1 + [ "$invariable" = "" ] && return 255 + local invariableint=$( echo $invariable | sed -e s/[^0-9]//g ) + [ "$invariable" = "$invariableint" ] || return 255 +} + +#generate LUSTRE_EXTRA_VERSION from EXTRA_VERSION +gen_lustre_version() +{ + local smptype=smp + [ "$KERNCONFSMPTYPE" = "" ] || smptype=$KERNCONFSMPTYPE + [ "$RPMSMPTYPE" = "" ] || smptype=$RPMSMPTYPE + + LUSTRE_EXTRA_VERSION="${lnxmaj}-${EXTRA_VERSION}" + if [ "$PATCHLESS" = "true" -a "$DISTRO" = "sles10" ]; then + LUSTRE_EXTRA_VERSION="${LUSTRE_EXTRA_VERSION}-${smptype}" + else + LUSTRE_EXTRA_VERSION="${LUSTRE_EXTRA_VERSION}${smptype}" + fi + LUSTRE_EXTRA_VERSION=$( echo $LUSTRE_EXTRA_VERSION | sed -e "s^-^_^g" ) +} + +#store RPMs and/or BUILD dir for feature reuse +store_for_reuse() +{ + local rpmonly=$1 + if [ ! "$REUSEBUILD" = "" ] && [ -d "/$REUSEBUILD/" ] ; then + [ -d "${REUSEBUILD}/${TIMESTAMP}" ] || mkdir "${REUSEBUILD}/${TIMESTAMP}" + [ -d "${REUSEBUILD}/${TIMESTAMP}" ] || return 255 + else + return 255 + fi + + local lnxrelnew=$( echo ${lnxrel} | sed s/-/_/g ) + local EXTRA_VERSIONnew=$( echo ${EXTRA_VERSION} | sed s/-/_/g ) + local KERNELRPMnew=$(basename "$KERNELRPM") + if [ ! "$rpmonly" = "rpmonly" ]; then + local builddir= + if [ ! "$KERNELCOMPILEDIR" = "" ]; then + builddir="$KERNELCOMPILEDIR" + else + builddir="BUILD/lustre-kernel-${lnxmaj}/lustre/linux-${lnxmaj}" + [ "$KERNELCOMPILEDIR" = "" ] || builddir="$KERNELCOMPILEDIR" + [ -d "$builddir" ] || builddir="BUILD/lustre-kernel-${lnxmaj}/lustre/linux-${lnxmaj}.${lnxrel}" + [ -d "$builddir" ] || builddir="BUILD/lustre-kernel-${lnxmaj}/lustre/linux-${lnxmaj}-${lnxrel}" + if [ ! -d "$builddir" ]; then + pushd "BUILD/lustre-kernel-${lnxmaj}/lustre/" || return 255 + local basebuilddir=$(ls -d linux-${lnxmaj}* | head -1) + [ "$basebuilddir" = "" ] || builddir="BUILD/lustre-kernel-${lnxmaj}/lustre/${basebuilddir}" + popd + fi + fi + [ -d "$builddir" ] || return 255 + local dstdir="${REUSEBUILD}/${TIMESTAMP}/linux-${KERNCONFSMPTYPE}-${lnxmaj}-${EXTRA_VERSIONnew}.${TARGET_ARCH}" + ( $PATCHLESS ) && dstdir="${REUSEBUILD}/${TIMESTAMP}/linux-$KERNELRPMnew" && \ + dstdir="${dstdir%.rpm}" + [ -d "$dstdir" ] && rm -rf "$dstdir" + mv "${builddir}" "$dstdir" || return 255 + fi + #store kernel rpm + local kernelrpmname="kernel-lustre-${KERNCONFSMPTYPE}-${lnxmaj}-${EXTRA_VERSIONnew}.${TARGET_ARCH}.rpm" + [ -f "RPMS/${TARGET_ARCH}/${kernelrpmname}" ] || kernelrpmname="kernel-${KERNCONFSMPTYPE}-${lnxmaj}-${EXTRA_VERSNnew}.${TARGET_ARCH}.rpm" + ( $PATCHLESS ) && [ -f "$KERNELRPM" ] && kernelrpmname="$KERNELRPMnew" + if [ "$rpmonly" = "rpmonly" ] && [ -f "${REUSEBUILD}/${TIMESTAMP}/${kernelrpmname}" ]; then + echo "RPM already exist in store directory tree" + else + [ -f "RPMS/${TARGET_ARCH}/${kernelrpmname}" ] && cp -f "RPMS/${TARGET_ARCH}/${kernelrpmname}" "${REUSEBUILD}/${TIMESTAMP}/" + fi + #store kernel source rpm + kernelrpmname="kernel-lustre-source-${lnxmaj}-${EXTRA_VERSIONnew}.${TARGET_ARCH}.rpm" + [ -f "RPMS/${TARGET_ARCH}/${kernelrpmname}" ] || kernelrpmname="kernel-source-${lnxmaj}-${EXTRA_VERSIONnew}.${TARGET_ARCH}.rpm" + ( $PATCHLESS ) && [ -f "$KERNELSOURCERPM" ] && kernelrpmname=$(basename "$KERNELSOURCERPM") + if [ "$rpmonly" = "rpmonly" ] && [ -f "${REUSEBUILD}/${TIMESTAMP}/${kernelrpmname}" ]; then + echo "RPM already exist in store directory tree" + else + [ -f "RPMS/${TARGET_ARCH}/${kernelrpmname}" ] && cp -f "RPMS/${TARGET_ARCH}/${kernelrpmname}" "${REUSEBUILD}/${TIMESTAMP}/" + fi +} + +#unpack kernel(/source/devel) RPM +unpack_linux_rpm() +{ + local prefix=$1 + local pathtorpms="${KERNELRPMSBASE}/${lnxmaj}/${DISTRO}" + local kernelbinaryrpm= + [ -d $pathtorpms ] || return 255 + local rpmfile= + local smptype= + for arch in $TARGET_ARCHS_ALL; do + for smptype in $SMPTYPES; do + [ "$smptype" = "''" ] && smptype= + [ "$smptype" = "" ] || smptype="-${smptype}" + case "$DISTRO" in + sles10) + rpmfile="kernel${smptype}${prefix}-${lnxmaj}.${lnxrel}.${arch}.rpm" + kernelbinaryrpm="kernel${smptype}-${lnxmaj}.${lnxrel}.${arch}.rpm" + ;; + *) + rpmfile="kernel${smptype}${prefix}-${lnxmaj}-${lnxrel}.${arch}.rpm" + kernelbinaryrpm="kernel${smptype}-${lnxmaj}-${lnxrel}.${arch}.rpm" + ;; + esac + [ -f "${pathtorpms}/${arch}/${kernelbinaryrpm}" ] && KERNELRPM="${pathtorpms}/${arch}/${kernelbinaryrpm}" + [ -f "${pathtorpms}/${arch}/${rpmfile}" ] && RPMSMPTYPE=${smptype/-/} && break + done + [ -f "${pathtorpms}/${arch}/${kernelbinaryrpm}" ] && KERNELRPM="${pathtorpms}/${arch}/${kernelbinaryrpm}" + [ -f "${pathtorpms}/${arch}/${rpmfile}" ] && TARGET_ARCH="$arch" && BUILD_ARCHS="$arch" + [ -f "${pathtorpms}/${arch}/${rpmfile}" ] && break + done + [ -f "${pathtorpms}/${TARGET_ARCH}/${rpmfile}" ] || return 255 + [ -d $TOPDIR/reused ] || mkdir $TOPDIR/reused + pushd $TOPDIR/reused || return 255 + rpm2cpio < "${pathtorpms}/${TARGET_ARCH}/${rpmfile}" | cpio -idc > /dev/null 2>&1 + if [ ${PIPESTATUS[0]} -eq 0 ]; then + local path2source=$(ls usr/src/kernels/ | grep ${lnxmaj}) + [ -d "usr/src/kernels/${path2source}" ] && LINUX="$(pwd)/usr/src/kernels/${path2source}" + else + popd + return 255 + fi + popd +} + +#look for kernel source RPM +find_linux_source_rpm() +{ + local rpmfile= + local findarch=true + local arch= + local pathtorpms= + [ ! "$TARGET_ARCH" = "" ] && arch=$TARGET_ARCH && findarch=false + + if ! $findarch; then + pathtorpms="${KERNELRPMSBASE}/${lnxmaj}/${DISTRO}/${arch}" + [ -d $pathtorpms ] || return 255 + case "$DISTRO" in + rhel4) + rpmfile="kernel-${lnxmaj}-${lnxrel}.src.rpm" + ;; + sles10) + rpmfile="kernel-source-${lnxmaj}.${lnxrel}.${arch}.rpm" + ;; + *) + rpmfile="kernel-source-${lnxmaj}-${lnxrel}.${arch}.rpm" + ;; + esac + [ -f "${pathtorpms}/${rpmfile}" ] || return 255 + KERNELSOURCERPM="${pathtorpms}/${rpmfile}" + else + for arch in $TARGET_ARCHS_ALL; do + pathtorpms="${KERNELRPMSBASE}/${lnxmaj}/${DISTRO}/${arch}" + [ -d $pathtorpms ] || continue + case "$DISTRO" in + rhel4) + rpmfile="kernel-${lnxmaj}-${lnxrel}.src.rpm" + ;; + sles10) + rpmfile="kernel-source-${lnxmaj}.${lnxrel}.${arch}.rpm" + ;; + *) + rpmfile="kernel-source-${lnxmaj}-${lnxrel}.${arch}.rpm" + ;; + esac + [ -f "${pathtorpms}/${rpmfile}" ] || continue + KERNELSOURCERPM="${pathtorpms}/${rpmfile}" + TARGET_ARCH=${arch} + break + done + fi + [ -f "${KERNELSOURCERPM}" ] || return 255 +} + +#unpack and make symlinks for reusing kernel RPM +reuse_kernel_rpm() +{ + local pathtorpm=$1 + [ "$pathtorpm" = "" ] && return 255 + [ -f "$pathtorpm" ] || return 255 + [ -d $TOPDIR/reused ] || mkdir $TOPDIR/reused + pushd $TOPDIR/reused || return 255 + + rpm2cpio < $pathtorpm | cpio -idc + [ ${PIPESTATUS[0]} -eq 0 ] || return 255 + + local smptype= + if pushd usr/src/linux-*-obj/${TARGET_ARCH}; then + local smptypes="$SMPTYPES" + [ "$RPMSMPTYPE" = "" ] || smptypes=$RPMSMPTYPE + ( ! $PATCHLESS ) && [ ! "$KERNCONFSMPTYPE" = "" ] && smptypes="$KERNCONFSMPTYPE" + local cursmptype= + for cursmptype in $smptypes; do + [ "$cursmptype" = "''" ] && continue + [ -d $cursmptype ] && smptype=$cursmptype + [ -d $smptype ] && break + done + popd + fi + if [ "${smptype}" = "" ]; then + popd + return 255 # cannot detect smp type + fi + if pushd usr/src/linux-*-obj/${TARGET_ARCH}/$smptype/include2; then + local base=$(readlink asm) + if [ ! -d "/${base}/" ]; then + rm -f asm + base=$(basename "$base") + if pushd ../../../../linux-*/include; then + local lsrc=$(pwd) + popd + [ -d "$lsrc/${base}" ] && ln -s $lsrc/${base} asm + fi + fi + popd + read a b < <(echo $(pwd)/usr/src/linux-*) + if [[ $a == $(pwd)/* && $b = $(pwd)/* ]]; then + cp -f $a/include/linux/config.h $b/${TARGET_ARCH}/$smptype/include/linux/ + cp $b/${TARGET_ARCH}/$smptype/.config $a/ + [ -f "$b/${TARGET_ARCH}/$smptype/.config" ] && KERNELRPMCONFIG="$b/${TARGET_ARCH}/$smptype/.config" + cp $b/${TARGET_ARCH}/$smptype/.kernelrelease $a/ + [ -f "$b/${TARGET_ARCH}/$smptype/.kernelrelease" ] && KERNELRPMRELEASE="$b/${TARGET_ARCH}/$smptype/.kernelrelease" + LINUX=$a + LINUXOBJ=$b/${TARGET_ARCH}/$smptype +# local fname=$(basename $kernel_rpm) +# KERNELRPMSDIR=${kernel_rpm%$fname} + fi + fi + popd + [ "$LINUX" = "" ] && return 255 + [ -d "$LINUX" ] || return 255 +} + +#build linux kernel rpm +build_linux_rpm() +{ + pushd $LINUX || return 255 + make binrpm-pkg || ( popd ; return 255 ) + local addlnxrel= + [ -f ".version" ] && addlnxrel="-$(cat .version)" + popd + local arch= + for arch in $TARGET_ARCHS_ALL; do + [ -f "/usr/src/rpm/RPMS/$arch/kernel-$lnxmaj.${lnxrel}${addlnxrel}.$arch.rpm" ] && \ + KERNELRPM="/usr/src/rpm/RPMS/$arch/kernel-$lnxmaj.${lnxrel}${addlnxrel}.$arch.rpm" && \ + TARGET_ARCH="$arch" + [ -f "/usr/src/packages/RPMS/$arch/kernel-$lnxmaj.${lnxrel}${addlnxrel}.$arch.rpm" ] && \ + KERNELRPM="/usr/src/packages/RPMS/$arch/kernel-$lnxmaj.${lnxrel}${addlnxrel}.$arch.rpm" && \ + TARGET_ARCH="$arch" + done + [ "$KERNELRPM" = "" ] || return + return 255 +} + +#build linux kernel +build_linux() +{ + local nofullmake=$1 + local nocopykernel=$2 + pushd $LINUX || fatal 1 "Kernel source not found" + [ "$nofullmake" = "nofullmake" ] || make mrproper + [ "$nofullmake" = "nofullmake" ] || rm -f rpm-release +# [ "$nocopykernel" = "copyrpmkernel" ] || rm -f localversion-* + [ "$nocopykernel" = "copykernel" ] && [ -f "$CONFIG_FILE" ] && cp $CONFIG_FILE .config + if [ "$nocopykernel" = "copyrpmkernel" ]; then + [ -f "$KERNELRPMCONFIG" ] && cp $KERNELRPMCONFIG .config + if [ -f "$KERNELRPMRELEASE" ]; then + cp $KERNELRPMRELEASE . + else + sed -e "s/^EXTRAVERSION\s\+=\s\+.*$/EXTRAVERSION = -${lnxrel}/" < Makefile > Makefile.new + [ -f "Makefile.new" ] && mv Makefile.new Makefile + fi + fi + [ -f ".config" ] || ( popd ; echo "Cannot find .config file"; return 255 ) + make oldconfig || ( popd ; return 255 ) + make include/linux/version.h + if [ ! "$nofullmake" = "nofullmake" ]; then + make || ( popd ; return 255 ) + fi + popd + return +} + +#build patchless lustre +patchless_build_sequence() +{ + #try to build from kernel-devel RPM (RHEL) + LINUX= + TARGET_ARCH= + local rpmfound=false + local buildsuccess=false + local storeforreuse=false + if [ "$KERNELRPMSBASE" = "" ] || [ ! -d "$KERNELRPMSBASE" ]; then + return 255 + fi + [ -d $TOPDIR/reused ] && rm -rf $TOPDIR/reused + unpack_linux_rpm "-devel" && rpmfound=true + ( $rpmfound ) && build_lustre && buildsuccess=true && find_linux_source_rpm + + if ! $buildsuccess; then + # cannot build from -devel. Try to build from kernel source RPM without rebuilding kernel + KERNELSOURCERPM= + # first unpack kernel binary RPM + rpmfound=false + [ -d $TOPDIR/reused ] && rm -rf $TOPDIR/reused + unpack_linux_rpm && rpmfound=true + ! ( $rpmfound ) && echo "Cannot find kernel RPM" + + if ! $rpmfound; then #kernel RPM not found. + # try to reuse + build_sequence_reuse && buildsuccess=true + if ! $buildsuccess; then + # reuse failed. Try to build from tarbal + KERNELRPM= + KERNELSOURCERPM= + unpack_linux && LINUX="$TOPDIR/linux" && \ + build_linux fullmake copykernel && build_linux_rpm + ! [ "$KERNELRPM" = "" ] && build_lustre && storeforreuse=true && buildsuccess=true + fi + else + # next find and unpack source RPM + rpmfound=false + find_linux_source_rpm && rpmfound=true + ( $rpmfound ) && \ + reuse_kernel_rpm "$KERNELSOURCERPM" && build_linux nofullmake copyrpmkernel && build_lustre && buildsuccess=true + fi + fi + + if ! $buildsuccess && $rpmfound; then + # build wrong. try to build kernel first + REUSEDKERNELMASK=$(basename $KERNELRPM | sed s/\\.rpm//) + build_sequence_reuse && buildsuccess=true + fi + + if ! $buildsuccess && $rpmfound; then + build_linux fullmake copyrpmkernel && build_lustre && buildsuccess=true && storeforreuse=true + fi + + if $buildsuccess; then + [ -d "RPMS/${TARGET_ARCH}" ] && [ -f "$KERNELRPM" ] && \ + cp "$KERNELRPM" RPMS/${TARGET_ARCH}/ + [ -d "RPMS/${TARGET_ARCH}" ] && [ -f "$KERNELSOURCERPM" ] && \ + cp "$KERNELSOURCERPM" RPMS/${TARGET_ARCH}/ + KERNELCOMPILEDIR="$LINUX" + if $storeforreuse; then + store_for_reuse || echo "Cannot store for feature reuse" + fi + return + fi + + # patchless build filed + return 255 +} + +#check timestamp value. should bi 14-digits string +check_timestamp() +{ + local invalue=$1 + local timestampnodig=$(echo $invalue | sed -e s/[0-9]*//g) + [ "$timestampnodig" = "" ] || return 255 + local timestamplength="${#invalue}" + [ $timestamplength -eq 14 ] || return 255 +} + +# get list of suitable directories with potential reused staff +get_reuse_dir_list() +{ + local rpmonly=$1 + local reusedkernelmasknew=$2 + local buildtimestamp= + local dirsforreuse= + local sourcelastdate=$(get_last_source_date) + for buildtimestamp in $(ls "$REUSEBUILD/" 2>&1); do + [ -d "$REUSEBUILD/$buildtimestamp" ] || continue + check_timestamp "$buildtimestamp" || continue + local buildtimestampstr=$(echo $buildtimestamp | \ + sed -e "s^\(....\)\(..\)\(..\)\(..\)\(..\)\(..\)^\1-\2-\3 \4:\5:\6 GMT^g") + local buildtimestampepoch=$(date --date="$buildtimestampstr" +%s ) + #check for suitable date + if ! $PATCHLESS; then + [ $buildtimestampepoch -ge $sourcelastdate ] || continue + fi + #check for suitable version + if [ "$rpmonly" = "rpmonly" ]; then + local reusedkernelprefix="kernel-lustre-" + ( $PATCHLESS ) && reusedkernelprefix= + local rpmmask="${reusedkernelprefix}${REUSEDKERNELMASK}" + [ "$reusedkernelmasknew" = "" ] || rpmmask="$reusedkernelmasknew" + [ -f $REUSEBUILD/$buildtimestamp/${rpmmask}.rpm ] && \ + dirsforreuse="$dirsforreuse $REUSEBUILD/$buildtimestamp" + else + local rpmmask="$REUSEDKERNELMASK" + [ "$reusedkernelmasknew" = "" ] || rpmmask="$reusedkernelmasknew" + pushd $REUSEBUILD/$buildtimestamp/linux-${rpmmask} > /dev/null 2>&1 || continue + local curdir=$(pwd) + dirsforreuse="$dirsforreuse $curdir" + popd + fi + done + echo "$dirsforreuse" +} + +#try to reuse old RPM +build_sequence_rpm_reuse() +{ + local sourcerpm=$1 + [ "$REUSERPM" = "" ] && [ "$REUSEBUILD" = "" ] && return 255 + local dirsforreuse= + if ! [ "$REUSEBUILD" = "" ] && [ -d "$REUSEBUILD" ]; then #try to reuse RPM + local REUSEDKERNELMASKnew=$(echo $REUSEDKERNELMASK | sed -e "s/^[^0-9]*//") + REUSEDKERNELMASKnew="kernel-lustre-source-${REUSEDKERNELMASKnew}" + local dirsforreuse="$(get_reuse_dir_list rpmonly $REUSEDKERNELMASKnew)" + local buildsuccess=false + LINUXOBJ= + for curdir in $(echo $dirsforreuse); do + [ -d "$curdir" ] || continue + local reusedkernelprefix="kernel-lustre-" + local reusedkernelrpm= + [ -f ${curdir}/${reusedkernelprefix}${REUSEDKERNELMASK}.rpm ] && \ + reusedkernelrpm=$(ls ${curdir}/${reusedkernelprefix}${REUSEDKERNELMASK}.rpm | head -1 ) + [ -f "$reusedkernelrpm" ] || continue + + local reusedkernelsourcerpm= + [ -f ${curdir}/${REUSEDKERNELMASKnew}.rpm ] && \ + reusedkernelsourcerpm=$(ls ${curdir}/${REUSEDKERNELMASKnew}.rpm | head -1 ) + [ -f "$reusedkernelsourcerpm" ] || continue + + [ -d $TOPDIR/reused ] && rm -rf $TOPDIR/reused + reuse_kernel_rpm "$reusedkernelsourcerpm" && build_linux nofullmake copyrpmkernel && build_lustre && buildsuccess=true + ( $buildsuccess ) || continue + if ( ! $NORPM ) && ( ! $PATCHLESS ) ; then + [ -f "$reusedkernelrpm" ] && \ + cp -f "$reusedkernelrpm" RPMS/${TARGET_ARCH}/ > /dev/null 2>&1 + + [ -f "$reusedkernelsourcerpm" ] && \ + cp -f "$reusedkernelsourcerpm" RPMS/${TARGET_ARCH}/ > /dev/null 2>&1 && \ + touch RPMS/${TARGET_ARCH}/kernel_was_reused + fi + return + done + fi + return 255 +} + +#try to reuse old BUILD dir +build_sequence_reuse() +{ + local sourcerpm=$1 + [ "$REUSERPM" = "" ] && [ "$REUSEBUILD" = "" ] && return 255 + local dirsforreuse= + if [ ! "$REUSEBUILD" = "" ] && [ -d "$REUSEBUILD" ]; then #try to reuse old kernel build directory + local dirsforreuse="$(get_reuse_dir_list)" + local buildsuccess=false + LINUXOBJ= + local REUSEDKERNELMASKnew=$(echo $REUSEDKERNELMASK | sed -e "s/^[^0-9]*//") + for curdir in $(echo $dirsforreuse); do + local reusedkernelrpm= + local reusedkernelsourcerpm= + [ -d "$curdir" ] || continue + local reusedkernelprefix="kernel-lustre-" + ( $PATCHLESS ) && reusedkernelprefix= + [ -f ${curdir}/../${reusedkernelprefix}${REUSEDKERNELMASK}.rpm ] && \ + reusedkernelrpm=$(ls ${curdir}/../${reusedkernelprefix}${REUSEDKERNELMASK}.rpm | head -1 ) + reusedkernelprefix="kernel-lustre-source-" + [ -f ${curdir}/../${reusedkernelprefix}${REUSEDKERNELMASKnew}.rpm ] && \ + reusedkernelsourcerpm=$(ls ${curdir}/../${reusedkernelprefix}${REUSEDKERNELMASKnew}.rpm | head -1 ) + if ! ( $NORPM ) && ! [ -f "$reusedkernelrpm" ]; then #kernel rpm not found. Build all + continue + fi + if ! ( $NORPM ) && ! [ -f "$reusedkernelsourcerpm" ]; then #kernel source rpm not found. Build all + continue + fi + LINUX="$curdir" + build_lustre || continue + touch "$curdir/../" + buildsuccess=true + if ( ! $NORPM ) && ( ! $PATCHLESS ) ; then + [ -f "$reusedkernelrpm" ] && \ + cp -f "$reusedkernelrpm" RPMS/${TARGET_ARCH}/ > /dev/null 2>&1 && \ + touch RPMS/${TARGET_ARCH}/kernel_was_reused + [ -f "$reusedkernelsourcerpm" ] && \ + cp -f "$reusedkernelsourcerpm" RPMS/${TARGET_ARCH}/ > /dev/null 2>&1 + fi + return + done + fi + return 255 +} + + +build_sequence() +{ + if (( $DO_SRC )) ; then + unpack_linux + patch_linux + pack_linux + clean_linux + fi + prep_kernel_build || return 255 + clean_lustre || return 255 + + build_kernel || return 255 +} + [ -r ~/.lbuildrc ] && . ~/.lbuildrc -options=$(getopt -o d:D:h -l disable-datestamp,external-patches:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,publish,release,src,stage:,tag:,target:,target-archs:,with-linux: -- "$@") +options=$(getopt -o d:D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ldiskfs,ccache,reuse:,norpm,disable-datestamp,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,publish,release,src,stage:,tag:,target:,target-archs:,with-linux: -- "$@") if [ $? != 0 ] ; then usage 1 @@ -620,6 +1396,10 @@ while [ "$1" ] ; do '') usage 1 ;; + --ccache) + CCACHE='ccache' + shift + ;; -d) CVSROOT=$2 shift 2 @@ -643,10 +1423,46 @@ while [ "$1" ] ; do KERNELDIR=$2 shift 2 ;; + --kerneltree) + KERNELTREE=$2 + shift 2 + ;; --linux | --with-linux) LINUX=$2 shift 2 ;; + --distro) + DISTRO=$2 + shift 2 + ;; + --reuserpm) + REUSERPM=$2 + shift 2 + ;; + --reusebuild) + REUSEBUILD=$2 + shift 2 + ;; + --norpm) + NORPM=true + shift + ;; + --ldiskfs) + LDISKFSRPM=true + shift + ;; + --patchless) + PATCHLESS=true + shift + ;; + --kernelrpm) + KERNELRPMSBASE=$2 + shift 2 + ;; + --timestamp) + TIMESTAMP=$2 + shift 2 + ;; --lustre) LUSTRE=$2 shift 2 @@ -663,7 +1479,7 @@ while [ "$1" ] ; do shift ;; --release) - RELEASE=1 + RELEASE=true shift ;; --src) @@ -693,6 +1509,7 @@ while [ "$1" ] ; do --) shift CONFIGURE_FLAGS=$@ + CONFIGURE_FLAGS="$CONFIGURE_FLAGS --enable-liblustre --enable-liblustre-tests" break ;; *) @@ -705,22 +1522,25 @@ check_options unpack_lustre -# prep_build needs the .spec.in from the lustre source -if [ -z "$LINUX" ] ; then - load_target - if (( $DO_SRC )) ; then - unpack_linux - patch_linux - pack_linux - clean_linux +load_target + +build_success=false +if $PATCHLESS; then + patchless_build_sequence && build_success=true +elif [ -z "$LINUX" ] ; then + [ "$DISTRO" = "sles9" ] && build_sequence_rpm_reuse && build_success=true + if ! $build_success; then + build_sequence_reuse && build_success=true + if ! $build_success; then + build_sequence && build_success=true + if $build_success; then + store_for_reuse || echo "Cannot store for feature reuse" + fi + fi fi - - prep_kernel_build - clean_lustre - - build_kernel else build_lustre fi +( $build_success ) || fatal 1 "Cannot build lustre" stage -- 1.8.3.1