X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=contrib%2Flbuild%2Flbuild;h=17f62b53ae51731d76ba885a64e817a2c7279c28;hp=b4a0a8d26579a7503670990f11ccfc5e1b6ceece;hb=28254f0b2726bc540d57dac1dbb16e8a3e45b411;hpb=a6c26e19f7f189c78c7efa9e682b78e0e5f80d73 diff --git a/contrib/lbuild/lbuild b/contrib/lbuild/lbuild index b4a0a8d..17f62b5 100755 --- a/contrib/lbuild/lbuild +++ b/contrib/lbuild/lbuild @@ -38,7 +38,6 @@ BUILD_GEN=6 # TT-1092: don't cache the BUILD dir, to rebuild external OFED TOPDIR=$PWD -# CVSROOT is inherited from the environment KERNELDIR= LINUX= LUSTRE= @@ -48,7 +47,6 @@ RELEASE=false # simply don't account for this option DO_SRC=true DOWNLOAD=true -TAG= CANONICAL_TARGET= TARGET= TARGET_ARCH="$(uname -m)" @@ -72,10 +70,8 @@ USE_BUILD_CACHE=true # what does this do exactly? does it imply no kernel build? NORPM=false IOKITRPM=true -LDISKFSRPM=true OSDLDISKFSRPM=true OSDZFSRPM=false -SKIPLDISKFSRPM="v1_4_* b1_4" SMPTYPES="smp bigsmp default ''" PATCHLESS=false XEN=false @@ -110,7 +106,6 @@ LUSTRE_TESTS=true DATE=$(date) -USE_DATESTAMP=1 RPMBUILD= export CC=${CC:-gcc} @@ -131,11 +126,6 @@ usage() { cat <] - -d CVSROOT - Specifies the CVS Root to use when pulling files from CVS. The - environment variable \$CVSROOT is used if this option is not - present. - --external-patches=EXTERNAL_PATCHES Directory similar to lustre/lustre/kernel_patches/ that lbuild should look for seres and config files in before looking in the lustre @@ -181,8 +171,7 @@ Usage: ${0##*/} [OPTION]... [-- ] Lustre modules and userspace are built. --lustre=LUSTRE - Path to an existing lustre source tarball to use instead of - pulling from CVS. + Path to an existing lustre source tarball to use. --nodownload Do not try to download a kernel from downloads.lustre.org @@ -192,7 +181,7 @@ Usage: ${0##*/} [OPTION]... [-- ] tarball. --ldiskfs - Do ldiskfs RPM. Now true by default + Build with ldiskfs support. (Deprecated, always true) --noiokit Do not build lustre-iokit RPM. Now true by default @@ -212,9 +201,6 @@ Usage: ${0##*/} [OPTION]... [-- ] more or less in DIR/-, and the tarball will be placed in DIR. - --tag=TAG - A CVS branch/tag name to build from when pulling from CVS. - --target=TARGET The name of the target to build. The available targets are listed below. @@ -229,11 +215,6 @@ Usage: ${0##*/} [OPTION]... [-- ] Also note that by using a non-"base" arch (eg, i386) only kernels will be built - there will be no lustre-lite-utils package. - --disable-datestamp - Prevents the datestamp flag (-D) from being passed to cvs for - checkouts. This is a workaround for a problem encountered when - using lbuild with tinderbox. - --xen Builds a Xen domX kernel. @@ -280,14 +261,8 @@ canon_path() { check_options() { - if [ "$LUSTRE" ]; then - [ -r "$LUSTRE" ] || \ - usage 1 "Could not find Lustre source tarball '$LUSTRE'." - else - [ "$CVSROOT" ] || \ - usage 1 "Either specify a CVS Root with -d, or a Lustre source tarball with --lustre." - [ "$TAG" ] || \ - usage 1 "A branch/tag name must be specified with --tag when not building from a tarball." + if [ -z "$LUSTRE" -o ! -r "$LUSTRE" ]; then + usage 1 "Could not find Lustre source tarball '$LUSTRE'." fi [ -z "$DISTRO" ] && DISTRO=$(autodetect_distro) @@ -299,11 +274,6 @@ check_options() { [ -d "$KERNELDIR" -o -d "$KERNELTREE" ] || \ usage 1 "$KERNELDIR and $KERNELTREE are not a directory." - if ! $RELEASE; then - [ "$TAG" ] || \ - usage 1 "When building a snapshot, a tag name must be used." - fi - [ "$TARGET" ] || TARGET=$(autodetect_target "$DISTRO") # TARGET_FILE="$TOPDIR/lustre/kernel_patches/targets/$TARGET.target" # [ -r "$TARGET_FILE" ] || \ @@ -568,9 +538,9 @@ download_ofed() { local location="http://www.openfabrics.org/downloads/OFED/ofed-${OFED_BASE_VERSION}/" - if [[ $OFED_VERSION = *-rc[0-9] ]]; then + if [[ $OFED_VERSION = *-[rR][cC][0-9] ]]; then local Mmv - Mmv=${OFED_VERSION%%-*} + Mmv=${OFED_VERSION%%-[rR][cC][0-9]} location="http://www.openfabrics.org/downloads/OFED/ofed-${Mmv}/" fi @@ -650,7 +620,7 @@ load_target() { fi if [ -f $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET-$TARGET_ARCH.config ]; then - CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET$XENPOSTFIX-$TARGET_ARCH${RPMSMPTYPE:+-}${RPMSMPTYPE}.config" + CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/kernel-$lnxmaj-$TARGET$XENPOSTFIX-$TARGET_ARCH.config" fi local lnxrelnew=${lnxrel//-/_} @@ -672,7 +642,6 @@ load_target() { # kernel. don't make it look like one if $PATCHLESS || [ -n "$SERIES" ]; then EXTRA_VERSION=$(echo $EXTRA_VERSION | sed -e "s/\(.*_lustre\)\..*/\1/") -# EXTRA_VERSION="${EXTRA_VERSION}-${TAG}.${TIMESTAMP}" if ! $PATCHLESS && [ -n "$BUILDID" ]; then EXTRA_VERSION="${EXTRA_VERSION}.${BUILDID}" fi @@ -737,29 +706,6 @@ unpack_ofed() { unpack_lustre() { - if [ -z "$LUSTRE" ]; then - local DATESTAMP="" - - if [ -n "$USE_DATESTAMP" ]; then - DATESTAMP="-D '$DATE'" - fi - - local DIRNAME="lustre-$TAG-$TIMESTAMP" - - cvs -d "$CVSROOT" -qz3 co $DATESTAMP -d "$DIRNAME" lustre || \ - fatal 1 "There was an error checking out toplevel Lustre from CVS." - pushd "$DIRNAME" > /dev/null - ./lustrecvs "$TAG" || \ - fatal 1 "There was an error checking out Lustre/Portals/Build from CVS." - echo "Creating lustre tarball..." - sh autogen.sh || fatal 1 "There was an error running autogen.sh." - ./configure --enable-dist || \ - fatal 1 "There was an error running ./configure to create makefiles." - make dist || fatal 1 "There was an error running 'make dist'." - LUSTRE=$PWD/lustre-*.tar.gz - popd > /dev/null - fi - untar "$LUSTRE" || fatal 1 "Error unpacking Lustre tarball" [ -d lustre ] || ln -sf lustre-[0-9].[0-9]* lustre @@ -872,59 +818,6 @@ build_lustre() { fi fi - ( $(skeep_ldiskfs_rpm $TAG) ) || { - - pushd ldiskfs > /dev/null || return 255 - - if ! ./configure --enable-dist; then - echo "failed to configure in ldiskfs" - popd >/dev/null # pushd ldiskfs - popd >/dev/null # pushd lustre - return 255 - fi - - if ! make dist 2>&1; then - popd >/dev/null # pushd ldiskfs - popd >/dev/null # pushd lustre - return 255 - fi - - #cp lustre-ldiskfs*.tar.gz $TOPDIR/SOURCES || \ - # fatal 1 "Could not copy lustre-ldiskfs*.tar.gz to $TOPDIR/SOURCES" - - if ! $RPMBUILD $targets $rpmbuildopt lustre-ldiskfs*.tar.gz \ - --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ - --define "kdir $linux" \ - ${linuxobj:+--define "kobjdir $linuxobj"} \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - popd >/dev/null # pushd ldiskfs - popd >/dev/null # pushd lustre - return 255 - fi - - if $DO_SRC; then - if ! $RPMBUILD -ts lustre-ldiskfs*.tar.gz \ - --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ - --define "kdir $linux" \ - ${linuxobj:+--define "kobjdir $linuxobj"} \ - --define "_tmppath /var/tmp" \ - --define "_topdir $TOPDIR" 2>&1; then - popd >/dev/null # pushd ldiskfs - popd >/dev/null # pushd lustre - return 255 - fi - fi - popd >/dev/null # pushd ldiskfs - - # tell lustre where ldiskfs is - # XXX - pointing to the RPM BUILD dir is a hack. we need to flesh - # out the ldiskfs RPM build so that it builds a - # lustre-ldiskfs-devel RPM and install that and point lustre - # to that instead - confoptions="$confoptions --with-ldiskfs=$(ls -d $TOPDIR/BUILD/lustre-ldiskfs-*)" - } - if $IOKITRPM; then if ! build_iokit $rpmbuildopt; then echo "failed to build lustre-iokit" @@ -958,10 +851,10 @@ build_lustre() { fi $RPMBUILD $targets $rpmbuildopt "$LUSTRE" \ - ${is_patchless:+--define "lustre_name lustre-client"} \ - ${lustre_tests:+--define "build_lustre_tests 0"} \ - ${osd_zfs:+--define "build_lustre_osd_zfs 1"} \ - ${osd_ldiskfs:+--define "build_lustre_osd_ldiskfs 1"} \ + ${is_patchless:+--without servers} \ + ${lustre_tests:+--without lustre_tests} \ + ${osd_zfs:+--with zfs} \ + ${osd_ldiskfs:+--with ldiskfs} \ ${FIND_REQUIRES:+--define "__find_requires $FIND_REQUIRES"} \ --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ --define "kdir $linux" \ @@ -972,8 +865,8 @@ build_lustre() { if $DO_SRC; then if ! $RPMBUILD -ts "$LUSTRE" \ - ${is_patchless:+--define "lustre_name lustre-client"} \ - ${lustre_tests:+--define "build_lustre_tests 0"} \ + ${is_patchless:+--without servers} \ + ${lustre_tests:+--without lustre_tests} \ ${FIND_REQUIRES:+--define "__find_requires $FIND_REQUIRES"} \ --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \ --define "kdir $linux" \ @@ -1008,11 +901,11 @@ build_spl_zfs() { # The spl/zfs spec files expect RPM_BUILD_ROOT to point to the root of the # destination for the rpms export RPM_BUILD_ROOT=$TOPDIR - SPLZFSVER=${SPLZFSVER:-0.6.1} - SPLZFSTAG=${SPLZFSTAG:-lustre-zfs} + SPLZFSVER=${SPLZFSVER:-0.6.2} + SPLZFSTAG=${SPLZFSTAG:-upstream-master} # The files expect a kver to be set to the kernel version . - local kver=$(find_linux_release ${LINUX}) + local kver=$(find_linux_release) # build and install the spl and zfs (and -devel) RPMs for lustre to use local pkg @@ -1046,7 +939,7 @@ build_spl_zfs() { sh autogen.sh || return 255 - if ! ./configure --with-linux=${LINUX} --with-linux-obj=${LINUX} \ + if ! ./configure --with-linux=${LINUX} --with-linux-obj=${LINUXOBJ:-$LINUX} \ ${spldir:+--with-spl="${spldir}"} 2>&1 || ! make dist 2>&1; then popd @@ -1081,7 +974,7 @@ build_spl_zfs() { --define "kver $kver" \ --define "kernels $kver" \ --define "_tmppath /var/tmp" \ - --define "kernelbuildroot $(find_linux_builddir $LINUX)" \ + --define "kernelbuildroot $TOPDIR/reused" \ --define "_topdir $TOPDIR" 2>&1; then return 255 fi @@ -1101,6 +994,16 @@ build_spl_zfs() { for rpm in $rpms; do rpm2cpio $rpm | cpio -id done + + if [ "$pkg" == "zfs" ]; then + mkdir -p $TOPDIR/zfsdevel + pushd $TOPDIR/zfsdevel + # We also need to extract zfs-devel rpm + local devel_rpm=$(ls -1 $TOPDIR/RPMS/*/$pkg-devel-*.rpm) + rpm2cpio $devel_rpm | cpio -id + popd + CONFIGURE_FLAGS="--with-$pkg-devel=$TOPDIR/zfsdevel ${CONFIGURE_FLAGS}" + fi popd CONFIGURE_FLAGS="--with-$pkg=$(ls -d $TOPDIR/usr/src/$pkg-*/|tail -1) ${CONFIGURE_FLAGS}" @@ -1130,26 +1033,6 @@ stage() { } -#check if we need to build separate ldiskfs RPM -skeep_ldiskfs_rpm() { - local tag="$1" - - local skip=false - - if ! $LDISKFSRPM; then - skip=true - elif $PATCHLESS; then - skip=true - else - for skiptag in $SKIPLDISKFSRPM; do - [[ $tag == $skiptag ]] && skip=true && break - done - fi - - echo $skip - -} - set_rpm_smp_type() { local infact_arch="${TARGET_ARCH}" @@ -1176,26 +1059,25 @@ set_rpm_smp_type() { } -# This function takes a linux source pool and digs out the linux release -# from it -find_linux_release() { - local SRCDIR="$1" +# This function takes a linux include tree and digs out the linux release +# from it. It is never called directly, only called from the distro +# specific function find_linux_release() in lbuild-{rhel,sles}. +_find_linux_release() { + local SRC="$1" + local LINUXRELEASEHEADER="" - local LINUXRELEASEHEADER=$SRCDIR/include/linux/version.h - if [ -s $SRCDIR/include/linux/utsrelease.h ]; then - LINUXRELEASEHEADER=$SRCDIR/include/linux/utsrelease.h + LINUXRELEASEHEADER=$SRC/include/linux/version.h + if [ -s $SRC/include/generated/utsrelease.h ]; then + LINUXRELEASEHEADER=$SRC/include/generated/utsrelease.h + elif [ -s $SRC/include/linux/utsrelease.h ]; then + LINUXRELEASEHEADER=$SRC/include/linux/utsrelease.h fi - sed -ne 's/#define UTS_RELEASE "\(.*\)"$/\1/p' $LINUXRELEASEHEADER -} - -# This funcition finds the buildroot directory for a given linux -# kernel directory -find_linux_builddir() { - local linux=${1:-${LINUX}} - local rel=$(find_linux_release $linux) + if [ ! -s $LINUXRELEASEHEADER ]; then + fatal 1 "could not find UTS_RELEASE" + fi - echo ${linux%/usr/src/kernels/${rel}*} + sed -ne 's/#define UTS_RELEASE "\(.*\)"$/\1/p' $LINUXRELEASEHEADER } # unpack kernel(/source/devel) RPM @@ -1273,12 +1155,12 @@ build_kernel_ib() { local SOURCE="${TOPDIR}/OFED/SRPMS/${kib_prefix}-*.src.rpm" # but switch to building from the SPEC if we need to apply patches - if ls ${TOPDIR}/lustre/build/patches/ofed/* >/dev/null; then + if ls ${TOPDIR}/lustre/contrib/patches/ofed/* >/dev/null; then BUILD_TYPE="-bb" rpm --define "_topdir ${TOPDIR}" -ivh $SOURCE SOURCE="${TOPDIR}/SPECS/${kib_prefix}.spec" local file ed_fragment1 ed_fragment2 n=1 - for file in $(ls ${TOPDIR}/lustre/build/patches/ofed/*.patch); do + for file in $(ls ${TOPDIR}/lustre/contrib/patches/ofed/*.patch); do ed_fragment1="$ed_fragment1 Patch$n: ${file%%*/}" ed_fragment2="$ed_fragment2 @@ -1286,9 +1168,9 @@ Patch$n: ${file%%*/}" cp $file ${TOPDIR}/SOURCES let n=$n+1 done - for file in $(ls ${TOPDIR}/lustre/build/patches/ofed/*.ed); do + for file in $(ls ${TOPDIR}/lustre/contrib/patches/ofed/*.ed); do # Only apply the ed-scripts that should be used for the canonical target - # ed-files in ${TOPDIR}/lustre/build/patches/ofed/ have to follow the naming + # ed-files in ${TOPDIR}/lustre/contrib/patches/ofed/ have to follow the naming # convention # -:: ...:.ed # To apply the same change to multiple canonical target simply specify @@ -1316,7 +1198,7 @@ EOF fi fi - local linuxrelease=$(find_linux_release "$linux") + local linuxrelease=$(find_linux_release) if compare_version $OFED_VERSION 3.0; then local OFA_KERNEL_RELEASE=$(echo -n ${linuxrelease} | sed -e 's/-/_/g') fi @@ -1481,7 +1363,7 @@ build_kernel_with_srpm() { local REUSE_SIGNATURE=$({ echo -en $release_str; echo $BUILD_GEN; cat $CONFIG_FILE $TARGET_FILE $FULL_PATCH; - cat ${0%/lbuild}/lbuild ${0%/lbuild}/lbuild-${DISTRO}; } | + cat $LBUILD_DIR/lbuild $LBUILD_DIR/lbuild-${DISTRO}; } | md5sum | cut -d" " -f1) # see if we can link to the reuse pool # XXX - hrm. i'm not convinced this doesn't belong in the reuse @@ -1534,7 +1416,7 @@ build_kernel_with_srpm() { # figure out the EXTRA_VERSION of the kernel we built or are re-using local KERNEL_RPM - if ! KERNEL_RPM=$(find_rpm "$TOPDIR/RPMS/$TARGET_ARCH/" provides "^kernel ="); then + if ! KERNEL_RPM=$(find_rpm "$TOPDIR/RPMS/$TARGET_ARCH/" provides "^kernel(-default)? ="); then fatal 1 "Failed to find a kernel RPM in $TOPDIR/RPMS/$TARGET_ARCH/" fi kernel_extra_version=$(rpm -q --queryformat "%{RELEASE}" -p $TOPDIR/RPMS/$TARGET_ARCH/$KERNEL_RPM) @@ -1595,11 +1477,11 @@ build_ofed() { # build kernel-ib/compat-rdma if $USE_BUILD_CACHE && [ -n "$REUSEBUILD" ]; then local REUSE_SIGNATURE=$({ echo "$ofed_version"; - echo "$(find_linux_release ${linux}; + echo "$(find_linux_release; echo "$BUILD_GEN")"; cat "${linux}/include/linux/autoconf.h"; - cat "${0%/lbuild}/lbuild" ; - cat "${0%/lbuild}/lbuild-${DISTRO}"; } | + cat "$LBUILD_DIR/lbuild" ; + cat "$LBUILD_DIR/lbuild-${DISTRO}"; } | md5sum | cut -d" " -f1) # see if we can link to the reuse pool # XXX - hrm. i'm not convinced this doesn't belong in the reuse @@ -1652,9 +1534,16 @@ build_ofed() { cd ${kib_rpm}-devel # the actual ofed RPMs don't have the -rc$n or -$date string appened that # might be present on the file - local linuxrelease=$(find_linux_release "$linux") + local linuxrelease=$(find_linux_release) ofed_version=$(echo $ofed_version | sed -re 's/-(20[0-9]{6,6}-[0-9]{4,4}|rc[0-9]*)$//') + # FIXME + # OFED version will have 'hyphen' for minor release. (e.g. 3.5-1, instead + # of 3.5.1) compat-rdma and compat-rdma-devel could have same version + # number, but currectly not. Once OFED fix this in the future release, we + # can remove following filter. + ofed_version=$(echo $ofed_version | + sed -re 's/-([0-9]*-[rR][cC][0-9]*)$//') local rpm=$(ls $TOPDIR/RPMS/*/${kib_rpm}-devel-${ofed_version}-${linuxrelease//-/_}.*.rpm) if ! rpm2cpio < $rpm | cpio -id; then fatal 1 "could not unpack the ${kib_rpm}-devel rpm." @@ -1889,7 +1778,7 @@ set -E [ -r ~/.lbuildrc ] && . ~/.lbuildrc -options=$(getopt -o d:D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ldiskfs,ccache,reuse:,norpm,disable-datestamp,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,ofed-version:,publish,release,set-value:,src,stage:,tag:,target:,target-archs:,with-linux:,xen -- "$@") +options=$(getopt -o D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ldiskfs,ccache,reuse:,norpm,disable-datestamp,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,noiokit,ofed-version:,publish,release,set-value:,src,stage:,tag:,target:,target-archs:,with-linux:,xen -- "$@") if [ $? != 0 ]; then usage 1 @@ -1906,10 +1795,6 @@ while [ "$1" ]; do CCACHE='ccache' shift ;; - -d) - CVSROOT=$2 - shift 2 - ;; -D) DATE=$2 shift 2 @@ -1960,7 +1845,7 @@ while [ "$1" ]; do shift ;; --ldiskfs) - LDISKFSRPM=true + echo "WARNING: \"--ldiskfs\" is deprecated" > 2 shift ;; --noiokit) @@ -2015,8 +1900,8 @@ while [ "$1" ]; do shift 2 ;; --tag) - TAG=$2 shift 2 + echo "WARNING: \"--tag\" is deprecated" > 2 ;; --target) TARGET=$2 @@ -2026,10 +1911,6 @@ while [ "$1" ]; do TARGET_ARCHS=$2 shift 2 ;; - --disable-datestamp) - USE_DATESTAMP= - shift - ;; --xen) XEN=true shift