Whamcloud - gitweb
b=23428 Fix lustre built with --enable-lu_ref
[fs/lustre-release.git] / build / lbuild
index 3478342..15b78be 100755 (executable)
@@ -48,10 +48,13 @@ LUSTRE_EXTRA_VERSION=
 STAGEDIR=
 TMPDIR=${TMPDIR:-"/var/tmp"}
 TIMESTAMP=
-# XXX - i think these two parameters/arguments/variables need to be
-#       cleaned up and merged.  they effectively do the same thing
+# this is a dir to try reuse old kernel RPMs in (although, it seems to be
+# unused in any real manner
 REUSERPM=
+# this is the dir that should be used to store reuse products
 REUSEBUILD=
+# should cached products be used or force rebuilding?
+USE_BUILD_CACHE=true
 # what does this do exactly?  does it imply no kernel build?
 NORPM=false
 LDISKFSRPM=true
@@ -83,6 +86,9 @@ UP_ARCHS=
 # not in the target file any more
 CONFIG=
 
+# build the lustre-tests rpm?
+LUSTRE_TESTS=true
+
 DATE=$(date)
 
 USE_DATESTAMP=1
@@ -510,7 +516,7 @@ download_ofed() {
         daily=${OFED_VERSION##$Mmv-}
         location="http://www.openfabrics.org/downloads/OFED/ofed-${Mmv}-daily/"
         # find the filename for the version for the date specified
-        OFED_VERSION=$(curl -s "$location" | sed -nre "/${daily}-/s/.*href=\"OFED-([0-9]+\.[0-9]+-${daily}-[0-9]{4,4}).tgz.*$/\1/p" | tail -1)
+        OFED_VERSION=$(curl -s "$location" | sed -nre "/${daily}-/s/.*href=\"OFED-(${Mmv//./\\.}-${daily}-[0-9]{4,4}).tgz.*$/\1/p" | tail -1)
         if [ -z "$OFED_VERSION" ]; then
             fatal 1 "Could not determine the filename of the OFED snapshot for ${daily}"
         fi
@@ -745,7 +751,7 @@ do_patch_linux() {
 
     local do_patch=${1:-true}
 
-    FULL_PATCH="$PWD/lustre-kernel-${TARGET}${EXTRA_VERSION_DELIMITER}${EXTRA_VERSION}.patch"
+    FULL_PATCH="$PWD/lustre-kernel-${TARGET}-${EXTRA_VERSION}.patch"
     [ -f "$FULL_PATCH" ] && rm -f "$FULL_PATCH"
     $do_patch && pushd linux >/dev/null
     for series in $SERIES; do
@@ -833,15 +839,24 @@ build_lustre() {
         echo NORPM mode. Only compiling.
     fi
 
-    # convert the $PATCHLESS boolean to an empty/no-empty boolean
+    # convert the $PATCHLESS boolean to an empty/not-empty boolean
     # as silly as this seems, it makes the syntax of the rpmbuild command
     # simpler and not need an eval to deal with the quotes in the quotes
     local is_patchless=""
     if $PATCHLESS; then
         is_patchless="yes"
     fi
+
+    # ditto for the lustre-tests boolean
+    local lustre_tests=""
+    if ! $LUSTRE_TESTS; then
+        lustre_tests="no"
+    fi
+
     $RPMBUILD $targets $rpmbuildopt ../lustre.spec \
         ${is_patchless:+--define "lustre_name lustre-client"} \
+        ${lustre_tests:+--define "build_lustre_tests 0"} \
+        --define "configure_args $confoptions ${CONFIGURE_FLAGS}" \
         --define "_tmppath $TMPDIR" \
         --define "_topdir $TOPDIR" 2>&1 || \
         fatal 1 "Error building rpms for $BUILD_ARCHS."
@@ -929,7 +944,7 @@ skeep_ldiskfs_rpm() {
 #generate LUSTRE_EXTRA_VERSION from EXTRA_VERSION
 gen_lustre_version() {
 
-    LUSTRE_EXTRA_VERSION="${lnxmaj}${EXTRA_VERSION_DELIMITER}${EXTRA_VERSION}${TARGET_DELIMITER}${RPMSMPTYPE}"
+    LUSTRE_EXTRA_VERSION="${lnxmaj}-${EXTRA_VERSION}${FLAVOR_DELIMITER}${RPMSMPTYPE}"
     LUSTRE_EXTRA_VERSION=${LUSTRE_EXTRA_VERSION//-/_}
 
 }
@@ -974,19 +989,19 @@ find_linux_release() {
 #     or just gotten rid of.  :-)
 find_linux_rpm() {
     local prefix="$1"
-    local delimiter=${2:-"-"}
 
     local pathtorpms="${KERNELRPMSBASE}/${lnxmaj}/${DISTRO}"
     [ -d $pathtorpms ] || return 255
 
     local kernelbinaryrpm rpmfile
-    local wanted_kernel="${lnxmaj}${delimiter}${lnxrel}"
+    local wanted_kernel="${lnxmaj}${lnxmin}-${lnxrel}"
 
     local arch ret=1
     for arch in $TARGET_ARCHS_ALL; do
         local found_rpm="" rpm
         for rpm in ${pathtorpms}/${arch}/*.rpm; do
-            if rpm -q --provides -p "$rpm" | grep -q "kernel${prefix} = $wanted_kernel"; then
+            if rpm -q --provides -p "$rpm" 2>&3 | grep -q "kernel${prefix} = $wanted_kernel" 2>&3; then
+
                 found_rpm="$rpm"
                 ret=0
                 break
@@ -1008,10 +1023,6 @@ find_linux_rpm() {
 # lustre itself)
 unpack_linux_devel_rpm() {
     local kernelrpm="${1}"
-    # it's worth noting that neither sles10 nor rhel5 appear to use their
-    # extra_version delimiter for the dirname under /usr/src, so we could
-    # probably just get rid of this parameter
-    local delimiter=${2:-"-"}
 
     [ -f "$kernelrpm" ] || return 255
     [ -d $TOPDIR/reused ] || mkdir $TOPDIR/reused || return 255
@@ -1024,7 +1035,9 @@ unpack_linux_devel_rpm() {
 
     # call a distro specific hook, if available
     if type -p unpack_linux_devel_rpm-$DISTRO; then
-        unpack_linux_devel_rpm-$DISTRO "$kernelrpm"
+        if ! unpack_linux_devel_rpm-$DISTRO "$kernelrpm"; then
+            return 255
+        fi
     fi
 
     popd &>/dev/null
@@ -1035,55 +1048,6 @@ unpack_linux_devel_rpm() {
 
 }
 
-# XXX - this rhel/sles goop needs abstracting out into the
-#       lbuild-{rhel5,sles10} method files
-find_linux_devel_paths() {
-    local path="$1"
-
-    local RC=0
-
-    pushd $path
-        # RHEL-style and SLES-style rpms
-        # XXX - until bug 19336 cleans this up, we need to extricate the
-        #       ${lnxmin}- from the $lnxrel
-        local paths="kernels/${lnxmaj}${lnxmin}${delimiter}${lnxrel}-${TARGET_ARCH} linux-${lnxmaj}${lnxmin}${delimiter}${lnxrel##${lnxmin#.}-}"
-
-        local path
-        for path in $paths; do
-            local src='usr/src'
-
-            if [ -d "$src/$path/" ]; then
-                LINUX="$(pwd)/$src/$path"
-            fi
-            # SLES has a separate -obj tree
-            if [ -d "$src/${path}-obj" ]; then
-                local src="$src/${path}-obj"
-                local objects="$TARGET_ARCH/$RPMSMPTYPE"
-
-                # Novell, are you *TRYING* to make life hard for me?
-                if [ -d "$src/powerpc" ]; then
-                    objects="powerpc/$TARGET_ARCH"
-                elif [ $TARGET_ARCH == 'i686' ]; then
-                    objects="i386/$RPMSMPTYPE"
-                fi
-
-                LINUXOBJ="$(pwd)/$src/$objects"
-            fi
-        done
-        if [ -z "$LINUX" ]; then
-            RC=255
-        else
-            # dig out the release version
-            LINUXRELEASE=$(find_linux_release ${LINUXOBJ:-$LINUX})
-            if [ -z "$LINUXRELEASE" ]; then
-                echo "Failed to find linux release in ${LINUXOBJ:-$LINUX}"
-                RC=255
-            fi
-        fi
-    popd
-    return $RC
-}
-
 build_kernel_ib() {
     local linux="$1"
 
@@ -1285,7 +1249,7 @@ find_rpm() {
         case "$match_type" in
             provides)
                 # match is any valid ERE (i.e. given to egrep) match
-                if rpm -q --provides -p "$file" | egrep -q "$match"; then
+                if rpm -q --provides -p "$file" 2>&3 | egrep -q "$match"; then
                     echo "$file"
                     popd >/dev/null
                     return 0
@@ -1319,21 +1283,27 @@ build_kernel_with_srpm() {
     if $RELEASE; then
         local release_str="RELEASE=$RELEASE\n"
     fi
-    local REUSE_SIGNATURE=$({ echo -en $release_str; echo $BUILD_GEN; cat $CONFIG_FILE $TARGET_FILE $FULL_PATCH; } | md5sum | cut -d" " -f1)
 
-    # see if we can link to the reuse pool
-    # XXX - hrm.  i'm not convinced this doesn't belong in the reuse "library"
-    local CAN_LINK_FOR_REUSE=false
-    touch $REUSEBUILD/$$
-    if cp -al $REUSEBUILD/$$ $TOPDIR/ 2>/dev/null; then
-        CAN_LINK_FOR_REUSE=true
+    if $USE_BUILD_CACHE && [ -n "$REUSEBUILD" ]; then
+        local REUSE_SIGNATURE=$({ echo -en $release_str;
+                                  echo $BUILD_GEN;
+                                  cat $CONFIG_FILE $TARGET_FILE $FULL_PATCH; } |
+                                md5sum | cut -d" " -f1)
+        # see if we can link to the reuse pool
+        # XXX - hrm.  i'm not convinced this doesn't belong in the reuse
+        #       "library"
+        local CAN_LINK_FOR_REUSE=false
+        touch $REUSEBUILD/$$
+        if cp -al $REUSEBUILD/$$ $TOPDIR/ 2>/dev/null; then
+            CAN_LINK_FOR_REUSE=true
+        fi
+        rm $REUSEBUILD/$$
     fi
-    rm $REUSEBUILD/$$
 
     # the extra version string to use for the kernel (which might be a reused
     # kernel, remember)
     local kernel_extra_version=""
-    if $REUSERPM && ! reuse kernel "$TOPDIR" "$CAN_LINK_FOR_REUSE" \
+    if ! $USE_BUILD_CACHE || ! reuse kernel "$TOPDIR" "$CAN_LINK_FOR_REUSE" \
                                    "$REUSE_SIGNATURE"; then
         # nothing cached, build from scratch
         if [ ! -r "$KERNELDIR/$KERNEL_SRPM" ]; then
@@ -1353,14 +1323,18 @@ build_kernel_with_srpm() {
         prepare_and_build_srpm >&${outfd} ||
             fatal 1 "failed to prepare_and_build_srpm"
 
-        # store the resulting kernel RPM build tree for future use
-        echo "caching the built kenel for future builds..." >&${outfd}
-        if ! store_for_reuse "$TOPDIR/{SPECS,SOURCES,BUILD,SRPMS,RPMS}" \
-                             "kernel" "$REUSEBUILD" "$REUSE_SIGNATURE" \
-                             "$CAN_LINK_FOR_REUSE"; then
-            error "Failed to store kernel RPMS for reuse"
-            echo "unknown"
-            return 1
+        if [ -z "$REUSE_SIGNATURE" ]; then
+            echo "No reuse signature was caculated so not storing the built kernel" >&${outfd}
+        else
+            # store the resulting kernel RPM build tree for future use
+            echo "Storing the built kernel for future reuse" >&${outfd}
+            if ! store_for_reuse "$TOPDIR/{SPECS,SOURCES,BUILD,SRPMS,RPMS}" \
+                                 "kernel" "$REUSEBUILD" "$REUSE_SIGNATURE" \
+                                 "$CAN_LINK_FOR_REUSE"; then
+                error "Failed to store kernel RPMS for reuse"
+                echo "unknown"
+                return 1
+            fi
         fi
     fi  # build reuse
 
@@ -1432,7 +1406,7 @@ build_mptlinux() {
                   --define "_tmppath /var/tmp" \
                   --define "_topdir ${TOPDIR}" \
                   --define "kernel_obj $linux" \
-                  ${TOPDIR}/SPECS/mptlinux.spec; then
+                  ${TOPDIR}/SPECS/mptlinux.spec 2>&1; then
         return 1
     fi
     if $DO_SRC; then
@@ -1440,7 +1414,7 @@ build_mptlinux() {
                       --define "_tmppath /var/tmp" \
                       --define "_topdir ${TOPDIR}" \
                       --define "kernel_obj $linux" \
-                      ${TOPDIR}/SPECS/mptlinux.spec; then
+                      ${TOPDIR}/SPECS/mptlinux.spec 2>&1; then
             return 1
         fi
     fi
@@ -1488,7 +1462,7 @@ build_rdac() {
                   --define "_tmppath /var/tmp" \
                   --define "_topdir ${TOPDIR}" \
                   --define "kernel_obj $linux" \
-                  ${TOPDIR}/SPECS/rdac.spec; then
+                  ${TOPDIR}/SPECS/rdac.spec 2>&1; then
         return 1
     fi
     if $DO_SRC; then
@@ -1496,7 +1470,7 @@ build_rdac() {
                       --define "_tmppath /var/tmp" \
                       --define "_topdir ${TOPDIR}" \
                       --define "kernel_obj $linux" \
-                      ${TOPDIR}/SPECS/rdac.spec; then
+                      ${TOPDIR}/SPECS/rdac.spec 2>&1; then
             return 1
         fi
     fi
@@ -1507,7 +1481,7 @@ build_rdac() {
 # build OFED
 # globals used:
 #    TOPDIR
-#    REUSEBUILD, REUSERPM
+#    REUSEBUILD, USE_BUILD_CACHE
 #    CONFIGURE_FLAGS
 
 build_ofed() {
@@ -1519,10 +1493,14 @@ build_ofed() {
         return 0
     fi
 
-    if [ -n "$REUSEBUILD" ]; then
+    if $USE_BUILD_CACHE && [ -n "$REUSEBUILD" ]; then
+        local REUSE_SIGNATURE=$({ echo "$ofed_version";
+                                  echo "$(find_linux_release ${linux})";
+                                  cat "${linux}/include/linux/autoconf.h"; } |
+                                md5sum | cut -d" " -f1)
         # see if we can link to the reuse pool
-            # XXX - hrm.  i'm not convinced this doesn't belong in the reuse
-            #       "library"
+        # XXX - hrm.  i'm not convinced this doesn't belong in the reuse
+        #       "library"
         local CAN_LINK_FOR_REUSE=false
         touch $REUSEBUILD/$$
         if cp -al $REUSEBUILD/$$ $TOPDIR/; then
@@ -1531,33 +1509,38 @@ build_ofed() {
         rm $REUSEBUILD/$$
     fi
 
-    local REUSE_SIGNATURE=$({ echo "$ofed_version";
-                              echo "$(find_linux_release ${linux})";
-                              cat "${linux}/include/linux/autoconf.h"; } |
-                            md5sum | cut -d" " -f1)
-    if ! $REUSERPM || ! reuse ofed "$TOPDIR" "$CAN_LINK_FOR_REUSE" \
+    if ! $USE_BUILD_CACHE || ! reuse ofed "$TOPDIR" "$CAN_LINK_FOR_REUSE" \
                                    "$REUSE_SIGNATURE"; then
-        # stash away the existing built articles for a moment
-        mkdir bak
-        mv {BUILD,{S,}RPMS,S{OURCE,PEC}S} bak
-        function mv_back {
-            pushd bak
-            find . | cpio -pudlm ..
-            popd
-            rm -rf bak
-        }
-        create_rpmbuild_dirs
+        if [ -n "$REUSE_SIGNATURE" ]; then
+            # stash away the existing built articles for a moment
+            mkdir bak
+            mv {BUILD,{S,}RPMS,S{OURCE,PEC}S} bak
+            function mv_back {
+                pushd bak
+                find . | cpio -pudlm ..
+                popd
+                rm -rf bak
+            }
+            create_rpmbuild_dirs
+        fi
         # build it
         build_kernel_ib "${linux}"
-        if ! store_for_reuse "$TOPDIR/{SPECS,SOURCES,BUILD,SRPMS,RPMS}" \
-                             "ofed" "$REUSEBUILD" "$REUSE_SIGNATURE" \
-                             "$CAN_LINK_FOR_REUSE"; then
-            error "Failed to store OFED RPMS for reuse"
+
+        if [ -z "$REUSE_SIGNATURE" ]; then
+            echo "No reuse signature was caculated so not storing the built ofed"
+        else
+            # store the resulting RPM build tree for future use
+            echo "Storing the built ofed for future reuse"
+            if ! store_for_reuse "$TOPDIR/{SPECS,SOURCES,BUILD,SRPMS,RPMS}" \
+                                 "ofed" "$REUSEBUILD" "$REUSE_SIGNATURE" \
+                                 "$CAN_LINK_FOR_REUSE"; then
+                error "Failed to store OFED RPMS for reuse"
+                mv_back
+                return 1
+            fi
+            # put the stuff we stashed away back
             mv_back
-            return 1
         fi
-        # put the stuff we stashed away back
-        mv_back
     fi
 
     pushd "$TOPDIR" >/dev/null
@@ -1591,10 +1574,8 @@ build_with_srpm() {
             fi
 
             # install the -devel RPM in preparation for the lustre build
-            # note that the EXTRA_VERSION_DELIMITER is *NOT* used in the
-            # version of the directory name under /usr/src
             if ! lnxrel="$kernel_extra_version" unpack_linux_devel_rpm \
-                           "$TOPDIR/RPMS/$arch/$kernel_devel_rpm" "-"; then
+                           "$TOPDIR/RPMS/$arch/$kernel_devel_rpm"; then
                 fatal 1 "Could not find the Linux tree in $TOPDIR/RPMS/$arch/$kernel_devel_rpm"
             fi
         done
@@ -1602,7 +1583,7 @@ build_with_srpm() {
         # need to find and unpack the vendor's own kernel-devel for patchless
         # client build
         local kernelrpm
-        if ! kernelrpm=$(find_linux_rpm "-$DEVEL_KERNEL_TYPE" ${EXTRA_VERSION_DELIMITER:-"-"}); then
+        if ! kernelrpm=$(find_linux_rpm "-$DEVEL_KERNEL_TYPE"); then
             fatal 1 "Could not find the kernel-$DEVEL_KERNEL_TYPE RPM in ${KERNELRPMSBASE}/${lnxmaj}/${DISTRO}"
         fi
         if ! lnxrel="$lnxrel" unpack_linux_devel_rpm "$kernelrpm" "-"; then
@@ -1911,6 +1892,11 @@ while [ "$1" ]; do
             ;;
         --)
             shift
+            # there are actually some lustre configure flags that we need to
+            # handle ourselves (but we still give them to configure)
+            if [[ \ $@\  == *\ --disable-tests\ * ]]; then
+                LUSTRE_TESTS=false
+            fi
             CONFIGURE_FLAGS=$@
             CONFIGURE_FLAGS="$CONFIGURE_FLAGS --enable-liblustre --enable-liblustre-tests"
             break
@@ -1926,7 +1912,6 @@ check_options
 unpack_lustre
 
 load_target
-EXTRA_VERSION_DELIMITER=${EXTRA_VERSION_DELIMITER:-"-"}
 
 if [ -n "$OFED_VERSION" -a "$OFED_VERSION" != "inkernel" ]; then
     download_ofed
@@ -1967,6 +1952,7 @@ else
 
         build_with_srpm || fatal 1 "Failed to build_with_srpm"
     else
+        EXTRA_VERSION_DELIMITER=${EXTRA_VERSION_DELIMITER:-"-"}
         source ${0%/*}/lbuild.old_school
 
         old_school_download_kernel