Whamcloud - gitweb
add a branch and its table named "b_iam" for "Index API Module" from #colibri
[fs/lustre-release.git] / build / lbuild
index fff2778..d6e95ca 100755 (executable)
@@ -4,16 +4,18 @@ TOPDIR=$PWD
 
 # CVSROOT is inherited from the environment
 KERNELDIR=
+LINUX=
 LUSTRE=
-PUBLISH=0
 RELEASE=0
-DO_SRC=1
+DO_SRC=0
+DOWNLOAD=1
 TAG=
 TARGET=
 TARGET_ARCHS=
 CONFIGURE_FLAGS=
 EXTERNAL_PATCHES=
 EXTRA_VERSION=
+STAGEDIR=
 
 # from target file
 KERNEL=
@@ -32,6 +34,7 @@ BOOT_ARCHS=
 JENSEN_ARCHS=
 SMP_ARCHS=
 BIGSMP_ARCHS=
+PSERIES64_ARCHS=
 UP_ARCHS=
 
 DATE=$(date)
@@ -39,6 +42,20 @@ DATE=$(date)
 USE_DATESTAMP=1
 RPMBUILD=
 
+export CC=${CC:-gcc}
+
+# Readlink is not present on some older distributions: emulate it.
+readlink() {
+    local path=$1 ll
+
+    if [ -L "$path" ]; then
+       ll="$(LC_ALL=C ls -l "$path" 2> /dev/null)" &&
+       echo "${ll/* -> }"
+    else
+       return 1
+    fi
+}
+
 cleanup()
 {
     true
@@ -56,11 +73,6 @@ fatal()
     exit $1
 }
 
-publishing()
-{
-    (( $PUBLISH )) || return 0
-}
-
 is_release()
 {
     (( $RELEASE )) || return 0
@@ -98,21 +110,36 @@ Usage: ${0##*/} [OPTION]... [-- <lustre configure options>]
     Directory containing Linux source tarballs referenced by target
     files.
 
+  --linux=LINUX
+    Directory of Linux kernel sources.  When this option is used, only
+    Lustre modules and userspace are built.
+
   --lustre=LUSTRE
     Path to an existing lustre source tarball to use instead of
     pulling from CVS.
 
+  --nodownload
+    Do not try to download a kernel from ftp.lustre.org
+
   --nosrc
     Do not build a .src.rpm, a full kernel patch, or a patched kernel
     tarball.
 
   --publish
-    Publish the packages, patches, and tarballs on the ftp server.
+    Unused.
 
   --release
     Specifies that the files generated do not include timestamps, and
     that this is an official release.
 
+  --src
+    Build a .src.rpm, a full kernel patch, and a patched kernel tarball.
+
+  --stage=DIR
+    Directory used to stage packages for release.  RPMs will be placed
+    more or less in DIR/<target>-<arch>, and the tarball will be
+    placed in DIR.
+
   --tag=TAG
     A CVS branch/tag name to build from when pulling from CVS.
 
@@ -154,23 +181,44 @@ check_options()
            usage 1 "A branch/tag name must be specified with --tag when not building from a tarball."
     fi
 
-    [ "$KERNELDIR" ] || \
-       usage 1 "A kernel directory must be specified with --kerneldir."
+    if [ -z "$LINUX" ] ; then
+       [ "$KERNELDIR" ] || \
+           usage 1 "A kernel directory must be specified with --kerneldir."
 
-    [ -d "$KERNELDIR" ] || \
-       usage 1 "$KERNELDIR is not a directory."
+       [ -d "$KERNELDIR" ] || \
+           usage 1 "$KERNELDIR is not a directory."
 
-    if ! (( $RELEASE )) ; then
-       [ "$TAG" ] || \
-           usage 1 "When building a snapshot, a tag name must be used."
+       if ! (( $RELEASE )) ; then
+           [ "$TAG" ] || \
+               usage 1 "When building a snapshot, a tag name must be used."
+       fi
+
+       [ "$TARGET" ] || usage 1 "A target must be specified with --target."
+#       TARGET_FILE="$TOPDIR/lustre/kernel_patches/targets/$TARGET.target"
+#       [ -r "$TARGET_FILE" ] || \
+#              usage 1 "Target '$TARGET' was not found."
     fi
 
-    TIMESTAMP=$(date -d "$DATE" "+%Y%m%d%H%M")
+    case $TARGET in
+       2.6-rhel4)
+           CANONICAL_TARGET="rhel-2.6"
+           ;;
+       2.6-suse)
+           CANONICAL_TARGET="sles-2.6"
+           ;;
+       hp_pnnl-2.4)
+           CANONICAL_TARGET="hp-pnnl-2.4"
+           ;;
+       2.6-vanilla \
+           | suse-2.4.21-2 \
+           | rh-2.4 \
+           | rhel-2.4 \
+           | sles-2.4)
+               CANONICAL_TARGET="$TARGET"
+               ;;
+    esac
 
-    [ "$TARGET" ] || usage 1 "A target must be specified with --target."
-#    TARGET_FILE="$TOPDIR/lustre/kernel_patches/targets/$TARGET.target"
-#    [ -r "$TARGET_FILE" ] || \
-#      usage 1 "Target '$TARGET' was not found."
+    TIMESTAMP=$(date -d "$DATE" "+%Y%m%d%H%M")
 
     RPMBUILD=$(which rpmbuild 2>/dev/null | head -1)
     if [ ! "$RPMBUILD" -o "$RPMBUILD" == "" ]; then
@@ -186,6 +234,38 @@ uniqify()
     echo $(echo "$*" | xargs -n 1 | sort -u)
 }
 
+download_and_build_tarball() {
+    local TARGET=$1
+    local KERNEL_FILE=$2
+
+    local SRPM=kernel-${lnxmaj}-${lnxrel}.src.rpm
+
+    echo "Downloading http://ftp.lustre.org/kernels/$TARGET/old/$SRPM..."
+    if ! wget -nv "http://ftp.lustre.org/kernels/$TARGET/old/$SRPM" \
+        -O "$KERNELDIR/$SRPM" ; then
+        fatal 1 "Could not download target $TARGET's kernel SRPM $SRPM from ftp.lustre.org."
+    fi
+
+    if [ "$TARGET" = "rhel-2.6" ]; then
+       RPMTOPDIR=$(mktemp -d $KERNELDIR/rpm_XXXXXX)
+       mkdir $RPMTOPDIR/BUILD/
+        rpm -ivh $KERNELDIR/$SRPM --define "_topdir $RPMTOPDIR" || \
+           { rm -rf $RPMTOPDIR; fatal 1 "Error installing kernel SRPM."; }
+       $RPMBUILD -bp --nodeps --target i686 $RPMTOPDIR/SPECS/kernel-2.6.spec --define "_topdir $RPMTOPDIR"
+        pushd $RPMTOPDIR/BUILD/kernel-${lnxmaj}/linux-${lnxmaj} && {
+            make mrproper
+            cp configs/kernel-${lnxmaj}-i686-smp.config .config
+            make nonint_oldconfig > /dev/null
+            make include/linux/version.h 
+            rm -f .config
+            cd ..
+            tar cjf $KERNEL_FILE linux-${lnxmaj}
+        }
+        popd
+       rm -rf $RPMTOPDIR
+    fi
+}
+
 load_target()
 {
     EXTRA_VERSION_save="$EXTRA_VERSION"
@@ -207,8 +287,17 @@ load_target()
 
     if [ "$KERNELDIR" ] ; then
        KERNEL_FILE="$KERNELDIR/$KERNEL"
-       [ -r "$KERNELDIR/$KERNEL" ] || \
-           fatal 1 "Target $TARGET's kernel file $KERNEL not found in kernel directory $KERNELDIR."
+       if [ ! -r "$KERNELDIR/$KERNEL" ] ; then
+           if (( $DOWNLOAD )) ; then
+               echo "Downloading http://ftp.lustre.org/kernels/$CANONICAL_TARGET/old/$KERNEL..."
+               if ! wget -nv "http://ftp.lustre.org/kernels/$CANONICAL_TARGET/old/$KERNEL" -O "$KERNELDIR/$KERNEL" ; then
+                   # see if we can do it with an SRPM from the download site
+                   download_and_build_tarball $CANONICAL_TARGET $KERNEL_FILE
+               fi
+           else
+               fatal 1 "Target $TARGET's kernel file $KERNEL not found in kernel directory $KERNELDIR."
+           fi
+       fi
     fi
 
     if [ "$SERIES" ] ; then
@@ -227,11 +316,13 @@ load_target()
     if [ "$EXTRA_VERSION_save" ] ; then
        EXTRA_VERSION="$EXTRA_VERSION_save"
     elif ! (( $RELEASE )) ; then
+       #remove the @VERSION@ (lustre version)
+       EXTRA_VERSION=$(echo $EXTRA_VERSION | sed -e "s/\(.*_lustre\)\..*/\1/")
        EXTRA_VERSION="${EXTRA_VERSION}-${TAG}.${TIMESTAMP}"
     fi
     # EXTRA_VERSION=${EXTRA_VERSION//-/_}
 
-    ALL_ARCHS="$BASE_ARCHS $BIGMEM_ARCHS $BOOT_ARCHS $JENSEN_ARCHS $SMP_ARCHS $BIGSMP_ARCHS $UP_ARCHS"
+    ALL_ARCHS="$BASE_ARCHS $BIGMEM_ARCHS $BOOT_ARCHS $JENSEN_ARCHS $SMP_ARCHS $BIGSMP_ARCHS $PSERIES64_ARCHS $UP_ARCHS"
 
     BUILD_ARCHS=
     for arch in $(uniqify "$ALL_ARCHS") ; do
@@ -249,12 +340,15 @@ tarflags()
        '')
            fatal 1 "tarflags(): File name argument missing."
            ;;
-       *.tar.gz)
+       *.tar.gz | *.tgz)
            echo 'zxf'
            ;;
        *.tar.bz2)
            echo 'jxf'
            ;;
+       *.tar)
+           echo 'xf'
+           ;;
        *)
            fatal 1 "tarflags(): Unrecognized tar extension in file: $1"
            ;;
@@ -361,14 +455,10 @@ clean_linux()
     rm -rf linux
 }
 
-prep_build()
+prep_kernel_build()
 {
     # make .spec file
-    if [ -f /etc/init.d/functions -a -f /etc/sysconfig/network ] ; then
-        ENABLE_INIT_SCRIPTS=""
-    else
-        ENABLE_INIT_SCRIPTS="# "
-    fi
+    ENABLE_INIT_SCRIPTS=""
     sed \
        -e "s^@BASE_ARCHS@^$BASE_ARCHS^g" \
        -e "s^@BIGMEM_ARCHS@^$BIGMEM_ARCHS^g" \
@@ -384,6 +474,7 @@ prep_build()
        -e "s^@LINUX26@^$LINUX26^g" \
        -e "s^@LUSTRE_SOURCE@^${LUSTRE##*/}^g" \
        -e "s^@LUSTRE_TARGET@^$TARGET^g" \
+       -e "s^@PSERIES64_ARCHS@^$PSERIES64_ARCHS^g" \
        -e "s^@RHBUILD@^$RHBUILD^g" \
        -e "s^@SMP_ARCHS@^$SMP_ARCHS^g" \
        -e "s^@SUSEBUILD@^$SUSEBUILD^g" \
@@ -396,7 +487,8 @@ prep_build()
     [ -d BUILD ] || mkdir BUILD
     [ -d SOURCES ] || mkdir SOURCES
     for script in linux-{rhconfig.h,merge-config.awk,merge-modules.awk} \
-       suse-{functions.sh,post.sh,postun.sh,trigger-script.sh.in} ; do
+       suse-{functions.sh,post.sh,postun.sh,trigger-script.sh.in} \
+       sles8-{pre,post,postun,update_{INITRD_MODULES,rcfile_setting}}.sh ; do
        cp $TOPDIR/lustre/build/$script SOURCES
     done
     cp "$LUSTRE" "$KERNEL_FILE" SOURCES
@@ -410,14 +502,14 @@ prep_build()
 clean_lustre()
 {
     [ -d lustre ] || return 0
-    echo "Cleaning lustre..."
+    echo "Cleaning Lustre..."
     [ -L lustre ] && rm -rf $(readlink lustre)
     rm -rf lustre
 }
 
-build()
+build_kernel()
 {
-    echo "Building rpms for: $BUILD_ARCHS..."
+    echo "Building kernel + Lustre RPMs for: $BUILD_ARCHS..."
     targets=
     for arch in $BUILD_ARCHS ; do
        targets="--target $arch $targets"
@@ -425,7 +517,7 @@ build()
 
     $RPMBUILD $targets -bb lustre-kernel-2.4.spec \
        --define "_topdir $TOPDIR" || \
-       fatal 1 "Error building rpms for $arch."
+       fatal 1 "Error building rpms for $BUILD_ARCHS."
 
     if (( $DO_SRC )) ; then
        $RPMBUILD -bs lustre-kernel-2.4.spec \
@@ -434,14 +526,56 @@ build()
     fi
 }
 
-publish()
+build_lustre()
 {
-    publishing || return 0
+    [ -d SRPMS ] || mkdir SRPMS
+    [ -d RPMS ] || mkdir RPMS
+    [ -d BUILD ] || mkdir BUILD
+    [ -d SOURCES ] || mkdir SOURCES
+
+    cp "$LUSTRE" SOURCES
+
+    pushd lustre >/dev/null
+
+    echo "Building Lustre RPMs for: $BUILD_ARCHS..."
+    targets=
+    for arch in $BUILD_ARCHS ; do
+       targets="--target $arch $targets"
+    done
+
+    ./configure "--with-linux=${LINUX}" ${CONFIGURE_FLAGS}
+
+    $RPMBUILD $targets -bb build/lustre.spec \
+       --define "_topdir $TOPDIR" || \
+       fatal 1 "Error building rpms for $BUILD_ARCHS."
+
+    popd >/dev/null
+}
+
+stage()
+{
+    [ "$STAGEDIR" ] || return 0
+
+    for arch in $BUILD_ARCHS ; do
+       rpmdir="${STAGEDIR}/${CANONICAL_TARGET}-${arch}"
+       echo "${0##*/}: Copying RPMs into ${rpmdir}"
+       mkdir -p "${rpmdir}"
+       cp -v RPMS/${arch}/*.rpm "${rpmdir}"
+       if [ -d RPMS/noarch ] ; then
+           cp -v RPMS/noarch/*.rpm "${rpmdir}"
+       fi
+    done
+
+    cp -v "$LUSTRE" "$STAGEDIR"
 }
 
 [ -r ~/.lbuildrc ] && . ~/.lbuildrc
 
-options=$(getopt -o d:D:h -l external-patches:,extraversion:,kerneldir:,lustre:,nosrc,publish,release,tag:,target:,target-archs:,disable-datestamp -- "$@")
+options=$(getopt -o d:D:h -l disable-datestamp,external-patches:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,publish,release,stage:,tag:,target:,target-archs:,with-linux: -- "$@")
+
+if [ $? != 0 ] ; then
+    usage 1
+fi
 
 eval set -- "$options"
     
@@ -473,22 +607,37 @@ while [ "$1" ] ; do
            KERNELDIR=$2
            shift 2
            ;;
+       --linux | --with-linux)
+           LINUX=$2
+           shift 2
+           ;;
        --lustre)
            LUSTRE=$2
            shift 2
            ;;
+       --nodownload)
+           DOWNLOAD=0
+           shift 1
+           ;;
        --nosrc)
            DO_SRC=0
            shift 1
            ;;
        --publish)
-           PUBLISH=1
            shift
            ;;
        --release)
            RELEASE=1
            shift
            ;;
+       --src)
+           DO_SRC=1
+           shift 1
+           ;;
+       --stage)
+           STAGEDIR=$2
+           shift 2
+           ;;
        --tag)
            TAG=$2
            shift 2
@@ -519,18 +668,24 @@ done
 check_options
 
 unpack_lustre
-load_target
-
-if (( $DO_SRC )) ; then
-    unpack_linux
-    patch_linux
-    pack_linux
-    clean_linux
-fi
 
 # prep_build needs the .spec.in from the lustre source
-prep_build
-clean_lustre
+if [ -z "$LINUX" ] ; then
+    load_target
+
+    if (( $DO_SRC )) ; then
+       unpack_linux
+       patch_linux
+       pack_linux
+       clean_linux
+    fi
+
+    prep_kernel_build
+    clean_lustre
+
+    build_kernel
+else
+    build_lustre
+fi
 
-build
-publish
+stage