build/lbuild

   1 #!/bin/sh
   2
   3 TOPDIR=$PWD
   4
   5 # CVSROOT is inherited from the environment
   6 KERNELDIR=
   7 LUSTRE=
   8 PUBLISH=0
   9 RELEASE=0
  10 DO_SRC=0
  11 DOWNLOAD=1
  12 TAG=
  13 TARGET=
  14 TARGET_ARCHS=
  15 CONFIGURE_FLAGS=
  16 EXTERNAL_PATCHES=
  17 EXTRA_VERSION=
  18
  19 # from target file
  20 KERNEL=
  21 SERIES=
  22 CONFIG=
  23 VERSION=
  24
  25 RHBUILD=0
  26 SUSEBUILD=0
  27 LINUX26=0
  28 SUSEBUILD=0
  29
  30 BASE_ARCHS=
  31 BIGMEM_ARCHS=
  32 BOOT_ARCHS=
  33 JENSEN_ARCHS=
  34 SMP_ARCHS=
  35 BIGSMP_ARCHS=
  36 PSERIES64_ARCHS=
  37 UP_ARCHS=
  38
  39 DATE=$(date)
  40
  41 USE_DATESTAMP=1
  42 RPMBUILD=
  43
  44 export CC=${CC:-gcc}
  45
  46 # Readlink is not present on some older distributions: emulate it.
  47 readlink() {
  48     local path=$1 ll
  49
  50     if [ -L "$path" ]; then
  51         ll="$(LC_ALL=C ls -l "$path" 2> /dev/null)" &&
  52         echo "${ll/* -> }"
  53     else
  54         return 1
  55     fi
  56 }
  57
  58 cleanup()
  59 {
  60     true
  61 }
  62
  63 error()
  64 {
  65     [ "$1" ] && echo -e "\n${0##*/}: $1"
  66 }
  67
  68 fatal()
  69 {
  70     cleanup
  71     error "$2"
  72     exit $1
  73 }
  74
  75 publishing()
  76 {
  77     (( $PUBLISH )) || return 0
  78 }
  79
  80 is_release()
  81 {
  82     (( $RELEASE )) || return 0
  83 }
  84
  85 list_targets()
  86 {
  87     echo -n "Available targets:"
  88     for target in $TOPDIR/lustre/lustre/kernel_patches/targets/*.target ; do
  89         target_file=${target##*/}
  90         echo -n " ${target_file%%.target}"
  91     done
  92     echo
  93 }
  94
  95 usage()
  96 {
  97     cat <<EOF
  98 Usage: ${0##*/} [OPTION]... [-- <lustre configure options>]
  99
 100   -d CVSROOT
 101     Specifies the CVS Root to use when pulling files from CVS.  The
 102     environment variable \$CVSROOT is used if this option is not
 103     present.
 104
 105   --external-patches=EXTERNAL_PATCHES
 106     Directory similar to lustre/lustre/kernel_patches/ that lbuild should
 107     look for seres and config files in before looking in the lustre
 108     tree.
 109
 110   --extraversion=EXTRAVERSION
 111     Text to use for the rpm release and kernel extraversion.
 112
 113   --kerneldir=KERNELDIR
 114     Directory containing Linux source tarballs referenced by target
 115     files.
 116
 117   --lustre=LUSTRE
 118     Path to an existing lustre source tarball to use instead of
 119     pulling from CVS.
 120
 121   --nodownload
 122     Do not try to download a kernel from ftp.lustre.org
 123
 124   --nosrc
 125     Do not build a .src.rpm, a full kernel patch, or a patched kernel
 126     tarball.
 127
 128   --publish
 129     Publish the packages, patches, and tarballs on the ftp server.
 130
 131   --release
 132     Specifies that the files generated do not include timestamps, and
 133     that this is an official release.
 134
 135   --tag=TAG
 136     A CVS branch/tag name to build from when pulling from CVS.
 137
 138   --target=TARGET
 139     The name of the target to build.  The available targets are listed
 140     below.
 141
 142   --target-archs=TARGET_ARCHS
 143     A (space delimited) list of architectures to build.  By default,
 144     all of the archs supported by the TARGET will be built, in
 145     addition to a .src.rpm.  This option can limit those, for machines
 146     that can only build certain archs or if you only want a certain
 147     arch built (for testing, or a one-off kernel).
 148
 149     Also note that by using a non-"base" arch (eg, i386) only kernels
 150     will be built - there will be no lustre-lite-utils package.
 151
 152   --disable-datestamp
 153     Prevents the datestamp flag (-D) from being passed to cvs for
 154     checkouts. This is a workaround for a problem encountered when
 155     using lbuild with tinderbox.
 156
 157 EOF
 158
 159 #   list_targets
 160
 161     fatal "$1" "$2"
 162 }
 163
 164 check_options()
 165 {
 166     if [ "$LUSTRE" ] ; then
 167         [ -r "$LUSTRE" ] || \
 168             usage 1 "Could not find Lustre source tarball '$LUSTRE'."
 169     else
 170         [ "$CVSROOT" ] || \
 171             usage 1 "Either specify a CVS Root with -d, or a Lustre source tarball with --lustre."
 172         [ "$TAG" ] || \
 173             usage 1 "A branch/tag name must be specified with --tag when not building from a tarball."
 174     fi
 175
 176     [ "$KERNELDIR" ] || \
 177         usage 1 "A kernel directory must be specified with --kerneldir."
 178
 179     [ -d "$KERNELDIR" ] || \
 180         usage 1 "$KERNELDIR is not a directory."
 181
 182     if ! (( $RELEASE )) ; then
 183         [ "$TAG" ] || \
 184             usage 1 "When building a snapshot, a tag name must be used."
 185     fi
 186
 187     TIMESTAMP=$(date -d "$DATE" "+%Y%m%d%H%M")
 188
 189     [ "$TARGET" ] || usage 1 "A target must be specified with --target."
 190 #    TARGET_FILE="$TOPDIR/lustre/kernel_patches/targets/$TARGET.target"
 191 #    [ -r "$TARGET_FILE" ] || \
 192 #       usage 1 "Target '$TARGET' was not found."
 193
 194     RPMBUILD=$(which rpmbuild 2>/dev/null | head -1)
 195     if [ ! "$RPMBUILD" -o "$RPMBUILD" == "" ]; then
 196         RPMBUILD=$(which rpm 2>/dev/null | head -1)
 197         if [ ! "$RPMBUILD" -o "$RPMBUILD" == "" ]; then
 198             usage 1 "Could not find binary for making rpms (tried rpmbuild and rpm)."
 199         fi
 200     fi
 201 }
 202
 203 uniqify()
 204 {
 205     echo $(echo "$*" | xargs -n 1 | sort -u)
 206 }
 207
 208 load_target()
 209 {
 210     EXTRA_VERSION_save="$EXTRA_VERSION"
 211     for patchesdir in "$EXTERNAL_PATCHES" "$TOPDIR/lustre/lustre/kernel_patches" ; do
 212         TARGET_FILE="$patchesdir/targets/$TARGET.target"
 213         [ -r "$TARGET_FILE" ] && break
 214     done
 215     [ -r "$TARGET_FILE" ] || \
 216         fatal 1 "Target $TARGET was not found."
 217
 218     echo "Loading target config file $TARGET.target..."
 219
 220     . "$TARGET_FILE"
 221
 222     [ "$KERNEL"  ] || fatal 1 "Target $TARGET did not specify a kernel."
 223 #    [ "$SERIES"  ] || fatal 1 "Target $TARGET did not specify a kernel patch series."
 224 #    [ "$CONFIG"  ] || fatal 1 "Target $TARGET did not specify a kernel config."
 225     [ "$VERSION" ] || fatal 1 "Target $TARGET did not specify a kernel version."
 226
 227     if [ "$KERNELDIR" ] ; then
 228         KERNEL_FILE="$KERNELDIR/$KERNEL"
 229         if [ ! -r "$KERNELDIR/$KERNEL" ] ; then
 230             case $TARGET in
 231                 2.6-rhel4)
 232                     dldir="rhel-2.6"
 233                     ;;
 234                 2.6-suse)
 235                     dldir="suse-2.6"
 236                     ;;
 237                 hp_pnnl-2.4)
 238                     dldir="hp-pnnl-2.4"
 239                     ;;
 240                 2.6-vanilla \
 241                 | suse-2.4.21-2 \
 242                 | rh-2.4 \
 243                 | rhel-2.4 \
 244                 | sles-2.4)
 245                     dldir="$TARGET"
 246                     ;;
 247             esac
 248             if (( $DOWNLOAD )) ; then
 249                 echo "Downloading http://ftp.lustre.org/kernels/$dldir/old/$KERNEL..."
 250                 if ! wget -nv "http://ftp.lustre.org/kernels/$dldir/old/$KERNEL" \
 251                     -O "$KERNELDIR/$KERNEL" ; then
 252                     fatal 1 "Could not download target $TARGET's kernel file $KERNEL from ftp.lustre.org."
 253                 fi
 254             else
 255                 fatal 1 "Target $TARGET's kernel file $KERNEL not found in kernel directory $KERNELDIR."
 256             fi
 257         fi
 258     fi
 259
 260     if [ "$SERIES" ] ; then
 261         for series in $SERIES ; do
 262             for patchesdir in "$EXTERNAL_PATCHES" "$TOPDIR/lustre/lustre/kernel_patches" ; do
 263                 [ -r "$patchesdir/series/$series" ] && continue 2
 264             done
 265             fatal 1 "Target $TARGET's series $SERIES could not be found.\nSearched:\n\t$EXTERNAL_PATCHES/series\n\t$TOPDIR/lustre/lustre/kernel_patches/series."
 266         done
 267     fi
 268
 269     CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/$CONFIG"
 270     [ -r "$CONFIG_FILE" ] || \
 271         fatal 1 "Target $TARGET's config file $CONFIG missing from $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/."
 272
 273     if [ "$EXTRA_VERSION_save" ] ; then
 274         EXTRA_VERSION="$EXTRA_VERSION_save"
 275     elif ! (( $RELEASE )) ; then
 276         #remove the @VERSION@ (lustre version)
 277         EXTRA_VERSION=$(echo $EXTRA_VERSION | sed -e "s/\(.*_lustre\)\..*/\1/")
 278         EXTRA_VERSION="${EXTRA_VERSION}-${TAG}.${TIMESTAMP}"
 279     fi
 280     # EXTRA_VERSION=${EXTRA_VERSION//-/_}
 281
 282     ALL_ARCHS="$BASE_ARCHS $BIGMEM_ARCHS $BOOT_ARCHS $JENSEN_ARCHS $SMP_ARCHS $BIGSMP_ARCHS $PSERIES64_ARCHS $UP_ARCHS"
 283
 284     BUILD_ARCHS=
 285     for arch in $(uniqify "$ALL_ARCHS") ; do
 286         if [ -z "$TARGET_ARCHS" ] || echo "$TARGET_ARCHS" | grep "$arch" >/dev/null 2>/dev/null ; then
 287             BUILD_ARCHS="$BUILD_ARCHS $arch"
 288         fi
 289     done
 290     [ "$BUILD_ARCHS" ] || usage 1 "No available target archs to build."
 291     echo "Building for: $BUILD_ARCHS"
 292 }
 293
 294 tarflags()
 295 {
 296     case "$1" in
 297         '')
 298             fatal 1 "tarflags(): File name argument missing."
 299             ;;
 300         *.tar.gz)
 301             echo 'zxf'
 302             ;;
 303         *.tar.bz2)
 304             echo 'jxf'
 305             ;;
 306         *)
 307             fatal 1 "tarflags(): Unrecognized tar extension in file: $1"
 308             ;;
 309     esac
 310 }
 311
 312 untar()
 313 {
 314     echo "Untarring ${1##*/}..."
 315     tar $(tarflags "$1") "$1"
 316 }
 317
 318 unpack_lustre()
 319 {
 320     DIRNAME="lustre-$TAG-$TIMESTAMP"
 321     if [ "$LUSTRE" ] ; then
 322         untar "$LUSTRE"
 323         [ -d lustre ] || ln -sf lustre* lustre
 324     else
 325         if [ "$USE_DATESTAMP" ]; then
 326             DATESTAMP="-D '$DATE'"
 327         else
 328             DATESTAMP=""
 329         fi
 330
 331         cvs -d "$CVSROOT" -qz3 co $DATESTAMP -d "$DIRNAME" lustre || \
 332             fatal 1 "There was an error checking out toplevel Lustre from CVS."
 333         pushd "$DIRNAME" > /dev/null
 334         ./lustrecvs "$TAG" || \
 335             fatal 1 "There was an error checking out Lustre/Portals/Build from CVS."
 336         echo "Creating lustre tarball..."
 337         sh autogen.sh || fatal 1 "There was an error running autogen.sh."
 338         ./configure --disable-{modules,utils,liblustre,tests,doc} || \
 339             fatal 1 "There was an error running ./configure to create makefiles."
 340         make dist || fatal 1 "There was an error running 'make dist'."
 341         popd > /dev/null
 342         fname=`basename $DIRNAME/lustre-*.tar.gz`
 343         cp $DIRNAME/$fname . || fatal 1 "There was an error copying lustre tarball."
 344         LUSTRE="$PWD/$fname"
 345         ln -sf "$DIRNAME" lustre
 346     fi
 347 }
 348
 349 unpack_linux()
 350 {
 351     untar "$KERNEL_FILE"
 352     [ -d linux ] || ln -sf linux* linux
 353 }
 354
 355 patch_linux()
 356 {
 357     [ "$SERIES" ] || return 0
 358     FULL_PATCH="$PWD/lustre-kernel-${TARGET}-${EXTRA_VERSION}.patch"
 359     [ -f "$FULL_PATCH" ] && rm -f "$FULL_PATCH"
 360     pushd linux >/dev/null
 361     for series in $SERIES ; do
 362         echo -n "Applying series $series:"
 363         for patchesdir in "$EXTERNAL_PATCHES" "$TOPDIR/lustre/lustre/kernel_patches" ; do
 364             [ -r "$patchesdir/series/$series" ] || continue
 365             SERIES_FILE="$patchesdir/series/$series"
 366             for patch in $(<"$SERIES_FILE") ; do
 367                 echo -n " $patch"
 368                 PATCH_FILE="$patchesdir/patches/$patch"
 369                 [ -r "$PATCH_FILE" ] || \
 370                     fatal 1 "Patch $patch does not exist in Lustre tree."
 371                 cat "$PATCH_FILE" >> "$FULL_PATCH" || \
 372                     fatal 1 "Error adding patch $patch to full patch."
 373                 patch -s -p1 < "$PATCH_FILE" || fatal 1 "Error applying patch $patch."
 374             done
 375             break
 376         done
 377         echo
 378     done
 379     popd >/dev/null
 380     echo "Full patch has been saved in ${FULL_PATCH##*/}."
 381     echo "Replacing .config files..."
 382     [ -d linux/configs ] || mkdir linux/configs || \
 383         fatal 1 "Error creating configs directory."
 384     rm -f linux/configs/*
 385     copysuccess=0
 386     for patchesdir in "$EXTERNAL_PATCHES" "lustre/lustre/kernel_patches" ; do
 387         [ "$patchesdir" ] && \
 388             cp -v $patchesdir/kernel_configs/kernel-${VERSION}-${TARGET}*.config linux/configs/ >/dev/null && copysuccess=1
 389     done
 390     [ "$copysuccess" = "1" ] || \
 391         fatal 1 "Error copying in kernel configs."
 392 }
 393
 394 pack_linux()
 395 {
 396     TARBALL="$(readlink linux)-$EXTRA_VERSION.tar.gz"
 397     echo "Creating patched linux tarball $TARBALL..."
 398     tar zcf "$TARBALL" "$(readlink linux)" \
 399         --exclude "CVS" --exclude ".cvsignore" || \
 400         --exclude "*.orig" --exclude "*~" --exclude "*.rej" || \
 401         fatal 1 "Error creating patched Linux tarball."
 402 }
 403
 404 clean_linux()
 405 {
 406     [ -d linux ] || return 0
 407     echo "Cleaning linux..."
 408     [ -L linux ] && rm -rf $(readlink linux)
 409     rm -rf linux
 410 }
 411
 412 prep_build()
 413 {
 414     # make .spec file
 415     ENABLE_INIT_SCRIPTS=""
 416     sed \
 417         -e "s^@BASE_ARCHS@^$BASE_ARCHS^g" \
 418         -e "s^@BIGMEM_ARCHS@^$BIGMEM_ARCHS^g" \
 419         -e "s^@BIGSMP_ARCHS@^$BIGSMP_ARCHS^g" \
 420         -e "s^@BOOT_ARCHS@^$BOOT_ARCHS^g" \
 421         -e "s^@CONFIGURE_FLAGS@^$CONFIGURE_FLAGS^g" \
 422         -e "s^@ENABLE_INIT_SCRIPTS@^$ENABLE_INIT_SCRIPTS^g" \
 423         -e "s^@JENSEN_ARCHS@^$BOOT_ARCHS^g" \
 424         -e "s^@KERNEL_EXTRA_VERSION@^$EXTRA_VERSION^g" \
 425         -e "s^@KERNEL_RELEASE@^${EXTRA_VERSION//-/_}^g" \
 426         -e "s^@KERNEL_SOURCE@^$KERNEL^g" \
 427         -e "s^@KERNEL_VERSION@^$VERSION^g" \
 428         -e "s^@LINUX26@^$LINUX26^g" \
 429         -e "s^@LUSTRE_SOURCE@^${LUSTRE##*/}^g" \
 430         -e "s^@LUSTRE_TARGET@^$TARGET^g" \
 431         -e "s^@PSERIES64_ARCHS@^$PSERIES64_ARCHS^g" \
 432         -e "s^@RHBUILD@^$RHBUILD^g" \
 433         -e "s^@SMP_ARCHS@^$SMP_ARCHS^g" \
 434         -e "s^@SUSEBUILD@^$SUSEBUILD^g" \
 435         -e "s^@SUSEBUILD@^$SUSEBUILD^g" \
 436         -e "s^@UP_ARCHS@^$UP_ARCHS^g" \
 437         < $TOPDIR/lustre/build/lustre-kernel-2.4.spec.in \
 438         > lustre-kernel-2.4.spec
 439     [ -d SRPMS ] || mkdir SRPMS
 440     [ -d RPMS ] || mkdir RPMS
 441     [ -d BUILD ] || mkdir BUILD
 442     [ -d SOURCES ] || mkdir SOURCES
 443     for script in linux-{rhconfig.h,merge-config.awk,merge-modules.awk} \
 444         suse-{functions.sh,post.sh,postun.sh,trigger-script.sh.in} \
 445         sles8-{pre,post,postun,update_{INITRD_MODULES,rcfile_setting}}.sh ; do
 446         cp $TOPDIR/lustre/build/$script SOURCES
 447     done
 448     cp "$LUSTRE" "$KERNEL_FILE" SOURCES
 449     if [ "$EXTERNAL_PATCHES" -a -d "$EXTERNAL_PATCHES" ] ; then
 450         tar zcf SOURCES/external-patches.tar.gz -C "$EXTERNAL_PATCHES" series targets patches kernel_configs
 451     else
 452         touch SOURCES/external-patches.tar.gz
 453     fi
 454 }
 455
 456 clean_lustre()
 457 {
 458     [ -d lustre ] || return 0
 459     echo "Cleaning lustre..."
 460     [ -L lustre ] && rm -rf $(readlink lustre)
 461     rm -rf lustre
 462 }
 463
 464 build()
 465 {
 466     echo "Building rpms for: $BUILD_ARCHS..."
 467     targets=
 468     for arch in $BUILD_ARCHS ; do
 469         targets="--target $arch $targets"
 470     done
 471
 472     $RPMBUILD $targets -bb lustre-kernel-2.4.spec \
 473         --define "_topdir $TOPDIR" || \
 474         fatal 1 "Error building rpms for $arch."
 475
 476     if (( $DO_SRC )) ; then
 477         $RPMBUILD -bs lustre-kernel-2.4.spec \
 478             --define "_topdir $TOPDIR" || \
 479             fatal 1 "Error building .src.rpm."
 480     fi
 481 }
 482
 483 publish()
 484 {
 485     publishing || return 0
 486 }
 487
 488 [ -r ~/.lbuildrc ] && . ~/.lbuildrc
 489
 490 options=$(getopt -o d:D:h -l external-patches:,extraversion:,kerneldir:,lustre:,nodownload,nosrc,publish,release,tag:,target:,target-archs:,disable-datestamp -- "$@")
 491
 492 eval set -- "$options"
 493
 494 while [ "$1" ] ; do
 495     case "$1" in
 496         '')
 497             usage 1
 498             ;;
 499         -d)
 500             CVSROOT=$2
 501             shift 2
 502             ;;
 503         -D)
 504             DATE=$2
 505             shift 2
 506             ;;
 507         --external-patches)
 508             EXTERNAL_PATCHES=$2
 509             shift 2
 510             ;;
 511         --extraversion)
 512             EXTRA_VERSION=$2
 513             shift 2
 514             ;;
 515         --help | -h)
 516             usage 0
 517             ;;
 518         --kerneldir)
 519             KERNELDIR=$2
 520             shift 2
 521             ;;
 522         --lustre)
 523             LUSTRE=$2
 524             shift 2
 525             ;;
 526         --nodownload)
 527             DOWNLOAD=0
 528             shift 1
 529             ;;
 530         --nosrc)
 531             DO_SRC=0
 532             shift 1
 533             ;;
 534         --publish)
 535             PUBLISH=1
 536             shift
 537             ;;
 538         --release)
 539             RELEASE=1
 540             shift
 541             ;;
 542         --tag)
 543             TAG=$2
 544             shift 2
 545             ;;
 546         --target)
 547             TARGET=$2
 548             shift 2
 549             ;;
 550         --target-archs)
 551             TARGET_ARCHS=$2
 552             shift 2
 553             ;;
 554         --disable-datestamp)
 555             USE_DATESTAMP=
 556             shift
 557             ;;
 558         --)
 559             shift
 560             CONFIGURE_FLAGS=$@
 561             break
 562             ;;
 563         *)
 564             usage 1 "Unrecognized option: $1"
 565             ;;
 566     esac
 567 done
 568
 569 check_options
 570
 571 unpack_lustre
 572 load_target
 573
 574 if (( $DO_SRC )) ; then
 575     unpack_linux
 576     patch_linux
 577     pack_linux
 578     clean_linux
 579 fi
 580
 581 # prep_build needs the .spec.in from the lustre source
 582 prep_build
 583 clean_lustre
 584
 585 build
 586 publish