X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=contrib%2Flbuild%2Flbuild;h=9d6a600397e632aab75e6b2a55a6b64a4daeb22e;hb=6e84dbce05959e854d886772841a5d6d0690eb65;hp=60bf747be5c456a8ed1e394d2c7125c7cadac2b5;hpb=bb07b07567654b8c8bfdc90c399ecac9d1e41444;p=fs%2Flustre-release.git diff --git a/contrib/lbuild/lbuild b/contrib/lbuild/lbuild index 60bf747..9d6a600 100755 --- a/contrib/lbuild/lbuild +++ b/contrib/lbuild/lbuild @@ -65,12 +65,12 @@ REUSEBUILD= # should cached products be used or force rebuilding? USE_BUILD_CACHE=true # what does this do exactly? does it imply no kernel build? -NORPM=false IOKITRPM=true OSDLDISKFSRPM=true OSDZFSRPM=false SMPTYPES="smp bigsmp default ''" PATCHLESS=false +WITH_ZFS="" XEN=false LINUXOBJ= DISTRO= @@ -144,7 +144,7 @@ Usage: ${0##*/} [OPTION]... [-- ] Use ccache --norpm - Do not build RPMs (compile only mode) + Unused. --patchless Build lustre client only @@ -187,6 +187,9 @@ Usage: ${0##*/} [OPTION]... [-- ] Specifies that the files generated do not include timestamps, and that this is an official release. + --disable-zfs + Build Lustre without ZFS. + --src Build a .src.rpm, a full kernel patch, and a patched kernel tarball. @@ -281,10 +284,13 @@ check_options() { fi case $TARGET in + 3.12-sles12) + CANONICAL_TARGET="sles12" + ;; 3.10-rhel7) CANONICAL_TARGET="rhel7" ;; - 2.6-rhel6) + 2.6-rhel6*) CANONICAL_TARGET="rhel6" ;; 2.6-rhel5) @@ -386,6 +392,8 @@ compare_version () { return 1 fi done + + return 0 } uniqify() { @@ -572,129 +580,6 @@ download_ofed() { } -parse_mpss_info() { - local file="$1" - local build="$2" - local distro="$3" - - [ -r "$file" ] || fatal 1 "Could not find MPSS info file" - - # convert to MPSS distro naming - distro=${distro/sles/suse} - distro_name=${distro%%-*} - - # extract links from MPSS info file - local urls=$(cat "$file" | \ - tr -d '\t\r\n' | sed -e 's?\(\)?\1\n?g' | \ - grep '.intel.com/' | \ - sed -e 's/^.*[hH][rR][eE][fF]="\([^>"]\+\)".*$/\1/g' | \ - grep '.tar') - local res2="" - local res3="" - for url in $urls; do - local ver2="" - local ver3="" - # Try to match with MPSS 3.x or 3.x.x distro package - if [[ $url =~ mpss-[0-9].[0-9](.[0-9]*)?-$distro.tar ]]; then - ver3=${url##*mpss-} - ver3=${ver3%%-$distro.tar} - # Try to match with MPSS 3.x or 3.x.x source package - elif [[ $url =~ mpss-src-[0-9].[0-9](.[0-9]*)?.tar ]]; then - ver3=${url##*mpss-src-} - ver3=${ver3%%.tar} - # Try to match with MPSS 3.x or 3.x.x cross compiler package - elif [[ $url =~ mpss-[0-9].[0-9](.[0-9]*)?-k1om.tar ]]; then - ver3=${url##*mpss-} - ver3=${ver3%%-k1om.tar} - # Try to match with MPSS 2.1.x source package - elif [[ $url =~ _src-[0-9].[0-9].[0-9]*-[0-9]*_$distro_name.tar ]]; then - ver2=${url##*_src-} - ver2=${ver2%%_$distro_name.tar} - fi - - if [ -n "$ver3" ]; then - # Check for MPSS 3.x or MPSS 3.x.x - if [[ $ver3 =~ [0-9].[0-9].[0-9]* ]]; then - ver3="${ver3}-0" - else - ver3="${ver3}.0-0" - fi - if [ -z "$build" ]; then - res3="$ver3" - elif [ "$build" = "$ver3" ]; then - res3="$res3 $url" - fi - elif [ -n "$ver2" ]; then - if [ -z "$build" ]; then - res2="$ver2" - elif [ "$build" = "$ver2" ]; then - res2="$res2 $url" - fi - fi - - done - - if [ -z "$build" ]; then - # return a last version of MPSS - if [ -n "$res3" ]; then - echo "$res3" - else - echo "$res2" - fi - elif [ "${build%%.*}" = "3" ]; then - local ver=${build%%-*} - local bid=${build##*-} - if [ $bid -eq 0 ]; then - # return URLs from MPSS info file - echo "$res3" - else - # Check for MPSS 3.x or MPSS 3.x.x - if [ ${ver##*.} -eq 0 ]; then - ver=${ver%.*} - fi - # return URLs from internal site with MPSS builds - # kernel sources: - echo "${MPSS_BUILDS:-"http://mic-bld.pdx.intel.com/release"}/$ver/$bid/release/knightscorner/package/mpss-src-$ver.tar" - # kernel configs: - echo "${MPSS_BUILDS:-"http://mic-bld.pdx.intel.com/release"}/$ver/$bid/release/knightscorner/package/mpss-$ver-k1om.tar" - # OFED headers: - echo "${MPSS_BUILDS:-"http://mic-bld.pdx.intel.com/release"}/$ver/$bid/release/knightscorner/package/mpss-$ver-$distro.tar" - fi - else - # return URLs from MPSS info file - echo "$res2" - fi -} - -# Get public information about last releases of -# Intel Manycore Platform Software Stack (MPSS) -download_mpss_info() { - local file="$1" - local force="${2:-true}" - local url=${MPSS_URL:-"http://software.intel.com/en-us/articles/intel-manycore-platform-software-stack-mpss"} - - download_file "$url" "$file" "$force" -} - -download_mpss() { - local file="$1" - local force="${2:-false}" - local urls=$(parse_mpss_info "$file" ${MPSS_VERSION} ${MPSS_DISTRO}) - - [ -z "$urls" ] && fatal 1 "Could not determine the URLs of MPSS $MPSS_VERSION" - - # force re-download if build number is zero - [[ $MPSS_VERSION = [0-9].[0-9].[0-9]*-0 ]] && force=true - - local url - local i=0 - for url in $urls; do - file="$KERNELTREE/mpss_src-${MPSS_VERSION}-${MPSS_DISTRO}-part$i.tar" - download_file "$url" "$file" "$force" - i=$((i+1)) - done -} - load_target() { EXTRA_VERSION_save="$EXTRA_VERSION" @@ -726,7 +611,7 @@ load_target() { # go away and the target just specify the $RPMSMPTYPE [ -z "$RPMSMPTYPE" ] && set_rpm_smp_type - # CC might have been overwriten in TARGET_FILE + # CC might have been overwritten in TARGET_FILE if [[ $CC != ccache\ * ]] && which "$CCACHE" &>/dev/null; then export CCACHE && export CC="ccache $CC" fi @@ -775,9 +660,6 @@ load_target() { # kernel. don't make it look like one if $PATCHLESS || [ -n "$SERIES" ]; then EXTRA_VERSION=$(echo $EXTRA_VERSION | sed -e "s/\(.*_lustre\)\..*/\1/") - if ! $PATCHLESS && [ -n "$BUILDID" ]; then - EXTRA_VERSION="${EXTRA_VERSION}.${BUILDID}" - fi fi fi # EXTRA_VERSION=${EXTRA_VERSION//-/_} @@ -845,73 +727,6 @@ unpack_lustre() { } -unpack_mpss() { - - [ -d mpss ] && return 0 - - if [ "${MPSS_VERSION%%.*}" = "3" ]; then - local ver=${MPSS_VERSION%%-*} - local file - local i - # Check for MPSS 3.x or MPSS 3.x.x - if [ ${ver##*.} -eq 0 ]; then - ver=${ver%.*} - fi - for i in $(seq 0 9); do - file="$KERNELTREE/mpss_src-${MPSS_VERSION}-${MPSS_DISTRO}-part$i.tar" - if [ -r "$file" ]; then - untar "$file" \ - "mpss-$ver/*/linux-*.tar.bz2" \ - "mpss-$ver/*/kernel-dev-*.rpm" \ - "mpss-$ver/*/ofed-driver-*-devel-*.rpm" - fi - done - # Extract kernel configs - file=$(find mpss-$ver -type f -path "*/kernel-dev-*.rpm") - if ! rpm2cpio "$file" | cpio -idm; then - echo "Error extracting MPSS kernel configs" - return 1 - fi - # Unpack kernel sources - file=$(find mpss-$ver -type f -path "*/linux-*.tar.bz2") - if ! untar "$file"; then - echo "Error unpacking MPSS kernel sources" - return 1 - fi - # Extract OFED headers - file=$(find mpss-$ver -type f -path "*/ofed-driver-*-devel-*.rpm") - if ! rpm2cpio "$file" | cpio -idm; then - echo "Error extracting MPSS OFED headers" - return 1 - fi - # Remove unpacked archives to save space - rm -rf mpss-$ver - # Make link to MPSS kernel sources - ln -sf linux-* mpss - else - local dir="mpss-${MPSS_VERSION}-${MPSS_DISTRO%%-*}" - - mkdir $dir || return 255 - pushd $dir >/dev/null || return 255 - if ! untar "$KERNELTREE/mpss_src-${MPSS_VERSION}-${MPSS_DISTRO}-part0.tar" "*/gpl/*full_src*"; then - popd >/dev/null - rm -rf $dir - echo "Error unpacking MPSS tarball 1" - return 1 - fi - local file=$(find . -type f -path "*/gpl/*full_src*") - if ! untar "$file"; then - popd >/dev/null - rm -rf $dir - echo "Error unpacking MPSS tarball 2" - return 1 - fi - popd >/dev/null - ln -sf $dir mpss - fi - -} - do_patch_linux() { local do_patch=${1:-true} @@ -952,6 +767,7 @@ do_patch_linux() { build_lustre() { local linux="$1" local linuxobj="$2" + local configure_args="" cp "$LUSTRE" SOURCES @@ -963,28 +779,12 @@ build_lustre() { fi echo "Building Lustre RPMs for: $BUILD_ARCHS..." - local targets arch - for arch in $BUILD_ARCHS; do - targets="--target $arch $targets" - done - - local confoptions="" - - if $PATCHLESS; then - confoptions="$confoptions --disable-server" - fi - - local rpmbuildopt='-tb' - if $NORPM; then - rpmbuildopt='-tc' - echo NORPM mode. Only compiling. - fi # If server we now build the spl and zfs modules against the lustre kernel. # These are required prior to the building of lustre server. Client does # not require spl/zfs. Use !PATCHLESS to indicate server which follows the # line above so is at least consistant. - if [ $PATCHLESS == false ] && [ "x$ZFSNOTSUPPORTED" == "x" ]; then + if [ $PATCHLESS == false ] && [ "x$WITH_ZFS" == "x" ]; then if ! build_spl_zfs; then popd >/dev/null # pushd lustre return 255 @@ -992,60 +792,61 @@ build_lustre() { fi if $PATCHLESS; then - RPMBUILD_DEFS="$RPMBUILD_DEFS --without servers" - if [ -n "$CROSS_SUFFIX" ]; then - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"lustre_name lustre-client$CROSS_SUFFIX\"" - fi + configure_args="$configure_args --disable-server" fi # ditto for the lustre-tests boolean if ! $LUSTRE_TESTS; then - RPMBUILD_DEFS="$RPMBUILD_DEFS --without lustre_tests" + configure_args="$configure_args --disable-tests" fi if ! $IOKITRPM; then - RPMBUILD_DEFS="$RPMBUILD_DEFS --without lustre_iokit" + configure_args="$configure_args --disable-iokit" fi - local osd_zfs="" - if $OSDZFSRPM; then - osd_zfs="yes" + if ! $OSDZFSRPM; then + configure_args="$configure_args --without-zfs" fi - local osd_ldiskfs="" - if $OSDLDISKFSRPM; then - osd_ldiskfs="yes" + if ! $OSDLDISKFSRPM; then + configure_args="$configure_args --disable-ldiskfs" fi - RPMBUILD_DEFS="$RPMBUILD_DEFS ${FIND_REQUIRES:+--define \"__find_requires $FIND_REQUIRES\"}" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"configure_args $confoptions ${CONFIGURE_FLAGS}\"" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"kdir $linux\"" - RPMBUILD_DEFS="$RPMBUILD_DEFS ${linuxobj:+--define \"kobjdir $linuxobj\"}" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"_tmppath $TMPDIR\"" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"_topdir $TOPDIR\"" + configure_args="$configure_args --with-linux=$linux" + configure_args="$configure_args ${linuxobj:+--with-linux-obj=$linuxobj}" + + # allow environment setting to override ldiskfs series selection + [ -n "$LDISKFS_SERIES" ] && export LDISKFS_SERIES - eval $RPMBUILD $targets $rpmbuildopt "$LUSTRE" $RPMBUILD_DEFS \ - ${osd_zfs:+--with zfs} \ - ${osd_ldiskfs:+--with ldiskfs} 2>&1 || \ + ./configure $configure_args $CONFIGURE_FLAGS 2>&1 || + fatal 1 "Error in configure." + make rpms 2>&1 || fatal 1 "Error building rpms for $BUILD_ARCHS." - if $DO_SRC; then - if ! eval $RPMBUILD -ts "$LUSTRE" $RPMBUILD_DEFS 2>&1; then - popd >/dev/null - return 255 - fi - fi + # move RPMs into place where they are expected to be + for arch in $BUILD_ARCHS; do + mv -f lustre-*.${arch}.rpm $TOPDIR/RPMS/${arch}/ + done + mv -f lustre-*.src.rpm $TOPDIR/SRPMS/ + popd >/dev/null + return 0 } +# Only zfs Lustre DKMS Server is supported build_lustre_dkms() { - [ -n "$MPSS_VERSION" ] && return 0 - local ver=$(eval echo $(awk '/LUSTRE_VERSION_STRING/ {print $3}' lustre/include/lustre_ver.h)) + local ver=$(sed -n -e 's/^LUSTRE_VERSION = //p' LUSTRE-VERSION-FILE) echo "Building Lustre DKMS RPMs for: $BUILD_ARCHS..." ./configure --enable-dist || return 255 - $RPMBUILD --define="_topdir $TOPDIR" -bs lustre-dkms.spec || return 255 - $RPMBUILD --rebuild --define="_topdir $TOPDIR" $TOPDIR/SRPMS/lustre-dkms-$ver-*.src.rpm || return 255 + + if $PATCHLESS; then + $RPMBUILD --define="_topdir $TOPDIR" --without servers -bs lustre-dkms.spec || return 255 + $RPMBUILD --rebuild --define="_topdir $TOPDIR" --without servers $TOPDIR/SRPMS/lustre-client-dkms-$ver-*.src.rpm || return 255 + else + $RPMBUILD --define="_topdir $TOPDIR" -bs lustre-dkms.spec || return 255 + $RPMBUILD --rebuild --define="_topdir $TOPDIR" $TOPDIR/SRPMS/lustre-dkms-$ver-*.src.rpm || return 255 + fi return 0 } @@ -1069,7 +870,7 @@ build_spl_zfs() { # The spl/zfs spec files expect RPM_BUILD_ROOT to point to the root of the # destination for the rpms export RPM_BUILD_ROOT=$TOPDIR - SPLZFSVER=${SPLZFSVER:-0.6.3} + SPLZFSVER=${SPLZFSVER:-0.6.4.2} SPLZFSTAG=${SPLZFSTAG:-} # The files expect a kver to be set to the kernel version . @@ -1185,34 +986,6 @@ build_spl_zfs() { return 0 } -prepare_mpss() { - - pushd mpss >/dev/null || return 255 - if [ "${MPSS_VERSION%%.*}" = "3" ]; then - cp -f ../boot/config-* .config - if ! make ARCH=k1om silentoldconfig ; then - popd >/dev/null - return 1 - fi - if ! make ARCH=k1om modules_prepare ; then - popd >/dev/null - return 1 - fi - cp -f ../boot/Module.symvers-* Module.symvers - else - if ! make defconfig-miclinux; then - popd >/dev/null - return 1 - fi - if ! make -C card/kernel ARCH=k1om modules_prepare ; then - popd >/dev/null - return 1 - fi - fi - popd >/dev/null - -} - stage() { [ "$STAGEDIR" ] || return 0 @@ -1221,10 +994,9 @@ stage() { rpmdir="${STAGEDIR}/${CANONICAL_TARGET}-${arch}" echo "${0##*/}: Copying RPMs into ${rpmdir}" mkdir -p "${rpmdir}" - cp -v RPMS/${arch}/*.rpm "${rpmdir}" - if [ -d RPMS/noarch ]; then - cp -v RPMS/noarch/*.rpm "${rpmdir}" - fi + for rpm in $(ls RPMS/${arch}/*.rpm RPMS/noarch/*.rpm); do + cp -v $rpm "${rpmdir}" + done done cp -v "$LUSTRE" "$STAGEDIR" @@ -1561,7 +1333,8 @@ build_kernel_with_srpm() { local REUSE_SIGNATURE=$({ echo -en $release_str; echo $BUILD_GEN; cat "$CONFIG_FILE"; - cat "$TARGET_FILE"; + cat "$TARGET_FILE" | + sed -e '/_VERSION=/s/_[0-9]*_g.*$//g'; cat "$FULL_PATCH"; cat "$LBUILD_DIR/lbuild"; cat "$LBUILD_DIR/lbuild-$DISTROMAJ"; } | @@ -1982,7 +1755,7 @@ set -E [ -r ~/.lbuildrc ] && . ~/.lbuildrc -options=$(getopt -o D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ccache,norpm,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,noiokit,ofed-version:,mpss-version:,publish,release,set-value:,src,stage:,target:,target-archs:,with-linux:,xen -- "$@") +options=$(getopt -o D:h -l kerneltree:,distro:,kernelrpm:,reusebuild:,patchless,ccache,norpm,external-patches:,timestamp:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,noiokit,ofed-version:,publish,disable-zfs,release,set-value:,src,stage:,target:,target-archs:,with-linux:,xen -- "$@") if [ $? != 0 ]; then usage 1 @@ -2041,7 +1814,6 @@ while [ "$1" ]; do shift 2 ;; --norpm) - NORPM=true shift ;; --noiokit) @@ -2080,13 +1852,13 @@ while [ "$1" ]; do OFED_VERSION="$2" shift 2 ;; - --mpss-version) - MPSS_VERSION="$2" - shift 2 - ;; --publish) shift ;; + --disable-zfs) + WITH_ZFS="no" + shift + ;; --release) RELEASE=true shift @@ -2135,10 +1907,6 @@ check_options unpack_lustre -# XXX - should we _always_ get the buildid from the META file? what are the -# other (i.e. non-lustre-tarball use cases of lbuild)? -BUILDID=$(sed -n -e 's/^BUILDID = //p' lustre/META) - load_target if [ -n "$OFED_VERSION" -a "$OFED_VERSION" != "inkernel" ]; then @@ -2146,65 +1914,6 @@ if [ -n "$OFED_VERSION" -a "$OFED_VERSION" != "inkernel" ]; then unpack_ofed || fatal 1 "Error unpacking OFED tarball" fi -if [ -n "$MPSS_VERSION" ]; then - [ -z "$MPSS_DISTRO" ] && MPSS_DISTRO=$(autodetect_distro) - - MPSS_INFO=$(mktemp -u mpss-info-XXXXXXXXXX.html) - download_mpss_info "$MPSS_INFO" - - if [ "$MPSS_VERSION" = "last" ]; then - MPSS_VERSION=$(parse_mpss_info "$MPSS_INFO" "" ${MPSS_DISTRO}) - [ -z "$MPSS_VERSION" ] && - fatal 1 "Could not determine the last MPSS version" - fi - - download_mpss "$MPSS_INFO" - [ -r "$MPSS_INFO" ] && rm -f "$MPSS_INFO" - echo "Building with MPSS $MPSS_VERSION" - unpack_mpss || fatal 1 "Error unpacking MPSS tarballs" - prepare_mpss || fatal 1 "Error preparing MPSS for kernel modules build" - - if [ -z "$MPSS_OFED" ]; then - MPSS_OFED=$(find $PWD -type f -path "*/ofed-driver-*/Module.symvers") - if [ -n "$MPSS_OFED" ]; then - MPSS_OFED="--with-o2ib=$(dirname $MPSS_OFED)" - else - MPSS_OFED="--without-o2ib" - fi - fi - - # disable unsupported parts: - PATCHLESS=true - IOKITRPM=false - LDISKFSRPM=false - ZFSNOTSUPPORTED="yes" - - # define variables for cross compilation: - CROSS_SUFFIX="-mic" - if [ "${MPSS_VERSION%%.*}" = "3" ]; then - CC_TARGET_ARCH=k1om-mpss-linux - LINUX="$TOPDIR/mpss" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"cross_requires /bin/sh\"" - else - CC_TARGET_ARCH=x86_64-k1om-linux - LINUX="$TOPDIR/mpss/card/kernel" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"cross_requires intel-mic-gpl = ${MPSS_VERSION%%-*}\"" - fi - LUSTRE_VERSION=$(sed -n -e 's/^VERSION = //p' lustre/META) - CONFIGURE_FLAGS="$CONFIGURE_FLAGS $MPSS_OFED --host=$CC_TARGET_ARCH --build=x86_64-pc-linux" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"post_script build/gen_filelist.sh\"" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"make_args ARCH=k1om CROSS_COMPILE=${CC_TARGET_ARCH}-\"" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"cross_path ${CROSS_PATH:=/opt/lustre/${LUSTRE_VERSION}/${CC_TARGET_ARCH}}\"" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"rootdir %{cross_path}\"" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"_prefix %{cross_path}/usr\"" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"_mandir %{_prefix}/share/man\"" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"_sysconfdir %{cross_path}/etc\"" - RPMBUILD_DEFS="$RPMBUILD_DEFS --define \"kmoddir extra\"" - - # redefine CC for proper ./configure during rpmbuild - export CC=${CC_TARGET_ARCH}-gcc -fi - # make sure the RPM build environment is set up create_rpmbuild_dirs