From 9e2944b0357f4636aef364524554116a66237f11 Mon Sep 17 00:00:00 2001 From: Gu Zheng Date: Fri, 8 May 2020 03:16:42 -0400 Subject: [PATCH] EX-1135 lipe: add support to build against centos8 There's a huge difference between centOS8 and centOS 7 series, especailly the strict distinction between python2 and python3, and related python rpms or pypi packages are the same condition. Following changes are introduced to add support to build against centOS8: 1. the python platform is strict to python2(python2.7) 2. use 'pip2' instead of 'pip' for pypi 3. improve dependency package list (rpm and pypi module), make it can be acceptable to centOS7.x and centOS8.x 4. fix code sytle issues to make pylint/pep8 on centOS8 happy 5. set encoding via environ "PYTHONIOENCODING" if sys.setdefaultencoding is gone (python2.7 on centOS8) 6. improve the lipe.spec to support "make rpms" against centOS8 Change-Id: Id172e2a6aa29f382c4d12ff0d2e748e8b0cde444 Signed-off-by: Gu Zheng Reviewed-on: https://review.whamcloud.com/43483 Tested-by: jenkins Tested-by: Minh Diep Tested-by: Maloo Reviewed-by: Minh Diep Reviewed-by: Gaurang Tapase Reviewed-by: John L. Hammond --- contrib/lbuild/lbuild | 19 ++++++++-- lipe/Makefile.am | 4 +- lipe/configure.ac | 6 +-- lipe/gen_lipe_test | 2 +- lipe/ldsync | 2 +- lipe/lipe.spec.in | 52 +++++++++++++------------- lipe/lipe_build | 2 +- lipe/lipe_copytool_manager | 2 +- lipe/lipe_expression_tests | 2 +- lipe/lipe_find | 2 +- lipe/lipe_hsm_check | 2 +- lipe/lipe_install | 2 +- lipe/lipe_install_build_deps | 2 +- lipe/lipe_launch | 2 +- lipe/lipe_run_action | 2 +- lipe/lipe_test | 2 +- lipe/lipe_test_console | 2 +- lipe/lipe_test_copytool | 2 +- lipe/lipe_test_launch | 2 +- lipe/lipe_test_scheduler | 2 +- lipe/lipe_virt | 2 +- lipe/loris_backup | 2 +- lipe/loris_crontab | 2 +- lipe/loris_test | 2 +- lipe/lpcc | 2 +- lipe/lpcc_cleanup | 2 +- lipe/lpcc_test | 2 +- lipe/pybuild/Makefile.am | 11 +----- lipe/pybuild/gen_lipe_test.py | 12 ++++-- lipe/pybuild/lipe_build.py | 69 +++++++++++++++++++++-------------- lipe/pybuild/lipe_expression_tests.py | 4 +- lipe/pylipe/Makefile.am | 2 +- lipe/pylipe/ldsync.py | 13 +++++-- lipe/pylipe/lipe.py | 11 ++---- lipe/pylipe/lipe_flist_handle.py | 13 +++++-- lipe/pylipe/lipe_install.py | 2 +- lipe/pylipe/lipe_launch.py | 2 +- lipe/pyloris/loris_backup.py | 10 ++++- lipe/pyloris/loris_test.py | 11 ++++-- lipe/pylpcc/lpcc.py | 11 ++++-- lipe/pylpcc/lpcc_cleanup.py | 11 ++++-- lipe/pylpcc/lpcc_test.py | 12 ++++-- lipe/pyltest_import_check | 2 +- lipe/pylustre/cmd_general.py | 11 ++++-- lipe/pylustre/hsm_check.py | 11 ++++-- lipe/pylustre/install_common.py | 6 ++- lipe/src/Makefile.am | 7 ++-- 47 files changed, 215 insertions(+), 143 deletions(-) diff --git a/contrib/lbuild/lbuild b/contrib/lbuild/lbuild index 91780bf..cea3057 100755 --- a/contrib/lbuild/lbuild +++ b/contrib/lbuild/lbuild @@ -881,14 +881,21 @@ build_lipe() { local source="$1" local lustre_install_root local lustre_rpm='' + local lustre_devel_rpm='' local name + # Extract headers and libraries from lustre RPM for build. + lustre_install_root=$(mktemp -d ${TOPDIR}/root.XXXXXXXX) + # Find the lustre rpm for rpm in ${source}/*.rpm; do name=$(rpm --query --queryformat '%{NAME}\n' --package "$rpm") - if [[ "$name" == "lustre-devel" ]]; then + if [[ "$name" == "lustre" ]]; then lustre_rpm="$rpm" - break + fi + + if [[ "$name" == "lustre-devel" ]]; then + lustre_devel_rpm="$rpm" fi done @@ -897,10 +904,14 @@ build_lipe() { return 255 fi - # Extract headers and liblustreapi.so from lustre RPM for build. - lustre_install_root=$(mktemp -d ${TOPDIR}/root.XXXXXXXX) + if [[ "$lustre_devel_rpm" == "" ]]; then + echo "cannot find lustre-devel RPM" >&2 + return 255 + fi + pushd $lustre_install_root > /dev/null rpm2cpio $lustre_rpm | cpio -idv + rpm2cpio $lustre_devel_rpm | cpio -idv popd > /dev/null # since lipe is not included in 'make dist', we have to assume two places diff --git a/lipe/Makefile.am b/lipe/Makefile.am index a9f0a1b..2a56277 100644 --- a/lipe/Makefile.am +++ b/lipe/Makefile.am @@ -96,12 +96,12 @@ PYTHON_CHECKS += $(PYLTEST_CHECKS) CHECKS = $(PYTHON_CHECKS) %.pyltest_import_checked: % - ./pyltest_import_check $< + python2 ./pyltest_import_check $< touch $@ %.python_checked: % pylipe/.pylintrc @if test $< != $(PYTHON_PROTOBUF); then \ - PYLINTRC=pylipe/.pylintrc pylint --disable=I $< || exit 1; \ + PYLINTRC=pylipe/.pylintrc $(PYLINT) --disable=I $< || exit 1; \ fi touch $@ diff --git a/lipe/configure.ac b/lipe/configure.ac index 88a47d5..1e76a0c 100644 --- a/lipe/configure.ac +++ b/lipe/configure.ac @@ -188,15 +188,15 @@ AC_MSG_RESULT([$DISTRO_RELEASE]) AC_SUBST(DISTRO_RELEASE) # ------- check for python -------- -AC_CHECK_PROGS([PYTHON], [python], []) +AC_CHECK_PROGS([PYTHON], [python2.7], []) if test "x$PYTHON" = "x"; then AC_MSG_ERROR([Python is needed for tests. Install python please.]) fi # ------- check for pylint -------- -AC_CHECK_PROGS([PYLINT], [pylint], []) +AC_CHECK_PROGS([PYLINT], [pylint-2.7], []) if test "x$PYLINT" = "x"; then - AC_MSG_ERROR([pylint is needed to check python coding style. Install pylint please.]) + AC_MSG_ERROR([pylint-2.7 is needed to check python coding style. Install pylint please.]) fi # ------- check for libssh -------- diff --git a/lipe/gen_lipe_test b/lipe/gen_lipe_test index f3572a3..f916980 100755 --- a/lipe/gen_lipe_test +++ b/lipe/gen_lipe_test @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2018 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/ldsync b/lipe/ldsync index f67d5d5..8f59824 100755 --- a/lipe/ldsync +++ b/lipe/ldsync @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2018 DataDirect Networks, Inc. # All Rights Reserved. # Author: Gu Zheng diff --git a/lipe/lipe.spec.in b/lipe/lipe.spec.in index a18d53a..c77b125 100644 --- a/lipe/lipe.spec.in +++ b/lipe/lipe.spec.in @@ -18,7 +18,9 @@ Version: @VERSION@ Vendor: DataDirect Networks Inc. Prefix: %{_prefix} -%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%define __python %{_bindir}/python2 + +%{!?python2_sitelib: %global python2_sitelib %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} Release: @LIPE_RELEASE@%{?dist} VCS: @LIPE_REVISION@ @@ -214,15 +216,15 @@ Generated using options: @ac_configure_args@ --includedir=%{_includedir} \ --prefix=%{_prefix} -make +make V=1 -python -m py_compile pyclownfish/*.py -python -m py_compile pylustre/*.py -python -m py_compile pylhsm/*.py -python -m py_compile pylipe/*.py -python -m py_compile pyloris/*.py -python -m py_compile pylpcc/*.py -python -m py_compile pyltest/*.py +python2 -m py_compile pyclownfish/*.py +python2 -m py_compile pylustre/*.py +python2 -m py_compile pylhsm/*.py +python2 -m py_compile pylipe/*.py +python2 -m py_compile pyloris/*.py +python2 -m py_compile pylpcc/*.py +python2 -m py_compile pyltest/*.py find pyclownfish pylustre pylhsm pylipe pyloris pylpcc pyltest -maxdepth 1 -type f -a -name "*.python_checked" -o -name "*.py" | xargs rm -f @@ -231,7 +233,7 @@ rm -rf $RPM_BUILD_ROOT mkdir -p $RPM_BUILD_ROOT%{_sbindir} mkdir -p $RPM_BUILD_ROOT%{_bindir} mkdir -p $RPM_BUILD_ROOT%{_libdir} -mkdir -p $RPM_BUILD_ROOT%{python_sitelib} +mkdir -p $RPM_BUILD_ROOT%{python2_sitelib} mkdir -p $RPM_BUILD_ROOT%{_mandir}/man1 mkdir -p $RPM_BUILD_ROOT%{_mandir}/man5 mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d @@ -277,18 +279,17 @@ mkdir -p $RPM_BUILD_ROOT%{ddntoolsdir}/ install -m 0755 scripts/*.sh $RPM_BUILD_ROOT%{ddntoolsdir}/ %endif -cp -a pyclownfish $RPM_BUILD_ROOT%{python_sitelib} -cp -a pylhsm $RPM_BUILD_ROOT%{python_sitelib} -cp -a pylipe $RPM_BUILD_ROOT%{python_sitelib} -cp -a pyloris $RPM_BUILD_ROOT%{python_sitelib} -cp -a pylpcc $RPM_BUILD_ROOT%{python_sitelib} -cp -a pylustre $RPM_BUILD_ROOT%{python_sitelib} -cp -a pyltest $RPM_BUILD_ROOT%{python_sitelib} +cp -a pyclownfish $RPM_BUILD_ROOT%{python2_sitelib} +cp -a pylhsm $RPM_BUILD_ROOT%{python2_sitelib} +cp -a pylipe $RPM_BUILD_ROOT%{python2_sitelib} +cp -a pyloris $RPM_BUILD_ROOT%{python2_sitelib} +cp -a pylpcc $RPM_BUILD_ROOT%{python2_sitelib} +cp -a pylustre $RPM_BUILD_ROOT%{python2_sitelib} +cp -a pyltest $RPM_BUILD_ROOT%{python2_sitelib} mkdir -p $RPM_BUILD_ROOT%{_sysconfdir} cp -a \ example_configs/clownfish/seperate_mgs/clownfish.conf \ example_configs/lipe/lipe_install.conf \ - example_configs/lipe/lipe_test.conf \ example_configs/lipe/lipe_launch.json \ example_configs/loris/loris.conf \ example_configs/ltest/lipe_test_scheduler.conf \ @@ -343,24 +344,24 @@ cp -a lhsm_tests $RPM_BUILD_ROOT%{_libdir}/ rm -rf $RPM_BUILD_ROOT %files clownfish -%{python_sitelib}/pyclownfish +%{python2_sitelib}/pyclownfish %{_bindir}/lcreatemany %config(noreplace) %{_sysconfdir}/clownfish.conf %files loris -%{python_sitelib}/pyloris +%{python2_sitelib}/pyloris %{_bindir}/loris_backup %{_bindir}/loris_crontab %{_bindir}/loris_test %config(noreplace) %{_sysconfdir}/loris.conf %files pylustre -%{python_sitelib}/pylustre +%{python2_sitelib}/pylustre %{_bindir}/lipe_virt %config(noreplace) %{_sysconfdir}/lipe_virt.conf %files pyltest -%{python_sitelib}/pyltest +%{python2_sitelib}/pyltest %{_bindir}/lipe_test_console %{_bindir}/lipe_test_launch %{_bindir}/lipe_test_scheduler @@ -373,7 +374,7 @@ rm -rf $RPM_BUILD_ROOT %files lpcc %defattr(-,root,root) -%{python_sitelib}/pylpcc +%{python2_sitelib}/pylpcc %{_bindir}/lpcc %{_bindir}/lpcc_cleanup %{_bindir}/lpcc_test @@ -386,7 +387,7 @@ rm -rf $RPM_BUILD_ROOT %files hsm %defattr(-,root,root) -%{python_sitelib}/pylhsm +%{python2_sitelib}/pylhsm %{_bindir}/lipe_hsm_remover %{_bindir}/lipe_hsm_check %{_bindir}/lipe_hsm_copytool @@ -436,9 +437,8 @@ rm -rf $RPM_BUILD_ROOT %{_bindir}/lfill %{_bindir}/lipe_scan %{_bindir}/lipe_find -%{python_sitelib}/pylipe +%{python2_sitelib}/pylipe %config(noreplace) %{_sysconfdir}/lipe_install.conf -%config(noreplace) %{_sysconfdir}/lipe_test.conf %config(noreplace) %{_sysconfdir}/lipe_launch.json %{_mandir}/man1/lipe_scan.1* %{_mandir}/man1/lipe_find.1* diff --git a/lipe/lipe_build b/lipe/lipe_build index 9c6aae7..0a21484 100755 --- a/lipe/lipe_build +++ b/lipe/lipe_build @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_copytool_manager b/lipe/lipe_copytool_manager index 69a40df..c64cb26 100755 --- a/lipe/lipe_copytool_manager +++ b/lipe/lipe_copytool_manager @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_expression_tests b/lipe/lipe_expression_tests index 3d09119..1116023 100755 --- a/lipe/lipe_expression_tests +++ b/lipe/lipe_expression_tests @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_find b/lipe/lipe_find index 41fb867..a3aed1d 100755 --- a/lipe/lipe_find +++ b/lipe/lipe_find @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2018 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_hsm_check b/lipe/lipe_hsm_check index e5441f1..b16a27a 100755 --- a/lipe/lipe_hsm_check +++ b/lipe/lipe_hsm_check @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_install b/lipe/lipe_install index 9f02b5c..2423b41 100755 --- a/lipe/lipe_install +++ b/lipe/lipe_install @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2018 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_install_build_deps b/lipe/lipe_install_build_deps index 64ca2ff..9e8b20e 100755 --- a/lipe/lipe_install_build_deps +++ b/lipe/lipe_install_build_deps @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u """ Install LiPE build dependent rpms and python libs """ diff --git a/lipe/lipe_launch b/lipe/lipe_launch index d6343d2..0d809f2 100755 --- a/lipe/lipe_launch +++ b/lipe/lipe_launch @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_run_action b/lipe/lipe_run_action index e574b25..a06a4a8 100755 --- a/lipe/lipe_run_action +++ b/lipe/lipe_run_action @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_test b/lipe/lipe_test index d8dc63e..671f4cd 100755 --- a/lipe/lipe_test +++ b/lipe/lipe_test @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2018 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_test_console b/lipe/lipe_test_console index 0335a45..560f917 100755 --- a/lipe/lipe_test_console +++ b/lipe/lipe_test_console @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2018 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_test_copytool b/lipe/lipe_test_copytool index f6290cb..3e73bdc 100755 --- a/lipe/lipe_test_copytool +++ b/lipe/lipe_test_copytool @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_test_launch b/lipe/lipe_test_launch index a2df41c..3cffd5c 100755 --- a/lipe/lipe_test_launch +++ b/lipe/lipe_test_launch @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2018 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_test_scheduler b/lipe/lipe_test_scheduler index 265fdc8..d83054c 100755 --- a/lipe/lipe_test_scheduler +++ b/lipe/lipe_test_scheduler @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2018 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lipe_virt b/lipe/lipe_virt index 88da567..2dce6da 100755 --- a/lipe/lipe_virt +++ b/lipe/lipe_virt @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2018 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/loris_backup b/lipe/loris_backup index b86d915..45c849e 100755 --- a/lipe/loris_backup +++ b/lipe/loris_backup @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/loris_crontab b/lipe/loris_crontab index 9e31a8b..f2e7123 100755 --- a/lipe/loris_crontab +++ b/lipe/loris_crontab @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/loris_test b/lipe/loris_test index 12fa4dd..730b763 100755 --- a/lipe/loris_test +++ b/lipe/loris_test @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lpcc b/lipe/lpcc index 0f3f4af..73922cc 100755 --- a/lipe/lpcc +++ b/lipe/lpcc @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lpcc_cleanup b/lipe/lpcc_cleanup index 8b244eb..c283a7a 100755 --- a/lipe/lpcc_cleanup +++ b/lipe/lpcc_cleanup @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/lpcc_test b/lipe/lpcc_test index 5c2b395..34e34a9 100755 --- a/lipe/lpcc_test +++ b/lipe/lpcc_test @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2017 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/pybuild/Makefile.am b/lipe/pybuild/Makefile.am index 6e698a1..51eccf6 100644 --- a/lipe/pybuild/Makefile.am +++ b/lipe/pybuild/Makefile.am @@ -2,13 +2,6 @@ lipe_expression_tests.checked: ../src/lipe_expression_test \ lipe_expression_tests.py ../lipe_expression_tests lipe_expression_tests.checked -../example_configs/lipe/lipe_test.conf: \ - ../gen_lipe_test gen_lipe_test.py \ - ../pylipe/lipe_test.py ../pylipe/lipe_common.py \ - ../pyclownfish/clownfish_common.py \ - ../pylustre/cstr.py ../pylustre/lipe_virt.py - ../gen_lipe_test ../example_configs/lipe/lipe_test.conf - ../pylipe/lipe_constant.py: ../src/generate_definition ../src/generate_definition ../pylipe/lipe_constant.py @@ -17,7 +10,7 @@ PYTHON_CHECKS = $(PYTHON_FILES:%=%.python_checked) CHECKS = $(PYTHON_CHECKS) %.python_checked: % ../.pylintrc - PYLINTRC=../.pylintrc pylint --disable=I $< + PYLINTRC=../.pylintrc $(PYLINT) --disable=I $< touch $@ check_clean-local: @@ -26,4 +19,4 @@ check_clean-local: check-local: $(CHECKS) all: all-am ../pylipe/lipe_constant.py \ - ../example_configs/lipe/lipe_test.conf $(CHECKS) + $(CHECKS) diff --git a/lipe/pybuild/gen_lipe_test.py b/lipe/pybuild/gen_lipe_test.py index 900dc09..ad4d9f0 100644 --- a/lipe/pybuild/gen_lipe_test.py +++ b/lipe/pybuild/gen_lipe_test.py @@ -5,6 +5,7 @@ """ Generate lipe_test.conf """ +import os import sys import yaml @@ -29,10 +30,15 @@ def main(): """ Generate lipe_test.conf """ - # pylint: disable=bare-except + # pylint: disable=bare-except,not-callable log = clog.get_log() - reload(sys) - sys.setdefaultencoding("utf-8") + if sys.version[0] == '2': + reload(sys) + if hasattr(sys, "setdefaultencoding"): + set_encoding = getattr(sys, "setdefaultencoding", None) + set_encoding('UTF-8') + else: + os.environ["PYTHONIOENCODING"] = 'UTF-8' if len(sys.argv) != 2: usage() diff --git a/lipe/pybuild/lipe_build.py b/lipe/pybuild/lipe_build.py index 47a9828..1bb41cf 100644 --- a/lipe/pybuild/lipe_build.py +++ b/lipe/pybuild/lipe_build.py @@ -44,12 +44,26 @@ STRING_MPIFILEUTILS_DEFAULT_BRANCH = "b_ddn0_0" SOURCE_DIR = None LIPE_BUILD_DEPENDENT_RPMS = [ - "automake", "libtool", "byacc", "flex", "json-c-devel", - "libattr-devel", "libtool-ltdl-devel", "libssh-devel", "libyaml-devel", - "openssl-devel", "pylint", "python-dateutil", - "python2-pip", "PyYAML", "python-prettytable"] - -LIPE_BUILD_DEPENDENT_PIP_LIBS = ["filelock", "python-crontab"] + "PyYAML", + "automake", + "byacc", + "flex", + "json-c-devel", + "libattr-devel", + "libssh-devel", + "libtool", + "libtool-ltdl-devel", + "libyaml-devel", + "openssl-devel", + "python-prettytable", + "python2-dateutil", + "python2-pip", + "python2-pylint", + "redhat-lsb-core" +] + +LIPE_BUILD_DEPENDENT_PIP_LIBS = ["filelock", "pyyaml", "pylint==1.9.5", "python-dateutil", + "PrettyTable", "pep8", "python-crontab"] def clone_src_from_git(log, build_dir, git_url, branch, @@ -852,7 +866,7 @@ def lipe_install_build_dependents(log): log.cl_info("installed package [%s]" % build_dependent_rpm) for pip_lib in LIPE_BUILD_DEPENDENT_PIP_LIBS: - command = ("pip install %s" % pip_lib) + command = ("%s install %s" % (install_common.PIP, pip_lib)) retval = utils.run(command) if retval.cr_exit_status: log.cl_error("failed to install python module [%s] via [%s], " @@ -1024,6 +1038,7 @@ def lipe_build_mpifileutils_tools(log, local_host, source_dir, config, if ret: log.cl_error("failed to build mpifileutils") return -1 + return 0 def download_pip(log, local_host, pip_dir): @@ -1055,27 +1070,27 @@ def download_pip(log, local_host, pip_dir): retval.cr_stderr) return -1 - pip_packages = [] - if len(pip_packages) > 0: - command = ("cd %s && pip download" % (pip_dir)) - for pip_package in pip_packages: - command += " " + pip_package - retval = local_host.sh_run(log, command) - if retval.cr_exit_status: - # upgrade pip and try again - retval = local_host.sh_run(log, "pip install --upgrade pip") + pip_packages = ["filelock", "pyyaml", "python-dateutil", + "PrettyTable", "python-crontab"] + command = ("cd %s && %s download" % (pip_dir, install_common.PIP)) + for pip_package in pip_packages: + command += " " + pip_package + retval = local_host.sh_run(log, command) + if retval.cr_exit_status: + # upgrade pip and try again + retval = local_host.sh_run(log, "%s install --upgrade pip" % install_common.PIP) + if retval.cr_exit_status == 0: + retval = local_host.sh_run(log, command) if retval.cr_exit_status == 0: - retval = local_host.sh_run(log, command) - if retval.cr_exit_status == 0: - return 0 - log.cl_error("failed to run command [%s] on host [%s], " - "ret = [%d], stdout = [%s], stderr = [%s]", - command, - local_host.sh_hostname, - retval.cr_exit_status, - retval.cr_stdout, - retval.cr_stderr) - return -1 + return 0 + log.cl_error("failed to run command [%s] on host [%s], " + "ret = [%d], stdout = [%s], stderr = [%s]", + command, + local_host.sh_hostname, + retval.cr_exit_status, + retval.cr_stdout, + retval.cr_stderr) + return -1 return 0 diff --git a/lipe/pybuild/lipe_expression_tests.py b/lipe/pybuild/lipe_expression_tests.py index 82baa14..653e3f6 100755 --- a/lipe/pybuild/lipe_expression_tests.py +++ b/lipe/pybuild/lipe_expression_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # pylint: disable=too-many-lines # Copyright (c) 2019 DataDirect Networks, Inc. # All Rights Reserved. @@ -29,7 +29,7 @@ RETVAL_HELP = 1 RETVAL_ARG_INVALID = 2 # The attribute name is invalid RETVAL_ATTR_NAME_INVALID = 3 -#The attribute value is invalid +# The attribute value is invalid RETVAL_ATTR_VALUE_INVALID = 4 # The expression is invalid RETVAL_EXPRESSION_INVALID = 5 diff --git a/lipe/pylipe/Makefile.am b/lipe/pylipe/Makefile.am index d07a83f..3bb5b3e 100644 --- a/lipe/pylipe/Makefile.am +++ b/lipe/pylipe/Makefile.am @@ -3,7 +3,7 @@ PYTHON_CHECKS = $(PYTHON_FILES:%=%.python_checked) CHECKS = $(PYTHON_CHECKS) %.python_checked: % ../.pylintrc - PYLINTRC=../.pylintrc pylint --disable=I $< + PYLINTRC=../.pylintrc $(PYLINT) --disable=I $< touch $@ check_clean-local: diff --git a/lipe/pylipe/ldsync.py b/lipe/pylipe/ldsync.py index 9341691..9c4720d 100644 --- a/lipe/pylipe/ldsync.py +++ b/lipe/pylipe/ldsync.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2018 DataDirect Networks, Inc. # All Rights Reserved. # Author: Gu Zheng @@ -94,9 +94,14 @@ def main(): """ # pylint: disable=too-many-branches,too-many-statements,unused-variable # pylint: disable=too-many-boolean-expressions,too-many-locals - # pylint: disable=global-statement - reload(sys) - sys.setdefaultencoding("utf-8") + # pylint: disable=global-statement,not-callable + if sys.version[0] == '2': + reload(sys) + if hasattr(sys, "setdefaultencoding"): + set_encoding = getattr(sys, "setdefaultencoding", None) + set_encoding('UTF-8') + else: + os.environ["PYTHONIOENCODING"] = 'UTF-8' global DSYNC global MPIRUN diff --git a/lipe/pylipe/lipe.py b/lipe/pylipe/lipe.py index f3560e1..8bb31a9 100644 --- a/lipe/pylipe/lipe.py +++ b/lipe/pylipe/lipe.py @@ -237,12 +237,10 @@ def flist_join_string(flist, pattern=None, start=None, end=None): if pattern is None: if start is None or end is None: return flist - else: - return "%s_%d_%d" % (flist, start, end) + return "%s_%d_%d" % (flist, start, end) elif start is None or end is None: return "%s_%s" % (flist, pattern) - else: - return "%s_%s_%d_%d" % (flist, pattern, start, end) + return "%s_%s_%d_%d" % (flist, pattern, start, end) def flist_prepare_flist_summary(log, workspace, @@ -531,7 +529,7 @@ def counter_apply_action(log, directory, clients, action, argument, utils.merge_files(fid_paths, fid_summary_file, cleanup=True) - if len(fid_paths) == 0: + if not fid_paths: log.cl_debug("no FID is dumped in counter [%s]", directory) return 0 @@ -626,8 +624,7 @@ def rule_counter_name(action, argument, action_counter_name): elif action in [LAT_ATTR_CLASSIFY, LAT_SHELL_CMD_FPATH, LAT_SHELL_CMD_FID, LAT_SHELL_CMD_FID_LIST]: return action_counter_name - else: - return None + return None def group_apply_action(log, config_group, result_json, flist_type, directory, diff --git a/lipe/pylipe/lipe_flist_handle.py b/lipe/pylipe/lipe_flist_handle.py index cbb7048..9882970 100644 --- a/lipe/pylipe/lipe_flist_handle.py +++ b/lipe/pylipe/lipe_flist_handle.py @@ -373,10 +373,15 @@ def main(): """ Main function """ - # pylint: disable=too-many-branches,too-many-statements,unused-variable - # pylint: disable=too-many-boolean-expressions,too-many-locals - reload(sys) - sys.setdefaultencoding("utf-8") + # pylint: disable=too-many-branches,too-many-statements,unused-variable,not-callable + # pylint: disable=too-many-boolean-expressions,too-many-locals,not-callable + if sys.version[0] == '2': + reload(sys) + if hasattr(sys, "setdefaultencoding"): + set_encoding = getattr(sys, "setdefaultencoding", None) + set_encoding('UTF-8') + else: + os.environ["PYTHONIOENCODING"] = 'UTF-8' try: options, remainder = getopt.getopt(sys.argv[1:], diff --git a/lipe/pylipe/lipe_install.py b/lipe/pylipe/lipe_install.py index fc77499..b3a59d5 100644 --- a/lipe/pylipe/lipe_install.py +++ b/lipe/pylipe/lipe_install.py @@ -39,7 +39,7 @@ def main(): if ret != 0: missing_dependencies.append(dependent_rpm) - if len(missing_dependencies): + if missing_dependencies: sys.stdout.write("installing dependency RPMs of %s" % missing_dependencies) ret = install_common.dependency_install(log, diff --git a/lipe/pylipe/lipe_launch.py b/lipe/pylipe/lipe_launch.py index b1179e9..2554d0a 100644 --- a/lipe/pylipe/lipe_launch.py +++ b/lipe/pylipe/lipe_launch.py @@ -143,7 +143,7 @@ def scan_thread(log, workspace, web_config, config_device, host, else: log.cl_debug("avoided getting file from local host") - if len(clients) == 0: + if not clients: log.cl_error("failed to apply action on device [%s] to because no " "Lustre client is available", device_id) return -1 diff --git a/lipe/pyloris/loris_backup.py b/lipe/pyloris/loris_backup.py index 412a13e..b14c0f3 100755 --- a/lipe/pyloris/loris_backup.py +++ b/lipe/pyloris/loris_backup.py @@ -169,8 +169,14 @@ def main(): """ Check a simple backup/copyback case """ - reload(sys) - sys.setdefaultencoding("utf-8") + # pylint: disable=not-callable + if sys.version[0] == '2': + reload(sys) + if hasattr(sys, "setdefaultencoding"): + set_encoding = getattr(sys, "setdefaultencoding", None) + set_encoding('UTF-8') + else: + os.environ["PYTHONIOENCODING"] = 'UTF-8' if len(sys.argv) == 2: config_file = sys.argv[1] else: diff --git a/lipe/pyloris/loris_test.py b/lipe/pyloris/loris_test.py index de98c44..9ce99c4 100755 --- a/lipe/pyloris/loris_test.py +++ b/lipe/pyloris/loris_test.py @@ -341,12 +341,17 @@ def run_test(log, json_file): def main(): - # pylint: disable=no-member + # pylint: disable=no-member,not-callable """ Check a simple backup/copyback case """ - reload(sys) - sys.setdefaultencoding("utf-8") + if sys.version[0] == '2': + reload(sys) + if hasattr(sys, "setdefaultencoding"): + set_encoding = getattr(sys, "setdefaultencoding", None) + set_encoding('UTF-8') + else: + os.environ["PYTHONIOENCODING"] = 'UTF-8' if len(sys.argv) == 2: json_file = sys.argv[1] diff --git a/lipe/pylpcc/lpcc.py b/lipe/pylpcc/lpcc.py index 48ee2fb..2f146ba 100644 --- a/lipe/pylpcc/lpcc.py +++ b/lipe/pylpcc/lpcc.py @@ -602,9 +602,14 @@ def main(): """ Run LPCC manager """ - # pylint: disable=unused-variable - reload(sys) - sys.setdefaultencoding("utf-8") + # pylint: disable=unused-variable,not-callable + if sys.version[0] == '2': + reload(sys) + if hasattr(sys, "setdefaultencoding"): + set_encoding = getattr(sys, "setdefaultencoding", None) + set_encoding('UTF-8') + else: + os.environ["PYTHONIOENCODING"] = 'UTF-8' config_fpath = LPCC_CONFIG if len(sys.argv) == 2: diff --git a/lipe/pylpcc/lpcc_cleanup.py b/lipe/pylpcc/lpcc_cleanup.py index e90c61c..42c30ef 100644 --- a/lipe/pylpcc/lpcc_cleanup.py +++ b/lipe/pylpcc/lpcc_cleanup.py @@ -60,9 +60,14 @@ def main(): """ Cleanup LPCC """ - # pylint: disable=unused-variable - reload(sys) - sys.setdefaultencoding("utf-8") + # pylint: disable=unused-variable,not-callable + if sys.version[0] == '2': + reload(sys) + if hasattr(sys, "setdefaultencoding"): + set_encoding = getattr(sys, "setdefaultencoding", None) + set_encoding('UTF-8') + else: + os.environ["PYTHONIOENCODING"] = 'UTF-8' options, remainder = getopt.getopt(sys.argv[1:], "h", diff --git a/lipe/pylpcc/lpcc_test.py b/lipe/pylpcc/lpcc_test.py index b98ec3f..754ccda 100644 --- a/lipe/pylpcc/lpcc_test.py +++ b/lipe/pylpcc/lpcc_test.py @@ -972,9 +972,15 @@ def main(): """ Run LPCC tests """ - # pylint: disable=unused-variable - reload(sys) - sys.setdefaultencoding("utf-8") + # pylint: disable=unused-variable,not-callable + if sys.version[0] == '2': + reload(sys) + if hasattr(sys, "setdefaultencoding"): + set_encoding = getattr(sys, "setdefaultencoding", None) + set_encoding('UTF-8') + else: + os.environ["PYTHONIOENCODING"] = 'UTF-8' + config_fpath = lpcc.LPCC_CONFIG if len(sys.argv) == 2: diff --git a/lipe/pyltest_import_check b/lipe/pyltest_import_check index 11b92f6..05edadb 100755 --- a/lipe/pyltest_import_check +++ b/lipe/pyltest_import_check @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/python2 -u # Copyright (c) 2019 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com diff --git a/lipe/pylustre/cmd_general.py b/lipe/pylustre/cmd_general.py index c7ebc47..43a6f44 100644 --- a/lipe/pylustre/cmd_general.py +++ b/lipe/pylustre/cmd_general.py @@ -30,9 +30,14 @@ def main(default_config_fpath, default_log_parent, main_func): """ The main function of a command """ - # pylint: disable=too-many-locals,too-many-branches,too-many-statements - reload(sys) - sys.setdefaultencoding("utf-8") + # pylint: disable=too-many-locals,too-many-branches,too-many-statements,not-callable + if sys.version[0] == '2': + reload(sys) + if hasattr(sys, "setdefaultencoding"): + set_encoding = getattr(sys, "setdefaultencoding", None) + set_encoding('UTF-8') + else: + os.environ["PYTHONIOENCODING"] = 'UTF-8' options, args = getopt.getopt(sys.argv[1:], "c:i:h", diff --git a/lipe/pylustre/hsm_check.py b/lipe/pylustre/hsm_check.py index 48057e2..f932a86 100644 --- a/lipe/pylustre/hsm_check.py +++ b/lipe/pylustre/hsm_check.py @@ -147,9 +147,14 @@ def main(): """ Main function """ - # pylint: disable=unused-variable - reload(sys) - sys.setdefaultencoding("utf-8") + # pylint: disable=unused-variable,not-callable + if sys.version[0] == '2': + reload(sys) + if hasattr(sys, "setdefaultencoding"): + set_encoding = getattr(sys, "setdefaultencoding", None) + set_encoding('UTF-8') + else: + os.environ["PYTHONIOENCODING"] = 'UTF-8' options, remainder = getopt.getopt(sys.argv[1:], "h", diff --git a/lipe/pylustre/install_common.py b/lipe/pylustre/install_common.py index 3112813..891a0d1 100644 --- a/lipe/pylustre/install_common.py +++ b/lipe/pylustre/install_common.py @@ -14,6 +14,8 @@ from pylustre import utils from pylustre import ssh_host from pylustre import cstr +PIP = "pip2" + RPM_PATTERN_RHEL7 = r"^%s-\d.+(\.el7|).*\.(x86_64|noarch)\.rpm$" RPM_PATTERN_RHEL6 = r"^%s-\d.+(\.el6|).*\.(x86_64|noarch)\.rpm$" LIPE_INSTALL_DEPENDENT_RPMS = ["rsync", @@ -336,8 +338,8 @@ class InstallationCluster(object): for pip_lib in pip_libs: log.cl_info("installing pip lib [%s] on host [%s]", pip_lib, hostname) - command = ("pip install --no-index --find-links %s %s" % - (self.ic_pip_dir, pip_lib)) + command = ("%s install --no-index --find-links %s %s" % + (PIP, self.ic_pip_dir, pip_lib)) retval = host.sh_run(log, command) if retval.cr_exit_status: log.cl_error("failed to run command [%s] on host [%s], " diff --git a/lipe/src/Makefile.am b/lipe/src/Makefile.am index 792c8ff..9ce6b9a 100644 --- a/lipe/src/Makefile.am +++ b/lipe/src/Makefile.am @@ -2,8 +2,8 @@ AUTOMAKE_OPTIONS = -Wall foreign ACLOCAL_AMFLAGS = ${ALOCAL_FLAGS} AM_CPPFLAGS = -D_GNU_SOURCE -include config.h -AM_CFLAGS = -Wall -Werror -g $(json_c_CFLAGS) $(json_c_LIBS) \ - -llustreapi -lpthread -Wno-error=deprecated-declarations +AM_CFLAGS = -g -Wall -Werror -Wno-deprecated-declarations $(json_c_CFLAGS) +AM_LDFLAGS = -llnetconfig -llustreapi -lpthread $(json_c_LIBS) bin_PROGRAMS = lipe_scan lfill ldumpstripe lipe_hsm_remover \ lipe_hsm_copytool ext4_inode2path lcreatemany @@ -34,7 +34,7 @@ LIPE_SOURCES = cmd.c cmd.h debug.c debug.h fake_lustre_disk.h \ lustre_ea_ldiskfs.c lustre_ea_ldiskfs.h \ posix.c posix.h posix_ea.c posix_ea.h LIPE_CFLAGS = -Wall -Werror -g -I/usr/include $(json_c_CFLAGS) -LIPE_LDFLAGS = -llustreapi -pthread $(json_c_LIBS) +LIPE_LDFLAGS = $(AM_LDFLAGS) if ZFS LIPE_SOURCES += lipe_zfs.h lipe_zfs.c zfs_read_ldd.c zfs_read_ldd.h @@ -80,6 +80,7 @@ lipe_expression_test_LDFLAGS = $(LIPE_LDFLAGS) generate_definition_SOURCES = generate_definition.c debug.c debug.h \ general_policy.h generate_definition_CFLAGS = -Wall -Werror -Wall -Werror -g +generate_definition_LDFLAGS = C_FILES = $(wildcard *.c *.h) C_CHECKS = $(C_FILES:%=%.c_checked) -- 1.8.3.1