From 075df01737d5980f52fbcef5ff7f04caaaf8e877 Mon Sep 17 00:00:00 2001 From: Alex Deiter Date: Fri, 5 Aug 2022 16:35:39 +0000 Subject: [PATCH] EX-5644 utils: fix lpcc service dependency on lipe-lpcc This patch makes changes to support python3 for lipe: * The long() function is no longer supported by Python 3. It only has one built-in integral type, named int(). * Octal literals are no longer of the form 0720 - use 0o720 instead. * The print statement has been replaced with a print() function, with keyword arguments to replace most of the special syntax of the old print statement (PEP 3105). * The dict.iterkeys(), dict.iteritems() and dict.itervalues() methods are no longer supported - use dict.items() instead. * The StringIO and StringIO modules are gone. Instead, import the io module and use io.StringIO or io.BytesIO for text and data respectively. * The builtin basestring abstract type was removed. Use str instead. * Use string.ascii_lowercase instead of string.lowercase. Test-Parameters: trivial testlist=sanity-lipe env=SANITY_EXCEPT="101j 130 244a" Test-Parameters: clientdistro=el7.9 testlist=sanity-pcc Test-Parameters: clientdistro=el8.6 testlist=sanity-pcc Change-Id: Ia5a3b6490fd4cebbd40327d5b2a431590c82cf00 Signed-off-by: Alex Deiter Reviewed-on: https://review.whamcloud.com/48149 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Jian Yu Reviewed-by: Feng, Lei Reviewed-by: Andreas Dilger --- lipe/configure.ac | 36 ++++++++++++----------- lipe/detect-distro.sh | 2 +- lipe/ldsync | 4 +-- lipe/lipe.spec.in | 55 ++++++++++++++++++++++++----------- lipe/lipe_convert_expr | 4 +-- lipe/lipe_expression_tests | 4 +-- lipe/lipe_find | 4 +-- lipe/lipe_launch | 4 +-- lipe/lipe_run_action | 4 +-- lipe/loris_backup | 4 +-- lipe/loris_crontab | 4 +-- lipe/loris_test | 4 +-- lipe/lpcc | 2 +- lipe/pybuild/lipe_expression_tests.py | 34 +++++++++++----------- lipe/pylipe/ldsync.py | 11 ++----- lipe/pylipe/lipe.py | 16 +++++----- lipe/pylipe/lipe_convert_expr.py | 7 ++--- lipe/pylipe/lipe_find.py | 11 ++++--- lipe/pylipe/lipe_flist_handle.py | 21 +++++-------- lipe/pyloris/lbackup.py | 3 +- lipe/pyloris/loris_backup.py | 12 ++------ lipe/pyloris/loris_test.py | 10 +------ lipe/pylustre/clog.py | 4 +-- lipe/pylustre/cmd_general.py | 10 +------ lipe/pylustre/cstr.py | 3 +- lipe/pylustre/lustre.py | 21 ++++++------- lipe/pylustre/ssh_host.py | 16 +++++----- lipe/pylustre/utils.py | 24 ++++++++------- lipe/pylustre/watched_io.py | 12 ++------ 29 files changed, 162 insertions(+), 184 deletions(-) diff --git a/lipe/configure.ac b/lipe/configure.ac index 302cb29..1ef9b9d 100644 --- a/lipe/configure.ac +++ b/lipe/configure.ac @@ -185,17 +185,28 @@ DISTRO_RELEASE=$(echo $DISTRO | awk -F 'rhel-' '{print $2}' | awk -F '.' '{print AC_MSG_RESULT([$DISTRO_RELEASE]) AC_SUBST(DISTRO_RELEASE) +# -------- check whether enable server build -------- +AC_MSG_CHECKING([whether enable server]) +AC_ARG_ENABLE([server], + AS_HELP_STRING([--disable-server], + [disable server]), + [enable_server="no"], [enable_server="yes"]) +AC_MSG_RESULT([$enable_server]) +AM_CONDITIONAL([BUILD_SERVER], [test x$enable_server = xyes]) + # ------- check for python -------- -AC_CHECK_PROGS([PYTHON], [python2.7], []) -if test "x$PYTHON" = "x"; then - AC_MSG_ERROR([Python is needed for tests. Install python please.]) -fi +AM_COND_IF([BUILD_SERVER], + [AC_CHECK_PROGS([PYTHON], [python3.6], []) + if test "x$PYTHON" = "x"; then + AC_MSG_ERROR([Python is needed for tests. Install python please.]) + fi]) # ------- check for pylint -------- -AC_CHECK_PROGS([PYLINT], [pylint-2.7], []) -if test "x$PYLINT" = "x"; then - AC_MSG_ERROR([pylint-2.7 is needed to check python coding style. Install pylint please.]) -fi +AM_COND_IF([BUILD_SERVER], + [AC_CHECK_PROGS([PYLINT], [pylint-3.6], []) + if test "x$PYLINT" = "x"; then + AC_MSG_ERROR([pylint-3.6 is needed to check python coding style. Install pylint please.]) + fi]) # ------- check for libssh -------- AC_CHECK_LIB([ssh], [ssh_new], [ @@ -212,15 +223,6 @@ PKG_CHECK_MODULES([libssh_threads], [libssh_threads], ) AM_CONDITIONAL([HAVE_SSH_THREADS], [test x$have_lssh_threads = xyes]) -# -------- check whether enable server build -------- -AC_MSG_CHECKING([whether enable server]) -AC_ARG_ENABLE([server], - AS_HELP_STRING([--disable-server], - [disable server]), - [enable_server="no"], [enable_server="yes"]) -AC_MSG_RESULT([$enable_server]) -AM_CONDITIONAL([BUILD_SERVER], [test x$enable_server = xyes]) - AM_COND_IF([BUILD_SERVER], [AC_DEFINE(HAVE_LDISKFS, 1, [enable ldiskfs support]) PKG_CHECK_MODULES([GUILE], [guile-2.0])]) diff --git a/lipe/detect-distro.sh b/lipe/detect-distro.sh index c2c64dd..beb1795 100755 --- a/lipe/detect-distro.sh +++ b/lipe/detect-distro.sh @@ -48,7 +48,7 @@ if which lsb_release >/dev/null 2>&1; then elif [ -f /etc/redhat-release ]; then #name=$(head -1 /etc/redhat-release) name=rhel - version=$(echo "$distroname" | + version=$(cat /etc/redhat-release | sed -e 's/^[^0-9.]*//g' | sed -e 's/[ ].*//') fi if [ -z "$name" -o -z "$version" ]; then diff --git a/lipe/ldsync b/lipe/ldsync index 8f59824..900f8b0 100755 --- a/lipe/ldsync +++ b/lipe/ldsync @@ -1,5 +1,5 @@ -#!/usr/bin/python2 -u -# Copyright (c) 2018 DataDirect Networks, Inc. +#!/usr/bin/python3 -u +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: Gu Zheng """ diff --git a/lipe/lipe.spec.in b/lipe/lipe.spec.in index 00a6dff..b27b566 100644 --- a/lipe/lipe.spec.in +++ b/lipe/lipe.spec.in @@ -25,8 +25,15 @@ Prefix: %{_prefix} %define lipe_site_dir %{guile_site_dir}/lipe %define lipe_site_ccache_dir %{guile_site_ccache_dir}/lipe -%{!?__python2: %global __python2 /usr/bin/python2} -%{!?python2_sitelib: %global python2_sitelib %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%{!?__python3: %global __python3 /usr/bin/python3} +%{!?python3_sitelib: %global python3_sitelib %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} + +# EPEL has different package naming convention for RHEL7 +%if 0%{?rhel} == 7 +%define python_suffix %(%{__python3} -c "import sys; sys.stdout.write(''.join(sys.version.split()[0].split('.')[0:2]))") +%else +%define python_suffix %(%{__python3} -c "import sys; sys.stdout.write(sys.version.split()[0].split('.')[0])") +%endif Release: @LIPE_RELEASE@%{?dist} VCS: @LIPE_REVISION@ @@ -36,7 +43,10 @@ Group: Applications/System Source0: @PACKAGE@-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) Requires: lipe-pylustre = %{version}-%{release} -Requires: rsync json-c libyaml python2-pyyaml python2-filelock python2-dateutil +Requires: rsync json-c libyaml +Requires: python%{python_suffix}-PyYAML +Requires: python%{python_suffix}-filelock +Requires: python%{python_suffix}-dateutil Requires: e2fsprogs >= 1.42.13.wc6 Provides: lipe = %{version}-%{release} %if %{with systemd} @@ -45,6 +55,8 @@ Requires(preun): systemd Requires(postun): systemd BuildRequires: systemd %endif +BuildRequires: python3-rpm-macros +BuildRequires: json-c-devel libyaml-devel %description LIPE(Lustre Integrated Policy Engine) is a policy engine which enables users @@ -61,7 +73,8 @@ Pylustre is a python library for managing Lustre file system. %package lpcc Summary: LPCC (Lustre Persisted Client Cache) -Requires: lipe-pylustre = %{version}-%{release} python2-pyyaml +Requires: lipe-pylustre = %{version}-%{release} +Requires: python%{python_suffix}-PyYAML Provides: lipe-lpcc = %{version}-%{release} Group: Applications/System @@ -94,7 +107,9 @@ Summary: Lustre Online Reliability Improvement System Requires: lipe-pylustre = %{version}-%{release} Provides: lipe-loris = %{version}-%{release} # python-crontab is generated by EXAScaler. -Requires: e2fsprogs >= 1.42.13.wc6, rsync, python2-filelock, openssh-clients, python-crontab +Requires: e2fsprogs >= 1.42.13.wc6, rsync, openssh-clients +Requires: python%{python_suffix}-filelock +Requires: python%{python_suffix}-crontab Group: Applications/System %description loris @@ -164,21 +179,19 @@ rm -rf "%{lipe_site_ccache_dir}" make V=1 -python2 -m py_compile pylustre/*.py +%__python3 -m compileall -f -b pylustre/*.py %if %{with server} -python2 -m py_compile pylipe/*.py -python2 -m py_compile pyloris/*.py +%__python3 -m compileall -f -b pylipe/*.py +%__python3 -m compileall -f -b pyloris/*.py %endif -find pylustre pylipe pyloris -maxdepth 1 -type f -a -name "*.python_checked" -o -name "*.py" | xargs rm -f - %install rm -rf $RPM_BUILD_ROOT mkdir -p $RPM_BUILD_ROOT%{_sbindir} mkdir -p $RPM_BUILD_ROOT%{_bindir} mkdir -p $RPM_BUILD_ROOT%{_libdir} mkdir -p $RPM_BUILD_ROOT%{guile_site_dir} -mkdir -p $RPM_BUILD_ROOT%{python2_sitelib} +mkdir -p $RPM_BUILD_ROOT%{python3_sitelib} mkdir -p $RPM_BUILD_ROOT%{_mandir}/man1 mkdir -p $RPM_BUILD_ROOT%{_mandir}/man5 mkdir -p $RPM_BUILD_ROOT%{_mandir}/man8 @@ -188,7 +201,9 @@ cp \ src/lpcc_purge \ $RPM_BUILD_ROOT%{_bindir} -cp -a pylustre $RPM_BUILD_ROOT%{python2_sitelib} +mkdir -p $RPM_BUILD_ROOT%{python3_sitelib}/pylustre +cp -a pylustre/*.pyc $RPM_BUILD_ROOT%{python3_sitelib}/pylustre + cp -a \ lpcc.conf \ $RPM_BUILD_ROOT%{_sysconfdir} @@ -233,8 +248,12 @@ mkdir -p $RPM_BUILD_ROOT%{ddntoolsdir}/ install -m 0755 scripts/*.sh $RPM_BUILD_ROOT%{ddntoolsdir}/ %endif # hotpool -cp -a pylipe $RPM_BUILD_ROOT%{python2_sitelib} -cp -a pyloris $RPM_BUILD_ROOT%{python2_sitelib} +mkdir -p $RPM_BUILD_ROOT%{python3_sitelib}/pylipe +cp -a pylipe/*.pyc $RPM_BUILD_ROOT%{python3_sitelib}/pylipe + +mkdir -p $RPM_BUILD_ROOT%{python3_sitelib}/pyloris +cp -a pyloris/*.pyc $RPM_BUILD_ROOT%{python3_sitelib}/pyloris + mkdir -p $RPM_BUILD_ROOT%{_sysconfdir} cp -a \ example_configs/lipe/lipe_launch.json \ @@ -290,7 +309,7 @@ install -m 0644 man/laudit.conf.5 $RPM_BUILD_ROOT%{_mandir}/man5/ rm -rf $RPM_BUILD_ROOT %files pylustre -%{python2_sitelib}/pylustre +%{python3_sitelib}/pylustre %files lpcc %defattr(-,root,root) @@ -310,7 +329,7 @@ rm -rf $RPM_BUILD_ROOT %if %{with server} %files loris -%{python2_sitelib}/pyloris +%{python3_sitelib}/pyloris %{_bindir}/loris_backup %{_bindir}/loris_crontab %{_bindir}/loris_test @@ -363,7 +382,7 @@ rm -rf $RPM_BUILD_ROOT %{_bindir}/lipe_purge %{_bindir}/lipe_scan %{_bindir}/lipe_scan2 -%{python2_sitelib}/pylipe +%{python3_sitelib}/pylipe %config(noreplace) %{_sysconfdir}/lipe_launch.json %{_mandir}/man1/lipe_scan.1* %{_mandir}/man1/lipe_find.1* @@ -371,6 +390,8 @@ rm -rf $RPM_BUILD_ROOT %endif # server %changelog +* Thu Jul 07 2022 Minh Diep [1.5] +- Use Python3 * Tue Jun 30 2020 Gu Zheng [1.5] - Update version to 1.5 * Wed Apr 03 2019 Gu Zheng [1.4] diff --git a/lipe/lipe_convert_expr b/lipe/lipe_convert_expr index 1036f45..04e29fd 100755 --- a/lipe/lipe_convert_expr +++ b/lipe/lipe_convert_expr @@ -1,5 +1,5 @@ -#!/usr/bin/python2 -u -# Copyright (c) 2021 DataDirect Networks, Inc. +#!/usr/bin/python3 -u +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: Jian Yu """ diff --git a/lipe/lipe_expression_tests b/lipe/lipe_expression_tests index 1116023..9a15896 100755 --- a/lipe/lipe_expression_tests +++ b/lipe/lipe_expression_tests @@ -1,5 +1,5 @@ -#!/usr/bin/python2 -u -# Copyright (c) 2017 DataDirect Networks, Inc. +#!/usr/bin/python3 -u +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ diff --git a/lipe/lipe_find b/lipe/lipe_find index a3aed1d..0a3805c 100755 --- a/lipe/lipe_find +++ b/lipe/lipe_find @@ -1,5 +1,5 @@ -#!/usr/bin/python2 -u -# Copyright (c) 2018 DataDirect Networks, Inc. +#!/usr/bin/python3 -u +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ diff --git a/lipe/lipe_launch b/lipe/lipe_launch index 0d809f2..a116dc7 100755 --- a/lipe/lipe_launch +++ b/lipe/lipe_launch @@ -1,5 +1,5 @@ -#!/usr/bin/python2 -u -# Copyright (c) 2017 DataDirect Networks, Inc. +#!/usr/bin/python3 -u +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ diff --git a/lipe/lipe_run_action b/lipe/lipe_run_action index a06a4a8..741bd0b 100755 --- a/lipe/lipe_run_action +++ b/lipe/lipe_run_action @@ -1,5 +1,5 @@ -#!/usr/bin/python2 -u -# Copyright (c) 2017 DataDirect Networks, Inc. +#!/usr/bin/python3 -u +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ diff --git a/lipe/loris_backup b/lipe/loris_backup index 45c849e..c44d5ab 100755 --- a/lipe/loris_backup +++ b/lipe/loris_backup @@ -1,5 +1,5 @@ -#!/usr/bin/python2 -u -# Copyright (c) 2017 DataDirect Networks, Inc. +#!/usr/bin/python3 -u +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ diff --git a/lipe/loris_crontab b/lipe/loris_crontab index f2e7123..303a282 100755 --- a/lipe/loris_crontab +++ b/lipe/loris_crontab @@ -1,5 +1,5 @@ -#!/usr/bin/python2 -u -# Copyright (c) 2017 DataDirect Networks, Inc. +#!/usr/bin/python3 -u +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ diff --git a/lipe/loris_test b/lipe/loris_test index 730b763..68c9094 100755 --- a/lipe/loris_test +++ b/lipe/loris_test @@ -1,5 +1,5 @@ -#!/usr/bin/python2 -u -# Copyright (c) 2017 DataDirect Networks, Inc. +#!/usr/bin/python3 -u +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ diff --git a/lipe/lpcc b/lipe/lpcc index a706d41..4917ec2 100755 --- a/lipe/lpcc +++ b/lipe/lpcc @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/python3 # Copyright (c) 2021 DataDirect Networks, Inc. # All Rights Reserved. # Author: flei@ddn.com diff --git a/lipe/pybuild/lipe_expression_tests.py b/lipe/pybuild/lipe_expression_tests.py index 6538eb4..be6e7a0 100755 --- a/lipe/pybuild/lipe_expression_tests.py +++ b/lipe/pybuild/lipe_expression_tests.py @@ -1,6 +1,6 @@ -#!/usr/bin/python2 -u +#!/usr/bin/python3 -u # pylint: disable=too-many-lines -# Copyright (c) 2019 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ @@ -166,7 +166,7 @@ def test_size_expressions(attribute="size", has_unit=True, bits=64): unit_dict["T"] = 1048576 * 1048576 unit_dict["P"] = 1048576 * 1048576 * 1024 unit_dict["E"] = 1048576 * 1048576 * 1048576 - for unit, size in unit_dict.iteritems(): + for unit, size in unit_dict.items(): find_size = (lipe_find.LipeFindSize.LFS_UNIT_EXPRESSION_INFIX % (attribute, 1, unit, attribute, 1, unit)) find_size_prefix = (lipe_find.LipeFindSize.LFS_UNIT_EXPRESSION_PREFIX % @@ -1160,7 +1160,7 @@ def main_with_exception(): for attr in lipe_constant.LIPE_POLICY_ATTRIBUTES: if attr == "type": # type bits are filtered by 00170000 - expected_value = 00170000 & value + expected_value = 0o00170000 & value elif attr == "stripe_count": # stripe count is 16 bit value = 0xFFFF & value @@ -1186,17 +1186,17 @@ def main_with_exception(): # S_ISUID 0004000 # S_ISGID 0002000 # S_ISVTX 0001000 - test_expression("S_IFMT", 00170000) - test_expression("S_IFSOCK", 0140000) - test_expression("S_IFLNK", 0120000) - test_expression("S_IFREG", 0100000) - test_expression("S_IFBLK", 0060000) - test_expression("S_IFDIR", 0040000) - test_expression("S_IFCHR", 0020000) - test_expression("S_IFIFO", 0010000) - test_expression("S_ISUID", 0004000) - test_expression("S_ISGID", 0002000) - test_expression("S_ISVTX", 0001000) + test_expression("S_IFMT", 0o00170000) + test_expression("S_IFSOCK", 0o0140000) + test_expression("S_IFLNK", 0o0120000) + test_expression("S_IFREG", 0o0100000) + test_expression("S_IFBLK", 0o0060000) + test_expression("S_IFDIR", 0o0040000) + test_expression("S_IFCHR", 0o0020000) + test_expression("S_IFIFO", 0o0010000) + test_expression("S_ISUID", 0o0004000) + test_expression("S_ISGID", 0o0002000) + test_expression("S_ISVTX", 0o0001000) # Attribute tests test_expression("1", options="--atime", expect_retval=RETVAL_ARG_INVALID) @@ -1210,8 +1210,8 @@ def main_with_exception(): # S_IFSOCK 0140000 # S_IFLNK 0120000 # "type" won't overwrite other bits of "mode", only the limited bits - test_expression("mode", 0140740, options="--mode 0120740 --type 0140000") - test_expression("mode", 0140740, options="--mode 0110740 --type 0140036") + test_expression("mode", 0o0140740, options="--mode 0o120740 --type 0o140000") + test_expression("mode", 0o0140740, options="--mode 0o110740 --type 0o140036") test_expression("sys_time", 1550310180000, options="--sys_time 1550310180000") for xtime in ["atime", "ctime", "mtime"]: diff --git a/lipe/pylipe/ldsync.py b/lipe/pylipe/ldsync.py index 9c4720d..fd24ce6 100644 --- a/lipe/pylipe/ldsync.py +++ b/lipe/pylipe/ldsync.py @@ -1,5 +1,5 @@ -#!/usr/bin/python2 -u -# Copyright (c) 2018 DataDirect Networks, Inc. +#!/usr/bin/python3 -u +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: Gu Zheng """ @@ -95,13 +95,6 @@ def main(): # pylint: disable=too-many-branches,too-many-statements,unused-variable # pylint: disable=too-many-boolean-expressions,too-many-locals # pylint: disable=global-statement,not-callable - if sys.version[0] == '2': - reload(sys) - if hasattr(sys, "setdefaultencoding"): - set_encoding = getattr(sys, "setdefaultencoding", None) - set_encoding('UTF-8') - else: - os.environ["PYTHONIOENCODING"] = 'UTF-8' global DSYNC global MPIRUN diff --git a/lipe/pylipe/lipe.py b/lipe/pylipe/lipe.py index 8bb31a9..5f7d65e 100644 --- a/lipe/pylipe/lipe.py +++ b/lipe/pylipe/lipe.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com @@ -139,8 +139,8 @@ class FlistActionContext(object): action, action_batch, arguments, action_flags=0): self.fac_fid_summary_file = fid_summary_file self.fac_output_files = output_files - self.fac_start_index = -1L - self.fac_end_index = -1L + self.fac_start_index = -1 + self.fac_end_index = -1 self.fac_context_empty = False self.fac_mutex = Lock() self.fac_action = action @@ -152,7 +152,7 @@ class FlistActionContext(object): """ Count lines of fid flist file, and init start&end index """ - count = 0L + count = 0 with open(self.fac_fid_summary_file, 'rb') as fidfile: while True: buff = fidfile.read(8 * 1024 * 1024) @@ -160,7 +160,7 @@ class FlistActionContext(object): break count += buff.count('\n') self.fac_mutex.acquire() - self.fac_start_index = 0L + self.fac_start_index = 0 self.fac_end_index = count self.fac_mutex.release() @@ -168,8 +168,8 @@ class FlistActionContext(object): """ Get valid range(line index) """ - start = -1L - end = -1L + start = -1 + end = -1 self.fac_mutex.acquire() if self.fac_context_empty: @@ -672,7 +672,7 @@ def group_apply_action(log, config_group, result_json, flist_type, directory, log.cl_debug("counter_dicts: %s, classify_names: %s", counter_dicts, classify_names) - for counter_name, (action, argument) in counter_dicts.iteritems(): + for counter_name, (action, argument) in counter_dicts.items(): fpath = directory + "/" + counter_name if not os.path.isdir(fpath): log.cl_error("path [%s] is not a directory for file list", fpath) diff --git a/lipe/pylipe/lipe_convert_expr.py b/lipe/pylipe/lipe_convert_expr.py index dd3229a..58b366d 100644 --- a/lipe/pylipe/lipe_convert_expr.py +++ b/lipe/pylipe/lipe_convert_expr.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com # Author: Jian Yu @@ -1570,7 +1570,7 @@ def perm_option_parse(log, arg, negative): mode_string) return None - if mode > 07777: + if mode > 0o7777: log.cl_error("argument [%s] of option [-perm] is larger than 07777", mode_string) return None @@ -1895,9 +1895,6 @@ def main(): """ Convert expression argument to an expression string """ - reload(sys) - sys.setdefaultencoding("utf-8") - argc = len(sys.argv) find_arg = LipeFindArg() end_of_leading_options = process_leading_options(argc, sys.argv) diff --git a/lipe/pylipe/lipe_find.py b/lipe/pylipe/lipe_find.py index e074138..24cd2c99 100644 --- a/lipe/pylipe/lipe_find.py +++ b/lipe/pylipe/lipe_find.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com # pylint: disable=too-many-lines @@ -1910,7 +1910,7 @@ def used_option_parse(log, arg, negative): days_string = arg try: - days = str(int(days_string, 10)) + days = int(days_string, 10) except ValueError: log.cl_error("invalid argument [%s] to [-used]", arg) return None @@ -1919,6 +1919,8 @@ def used_option_parse(log, arg, negative): log.cl_error("negative [%s] to [-used]", arg) return None + days = str(days) + return LipeFindUsed(days, greater_than=greater_than, less_than=less_than, exclude=negative) @@ -2000,7 +2002,7 @@ def perm_option_parse(log, arg, negative): mode_string) return None - if mode > 07777: + if mode > 0o7777: log.cl_error("argument [%s] of option [-perm] is larger than 07777", mode_string) return None @@ -2325,9 +2327,6 @@ def main(): """ Find files by calling lipe command """ - reload(sys) - sys.setdefaultencoding("utf-8") - argc = len(sys.argv) find_arg = LipeFindArg() end_of_leading_options = process_leading_options(argc, sys.argv, find_arg) diff --git a/lipe/pylipe/lipe_flist_handle.py b/lipe/pylipe/lipe_flist_handle.py index 9882970..7d3bd1c 100644 --- a/lipe/pylipe/lipe_flist_handle.py +++ b/lipe/pylipe/lipe_flist_handle.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ @@ -375,13 +375,6 @@ def main(): """ # pylint: disable=too-many-branches,too-many-statements,unused-variable,not-callable # pylint: disable=too-many-boolean-expressions,too-many-locals,not-callable - if sys.version[0] == '2': - reload(sys) - if hasattr(sys, "setdefaultencoding"): - set_encoding = getattr(sys, "setdefaultencoding", None) - set_encoding('UTF-8') - else: - os.environ["PYTHONIOENCODING"] = 'UTF-8' try: options, remainder = getopt.getopt(sys.argv[1:], @@ -407,8 +400,8 @@ def main(): dry_run = False fsname_rootpath = None src_flist = None - start = -1L - end = -1L + start = -1 + end = -1 dest_flist = None logdir = None action = None @@ -442,9 +435,9 @@ def main(): elif opt == "--argument": argument = arg elif opt == "--start": - start = long(arg) + start = int(arg) elif opt == "--end": - end = long(arg) + end = int(arg) elif opt == '-h' or opt == "--help": usage() sys.exit(0) @@ -540,8 +533,8 @@ def main(): if item.lfi_size > 0: total_size += item.lfi_size - print "Total apparent size: ~%d" % total_apparent_size - print "Total disk usage size: ~%d" % total_size + print("Total apparent size: ~%d" % total_apparent_size) + print("Total disk usage size: ~%d" % total_size) if invalid_fd: invalid_fd.close() diff --git a/lipe/pyloris/lbackup.py b/lipe/pyloris/lbackup.py index 7597330..36f245f 100755 --- a/lipe/pyloris/lbackup.py +++ b/lipe/pyloris/lbackup.py @@ -1,5 +1,5 @@ # pylint: disable=too-many-lines,too-many-public-methods -# Copyright (c) 2017 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ @@ -1136,7 +1136,6 @@ def backup_service_init(log, lustre_device, backup_type, host, path): """ Init backup service according to type """ - # pylint: disable=redefined-variable-type if backup_type == "rsync_local": backup_service = BackupServiceRsyncLocal(log, lustre_device, host, path) diff --git a/lipe/pyloris/loris_backup.py b/lipe/pyloris/loris_backup.py index b14c0f3..393afa5 100755 --- a/lipe/pyloris/loris_backup.py +++ b/lipe/pyloris/loris_backup.py @@ -1,13 +1,14 @@ -# Copyright (c) 2017 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ Backup the MDT """ +import json import os import sys import traceback -import json + import filelock import yaml @@ -170,13 +171,6 @@ def main(): Check a simple backup/copyback case """ # pylint: disable=not-callable - if sys.version[0] == '2': - reload(sys) - if hasattr(sys, "setdefaultencoding"): - set_encoding = getattr(sys, "setdefaultencoding", None) - set_encoding('UTF-8') - else: - os.environ["PYTHONIOENCODING"] = 'UTF-8' if len(sys.argv) == 2: config_file = sys.argv[1] else: diff --git a/lipe/pyloris/loris_test.py b/lipe/pyloris/loris_test.py index 9ce99c4..dd59d68 100755 --- a/lipe/pyloris/loris_test.py +++ b/lipe/pyloris/loris_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ @@ -345,14 +345,6 @@ def main(): """ Check a simple backup/copyback case """ - if sys.version[0] == '2': - reload(sys) - if hasattr(sys, "setdefaultencoding"): - set_encoding = getattr(sys, "setdefaultencoding", None) - set_encoding('UTF-8') - else: - os.environ["PYTHONIOENCODING"] = 'UTF-8' - if len(sys.argv) == 2: json_file = sys.argv[1] else: diff --git a/lipe/pylustre/clog.py b/lipe/pylustre/clog.py index 03de76a..a068002 100644 --- a/lipe/pylustre/clog.py +++ b/lipe/pylustre/clog.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ @@ -154,7 +154,7 @@ def get_message(msg, args): if not _UNICODE: # if no unicode support... msg = str(msg) else: - if not isinstance(msg, basestring): + if not isinstance(msg, str): try: msg = str(msg) except UnicodeError: diff --git a/lipe/pylustre/cmd_general.py b/lipe/pylustre/cmd_general.py index 43a6f44..c53a21f 100644 --- a/lipe/pylustre/cmd_general.py +++ b/lipe/pylustre/cmd_general.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ @@ -31,14 +31,6 @@ def main(default_config_fpath, default_log_parent, main_func): The main function of a command """ # pylint: disable=too-many-locals,too-many-branches,too-many-statements,not-callable - if sys.version[0] == '2': - reload(sys) - if hasattr(sys, "setdefaultencoding"): - set_encoding = getattr(sys, "setdefaultencoding", None) - set_encoding('UTF-8') - else: - os.environ["PYTHONIOENCODING"] = 'UTF-8' - options, args = getopt.getopt(sys.argv[1:], "c:i:h", ["config=", diff --git a/lipe/pylustre/cstr.py b/lipe/pylustre/cstr.py index a47a97c..ff68a48 100644 --- a/lipe/pylustre/cstr.py +++ b/lipe/pylustre/cstr.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ @@ -34,4 +34,3 @@ CSTR_STATUS = "status" CSTR_TRUE = "true" CSTR_UNKNOWN = "unknown" CSTR_UPDATE_TIME = "update_time" - diff --git a/lipe/pylustre/lustre.py b/lipe/pylustre/lustre.py index 21a2be1..f80a680 100644 --- a/lipe/pylustre/lustre.py +++ b/lipe/pylustre/lustre.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ @@ -1251,14 +1251,14 @@ class LustreFilesystem(object): "configured, not able to format", self.lf_fsname) return -1 - for service_name, mdt in self.lf_mdts.iteritems(): + for service_name, mdt in self.lf_mdts.items(): ret = mdt.ls_format(log) if ret: log.cl_stderr("failed to format MDT [%s] of Lustre file " "system [%s]", service_name, self.lf_fsname) return -1 - for service_name, ost in self.lf_osts.iteritems(): + for service_name, ost in self.lf_osts.items(): ret = ost.ls_format(log) if ret: log.cl_stderr("failed to format OST [%s] of Lustre file " @@ -1280,21 +1280,21 @@ class LustreFilesystem(object): "system [%s]", self.lf_fsname) return -1 - for service_name, mdt in self.lf_mdts.iteritems(): + for service_name, mdt in self.lf_mdts.items(): ret = mdt.ls_mount(log) if ret: log.cl_stderr("failed to mount MDT [%s] of Lustre file " "system [%s]", service_name, self.lf_fsname) return -1 - for service_name, ost in self.lf_osts.iteritems(): + for service_name, ost in self.lf_osts.items(): ret = ost.ls_mount(log) if ret: log.cl_stderr("failed to mount OST [%s] of Lustre file " "system [%s]", service_name, self.lf_fsname) return -1 - for client_index, client in self.lf_clients.iteritems(): + for client_index, client in self.lf_clients.items(): ret = client.lc_mount(log) if ret: log.cl_stderr("failed to mount client [%s] of Lustre file " @@ -1400,21 +1400,21 @@ class LustreFilesystem(object): Write lock of the file system should be held """ # pylint: disable=too-many-branches - for client_index, client in self.lf_clients.iteritems(): + for client_index, client in self.lf_clients.items(): ret = client.lc_umount(log) if ret: log.cl_stderr("failed to umount client [%s] of Lustre file " "system [%s]", client_index, self.lf_fsname) return -1 - for service_name, ost in self.lf_osts.iteritems(): + for service_name, ost in self.lf_osts.items(): ret = ost.ls_umount(log) if ret: log.cl_stderr("failed to umount OST [%s] of Lustre file " "system [%s]", service_name, self.lf_fsname) return -1 - for service_name, mdt in self.lf_mdts.iteritems(): + for service_name, mdt in self.lf_mdts.items(): ret = mdt.ls_umount(log) if ret: log.cl_stderr("failed to umount MDT [%s] of Lustre file " @@ -3872,7 +3872,7 @@ def get_underlay_device(log, host, target): for mdt_instance in mdts.values(): mnt = mdt_instance.lsi_mnt - if mnt and mnt.rstrip('/') == target.rstrip('/'): + if mnt and mnt.rstrip('/') == target.rstrip('/'): return mdt_instance.lsi_device # no mdt services found, leave it as it is. @@ -3885,6 +3885,7 @@ def lfs_fid2path(log, fid, fsname_rootpath): """ args = ("lfs", "fid2path", "--link=0", fsname_rootpath, fid) proc = subprocess.Popen(args, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate() diff --git a/lipe/pylustre/ssh_host.py b/lipe/pylustre/ssh_host.py index e3de948a..2a95e77 100644 --- a/lipe/pylustre/ssh_host.py +++ b/lipe/pylustre/ssh_host.py @@ -1,5 +1,5 @@ # pylint: disable=too-many-lines -# Copyright (c) 2018 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ @@ -124,8 +124,8 @@ def ssh_run(hostname, command, login_name="root", timeout=None, Use ssh to run command on a remote host """ # pylint: disable=too-many-arguments - if not isinstance(command, basestring): - stderr = "type of command argument is not a basestring" + if not isinstance(command, str): + stderr = "type of command argument is not a string" return utils.CommandResult(stderr=stderr, exit_status=-1) full_command = ssh_command(hostname, command, login_name, identity_file) @@ -544,7 +544,7 @@ class SSHHost(object): umask = os.umask(0) os.umask(umask) - max_privs = 0777 & ~umask + max_privs = 0o777 & ~umask def set_file_privs(filename): """ @@ -555,8 +555,8 @@ class SSHHost(object): file_privs = max_privs # if the original file permissions do not have at least one # executable bit then do not set it anywhere - if not file_stat.st_mode & 0111: - file_privs &= ~0111 + if not file_stat.st_mode & 0o111: + file_privs &= ~0o111 os.chmod(filename, file_privs) @@ -589,7 +589,7 @@ class SSHHost(object): """ # pylint: disable=too-many-arguments dest = os.path.abspath(dest) - if isinstance(source, basestring): + if isinstance(source, str): source = [source] if delete_dest and os.path.isdir(dest): @@ -781,7 +781,7 @@ class SSHHost(object): return -1 self.sh_cached_has_commands["rsync"] = True - if isinstance(source, basestring): + if isinstance(source, str): source = [source] if remote_host is None: remote_host = self diff --git a/lipe/pylustre/utils.py b/lipe/pylustre/utils.py index a0ee703..1361210 100644 --- a/lipe/pylustre/utils.py +++ b/lipe/pylustre/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ @@ -10,12 +10,12 @@ library to install python packages. """ from __future__ import print_function +import io import os import errno import time import signal import subprocess -import StringIO import select import logging import logging.handlers @@ -150,16 +150,16 @@ class CommandJob(object): self.cj_cwd = cwd # allow for easy stdin input by string, we'll let subprocess create # a pipe for stdin input and we'll write to it in the wait loop - if isinstance(stdin, basestring): + if isinstance(stdin, str): self.cj_string_stdin = stdin self.cj_stdin = subprocess.PIPE else: self.cj_string_stdin = None self.cj_stdin = None if return_stdout: - self.cj_stdout_file = StringIO.StringIO() + self.cj_stdout_file = io.StringIO() if return_stderr: - self.cj_stderr_file = StringIO.StringIO() + self.cj_stderr_file = io.StringIO() self.cj_started = False self.cj_killed = False self.cj_start_time = None @@ -189,6 +189,7 @@ class CommandJob(object): self.cj_subprocess = subprocess.Popen(self.cj_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, shell=True, executable=shell, cwd=self.cj_cwd, @@ -198,6 +199,7 @@ class CommandJob(object): self.cj_subprocess = subprocess.Popen(self.cj_command, cwd=self.cj_cwd, env=self.cj_env, + universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=self.cj_stdin) @@ -282,13 +284,13 @@ class CommandJob(object): # read in all the data we can from pipe and then stop tmp_data = [] while select.select([pipe], [], [], 0)[0]: - tmp_data.append(os.read(pipe.fileno(), 1024)) + tmp_data.append(pipe.read(1024)) if len(tmp_data[-1]) == 0: break data = "".join(tmp_data) else: # perform a single read - data = os.read(pipe.fileno(), 1024) + data = pipe.read(1024) if buf is not None: buf.write(data) if tee: @@ -382,8 +384,8 @@ def run(command, timeout=None, stdout_tee=None, stderr_tee=None, stdin=None, If shell=False, the command must be a preserved list rather than a basestring. """ # pylint: disable=too-many-arguments,too-many-locals - if shell is True and not isinstance(command, basestring): - stderr = "type of command argument is not a basestring" + if shell is True and not isinstance(command, str): + stderr = "type of command argument is not a string" return CommandResult(stderr=stderr, exit_status=-1) if shell is False and not isinstance(command, list): @@ -413,7 +415,7 @@ def random_word(length): """ Return random lowercase word with given length """ - return ''.join(random.choice(string.lowercase) for i in range(length)) + return ''.join(random.choice(string.ascii_lowercase) for i in range(length)) def is_exe(fpath): @@ -624,5 +626,5 @@ def dir_is_mountpoint(target): if st.st_dev != pst.st_dev: return True except IOError: - log.cl_error("failed to check path [%s]", target) + logging.error("failed to check path [%s]", target) return False diff --git a/lipe/pylustre/watched_io.py b/lipe/pylustre/watched_io.py index 7e109da..680c1bb 100644 --- a/lipe/pylustre/watched_io.py +++ b/lipe/pylustre/watched_io.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 DataDirect Networks, Inc. +# Copyright (c) 2022 DataDirect Networks, Inc. # All Rights Reserved. # Author: lixi@ddn.com """ @@ -24,7 +24,7 @@ def watched_io_open(fname, func, args): """Open watched IO file. Codes copied from io.py """ - if not isinstance(fname, (basestring, int)): + if not isinstance(fname, (str, int)): raise TypeError("invalid file: %r" % fname) mode = "w" raw = io.FileIO(fname, mode) @@ -54,14 +54,8 @@ class WatchedIO(io.TextIOWrapper): self.wi_fname = fname def write(self, data): - """ - Need unicode() otherwise will hit problem: - TypeError: can't write str to text stream - And also, even the encoding should be utf-8 - there will be some error, so need to ignore it. - """ #pylint: disable=bare-except - data = unicode(data, encoding='utf-8', errors='ignore') + data = str(data) try: super(WatchedIO, self).write(data) except: -- 1.8.3.1