EX-5644 utils: fix lpcc service dependency on lipe-lpcc

This patch makes changes to support python3 for lipe:
* The long() function is no longer supported by Python 3.
  It only has one built-in integral type, named int().
* Octal literals are no longer of the form 0720 - use
  0o720 instead.
* The print statement has been replaced with a print()
  function, with keyword arguments to replace most of the
  special syntax of the old print statement (PEP 3105).
* The dict.iterkeys(), dict.iteritems() and dict.itervalues()
  methods are no longer supported - use dict.items() instead.
* The StringIO and StringIO modules are gone. Instead,
  import the io module and use io.StringIO or io.BytesIO
  for text and data respectively.
* The builtin basestring abstract type was removed.
  Use str instead.
* Use string.ascii_lowercase instead of string.lowercase.

Test-Parameters: trivial testlist=sanity-lipe env=SANITY_EXCEPT="101j 130 244a"
Test-Parameters: clientdistro=el7.9 testlist=sanity-pcc
Test-Parameters: clientdistro=el8.6 testlist=sanity-pcc
Change-Id: Ia5a3b6490fd4cebbd40327d5b2a431590c82cf00
Signed-off-by: Alex Deiter <alex.deiter@gmail.com>
Reviewed-on: https://review.whamcloud.com/48149
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Feng, Lei <flei@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lipe/configure.ac b/lipe/configure.ac
index 302cb29..1ef9b9d 100644 (file)
--- a/lipe/configure.ac
+++ b/lipe/configure.ac
@@ -185,17 +185,28 @@ DISTRO_RELEASE=$(echo $DISTRO | awk -F 'rhel-' '{print $2}' | awk -F '.' '{print
 AC_MSG_RESULT([$DISTRO_RELEASE])
 AC_SUBST(DISTRO_RELEASE)
 
+# -------- check whether enable server build --------
+AC_MSG_CHECKING([whether enable server])
+AC_ARG_ENABLE([server],
+       AS_HELP_STRING([--disable-server],
+                       [disable server]),
+        [enable_server="no"], [enable_server="yes"])
+AC_MSG_RESULT([$enable_server])
+AM_CONDITIONAL([BUILD_SERVER], [test x$enable_server = xyes])
+
 # ------- check for python --------
-AC_CHECK_PROGS([PYTHON], [python2.7], [])
-if test "x$PYTHON" = "x"; then
-       AC_MSG_ERROR([Python is needed for tests. Install python please.])
-fi
+AM_COND_IF([BUILD_SERVER],
+       [AC_CHECK_PROGS([PYTHON], [python3.6], [])
+       if test "x$PYTHON" = "x"; then
+               AC_MSG_ERROR([Python is needed for tests. Install python please.])
+       fi])
 
 # ------- check for pylint --------
-AC_CHECK_PROGS([PYLINT], [pylint-2.7], [])
-if test "x$PYLINT" = "x"; then
-       AC_MSG_ERROR([pylint-2.7 is needed to check python coding style. Install pylint please.])
-fi
+AM_COND_IF([BUILD_SERVER],
+       [AC_CHECK_PROGS([PYLINT], [pylint-3.6], [])
+       if test "x$PYLINT" = "x"; then
+               AC_MSG_ERROR([pylint-3.6 is needed to check python coding style. Install pylint please.])
+       fi])
 
 # ------- check for libssh --------
 AC_CHECK_LIB([ssh], [ssh_new], [
@@ -212,15 +223,6 @@ PKG_CHECK_MODULES([libssh_threads], [libssh_threads],
 )
 AM_CONDITIONAL([HAVE_SSH_THREADS], [test x$have_lssh_threads = xyes])
 
-# -------- check whether enable server build --------
-AC_MSG_CHECKING([whether enable server])
-AC_ARG_ENABLE([server],
-       AS_HELP_STRING([--disable-server],
-                       [disable server]),
-        [enable_server="no"], [enable_server="yes"])
-AC_MSG_RESULT([$enable_server])
-AM_CONDITIONAL([BUILD_SERVER], [test x$enable_server = xyes])
-
 AM_COND_IF([BUILD_SERVER],
        [AC_DEFINE(HAVE_LDISKFS, 1, [enable ldiskfs support])
         PKG_CHECK_MODULES([GUILE], [guile-2.0])])

diff --git a/lipe/detect-distro.sh b/lipe/detect-distro.sh
index c2c64dd..beb1795 100755 (executable)
--- a/lipe/detect-distro.sh
+++ b/lipe/detect-distro.sh
@@ -48,7 +48,7 @@ if which lsb_release >/dev/null 2>&1; then
                elif [ -f /etc/redhat-release ]; then
                        #name=$(head -1 /etc/redhat-release)
                        name=rhel
-                       version=$(echo "$distroname" |
+                       version=$(cat /etc/redhat-release |
                                sed -e 's/^[^0-9.]*//g' | sed -e 's/[ ].*//')
         fi
                if [ -z "$name" -o -z "$version" ]; then

diff --git a/lipe/ldsync b/lipe/ldsync
index 8f59824..900f8b0 100755 (executable)
--- a/lipe/ldsync
+++ b/lipe/ldsync
@@ -1,5 +1,5 @@
-#!/usr/bin/python2 -u
-# Copyright (c) 2018 DataDirect Networks, Inc.
+#!/usr/bin/python3 -u
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: Gu Zheng <gzheng@ddn.com>
 """

diff --git a/lipe/lipe.spec.in b/lipe/lipe.spec.in
index 00a6dff..b27b566 100644 (file)
--- a/lipe/lipe.spec.in
+++ b/lipe/lipe.spec.in
@@ -25,8 +25,15 @@ Prefix: %{_prefix}
 %define lipe_site_dir %{guile_site_dir}/lipe
 %define lipe_site_ccache_dir %{guile_site_ccache_dir}/lipe
 
-%{!?__python2: %global __python2 /usr/bin/python2}
-%{!?python2_sitelib: %global python2_sitelib %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")}
+%{!?__python3: %global __python3 /usr/bin/python3}
+%{!?python3_sitelib: %global python3_sitelib %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")}
+
+# EPEL has different package naming convention for RHEL7
+%if 0%{?rhel} == 7
+%define python_suffix %(%{__python3} -c "import sys; sys.stdout.write(''.join(sys.version.split()[0].split('.')[0:2]))")
+%else
+%define python_suffix %(%{__python3} -c "import sys; sys.stdout.write(sys.version.split()[0].split('.')[0])")
+%endif
 
 Release: @LIPE_RELEASE@%{?dist}
 VCS: @LIPE_REVISION@
@@ -36,7 +43,10 @@ Group: Applications/System
 Source0: @PACKAGE@-%{version}.tar.gz
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 Requires: lipe-pylustre = %{version}-%{release}
-Requires: rsync json-c libyaml python2-pyyaml python2-filelock python2-dateutil
+Requires: rsync json-c libyaml
+Requires: python%{python_suffix}-PyYAML
+Requires: python%{python_suffix}-filelock
+Requires: python%{python_suffix}-dateutil
 Requires: e2fsprogs >= 1.42.13.wc6
 Provides: lipe = %{version}-%{release}
 %if %{with systemd}
@@ -45,6 +55,8 @@ Requires(preun): systemd
 Requires(postun): systemd
 BuildRequires: systemd
 %endif
+BuildRequires: python3-rpm-macros
+BuildRequires: json-c-devel libyaml-devel
 
 %description
 LIPE(Lustre Integrated Policy Engine) is a policy engine which enables users
@@ -61,7 +73,8 @@ Pylustre is a python library for managing Lustre file system.
 
 %package lpcc
 Summary: LPCC (Lustre Persisted Client Cache)
-Requires: lipe-pylustre = %{version}-%{release} python2-pyyaml
+Requires: lipe-pylustre = %{version}-%{release}
+Requires: python%{python_suffix}-PyYAML
 Provides: lipe-lpcc = %{version}-%{release}
 Group: Applications/System
 
@@ -94,7 +107,9 @@ Summary: Lustre Online Reliability Improvement System
 Requires: lipe-pylustre = %{version}-%{release}
 Provides: lipe-loris = %{version}-%{release}
 # python-crontab is generated by EXAScaler.
-Requires: e2fsprogs >= 1.42.13.wc6, rsync, python2-filelock, openssh-clients, python-crontab
+Requires: e2fsprogs >= 1.42.13.wc6, rsync, openssh-clients
+Requires: python%{python_suffix}-filelock
+Requires: python%{python_suffix}-crontab
 Group: Applications/System
 
 %description loris
@@ -164,21 +179,19 @@ rm -rf "%{lipe_site_ccache_dir}"
 
 make V=1
 
-python2 -m py_compile pylustre/*.py
+%__python3 -m compileall -f -b pylustre/*.py
 %if %{with server}
-python2 -m py_compile pylipe/*.py
-python2 -m py_compile pyloris/*.py
+%__python3 -m compileall -f -b pylipe/*.py
+%__python3 -m compileall -f -b pyloris/*.py
 %endif
 
-find pylustre pylipe pyloris -maxdepth 1 -type f -a -name "*.python_checked" -o -name "*.py" | xargs rm -f
-
 %install
 rm -rf $RPM_BUILD_ROOT
 mkdir -p $RPM_BUILD_ROOT%{_sbindir}
 mkdir -p $RPM_BUILD_ROOT%{_bindir}
 mkdir -p $RPM_BUILD_ROOT%{_libdir}
 mkdir -p $RPM_BUILD_ROOT%{guile_site_dir}
-mkdir -p $RPM_BUILD_ROOT%{python2_sitelib}
+mkdir -p $RPM_BUILD_ROOT%{python3_sitelib}
 mkdir -p $RPM_BUILD_ROOT%{_mandir}/man1
 mkdir -p $RPM_BUILD_ROOT%{_mandir}/man5
 mkdir -p $RPM_BUILD_ROOT%{_mandir}/man8
@@ -188,7 +201,9 @@ cp \
        src/lpcc_purge \
        $RPM_BUILD_ROOT%{_bindir}
 
-cp -a pylustre $RPM_BUILD_ROOT%{python2_sitelib}
+mkdir -p $RPM_BUILD_ROOT%{python3_sitelib}/pylustre
+cp -a pylustre/*.pyc $RPM_BUILD_ROOT%{python3_sitelib}/pylustre
+
 cp -a \
        lpcc.conf \
        $RPM_BUILD_ROOT%{_sysconfdir}
@@ -233,8 +248,12 @@ mkdir -p $RPM_BUILD_ROOT%{ddntoolsdir}/
 install -m 0755 scripts/*.sh $RPM_BUILD_ROOT%{ddntoolsdir}/
 %endif # hotpool
 
-cp -a pylipe $RPM_BUILD_ROOT%{python2_sitelib}
-cp -a pyloris $RPM_BUILD_ROOT%{python2_sitelib}
+mkdir -p $RPM_BUILD_ROOT%{python3_sitelib}/pylipe
+cp -a pylipe/*.pyc $RPM_BUILD_ROOT%{python3_sitelib}/pylipe
+
+mkdir -p $RPM_BUILD_ROOT%{python3_sitelib}/pyloris
+cp -a pyloris/*.pyc $RPM_BUILD_ROOT%{python3_sitelib}/pyloris
+
 mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}
 cp -a \
        example_configs/lipe/lipe_launch.json \
@@ -290,7 +309,7 @@ install -m 0644 man/laudit.conf.5 $RPM_BUILD_ROOT%{_mandir}/man5/
 rm -rf $RPM_BUILD_ROOT
 
 %files pylustre
-%{python2_sitelib}/pylustre
+%{python3_sitelib}/pylustre
 
 %files lpcc
 %defattr(-,root,root)
@@ -310,7 +329,7 @@ rm -rf $RPM_BUILD_ROOT
 
 %if %{with server}
 %files loris
-%{python2_sitelib}/pyloris
+%{python3_sitelib}/pyloris
 %{_bindir}/loris_backup
 %{_bindir}/loris_crontab
 %{_bindir}/loris_test
@@ -363,7 +382,7 @@ rm -rf $RPM_BUILD_ROOT
 %{_bindir}/lipe_purge
 %{_bindir}/lipe_scan
 %{_bindir}/lipe_scan2
-%{python2_sitelib}/pylipe
+%{python3_sitelib}/pylipe
 %config(noreplace) %{_sysconfdir}/lipe_launch.json
 %{_mandir}/man1/lipe_scan.1*
 %{_mandir}/man1/lipe_find.1*
@@ -371,6 +390,8 @@ rm -rf $RPM_BUILD_ROOT
 %endif # server
 
 %changelog
+* Thu Jul 07 2022 Minh Diep <mdiep@ddn.com> [1.5]
+- Use Python3
 * Tue Jun 30 2020 Gu Zheng <gzheng@ddn.com> [1.5]
 - Update version to 1.5
 * Wed Apr 03 2019 Gu Zheng <gzheng@ddn.com> [1.4]

diff --git a/lipe/lipe_convert_expr b/lipe/lipe_convert_expr
index 1036f45..04e29fd 100755 (executable)
--- a/lipe/lipe_convert_expr
+++ b/lipe/lipe_convert_expr
@@ -1,5 +1,5 @@
-#!/usr/bin/python2 -u
-# Copyright (c) 2021 DataDirect Networks, Inc.
+#!/usr/bin/python3 -u
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: Jian Yu <yujian@whamcloud.com>
 """

diff --git a/lipe/lipe_expression_tests b/lipe/lipe_expression_tests
index 1116023..9a15896 100755 (executable)
--- a/lipe/lipe_expression_tests
+++ b/lipe/lipe_expression_tests
@@ -1,5 +1,5 @@
-#!/usr/bin/python2 -u
-# Copyright (c) 2017 DataDirect Networks, Inc.
+#!/usr/bin/python3 -u
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """

diff --git a/lipe/lipe_find b/lipe/lipe_find
index a3aed1d..0a3805c 100755 (executable)
--- a/lipe/lipe_find
+++ b/lipe/lipe_find
@@ -1,5 +1,5 @@
-#!/usr/bin/python2 -u
-# Copyright (c) 2018 DataDirect Networks, Inc.
+#!/usr/bin/python3 -u
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """

diff --git a/lipe/lipe_launch b/lipe/lipe_launch
index 0d809f2..a116dc7 100755 (executable)
--- a/lipe/lipe_launch
+++ b/lipe/lipe_launch
@@ -1,5 +1,5 @@
-#!/usr/bin/python2 -u
-# Copyright (c) 2017 DataDirect Networks, Inc.
+#!/usr/bin/python3 -u
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """

diff --git a/lipe/lipe_run_action b/lipe/lipe_run_action
index a06a4a8..741bd0b 100755 (executable)
--- a/lipe/lipe_run_action
+++ b/lipe/lipe_run_action
@@ -1,5 +1,5 @@
-#!/usr/bin/python2 -u
-# Copyright (c) 2017 DataDirect Networks, Inc.
+#!/usr/bin/python3 -u
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """

diff --git a/lipe/loris_backup b/lipe/loris_backup
index 45c849e..c44d5ab 100755 (executable)
--- a/lipe/loris_backup
+++ b/lipe/loris_backup
@@ -1,5 +1,5 @@
-#!/usr/bin/python2 -u
-# Copyright (c) 2017 DataDirect Networks, Inc.
+#!/usr/bin/python3 -u
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """

diff --git a/lipe/loris_crontab b/lipe/loris_crontab
index f2e7123..303a282 100755 (executable)
--- a/lipe/loris_crontab
+++ b/lipe/loris_crontab
@@ -1,5 +1,5 @@
-#!/usr/bin/python2 -u
-# Copyright (c) 2017 DataDirect Networks, Inc.
+#!/usr/bin/python3 -u
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """

diff --git a/lipe/loris_test b/lipe/loris_test
index 730b763..68c9094 100755 (executable)
--- a/lipe/loris_test
+++ b/lipe/loris_test
@@ -1,5 +1,5 @@
-#!/usr/bin/python2 -u
-# Copyright (c) 2017 DataDirect Networks, Inc.
+#!/usr/bin/python3 -u
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """

diff --git a/lipe/lpcc b/lipe/lpcc
index a706d41..4917ec2 100755 (executable)
--- a/lipe/lpcc
+++ b/lipe/lpcc
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/python3
 # Copyright (c) 2021 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: flei@ddn.com

diff --git a/lipe/pybuild/lipe_expression_tests.py b/lipe/pybuild/lipe_expression_tests.py
index 6538eb4..be6e7a0 100755 (executable)
--- a/lipe/pybuild/lipe_expression_tests.py
+++ b/lipe/pybuild/lipe_expression_tests.py
@@ -1,6 +1,6 @@
-#!/usr/bin/python2 -u
+#!/usr/bin/python3 -u
 # pylint: disable=too-many-lines
-# Copyright (c) 2019 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """
@@ -166,7 +166,7 @@ def test_size_expressions(attribute="size", has_unit=True, bits=64):
     unit_dict["T"] = 1048576 * 1048576
     unit_dict["P"] = 1048576 * 1048576 * 1024
     unit_dict["E"] = 1048576 * 1048576 * 1048576
-    for unit, size in unit_dict.iteritems():
+    for unit, size in unit_dict.items():
         find_size = (lipe_find.LipeFindSize.LFS_UNIT_EXPRESSION_INFIX %
                      (attribute, 1, unit, attribute, 1, unit))
         find_size_prefix = (lipe_find.LipeFindSize.LFS_UNIT_EXPRESSION_PREFIX %
@@ -1160,7 +1160,7 @@ def main_with_exception():
         for attr in lipe_constant.LIPE_POLICY_ATTRIBUTES:
             if attr == "type":
                 # type bits are filtered by 00170000
-                expected_value = 00170000 & value
+                expected_value = 0o00170000 & value
             elif attr == "stripe_count":
                 # stripe count is 16 bit
                 value = 0xFFFF & value
@@ -1186,17 +1186,17 @@ def main_with_exception():
     # S_ISUID 0004000
     # S_ISGID 0002000
     # S_ISVTX 0001000
-    test_expression("S_IFMT", 00170000)
-    test_expression("S_IFSOCK", 0140000)
-    test_expression("S_IFLNK", 0120000)
-    test_expression("S_IFREG", 0100000)
-    test_expression("S_IFBLK", 0060000)
-    test_expression("S_IFDIR", 0040000)
-    test_expression("S_IFCHR", 0020000)
-    test_expression("S_IFIFO", 0010000)
-    test_expression("S_ISUID", 0004000)
-    test_expression("S_ISGID", 0002000)
-    test_expression("S_ISVTX", 0001000)
+    test_expression("S_IFMT", 0o00170000)
+    test_expression("S_IFSOCK", 0o0140000)
+    test_expression("S_IFLNK", 0o0120000)
+    test_expression("S_IFREG", 0o0100000)
+    test_expression("S_IFBLK", 0o0060000)
+    test_expression("S_IFDIR", 0o0040000)
+    test_expression("S_IFCHR", 0o0020000)
+    test_expression("S_IFIFO", 0o0010000)
+    test_expression("S_ISUID", 0o0004000)
+    test_expression("S_ISGID", 0o0002000)
+    test_expression("S_ISVTX", 0o0001000)
 
     # Attribute tests
     test_expression("1", options="--atime", expect_retval=RETVAL_ARG_INVALID)
@@ -1210,8 +1210,8 @@ def main_with_exception():
     # S_IFSOCK 0140000
     # S_IFLNK 0120000
     # "type" won't overwrite other bits of "mode", only the limited bits
-    test_expression("mode", 0140740, options="--mode 0120740 --type 0140000")
-    test_expression("mode", 0140740, options="--mode 0110740 --type 0140036")
+    test_expression("mode", 0o0140740, options="--mode 0o120740 --type 0o140000")
+    test_expression("mode", 0o0140740, options="--mode 0o110740 --type 0o140036")
     test_expression("sys_time", 1550310180000, options="--sys_time 1550310180000")
 
     for xtime in ["atime", "ctime", "mtime"]:

diff --git a/lipe/pylipe/ldsync.py b/lipe/pylipe/ldsync.py
index 9c4720d..fd24ce6 100644 (file)
--- a/lipe/pylipe/ldsync.py
+++ b/lipe/pylipe/ldsync.py
@@ -1,5 +1,5 @@
-#!/usr/bin/python2 -u
-# Copyright (c) 2018 DataDirect Networks, Inc.
+#!/usr/bin/python3 -u
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: Gu Zheng <gzheng@ddn.com>
 """
@@ -95,13 +95,6 @@ def main():
     # pylint: disable=too-many-branches,too-many-statements,unused-variable
     # pylint: disable=too-many-boolean-expressions,too-many-locals
     # pylint: disable=global-statement,not-callable
-    if sys.version[0] == '2':
-        reload(sys)
-        if hasattr(sys, "setdefaultencoding"):
-            set_encoding = getattr(sys, "setdefaultencoding", None)
-            set_encoding('UTF-8')
-        else:
-            os.environ["PYTHONIOENCODING"] = 'UTF-8'
 
     global DSYNC
     global MPIRUN

diff --git a/lipe/pylipe/lipe.py b/lipe/pylipe/lipe.py
index 8bb31a9..5f7d65e 100644 (file)
--- a/lipe/pylipe/lipe.py
+++ b/lipe/pylipe/lipe.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 
@@ -139,8 +139,8 @@ class FlistActionContext(object):
                  action, action_batch, arguments, action_flags=0):
         self.fac_fid_summary_file = fid_summary_file
         self.fac_output_files = output_files
-        self.fac_start_index = -1L
-        self.fac_end_index = -1L
+        self.fac_start_index = -1
+        self.fac_end_index = -1
         self.fac_context_empty = False
         self.fac_mutex = Lock()
         self.fac_action = action
@@ -152,7 +152,7 @@ class FlistActionContext(object):
         """
         Count lines of fid flist file, and init start&end index
         """
-        count = 0L
+        count = 0
         with open(self.fac_fid_summary_file, 'rb') as fidfile:
             while True:
                 buff = fidfile.read(8 * 1024 * 1024)
@@ -160,7 +160,7 @@ class FlistActionContext(object):
                     break
                 count += buff.count('\n')
         self.fac_mutex.acquire()
-        self.fac_start_index = 0L
+        self.fac_start_index = 0
         self.fac_end_index = count
         self.fac_mutex.release()
 
@@ -168,8 +168,8 @@ class FlistActionContext(object):
         """
         Get valid range(line index)
         """
-        start = -1L
-        end = -1L
+        start = -1
+        end = -1
 
         self.fac_mutex.acquire()
         if self.fac_context_empty:
@@ -672,7 +672,7 @@ def group_apply_action(log, config_group, result_json, flist_type, directory,
     log.cl_debug("counter_dicts: %s, classify_names: %s",
                  counter_dicts, classify_names)
 
-    for counter_name, (action, argument) in counter_dicts.iteritems():
+    for counter_name, (action, argument) in counter_dicts.items():
         fpath = directory + "/" + counter_name
         if not os.path.isdir(fpath):
             log.cl_error("path [%s] is not a directory for file list", fpath)

diff --git a/lipe/pylipe/lipe_convert_expr.py b/lipe/pylipe/lipe_convert_expr.py
index dd3229a..58b366d 100644 (file)
--- a/lipe/pylipe/lipe_convert_expr.py
+++ b/lipe/pylipe/lipe_convert_expr.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 # Author: Jian Yu <yujian@whamcloud.com>
@@ -1570,7 +1570,7 @@ def perm_option_parse(log, arg, negative):
                      mode_string)
         return None
 
-    if mode > 07777:
+    if mode > 0o7777:
         log.cl_error("argument [%s] of option [-perm] is larger than 07777",
                      mode_string)
         return None
@@ -1895,9 +1895,6 @@ def main():
     """
     Convert expression argument to an expression string
     """
-    reload(sys)
-    sys.setdefaultencoding("utf-8")
-
     argc = len(sys.argv)
     find_arg = LipeFindArg()
     end_of_leading_options = process_leading_options(argc, sys.argv)

diff --git a/lipe/pylipe/lipe_find.py b/lipe/pylipe/lipe_find.py
index e074138..24cd2c9 100644 (file)
--- a/lipe/pylipe/lipe_find.py
+++ b/lipe/pylipe/lipe_find.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 # pylint: disable=too-many-lines
@@ -1910,7 +1910,7 @@ def used_option_parse(log, arg, negative):
         days_string = arg
 
     try:
-        days = str(int(days_string, 10))
+        days = int(days_string, 10)
     except ValueError:
         log.cl_error("invalid argument [%s] to [-used]", arg)
         return None
@@ -1919,6 +1919,8 @@ def used_option_parse(log, arg, negative):
         log.cl_error("negative [%s] to [-used]", arg)
         return None
 
+    days = str(days)
+
     return LipeFindUsed(days, greater_than=greater_than, less_than=less_than,
                         exclude=negative)
 
@@ -2000,7 +2002,7 @@ def perm_option_parse(log, arg, negative):
                      mode_string)
         return None
 
-    if mode > 07777:
+    if mode > 0o7777:
         log.cl_error("argument [%s] of option [-perm] is larger than 07777",
                      mode_string)
         return None
@@ -2325,9 +2327,6 @@ def main():
     """
     Find files by calling lipe command
     """
-    reload(sys)
-    sys.setdefaultencoding("utf-8")
-
     argc = len(sys.argv)
     find_arg = LipeFindArg()
     end_of_leading_options = process_leading_options(argc, sys.argv, find_arg)

diff --git a/lipe/pylipe/lipe_flist_handle.py b/lipe/pylipe/lipe_flist_handle.py
index 9882970..7d3bd1c 100644 (file)
--- a/lipe/pylipe/lipe_flist_handle.py
+++ b/lipe/pylipe/lipe_flist_handle.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """
@@ -375,13 +375,6 @@ def main():
     """
     # pylint: disable=too-many-branches,too-many-statements,unused-variable,not-callable
     # pylint: disable=too-many-boolean-expressions,too-many-locals,not-callable
-    if sys.version[0] == '2':
-        reload(sys)
-        if hasattr(sys, "setdefaultencoding"):
-            set_encoding = getattr(sys, "setdefaultencoding", None)
-            set_encoding('UTF-8')
-        else:
-            os.environ["PYTHONIOENCODING"] = 'UTF-8'
 
     try:
         options, remainder = getopt.getopt(sys.argv[1:],
@@ -407,8 +400,8 @@ def main():
     dry_run = False
     fsname_rootpath = None
     src_flist = None
-    start = -1L
-    end = -1L
+    start = -1
+    end = -1
     dest_flist = None
     logdir = None
     action = None
@@ -442,9 +435,9 @@ def main():
         elif opt == "--argument":
             argument = arg
         elif opt == "--start":
-            start = long(arg)
+            start = int(arg)
         elif opt == "--end":
-            end = long(arg)
+            end = int(arg)
         elif opt == '-h' or opt == "--help":
             usage()
             sys.exit(0)
@@ -540,8 +533,8 @@ def main():
                 if item.lfi_size > 0:
                     total_size += item.lfi_size
 
-    print "Total apparent size: ~%d" % total_apparent_size
-    print "Total disk usage size: ~%d" % total_size
+    print("Total apparent size: ~%d" % total_apparent_size)
+    print("Total disk usage size: ~%d" % total_size)
 
     if invalid_fd:
         invalid_fd.close()

diff --git a/lipe/pyloris/lbackup.py b/lipe/pyloris/lbackup.py
index 7597330..36f245f 100755 (executable)
--- a/lipe/pyloris/lbackup.py
+++ b/lipe/pyloris/lbackup.py
@@ -1,5 +1,5 @@
 # pylint: disable=too-many-lines,too-many-public-methods
-# Copyright (c) 2017 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """
@@ -1136,7 +1136,6 @@ def backup_service_init(log, lustre_device, backup_type, host, path):
     """
     Init backup service according to type
     """
-    # pylint: disable=redefined-variable-type
     if backup_type == "rsync_local":
         backup_service = BackupServiceRsyncLocal(log, lustre_device,
                                                  host, path)

diff --git a/lipe/pyloris/loris_backup.py b/lipe/pyloris/loris_backup.py
index b14c0f3..393afa5 100755 (executable)
--- a/lipe/pyloris/loris_backup.py
+++ b/lipe/pyloris/loris_backup.py
@@ -1,13 +1,14 @@
-# Copyright (c) 2017 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """
 Backup the MDT
 """
+import json
 import os
 import sys
 import traceback
-import json
+
 import filelock
 import yaml
 
@@ -170,13 +171,6 @@ def main():
     Check a simple backup/copyback case
     """
     # pylint: disable=not-callable
-    if sys.version[0] == '2':
-        reload(sys)
-        if hasattr(sys, "setdefaultencoding"):
-            set_encoding = getattr(sys, "setdefaultencoding", None)
-            set_encoding('UTF-8')
-        else:
-            os.environ["PYTHONIOENCODING"] = 'UTF-8'
     if len(sys.argv) == 2:
         config_file = sys.argv[1]
     else:

diff --git a/lipe/pyloris/loris_test.py b/lipe/pyloris/loris_test.py
index 9ce99c4..dd59d68 100755 (executable)
--- a/lipe/pyloris/loris_test.py
+++ b/lipe/pyloris/loris_test.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """
@@ -345,14 +345,6 @@ def main():
     """
     Check a simple backup/copyback case
     """
-    if sys.version[0] == '2':
-        reload(sys)
-        if hasattr(sys, "setdefaultencoding"):
-            set_encoding = getattr(sys, "setdefaultencoding", None)
-            set_encoding('UTF-8')
-        else:
-            os.environ["PYTHONIOENCODING"] = 'UTF-8'
-
     if len(sys.argv) == 2:
         json_file = sys.argv[1]
     else:

diff --git a/lipe/pylustre/clog.py b/lipe/pylustre/clog.py
index 03de76a..a068002 100644 (file)
--- a/lipe/pylustre/clog.py
+++ b/lipe/pylustre/clog.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """
@@ -154,7 +154,7 @@ def get_message(msg, args):
     if not _UNICODE:  # if no unicode support...
         msg = str(msg)
     else:
-        if not isinstance(msg, basestring):
+        if not isinstance(msg, str):
             try:
                 msg = str(msg)
             except UnicodeError:

diff --git a/lipe/pylustre/cmd_general.py b/lipe/pylustre/cmd_general.py
index 43a6f44..c53a21f 100644 (file)
--- a/lipe/pylustre/cmd_general.py
+++ b/lipe/pylustre/cmd_general.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """
@@ -31,14 +31,6 @@ def main(default_config_fpath, default_log_parent, main_func):
     The main function of a command
     """
     # pylint: disable=too-many-locals,too-many-branches,too-many-statements,not-callable
-    if sys.version[0] == '2':
-        reload(sys)
-        if hasattr(sys, "setdefaultencoding"):
-            set_encoding = getattr(sys, "setdefaultencoding", None)
-            set_encoding('UTF-8')
-        else:
-            os.environ["PYTHONIOENCODING"] = 'UTF-8'
-
     options, args = getopt.getopt(sys.argv[1:],
                                   "c:i:h",
                                   ["config=",

diff --git a/lipe/pylustre/cstr.py b/lipe/pylustre/cstr.py
index a47a97c..ff68a48 100644 (file)
--- a/lipe/pylustre/cstr.py
+++ b/lipe/pylustre/cstr.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """
@@ -34,4 +34,3 @@ CSTR_STATUS = "status"
 CSTR_TRUE = "true"
 CSTR_UNKNOWN = "unknown"
 CSTR_UPDATE_TIME = "update_time"
-

diff --git a/lipe/pylustre/lustre.py b/lipe/pylustre/lustre.py
index 21a2be1..f80a680 100644 (file)
--- a/lipe/pylustre/lustre.py
+++ b/lipe/pylustre/lustre.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2018 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """
@@ -1251,14 +1251,14 @@ class LustreFilesystem(object):
                           "configured, not able to format", self.lf_fsname)
             return -1
 
-        for service_name, mdt in self.lf_mdts.iteritems():
+        for service_name, mdt in self.lf_mdts.items():
             ret = mdt.ls_format(log)
             if ret:
                 log.cl_stderr("failed to format MDT [%s] of Lustre file "
                               "system [%s]", service_name, self.lf_fsname)
                 return -1
 
-        for service_name, ost in self.lf_osts.iteritems():
+        for service_name, ost in self.lf_osts.items():
             ret = ost.ls_format(log)
             if ret:
                 log.cl_stderr("failed to format OST [%s] of Lustre file "
@@ -1280,21 +1280,21 @@ class LustreFilesystem(object):
                               "system [%s]", self.lf_fsname)
                 return -1
 
-        for service_name, mdt in self.lf_mdts.iteritems():
+        for service_name, mdt in self.lf_mdts.items():
             ret = mdt.ls_mount(log)
             if ret:
                 log.cl_stderr("failed to mount MDT [%s] of Lustre file "
                               "system [%s]", service_name, self.lf_fsname)
                 return -1
 
-        for service_name, ost in self.lf_osts.iteritems():
+        for service_name, ost in self.lf_osts.items():
             ret = ost.ls_mount(log)
             if ret:
                 log.cl_stderr("failed to mount OST [%s] of Lustre file "
                               "system [%s]", service_name, self.lf_fsname)
                 return -1
 
-        for client_index, client in self.lf_clients.iteritems():
+        for client_index, client in self.lf_clients.items():
             ret = client.lc_mount(log)
             if ret:
                 log.cl_stderr("failed to mount client [%s] of Lustre file "
@@ -1400,21 +1400,21 @@ class LustreFilesystem(object):
         Write lock of the file system should be held
         """
         # pylint: disable=too-many-branches
-        for client_index, client in self.lf_clients.iteritems():
+        for client_index, client in self.lf_clients.items():
             ret = client.lc_umount(log)
             if ret:
                 log.cl_stderr("failed to umount client [%s] of Lustre file "
                               "system [%s]", client_index, self.lf_fsname)
                 return -1
 
-        for service_name, ost in self.lf_osts.iteritems():
+        for service_name, ost in self.lf_osts.items():
             ret = ost.ls_umount(log)
             if ret:
                 log.cl_stderr("failed to umount OST [%s] of Lustre file "
                               "system [%s]", service_name, self.lf_fsname)
                 return -1
 
-        for service_name, mdt in self.lf_mdts.iteritems():
+        for service_name, mdt in self.lf_mdts.items():
             ret = mdt.ls_umount(log)
             if ret:
                 log.cl_stderr("failed to umount MDT [%s] of Lustre file "
@@ -3872,7 +3872,7 @@ def get_underlay_device(log, host, target):
 
     for mdt_instance in mdts.values():
         mnt = mdt_instance.lsi_mnt
-       if mnt and mnt.rstrip('/') == target.rstrip('/'):
+        if mnt and mnt.rstrip('/') == target.rstrip('/'):
             return mdt_instance.lsi_device
 
     # no mdt services found, leave it as it is.
@@ -3885,6 +3885,7 @@ def lfs_fid2path(log, fid, fsname_rootpath):
     """
     args = ("lfs", "fid2path", "--link=0", fsname_rootpath, fid)
     proc = subprocess.Popen(args,
+                            universal_newlines=True,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
     out, err = proc.communicate()

diff --git a/lipe/pylustre/ssh_host.py b/lipe/pylustre/ssh_host.py
index e3de948..2a95e77 100644 (file)
--- a/lipe/pylustre/ssh_host.py
+++ b/lipe/pylustre/ssh_host.py
@@ -1,5 +1,5 @@
 # pylint: disable=too-many-lines
-# Copyright (c) 2018 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """
@@ -124,8 +124,8 @@ def ssh_run(hostname, command, login_name="root", timeout=None,
     Use ssh to run command on a remote host
     """
     # pylint: disable=too-many-arguments
-    if not isinstance(command, basestring):
-        stderr = "type of command argument is not a basestring"
+    if not isinstance(command, str):
+        stderr = "type of command argument is not a string"
         return utils.CommandResult(stderr=stderr, exit_status=-1)
 
     full_command = ssh_command(hostname, command, login_name, identity_file)
@@ -544,7 +544,7 @@ class SSHHost(object):
         umask = os.umask(0)
         os.umask(umask)
 
-        max_privs = 0777 & ~umask
+        max_privs = 0o777 & ~umask
 
         def set_file_privs(filename):
             """
@@ -555,8 +555,8 @@ class SSHHost(object):
             file_privs = max_privs
             # if the original file permissions do not have at least one
             # executable bit then do not set it anywhere
-            if not file_stat.st_mode & 0111:
-                file_privs &= ~0111
+            if not file_stat.st_mode & 0o111:
+                file_privs &= ~0o111
 
             os.chmod(filename, file_privs)
 
@@ -589,7 +589,7 @@ class SSHHost(object):
         """
         # pylint: disable=too-many-arguments
         dest = os.path.abspath(dest)
-        if isinstance(source, basestring):
+        if isinstance(source, str):
             source = [source]
 
         if delete_dest and os.path.isdir(dest):
@@ -781,7 +781,7 @@ class SSHHost(object):
                 return -1
             self.sh_cached_has_commands["rsync"] = True
 
-        if isinstance(source, basestring):
+        if isinstance(source, str):
             source = [source]
         if remote_host is None:
             remote_host = self

diff --git a/lipe/pylustre/utils.py b/lipe/pylustre/utils.py
index a0ee703..1361210 100644 (file)
--- a/lipe/pylustre/utils.py
+++ b/lipe/pylustre/utils.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """
@@ -10,12 +10,12 @@ library to install python packages.
 """
 
 from __future__ import print_function
+import io
 import os
 import errno
 import time
 import signal
 import subprocess
-import StringIO
 import select
 import logging
 import logging.handlers
@@ -150,16 +150,16 @@ class CommandJob(object):
         self.cj_cwd = cwd
         # allow for easy stdin input by string, we'll let subprocess create
         # a pipe for stdin input and we'll write to it in the wait loop
-        if isinstance(stdin, basestring):
+        if isinstance(stdin, str):
             self.cj_string_stdin = stdin
             self.cj_stdin = subprocess.PIPE
         else:
             self.cj_string_stdin = None
             self.cj_stdin = None
         if return_stdout:
-            self.cj_stdout_file = StringIO.StringIO()
+            self.cj_stdout_file = io.StringIO()
         if return_stderr:
-            self.cj_stderr_file = StringIO.StringIO()
+            self.cj_stderr_file = io.StringIO()
         self.cj_started = False
         self.cj_killed = False
         self.cj_start_time = None
@@ -189,6 +189,7 @@ class CommandJob(object):
             self.cj_subprocess = subprocess.Popen(self.cj_command,
                                                   stdout=subprocess.PIPE,
                                                   stderr=subprocess.PIPE,
+                                                  universal_newlines=True,
                                                   shell=True,
                                                   executable=shell,
                                                   cwd=self.cj_cwd,
@@ -198,6 +199,7 @@ class CommandJob(object):
             self.cj_subprocess = subprocess.Popen(self.cj_command,
                                                   cwd=self.cj_cwd,
                                                   env=self.cj_env,
+                                                  universal_newlines=True,
                                                   stdout=subprocess.PIPE,
                                                   stderr=subprocess.PIPE,
                                                   stdin=self.cj_stdin)
@@ -282,13 +284,13 @@ class CommandJob(object):
             # read in all the data we can from pipe and then stop
             tmp_data = []
             while select.select([pipe], [], [], 0)[0]:
-                tmp_data.append(os.read(pipe.fileno(), 1024))
+                tmp_data.append(pipe.read(1024))
                 if len(tmp_data[-1]) == 0:
                     break
             data = "".join(tmp_data)
         else:
             # perform a single read
-            data = os.read(pipe.fileno(), 1024)
+            data = pipe.read(1024)
         if buf is not None:
             buf.write(data)
         if tee:
@@ -382,8 +384,8 @@ def run(command, timeout=None, stdout_tee=None, stderr_tee=None, stdin=None,
     If shell=False, the command must be a preserved list rather than a basestring.
     """
     # pylint: disable=too-many-arguments,too-many-locals
-    if shell is True and not isinstance(command, basestring):
-        stderr = "type of command argument is not a basestring"
+    if shell is True and not isinstance(command, str):
+        stderr = "type of command argument is not a string"
         return CommandResult(stderr=stderr, exit_status=-1)
 
     if shell is False and not isinstance(command, list):
@@ -413,7 +415,7 @@ def random_word(length):
     """
     Return random lowercase word with given length
     """
-    return ''.join(random.choice(string.lowercase) for i in range(length))
+    return ''.join(random.choice(string.ascii_lowercase) for i in range(length))
 
 
 def is_exe(fpath):
@@ -624,5 +626,5 @@ def dir_is_mountpoint(target):
         if st.st_dev != pst.st_dev:
             return True
     except IOError:
-        log.cl_error("failed to check path [%s]", target)
+        logging.error("failed to check path [%s]", target)
     return False

diff --git a/lipe/pylustre/watched_io.py b/lipe/pylustre/watched_io.py
index 7e109da..680c1bb 100644 (file)
--- a/lipe/pylustre/watched_io.py
+++ b/lipe/pylustre/watched_io.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2016 DataDirect Networks, Inc.
+# Copyright (c) 2022 DataDirect Networks, Inc.
 # All Rights Reserved.
 # Author: lixi@ddn.com
 """
@@ -24,7 +24,7 @@ def watched_io_open(fname, func, args):
     """Open watched IO file.
     Codes copied from io.py
     """
-    if not isinstance(fname, (basestring, int)):
+    if not isinstance(fname, (str, int)):
         raise TypeError("invalid file: %r" % fname)
     mode = "w"
     raw = io.FileIO(fname, mode)
@@ -54,14 +54,8 @@ class WatchedIO(io.TextIOWrapper):
         self.wi_fname = fname
 
     def write(self, data):
-        """
-        Need unicode() otherwise will hit problem:
-        TypeError: can't write str to text stream
-        And also, even the encoding should be utf-8
-        there will be some error, so need to ignore it.
-        """
         #pylint: disable=bare-except
-        data = unicode(data, encoding='utf-8', errors='ignore')
+        data = str(data)
         try:
             super(WatchedIO, self).write(data)
         except: