From: John L. Hammond Date: Mon, 23 Aug 2021 22:28:27 +0000 (-0500) Subject: EX-2921 lipe: merge lipe changes from b_es6_0 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=10a8c7f59bcd25c329041c97c8ef2d1220af99ff;p=fs%2Flustre-release.git EX-2921 lipe: merge lipe changes from b_es6_0 Merge commit '7d665cba3b245f3d49166de48c39fd9df9843633' into b_es6_0 $ git checkout b_es5_2 $ git subtree split --prefix=lipe 2a015e67c4d3cdc7802178f1ad0fd85fc251fb0a $ git checkout b_es6_0 $ git subtree merge --prefix=lipe --squash 2a015e67c4d3cdc7802178f1ad0fd85fc251fb0a Signed-off-by: John L. Hammond Change-Id: Ie8fdc48bad7dba428a247350cf0662524225660f --- 10a8c7f59bcd25c329041c97c8ef2d1220af99ff diff --cc lipe/.gitignore index b4937dc,0000000..8289cdd mode 100644,000000..100644 --- a/lipe/.gitignore +++ b/lipe/.gitignore @@@ -1,69 -1,0 +1,70 @@@ +*.pyc +*.c_checked +*.python_checked +*.pyltest_import_checked +*.checked +*~ +/build +/example_configs/clownfish/clownfish_test.conf +/example_configs/lipe/lipe_test.conf +/lipe.spec +/lipe-*.tar.bz2 +/lipe-*.tar.gz +/pyclownfish/clownfish_pb2.py +/pylipe/lipe_constant.py +/src/*.o +/src/ext4_inode2path +/src/generate_definition +/src/laudit +/src/laudit-report +/src/lcreatemany +/src/ldumpstripe +/src/lfill +/src/lipe_expression_test +/src/lipe_hsm_copytool +/src/lipe_hsm_remover +/src/lipe_scan ++/src/lipe_scan2 +/src/lpcc_purge +/src/lpurge + +# Automake + +Makefile.in +/ar-lib +/mdate-sh +/py-compile +/test-driver +/ylwrap +.deps/ +.dirstamp + +# AutoConf + +autom4te.cache +/autoscan.log +/autoscan-*.log +/aclocal.m4 +/compile +/config.guess +/config.h.in +/config.log +/config.status +/config.sub +/configure +/configure.scan +/depcomp +/install-sh +/missing +/stamp-h1 + +# libtool/libltdl + +/ltmain.sh +/libtool +/libltdl + +# Generated Files from automake/autoconf + +/config.h +Makefile diff --cc lipe/Makefile.am index 81f257b,0000000..558a214d mode 100644,000000..100644 --- a/lipe/Makefile.am +++ b/lipe/Makefile.am @@@ -1,164 -1,0 +1,165 @@@ +SUBDIRS = src pybuild pylipe . + +build_dir = `pwd`/build +rpmbuild_opt = +ISO_PATH = `pwd`/ISO +PACKAGE_PATH = ${ISO_PATH}/Packages + +AUTOMAKE_OPTIONS = -Wall foreign +ACLOCAL_AMFLAGS = ${ALOCAL_FLAGS} + +if BUILD_LAUDIT +rpmbuild_opt += --with laudit +else +rpmbuild_opt += --without laudit +endif + +if ZFS +rpmbuild_opt += --with zfs +else +rpmbuild_opt += --without zfs +endif + +if BUILD_HOTPOOL_UTILS +rpmbuild_opt += --with hotpool +else +rpmbuild_opt += --without hotpool +endif + +PYTHON_COMMANDS = \ + gen_lipe_test \ + ldsync \ + lipe_build \ + lipe_copytool_manager \ + lipe_expression_tests \ + lipe_find \ ++ lipe_convert_expr \ + lipe_run_action \ + lipe_hsm_check \ + lipe_install \ + lipe_install_build_deps \ + lipe_launch \ + lipe_test \ + lipe_test_console \ + lipe_test_copytool \ + lipe_test_launch \ + lipe_test_scheduler \ + lipe_virt \ + loris_backup \ + loris_crontab \ + loris_test \ + lpcc \ + pyltest_import_check + +EXTRA_DIST= \ + $(PYTHON_COMMANDS) \ + detect-distro.sh \ + lipe-revision.sh \ + example_configs/clownfish/seperate_mgs/clownfish.conf \ + example_configs/clownfish/seperate_mgs/lipe_virt.conf \ + example_configs/lipe/lipe_install.conf \ + example_configs/lipe/lipe_launch.json \ + example_configs/loris/loris.conf \ + example_configs/ltest/lipe_test_scheduler.conf \ + example_configs/hotpool/* \ + init.d/* \ + lipe_copytool_manager.conf \ + lipe.conf \ + lipe.spec \ + lipe.spec.in \ + lipe_c_check.pl \ + lpcc.conf \ + laudit.conf.example \ + pybuild/*.py \ + pyclownfish/*.py \ + pylhsm/*.py \ + pylipe/.pylintrc \ + pylipe/*.py \ + pyloris/*.py \ + pylustre/*.py \ + pyltest/*.py \ + scripts/*.sh \ + systemd/* \ + man/* \ + lhsm_tests/* \ + .pylintrc + +PYLTEST_FILES = $(wildcard pyltest/*.py) +PYTHON_LIB_FILES = $(wildcard pyclownfish/*.py pylustre/*.py pyloris/*.py pylhsm/*.py) +PYTHON_LIB_FILES += $(PYLTEST_FILES) +PYTHON_FILES = $(PYTHON_LIB_FILES) $(PYTHON_COMMANDS) +PYTHON_CHECKS = $(PYTHON_FILES:%=%.python_checked) +PYLTEST_CHECKS = $(PYLTEST_FILES:%=%.pyltest_import_checked) +PYTHON_CHECKS += $(PYLTEST_CHECKS) +CHECKS = $(PYTHON_CHECKS) + +%.pyltest_import_checked: % + python2 ./pyltest_import_check $< + touch $@ + +%.python_checked: % pylipe/.pylintrc + @if test $< != $(PYTHON_PROTOBUF); then \ + PYLINTRC=pylipe/.pylintrc $(PYLINT) --disable=I $< || exit 1; \ + fi + touch $@ + +check_clean-local: + rm -f $(CHECKS) + +check-local: $(CHECKS) + +all: all-am + +# Clean up all the generated files that are ignored in the source repo +# +mrproper: maintainer-clean + rm -f Makefile.in aclocal.m4 configure + rm -f compile depcomp install-sh missing + +PYLUSTRE_RPM = build/RPMS/x86_64/lipe-pylustre-$(PACKAGE_VERSION)-$(LIPE_RELEASE).el$(DISTRO_RELEASE)*.x86_64.rpm +PYLTEST_RPM = build/RPMS/x86_64/lipe-pyltest-$(PACKAGE_VERSION)-$(LIPE_RELEASE).el$(DISTRO_RELEASE)*.x86_64.rpm +CLOWNFISH_RPM = build/RPMS/x86_64/lipe-clownfish-$(PACKAGE_VERSION)-$(LIPE_RELEASE).el$(DISTRO_RELEASE)*.x86_64.rpm +LIPE_RPM = build/RPMS/x86_64/lipe-$(PACKAGE_VERSION)-$(LIPE_RELEASE).el$(DISTRO_RELEASE)*.x86_64.rpm +LIPE_DEBUGINFO_RPM = build/RPMS/x86_64/lipe-debuginfo-$(PACKAGE_VERSION)-$(LIPE_RELEASE).el$(DISTRO_RELEASE)*.x86_64.rpm +LORIS_RPM = build/RPMS/x86_64/lipe-loris-$(PACKAGE_VERSION)-$(LIPE_RELEASE).el$(DISTRO_RELEASE)*.x86_64.rpm +LIPE_CLIENT_RPM = build/RPMS/x86_64/lipe-client-$(PACKAGE_VERSION)-$(LIPE_RELEASE).el$(DISTRO_RELEASE)*.x86_64.rpm +LIPE_SERVER_RPM = build/RPMS/x86_64/lipe-server-$(PACKAGE_VERSION)-$(LIPE_RELEASE).el$(DISTRO_RELEASE)*.x86_64.rpm +LIPE_HSM_RPM = build/RPMS/x86_64/lipe-hsm-$(PACKAGE_VERSION)-$(LIPE_RELEASE).el$(DISTRO_RELEASE)*.x86_64.rpm +LIPE_LPCC_RPM = build/RPMS/x86_64/lipe-lpcc-$(PACKAGE_VERSION)-$(LIPE_RELEASE).el$(DISTRO_RELEASE)*.x86_64.rpm + +rpms: lipe.spec dist + mkdir -p $(build_dir)/BUILD $(build_dir)/SPECS $(build_dir)/SRPMS $(build_dir)/RPMS \ + && rpmbuild $(rpmbuild_opt) --define="_topdir $(build_dir)" \ + --define="_prefix $(prefix)" -tb $(distdir).tar.gz \ + && echo "RPMs successfully generated in $(build_dir)/RPMS" + +lipe-$(PACKAGE_VERSION).x86_64.iso: rpms + @if test -z "$(CACHED_ISO_PATH)"; then \ + echo -e "Error: Can not build ISO without the cached ISO path," \ + "please reconfigure using --with-cached-iso=path option"; \ + exit 1; \ + fi + rm $(ISO_PATH) -fr + rm -f lipe-*.iso + rm -f lipe-*.md5 + cp -a $(CACHED_ISO_PATH) $(ISO_PATH) + mkdir -p $(PACKAGE_PATH) + cp $(CLOWNFISH_RPM) $(PACKAGE_PATH) + cp $(LIPE_RPM) $(PACKAGE_PATH) + cp $(LIPE_DEBUGINFO_RPM) $(PACKAGE_PATH) + cp $(PYLUSTRE_RPM) $(PACKAGE_PATH) + cp $(PYLTEST_RPM) $(PACKAGE_PATH) + cp $(LORIS_RPM) $(PACKAGE_PATH) + cp $(LIPE_CLIENT_RPM) $(PACKAGE_PATH) + cp $(LIPE_SERVER_RPM) $(PACKAGE_PATH) + cp $(LIPE_HSM_RPM) $(PACKAGE_PATH) + cp $(LIPE_LPCC_RPM) $(PACKAGE_PATH) + + createrepo $(PACKAGE_PATH) + mkisofs -joliet-long -R -o lipe-$(PACKAGE_VERSION).x86_64.iso $(ISO_PATH) + +lipe-$(PACKAGE_VERSION).x86_64.md5: lipe-$(PACKAGE_VERSION).x86_64.iso + md5sum lipe-$(PACKAGE_VERSION).x86_64.iso \ + > lipe-$(PACKAGE_VERSION).x86_64.md5 + +iso: lipe-$(PACKAGE_VERSION).x86_64.iso lipe-$(PACKAGE_VERSION).x86_64.md5 diff --cc lipe/lipe.spec.in index 5792820,0000000..959a00f mode 100644,000000..100644 --- a/lipe/lipe.spec.in +++ b/lipe/lipe.spec.in @@@ -1,469 -1,0 +1,473 @@@ +# LIPE specfile + +# Declare rpmbuild --with/--without parameters +%bcond_with laudit +%bcond_with zfs +%bcond_with hotpool + +# RHEL >= 7 comes with systemd +%if 0%{?rhel} >= 7 +%define with_systemd 1 +%endif + +%define ddntoolsdir /opt/ddn/es/tools + +Name: @PACKAGE@ +Version: @VERSION@ + +Vendor: DataDirect Networks Inc. +Prefix: %{_prefix} + +%define __python %{_bindir}/python2 + +%{!?python2_sitelib: %global python2_sitelib %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} + +Release: @LIPE_RELEASE@%{?dist} +VCS: @LIPE_REVISION@ +Summary: lipe - Lustre Integrated Policy Engine +License: All rights reserved DataDirect Networks Inc. +Group: Applications/System +Source0: @PACKAGE@-%{version}.tar.gz +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +Requires: lipe-pylustre = %{version}-%{release} +Requires: rsync json-c libyaml PyYAML python2-filelock python-dateutil +Requires: e2fsprogs >= 1.42.13.wc6 +Provides: lipe = %{version}-%{release} +%if %{with systemd} +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd +BuildRequires: systemd +%endif + +%description +LIPE(Lustre Integrated Policy Engine) is a policy engine which enables users +to use predefined rules to do the daily data management of Lustre file system +automatically. + +%package pylustre +Summary: Python Library of Lustre - General Python Library to manage Lustre +Provides: lipe-pylustre = %{version}-%{release} +Group: Applications/System + +%description pylustre +Pylustre is a python library for managing Lustre file system. + +%package loris +Summary: Lustre Online Reliability Improvement System +Requires: lipe-pylustre = %{version}-%{release} +Provides: lipe-loris = %{version}-%{release} +# python-crontab is generated by EXAScaler. +Requires: e2fsprogs >= 1.42.13.wc6, rsync, python2-filelock, openssh-clients, python-crontab +Group: Applications/System + +%description loris +LORIS(Lustre Online Reliability Improvement System) backups MDTs for purposes +of disaster recovery. + +%package clownfish +Summary: Lustre Management System +Requires: lipe-pylustre = %{version}-%{release} +Requires: rsync +Provides: lipe-clownfish = %{version}-%{release} +Group: Applications/System + +%description clownfish +Clownfish manages Lustre clusters for HA purposes. + +%package pyltest +Summary: Python Library of LiPE common test framework +Requires: lipe-pylustre = %{version}-%{release} +Provides: lipe-pyltest = %{version}-%{release} +%if %{with systemd} +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd +BuildRequires: systemd +%else +Requires(post): chkconfig +Requires(preun): chkconfig +%endif +Group: Applications/System + +%description pyltest +Pyltest is a common test framework for LiPE + +%post pyltest +%if %{with systemd} +%systemd_post lipe_test_scheduler.service +%endif + +%preun pyltest +%if %{with systemd} +%systemd_preun lipe_test_scheduler.service +%else +/sbin/service lipe_test_scheduler stop >/dev/null 2>&1 ||: +/sbin/chkconfig --del lipe_test_scheduler +%endif + +%postun pyltest +%if %{with systemd} +%systemd_postun_with_restart lipe_test_scheduler.service +%else +/sbin/service lipe_test_scheduler condrestart >/dev/null 2>&1 ||: +%endif + +%package lpcc +Summary: LPCC (Lustre Persisted Client Cache) +Requires: lipe-pylustre = %{version}-%{release} +Provides: lipe-lpcc = %{version}-%{release} +Group: Applications/System + +%description lpcc +Tools for LPCC (Lustre Persisted Client Cache). + +%post lpcc +%if %{with systemd} +%systemd_post lipe.service +%endif + +%preun lpcc +%if %{with systemd} +%systemd_preun lpcc.service +%else +/sbin/service lpcc stop >/dev/null 2>&1 ||: +/sbin/chkconfig --del lpcc +%endif + +%postun lpcc +%if %{with systemd} +%systemd_postun_with_restart lpcc.service +%else +/sbin/service lpcc condrestart >/dev/null 2>&1 ||: +%endif + + +%package hsm +Summary: Lightweight hsm (Hierarchical Storage Management) tools. +Requires: lipe-pylustre = %{version}-%{release} +Requires: gzip +Provides: lipe-hsm = %{version}-%{release} +Group: Applications/System + +%description hsm +lightweight hsm (Hierarchical Storage Management) tools. + +%post hsm +%if %{with systemd} +%systemd_post lipe_copytool_manager.service +%endif + +%preun hsm +%if %{with systemd} +%systemd_preun lipe_copytool_manager.service +%else +/sbin/service lipe_copytool_manager stop >/dev/null 2>&1 ||: +/sbin/chkconfig --del lipe_copytool_manager +%endif + +%postun hsm +%if %{with systemd} +%systemd_postun_with_restart lipe_copytool_manager.service +%else +/sbin/service lipe_copytool_manager condrestart >/dev/null 2>&1 ||: +%endif + + +%package server +Summary: Lipe Server Package +Requires: lustre +Requires: e2fsprogs >= 1.42.13.wc6 +%if %{with zfs} +Requires: libzfs2 +%endif +Provides: lipe-server = %{version}-%{release} +Group: Applications/System + +%description server +Provides lipe tools run on lustre server. + +%package client +Summary: Lipe Client Package +Requires: lipe = %{version}-%{release} +Requires: /usr/lib64/liblustreapi.so +Provides: lipe-client = %{version}-%{release} +Group: Applications/System + +%description client +Provides lipe tools run on lustre client. + +Generated using options: @ac_configure_args@ + +%prep +%setup -q -n @PACKAGE@-%{version} + +%post + +%preun + +%postun + +%build +./configure @ac_configure_args@ %{?configure_flags:configure_flags} \ + --sysconfdir=%{_sysconfdir} \ + --mandir=%{_mandir} \ + --libdir=%{_libdir} \ + --includedir=%{_includedir} \ + --prefix=%{_prefix} + +make V=1 + +python2 -m py_compile pyclownfish/*.py +python2 -m py_compile pylustre/*.py +python2 -m py_compile pylhsm/*.py +python2 -m py_compile pylipe/*.py +python2 -m py_compile pyloris/*.py +python2 -m py_compile pyltest/*.py + +find pyclownfish pylustre pylhsm pylipe pyloris pyltest -maxdepth 1 -type f -a -name "*.python_checked" -o -name "*.py" | xargs rm -f + +%install +rm -rf $RPM_BUILD_ROOT +mkdir -p $RPM_BUILD_ROOT%{_sbindir} +mkdir -p $RPM_BUILD_ROOT%{_bindir} +mkdir -p $RPM_BUILD_ROOT%{_libdir} +mkdir -p $RPM_BUILD_ROOT%{python2_sitelib} +mkdir -p $RPM_BUILD_ROOT%{_mandir}/man1 +mkdir -p $RPM_BUILD_ROOT%{_mandir}/man5 +mkdir -p $RPM_BUILD_ROOT%{_mandir}/man8 +mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d +cp \ + ldsync \ + lipe_copytool_manager \ + lipe_find \ ++ lipe_convert_expr \ + lipe_run_action \ + lipe_hsm_check \ + lipe_install \ + lipe_launch \ + lipe_test \ + lipe_test_console \ + lipe_test_launch \ + lipe_test_scheduler \ + lipe_test_copytool \ + lipe_virt \ + loris_backup \ + loris_crontab \ + loris_test \ + lpcc \ + src/lpcc_purge \ + src/ext4_inode2path \ + src/lcreatemany \ + src/ldumpstripe \ + src/lfill \ + src/lipe_scan \ ++ src/lipe_scan2 \ + src/lipe_hsm_copytool \ + src/lipe_hsm_remover \ + $RPM_BUILD_ROOT%{_bindir} +%if %{with laudit} +cp src/laudit \ + src/laudit-report \ + $RPM_BUILD_ROOT%{_bindir} +%endif + +%if %{with hotpool} +cp src/lamigo \ + src/lpurge \ + $RPM_BUILD_ROOT%{_sbindir} +mkdir -p $RPM_BUILD_ROOT%{ddntoolsdir}/ +install -m 0755 scripts/*.sh $RPM_BUILD_ROOT%{ddntoolsdir}/ +%endif + +cp -a pyclownfish $RPM_BUILD_ROOT%{python2_sitelib} +cp -a pylhsm $RPM_BUILD_ROOT%{python2_sitelib} +cp -a pylipe $RPM_BUILD_ROOT%{python2_sitelib} +cp -a pyloris $RPM_BUILD_ROOT%{python2_sitelib} +cp -a pylustre $RPM_BUILD_ROOT%{python2_sitelib} +cp -a pyltest $RPM_BUILD_ROOT%{python2_sitelib} +mkdir -p $RPM_BUILD_ROOT%{_sysconfdir} +cp -a \ + example_configs/clownfish/seperate_mgs/clownfish.conf \ + example_configs/lipe/lipe_install.conf \ + example_configs/lipe/lipe_launch.json \ + example_configs/loris/loris.conf \ + example_configs/ltest/lipe_test_scheduler.conf \ + lipe.conf \ + lipe_copytool_manager.conf \ + example_configs/clownfish/seperate_mgs/lipe_virt.conf \ + lpcc.conf \ + $RPM_BUILD_ROOT%{_sysconfdir} + +%if %{with laudit} +mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/laudit +cp -a laudit.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/laudit +%endif + +%if %{with hotpool} +cp -a example_configs/hotpool/* $RPM_BUILD_ROOT%{_sysconfdir}/ +%endif + +%if %{with systemd} + mkdir -p $RPM_BUILD_ROOT%{_unitdir}/ + install -m 0644 -D systemd/lpcc.service $RPM_BUILD_ROOT%{_unitdir}/lpcc.service + install -m 0644 -D systemd/lipe_copytool_manager.service \ + $RPM_BUILD_ROOT%{_unitdir}/lipe_copytool_manager.service + install -m 0644 -D systemd/lipe_test_scheduler.service \ + $RPM_BUILD_ROOT%{_unitdir}/lipe_test_scheduler.service +%if %{with hotpool} + install -m 0644 -D systemd/lpurge@.service \ + $RPM_BUILD_ROOT%{_unitdir}/lpurge@.service + install -m 0644 -D systemd/lamigo@.service \ + $RPM_BUILD_ROOT%{_unitdir}/lamigo@.service +%endif +%else + mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/rc.d/init.d + install -m 0744 -D init.d/lpcc \ + $RPM_BUILD_ROOT%{_sysconfdir}/rc.d/init.d/lpcc + install -m 0744 -D init.d/lipe_copytool_manager \ + $RPM_BUILD_ROOT%{_sysconfdir}/rc.d/init.d/lipe_copytool_manager + install -m 0744 -D init.d/lipe_test_scheduler \ + $RPM_BUILD_ROOT%{_sysconfdir}/rc.d/init.d/lipe_test_scheduler +%endif +install -m 0644 man/lipe_scan.1 $RPM_BUILD_ROOT%{_mandir}/man1/ +install -m 0644 man/lipe_find.1 $RPM_BUILD_ROOT%{_mandir}/man1/ +install -m 0644 man/lfill.1 $RPM_BUILD_ROOT%{_mandir}/man1/ +install -m 0644 man/lpcc.8 $RPM_BUILD_ROOT%{_mandir}/man8/ +install -m 0644 man/lpcc-start.8 $RPM_BUILD_ROOT%{_mandir}/man8/ +install -m 0644 man/lpcc-stop.8 $RPM_BUILD_ROOT%{_mandir}/man8/ +install -m 0644 man/lpcc-status.8 $RPM_BUILD_ROOT%{_mandir}/man8/ +install -m 0644 man/lpcc.conf.5 $RPM_BUILD_ROOT%{_mandir}/man5/ +%if %{with laudit} +install -m 0644 man/laudit.1 $RPM_BUILD_ROOT%{_mandir}/man1/ +install -m 0644 man/laudit-report.1 $RPM_BUILD_ROOT%{_mandir}/man1/ +install -m 0644 man/laudit.conf.5 $RPM_BUILD_ROOT%{_mandir}/man5/ +%endif +cp -a lhsm_tests $RPM_BUILD_ROOT%{_libdir}/ + + +%clean +rm -rf $RPM_BUILD_ROOT + +%files clownfish +%{python2_sitelib}/pyclownfish +%{_bindir}/lcreatemany +%config(noreplace) %{_sysconfdir}/clownfish.conf + +%files loris +%{python2_sitelib}/pyloris +%{_bindir}/loris_backup +%{_bindir}/loris_crontab +%{_bindir}/loris_test +%config(noreplace) %{_sysconfdir}/loris.conf + +%files pylustre +%{python2_sitelib}/pylustre +%{_bindir}/lipe_virt +%config(noreplace) %{_sysconfdir}/lipe_virt.conf + +%files pyltest +%{python2_sitelib}/pyltest +%{_bindir}/lipe_test_console +%{_bindir}/lipe_test_launch +%{_bindir}/lipe_test_scheduler +%config(noreplace) %{_sysconfdir}/lipe_test_scheduler.conf +%if %{with systemd} + %{_unitdir}/lipe_test_scheduler.service +%else + %{_sysconfdir}/rc.d/init.d/lipe_test_scheduler +%endif + +%files lpcc +%defattr(-,root,root) +%{_bindir}/lpcc +%{_bindir}/lpcc_purge +%config(noreplace) %{_sysconfdir}/lpcc.conf +%if %{with systemd} + %{_unitdir}/lpcc.service +%else + %{_sysconfdir}/rc.d/init.d/lpcc +%endif +%{_mandir}/man8/lpcc.8* +%{_mandir}/man8/lpcc-start.8* +%{_mandir}/man8/lpcc-stop.8* +%{_mandir}/man8/lpcc-status.8* +%{_mandir}/man5/lpcc.conf.5* + + + +%files hsm +%defattr(-,root,root) +%{python2_sitelib}/pylhsm +%{_bindir}/lipe_hsm_remover +%{_bindir}/lipe_hsm_check +%{_bindir}/lipe_hsm_copytool +%{_bindir}/lipe_copytool_manager +%{_bindir}/lipe_test_copytool +%config(noreplace) %{_sysconfdir}/lipe_copytool_manager.conf +%if %{with systemd} + %{_unitdir}/lipe_copytool_manager.service +%else + %{_sysconfdir}/rc.d/init.d/lipe_copytool_manager +%endif +%{_libdir}/lhsm_tests + +%files server +%defattr(-,root,root) +%{_bindir}/ext4_inode2path +%{_bindir}/ldumpstripe +%config(noreplace) %{_sysconfdir}/lipe.conf +%if %{with hotpool} +%{_sbindir}/lamigo +%{_sbindir}/lpurge +%config(noreplace) %{_sysconfdir}/lamigo.conf.example +%config(noreplace) %{_sysconfdir}/lpurge.conf.example +%{_unitdir}/lpurge@.service +%{_unitdir}/lamigo@.service +%{ddntoolsdir}/*.sh +%endif + +%files client +%defattr(-,root,root) +%{_bindir}/lipe_run_action +%if %{with laudit} +%{_bindir}/laudit +%{_bindir}/laudit-report +%{_sysconfdir}/laudit/laudit.conf.example +%{_mandir}/man1/laudit.1* +%{_mandir}/man1/laudit-report.1* +%{_mandir}/man5/laudit.conf.5* +%endif + +%files +%defattr(-,root,root) +%{_bindir}/ldsync ++%{_bindir}/lipe_convert_expr +%{_bindir}/lipe_install +%{_bindir}/lipe_launch +%{_bindir}/lipe_test +%{_bindir}/lfill +%{_bindir}/lipe_scan ++%{_bindir}/lipe_scan2 +%{_bindir}/lipe_find +%{python2_sitelib}/pylipe +%config(noreplace) %{_sysconfdir}/lipe_install.conf +%config(noreplace) %{_sysconfdir}/lipe_launch.json +%{_mandir}/man1/lipe_scan.1* +%{_mandir}/man1/lipe_find.1* +%{_mandir}/man1/lfill.1* + + +%changelog +* Tue Jun 30 2020 Gu Zheng [1.5] +- Update version to 1.5 +* Wed Apr 03 2019 Gu Zheng [1.4] +- Devide RPM by function +* Thu Jan 03 2019 Gu Zheng [1.4] +- Add pyltest RPM +* Wed Mar 07 2018 Sebastien Buisson [1.1] +- Add laudit and laudit-report +* Wed Oct 11 2017 Li Xi [1.0] +- Add license-server RPM +* Mon Feb 27 2017 Qian Yingjin [0.1] +- Initial import diff --cc lipe/lipe_convert_expr index 0000000,0fa2320..0fa2320 mode 000000,100755..100755 --- a/lipe/lipe_convert_expr +++ b/lipe/lipe_convert_expr diff --cc lipe/pybuild/lipe_expression_tests.py index 653e3f6,0000000..6538eb4 mode 100755,000000..100755 --- a/lipe/pybuild/lipe_expression_tests.py +++ b/lipe/pybuild/lipe_expression_tests.py @@@ -1,1310 -1,0 +1,1346 @@@ +#!/usr/bin/python2 -u +# pylint: disable=too-many-lines +# Copyright (c) 2019 DataDirect Networks, Inc. +# All Rights Reserved. +# Author: lixi@ddn.com +""" +Script to check whether expression evaluation works well +""" +import os +import sys + +# Local libs +from pylustre import clog +from pylustre import utils +from pylipe import lipe_constant +from pylipe import lipe_find + +LOG = None +EXE_FPATH = None + +DONT_EVALUATE = "--dont-evaluate" + +# Please refer to lipe_expression_test.c for the usage of the RETVAL_* +# Success +RETVAL_SUCCESS = 0 +# Help message without any other operation +RETVAL_HELP = 1 +# The argument to this program is invalid +RETVAL_ARG_INVALID = 2 +# The attribute name is invalid +RETVAL_ATTR_NAME_INVALID = 3 +# The attribute value is invalid +RETVAL_ATTR_VALUE_INVALID = 4 +# The expression is invalid +RETVAL_EXPRESSION_INVALID = 5 +# The expression value is not expected +RETVAL_UNEXPECTED_VALUE = 6 +# Unexpected internal error +RETVAL_INTERNAL_ERROR = 7 +# Attribute has no value +RETVAL_ATTR_NO_VALUE = 8 + + +def test_expression_invalid(expression): + """ + Run lipe_expression_test on the expression + """ + test_expression(expression, expect_retval=RETVAL_EXPRESSION_INVALID) + + +def test_expression_no_evaluate(expression): + """ + Run lipe_expression_test on the expression + """ + test_expression(expression, dont_evaluate=True) + + +def test_expression(expression, expected_value=None, options="", + dont_evaluate=False, expect_retval=RETVAL_SUCCESS): + """ + Run lipe_expression_test on the expression + """ + if dont_evaluate: + no_evaluation_string = " " + DONT_EVALUATE + else: + no_evaluation_string = "" + + command = ("%s%s %s '%s'" % + (EXE_FPATH, no_evaluation_string, options, expression)) + + if not dont_evaluate: + command += " '%s'" % expected_value + + retval = utils.run(command) + if retval.cr_exit_status != expect_retval: + LOG.cl_error("unexpected ret [%s] (expected [%s]) of command " + "[%s], stdout = [%s], stderr = [%s], ", + retval.cr_exit_status, expect_retval, + command, retval.cr_stdout, retval.cr_stderr) + raise Exception("test failure") + else: + if options != "": + expression_string = "[%s] on [%s]" % (expression, options) + else: + expression_string = "[%s]" % expression + if expect_retval == RETVAL_SUCCESS: + if dont_evaluate: + LOG.cl_info("%s -> valid ... checked", expression_string) + else: + LOG.cl_info("%s == %s ... checked", expression_string, expected_value) + else: + LOG.cl_info("%s --> failure(%s) ... checked", expression_string, + expect_retval) + + +def test_size_expressions(attribute="size", has_unit=True, bits=64): + """ + test expressions of size/blocks + """ + # pylint: disable=too-many-statements + sizes = [0, 1, 1023, 1024, 65534] + if bits == 32: + # 1048576 = 1M, 1024 * 1048576 = 1G = 2^30 + sizes += [65535, 65536, 1048576, 1024 * 1048576, 1024 * 1048576 * 4 - 2] + if bits == 64: + # 1048576 * 1048576 = 1T, 1048576 * 1048576 * 1024 = 1P, + # 1048576 * 1048576 * 1048576 = 1E = 2^60 + sizes += [1024 * 1048576 * 4 - 1, 1024 * 1048576 * 4, + 1024 * 1048576 * 4 + 1, 1048576 * 1048576, + 1048576 * 1048576 * 1024, 1048576 * 1048576 * 1048576] + for size in sizes: + find_size = (lipe_find.LipeFindSize.LFS_EXPRESSION_INFIX % (attribute, size)) + find_size_prefix = (lipe_find.LipeFindSize.LFS_EXPRESSION_PREFIX % (attribute, size)) + find_size_greater = (lipe_find.LipeFindSize.LFS_GREATER_EXPRESSION_INFIX % + (attribute, size)) + find_size_greater_prefix = (lipe_find.LipeFindSize.LFS_GREATER_EXPRESSION_PREFIX % + (attribute, size)) + find_size_less = (lipe_find.LipeFindSize.LFS_LESS_EXPRESSION_INFIX % + (attribute, size)) + find_size_less_prefix = (lipe_find.LipeFindSize.LFS_LESS_EXPRESSION_PREFIX % + (attribute, size)) + + test_expression(find_size, 1, options="--%s %s" % (attribute, size)) + test_expression(find_size_prefix, 1, options="--%s %s" % (attribute, size)) + test_expression(find_size_greater, 0, options="--%s %s" % (attribute, size)) + test_expression(find_size_greater_prefix, 0, options="--%s %s" % (attribute, size)) + test_expression(find_size_less, 0, options="--%s %s" % (attribute, size)) + test_expression(find_size_less_prefix, 0, options="--%s %s" % (attribute, size)) + + test_expression(find_size, 0, + options="--%s %s" % (attribute, (size + 1))) + test_expression(find_size_prefix, 0, + options="--%s %s" % (attribute, (size + 1))) + test_expression(find_size_greater, 1, + options="--%s %s" % (attribute, (size + 1))) + test_expression(find_size_greater_prefix, 1, + options="--%s %s" % (attribute, (size + 1))) + test_expression(find_size_less, 0, + options="--%s %s" % (attribute, (size + 1))) + test_expression(find_size_less_prefix, 0, + options="--%s %s" % (attribute, (size + 1))) + + if size < 1: + continue + test_expression(find_size, 0, + options="--%s %s" % (attribute, (size - 1))) + test_expression(find_size_prefix, 0, + options="--%s %s" % (attribute, (size - 1))) + test_expression(find_size_greater, 0, + options="--%s %s" % (attribute, (size - 1))) + test_expression(find_size_greater_prefix, 0, + options="--%s %s" % (attribute, (size - 1))) + test_expression(find_size_less, 1, + options="--%s %s" % (attribute, (size - 1))) + test_expression(find_size_less_prefix, 1, + options="--%s %s" % (attribute, (size - 1))) + + if not has_unit: + return + + unit_dict = {} + unit_dict["512"] = 512 + unit_dict["K"] = 1024 + unit_dict["M"] = 1048576 + unit_dict["G"] = 1048576 * 1024 + unit_dict["T"] = 1048576 * 1048576 + unit_dict["P"] = 1048576 * 1048576 * 1024 + unit_dict["E"] = 1048576 * 1048576 * 1048576 + for unit, size in unit_dict.iteritems(): + find_size = (lipe_find.LipeFindSize.LFS_UNIT_EXPRESSION_INFIX % + (attribute, 1, unit, attribute, 1, unit)) + find_size_prefix = (lipe_find.LipeFindSize.LFS_UNIT_EXPRESSION_PREFIX % + (attribute, 1, unit, attribute, 1, unit)) + find_size_greater = (lipe_find.LipeFindSize.LFS_GREATER_UNIT_EXPRESSION_INFIX % + (attribute, 1, unit)) + find_size_greater_prefix = (lipe_find.LipeFindSize.LFS_GREATER_UNIT_EXPRESSION_PREFIX % + (attribute, 1, unit)) + find_size_less = (lipe_find.LipeFindSize.LFS_LESS_UNIT_EXPRESSION_INFIX % + (attribute, 1, unit)) + find_size_less_prefix = (lipe_find.LipeFindSize.LFS_LESS_UNIT_EXPRESSION_PREFIX % + (attribute, 1, unit)) + + test_expression(find_size, 0, options="--%s 0" % attribute) + test_expression(find_size_prefix, 0, options="--%s 0" % attribute) + test_expression(find_size_greater, 0, options="--%s 0" % attribute) + test_expression(find_size_greater_prefix, 0, options="--%s 0" % attribute) + test_expression(find_size_less, 1, options="--%s 0" % attribute) + test_expression(find_size_less_prefix, 1, options="--%s 0" % attribute) + + test_expression(find_size, 0, + options="--%s %s" % (attribute, (size - 1))) + test_expression(find_size_prefix, 0, + options="--%s %s" % (attribute, (size - 1))) + test_expression(find_size_greater, 0, + options="--%s %s" % (attribute, (size - 1))) + test_expression(find_size_greater_prefix, 0, + options="--%s %s" % (attribute, (size - 1))) + test_expression(find_size_less, 1, + options="--%s %s" % (attribute, (size - 1))) + test_expression(find_size_less_prefix, 1, + options="--%s %s" % (attribute, (size - 1))) + + test_expression(find_size, 1, options="--%s %s" % (attribute, size)) + test_expression(find_size_prefix, 1, options="--%s %s" % (attribute, size)) + test_expression(find_size_greater, 0, options="--%s %s" % (attribute, size)) + test_expression(find_size_greater_prefix, 0, options="--%s %s" % (attribute, size)) + test_expression(find_size_less, 0, options="--%s %s" % (attribute, size)) + test_expression(find_size_less_prefix, 0, options="--%s %s" % (attribute, size)) + + test_expression(find_size, 1, + options="--%s %s" % (attribute, (size + 1))) + test_expression(find_size_prefix, 1, + options="--%s %s" % (attribute, (size + 1))) + test_expression(find_size_greater, 0, + options="--%s %s" % (attribute, (size + 1))) + test_expression(find_size_greater_prefix, 0, + options="--%s %s" % (attribute, (size + 1))) + test_expression(find_size_less, 0, + options="--%s %s" % (attribute, (size + 1))) + test_expression(find_size_less_prefix, 0, + options="--%s %s" % (attribute, (size + 1))) + + test_expression(find_size, 1, + options="--%s %s" % (attribute, (size * 2 - 1))) + test_expression(find_size_prefix, 1, + options="--%s %s" % (attribute, (size * 2 - 1))) + test_expression(find_size_greater, 0, + options="--%s %s" % (attribute, (size * 2 - 1))) + test_expression(find_size_greater_prefix, 0, + options="--%s %s" % (attribute, (size * 2 - 1))) + test_expression(find_size_less, 0, + options="--%s %s" % (attribute, (size * 2 - 1))) + test_expression(find_size_less_prefix, 0, + options="--%s %s" % (attribute, (size * 2 - 1))) + + test_expression(find_size, 0, + options="--%s %s" % (attribute, (size * 2))) + test_expression(find_size_prefix, 0, + options="--%s %s" % (attribute, (size * 2))) + test_expression(find_size_greater, 1, + options="--%s %s" % (attribute, (size * 2))) + test_expression(find_size_greater_prefix, 1, + options="--%s %s" % (attribute, (size * 2))) + test_expression(find_size_less, 0, + options="--%s %s" % (attribute, (size * 2))) + test_expression(find_size_less_prefix, 0, + options="--%s %s" % (attribute, (size * 2))) + + test_expression(find_size, 0, + options="--%s %s" % (attribute, (size * 2 + 1))) + test_expression(find_size_prefix, 0, + options="--%s %s" % (attribute, (size * 2 + 1))) + test_expression(find_size_greater, 1, + options="--%s %s" % (attribute, (size * 2 + 1))) + test_expression(find_size_greater_prefix, 1, + options="--%s %s" % (attribute, (size * 2 + 1))) + test_expression(find_size_less, 0, + options="--%s %s" % (attribute, (size * 2 + 1))) + test_expression(find_size_less_prefix, 0, + options="--%s %s" % (attribute, (size * 2 + 1))) + + test_expression(find_size, 0, + options="--%s %s" % (attribute, (size * 3))) + test_expression(find_size_prefix, 0, + options="--%s %s" % (attribute, (size * 3))) + test_expression(find_size_greater, 1, + options="--%s %s" % (attribute, (size * 3))) + test_expression(find_size_greater_prefix, 1, + options="--%s %s" % (attribute, (size * 3))) + test_expression(find_size_less, 0, + options="--%s %s" % (attribute, (size * 3))) + test_expression(find_size_less_prefix, 0, + options="--%s %s" % (attribute, (size * 3))) + + +def main_with_exception(): + """ + Run tests of expression evaluation, do no catch exception + """ + # pylint: disable=global-statement,too-many-statements,too-many-locals + global LOG, EXE_FPATH + LOG = clog.get_log(simple_console=True) + my_exe_fpath = os.path.abspath(sys.argv[0]) + dir_path = os.path.dirname(my_exe_fpath) + EXE_FPATH = dir_path + "/src/lipe_expression_test" + + # Option tests for lipe_expression_test + test_expression("1", options="-h", expect_retval=RETVAL_HELP) + test_expression("1", options="--help", expect_retval=RETVAL_HELP) + test_expression("1", options="--help", expect_retval=RETVAL_HELP) + + # Number tests + test_expression("4294967296", 4294967296) + test_expression("4294967297", 4294967297) + + # 2^64 = 18446744073709551616 + # 2^63 = 9223372036854775808 + # 2^32 = 4294967296 + # 2^31 = 2147483648 + test_expression("9223372036854775807", 9223372036854775807) + test_expression_invalid("9223372036854775808") + test_expression("-9223372036854775808", -9223372036854775808) + test_expression("-9223372036854775808 == -9223372036854775808", 1) + test_expression("== -9223372036854775808 -9223372036854775808", 1) + test_expression("9223372036854775807 + 1", -9223372036854775808) + test_expression("+ 9223372036854775807 1", -9223372036854775808) + test_expression("9223372036854775807 + 2", -9223372036854775807) + test_expression("+ 9223372036854775807 2", -9223372036854775807) + test_expression("9223372036854775807 + 3", -9223372036854775806) + test_expression("+ 9223372036854775807 3", -9223372036854775806) + test_expression("2147483648 * 2147483648 * 2", -9223372036854775808) + test_expression("* * 2147483648 2147483648 2", -9223372036854775808) + test_expression("4294967296 * 4294967296", 0) + test_expression("* 4294967296 4294967296", 0) + # Overflowed value won't be accepted + test_expression_invalid("18446744073709551615") + test_expression_invalid("18446744073709551616") + test_expression_invalid("18446744073709551617") + # Not allow to seperate - with the number + test_expression_invalid("- 2") + test_expression("-2", -2) + test_expression_invalid("1 - - 2") + test_expression_invalid("- 1 - 2") + test_expression("1 - -2", 3) + test_expression("- 1 -2", 3) + test_expression_invalid("1 * - 2") + test_expression_invalid("* 1 - 2") + test_expression("1 * -2", -2) + test_expression("* 1 -2", -2) + + # Base 16 support + # + # 2^64 = 0x10000000000000000 = 18446744073709551616 + # 2^63 = 0x8000000000000000 = 9223372036854775808 + # 2^32 = 0x100000000 = 4294967296 + # 2^31 = 0x08 = 2147483648 + test_expression("0x00", 0) + test_expression("-0x0", 0) + test_expression("-0x1", -1) + test_expression("-0x1 + 0", -1) + test_expression("+ -0x1 0", -1) + test_expression("-0x02 + -0", -2) + test_expression("+ -0x02 -0", -2) + test_expression("-0X02 + -0", -2) + test_expression("+ -0X02 -0", -2) + test_expression("0x7FFFFFFFFFFFFFFF", 9223372036854775807) + test_expression_invalid("0x8000000000000000") + test_expression("-0x7fffffffffffffff", -9223372036854775807) + test_expression("-0x7fffffffffffffff == -9223372036854775807", 1) + test_expression("== -0x7fffffffffffffff -9223372036854775807", 1) + test_expression("-0x8000000000000000", -9223372036854775808) + test_expression("-0x8000000000000000 == -9223372036854775808", 1) + test_expression("== -0x8000000000000000 -9223372036854775808", 1) + test_expression_invalid("-0x8000000000000001") + test_expression("-0x01 + 0x02", 1) + test_expression("+ -0x01 0x02", 1) + test_expression("-0x02 + 0x1", -1) + test_expression("+ -0x02 0x1", -1) + test_expression("0x1 - 0x2", -1) + test_expression("- 0x1 0x2", -1) + test_expression("0x1 - -0x2", 3) + test_expression("- 0x1 -0x2", 3) + test_expression("0xff + 0x1 == 0x100", 1) + test_expression("== + 0xff 0x1 0x100", 1) + test_expression("0xaf + 0x2 == 0xb1", 1) + test_expression("== + 0xaf 0x2 0xb1", 1) + + # Base 8 support + # + # 2^64 = 02000000000000000000000 = 18446744073709551616 + # 0123456789012345678901 + # 2^63 = 01000000000000000000000 = 9223372036854775808 + # 0123456789012345678901 + # 2^63 - 1 = 0777777777777777777777 = 9223372036854775807 + # 0123456789012345678901 + # 2^32 = 040000000000 = 4294967296 + # 01234567890 + test_expression("00", 0) + test_expression("-00", 0) + test_expression("-01", -1) + test_expression("-01 + 0", -1) + test_expression("+ -01 0", -1) + test_expression("-02 + -0", -2) + test_expression("+ -02 -0", -2) + test_expression("-02 + -0", -2) + test_expression("+ -02 -0", -2) + test_expression("-07 + -0", -7) + test_expression("+ -07 -0", -7) + test_expression_invalid("08") + test_expression_invalid("09") + test_expression_invalid("0A") + test_expression_invalid("0a") + test_expression("0777777777777777777777", 9223372036854775807) + test_expression_invalid("01000000000000000000000") + test_expression("-0777777777777777777777", -9223372036854775807) + test_expression("-0777777777777777777777 == -9223372036854775807", 1) + test_expression(" == -0777777777777777777777 -9223372036854775807", 1) + test_expression("-01000000000000000000000", -9223372036854775808) + test_expression("-01000000000000000000000 == -9223372036854775808", 1) + test_expression("== -01000000000000000000000 -9223372036854775808", 1) + test_expression("-01 + 02", 1) + test_expression("+ -01 02", 1) + test_expression("-02 + 01", -1) + test_expression("+ -02 01", -1) + test_expression("01 - 02", -1) + test_expression("- 01 02", -1) + test_expression("01 - -02", 3) + test_expression("- 01 -02", 3) + test_expression("077 + 01 == 0100", 1) + test_expression("== + 077 01 0100", 1) + test_expression("067 + 012 == 0101", 1) + test_expression("== + 067 012 0101", 1) + test_expression_no_evaluate('(mode & S_IRWXG) == 060') + test_expression_no_evaluate('== & mode S_IRWXG 060') + test_expression_no_evaluate('(mode & (S_IRWXU | S_IRWXG | S_IRWXO)) == 0770') + test_expression_no_evaluate('== & mode | | S_IRWXU S_IRWXG S_IRWXO 0770') + + # !!! Do not crash when divided by 0 + test_expression("1 / 0", 9223372036854775807) + test_expression("/ 1 0", 9223372036854775807) + test_expression("99 / 0", 9223372036854775807) + test_expression("/ 99 0", 9223372036854775807) + test_expression("9223372036854775807 / 0", 9223372036854775807) + test_expression("/ 9223372036854775807 0", 9223372036854775807) + test_expression("1 % 0", 1) + test_expression("% 1 0", 1) + test_expression("99 % 0", 99) + test_expression("% 99 0", 99) + test_expression("9223372036854775807 % 0", 9223372036854775807) + test_expression("% 9223372036854775807 0", 9223372036854775807) + + # Bool + test_expression("100 == 100", 1) + test_expression("== 100 100", 1) + test_expression("99 == 100", 0) + test_expression("== 99 100", 0) + + test_expression("1 + 2", 3) + test_expression("1 + 2", 3) + test_expression("+ 1 2", 3) + + # Byte constants + test_expression("KB", 1000) + test_expression("MB", 1000 * 1000) + test_expression("GB", 1000 * 1000 * 1000) + test_expression("TB", 1000 * 1000 * 1000 * 1000) + test_expression("PB", 1000 * 1000 * 1000 * 1000 * 1000) + test_expression("KB * 1000 == MB", 1) + test_expression("MB * 1000 == GB", 1) + test_expression("GB * 1000 == TB", 1) + test_expression("TB * 1000 == PB", 1) + test_expression("K", 1024) + test_expression("M", 1024 * 1024) + test_expression("G", 1024 * 1024 * 1024) + test_expression("T", 1024 * 1024 * 1024 * 1024) + test_expression("P", 1024 * 1024 * 1024 * 1024 * 1024) + test_expression("K * 1024 == M", 1) + test_expression("== * K 1024 M", 1) + test_expression("M * 1024 == G", 1) + test_expression("== * M 1024 G", 1) + test_expression("G * 1024 == T", 1) + test_expression("== * G 1024 T", 1) + test_expression("T * 1024 == P", 1) + test_expression("== * T 1024 P", 1) + test_expression("KB + 24 == K", 1) + test_expression("== + KB 24 K", 1) + + # Time constants + test_expression("second", 1000) + test_expression("minute", 60 * 1000) + test_expression("hour", 3600 * 1000) + test_expression("day", 24 * 3600 * 1000) + test_expression("week", 7 * 24 * 3600 * 1000) + test_expression("month", 30 * 24 * 3600 * 1000) + test_expression("year", 365 * 24 * 3600 * 1000) + test_expression("seconds", 1000) + test_expression("minutes", 60 * 1000) + test_expression("hours", 3600 * 1000) + test_expression("days", 24 * 3600 * 1000) + test_expression("weeks", 7 * 24 * 3600 * 1000) + test_expression("months", 30 * 24 * 3600 * 1000) + test_expression("years", 365 * 24 * 3600 * 1000) + test_expression("s", 1000) + test_expression("m", 60 * 1000) + test_expression("h", 3600 * 1000) + test_expression("d", 24 * 3600 * 1000) + test_expression("w", 7 * 24 * 3600 * 1000) + test_expression("y", 365 * 24 * 3600 * 1000) + test_expression("10 * seconds", 10 * 1000) + test_expression("15 * minutes", 15 * 60 * 1000) + test_expression("8 * hours", 8 * 3600 * 1000) + test_expression("12 * days", 12 * 24 * 3600 * 1000) + test_expression("7 * weeks", 7 * 7 * 24 * 3600 * 1000) + test_expression("4 * months", 4 * 30 * 24 * 3600 * 1000) + test_expression("2 * years", 2 * 365 * 24 * 3600 * 1000) + test_expression("90 * s == m / 2 * 3", 1) + test_expression("60 * seconds == minute", 1) + test_expression("90 * minutes == hour * 3 / 2", 1) + test_expression("72 * hours == 3 * days", 1) + test_expression("7 * days == week", 1) + test_expression("4 * weeks + 2 * days == month", 1) + test_expression("2 * months == 60 * days", 1) + test_expression("12 * months + 5 * day == year", 1) + test_expression("52 * weeks + 1 * day == year", 1) + + # Functions + test_expression_invalid('fname_reg') + test_expression_invalid('fname_reg(') + test_expression_invalid('fname_reg(x') + test_expression_invalid('fname_reg(x)') + test_expression_invalid('fname_reg(x) ') + test_expression_invalid('fname_reg("') + test_expression_invalid('fname_reg("x') + test_expression_invalid('fname_reg("x"') + test_expression_invalid('fname_reg("x)') + test_expression_invalid('fname_reg(x")') + test_expression_invalid('fname_reg("x" ') + # Escaped \" should not be considered as the end. \\ will be transfered + # to \, so the actual expression is fname_reg("x\"), which is invalid. + test_expression_invalid('fname_reg("x\\")') + # fname_reg(\"x") is invalid + test_expression_invalid('fname_reg(\\"x")') + # fname_reg("\") is invalid + test_expression_invalid('fname_match("\\")') + # fname_reg("\\\") is invalid + test_expression_invalid('fname_match("\\\\\\")') + # fname_reg("\\\\\") is invalid + test_expression_invalid('fname_match("\\\\\\\\\\")') + test_expression_invalid("fname_match('x')") + + test_expression_no_evaluate('fname_reg("x")') + test_expression_no_evaluate('fname_reg("x") ') + test_expression_no_evaluate('fname_reg("x") ') + test_expression_no_evaluate(' fname_reg("x")') + test_expression_no_evaluate(' fname_reg("x")') + test_expression_no_evaluate(' fname_reg("x") ') + test_expression_no_evaluate(' fname_reg("x") ') + test_expression_no_evaluate('fname_reg("x") + 1') + test_expression_no_evaluate('+ fname_reg("x") 1') + test_expression_no_evaluate('(fname_reg("x"))') + test_expression_no_evaluate('(fname_reg("x") )') + test_expression_no_evaluate('(fname_reg("x") )') + test_expression_no_evaluate('( fname_reg("x"))') + test_expression_no_evaluate('( fname_reg("x"))') + test_expression_no_evaluate('( fname_reg("x") )') + test_expression_no_evaluate('( fname_reg("x") )') + test_expression_no_evaluate('fname_reg( "x")') + test_expression_no_evaluate('fname_reg( "x")') + test_expression_no_evaluate('fname_reg("x" )') + test_expression_no_evaluate('fname_reg("x" )') + test_expression_no_evaluate('fname_reg( "x" )') + test_expression_no_evaluate('fname_reg( "x" )') + test_expression_no_evaluate('fname_reg( "x" )') + test_expression_no_evaluate('fname_reg("x ")') + test_expression_no_evaluate('fname_reg("x ")') + test_expression_no_evaluate('fname_reg(" x")') + test_expression_no_evaluate('fname_reg(" x ")') + test_expression_no_evaluate('fname_reg(" x")') + test_expression_no_evaluate('fname_reg(" x ")') + test_expression_no_evaluate('fname_reg(" x ")') + test_expression_no_evaluate('fname_reg(".+")') + # testing escape of \". \\ will be changed to \, thus actual expression is + # fname_reg("\""). + test_expression_no_evaluate('fname_reg("\\"")') + test_expression_no_evaluate('fname_reg("\\"\\"")') + test_expression_no_evaluate('fname_reg("\\"\\"\\"\\"")') + test_expression_no_evaluate('fname_reg("\\"\\"\\"\\"\\"")') + # testing escape of \\, \\ will be changed to \, thus actual expression is + # fname_match("\\"). And this actually matches file that has name \. Note that + # \ is not a valid expression, so fname_reg() would fail. + test_expression_no_evaluate('fname_match("\\\\")') + # fname_match("\\\\") is valid + test_expression_no_evaluate('fname_match("\\\\\\\\")') + # fname_match("\\\\\\") is valid + test_expression_no_evaluate('fname_match("\\\\\\\\\\\\")') + # fname_match("\\x\\") is valid + test_expression_no_evaluate('fname_match("\\\\x\\\\")') + # fname_match("\\x\x\\") is valid + test_expression_no_evaluate('fname_match("\\\\x\\x\\\\")') + # fname_match("\\x\x\x\x\x\\") is valid + test_expression_no_evaluate('fname_match("\\\\x\\x\\x\\x\\x\\\\")') + test_expression_no_evaluate('fname_match("fname")') + test_expression_no_evaluate('fname_match("fname*")') + test_expression_no_evaluate('fname_match("fname?0")') + test_expression_no_evaluate('fname_match("fname[0-9]")') + test_expression_no_evaluate('fname_match("fname[A-Z]")') + test_expression_no_evaluate('fname_match("fname[^0-8]")') + test_expression_no_evaluate('fname_match("fname[!a-d]")') + test_expression_no_evaluate('fname_match("fname[]4]")') + + # ost() tests + test_expression_invalid('ost') + test_expression_invalid('ost(') + test_expression_invalid('ost(0') + test_expression_invalid('ost()') + test_expression_invalid('ost)') + test_expression_invalid('ost("') + test_expression_invalid('ost(""') + test_expression_invalid('ost("")') + test_expression_invalid('ost("0")') + test_expression_invalid('ost(-1)') + test_expression_invalid('ost(-9223372036854775807)') + test_expression_invalid('ost(-9223372036854775808)') + test_expression_invalid('ost(9223372036854775808)') + test_expression_no_evaluate('ost(9223372036854775807)') + test_expression_no_evaluate('ost(0)') + test_expression_no_evaluate('ost( 0) ') + test_expression_no_evaluate('ost(0 ) ') + test_expression_no_evaluate('ost( 0 ) ') + test_expression_no_evaluate('ost( 0 ) ') + test_expression_no_evaluate('ost(0) + 1') + test_expression_no_evaluate('+ ost(0) 1') + test_expression_no_evaluate('ost(0xFF)') + test_expression_no_evaluate('ost(0xEA)') + test_expression_no_evaluate('ost(0x000a)') + test_expression_no_evaluate('ost(07)') + test_expression_no_evaluate('ost(0777)') + test_expression_invalid("ost(08)") + test_expression_invalid("ost(0a)") + + # pool_match() tests + test_expression_no_evaluate('pool_match("pool")') + test_expression_no_evaluate('pool_match("pool*")') + test_expression_no_evaluate('pool_match("pool?0")') + test_expression_no_evaluate('pool_match("pool[0-9]")') + test_expression_no_evaluate('pool_match("pool9[A-Z]")') + test_expression_no_evaluate('pool_match("pool9[^0-8]")') + test_expression_no_evaluate('pool_match("pool9[!1-3]")') + test_expression_no_evaluate('pool_match("pool0[]4]")') + + # pool_reg() tests + test_expression_no_evaluate('pool_match("pool")') + test_expression_no_evaluate('pool_match("pool.*")') + test_expression_no_evaluate('pool_match("pool.+")') + + # user() tests + test_expression_no_evaluate('user("root")') + test_expression_no_evaluate('user("0")') + # user() doesn't support base 16 + test_expression_invalid('user("0x0")') + test_expression_invalid('user("0xA")') + test_expression_invalid('user("this_user_doesnot_exist")') + + # group() tests + test_expression_no_evaluate('group("root")') + test_expression_no_evaluate('group("0")') + test_expression_invalid('group(0)') + # group() doesn't support base 16 number + test_expression_invalid('group(root)') + test_expression_invalid('group("0x0")') + test_expression_invalid('group("0xA")') + test_expression_invalid('group("this_group_doesnot_exist")') + + # user2id() tests + test_expression('user2id("root")', 0) + test_expression('user2id("root") == uid', 1, options="--uid 0") + test_expression('user2id("root") == uid', 0, options="--uid 100") + + # group2id() tests + test_expression('group2id("root")', 0) + test_expression('group2id("root") == gid', 1, options="--gid 0") + test_expression('group2id("root") == gid', 0, options="--gid 100") + + # xattr_match() tests - test_expression_no_evaluate('xattr_match("value", "name")') - test_expression_no_evaluate('xattr_match("value*", "name")') - test_expression_no_evaluate('xattr_match("value*", "name*")') - test_expression_no_evaluate('xattr_match("*value*", "*name*")') - test_expression_no_evaluate('xattr_match("value" , "*name*")') - test_expression_no_evaluate('xattr_match("value" , "name")') - test_expression_no_evaluate('xattr_match("value","name")') - test_expression_no_evaluate('xattr_match( "value","name")') - test_expression_no_evaluate('xattr_match( "value","name")') - test_expression_no_evaluate('xattr_match("value","name" )') - test_expression_no_evaluate('xattr_match("value","name" )') - # xattr_match("\"value", "name") - test_expression_no_evaluate('xattr_match("\\\"value", "name")') - # xattr_match("value\"", "name") - test_expression_no_evaluate('xattr_match("value\\\"", "name")') - # xattr_match("value", "\"name") - test_expression_no_evaluate('xattr_match("value", "\\\"name")') - # xattr_match("value", "\"name\"") - test_expression_no_evaluate('xattr_match("value", "name\\\"")') - # xattr_match("\\value", "name") - test_expression_no_evaluate('xattr_match("\\\\value", "name")') - # xattr_match("value", "\\name") - test_expression_no_evaluate('xattr_match("value", "\\\\name")') - # xattr_match("\\value\\", "\\name\\") - test_expression_no_evaluate('xattr_match("\\\\value\\\\", "\\\\name\\\\")') - test_expression_invalid('xattr_match(value, name)') - test_expression_invalid('xattr_match(value, "name")') - test_expression_invalid('xattr_match("value", name)') - test_expression_invalid('xattr_match("value", "name)') - test_expression_invalid('xattr_match("value", name")') - test_expression_invalid('xattr_match("value" "name")') - test_expression_invalid('xattr_match"value", "name")') - test_expression_invalid('xattr_match("value", "name"') - test_expression_invalid('xattr_match("value")') - test_expression_invalid('xattr_match(value)') - test_expression_invalid('xattr_match(\\"value", "name")') - test_expression_invalid('xattr_match("value\\", "name")') - test_expression_invalid('xattr_match("value", \\"name")') - test_expression_invalid('xattr_match("value", "name\\")') ++ # xattr_match() tests ++ test_expression_no_evaluate('xattr_match("name", "value")') ++ test_expression_no_evaluate('xattr_match("name*", "value")') ++ test_expression_no_evaluate('xattr_match("name*", "value*")') ++ test_expression_no_evaluate('xattr_match("*name*", "*value*")') ++ test_expression_no_evaluate('xattr_match("name" , "*value*")') ++ test_expression_no_evaluate('xattr_match("name" , "value")') ++ test_expression_no_evaluate('xattr_match("name","value")') ++ test_expression_no_evaluate('xattr_match( "name","value")') ++ test_expression_no_evaluate('xattr_match( "name","value")') ++ test_expression_no_evaluate('xattr_match("name","value" )') ++ test_expression_no_evaluate('xattr_match("name","value" )') ++ # xattr_match("\"name", "value") ++ test_expression_no_evaluate('xattr_match("\\\"name", "value")') ++ # xattr_match("name\"", "value") ++ test_expression_no_evaluate('xattr_match("name\\\"", "value")') ++ # xattr_match("name", "\"value") ++ test_expression_no_evaluate('xattr_match("name", "\\\"value")') ++ # xattr_match("name", "\"value\"") ++ test_expression_no_evaluate('xattr_match("name", "value\\\"")') ++ # xattr_match("\\name", "value") ++ test_expression_no_evaluate('xattr_match("\\\\name", "value")') ++ # xattr_match("name", "\\value") ++ test_expression_no_evaluate('xattr_match("name", "\\\\value")') ++ # xattr_match("\\name\\", "\\value\\") ++ test_expression_no_evaluate('xattr_match("\\\\name\\\\", "\\\\value\\\\")') ++ test_expression_invalid('xattr_match(name, value)') ++ test_expression_invalid('xattr_match(name, "value")') ++ test_expression_invalid('xattr_match("name", value)') ++ test_expression_invalid('xattr_match("name", "value)') ++ test_expression_invalid('xattr_match("name", value")') ++ test_expression_invalid('xattr_match("name" "value")') ++ test_expression_invalid('xattr_match"name", "value")') ++ test_expression_invalid('xattr_match("name", "value"') ++ test_expression_invalid('xattr_match("name")') ++ test_expression_invalid('xattr_match(name)') ++ test_expression_invalid('xattr_match(\\"name", "value")') ++ test_expression_invalid('xattr_match("name\\", "value")') ++ test_expression_invalid('xattr_match("name", \\"value")') ++ test_expression_invalid('xattr_match("name", "value\\")') ++ ++ # xattr_name() tests ++ test_expression_no_evaluate('xattr_name("name")') ++ test_expression_no_evaluate('xattr_name("name*")') ++ test_expression_no_evaluate('xattr_name("name*")') ++ test_expression_no_evaluate('xattr_name("*name*")') ++ test_expression_no_evaluate('xattr_name("name" )') ++ test_expression_no_evaluate('xattr_name("name" )') ++ test_expression_no_evaluate('xattr_name( "name")') ++ test_expression_no_evaluate('xattr_name( "name")') ++ test_expression_no_evaluate('xattr_name("name")') ++ test_expression_no_evaluate('xattr_name("name")') ++ # xattr_name("\"name")') ++ test_expression_no_evaluate('xattr_name("\\\"name")') ++ # xattr_name("name\"")') ++ test_expression_no_evaluate('xattr_name("name\\\"")') ++ # xattr_name("name")') ++ test_expression_no_evaluate('xattr_name("name")') ++ # xattr_name("name")') ++ test_expression_no_evaluate('xattr_name("name")') ++ # xattr_name("\\name")') ++ test_expression_no_evaluate('xattr_name("\\\\name")') ++ # xattr_name("name")') ++ test_expression_no_evaluate('xattr_name("name")') ++ # xattr_name("\\name\\")') ++ test_expression_no_evaluate('xattr_name("\\\\name\\\\")') ++ test_expression_invalid('xattr_name(name)') ++ test_expression_invalid('xattr_name(name)') ++ test_expression_invalid('xattr_name("name", "value")') ++ test_expression_invalid('xattr_name("name" "value")') ++ test_expression_invalid('xattr_name("name",)') ++ test_expression_invalid('xattr_name"name")') ++ test_expression_invalid('xattr_name(name)') ++ test_expression_invalid('xattr_name(\\"name")') ++ test_expression_invalid('xattr_name("name\\")') + + test_expression_invalid('perm_equal()') + test_expression_invalid('perm_equal(8)') + test_expression_invalid('perm_equal(0778)') + test_expression_invalid('perm_equal(0877)') + test_expression_invalid('perm_equal(010000)') + test_expression_invalid('perm_equal("07777")') + + test_expression('perm_equal(07777)', 1, options="--mode 07777") + test_expression('perm_equal(07777)', 0, options="--mode 07776") + test_expression('perm_equal(04567)', 1, options="--mode 04567") + test_expression('perm_equal(03567)', 0, options="--mode 04567") + test_expression('perm_equal(04467)', 0, options="--mode 04567") + test_expression('perm_equal(04577)', 0, options="--mode 04567") + + test_expression_invalid('perm_at_least()') + test_expression_invalid('perm_at_least(8)') + test_expression_invalid('perm_at_least(0778)') + test_expression_invalid('perm_at_least(0877)') + test_expression_invalid('perm_at_least(010000)') + test_expression_invalid('perm_at_least("07777")') + test_expression('perm_at_least(07777)', 1, options="--mode 07777") + test_expression('perm_at_least(07777)', 0, options="--mode 07776") + test_expression('perm_at_least(07776)', 1, options="--mode 07777") + test_expression('perm_at_least(04242)', 1, options="--mode 06666") + test_expression('perm_at_least(02424)', 1, options="--mode 06666") + test_expression('perm_at_least(03424)', 0, options="--mode 06666") + + test_expression_invalid('perm_any()') + test_expression_invalid('perm_any(8)') + test_expression_invalid('perm_any(0778)') + test_expression_invalid('perm_any(0877)') + test_expression_invalid('perm_any(010000)') + test_expression_invalid('perm_any("07777")') + test_expression('perm_any(07777)', 1, options="--mode 07777") + test_expression('perm_any(07777)', 1, options="--mode 01000") + test_expression('perm_any(01010)', 1, options="--mode 07006") + test_expression('perm_any(01010)', 1, options="--mode 01000") + test_expression('perm_any(01010)', 1, options="--mode 00010") + test_expression('perm_any(00000)', 1, options="--mode 07777") + test_expression('perm_any(00000)', 1, options="--mode 00000") + test_expression('perm_any(01000)', 0, options="--mode 00777") + test_expression('perm_any(00001)', 0, options="--mode 07770") + + test_expression_no_evaluate('layout("released")') + test_expression_no_evaluate('layout("raid0")') + test_expression_no_evaluate('layout("mdt")') + test_expression_no_evaluate('layout("mdt,released")') + test_expression_no_evaluate('layout("mdt,raid0,released")') + test_expression_invalid('layout(",")') + test_expression_invalid('layout("released,")') + test_expression_invalid('layout("raid0x")') + test_expression_invalid('layout(",mdt")') + test_expression_invalid('layout("released, raid0")') + test_expression_invalid('layout("raid0xraid0")') + test_expression_invalid('layout("mdt raid0")') + # Operator precedences: + # + # 9: * / % + # 8: + - + # 7: >> << + # 6: < > <= >= + # 5: == != + # 4: & + # 3: ^ + # 2: | + # 1: && + # 0: || + + test_expression("1 + 2 * 3", 7) + test_expression("+ 1 * 2 3", 7) + + test_expression("1 + -2 * 3", -5) + test_expression("+ 1 * -2 3", -5) + test_expression("1 + -2 * -3", 7) + test_expression("+ 1 * -2 -3", 7) + test_expression("-1 + -2 * -3", 5) + test_expression("+ -1 * -2 -3", 5) + + test_expression("1 + 6 / 2", 4) + test_expression("+ 1 / 6 2", 4) + + test_expression("1 + 6 % 2", 1) + test_expression("+ 1 % 6 2", 1) + + # + - from left to right + test_expression("3 + 2 - 1", 4) + test_expression("- + 3 2 1", 4) + + # + - have greater precedences than >> << + test_expression("1 + 1 << 1", 4) + test_expression("<< + 1 1 1", 4) + test_expression("2 - 1 << 1", 2) + test_expression("<< - 2 1 1", 2) + test_expression("4 + 4 >> 1", 4) + test_expression(">> + 4 4 1", 4) + test_expression("8 - 4 >> 1", 2) + test_expression(">> - 8 4 1", 2) + + # >> << from left to right + # 3 >> 1: 1 + test_expression("4 << 3 >> 1", 16) + test_expression(">> << 4 3 1", 16) + test_expression("4 << (3 >> 1)", 8) + test_expression("<< 4 >> 3 1", 8) + test_expression("4 >> 1 << 3", 16) + test_expression("<< >> 4 1 3", 16) + test_expression("4 >> (1 << 3)", 0) + test_expression(">> 4 << 1 3", 0) + + # >> << have greater precedences than < > <= >= + test_expression("8 >> 1 < 4 + 1", 1) + test_expression("< >> 8 1 + 4 1", 1) + test_expression("8 >> 1 < 4 - 1", 0) + test_expression("< >> 8 1 - 4 1", 0) + test_expression("8 >> 1 > 4 - 1", 1) + test_expression("> >> 8 1 - 4 1", 1) + test_expression("8 >> 1 > 4 + 0", 0) + test_expression("> >> 8 1 + 4 0", 0) + test_expression("8 >> 1 > 4 + 1", 0) + test_expression("> >> 8 1 + 4 1", 0) + test_expression("8 >> 1 <= 4 + 1", 1) + test_expression("<= >> 8 1 + 4 1", 1) + test_expression("8 >> 1 <= 4 + 0", 1) + test_expression("<= >> 8 1 + 4 0", 1) + test_expression("8 >> 1 <= 4 - 1", 0) + test_expression("<= >> 8 1 - 4 1", 0) + test_expression("8 >> 1 >= 4 - 1", 1) + test_expression(">= >> 8 1 - 4 1", 1) + test_expression("8 >> 1 >= 4 + 0", 1) + test_expression(">= >> 8 1 + 4 0", 1) + test_expression("8 >> 1 >= 4 + 1", 0) + test_expression(">= >> 8 1 + 4 1", 0) + + # >> << have greater precedences than < > <= >= + test_expression("8 << 1 < 16 + 1", 1) + test_expression("< << 8 1 + 16 1", 1) + test_expression("8 << 1 < 16 - 1", 0) + test_expression("< << 8 1 - 16 1", 0) + test_expression("8 << 1 > 16 - 1", 1) + test_expression("> << 8 1 - 16 1", 1) + test_expression("8 << 1 > 16 + 0", 0) + test_expression("> << 8 1 + 16 0", 0) + test_expression("8 << 1 > 16 + 1", 0) + test_expression("> << 8 1 + 16 1", 0) + test_expression("8 << 1 <= 16 + 1", 1) + test_expression("<= << 8 1 + 16 1", 1) + test_expression("8 << 1 <= 16 + 0", 1) + test_expression("<= << 8 1 + 16 0", 1) + test_expression("8 << 1 <= 16 - 1", 0) + test_expression("<= << 8 1 - 16 1", 0) + test_expression("8 << 1 >= 16 - 1", 1) + test_expression(">= << 8 1 - 16 1", 1) + test_expression("8 << 1 >= 16 + 0", 1) + test_expression(">= << 8 1 + 16 0", 1) + test_expression("8 << 1 >= 16 + 1", 0) + test_expression(">= << 8 1 + 16 1", 0) + + # < > <= >= have greater precedences than == != + # Test < and == + test_expression("1 == 3 < 3 + 1", 1) + test_expression("== 1 < 3 + 3 1", 1) + test_expression("0 == 3 < 3 + 1", 0) + test_expression("== 0 < 3 + 3 1", 0) + test_expression("1 == 3 < 3 - 1", 0) + test_expression("== 1 < 3 - 3 1", 0) + test_expression("0 == 3 < 3 - 1", 1) + test_expression("== 0 < 3 - 3 1", 1) + test_expression("1 == 3 < 3 + 0", 0) + test_expression("== 1 < 3 + 3 0", 0) + test_expression("0 == 3 < 3 + 0", 1) + test_expression("== 0 < 3 + 3 0", 1) + + # < > <= >= have greater precedences than == != + # Test > and == + test_expression("1 == 3 > 3 + 1", 0) + test_expression("== 1 > 3 + 3 1", 0) + test_expression("0 == 3 > 3 + 1", 1) + test_expression("== 0 > 3 + 3 1", 1) + test_expression("1 == 3 > 3 - 1", 1) + test_expression("== 1 > 3 - 3 1", 1) + test_expression("0 == 3 > 3 - 1", 0) + test_expression("== 0 > 3 - 3 1", 0) + test_expression("1 == 3 > 3 + 0", 0) + test_expression("== 1 > 3 + 3 0", 0) + test_expression("0 == 3 > 3 + 0", 1) + test_expression("== 0 > 3 + 3 0", 1) + + # < > <= >= have greater precedences than == != + # Test <= and == + test_expression("1 == 3 <= 3 + 1", 1) + test_expression("== 1 <= 3 + 3 1", 1) + test_expression("0 == 3 <= 3 + 1", 0) + test_expression("== 0 <= 3 + 3 1", 0) + test_expression("1 == 3 <= 3 - 1", 0) + test_expression("== 1 <= 3 - 3 1", 0) + test_expression("0 == 3 <= 3 - 1", 1) + test_expression("== 0 <= 3 - 3 1", 1) + test_expression("1 == 3 <= 3 + 0", 1) + test_expression("== 1 <= 3 + 3 0", 1) + test_expression("0 == 3 <= 3 + 0", 0) + test_expression("== 0 <= 3 + 3 0", 0) + + # < > <= >= have greater precedences than == != + # Test >= and == + test_expression("1 == 3 >= 3 + 1", 0) + test_expression("== 1 >= 3 + 3 1", 0) + test_expression("0 == 3 >= 3 + 1", 1) + test_expression("== 0 >= 3 + 3 1", 1) + test_expression("1 == 3 >= 3 - 1", 1) + test_expression("== 1 >= 3 - 3 1", 1) + test_expression("0 == 3 >= 3 - 1", 0) + test_expression("== 0 >= 3 - 3 1", 0) + test_expression("1 == 3 >= 3 + 0", 1) + test_expression("== 1 >= 3 + 3 0", 1) + test_expression("0 == 3 >= 3 + 0", 0) + test_expression("== 0 >= 3 + 3 0", 0) + + # < > <= >= have greater precedences than == != + # Test < and != + test_expression("1 != 3 < 3 + 1", 0) + test_expression("!= 1 < 3 + 3 1", 0) + test_expression("0 != 3 < 3 + 1", 1) + test_expression("!= 0 < 3 + 3 1", 1) + test_expression("1 != 3 < 3 - 1", 1) + test_expression("!= 1 < 3 - 3 1", 1) + test_expression("0 != 3 < 3 - 1", 0) + test_expression("!= 0 < 3 - 3 1", 0) + test_expression("1 != 3 < 3 + 0", 1) + test_expression("!= 1 < 3 + 3 0", 1) + test_expression("0 != 3 < 3 + 0", 0) + test_expression("!= 0 < 3 + 3 0", 0) + + # < > <= >= have greater precedences than == != + # Test > and != + test_expression("1 != 3 > 3 + 1", 1) + test_expression("!= 1 > 3 + 3 1", 1) + test_expression("0 != 3 > 3 + 1", 0) + test_expression("!= 0 > 3 + 3 1", 0) + test_expression("1 != 3 > 3 - 1", 0) + test_expression("!= 1 > 3 - 3 1", 0) + test_expression("0 != 3 > 3 - 1", 1) + test_expression("!= 0 > 3 - 3 1", 1) + test_expression("1 != 3 > 3 + 0", 1) + test_expression("!= 1 > 3 + 3 0", 1) + test_expression("0 != 3 > 3 + 0", 0) + test_expression("!= 0 > 3 + 3 0", 0) + + # < > <= >= have greater precedences than == != + # Test <= and != + test_expression("1 != 3 <= 3 + 1", 0) + test_expression("!= 1 <= 3 + 3 1", 0) + test_expression("0 != 3 <= 3 + 1", 1) + test_expression("!= 0 <= 3 + 3 1", 1) + test_expression("1 != 3 <= 3 - 1", 1) + test_expression("!= 1 <= 3 - 3 1", 1) + test_expression("0 != 3 <= 3 - 1", 0) + test_expression("!= 0 <= 3 - 3 1", 0) + test_expression("1 != 3 <= 3 + 0", 0) + test_expression("!= 1 <= 3 + 3 0", 0) + test_expression("0 != 3 <= 3 + 0", 1) + test_expression("!= 0 <= 3 + 3 0", 1) + + # < > <= >= have greater precedences than == != + # Test >= and != + test_expression("1 != 3 >= 3 + 1", 1) + test_expression("!= 1 >= 3 + 3 1", 1) + test_expression("0 != 3 >= 3 + 1", 0) + test_expression("!= 0 >= 3 + 3 1", 0) + test_expression("1 != 3 >= 3 - 1", 0) + test_expression("!= 1 >= 3 - 3 1", 0) + test_expression("0 != 3 >= 3 - 1", 1) + test_expression("!= 0 >= 3 - 3 1", 1) + test_expression("1 != 3 >= 3 + 0", 0) + test_expression("!= 1 >= 3 + 3 0", 0) + test_expression("0 != 3 >= 3 + 0", 1) + test_expression("!= 0 >= 3 + 3 0", 1) + + # == != have higher precedences than & + # 3: 0011 + # 6: 0110 + # 5: 0101 + # 6 & 5: 4 + # 5 & 3: 1 + test_expression("6 & 5 == 4", 0) + test_expression("& 6 == 5 4", 0) + test_expression("4 == 5 & 6", 0) + test_expression("& 4 == 5 6", 0) + test_expression("6 & 5 != 3", 0) + test_expression("& 6 != 5 3", 0) + test_expression("3 != 5 & 6", 0) + test_expression("& != 3 5 6", 0) + test_expression("6 & 5 != 4", 0) + test_expression("& 6 != 5 4", 0) + test_expression("4 != 5 & 6", 0) + test_expression("& != 4 5 6", 0) + test_expression("3 & 5 != 1", 1) + test_expression("& != 3 5 1", 1) + test_expression("1 != 5 & 3", 1) + test_expression("& != 1 5 3", 1) + test_expression("3 & 2 != 2", 0) + test_expression("& 3 != 2 2", 0) + test_expression("2 != 2 & 3", 0) + test_expression("& != 2 2 3", 0) + + # & has higher precedence than ^ + # 7: 0111 + # 6: 0110 + # 2: 0010 + # 6 ^ 7: 1 + # 3 & 6: 2 + # 2 ^ 1: 3 + # 2 ^ 7: 5 + test_expression("3 & 6 ^ 7", 5) + test_expression("^ & 3 6 7", 5) + test_expression("6 & 3 ^ 7", 5) + test_expression("^ & 6 3 7", 5) + test_expression("7 ^ 6 & 3", 5) + test_expression("^ 7 & 6 3", 5) + + # ^ has higher precedence than | + # 5: 0101 + # 6: 0110 + # 4: 0100 + # 5 ^ 6: 3 + # 3 | 4: 7 + # 6 | 4: 6 + # 6 ^ 7: 1 + test_expression("5 ^ 6 | 4", 7) + test_expression("| ^ 5 6 4", 7) + test_expression("6 ^ 5 | 4", 7) + test_expression("| ^ 6 5 4", 7) + test_expression("4 | 5 ^ 6", 7) + test_expression("| 4 ^ 5 6", 7) + test_expression("4 | 6 ^ 5", 7) + test_expression("| 4 ^ 6 5", 7) + + # | has higher precedence than && + # 5 | 0: 5 + # 5 && 4: 1 + test_expression("5 | 0 && 4", 1) + test_expression("&& | 5 0 4", 1) + test_expression("0 | 5 && 4", 1) + test_expression("&& | 0 5 4", 1) + test_expression("4 && 0 | 5", 1) + test_expression("&& 4 | 0 5", 1) + test_expression("4 && 5 | 0", 1) + test_expression("&& 4 | 5 0", 1) + + # && has higher precedence than || + test_expression("0 && 1 || 1", 1) + test_expression("|| && 0 1 1", 1) + test_expression("1 && 0 || 1", 1) + test_expression("|| && 1 0 1", 1) + test_expression("1 || 0 && 1", 1) + test_expression("|| 1 && 0 1", 1) + test_expression("1 || 1 && 0", 1) + test_expression("|| 1 && 1 0", 1) + + # Precedence sequence: + # 9: * / % + # 8: + - + # 7: >> << + # 6: < > <= >= + # 5: == != + # 4: & + # 3: ^ + # 2: | + # 1: && + # 0: || + test_expression("1 + 2 * 3 - 1", 6) + test_expression("- + 1 * 2 3 1", 6) + test_expression("2 << 1 + 2 * 3 - 1", 128) + test_expression("<< 2 - + 1 * 2 3 1", 128) + test_expression("2 << 1 + 2 * 3 - 1 >> 5", 4) + test_expression(">> << 2 - + 1 * 2 3 1 5", 4) + test_expression("3 < 2 << 1 + 2 * 3 - 1 >> 5", 1) + test_expression("< 3 >> << 2 - + 1 * 2 3 1 5", 1) + test_expression("3 < 2 << 1 + 2 * 3 - 1 >> 5 > 1", 0) + test_expression("> < 3 >> << 2 - + 1 * 2 3 1 5 1", 0) + test_expression("3 < 2 << 1 + 2 * 3 - 1 >> 5 > 1 <= 0", 1) + test_expression("<= > < 3 >> << 2 - + 1 * 2 3 1 5 1 0", 1) + test_expression("3 < 2 << 1 + 2 * 3 - 1 >> 5 > 1 <= 0 >= 100", 0) + test_expression(">= <= > < 3 >> << 2 - + 1 * 2 3 1 5 1 0 100", 0) + test_expression("0 == 3 < 2 << 1 + 2 * 3 - 1 >> 5 > 1 <= 0 >= 100", 1) + test_expression("== 0 >= <= > < 3 >> << 2 - + 1 * 2 3 1 5 1 0 100", 1) + test_expression("0 == 3 < 2 << 1 + 2 * 3 - 1 >> 5 > 1 <= 0 >= 100 != 0", 1) + test_expression("!= == 0 >= <= > < 3 >> << 2 - + 1 * 2 3 1 5 1 0 100 0", 1) + test_expression("7 & 0 == 3 < 2 << 1 + 2 * 3 - 1 >> 5 > 1 <= 0 >= 100 != 0", 1) + test_expression("& 7 != == 0 >= <= > < 3 >> << 2 - + 1 * 2 3 1 5 1 0 100 0", 1) + test_expression("3 ^ 7 & 0 == 3 < 2 << 1 + 2 * 3 - 1 >> 5 > 1 <= 0 >= 100 != 0", 2) + test_expression("^ 3 & 7 != == 0 >= <= > < 3 >> << 2 - + 1 * 2 3 1 5 1 0 100 0", 2) + test_expression("5 | 3 ^ 7 & 0 == 3 < 2 << 1 + 2 * 3 - 1 >> 5 > 1 <= 0 >= 100 != 0", 7) + test_expression("| 5 ^ 3 & 7 != == 0 >= <= > < 3 >> << 2 - + 1 * 2 3 1 5 1 0 100 0", 7) + test_expression("0 && 5 | 3 ^ 7 & 0 == 3 < 2 << 1 + 2 * 3 - 1 >> 5 > 1 <= 0 >= 100 != 0", 0) + test_expression("&& 0 | 5 ^ 3 & 7 != == 0 >= <= > < 3 >> << 2 - + 1 * 2 3 1 5 1 0 100 0", 0) + test_expression("4 || 0 && 5 | 3 ^ 7 & 0 == 3 < 2 << 1 + 2 * 3 - 1 >> 5 > 1 <= 0 >= 100 != 0", 1) + test_expression("|| 4 && 0 | 5 ^ 3 & 7 != == 0 >= <= > < 3 >> << 2 - + 1 * 2 3 1 5 1 0 100 0", 1) + + # Parenthesises + test_expression("(1 + 2)", 3) + test_expression("+ 1 2", 3) + test_expression("((1 + 2))", 3) + test_expression("+ 1 2", 3) + test_expression("(1 + 2) * 3", 9) + test_expression("* + 1 2 3", 9) + test_expression("(1 + 2) * (3 + 4)", 21) + test_expression("* + 1 2 + 3 4", 21) + test_expression("(1 + 2) * (((3 + 4)))", 21) + test_expression("* + 1 2 + 3 4", 21) + test_expression("((6 + 2) - (5 - 3)) * (3 - 1)", 12) + test_expression("* - + 6 2 - 5 3 - 3 1", 12) + test_expression("((6 + 2) - (5 - 3)) * ((3 - 1) / (3 - 2 + 1))", 6) + test_expression("* - + 6 2 - 5 3 / - 3 1 + - 3 2 1", 6) + + for attr in lipe_constant.LIPE_POLICY_ATTRIBUTES: + test_expression_no_evaluate(attr) + + # Attributes + for value in [12345691, 0]: + for attr in lipe_constant.LIPE_POLICY_ATTRIBUTES: + if attr == "type": + # type bits are filtered by 00170000 + expected_value = 00170000 & value + elif attr == "stripe_count": + # stripe count is 16 bit + value = 0xFFFF & value + expected_value = value + elif attr == "empty": + # empty is bool + value = int(bool(value)) + expected_value = value + else: + expected_value = value + test_expression(attr, expected_value, + options="--%s %s" % (attr, value)) + + # Contants + # S_IFMT 00170000 + # S_IFSOCK 0140000 + # S_IFLNK 0120000 + # S_IFREG 0100000 + # S_IFBLK 0060000 + # S_IFDIR 0040000 + # S_IFCHR 0020000 + # S_IFIFO 0010000 + # S_ISUID 0004000 + # S_ISGID 0002000 + # S_ISVTX 0001000 + test_expression("S_IFMT", 00170000) + test_expression("S_IFSOCK", 0140000) + test_expression("S_IFLNK", 0120000) + test_expression("S_IFREG", 0100000) + test_expression("S_IFBLK", 0060000) + test_expression("S_IFDIR", 0040000) + test_expression("S_IFCHR", 0020000) + test_expression("S_IFIFO", 0010000) + test_expression("S_ISUID", 0004000) + test_expression("S_ISGID", 0002000) + test_expression("S_ISVTX", 0001000) + + # Attribute tests + test_expression("1", options="--atime", expect_retval=RETVAL_ARG_INVALID) + test_expression("1", options="--atime x", expect_retval=RETVAL_ATTR_VALUE_INVALID) + test_expression("1", options="--atime 9223372036854775808", + expect_retval=RETVAL_ATTR_VALUE_INVALID) + test_expression("1", options="--nonexist_attr 1", expect_retval=RETVAL_ATTR_NAME_INVALID) + test_expression("hsm_states", expect_retval=RETVAL_ATTR_NO_VALUE) + # hsm_states is 32 bit, value larger than 32 bit will overflow + test_expression("hsm_states", 4294967295, options="--hsm_states 4294967295") + # S_IFSOCK 0140000 + # S_IFLNK 0120000 + # "type" won't overwrite other bits of "mode", only the limited bits + test_expression("mode", 0140740, options="--mode 0120740 --type 0140000") + test_expression("mode", 0140740, options="--mode 0110740 --type 0140036") + test_expression("sys_time", 1550310180000, options="--sys_time 1550310180000") + + for xtime in ["atime", "ctime", "mtime"]: + test_expression(xtime, expect_retval=RETVAL_ATTR_NO_VALUE) + test_expression(xtime, 9223372036854775807, options="--%s 9223372036854775807" % xtime) + test_expression(xtime, 9223372036854775807, options="--%s 9223372036854775807" % xtime) + test_expression(xtime, expect_retval=RETVAL_ATTR_VALUE_INVALID, + options="--%s 9223372036854775808" % xtime) + # Same attribute will overwrite the previous ones + test_expression(xtime, 1550202026000, + options="--%s 9223372036854775807 --%s 1550202026000" % (xtime, xtime)) + + # 1 hour: 3600000 + # 1 day: 86400000 + # 100 days: 8640000000 + # 90 days: 7776000000 + # 89 days: 7689600000 + find_xtime_10days = (lipe_find.LipeFindXtime.LFX_EXPRESSION_INFIX % + (xtime, 10, "days", xtime, 10, "days")) + find_xtime_10days_prefix = (lipe_find.LipeFindXtime.LFX_EXPRESSION_PREFIX % + (xtime, 10, "days", xtime, 10, "days")) + test_expression(find_xtime_10days, 0, + options="--sys_time 8640000000 --%s 7776000001" % xtime) + test_expression(find_xtime_10days_prefix, 0, + options="--sys_time 8640000000 --%s 7776000001" % xtime) + test_expression(find_xtime_10days, 1, + options="--sys_time 8640000000 --%s 7776000000" % xtime) + test_expression(find_xtime_10days_prefix, 1, + options="--sys_time 8640000000 --%s 7776000000" % xtime) + test_expression(find_xtime_10days, 1, + options="--sys_time 8640000000 --%s 7775999999" % xtime) + test_expression(find_xtime_10days_prefix, 1, + options="--sys_time 8640000000 --%s 7775999999" % xtime) + test_expression(find_xtime_10days, 0, + options="--sys_time 8640000000 --%s 7689600000" % xtime) + test_expression(find_xtime_10days_prefix, 0, + options="--sys_time 8640000000 --%s 7689600000" % xtime) + test_expression(find_xtime_10days, 1, + options="--sys_time 8640000000 --%s 7689600001" % xtime) + test_expression(find_xtime_10days_prefix, 1, + options="--sys_time 8640000000 --%s 7689600001" % xtime) + + find_xtime_greater_10days = (lipe_find.LipeFindXtime.LFX_GREATER_EXPRESSION_INFIX % + (xtime, 10, "days")) + find_xtime_greater_10days_prefix = (lipe_find.LipeFindXtime.LFX_GREATER_EXPRESSION_PREFIX % + (xtime, 10, "days")) + test_expression(find_xtime_greater_10days, 0, + options="--sys_time 8640000000 --%s 7776000001" % xtime) + test_expression(find_xtime_greater_10days_prefix, 0, + options="--sys_time 8640000000 --%s 7776000001" % xtime) + test_expression(find_xtime_greater_10days, 0, + options="--sys_time 8640000000 --%s 7776000000" % xtime) + test_expression(find_xtime_greater_10days_prefix, 0, + options="--sys_time 8640000000 --%s 7776000000" % xtime) + test_expression(find_xtime_greater_10days, 0, + options="--sys_time 8640000000 --%s 7775999999" % xtime) + test_expression(find_xtime_greater_10days_prefix, 0, + options="--sys_time 8640000000 --%s 7775999999" % xtime) + test_expression(find_xtime_greater_10days, 0, + options="--sys_time 8640000000 --%s 7689600000" % xtime) + test_expression(find_xtime_greater_10days_prefix, 0, + options="--sys_time 8640000000 --%s 7689600000" % xtime) + test_expression(find_xtime_greater_10days, 0, + options="--sys_time 8640000000 --%s 7689600001" % xtime) + test_expression(find_xtime_greater_10days_prefix, 0, + options="--sys_time 8640000000 --%s 7689600001" % xtime) + test_expression(find_xtime_greater_10days, 1, + options="--sys_time 8640000000 --%s 7689599999" % xtime) + test_expression(find_xtime_greater_10days_prefix, 1, + options="--sys_time 8640000000 --%s 7689599999" % xtime) + test_expression(find_xtime_greater_10days, 1, + options="--sys_time 8640000000 --%s 0" % xtime) + test_expression(find_xtime_greater_10days_prefix, 1, + options="--sys_time 8640000000 --%s 0" % xtime) + + find_xtime_less_10days = (lipe_find.LipeFindXtime.LFX_LESS_EXPRESSION_INFIX % + (xtime, 10, "days")) + find_xtime_less_10days_prefix = (lipe_find.LipeFindXtime.LFX_LESS_EXPRESSION_PREFIX % + (xtime, 10, "days")) + test_expression(find_xtime_less_10days, 0, + options="--sys_time 8640000000 --%s 7776000000" % xtime) + test_expression(find_xtime_less_10days_prefix, 0, + options="--sys_time 8640000000 --%s 7776000000" % xtime) + test_expression(find_xtime_less_10days, 0, + options="--sys_time 8640000000 --%s 7775999999" % xtime) + test_expression(find_xtime_less_10days_prefix, 0, + options="--sys_time 8640000000 --%s 7775999999" % xtime) + test_expression(find_xtime_less_10days, 1, + options="--sys_time 8640000000 --%s 7776000001" % xtime) + test_expression(find_xtime_less_10days_prefix, 1, + options="--sys_time 8640000000 --%s 7776000001" % xtime) + test_expression(find_xtime_less_10days, 1, + options="--sys_time 8640000000 --%s 8640000000" % xtime) + test_expression(find_xtime_less_10days_prefix, 1, + options="--sys_time 8640000000 --%s 8640000000" % xtime) + test_expression(find_xtime_less_10days, 1, + options="--sys_time 8640000000 --%s 8640000001" % xtime) + test_expression(find_xtime_less_10days_prefix, 1, + options="--sys_time 8640000000 --%s 8640000001" % xtime) + test_expression(find_xtime_less_10days, 0, + options="--sys_time 8640000000 --%s 7689600000" % xtime) + test_expression(find_xtime_less_10days_prefix, 0, + options="--sys_time 8640000000 --%s 7689600000" % xtime) + test_expression(find_xtime_less_10days, 0, + options="--sys_time 8640000000 --%s 7689600001" % xtime) + test_expression(find_xtime_less_10days_prefix, 0, + options="--sys_time 8640000000 --%s 7689600001" % xtime) + test_expression(find_xtime_less_10days, 0, + options="--sys_time 8640000000 --%s 7689599999" % xtime) + test_expression(find_xtime_less_10days_prefix, 0, + options="--sys_time 8640000000 --%s 7689599999" % xtime) + + test_size_expressions(attribute="size") + test_size_expressions(attribute="blocks") + test_size_expressions(attribute="stripe_count", has_unit=False, + bits=16) + test_size_expressions(attribute="projid", has_unit=False) + test_size_expressions(attribute="stripe_size", has_unit=False, + bits=32) + test_size_expressions(attribute="comp_count", has_unit=False, + bits=32) + + +def main(): + """ + Run tests of expression evaluation, if exception, return -1 + """ + main_with_exception() + if len(sys.argv) > 1: + fd = open(sys.argv[1], 'w') + fd.close() + return 0 diff --cc lipe/pylipe/lipe_convert_expr.py index 0000000,dd3229a..dd3229a mode 000000,100644..100644 --- a/lipe/pylipe/lipe_convert_expr.py +++ b/lipe/pylipe/lipe_convert_expr.py diff --cc lipe/src/Makefile.am index b4dbb48,0000000..1f890a0 mode 100644,000000..100644 --- a/lipe/src/Makefile.am +++ b/lipe/src/Makefile.am @@@ -1,106 -1,0 +1,118 @@@ +AUTOMAKE_OPTIONS = -Wall foreign +ACLOCAL_AMFLAGS = ${ALOCAL_FLAGS} + +AM_CPPFLAGS = -D_GNU_SOURCE -include config.h +AM_CFLAGS = -g -Wall -Werror -Wno-deprecated-declarations $(json_c_CFLAGS) +AM_LDFLAGS = -llnetconfig -llustreapi -lpthread $(json_c_LIBS) + - bin_PROGRAMS = lipe_scan lfill ldumpstripe lipe_hsm_remover \ - lipe_hsm_copytool ext4_inode2path lcreatemany lpcc_purge - ++bin_PROGRAMS = \ ++ ext4_inode2path \ ++ lcreatemany \ ++ ldumpstripe \ ++ lfill \ ++ lipe_hsm_copytool \ ++ lipe_hsm_remover \ ++ lipe_scan \ ++ lipe_scan2 \ ++ lpcc_purge + +noinst_PROGRAMS = lipe_expression_test generate_definition + +if BUILD_LAUDIT +bin_PROGRAMS += laudit laudit-report +endif + +if BUILD_HOTPOOL_UTILS +bin_PROGRAMS += lamigo lpurge +endif + +ext4_inode2path_SOURCES = debug.c debug.h ext4_inode2path.c + +LIPE_SOURCES = cmd.c cmd.h debug.c debug.h fake_lustre_disk.h \ + fake_lustre_idl.h flist.c flist.h misc.c misc.h \ + lustre_ea.c lustre_ea.h \ + lipe_ldiskfs.c lipe_ldiskfs.h \ ++ lipe_object_attrs.c \ + lipe_object_attrs.h \ + lipe_version.c \ + lipe_version.h \ + list.h policy.h policy.c \ + ldiskfs_read_ldd.c ldiskfs_read_ldd.h \ + general_policy.h general_policy.c \ + lustre_ea_ldiskfs.c lustre_ea_ldiskfs.h \ + posix.c posix.h posix_ea.c posix_ea.h +LIPE_CFLAGS = -Wall -Werror -g -I/usr/include $(json_c_CFLAGS) +LIPE_LDFLAGS = $(AM_LDFLAGS) + +if ZFS +LIPE_SOURCES += lipe_zfs.h lipe_zfs.c zfs_read_ldd.c zfs_read_ldd.h +LIPE_LDFLAGS += -lzfs -lzpool -lnvpair +LIPE_CFLAGS += $(zfs_CFLAGS) +endif + - lipe_scan_SOURCES = lipe.c lipe_config.c lipe_config.h lipe_result.c lipe_result.h $(LIPE_SOURCES) ++lipe_scan_SOURCES = lipe_scan.c lipe_config.c lipe_config.h lipe_result.c lipe_result.h $(LIPE_SOURCES) +lipe_scan_CFLAGS = $(LIPE_CFLAGS) +lipe_scan_LDFLAGS = $(LIPE_LDFLAGS) -lssl -lcrypto -lyaml + ++lipe_scan2_SOURCES = lipe_scan2.c lipe_scan2.h $(LIPE_SOURCES) ++lipe_scan2_CFLAGS = $(LIPE_CFLAGS) ++lipe_scan2_LDFLAGS = $(LIPE_LDFLAGS) ++ +lipe_hsm_remover_SOURCES = lipe_hsm_remover.c debug.c debug.h + +lipe_hsm_copytool_SOURCES = lipe_hsm_copytool.c debug.c debug.h lipe_string.h \ + lipe_string.c + +lfill_SOURCES = cmd.c cmd.h debug.c debug.h lfill.c ldiskfs_read_ldd.c \ + ldiskfs_read_ldd.h misc.c misc.h + +lcreatemany_SOURCES = lcreatemany.c debug.c debug.h + +ldumpstripe_SOURCES = ldumpstripe.c lustre_ea.h lustre_ea.c debug.c debug.h + +laudit_SOURCES = laudit.c + +laudit_report_SOURCES = laudit-report.c + +lipe_ssh_SOURCES = lipe_ssh.c lipe_ssh.h + +lamigo_SOURCES = lamigo.c lamigo.h lamigo_alr.c lamigo_hash.c lamigo_hash.h \ + $(LIPE_SOURCES) $(lipe_ssh_SOURCES) +lamigo_CFLAGS = $(LIPE_CFLAGS) +lamigo_LDFLAGS = $(LIPE_LDFLAGS) -lssh + +if HAVE_SSH_THREADS +lamigo_LDFLAGS += -lssh_threads +endif + +lpurge_SOURCES = lpurge.c $(LIPE_SOURCES) +lpurge_CFLAGS = $(LIPE_CFLAGS) +lpurge_LDFLAGS = $(LIPE_LDFLAGS) + +lpcc_purge_SOURCES = lpcc_purge.c $(LIPE_SOURCES) +lpcc_purge_CFLAGS = $(LIPE_CFLAGS) +lpcc_purge_LDFLAGS = $(LIPE_LDFLAGS) + +lipe_expression_test_SOURCES = lipe_expression_test.c $(LIPE_SOURCES) +lipe_expression_test_CFLAGS = $(LIPE_CFLAGS) +lipe_expression_test_LDFLAGS = $(LIPE_LDFLAGS) + +generate_definition_SOURCES = generate_definition.c debug.c debug.h \ + general_policy.h +generate_definition_CFLAGS = -Wall -Werror -Wall -Werror -g +generate_definition_LDFLAGS = + +C_FILES = $(wildcard *.c *.h) +C_CHECKS = $(C_FILES:%=%.c_checked) +CHECKS = $(C_CHECKS) + +%.c_checked: % ../lipe_c_check.pl + ../lipe_c_check.pl -f $< + touch $@ + +check_clean-local: + rm -f $(CHECKS) + +check-local: $(CHECKS) + +all: all-am diff --cc lipe/src/lipe_object_attrs.c index 0000000,fb5e4df..fb5e4df mode 000000,100644..100644 --- a/lipe/src/lipe_object_attrs.c +++ b/lipe/src/lipe_object_attrs.c diff --cc lipe/src/lipe_scan2.c index 0000000,ff72b90..ff72b90 mode 000000,100644..100644 --- a/lipe/src/lipe_scan2.c +++ b/lipe/src/lipe_scan2.c diff --cc lipe/src/lipe_scan2.h index 0000000,a33cf76..a33cf76 mode 000000,100644..100644 --- a/lipe/src/lipe_scan2.h +++ b/lipe/src/lipe_scan2.h diff --cc lipe/src/lpcc_purge.c index f07d594,0000000..8efc49e mode 100644,000000..100644 --- a/lipe/src/lpcc_purge.c +++ b/lipe/src/lpcc_purge.c @@@ -1,1111 -1,0 +1,1117 @@@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "lipe_object_attrs.h" +#include "policy.h" + + +#define DEF_HIGH_USAGE 90 +#define DEF_LOW_USAGE 75 +#define DEF_INTERVAL 5 +#define DEF_SCAN_THREADS 1 +#define DEF_CANDIDATE_NUM (128 * 1024) +#define MIN_CANDIDATE_NUM 1024 +#define MAX_CANDIDATE_NUM (100 * 1024 * 1024) +#define DEF_MAX_SCAN_SECS 30 +#define MIN_MAX_SCAN_SECS 5 +#define MAX_MAX_SCAN_SECS 300 + +#define MAX_ATIME_FID_LEN 128 + +#define OPT_DRY_RUN 1 +#define OPT_CANDIDATE_NUM 2 +#define OPT_CLEAR_HASHDIR 3 +#define OPT_LOG_LEVEL 4 +#define OPT_MAX_SCAN_SECS 5 +#define OPT_PIDFILE 6 + +struct lpcc_purge_options { + char *o_cache; + char *o_mount; + unsigned int o_rwid; + + double o_high_usage; + double o_low_usage; + + int o_interval; + int o_scan_threads; + int o_candidate_num; + int o_max_scan_secs; + + char *o_dumpfile; + char *o_pidfile; + bool o_dry_run; + bool o_clear_hashdir; +}; +static struct lpcc_purge_options opt = { + .o_rwid = -1, + .o_high_usage = DEF_HIGH_USAGE, + .o_low_usage = DEF_LOW_USAGE, + .o_interval = DEF_INTERVAL, + .o_scan_threads = DEF_SCAN_THREADS, + .o_candidate_num = DEF_CANDIDATE_NUM, + .o_max_scan_secs = DEF_MAX_SCAN_SECS, + .o_dry_run = false, + .o_clear_hashdir = false, +}; + +bool exit_flag = false; +struct lpcc_purge_stats { + double s_start_usage; + uint64_t s_scan_times; + time_t s_start_time, s_end_time; + time_t s_scan_start_sec; + + pthread_mutex_t s_lock; + uint64_t s_total_purged_objs; + uint64_t s_total_failed_objs; + uint64_t s_scanned_objs; /* scanned objects in current scanning */ + uint64_t s_purged_objs; /* purged objects in current scanning */ +}; + +static struct lpcc_purge_stats stats = { + .s_start_usage = -1, + .s_lock = PTHREAD_MUTEX_INITIALIZER, +}; + +static struct lipe_instance *instance = NULL; + +struct lpcc_purge_candidate { + uint64_t c_atime_ms; + struct lu_fid c_fid; + char c_path[0]; +}; + +struct lpcc_purge_candidate_set { + struct lpcc_purge_candidate ** cs_arr; + int cs_capacity; /* the total capacity of array */ + int cs_count; /* the number of valid elements in array */ + pthread_mutex_t cs_lock; +}; +static struct lpcc_purge_candidate_set candidate_set; + +struct lpcc_purge_candidate *lpcc_purge_candidate_new( + uint64_t atime_ms, struct lu_fid fid, const char *path) +{ + struct lpcc_purge_candidate *candidate = + (struct lpcc_purge_candidate *)malloc( + sizeof(struct lpcc_purge_candidate) + strlen(path) + 1); + + if (candidate == NULL) { + llapi_error(LLAPI_MSG_FATAL, errno, + "cannot allocate memory for candidate!"); + exit(1); + } + + candidate->c_atime_ms = atime_ms; + candidate->c_fid = fid; + strcpy(candidate->c_path, path); + + return candidate; +} + +void lpcc_purge_candidate_destroy(struct lpcc_purge_candidate *candidate) +{ + free(candidate); +} + +static int _compare_candidate(const void *p1, const void *p2) +{ + const struct lpcc_purge_candidate *c1 = *(const struct lpcc_purge_candidate **)(p1); + const struct lpcc_purge_candidate *c2 = *(const struct lpcc_purge_candidate **)(p2); + + if (c1->c_atime_ms < c2->c_atime_ms) + return -1; + if (c1->c_atime_ms > c2->c_atime_ms) + return 1; + return 0; +} + +static void lpcc_purge_candidate_set_init(int capacity) +{ + candidate_set.cs_arr = calloc(capacity, sizeof(candidate_set.cs_arr[0])); + if (candidate_set.cs_arr == NULL) { + llapi_error(LLAPI_MSG_FATAL, errno, + "cannot allocate memory for candidate set!"); + exit(1); + } + candidate_set.cs_capacity = capacity; + candidate_set.cs_count = 0; + pthread_mutex_init(&candidate_set.cs_lock, NULL); +} + +static void lpcc_purge_candidate_set_destroy(void) +{ + int i; + for (i = 0; i < candidate_set.cs_count; i++) + lpcc_purge_candidate_destroy(candidate_set.cs_arr[i]); + + free(candidate_set.cs_arr); +} + +static void lpcc_purge_candidate_set_clear(void) +{ + int i; + for (i = 0; i < candidate_set.cs_count; i++) + lpcc_purge_candidate_destroy(candidate_set.cs_arr[i]); + + candidate_set.cs_count = 0; +} + +static void lpcc_purge_candidate_set_append( + struct lpcc_purge_candidate *candidate) +{ + if (candidate_set.cs_count >= candidate_set.cs_capacity) { + llapi_error(LLAPI_MSG_FATAL, EFAULT, + "Out of boundary of array"); + exit(1); + } + + candidate_set.cs_arr[candidate_set.cs_count++] = candidate; +} + +void lpcc_purge_candidate_set_sort(void) +{ + qsort(candidate_set.cs_arr, candidate_set.cs_count, + sizeof(candidate_set.cs_arr[0]), &_compare_candidate); +} + +static void lpcc_purge_null_handler(int signal) +{ +} + +static void lpcc_purge_sigint_handler(int signal) +{ + psignal(signal, "exiting"); + _exit(0); +} + +/** + * Dump: + * - config + * - stats + */ +static void lpcc_purge_dump_config_stats(FILE *f) +{ + int rc; + char buff[64]; + time_t curr; + + json_object *j_all = json_object_new_object(); + + curr = time(NULL); + ctime_r(&curr, buff); + json_object *j_curr_time = json_object_new_string(buff); + json_object_object_add(j_all, "curr_time", j_curr_time); + rc = llapi_get_fsname(opt.o_mount, buff, sizeof(buff)); + if (rc) { + llapi_error(LLAPI_MSG_FATAL, rc, "Cannot get fsname."); + exit(1); + } + json_object_object_add(j_all, "fsname", json_object_new_string(buff)); + + json_object *j_config = json_object_new_object(); + json_object_object_add(j_config, "mount", json_object_new_string(opt.o_mount)); + json_object_object_add(j_config, "cache", json_object_new_string(opt.o_cache)); + json_object_object_add(j_config, "rwid", json_object_new_int64(opt.o_rwid)); + json_object_object_add(j_config, "high_usage", json_object_new_double(opt.o_high_usage)); + json_object_object_add(j_config, "low_usage", json_object_new_double(opt.o_low_usage)); + json_object_object_add(j_config, "interval", json_object_new_int64(opt.o_interval)); + json_object_object_add(j_config, "scan_threads", json_object_new_int64(opt.o_scan_threads)); + json_object_object_add(j_config, "candidate_num", json_object_new_int64(opt.o_candidate_num)); + json_object_object_add(j_config, "clear_hashdir", json_object_new_boolean(opt.o_clear_hashdir)); + json_object_object_add(j_config, "max_scan_secs", json_object_new_int64(opt.o_max_scan_secs)); + json_object_object_add(j_all, "config", j_config); + + json_object *j_stats = json_object_new_object(); + json_object_object_add(j_stats, "scan_times", json_object_new_int64(stats.s_scan_times)); + json_object_object_add(j_stats, "total_purged_objs", json_object_new_int64(stats.s_total_purged_objs)); + json_object_object_add(j_stats, "total_failed_objs", json_object_new_int64(stats.s_total_failed_objs)); + if (stats.s_start_time != 0) { + ctime_r(&stats.s_start_time, buff); + json_object_object_add(j_stats, "start_time", json_object_new_string(buff)); + json_object_object_add(j_stats, "start_usage", json_object_new_double(stats.s_start_usage)); + } + if (stats.s_end_time != 0) { + ctime_r(&stats.s_end_time, buff); + json_object_object_add(j_stats, "end_time", json_object_new_string(buff)); + } + json_object_object_add(j_stats, "scanned_objs", json_object_new_int64(stats.s_scanned_objs)); + json_object_object_add(j_stats, "purged_objs", json_object_new_int64(stats.s_purged_objs)); + json_object_object_add(j_all, "stats", j_stats); + + const char *str = json_object_to_json_string_ext(j_all, + JSON_C_TO_STRING_PRETTY | + JSON_C_TO_STRING_SPACED | + JSON_C_TO_STRING_NOZERO); + + fprintf(f, "%s\n", str); + fflush(f); + + json_object_put(j_all); +} + +static void lpcc_purge_usr1_handler(int sig) +{ + FILE *f = NULL; + + f = fopen(opt.o_dumpfile, "w"); + if (!f) { + llapi_printf(LLAPI_MSG_DEBUG, "cannot open dumpfile '%s'\n", + opt.o_dumpfile); + return; + }; + + lpcc_purge_dump_config_stats(f); + + fclose(f); +} + +static void usage(void) +{ + printf("Usage: %s [options]\n" + "\t-b, --debug, enable debugging output\n" + "\t --log-level={debug|info|normal|warn|error|fatal|off}, set log level (default: info)\n" + "\t-f, --config-file=FILE\n" + "\t-C, --cache=DIR root directory of PCC\n" + "\t-M, --mount=DIR local Lustre client's mountpoint\n" + "\t-A, --rwid, --roid=NUM the roid/rwid of PCC\n" + "\t-H, --high-usage=NUM %% of space or inode to start purging (default: %u)\n" + "\t-L, --low-usage=NUM %% of space or inode to stop purging (default: %u)\n" + "\t-i, --interval=NUM, seconds to next check (default: %u)\n" + "\t-t, --scan-threads=NUM scanning threads (default: %u)\n" + "\t --candidate-num=NUM, candidate number of approximate LRU (default: %d, min: %d, max: %d)\n" + "\t --max-scan-secs, max seconds to scan continously before purging (default: %d, min: %d, max: %d)\n" + "\t-w, --dump=FILE, dump stats to FILE when signal USR1 is recieved (default: /var/run/lpcc_purge-RWID.stats)\n" + "\t --pidfile=FILE, the pidfile name (default: /var/run/lpcc_purge-RWID.pid)\n" + "\t --clear-hashdir, clear empty hash dir after detaching file\n" + "\t --dry-run, scan once but do not detach file really\n" + "\t-h, --help, print this help message\n", + + program_invocation_short_name, + DEF_HIGH_USAGE, + DEF_LOW_USAGE, + DEF_INTERVAL, + DEF_SCAN_THREADS, + DEF_CANDIDATE_NUM, MIN_CANDIDATE_NUM, MAX_CANDIDATE_NUM, + DEF_MAX_SCAN_SECS, MIN_MAX_SCAN_SECS, MAX_MAX_SCAN_SECS + ); +} + +static struct option long_options[] = { + { "debug", no_argument, NULL, 'b'}, + { "config-file", required_argument, NULL, 'f'}, + { "log-level", required_argument, NULL, OPT_LOG_LEVEL}, + { "cache", required_argument, NULL, 'C'}, + { "mount", required_argument, NULL, 'M'}, + { "rwid", required_argument, NULL, 'A'}, + { "roid", required_argument, NULL, 'A'}, + { "high-usage", required_argument, NULL, 'H'}, + { "low-usage", required_argument, NULL, 'L'}, + { "interval", required_argument, NULL, 'i'}, + { "scan-threads", required_argument, NULL, 't'}, + { "dry-run", no_argument, NULL, OPT_DRY_RUN}, + { "candidate-num", required_argument, NULL, OPT_CANDIDATE_NUM}, + { "dump", required_argument, NULL, 'w'}, + { "pidfile", required_argument, NULL, OPT_PIDFILE}, + { "clear-hashdir", no_argument, NULL, OPT_CLEAR_HASHDIR}, + { "max-scan-secs", required_argument, NULL, OPT_MAX_SCAN_SECS}, + { "help", no_argument, NULL, 'h' }, + { NULL } +}; + +static struct option *lpcc_purge_keyword_lookup(const char *keyword) +{ + int i; + char keyword2[PATH_MAX]; + + for (i = 0; i <= strlen(keyword) && i < PATH_MAX; i++) { + if (keyword[i] == '_') + keyword2[i] = '-'; + else + keyword2[i] = keyword[i]; + } + + i = 0; + while (long_options[i].name) { + if (strcmp(keyword2, long_options[i].name) == 0) + return long_options + i; + i++; + } + return NULL; +} + +static void lpcc_purge_process_opt(int c, char *optarg); + +static void load_config(const char *path) +{ + size_t len = 0; + char *buf = NULL; + + FILE *f = fopen(path, "r"); + if (!f) { + llapi_error(LLAPI_MSG_FATAL, errno, + "cannot open config file '%s'", path); + exit(1); + } + + while (!feof(f)) { + struct option *opt; + char *s, *t; + len = PATH_MAX; + + if (getline(&buf, &len, f) <= 0) + break; + s = buf; + while (isspace(*s)) + s++; + t = strchr(s, '#'); + if (t) + *t = '\0'; + if (*s == '#') + continue; + t = strsep(&s, "=\n"); + if (!t || *t == 0) + continue; + opt = lpcc_purge_keyword_lookup(t); + if (!opt) { + llapi_error(LLAPI_MSG_ERROR, EINVAL, + "unknown tunable '%s'", t); + continue; + } + if (opt->val == 'f') { + /* you shall not pass! */ + continue; + } + + if (opt->has_arg == required_argument || + opt->has_arg == optional_argument) { + optarg = strsep(&s, "\n "); + if (!optarg && + opt->has_arg == required_argument) { + llapi_error(LLAPI_MSG_FATAL, EINVAL, + "no argument for '%s'", t); + exit(1); + } + } + else { + optarg = NULL; + } + llapi_printf(LLAPI_MSG_DEBUG, "conf: %s %s\n", t, + optarg ? optarg : ""); + lpcc_purge_process_opt(opt->val, optarg); + } + + free(buf); + fclose(f); +} + +static const char *log_level_strs[] = { + [LLAPI_MSG_OFF] = "off", + [LLAPI_MSG_FATAL] = "fatal", + [LLAPI_MSG_ERROR] = "error", + [LLAPI_MSG_WARN] = "warn", + [LLAPI_MSG_NORMAL] = "normal", + [LLAPI_MSG_INFO] = "info", + [LLAPI_MSG_DEBUG] = "debug", +}; + +static enum llapi_message_level parse_log_level(const char *level_str) +{ + enum llapi_message_level level = LLAPI_MSG_INFO; + int i; + + for (i = 0; i < ARRAY_SIZE(log_level_strs); i++) { + if (strcasecmp(log_level_strs[i], level_str) == 0) { + level = i; + break; + } + } + + if (i == sizeof(log_level_strs) / sizeof(char *)) { + llapi_error(LLAPI_MSG_WARN, -EINVAL, + "Unrecognized log level string '%s'", + optarg); + } + + return level; +} + +static void parse_mountpoint(const char *name) +{ + int rc; + struct statfs statbuf; + + rc = statfs(name, &statbuf); + if (rc != 0) { + llapi_error(LLAPI_MSG_FATAL, -rc, + "cannot statfs '%s'", name); + exit(1); + } + if (statbuf.f_type != LL_SUPER_MAGIC) { + llapi_error(LLAPI_MSG_FATAL, EINVAL, + "'%s' is not a lustre mount point", name); + exit(1); + } +} + +static void parse_cache_dir(const char *name) +{ + struct stat stat1; + int rc; + + rc = stat(name, &stat1); + if (rc) { + llapi_error(LLAPI_MSG_FATAL, errno, + "cannot stat cache dir '%s'", name); + exit(1); + } + + if (!S_ISDIR(stat1.st_mode)) { + llapi_error(LLAPI_MSG_FATAL, errno, + "cache path '%s' is not a directory", name); + exit(1); + } +} + +static void lpcc_purge_process_opt(int c, char *optarg) +{ + long value; + char *endptr = NULL; + enum llapi_message_level log_level; + + switch(c) { + case 'f': + load_config(optarg); + break; + case 'b': + llapi_msg_set_level(LLAPI_MSG_MAX); + break; + case OPT_LOG_LEVEL: + log_level = parse_log_level(optarg); + llapi_printf(LLAPI_MSG_INFO, "set log level: %s\n", + log_level_strs[log_level]); + llapi_msg_set_level(log_level); + + break; + case 'C': + parse_cache_dir(optarg); + opt.o_cache = strdup(optarg); + break; + case 'M': + parse_mountpoint(optarg); + opt.o_mount = strdup(optarg); + break; + case 'A': + value = strtol(optarg, &endptr, 10); + if (*endptr != '\0' || value <= 0) { + llapi_error(LLAPI_MSG_FATAL, -EINVAL, + "invalid roid/rwid: '%s'", + optarg); + exit(1); + } + opt.o_rwid = value; + break; + case 'H': + value = strtol(optarg, &endptr, 10); + if (*endptr != '\0' || value < 1 || value > 99) { + llapi_error(LLAPI_MSG_FATAL, -EINVAL, + "invalid high watermark: '%s'", + optarg); + exit(1); + } + opt.o_high_usage = value; + break; + case 'L': + value = strtol(optarg, &endptr, 10); + if (*endptr != '\0' || value < 1 || value > 99) { + llapi_error(LLAPI_MSG_FATAL, -EINVAL, + "invalid low watermark: '%s'", + optarg); + exit(1); + } + opt.o_low_usage = value; + break; + case 'i': + value = strtol(optarg, &endptr, 10); + if (*endptr != '\0' || value <= 0) { + llapi_error(LLAPI_MSG_FATAL, -EINVAL, + "invalid check interval: '%s'", + optarg); + exit(1); + } + opt.o_interval = value; + break; + case 't': + value = strtol(optarg, &endptr, 10); + if (*endptr != '\0' || value <= 0) { + llapi_error(LLAPI_MSG_FATAL, -EINVAL, + "invalid scan threads count: '%s'", + optarg); + exit(1); + } + opt.o_scan_threads = value; + break; + case 'w': + opt.o_dumpfile = strdup(optarg); + break; + case OPT_PIDFILE: + opt.o_pidfile = strdup(optarg); + break; + case OPT_DRY_RUN: + opt.o_dry_run = true; + break; + case OPT_CANDIDATE_NUM: + value = strtol(optarg, &endptr, 10); + if (*endptr != '\0' || value < MIN_CANDIDATE_NUM || + value > MAX_CANDIDATE_NUM) { + llapi_error(LLAPI_MSG_FATAL, -EINVAL, + "invalid candidate number: '%s'", + optarg); + exit(1); + } + opt.o_candidate_num = value; + break; + case OPT_CLEAR_HASHDIR: + opt.o_clear_hashdir = true; + break; + case OPT_MAX_SCAN_SECS: + value = strtol(optarg, &endptr, 10); + if (*endptr != '\0' || value < MIN_MAX_SCAN_SECS || + value > MAX_MAX_SCAN_SECS) { + llapi_error(LLAPI_MSG_FATAL, -EINVAL, + "invalid max_scan_secs: '%s'\n", + optarg); + exit(1); + } + opt.o_max_scan_secs = value; + break; + default: + llapi_error(LLAPI_MSG_FATAL, -EINVAL, + "invalid argument: '%s'", + optarg); + exit(1); + break; + } +} + +static void lpcc_purge_parse_opts(int argc, char **argv) +{ + int c; + + while ((c = getopt_long(argc, argv, + "bf:C:M:A:H:L:i:t:w:h", + long_options, NULL)) + != EOF) { + switch(c) { + case '?': + /* Don't do further process if invalid option found */ + fprintf(stderr, + "Try '%s --help' for more information.\n", + program_invocation_short_name); + exit(1); + case 'h': + usage(); + exit(0); + default: + lpcc_purge_process_opt(c, optarg); + break; + } + } +} + +void lpcc_purge_verify_opts(void) +{ + char buf[PATH_MAX]; + + if (opt.o_mount == NULL) { + llapi_error(LLAPI_MSG_FATAL, EINVAL, + "lustre mount point must be specified"); + exit(1); + } + if (opt.o_cache == NULL) { + llapi_error(LLAPI_MSG_FATAL, EINVAL, + "pcc cache dir must be specified"); + exit(1); + } + if (opt.o_rwid == -1) { + llapi_error(LLAPI_MSG_FATAL, EINVAL, + "roid/rwid must be specified"); + exit(1); + } + if (opt.o_dumpfile == NULL) { + snprintf(buf, sizeof(buf), "/var/run/lpcc_purge-%d.stats", opt.o_rwid); + opt.o_dumpfile = strdup(buf); + } + if (opt.o_pidfile == NULL) { + snprintf(buf, sizeof(buf), "/var/run/lpcc_purge-%d.pid", opt.o_rwid); + opt.o_pidfile = strdup(buf); + } + + /* check freehi > freelo */ + if (opt.o_high_usage <= opt.o_low_usage) { + llapi_error(LLAPI_MSG_FATAL, EINVAL, + "high usage (%.1f) must be larger than low usage (%.1f)", + opt.o_high_usage, opt.o_low_usage); + exit(1); + } +} + +/* It seems that lustre/utils/pid_file.c does not exist in RHEL7.x, + * so copy the function here + */ +static int create_pid_file(const char *path) +{ + char buf[3 * sizeof(long long) + 2]; + size_t buf_len; + int fd = -1; + int rc2; + + fd = open(path, O_RDWR|O_CREAT|O_CLOEXEC, 0600); + if (fd < 0) { + fprintf(stderr, "%s: cannot open '%s': %s\n", + program_invocation_short_name, path, strerror(errno)); + return -1; + } + + struct flock fl = { + .l_type = F_WRLCK, + .l_whence = SEEK_SET, + }; + + rc2 = fcntl(fd, F_SETLK, &fl); + if (rc2 < 0) { + fprintf(stderr, "%s: cannot lock '%s': %s\n", + program_invocation_short_name, path, strerror(errno)); + goto out; + } + + rc2 = ftruncate(fd, 0); + if (rc2 < 0) { + fprintf(stderr, "%s: cannot truncate '%s': %s\n", + program_invocation_short_name, path, strerror(errno)); + goto out; + } + + buf_len = snprintf(buf, sizeof(buf), "%lld\n", (long long)getpid()); + rc2 = write(fd, buf, buf_len); + if (rc2 < 0) { + fprintf(stderr, "%s: cannot write '%s': %s\n", + program_invocation_short_name, path, strerror(errno)); + goto out; + } +out: + if (rc2 < 0 && !(fd < 0)) { + close(fd); + fd = -1; + } + + return fd; +} + +static void lpcc_purge_lock_pidfile(void) +{ + int fd; + + fd = create_pid_file(opt.o_pidfile); + if (fd < 0) { + llapi_error(LLAPI_MSG_FATAL, errno, + "cannot create pidfile '%s'", opt.o_pidfile); + exit(1); + } + /* we keep the fd open to hold the flock, + it will be closed automatically when the process exits */ +} + +static double lpcc_purge_get_fs_usage(const char *fs) +{ + int rc; + double usage, i_usage; + struct statfs statfs_buf; + + rc = statfs(opt.o_cache, &statfs_buf); + if (rc) { + llapi_error(LLAPI_MSG_FATAL, errno, "cannot statfs '%s'", fs); + exit(1); + } + + usage = 100.0 * (statfs_buf.f_blocks - statfs_buf.f_bavail) / statfs_buf.f_blocks; + i_usage = 100.0 * (statfs_buf.f_files - statfs_buf.f_ffree) / statfs_buf.f_files; + + return (usage > i_usage) ? usage : i_usage; +} + +static void lpcc_purge_wait_for_scan(void) +{ + while (1) { + double usage = lpcc_purge_get_fs_usage(opt.o_cache); + llapi_printf(LLAPI_MSG_DEBUG, "usage: %.1f\n", usage); + if (usage >= opt.o_high_usage || opt.o_dry_run) { + stats.s_start_usage = usage; + break; + } else { + sleep(opt.o_interval); + } + } +} + +static int clear_hash_dir(const char *path) +{ + int rc; + char dir[PATH_MAX], tmp[PATH_MAX]; + + strcpy(tmp, path); + strcpy(dir, dirname(tmp)); + + while (strlen(dir) > strlen(opt.o_cache)) { + rc = rmdir(dir); + if (rc) { + if (errno == ENOTEMPTY || errno == EBUSY) { + rc = 0; + } + else { + rc = -errno; + llapi_error(LLAPI_MSG_WARN, -rc, + "cannot remove hash dir: %s", dir); + } + break; + } + else { + strcpy(tmp, dir); + strcpy(dir, dirname(tmp)); + } + } + + return rc; +} + +static int lpcc_purge_detach_by_fid(const char *mntpath, + const struct lu_fid *fid, const char *cache_file) +{ + int rc; + int fd; + struct lu_pcc_detach_fid detach; + + fd = open(mntpath, O_DIRECTORY | O_RDONLY); + if (fd < 0) { + rc = -errno; + llapi_error(LLAPI_MSG_ERROR, rc, "cannot open root path: %s", + mntpath); + return rc; + } + + detach.pccd_fid = *fid; + detach.pccd_flags = PCC_DETACH_FL_UNCACHE; + rc = ioctl(fd, LL_IOC_PCC_DETACH_BY_FID, &detach); + close(fd); + + if (rc) { + /* skip */ + llapi_error(LLAPI_MSG_DEBUG, rc, + "cannot detach fid: "DFID"", PFID(fid)); + return rc; + } + + if (detach.pccd_flags & PCC_DETACH_FL_CACHE_REMOVED) { + llapi_printf(LLAPI_MSG_DEBUG, + "Detach and remove the PCC cached fid: "DFID"\n", + PFID(fid)); + } + else if (detach.pccd_flags & PCC_DETACH_FL_ATTACHING) { + llapi_printf(LLAPI_MSG_DEBUG, + "fid "DFID" is being attached, skip it", PFID(fid)); + } + else { + llapi_printf(LLAPI_MSG_DEBUG, + "Remove non-cached file: %s flags: %X\n", + cache_file, detach.pccd_flags); + rc = unlink(cache_file); + if (rc < 0 && errno != ENOENT) { + rc = -errno; + llapi_error(LLAPI_MSG_ERROR, rc, + "Unlink %s failed", cache_file); + } + + } + + return rc; +} + +static int lpcc_purge_detach_candidate(const char *mnt, + const struct lpcc_purge_candidate *candidate) +{ + int rc; + struct stat statbuf; + uint64_t atime_ms; + + llapi_printf(LLAPI_MSG_DEBUG, "detach fid: "DFID"\n", PFID(&candidate->c_fid)); + + /* double confirm the atime. If it's changed, discard this entry */ + rc = stat(candidate->c_path, &statbuf); + if (rc) { + rc = -errno; + llapi_error(LLAPI_MSG_WARN, -rc, "cannot stat file '%s'", candidate->c_path); + return rc; + } + atime_ms = statbuf.st_atim.tv_sec * 1000 + statbuf.st_atim.tv_nsec / 1000000; + if (candidate->c_atime_ms != atime_ms) + /* skip entirely */ + return 0; + + pthread_mutex_lock(&stats.s_lock); + stats.s_purged_objs++; + stats.s_total_purged_objs++; + pthread_mutex_unlock(&stats.s_lock); + + if (opt.o_dry_run) + return rc; + + rc = lpcc_purge_detach_by_fid(mnt, &candidate->c_fid, candidate->c_path); + if (rc) { + llapi_error(LLAPI_MSG_WARN, -rc, "cannot detach fid: "DFID, + PFID(&candidate->c_fid)); + pthread_mutex_lock(&stats.s_lock); + stats.s_total_failed_objs++; + pthread_mutex_unlock(&stats.s_lock); + return rc; + } + + if (opt.o_clear_hashdir) + rc = clear_hash_dir(candidate->c_path); + + return 0; +} + +static int lpcc_purge_scan_callback(struct lipe_instance *instance, + struct lipe_object *object, + struct lipe_object_attrs *attrs) +{ + int rc; + struct lu_fid fid; + char *path = NULL; + struct lpcc_purge_candidate *candidate = NULL, **to_detach = NULL; + uint64_t scan_secs; + int i, j, n_detach, n_discard; + + if (!S_ISREG(attrs->loa_mode)) { + /* skip non-regular file */ + return 0; + } + + path = strdup(object->u.lo_posix.lop_path); + if (path == NULL) { + rc = -errno; + goto out; + } + + const char * name = basename(path); + rc = sscanf(name, SFID, RFID(&fid)); + if (rc != 3) { + /* Not an valid fid string, skip it */ + rc = 0; + goto out; + } + + candidate = lpcc_purge_candidate_new(attrs->loa_atime_ms, fid, object->u.lo_posix.lop_path); + if (candidate == NULL) { + rc = -ENOMEM; + goto out; + } + + /* Increase statistic data */ + pthread_mutex_lock(&stats.s_lock); + stats.s_scanned_objs++; + pthread_mutex_unlock(&stats.s_lock); + + llapi_printf(LLAPI_MSG_DEBUG, "append fid: "DFID"\n", PFID(&fid)); + pthread_mutex_lock(&candidate_set.cs_lock); + lpcc_purge_candidate_set_append(candidate); + candidate = NULL; + + /* + * check whether the pool is full or has scanned continously + * for too long time + */ + scan_secs = time(NULL) - stats.s_scan_start_sec; + if (candidate_set.cs_count < candidate_set.cs_capacity && + scan_secs < opt.o_max_scan_secs) { + pthread_mutex_unlock(&candidate_set.cs_lock); + rc = 0; + goto out; + } + + /* pool is full or scan for too long time */ + llapi_printf(LLAPI_MSG_DEBUG, "start purging, candidate number: %u, " + "scan secs: %lu\n", candidate_set.cs_count, scan_secs); + lpcc_purge_candidate_set_sort(); + + /* (n_detach + n_discard) = 10% * candidate_num if start_ussage is 100 */ + n_detach = (stats.s_start_usage - opt.o_low_usage) / 1000 * + candidate_set.cs_capacity; + n_discard = opt.o_low_usage / 1000 * candidate_set.cs_count; + + if (n_detach <= 0) { + n_detach = 0; + } else { + to_detach = calloc(n_detach, sizeof(to_detach[0])); + if (to_detach == NULL) { + llapi_error(LLAPI_MSG_FATAL, errno, + "cannot allocate memory"); + exit(1); + } + } + + llapi_printf(LLAPI_MSG_DEBUG, + "copy out %d elements from the head\n", n_detach); + for (i = 0; i < n_detach; i++) { + to_detach[i] = candidate_set.cs_arr[i]; + } + + llapi_printf(LLAPI_MSG_DEBUG, + "shift pool and remove %d elements from the tail\n", n_discard); + memmove(&candidate_set.cs_arr[0], &candidate_set.cs_arr[n_detach], + (candidate_set.cs_count - n_detach - n_discard) * sizeof(candidate_set.cs_arr[0])); + + for (j = candidate_set.cs_count - n_discard; j < candidate_set.cs_count; j++) { + lpcc_purge_candidate_destroy(candidate_set.cs_arr[j]); + } + candidate_set.cs_count = candidate_set.cs_count - n_detach - n_discard; + + stats.s_scan_start_sec = time(NULL); + pthread_mutex_unlock(&candidate_set.cs_lock); + + /* Detach files from candidates */ + llapi_printf(LLAPI_MSG_DEBUG, "detach files...\n"); + for (i = 0; i < n_detach; i++) { + rc = lpcc_purge_detach_candidate(opt.o_mount, to_detach[i]); + lpcc_purge_candidate_destroy(to_detach[i]); + } + + rc = 0; +out: + free(path); + lpcc_purge_candidate_destroy(candidate); + free(to_detach); + + return rc; +} + +static void lpcc_purge_scan(void) +{ + int rc; + struct lipe_policy policy; + struct scan_result result; + bool ldd_err; + + llapi_printf(LLAPI_MSG_INFO, "do scanning...\n"); + + stats.s_scan_times++; + stats.s_start_usage = lpcc_purge_get_fs_usage(opt.o_cache); + stats.s_scanned_objs = 0; + stats.s_purged_objs = 0; + stats.s_scan_start_sec = time(NULL); + + lipe_policy_init(&policy); + policy.lp_attr_bits = LIPE_OBJECT_ATTR_ATTR; + memset(&result, 0x00, sizeof(result)); + + llapi_printf(LLAPI_MSG_DEBUG, "start scanning...\n"); + - rc = lipe_scan(instance, &policy, &result, NULL, - NULL, opt.o_scan_threads, "lpcc_purge", - NULL, true, false, &ldd_err); ++ rc = lipe_scan(instance, ++ &policy, ++ &result, ++ NULL /* sum_counter_list */, ++ NULL /* sum_classify_list */, ++ opt.o_scan_threads, ++ "lpcc_purge" /* workspace */, ++ true /* abort_failure */, ++ &ldd_err); + if (rc < 0) { + llapi_error(LLAPI_MSG_ERROR, rc, + "failed to run lipe_scan"); + } + + llapi_printf(LLAPI_MSG_INFO, "scanning end\n"); +} + +static void lpcc_purge_free_space(void) +{ + int i; + + llapi_printf(LLAPI_MSG_INFO, "free space...\n"); + + lpcc_purge_candidate_set_sort(); + + for (i = 0; i < candidate_set.cs_count; i++) { + double usage = lpcc_purge_get_fs_usage(opt.o_cache); + if (usage >= opt.o_low_usage || opt.o_dry_run) + lpcc_purge_detach_candidate(opt.o_mount, candidate_set.cs_arr[i]); + else + break; + } + + fflush(stdout); + lpcc_purge_candidate_set_clear(); +} + +int main(int argc, char *argv[]) +{ + int rc = 0; + llapi_msg_set_level(LLAPI_MSG_INFO); + + signal(SIGUSR1, &lpcc_purge_usr1_handler); + signal(SIGUSR2, &lpcc_purge_null_handler); + + signal(SIGINT, &lpcc_purge_sigint_handler); + signal(SIGTERM, &lpcc_purge_sigint_handler); + + lpcc_purge_parse_opts(argc, argv); + lpcc_purge_verify_opts(); + lpcc_purge_lock_pidfile(); + + lpcc_purge_candidate_set_init(opt.o_candidate_num); + + instance = lipe_instance_alloc(0, opt.o_cache); + instance->li_callback = lpcc_purge_scan_callback; + + while(1) { + lpcc_purge_wait_for_scan(); + + stats.s_end_time = 0; + stats.s_start_time = time(NULL); + + lpcc_purge_scan(); + + lpcc_purge_free_space(); + + stats.s_end_time = time(NULL); + + if (opt.o_dry_run) { + break; + } + } + + lipe_instance_free(instance); + lpcc_purge_candidate_set_destroy(); + + return rc == 0 ? 0 : EXIT_FAILURE; +} diff --cc lipe/src/posix.c index a72cdab,0000000..be926db9 mode 100644,000000..100644 --- a/lipe/src/posix.c +++ b/lipe/src/posix.c @@@ -1,392 -1,0 +1,369 @@@ +/* + * Copyright (c) 2019, DDN Storage Corporation. + */ +/* + * + * Support for scanning POSIX directory. + * + * Author: Li Xi + */ +#include +#include +#include +#include +#include "debug.h" +#include "policy.h" +#include "posix_ea.h" +#include "lipe_object_attrs.h" + +static struct posix_scan_work * +posix_scan_get_work(struct global_info *ginfo, + struct thread_info *info, + struct lipe_list_head *local_queue) +{ + bool is_idle = false; + struct posix_scan_work *top = NULL; + struct lipe_list_head *queue = &ginfo->gi_works; + + if (!lipe_list_empty(local_queue)) { + top = lipe_list_entry(local_queue->next, struct posix_scan_work, + psw_linkage); + lipe_list_del(&top->psw_linkage); + return top; + } + + pthread_mutex_lock(&ginfo->gi_mutex); + while (lipe_list_empty(queue)) { + if (info->ti_stopping) + goto out; + + assert(ginfo->gi_work_number == 0); + if (!is_idle) { + ginfo->gi_idle_num_threads++; + pthread_cond_broadcast(&ginfo->gi_cond); + is_idle = true; + } + if (ginfo->gi_idle_num_threads >= ginfo->gi_num_threads) + goto out; + pthread_cond_wait(&ginfo->gi_cond, &ginfo->gi_mutex); + } + + if (is_idle) + ginfo->gi_idle_num_threads--; + + assert(ginfo->gi_work_number > 0); + top = lipe_list_entry(queue->next, struct posix_scan_work, psw_linkage); + lipe_list_del(&top->psw_linkage); + ginfo->gi_work_number--; +out: + pthread_mutex_unlock(&ginfo->gi_mutex); + return top; +} + +static void posix_scan_add_global_work(struct global_info *ginfo, + struct posix_scan_work *work) +{ + struct lipe_list_head *queue = &ginfo->gi_works; + + pthread_mutex_lock(&ginfo->gi_mutex); + lipe_list_add_tail(&work->psw_linkage, queue); + ginfo->gi_work_number++; + pthread_mutex_unlock(&ginfo->gi_mutex); + pthread_cond_broadcast(&ginfo->gi_cond); +} + +static struct posix_scan_work *posix_scan_new_work(char *dir) +{ + struct posix_scan_work *work; + + LIPE_ALLOC_PTR(work); + if (work == NULL) + return NULL; + strncpy(work->psw_path, dir, sizeof(work->psw_path)); + return work; +} + +/* + * Return 0 if offload the work + * Return 1 if not + * Return negative value if error + */ +static int posix_scan_offload_work(struct global_info *ginfo, + struct lipe_list_head *local_queue, + char *dir) +{ + struct posix_scan_work *work; + + work = posix_scan_new_work(dir); + if (work == NULL) + return -ENOMEM; + + /* + * Checking the waiting work number without lock is racy but + * should be fine. + */ + if (ginfo->gi_work_number <= ginfo->gi_num_threads) + posix_scan_add_global_work(ginfo, work); + else + lipe_list_add(&work->psw_linkage, local_queue); + return 0; +} + +static void posix_scan_free_works(struct global_info *ginfo) +{ + struct lipe_list_head *queue = &ginfo->gi_works; + struct posix_scan_work *work, *n; + + lipe_list_for_each_entry_safe(work, n, queue, psw_linkage) { + lipe_list_del_init(&work->psw_linkage); + LIPE_FREE_PTR(work); + } +} + +static int64_t timespec2ms(struct timespec tim) +{ + return ((int64_t)tim.tv_sec * 1000) + ((int64_t)tim.tv_nsec / 1000000); +} + +static void posix_scan_copy_attrs(struct lipe_object_attrs *attrs, + struct stat *sb) +{ + attrs->loa_ino = (int64_t)sb->st_ino; + attrs->loa_atime_ms = timespec2ms(sb->st_atim); + attrs->loa_mtime_ms = timespec2ms(sb->st_mtim); + attrs->loa_ctime_ms = timespec2ms(sb->st_ctim); + attrs->loa_mode = (int64_t)sb->st_mode; + attrs->loa_uid = sb->st_uid; + attrs->loa_gid = sb->st_gid; + attrs->loa_flags = 0; + attrs->loa_nlinks = (int64_t)sb->st_nlink; + + attrs->loa_size = (int64_t)sb->st_size; + attrs->loa_blocks = (int64_t)sb->st_blocks * 512; + attrs->loa_attr_bits = LIPE_OBJECT_ATTR_ATTR | LIPE_OBJECT_ATTR_SIZE; +} + +static int posix_scan_match_policy(struct global_info *ginfo, + struct thread_info *info, + const char *path, + struct lipe_object_attrs *attrs, + struct lipe_policy_sysattrs *sysattrs, + struct lipe_object *object) +{ + int rc; + struct stat sb; + bool lustre = ginfo->gi_lustre; - bool all_inode = ginfo->gi_all_inode; + bool abort_failure = ginfo->gi_abort_failure; - struct lu_fid *watch_fid = ginfo->gi_watch_fid; - struct scan_result *result = &info->ti_result; + struct lipe_policy *policy = info->ti_policy; + struct lipe_instance *instance = info->ti_instance; - struct counter_list *counter_list = &info->ti_counter_list; - struct classify_list *classify_list = &info->ti_classify_list; + + rc = lstat(path, &sb); + if (rc) { + LDEBUG("failed to stat [%s]\n", path); + return rc; + } + + lipe_object_attrs_reset(attrs); + posix_scan_copy_attrs(attrs, &sb); + object->u.lo_posix.lop_path = path; + - rc = lipe_policy_apply(object, policy, instance, attrs, - sysattrs, result, abort_failure, counter_list, - classify_list, watch_fid, all_inode, ++ rc = lipe_policy_apply(info, object, policy, instance, attrs, ++ sysattrs, abort_failure, + lustre); + if (rc) + LDEBUG("failed to apply policy on [%s]\n", path); + return rc; +} + +/* + * The buffer of $dir will be changed from time to time, but at the time of + * returning, it remain the same as being passed in. + */ +static int posix_scan_traverse(struct global_info *ginfo, + struct thread_info *info, + struct lipe_list_head *local_queue, + char *path, int size, + struct lipe_object_attrs *attrs, + struct lipe_policy_sysattrs *sysattrs, + struct lipe_object *object) +{ + DIR *d; + int len; + int rc = 0; + int ret = 0; + struct dirent64 *dent; + + ret = posix_scan_match_policy(ginfo, info, path, attrs, sysattrs, + object); + if (ret) + LDEBUG("failed to match policy to path [%s]\n", path); + + d = opendir(path); + if (d == NULL) { + if (errno == ENOTDIR) { + return 0; + } else { + LERROR("failed to opendir [%s]: %s\n", + path, strerror(errno)); + return -errno; + } + } + + len = strlen(path); + while ((dent = readdir64(d)) != NULL) { + if (info->ti_stopping) + break; + + if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) + continue; + + if ((len + dent->d_reclen + 2) > size) { + LERROR("string buffer with size [%d] is too small\n", + size); + if (ret == 0) + ret = -E2BIG; + break; + } + + path[len] = 0; + strcat(path, "/"); + strcat(path, dent->d_name); + + switch (dent->d_type) { + case DT_UNKNOWN: + LERROR("unknown file type for [%s]\n", + path); + break; + case DT_DIR: + rc = posix_scan_offload_work(ginfo, local_queue, path); + if (rc < 0) { + LERROR("failed to offload work [%s]\n", path); + if (ret == 0) + ret = rc; + } + break; + default: + rc = posix_scan_match_policy(ginfo, info, path, attrs, + sysattrs, object); + if (rc) { + LDEBUG("failed to match policy to path [%s]\n", + path); + if (ret == 0) + ret = rc; + } + break; + } + } + + path[len] = 0; + closedir(d); + return ret; +} + +void *posix_scan_thread(void *arg) +{ + struct thread_info *info = arg; - struct counter_list *counter_list = &info->ti_counter_list; - struct classify_list *classify_list = &info->ti_classify_list; + struct scan_result *result = &info->ti_result; + struct lipe_object object; + struct lipe_object_attrs attrs; + struct lipe_policy_sysattrs sysattrs; + struct global_info *ginfo = info->ti_global_info; + int rc; + struct posix_scan_work *work; + struct lipe_list_head local_queue; + + LIPE_INIT_LIST_HEAD(&local_queue); + if (lipe_object_attrs_init(&attrs)) { + LERROR("failed to init atttrs: not enough memory\n"); + goto out; + } - classify_list = &info->ti_classify_list; - - result->sr_number_dirs = 0; - result->sr_number_files = 0; - result->sr_number_inodes = 0; - result->sr_number_groups = 0; + + gettimeofday(&result->sr_time_start, NULL); + sysattrs.lps_sys_time = result->sr_time_start.tv_sec * 1000 + + result->sr_time_start.tv_usec / 1000; + sysattrs.lps_attr_bits = LIPE_OBJECT_ATTR_SYSATTR_TIME; + + object.lo_backfs_ops = &posix_operations; + object.lo_backfs_type = LBT_POSIX; + + while (1) { + if (info->ti_stopping) { + LERROR("thread [%d] aborts without finishing\n", + info->ti_thread_index); + break; + } + + work = posix_scan_get_work(ginfo, info, &local_queue); + if (work == NULL) + break; + + /* Do not quit since more work might come soon */ + rc = posix_scan_traverse(ginfo, info, &local_queue, + work->psw_path, sizeof(work->psw_path), + &attrs, &sysattrs, &object); + if (rc) + LERROR("failed to handling work of [%s]\n", + work->psw_path); + LIPE_FREE_PTR(work); + } + - rc = counter_list_flush(counter_list); - if (rc) - LERROR("failed to flush counter list\n"); - - rc = classify_list_flush(classify_list); - if (rc) - LERROR("failed to flush classify list\n"); - + lipe_object_attrs_fini(&attrs); +out: + gettimeofday(&result->sr_time_end, NULL); + diff_timevals(&result->sr_time_start, &result->sr_time_end, + &result->sr_time_diff); + + return NULL; +} + +int posix_scan(struct lipe_instance *instance, + struct lipe_policy *policy, + struct scan_result *result, + struct counter_list *sum_counter_list, + struct classify_list *sum_classify_list, ++ void **thread_data, + int num_threads, + const char *workspace, - struct lu_fid *watch_fid, + bool abort_failure) +{ + int rc; + struct thread_info *infos = NULL; + struct global_info global_info; + struct posix_scan_work *work; + + global_info.gi_workspace = workspace; - global_info.gi_watch_fid = watch_fid; + global_info.gi_abort_failure = abort_failure; - global_info.gi_all_inode = true; + global_info.gi_lustre = true; + global_info.gi_num_threads = num_threads; + pthread_mutex_init(&global_info.gi_mutex, NULL); + pthread_cond_init(&global_info.gi_cond, NULL); + + LIPE_INIT_LIST_HEAD(&global_info.gi_works); + global_info.gi_work_number = 0; + global_info.gi_idle_num_threads = 0; + + work = posix_scan_new_work(instance->li_device); + if (work == NULL) { + rc = -ENOMEM; + LERROR("failed to add work\n"); + goto out; + } + posix_scan_add_global_work(&global_info, work); + - rc = scan_threads_start(&infos, num_threads, instance, NULL, policy, ++ rc = scan_threads_start(&infos, thread_data, num_threads, ++ instance, NULL, policy, + &global_info, LBT_POSIX); + if (rc) { + LERROR("failed to start threads\n"); + goto out; + } + + scan_threads_join(infos, num_threads, result, sum_counter_list, + sum_classify_list, LBT_POSIX); + +out: + posix_scan_free_works(&global_info); + pthread_mutex_destroy(&global_info.gi_mutex); + pthread_cond_destroy(&global_info.gi_cond); + return rc; +} diff --cc lipe/src/posix_ea.c index 34e2f30,0000000..57b0e2e mode 100644,000000..100644 --- a/lipe/src/posix_ea.c +++ b/lipe/src/posix_ea.c @@@ -1,416 -1,0 +1,402 @@@ +/* + * Copyright (c) 2019, DDN Storage Corporation. + */ +/* + * + * Get the EAs from POSIX file system + * + * Author: Li Xi + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "fake_lustre_idl.h" ++#include "lipe_object_attrs.h" +#include "lustre_ea.h" +#include "posix_ea.h" +#include "debug.h" +#include "policy.h" +#include "misc.h" + +static int get_link_ea_posix(struct lipe_object *object, + struct lipe_object_attrs *attrs) +{ + int rc; + ssize_t size; - char *buf = attrs->loa_leh_buf; + const char *path = object->u.lo_posix.lop_path; + - /* - * Need to clear the buffer, otherwise lee->lee_name will keep dirty - * data of last inode - */ - memset(buf, 0, MAX_LINKEA_SIZE); - - size = lgetxattr(path, XATTR_NAME_LINK, buf, - MAX_LINKEA_SIZE); ++ size = lgetxattr(path, XATTR_NAME_LINK, attrs->loa_leh_buf, ++ sizeof(attrs->loa_leh_buf)); + if (size < 0) { + if (errno == ENOATTR) + LDEBUG("path [%s] has no [%s] xattr, ignoring\n", + path, XATTR_NAME_LINK); + else + LDEBUG("failed to get [%s] xattr of path [%d]: %s, ignoring\n", + XATTR_NAME_LINK, path, strerror(errno)); + return -errno; + } + - rc = decode_linkea(buf, size); ++ rc = lipe_object_attrs_set_links(attrs, attrs->loa_leh_buf, size); + if (rc) { + LERROR("failed to decode linkea for path [%s]\n", path); + return -1; + } - attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_LINKEA; ++ + return 0; +} + +static int get_lmv_ea_posix(struct lipe_object *object, + struct lipe_object_attrs *attrs) +{ + ssize_t rc; + const char *path = object->u.lo_posix.lop_path; + + memset(attrs->loa_lmv_buf, 0, sizeof(attrs->loa_lmv_buf)); + + rc = lgetxattr(path, XATTR_NAME_LMV, attrs->loa_lmv_buf, + sizeof(attrs->loa_lmv_buf)); + if (rc < 0) { + if (errno == ENOATTR) { + LDEBUG("path [%s] has no [%s] xattr, ignoring\n", + path, XATTR_NAME_LMV); + return 1; + } else { + LDEBUG("failed to get [%s] xattr of path [%d]: %s, ignoring\n", + XATTR_NAME_LMV, path, strerror(errno)); + return -errno; + } + } + + return 0; +} + +static int get_lma_ea_posix(struct lipe_object *object, + struct lipe_object_attrs *attrs) +{ + char buf[MAX_LINKEA_SIZE]; + struct lustre_mdt_attrs *lma; + ssize_t size; + const char *path = object->u.lo_posix.lop_path; + + size = lgetxattr(path, XATTR_NAME_LMA, buf, + MAX_LINKEA_SIZE); + if (size < 0) { + if (errno == ENOATTR) + LDEBUG("path [%d] has no [%s] xattr, ignoring\n", + path, XATTR_NAME_LMA); + else + LDEBUG("failed to get [%s] xattr of path [%d]: %s, ignoring\n", + XATTR_NAME_LMA, path, strerror(errno)); + return -errno; + } + + lma = (struct lustre_mdt_attrs *)buf; + fid_le_to_cpu(&attrs->loa_fid, &lma->lma_self_fid); + snprintf(attrs->loa_fid_str, sizeof(attrs->loa_fid_str), DFID_NOBRACE, + PFID(&attrs->loa_fid)); + attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_LMAEA; + return 0; +} + +static int get_hsm_ea_posix(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) ++ struct lipe_object_attrs *attrs) +{ + char buf[MAX_LINKEA_SIZE]; + struct hsm_attrs *hsm; + int rc; + int size; + struct hsm_user_state *hus = &attrs->loa_hsm_state; + const char *path = object->u.lo_posix.lop_path; + + LASSERT(sizeof(*hsm) < MAX_LINKEA_SIZE); + size = lgetxattr(path, XATTR_NAME_HSM, buf, + MAX_LINKEA_SIZE); + if (size < 0) { + if (errno == ENOATTR) { - LDEBUGW(watch, ++ LDEBUGW(&attrs->loa_fid, + "path [%d] has no [%s] xattr, ignoring\n", + path, XATTR_NAME_HSM); + hus->hus_states = 0; + hus->hus_archive_id = 0; + } else { + LDEBUG("failed to get [%s] xattr of path [%s]: %s, ignoring\n", + XATTR_NAME_HSM, path, strerror(errno)); + return -errno; + } + } else { + hsm = (struct hsm_attrs *)buf; + rc = lustre_hsm2user(hsm, hus); + if (rc) { + LERROR("failed to extract [%s] xattr for path [%s], rc = %d\n", + XATTR_NAME_HSM, path, rc); + return rc; + } + } + attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_HSMEA; + return 0; +} + +static int get_lum_ea_posix(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) ++ struct lipe_object_attrs *attrs) +{ + int rc; + int size; + struct lov_user_md *lov = attrs->loa_lum; + const char *path = object->u.lo_posix.lop_path; + + size = lgetxattr(path, XATTR_NAME_LOV, + (char *)lov, attrs->loa_lum_size); + if (size < 0) { + if (errno == ENOATTR) + LDEBUG("path [%d] has no [%s] xattr, ignoring\n", + path, XATTR_NAME_LMA); + else + LDEBUG("failed to get [%s] xattr of path [%s]: %s, ignoring\n", + XATTR_NAME_LMA, path, strerror(errno)); + return -errno; + } + + rc = decode_lum(lov, size); + if (rc) { + LERROR("failed to decode lovea for path [%s]\n", path); + return rc; + } + attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_LOVEA; + return 0; +} + +#ifdef HAVE_LAZY_SIZE_ON_MDT +/* + * llapi_layout_get_by_xattr() and LSoM are both included in Lustre-2.12, + * so no need to duplicate the macros. + */ +static int get_som_ea_posix(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) ++ struct lipe_object_attrs *attrs) +{ + int size; + struct lustre_som_attrs *som = &attrs->loa_som; + const char *path = object->u.lo_posix.lop_path; + + size = lgetxattr(path, XATTR_NAME_SOM, + (char *)som, sizeof(*som)); + if (size < 0) { + if (errno == ENOATTR) + LDEBUG("path [%d] has no [%s] xattr, ignoring\n", + path, XATTR_NAME_LMA); + else + LDEBUG("failed to get [%s] xattr of path [%s]: %s, ignoring\n", + XATTR_NAME_LMA, path, strerror(errno)); + return -errno; + } + + if (size != sizeof(*som)) { + LERROR("unexpected size of [%s] xattr for path [%s], expected [%d], got [%d]\n", + XATTR_NAME_SOM, path, sizeof(*som), size); + return -ENODATA; + } + + lustre_som_swab(som); + attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_SOM; + return 0; +} +#endif /* HAVE_LAZY_SIZE_ON_MDT */ + +static int check_dir_empty_posix(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) ++ struct lipe_object_attrs *attrs) +{ + DIR *d; + struct dirent64 *dent; + bool is_empty = true; + const char *path = object->u.lo_posix.lop_path; + + d = opendir(path); + if (d == NULL) { + LDEBUG("failed to opendir [%s]: %s\n", path, strerror(errno)); + return -errno; + } + + while ((dent = readdir64(d)) != NULL) { + if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) + continue; + is_empty = false; + break; + } + closedir(d); + + attrs->loa_is_empty = is_empty; + attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_EMPTY; + return 0; +} + +static int get_dir_entries_posix(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) ++ struct lipe_object_attrs *attrs) +{ + DIR *d; + struct dirent64 *dent; + int64_t entry_number = 0; + const char *path = object->u.lo_posix.lop_path; + + d = opendir(path); + if (d == NULL) { + LDEBUG("failed to opendir [%s]: %s\n", path, strerror(errno)); + return -errno; + } + + while ((dent = readdir64(d)) != NULL) { + if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) + continue; + entry_number++; + } + closedir(d); + + attrs->loa_entries = entry_number; + attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_ENTRIES; + + attrs->loa_is_empty = (entry_number == 0); + attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_EMPTY; + return 0; +} + - void print_object_prefix_posix(struct lipe_object *object, int level, - bool watch) ++void print_object_prefix_posix(struct lipe_object *object, int level) +{ - _lipe_logging(level, watch, "path [%s], ", object->u.lo_posix.lop_path); ++ _lipe_logging(level, false, "path [%s], ", object->u.lo_posix.lop_path); +} + +static int posix_get_xattr(struct lipe_object *object, + struct lipe_object_attrs *attrs, const char *path, + const char *name) +{ + int rc; + char *value; + size_t value_len; + struct lipe_xattr *xattr; + + rc = get_xattr_value(path, name, &value, &value_len); + if (rc) { + OBJ_ERROR(object, "failed to get value for xattr [%s]\n", + name); + return rc; + } + + LIPE_ALLOC_PTR(xattr); + if (xattr == NULL) { + OBJ_ERROR(object, "not enough memory\n"); + rc = -ENOMEM; + goto out; + } + + xattr->lx_name = strdup(name); + if (xattr->lx_name == NULL) { + OBJ_ERROR(object, "not enough memory\n"); + rc = -ENOMEM; + goto out_xattr; + } + xattr->lx_value = value; + xattr->lx_value_len = value_len; + lipe_list_add_tail(&xattr->lx_linkage, &attrs->loa_xattrs); + return 0; +out_xattr: + LIPE_FREE_PTR(xattr); +out: + free(value); + return rc; +} + +static int posix_get_all_xattrs(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) ++ struct lipe_object_attrs *attrs) +{ + int rc; + char *name; + size_t namelen; + char *xattr_list; + ssize_t list_size; + const char *path = object->u.lo_posix.lop_path; + + rc = get_xattr_list(path, &xattr_list, &list_size); + if (rc) { + OBJ_ERROR(object, "failed to get xattr list\n"); + return rc; + } + + name = xattr_list; + while (name < xattr_list + list_size) { + rc = posix_get_xattr(object, attrs, path, name); + if (rc) { + OBJ_ERROR(object, "failed to get xattr of [%s]\n", + name); + goto out; + } + namelen = strlen(name) + 1; + name += namelen; + } + + attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_ALL_XATTR; +out: + free(xattr_list); + return rc; +} + +static int posix_get_projid(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) ++ struct lipe_object_attrs *attrs) +{ + int fd; + int rc; + struct fsxattr fsx; + const char *path = object->u.lo_posix.lop_path; + + LASSERT(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_ATTR); + if (!S_ISREG(attrs->loa_mode) && !S_ISDIR(attrs->loa_mode)) { + attrs->loa_projid = 0; + attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_PROJID; + return 0; + } + + fd = open(path, O_RDONLY | O_NOCTTY | O_NDELAY); + if (fd < 0) { + OBJ_ERROR(object, "failed to open file: %s\n", + strerror(errno)); + return -errno; + } + +#if defined(FS_IOC_FSGETXATTR) + rc = ioctl(fd, FS_IOC_FSGETXATTR, &fsx); +#elif defined(LL_IOC_FSGETXATTR) + rc = ioctl(fd, LL_IOC_FSGETXATTR, &fsx); +#else +#error IOC_FSGETXATTR is not defined +#endif + rc = 0; + if (rc) { + OBJ_ERROR(object, "failed to ioctl fsgetxattr: %s\n", + strerror(errno)); + rc = -errno; + goto out; + } + attrs->loa_projid = fsx.fsx_projid; + attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_PROJID; +out: + close(fd); + return rc; +} + +struct lipe_backfs_operations posix_operations = { + .print_object_prefix = print_object_prefix_posix, + .get_lma_ea = get_lma_ea_posix, + .get_link_ea = get_link_ea_posix, + .get_lmv_ea = get_lmv_ea_posix, + .get_hsm_ea = get_hsm_ea_posix, + .get_lum_ea = get_lum_ea_posix, +#ifdef HAVE_LAZY_SIZE_ON_MDT + .get_som_ea = get_som_ea_posix, +#endif + .check_dir_empty = check_dir_empty_posix, + .get_dir_entries = get_dir_entries_posix, + .get_all_xattrs = posix_get_all_xattrs, + .get_projid = posix_get_projid, +};