From 7d665cba3b245f3d49166de48c39fd9df9843633 Mon Sep 17 00:00:00 2001 From: "John L. Hammond" Date: Mon, 23 Aug 2021 17:23:19 -0500 Subject: [PATCH] Squashed 'lipe/' changes from 39ab2a0..2a015e6 2a015e6 EX-2453 lipe: add lipe_scan2 to RPM 989583f EX-2453 lipe: add xattr_name() test d9cff4d EX-2453 lipe: lipe_scan2 attribute handling 8c46fbb EX-2453 lipe: defer getting object paths 3fad488 EX-2453 lipe: remove 'all_inode' parameters 6445f44 EX-2453 lipe: remove watch_fid parameters c40e612 EX-2453 lipe: add lipe_scan2 8128fe0 EX-1325 lamigo: improve debug/error messages 7055a64 EX-2453 lipe: add paths handling to lipe_scan() 307dbcb EX-2453 lipe: move lipe_scan2() wrapper to policy.c 788a15c EX-3623 lipe: include errno.h in lipe_expression_test.c 288b18b EX-3476 lipe-scripts: Add --now to hp stop 1f01cf2 EX-2797 lpurge: initial support for DoM 2327aad EX-2853 lamigo: initial supoprt for DoM e524af6 EX-3198 lipe: add lipe_convert_expr 3981e3b EX-2453 lipe: add LIPE_OBJECT_ATTR_PATHS d87813e EX-2453 lipe: add json formatting to lipe_object_attrs 071e07b EX-2600 lipe: parse link xattr once f1c8142 EX-2453 lipe: add lipe_scan2() 61f28b5 EX-2453 lipe: reduce surface area of results and counters git-subtree-dir: lipe git-subtree-split: 2a015e67c4d3cdc7802178f1ad0fd85fc251fb0a --- .gitignore | 1 + Makefile.am | 1 + lipe.spec.in | 4 + lipe_convert_expr | 12 + man/lipe_find.1 | 4 +- man/lipe_scan.1 | 3 - pybuild/lipe_expression_tests.py | 114 ++- pylipe/__init__.py | 1 + pylipe/lipe_convert_expr.py | 1914 ++++++++++++++++++++++++++++++++++++++ pylipe/lipe_hotpool_test.py | 5 + scripts/stratagem-hp-stop.sh | 65 +- src/Makefile.am | 19 +- src/debug.c | 3 +- src/debug.h | 64 +- src/flist.c | 42 +- src/general_policy.c | 135 +-- src/general_policy.h | 8 +- src/lamigo.c | 344 ++++--- src/lamigo.h | 1 + src/ldiskfs_read_ldd.h | 5 + src/lipe_expression_test.c | 3 +- src/lipe_ldiskfs.c | 43 +- src/lipe_ldiskfs.h | 12 +- src/lipe_object_attrs.c | 533 +++++++++++ src/lipe_object_attrs.h | 37 + src/{lipe.c => lipe_scan.c} | 14 +- src/lipe_scan2.c | 446 +++++++++ src/lipe_scan2.h | 54 ++ src/lipe_ssh.c | 20 +- src/lipe_zfs.c | 46 +- src/lipe_zfs.h | 12 +- src/lpurge.c | 146 ++- src/lustre_ea.c | 107 --- src/lustre_ea.h | 4 - src/lustre_ea_ldiskfs.c | 48 +- src/policy.c | 743 ++++++++------- src/policy.h | 93 +- src/posix.c | 33 +- src/posix.h | 11 +- src/posix_ea.c | 44 +- 40 files changed, 4180 insertions(+), 1014 deletions(-) create mode 100755 lipe_convert_expr create mode 100644 pylipe/lipe_convert_expr.py create mode 100644 src/lipe_object_attrs.c rename src/{lipe.c => lipe_scan.c} (94%) create mode 100644 src/lipe_scan2.c create mode 100644 src/lipe_scan2.h diff --git a/.gitignore b/.gitignore index cf44836..2dd465f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ /src/lipe_hsm_copytool /src/lipe_hsm_remover /src/lipe_scan +/src/lipe_scan2 /src/lpurge # Automake diff --git a/Makefile.am b/Makefile.am index a9f0a1b..3fa8b6c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -33,6 +33,7 @@ PYTHON_COMMANDS = \ lipe_copytool_manager \ lipe_expression_tests \ lipe_find \ + lipe_convert_expr \ lipe_run_action \ lipe_hsm_check \ lipe_install \ diff --git a/lipe.spec.in b/lipe.spec.in index 161bd4d..038c6ea 100644 --- a/lipe.spec.in +++ b/lipe.spec.in @@ -239,6 +239,7 @@ cp \ ldsync \ lipe_copytool_manager \ lipe_find \ + lipe_convert_expr \ lipe_run_action \ lipe_hsm_check \ lipe_install \ @@ -260,6 +261,7 @@ cp \ src/ldumpstripe \ src/lfill \ src/lipe_scan \ + src/lipe_scan2 \ src/lipe_hsm_copytool \ src/lipe_hsm_remover \ $RPM_BUILD_ROOT%{_bindir} @@ -435,7 +437,9 @@ rm -rf $RPM_BUILD_ROOT %{_bindir}/lipe_test %{_bindir}/lfill %{_bindir}/lipe_scan +%{_bindir}/lipe_scan2 %{_bindir}/lipe_find +%{_bindir}/lipe_convert_expr %{python_sitelib}/pylipe %config(noreplace) %{_sysconfdir}/lipe_install.conf %config(noreplace) %{_sysconfdir}/lipe_test.conf diff --git a/lipe_convert_expr b/lipe_convert_expr new file mode 100755 index 0000000..0fa2320 --- /dev/null +++ b/lipe_convert_expr @@ -0,0 +1,12 @@ +#!/usr/bin/python -u +# Copyright (c) 2021 DataDirect Networks, Inc. +# All Rights Reserved. +# Author: Jian Yu +""" +Convert expression arguments to a single expression string +with Polish notation (prefix) or infix notation +""" +from pylipe import lipe_convert_expr + +if __name__ == "__main__": + lipe_convert_expr.main() diff --git a/man/lipe_find.1 b/man/lipe_find.1 index 3f6cddf..a933822 100644 --- a/man/lipe_find.1 +++ b/man/lipe_find.1 @@ -529,7 +529,9 @@ perm_at_least(07776) .IP fname_match() fname_match("fname[^0-8]"),fname_match("fname*") .IP xattr_match() -xattr_match("value", "name") +xattr_match("name", "value") +.IP xattr_name() +xattr_name("name") .IP ost() ost(0), ost(1) .IP user2id() diff --git a/man/lipe_scan.1 b/man/lipe_scan.1 index 1be1e0a..d9fb0f4 100644 --- a/man/lipe_scan.1 +++ b/man/lipe_scan.1 @@ -26,9 +26,6 @@ uses multiple threads to scan the MDT/OST in parallel, which is usually also fas .B -a, --abort-failure Abort on failure .TP -.B -A, --all-inode -All inode will be checked -.TP .B -c FILE,--config-file=FILE Use FILE as the configure file instead of /etc/lipe.conf .TP diff --git a/pybuild/lipe_expression_tests.py b/pybuild/lipe_expression_tests.py index 82baa14..4b9b91f 100755 --- a/pybuild/lipe_expression_tests.py +++ b/pybuild/lipe_expression_tests.py @@ -662,45 +662,81 @@ def main_with_exception(): test_expression('group2id("root") == gid', 0, options="--gid 100") # xattr_match() tests - test_expression_no_evaluate('xattr_match("value", "name")') - test_expression_no_evaluate('xattr_match("value*", "name")') - test_expression_no_evaluate('xattr_match("value*", "name*")') - test_expression_no_evaluate('xattr_match("*value*", "*name*")') - test_expression_no_evaluate('xattr_match("value" , "*name*")') - test_expression_no_evaluate('xattr_match("value" , "name")') - test_expression_no_evaluate('xattr_match("value","name")') - test_expression_no_evaluate('xattr_match( "value","name")') - test_expression_no_evaluate('xattr_match( "value","name")') - test_expression_no_evaluate('xattr_match("value","name" )') - test_expression_no_evaluate('xattr_match("value","name" )') - # xattr_match("\"value", "name") - test_expression_no_evaluate('xattr_match("\\\"value", "name")') - # xattr_match("value\"", "name") - test_expression_no_evaluate('xattr_match("value\\\"", "name")') - # xattr_match("value", "\"name") - test_expression_no_evaluate('xattr_match("value", "\\\"name")') - # xattr_match("value", "\"name\"") - test_expression_no_evaluate('xattr_match("value", "name\\\"")') - # xattr_match("\\value", "name") - test_expression_no_evaluate('xattr_match("\\\\value", "name")') - # xattr_match("value", "\\name") - test_expression_no_evaluate('xattr_match("value", "\\\\name")') - # xattr_match("\\value\\", "\\name\\") - test_expression_no_evaluate('xattr_match("\\\\value\\\\", "\\\\name\\\\")') - test_expression_invalid('xattr_match(value, name)') - test_expression_invalid('xattr_match(value, "name")') - test_expression_invalid('xattr_match("value", name)') - test_expression_invalid('xattr_match("value", "name)') - test_expression_invalid('xattr_match("value", name")') - test_expression_invalid('xattr_match("value" "name")') - test_expression_invalid('xattr_match"value", "name")') - test_expression_invalid('xattr_match("value", "name"') - test_expression_invalid('xattr_match("value")') - test_expression_invalid('xattr_match(value)') - test_expression_invalid('xattr_match(\\"value", "name")') - test_expression_invalid('xattr_match("value\\", "name")') - test_expression_invalid('xattr_match("value", \\"name")') - test_expression_invalid('xattr_match("value", "name\\")') + # xattr_match() tests + test_expression_no_evaluate('xattr_match("name", "value")') + test_expression_no_evaluate('xattr_match("name*", "value")') + test_expression_no_evaluate('xattr_match("name*", "value*")') + test_expression_no_evaluate('xattr_match("*name*", "*value*")') + test_expression_no_evaluate('xattr_match("name" , "*value*")') + test_expression_no_evaluate('xattr_match("name" , "value")') + test_expression_no_evaluate('xattr_match("name","value")') + test_expression_no_evaluate('xattr_match( "name","value")') + test_expression_no_evaluate('xattr_match( "name","value")') + test_expression_no_evaluate('xattr_match("name","value" )') + test_expression_no_evaluate('xattr_match("name","value" )') + # xattr_match("\"name", "value") + test_expression_no_evaluate('xattr_match("\\\"name", "value")') + # xattr_match("name\"", "value") + test_expression_no_evaluate('xattr_match("name\\\"", "value")') + # xattr_match("name", "\"value") + test_expression_no_evaluate('xattr_match("name", "\\\"value")') + # xattr_match("name", "\"value\"") + test_expression_no_evaluate('xattr_match("name", "value\\\"")') + # xattr_match("\\name", "value") + test_expression_no_evaluate('xattr_match("\\\\name", "value")') + # xattr_match("name", "\\value") + test_expression_no_evaluate('xattr_match("name", "\\\\value")') + # xattr_match("\\name\\", "\\value\\") + test_expression_no_evaluate('xattr_match("\\\\name\\\\", "\\\\value\\\\")') + test_expression_invalid('xattr_match(name, value)') + test_expression_invalid('xattr_match(name, "value")') + test_expression_invalid('xattr_match("name", value)') + test_expression_invalid('xattr_match("name", "value)') + test_expression_invalid('xattr_match("name", value")') + test_expression_invalid('xattr_match("name" "value")') + test_expression_invalid('xattr_match"name", "value")') + test_expression_invalid('xattr_match("name", "value"') + test_expression_invalid('xattr_match("name")') + test_expression_invalid('xattr_match(name)') + test_expression_invalid('xattr_match(\\"name", "value")') + test_expression_invalid('xattr_match("name\\", "value")') + test_expression_invalid('xattr_match("name", \\"value")') + test_expression_invalid('xattr_match("name", "value\\")') + + # xattr_name() tests + test_expression_no_evaluate('xattr_name("name")') + test_expression_no_evaluate('xattr_name("name*")') + test_expression_no_evaluate('xattr_name("name*")') + test_expression_no_evaluate('xattr_name("*name*")') + test_expression_no_evaluate('xattr_name("name" )') + test_expression_no_evaluate('xattr_name("name" )') + test_expression_no_evaluate('xattr_name( "name")') + test_expression_no_evaluate('xattr_name( "name")') + test_expression_no_evaluate('xattr_name("name")') + test_expression_no_evaluate('xattr_name("name")') + # xattr_name("\"name")') + test_expression_no_evaluate('xattr_name("\\\"name")') + # xattr_name("name\"")') + test_expression_no_evaluate('xattr_name("name\\\"")') + # xattr_name("name")') + test_expression_no_evaluate('xattr_name("name")') + # xattr_name("name")') + test_expression_no_evaluate('xattr_name("name")') + # xattr_name("\\name")') + test_expression_no_evaluate('xattr_name("\\\\name")') + # xattr_name("name")') + test_expression_no_evaluate('xattr_name("name")') + # xattr_name("\\name\\")') + test_expression_no_evaluate('xattr_name("\\\\name\\\\")') + test_expression_invalid('xattr_name(name)') + test_expression_invalid('xattr_name(name)') + test_expression_invalid('xattr_name("name", "value")') + test_expression_invalid('xattr_name("name" "value")') + test_expression_invalid('xattr_name("name",)') + test_expression_invalid('xattr_name"name")') + test_expression_invalid('xattr_name(name)') + test_expression_invalid('xattr_name(\\"name")') + test_expression_invalid('xattr_name("name\\")') test_expression_invalid('perm_equal()') test_expression_invalid('perm_equal(8)') diff --git a/pylipe/__init__.py b/pylipe/__init__.py index c758855..6e858a1 100644 --- a/pylipe/__init__.py +++ b/pylipe/__init__.py @@ -6,6 +6,7 @@ __all__ = ["ldsync", "lipe_common", "lipe_constant", "lipe_find", + "lipe_convert_expr", "lipe_flist_handle", "lipe_install", "lipe_install_nodeps", diff --git a/pylipe/lipe_convert_expr.py b/pylipe/lipe_convert_expr.py new file mode 100644 index 0000000..dd3229a --- /dev/null +++ b/pylipe/lipe_convert_expr.py @@ -0,0 +1,1914 @@ +# Copyright (c) 2017 DataDirect Networks, Inc. +# All Rights Reserved. +# Author: lixi@ddn.com +# Author: Jian Yu +# pylint: disable=too-many-lines +# pylint: disable=too-few-public-methods +# pylint: disable=protected-access +""" +Convert expression arguments to a single expression string +with Polish notation (prefix) or infix notation +""" + +import sys +import stat +import re + +# Local libs +from pylustre import utils +from pylustre import clog +from pylipe import lipe +from pylipe import lipe_constant + + +class LipeFindPredicate(object): + """ + Each single part of expression has an object of this type + """ + # pylint: disable=too-few-public-methods + def __init__(self, exclude=False): + self.lfp_exclude = exclude + + +class LipeFindType(LipeFindPredicate): + """ + The arg of type + """ + # pylint: disable=too-few-public-methods + def __init__(self, file_type, exclude=False): + self.lft_file_type = file_type + super(LipeFindType, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ("%s %s %s" % (lipe_constant.LIPE_POLICY_OPERATOR_NOT_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_TYPE, + self.lft_file_type)) + else: + return ("%s %s %s" % (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_TYPE, + self.lft_file_type)) + + +class LipeFindFid(LipeFindPredicate): + """ + The arg of fid + """ + # pylint: disable=too-few-public-methods + def __init__(self, fid_pattern, exclude=False): + self.lff_pattern = fid_pattern + super(LipeFindFid, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s("%s") 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_FUNCTION_FID_MATCH, self.lff_pattern)) + else: + return ('%s("%s")' % + (lipe_constant.LIPE_POLICY_FUNCTION_FID_MATCH, self.lff_pattern)) + + +class LipeFindFnmatch(LipeFindPredicate): + """ + The arg of fnamtch + """ + # pylint: disable=too-few-public-methods + def __init__(self, pattern, case_insensitive=False, exclude=False): + self.lff_pattern = pattern + self.lff_case_insensitive = case_insensitive + super(LipeFindFnmatch, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lff_case_insensitive: + function = lipe_constant.LIPE_POLICY_FUNCTION_FNAME_IMATCH + else: + function = lipe_constant.LIPE_POLICY_FUNCTION_FNAME_MATCH + + if self.lfp_exclude: + return ('%s %s("%s") 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + function, self.lff_pattern)) + else: + return ('%s("%s")' % + (function, self.lff_pattern)) + + +class LipeFindInum(LipeFindPredicate): + """ + The arg of inum + """ + # pylint: disable=too-few-public-methods + def __init__(self, inum, exclude=False): + self.lfi_inum = inum + super(LipeFindInum, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s %s' % + (lipe_constant.LIPE_POLICY_OPERATOR_NOT_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_INUM, + self.lfi_inum)) + else: + return ('%s %s %s' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_INUM, + self.lfi_inum)) + + +def find_equal(target): + """ + Return the index of the first = without excape, need to escape \\ and \= + """ + equal_index = -1 + escaped = False + for char in target: + equal_index += 1 + if char == '\\': + escaped = not escaped + else: + if char == '=': + if not escaped: + return equal_index + escaped = False + return -1 + + +class LipeFindXattr(LipeFindPredicate): + """ + The arg of xattr + """ + # pylint: disable=too-few-public-methods + def __init__(self, name_pattern, value_pattern, exclude=False): + self.lfx_name_pattern = name_pattern + self.lfx_value_pattern = value_pattern + super(LipeFindXattr, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s("%s", "%s") 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_FUNCTION_XATTR_MATCH, + self.lfx_name_pattern, self.lfx_value_pattern)) + else: + return ('%s("%s", "%s")' % + (lipe_constant.LIPE_POLICY_FUNCTION_XATTR_MATCH, + self.lfx_name_pattern, self.lfx_value_pattern)) + + +def xattr_option_parse(log, arg, negative): + """ + Parse the option and return an LipeFindXattr object + + arg is a pair of $name=$value. Both $name and $value can be a pattern. + $name and $value can contain = as long as it is escaped by \= + \ itself can be escaped too by \\ + """ + equal_index = find_equal(arg) + if equal_index == -1: + log.cl_error("cannot find [=] in argument [%s] of -xattr", arg) + return None + + if equal_index == 0: + log.cl_error("no name pattern before [=] in argument [%s] of -xattr", arg) + return None + + if equal_index == len(arg) - 1: + log.cl_error("no value pattern after [=] in argument [%s] of -xattr", arg) + return None + + name_pattern = arg[0:equal_index] + # Remove the escape \\ or \= + name_pattern = name_pattern.replace("\\\\", "\\").replace("\\=", "=") + + value_pattern = arg[equal_index + 1:] + # Remove the escape \\ or \= + value_pattern = value_pattern.replace("\\\\", "\\").replace("\\=", "=") + + return LipeFindXattr(name_pattern, value_pattern, exclude=negative) + + +def inum_option_parse(log, arg, negative): + """ + Parse the option and return an LipeFindInum object + + arg is a decimal GID. + """ + try: + inum = str(int(arg, 10)) + except ValueError: + inum = None + + # Not a index number + if inum is None: + log.cl_error("invalid argument [%s] to [-inum]", arg) + return None + + return LipeFindInum(inum, exclude=negative) + + +class LipeFindLinks(LipeFindPredicate): + """ + The arg of links + """ + # pylint: disable=too-few-public-methods + def __init__(self, links, exclude=False): + self.lfl_links = links + super(LipeFindLinks, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s %s' % + (lipe_constant.LIPE_POLICY_OPERATOR_NOT_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_NLINK, + self.lfl_links)) + else: + return ('%s %s %s' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_NLINK, + self.lfl_links)) + + +def links_option_parse(log, arg, negative): + """ + Parse the option and return an LipeFindLinks object + + arg is a decimal GID. + """ + try: + links = str(int(arg, 10)) + except ValueError: + log.cl_error("invalid argument [%s] to [-links]", arg) + return None + + return LipeFindLinks(links, exclude=negative) + + +class LipeFindArg(object): + """ + The arg of lipe_convert_expr + """ + # pylint: disable=too-few-public-methods,too-many-instance-attributes + def __init__(self): + self.lfa_predicates = [] + self.lfa_super_predicate = None + + def lfa_expression(self): + """ + Return the string of expression + """ + if self.lfa_super_predicate: + return self.lfa_super_predicate + + if len(self.lfa_predicates) == 0: + return "1" + expression = "" + for predicate in self.lfa_predicates: + if expression == "": + expression = predicate.lfp_expression() + else: + expression = (lipe_constant.LIPE_POLICY_OPERATOR_AND + " " + + expression + " " + predicate.lfp_expression()) + return expression + + +def exit_with_tip(log): + """ + Exit with tip + """ + log.cl_error("Try '%s --help' for more", sys.argv[0]) + sys.exit(-1) + + +def lipe_find_expression_usage(): + """ + Show expression usage + """ + buff = " Following attributes can be used in the expression:\n" + for k, v in lipe_constant.LIPE_POLICY_ATTRIBUTE_DESCRIPTIONS.items(): + buff += " %s, %s\n" % (k, v) + + buff += " Following functions can be used in the expression:\n" + for k, v in lipe_constant.LIPE_POLICY_FUNCTION_EXAMPLES.items(): + buff += " %s(), %s\n" % (k, v) + + buff += " Following operators can be used in the expression:\n" + for k, v in lipe_constant.LIPE_POLICY_OPERATOR_NAMES.items(): + buff += " %s, %s\n" % (k, v) + return buff + + +def usage(): + """ + Print usage string + """ + utils.oprint("Usage: %s [options] \n" + "options:\n" + " -h|-help|--help, print this message\n" + "expressions:\n" + " [[!] -amin [+-]\n" + " [[!] -atime [+-]\n" + " [[!] -blocks [+-][bcKMGTPE]\n" + " [[!] -cmin [+-]\n" + " [[!] -comp-count|-component-count [+-]\n" + " [[!] -ctime [+-]\n" + " [[!] -empty\n" + " [[!] -entries [+-]\n" + " [[!] -fid \n" + " [[!] -gid ]\n" + " [[!] -group ]\n" + " [[!] -iname ]\n" + " [[!] -inum ]\n" + " [[!] -layout mdt,raid0,released]\n" + " [[!] -links ]\n" + " [[!] -mmin [+-]\n" + " [[!] -mtime [+-]\n" + " [[!] -name ]\n" + " [[!] -nogroup]\n" + " [[!] -nouser]\n" + " [[!] -ost ]\n" + " [[!] -perm [-/]\n" + " [[!] -pool \n" + " [[!] -pool-regex \n" + " [[!] -projid [+-]\n" + " [[!] -size [+-][bcKMGTPE]]\n" + " [[!] -stripe-count [+-]\n" + " [[!] -stripe-index \n" + " [[!] -stripe-size [+-][bcKMGTPE]]\n" + " [[!] -type ]\n" + " [[!] -uid ]\n" + " [[!] -used [+-]]\n" + " [[!] -user ]\n" + " [[!] -xattr =\n" + " !: used before an option indicates 'NOT' requested attribute\n" + " minutes: a decimal number of minutes\n" + " days: a decimal number of days\n" + " empty: File is empty and is either a regular file or a directory.\n" + " entry_number: a decimal entry number.\n" + " pattern: a shell wildcard pattern, e.g. '*'\n" + " inode_number: a decimal inode number\n" + " link_number: a decimal link number\n" + " filetype:\n" + " b: block\n" + " c: char (default)\n" + " d: dir\n" + " f: file\n" + " l: symlink\n" + " p: fifo\n" + " s: socket\n" + " gid: decimal group ID\n" + " group: group name\n" + " ost_index: index of OST that starts from 0\n" + " ost_uuid: UUID of OST, e.g. fsname-OST000a or OST000a\n" + " size: N[bKMGTPE], file uses N units of space\n" + " b: block, 512 bytes\n" + " c: byte\n" + " K|k: kibibyte, 1024 bytes (default)\n" + " M|m: mebibyte, 1024 kibibytes\n" + " G|g: gibibyte, 1024 mebibytes\n" + " T|t: tebibyte, 1024 gibibytes\n" + " P|p: pebibyte, 1024 tebibytes\n" + " E|e: exbibyte, 1024 pebibytes\n" + " uid: decimal user ID\n" + " user: user name\n" + " mode: permission bits, e.g. 07777\n" + " -expr \n" + " Specify a LiPE policy expression. When -expr expression is specified, other\n" + " pattern option (such as -atime or -user) should NOT be specified, since\n" + " they will all be ignored.\n" + " The expression can either have infix notation or have prefix (Polish) notation.\n" + " Following two expressions will both find all the files that are modified in\n" + " the last 10 days:" + " infix expression: -expr \"mtime > sys_time - 10 * days\"\n" + " prefix expression: -expr \"> mtime - sys_time * 10 days\"\n%s" + % (sys.argv[0], lipe_find_expression_usage())) + + +def process_exec_args(arg_index, arg_max, argv): + """ + Process arguments of exec option + :return argc, count of exec arguments, 0 for no argument, -1 for no EOF + :return args, arguments list + """ + exec_args = list() + eof = None + while arg_index < arg_max: + arg = argv[arg_index] + if arg in lipe.LAT_SHELL_CMD_EXEC_EOF_MARKS: + eof = arg + break + exec_args.append(arg) + arg_index += 1 + + if not exec_args: + return 0, None + + if eof is None: + return -1, None + + exec_args.append(eof) + + return len(exec_args), exec_args + + +def process_leading_options(argc, argv): + """ + Process the leading options + """ + # pylint: disable=too-many-branches,too-many-statements + end_of_leading_options = 1 + while end_of_leading_options < argc: + arg = argv[end_of_leading_options] + if arg == '-h' or arg == "--help" or arg == "-help": + usage() + sys.exit(0) + else: + break + end_of_leading_options += 1 + + return end_of_leading_options + + +def file_option2lipe_constant(arg): + """ + Convert the file type option to the string used in lipe expression tool + """ + if arg == "d": + return lipe_constant.LPC_S_IFDIR + elif arg == "c": + return lipe_constant.LPC_S_IFCHR + elif arg == "b": + return lipe_constant.LPC_S_IFBLK + elif arg == "f": + return lipe_constant.LPC_S_IFREG + elif arg == "p": + return lipe_constant.LPC_S_IFIFO + elif arg == "l": + return lipe_constant.LPC_S_IFLNK + elif arg == "s": + return lipe_constant.LPC_S_IFSOCK + else: + return None + + +def file_type2option(filetype): + """ + Convert the file type option to the string used in lipe expression tool + """ + if filetype == stat.S_IFDIR: + return "d" + elif filetype == stat.S_IFCHR: + return "c" + elif filetype == stat.S_IFBLK: + return "b" + elif filetype == stat.S_IFREG: + return "f" + elif filetype == stat.S_IFIFO: + return "p" + elif filetype == stat.S_IFLNK: + return "l" + elif filetype == stat.S_IFSOCK: + return "s" + else: + return None + + +class LipeFindOST(LipeFindPredicate): + """ + The predicate of OST + """ + # pylint: disable=too-few-public-methods + def __init__(self, ost_index, exclude=False): + self.lfost_index = ost_index + super(LipeFindOST, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s(%s) 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_FUNCTION_OST, + self.lfost_index)) + else: + return ('%s(%s)' % + (lipe_constant.LIPE_POLICY_FUNCTION_OST, + self.lfost_index)) + + +def ost_option_parse(log, arg, negative): + """ + Parse the option and return an LipeFindOST object + + arg is either 1) a decimal/hexadecimal index of OST that starts from 0 or + 2) a UUID of OST, e.g. fsname-OST000a or OST000a. + """ + try: + ost_index = str(int(arg)) + except ValueError: + ost_index = None + + # Not a index number + if ost_index is None: + # First get rid of the leading "fsname-" + fields = arg.split("-") + field_number = len(fields) + if field_number == 1: + # No leading "fsname-" + ost_uuid = fields[0] + else: + # '-' is allowed in fsname, so need to find the last '-' in the + # OST name as the separator + ost_uuid = fields[field_number - 1] + + if not ost_uuid.startswith("OST"): + log.cl_error("invalid UUID of OST [%s]: should start with 'OST'", + ost_uuid) + return None + + index_string = ost_uuid[3:] + try: + ost_index = str(int(index_string, 16)) + except ValueError: + ost_index = None + if ost_index is None: + log.cl_error("invalid UUID of OST [%s]: should end with hexadecimal index", + ost_uuid) + return None + + return LipeFindOST(ost_index, exclude=negative) + + +class LipeFindUID(LipeFindPredicate): + """ + The predicate of user ID + """ + # pylint: disable=too-few-public-methods + def __init__(self, uid, exclude=False): + self.lfuid = uid + super(LipeFindUID, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s %s' % + (lipe_constant.LIPE_POLICY_OPERATOR_NOT_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_UID, + self.lfuid)) + else: + return ('%s %s %s' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_UID, + self.lfuid)) + + +def uid_option_parse(log, arg, negative): + """ + Parse the option and return an LipeFindUID object + + arg is a decimal UID. + """ + try: + uid = str(int(arg, 10)) + except ValueError: + uid = None + + # Not a index number + if uid is None: + log.cl_error("invalid argument [%s] to [-uid]", arg) + return None + + return LipeFindUID(uid, exclude=negative) + + +class LipeFindUser(LipeFindPredicate): + """ + The predicate of user + """ + # pylint: disable=too-few-public-methods + def __init__(self, username, exclude=False): + self.lfuser = username + super(LipeFindUser, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + # lipe_find is lazy, it depends on the core to transfer user to uid + if self.lfp_exclude: + return ('%s %s("%s") 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_FUNCTION_USER, + self.lfuser)) + else: + return ('%s("%s")' % + (lipe_constant.LIPE_POLICY_FUNCTION_USER, + self.lfuser)) + + +def user_option_parse(arg, negative): + """ + Parse the option and return an LipeFindUser object + + arg can be a user name or a decimal user ID. + """ + return LipeFindUser(arg, exclude=negative) + + +def lipe_find_user_expression_format(expr): + """ + Reformat user input expression if necessary to make sure it can be + well processed by lipe expression engine. + """ + assert isinstance(expr, str) + + # only cleanup extra space for prefix expression + if expr.startswith(tuple(lipe_constant.LIPE_POLICY_OPERATORS)): + return re.sub(' +', ' ', expr) + + expr_parentheses = ('(', ')') + expr_stack = list() + expr = expr.strip().replace(' ', '') + start = 0 + expr_len = len(expr) + idx = 0 + while (idx < expr_len): + if idx + 1 < expr_len: + double_char_operator = expr[idx: idx + 2] + if double_char_operator in lipe_constant.LIPE_POLICY_OPERATORS: + item = expr[start: idx] + expr_stack.append(item) + expr_stack.append(" %s " % double_char_operator) + idx += 2 + start = idx + continue + + single_char_operator = expr[idx: idx + 1] + if single_char_operator in lipe_constant.LIPE_POLICY_OPERATORS or \ + single_char_operator in expr_parentheses or \ + single_char_operator == ',': + item = expr[start: idx] + expr_stack.append(item) + if single_char_operator in expr_parentheses: + expr_stack.append(single_char_operator) + elif single_char_operator in lipe_constant.LIPE_POLICY_OPERATORS: + expr_stack.append(" %s " % single_char_operator) + else: + expr_stack.append("%s " % single_char_operator) + idx += 1 + start = idx + continue + idx += 1 + expr_stack.append(expr[start: idx]) + return "".join(expr_stack) + + +class LipeFindGID(LipeFindPredicate): + """ + The predicate of group ID + """ + # pylint: disable=too-few-public-methods + def __init__(self, gid, exclude=False): + self.lfgid = gid + super(LipeFindGID, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s %s' % + (lipe_constant.LIPE_POLICY_OPERATOR_NOT_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_GID, + self.lfgid)) + else: + return ('%s %s %s' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_GID, + self.lfgid)) + + +def gid_option_parse(log, arg, negative): + """ + Parse the option and return an LipeFindGID object + + arg is a decimal GID. + """ + try: + gid = str(int(arg, 10)) + except ValueError: + gid = None + + # Not a index number + if gid is None: + log.cl_error("invalid argument [%s] to [-gid]", arg) + return None + + return LipeFindGID(gid, exclude=negative) + + +class LipeFindGroup(LipeFindPredicate): + """ + The predicate of group + """ + # pylint: disable=too-few-public-methods + def __init__(self, group, exclude=False): + self.lfgroup = group + super(LipeFindGroup, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + # lipe_find is lazy, it depends on the core to transfer group to gid + if self.lfp_exclude: + return ('%s %s("%s") 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_FUNCTION_GROUP, + self.lfgroup)) + else: + return ('%s("%s")' % + (lipe_constant.LIPE_POLICY_FUNCTION_GROUP, + self.lfgroup)) + + +def group_option_parse(arg, negative): + """ + Parse the option and return an LipeFindGroup object + + arg can be a group name or a decimal group ID. + """ + return LipeFindGroup(arg, exclude=negative) + + +class LipeFindXtime(LipeFindPredicate): + """ + The predicate of atime/ctime/mtime + """ + # Expression for N: + # (xtime > sys_time - (N + 1) * days) && (xtime <= sys_time - N * days) + # && > xtime - sys_time * + N 1 days <= xtime - sys_time * N days + # + # Expression for +N: + # xtime < sys_time - (N + 1) * days + # < xtime - sys_time * + N 1 days + # + # Expression for -N: + # xtime > sys_time - N * days + # > xtime - sys_time * N days + # + LFX_EXPRESSION_INFIX = ("(%s > sys_time - (%s + 1) * %s) && " + "(%s <= sys_time - %s * %s)") + LFX_EXPRESSION_PREFIX = ("&& > %s - sys_time * + %s 1 %s <= %s - sys_time * %s %s") + + LFX_GREATER_EXPRESSION_INFIX = "%s < sys_time - (%s + 1) * %s" + LFX_GREATER_EXPRESSION_PREFIX = "< %s - sys_time * + %s 1 %s" + + LFX_LESS_EXPRESSION_INFIX = "%s > sys_time - %s * %s" + LFX_LESS_EXPRESSION_PREFIX = "> %s - sys_time * %s %s" + + # pylint: disable=too-few-public-methods + def __init__(self, time_type, number, unit="days", greater_than=False, + less_than=False, exclude=False): + # pylint: disable=too-many-arguments + self.lfx_time_type = time_type + self.lfx_number = number + self.lfx_greater_than = greater_than + self.lfx_less_than = less_than + self.lfx_unit = unit + super(LipeFindXtime, self).__init__(exclude=exclude) + + def lfx_expression_no_exclude(self): + """ + Return the string of expression without exclude + """ + # lipe_find is lazy, it depends on the core to get the systime + # + if self.lfx_greater_than: + return (self.LFX_GREATER_EXPRESSION_PREFIX % + (self.lfx_time_type, self.lfx_number, self.lfx_unit)) + elif self.lfx_less_than: + return (self.LFX_LESS_EXPRESSION_PREFIX % + (self.lfx_time_type, self.lfx_number, self.lfx_unit)) + else: + return (self.LFX_EXPRESSION_PREFIX % + (self.lfx_time_type, self.lfx_number, self.lfx_unit, + self.lfx_time_type, self.lfx_number, self.lfx_unit)) + + def lfp_expression(self): + """ + Return the string of expression + """ + + if self.lfp_exclude: + return ('%s %s 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + self.lfx_expression_no_exclude())) + else: + return self.lfx_expression_no_exclude() + + +def xtime_option_parse(log, time_type, number_string, negative): + """ + Parse the option and return an LipeFindXtime object + + time_type is either "-atime", "-ctime" or "-mtime" + number_string is a decimal/hexadecimal day/minute number. + """ + if time_type == "amin": + unit = "minutes" + time_name = "atime" + elif time_type == "cmin": + unit = "minutes" + time_name = "ctime" + elif time_type == "mmin": + unit = "minutes" + time_name = "mtime" + elif (time_type == "atime" or time_type == "ctime" or + time_type == "mtime"): + time_name = time_type + unit = "days" + else: + log.cl_error("invalid time type [%s]", time_type) + return None + + if len(number_string) < 1: + log.cl_error("invalid argument [%s] to [%s]", number_string, time_type) + return None + + greater_than = False + less_than = False + if number_string[0] == "+": + greater_than = True + number_string = number_string[1:] + elif number_string[0] == "-": + less_than = True + number_string = number_string[1:] + + try: + number = str(int(number_string)) + except ValueError: + number = None + + # Not a index number + if number is None: + log.cl_error("invalid argument [%s] to [%s]", number_string, time_type) + return None + + return LipeFindXtime(time_name, number, unit=unit, + greater_than=greater_than, + less_than=less_than, exclude=negative) + + +class LipeFindSize(LipeFindPredicate): + """ + The predicate of size or blocks + """ + # Expression for N without unit: + # size == N + # == size N + # + # Expression for +N: + # size > N + # > size N + # + # Expression for -N: + # size < N + # < size N + LFS_EXPRESSION_INFIX = ("%s == %s") + LFS_EXPRESSION_PREFIX = ("== %s %s") + + LFS_GREATER_EXPRESSION_INFIX = "%s > %s" + LFS_GREATER_EXPRESSION_PREFIX = "> %s %s" + + LFS_LESS_EXPRESSION_INFIX = "%s < %s" + LFS_LESS_EXPRESSION_PREFIX = "< %s %s" + # Expression for N: + # (size >= N * unit) && (size < (N + 1) * unit) + # && >= size * N unit < size * + N 1 unit + # + # Expression for +N: + # size >= (N + 1) * unit + # >= size * + N 1 unit + # + # Expression for -N: + # size < N * unit + # < size * N unit + LFS_UNIT_EXPRESSION_INFIX = ("(%s >= %s * %s) && " + "(%s < (%s + 1) * %s)") + LFS_UNIT_EXPRESSION_PREFIX = ("&& >= %s * %s %s < %s * + %s 1 %s") + + LFS_GREATER_UNIT_EXPRESSION_INFIX = "%s >= (%s + 1) * %s" + LFS_GREATER_UNIT_EXPRESSION_PREFIX = ">= %s * + %s 1 %s" + + LFS_LESS_UNIT_EXPRESSION_INFIX = "%s < %s * %s" + LFS_LESS_UNIT_EXPRESSION_PREFIX = "< %s * %s %s" + + # pylint: disable=too-few-public-methods + def __init__(self, number, unit, attribute="size", greater_than=False, + less_than=False, exclude=False): + # pylint: disable=too-many-arguments + self.lfs_number = number + self.lfs_unit = unit + self.lfs_greater_than = greater_than + self.lfs_less_than = less_than + self.lfs_attribute = attribute + super(LipeFindSize, self).__init__(exclude=exclude) + + def lfs_expression_no_exclude(self): + """ + Return the string of expression without exclude + """ + # lipe_find is lazy, it depends on the core to calculate the + # number * unit + if self.lfs_unit is None: + if self.lfs_greater_than: + return (self.LFS_GREATER_EXPRESSION_PREFIX % + (self.lfs_attribute, self.lfs_number)) + elif self.lfs_less_than: + return (self.LFS_LESS_EXPRESSION_PREFIX % + (self.lfs_attribute, self.lfs_number)) + else: + return (self.LFS_EXPRESSION_PREFIX % + (self.lfs_attribute, self.lfs_number)) + else: + if self.lfs_greater_than: + return (self.LFS_GREATER_UNIT_EXPRESSION_PREFIX % + (self.lfs_attribute, self.lfs_number, self.lfs_unit)) + elif self.lfs_less_than: + return (self.LFS_LESS_UNIT_EXPRESSION_PREFIX % + (self.lfs_attribute, self.lfs_number, self.lfs_unit)) + else: + return (self.LFS_UNIT_EXPRESSION_PREFIX % + (self.lfs_attribute, self.lfs_number, self.lfs_unit, + self.lfs_attribute, self.lfs_number, self.lfs_unit)) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + self.lfs_expression_no_exclude())) + else: + return self.lfs_expression_no_exclude() + + +def size_option_parse(log, origin_value, negative, attribute="size", default_unit="K"): + """ + Parse the option and return an LipeFindSize object + + origin_value: [+-]N[bckKmMgGtTPpEe] + N: a decimal/hexadecimal number. + """ + # pylint: disable=too-many-branches + if len(origin_value) < 1: + log.cl_error("invalid argument [%s] to [%s]", origin_value, attribute) + return None + value_string = origin_value[:] + + greater_than = False + less_than = False + if value_string[0] == "+": + greater_than = True + value_string = value_string[1:] + elif value_string[0] == "-": + less_than = True + value_string = value_string[1:] + + if len(value_string) < 1: + log.cl_error("invalid argument [%s] to [%s]", origin_value, attribute) + return None + + have_unit = True + unit = value_string[-1] + if unit == 'b': + unit_string = "512" + elif unit == 'c': + unit_string = None + elif unit == 'k' or unit == 'K': + unit_string = "K" + elif unit == 'm' or unit == 'M': + unit_string = "M" + elif unit == 'g' or unit == 'G': + unit_string = "G" + elif unit == 't' or unit == 'T': + unit_string = "T" + elif unit == 'p' or unit == 'P': + unit_string = "P" + elif unit == 'e' or unit == 'E': + unit_string = "E" + else: + unit_string = default_unit + have_unit = False + + if have_unit: + value_string = value_string[0:-1] + if len(value_string) < 1: + log.cl_error("invalid argument [%s] to [%s]", origin_value, + attribute) + return None + + try: + number = str(int(value_string)) + except ValueError: + number = None + + # Not a number + if number is None: + log.cl_error("invalid argument [%s] to [%s]", origin_value, + attribute) + return None + + return LipeFindSize(number, unit_string, attribute=attribute, + greater_than=greater_than, less_than=less_than, + exclude=negative) + + +class LipeFindCount(LipeFindPredicate): + """ + The predicate of count without any unit, e.g. -stripe-count + """ + # Expression for N without unit: + # size == N + # == size N + # + # Expression for +N: + # size > N + # > size N + # + # Expression for -N: + # size < N + # < size N + LFC_EXPRESSION_INFIX = ("%s == %s") + LFC_EXPRESSION_PREFIX = ("== %s %s") + + LFC_GREATER_EXPRESSION_INFIX = "%s > %s" + LFC_GREATER_EXPRESSION_PREFIX = "> %s %s" + + LFC_LESS_EXPRESSION_INFIX = "%s < %s" + LFC_LESS_EXPRESSION_PREFIX = "< %s %s" + + # pylint: disable=too-few-public-methods + def __init__(self, number, attribute="stripe_count", greater_than=False, + less_than=False, exclude=False): + # pylint: disable=too-many-arguments + self.lfc_number = number + self.lfc_greater_than = greater_than + self.lfc_less_than = less_than + self.lfc_attribute = attribute + super(LipeFindCount, self).__init__(exclude=exclude) + + def lfc_expression_no_exclude(self): + """ + Return the string of expression without exclude + """ + if self.lfc_greater_than: + return (self.LFC_GREATER_EXPRESSION_PREFIX % + (self.lfc_attribute, self.lfc_number)) + elif self.lfc_less_than: + return (self.LFC_LESS_EXPRESSION_PREFIX % + (self.lfc_attribute, self.lfc_number)) + else: + return (self.LFC_EXPRESSION_PREFIX % + (self.lfc_attribute, self.lfc_number)) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + self.lfc_expression_no_exclude())) + else: + return self.lfc_expression_no_exclude() + + +def count_option_parse(log, origin_value, negative, attribute="stripe_count"): + """ + Parse the option and return a LipeFindCount object + + origin_value: [+-]N + N: a decimal/hexadecimal number. + """ + # pylint: disable=too-many-branches + if len(origin_value) < 1: + log.cl_error("invalid argument [%s] to [-%s]", origin_value, attribute) + return None + value_string = origin_value[:] + + greater_than = False + less_than = False + if value_string[0] == "+": + greater_than = True + value_string = value_string[1:] + elif value_string[0] == "-": + less_than = True + value_string = value_string[1:] + + if len(value_string) < 1: + log.cl_error("invalid argument [%s] to [-%s]", origin_value, attribute) + return None + + try: + number = str(int(value_string)) + except ValueError: + number = None + + # Not a number + if number is None: + log.cl_error("invalid argument [%s] to [-%s]", origin_value, + attribute) + return None + + return LipeFindCount(number, attribute=attribute, + greater_than=greater_than, less_than=less_than, + exclude=negative) + + +class LipeFindEmpty(LipeFindPredicate): + """ + The predicate of empty + """ + # pylint: disable=too-few-public-methods + def __init__(self, exclude=False): + # pylint: disable=too-many-arguments + super(LipeFindEmpty, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_EMPTY)) + else: + return lipe_constant.LIPE_POLICY_ATTRIBUTE_EMPTY + + +class LipeFindNogroup(LipeFindPredicate): + """ + The predicate of nogroup + """ + # pylint: disable=too-few-public-methods + def __init__(self, exclude=False): + # pylint: disable=too-many-arguments + super(LipeFindNogroup, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_NOGROUP)) + else: + return lipe_constant.LIPE_POLICY_ATTRIBUTE_NOGROUP + + +class LipeFindEntries(LipeFindPredicate): + """ + The predicate of entries + """ + # Expression for N: + # entries == N + # == entries N + # + # Expression for +N: + # entries > N + # > entries N + # + # Expression for -N: + # entries < N + # < entries N + LFE_EXPRESSION_INFIX = ("entries == %s") + LFE_EXPRESSION_PREFIX = ("== entries %s") + + LFE_GREATER_EXPRESSION_INFIX = "entries > %s" + LFE_GREATER_EXPRESSION_PREFIX = "> entries %s" + + LFE_LESS_EXPRESSION_INFIX = "entries < %s" + LFE_LESS_EXPRESSION_PREFIX = "< entries %s" + + # pylint: disable=too-few-public-methods + def __init__(self, number, greater_than=False, + less_than=False, exclude=False): + # pylint: disable=too-many-arguments + self.lfe_number = number + self.lfe_greater_than = greater_than + self.lfe_less_than = less_than + super(LipeFindEntries, self).__init__(exclude=exclude) + + def lfe_expression_no_exclude(self): + """ + Return the string of expression without exclude + """ + if self.lfe_greater_than: + return (self.LFE_GREATER_EXPRESSION_PREFIX % + (self.lfe_number)) + elif self.lfe_less_than: + return (self.LFE_LESS_EXPRESSION_PREFIX % + (self.lfe_number)) + else: + return (self.LFE_EXPRESSION_PREFIX % + (self.lfe_number)) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + self.lfe_expression_no_exclude())) + else: + return self.lfe_expression_no_exclude() + + +def entries_option_parse(log, origin_value, negative): + """ + Parse the option and return an LipeFindEntries object + + origin_value: [+-]N + N: a decimal/hexadecimal number. + """ + # pylint: disable=too-many-branches + if len(origin_value) < 1: + log.cl_error("invalid argument [%s] to [-entries]", origin_value) + return None + value_string = origin_value[:] + + greater_than = False + less_than = False + if value_string[0] == "+": + greater_than = True + value_string = value_string[1:] + elif value_string[0] == "-": + less_than = True + value_string = value_string[1:] + + if len(value_string) < 1: + log.cl_error("invalid argument [%s] to [-entries]", origin_value) + return None + + try: + number = str(int(value_string)) + except ValueError: + number = None + + # Not a number + if number is None: + log.cl_error("invalid argument [%s] to [-entries]", origin_value) + return None + + return LipeFindEntries(number, greater_than=greater_than, + less_than=less_than, exclude=negative) + + +class LipeFindNouser(LipeFindPredicate): + """ + The predicate of nouser + """ + # pylint: disable=too-few-public-methods + def __init__(self, exclude=False): + # pylint: disable=too-many-arguments + super(LipeFindNouser, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_ATTRIBUTE_NOUSER)) + else: + return lipe_constant.LIPE_POLICY_ATTRIBUTE_NOUSER + + +class LipeFindPoolMatch(LipeFindPredicate): + """ + The predicate of matching the OST pool with fnmatch (shell wildcard) + """ + # pylint: disable=too-few-public-methods + def __init__(self, pool_pattern, exclude=False): + self.lfpm_pool_pattern = pool_pattern + super(LipeFindPoolMatch, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s("%s") 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_FUNCTION_POOL_MATCH, + self.lfpm_pool_pattern)) + else: + return ('%s("%s")' % + (lipe_constant.LIPE_POLICY_FUNCTION_POOL_MATCH, + self.lfpm_pool_pattern)) + + +def pool_match_option_parse(arg, negative): + """ + Parse the option and return an LipeFindPoolMatch object + + arg is a regular expression. + """ + return LipeFindPoolMatch(arg, exclude=negative) + + +class LipeFindPoolReg(LipeFindPredicate): + """ + The predicate of matching the OST pool with regular expression + """ + # pylint: disable=too-few-public-methods + def __init__(self, pool_reg, exclude=False): + self.lfpr_pool_reg = pool_reg + super(LipeFindPoolReg, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s("%s") 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + lipe_constant.LIPE_POLICY_FUNCTION_POOL_REG, + self.lfpr_pool_reg)) + else: + return ('%s("%s")' % + (lipe_constant.LIPE_POLICY_FUNCTION_POOL_REG, + self.lfpr_pool_reg)) + + +def pool_reg_option_parse(arg, negative): + """ + Parse the option and return a LipeFindPoolReg object + + arg is a regular expression. + """ + return LipeFindPoolReg(arg, exclude=negative) + + +class LipeFindPermReg(LipeFindPredicate): + """ + The predicate of checking permission + """ + # pylint: disable=too-few-public-methods + def __init__(self, mode, function, exclude=False): + self.lfpr_mode = mode + self.lfpr_function = function + super(LipeFindPermReg, self).__init__(exclude=exclude) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s(%s) 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + self.lfpr_function, self.lfpr_mode)) + else: + return ('%s(%s)' % (self.lfpr_function, self.lfpr_mode)) + + +class LipeFindUsed(LipeFindPredicate): + """ + The predicate of checking whether file was last accessed + N days after its status was last changed + """ + # Expression for N: + # (ctime + (N + 1) * days > atime) && (ctime + N * days <= atime) + # && > + ctime * + N 1 days atime <= + ctime * N days atime + # + # Expression for +N: + # ctime + (N + 1) * days < atime + # < + ctime * + N 1 days atime + # + # Expression for -N: + # ctime + N * days > atime + # > + ctime * N days atime + # + LFU_EXPRESSION_INFIX = ("(ctime + (%s + 1) * days > atime) && " + "(ctime + %s * days <= atime)") + LFU_EXPRESSION_PREFIX = ("&& > + ctime * + %s 1 days atime <= + ctime * %s days atime") + + LFU_GREATER_EXPRESSION_INFIX = "ctime + (%s + 1) * days < atime " + LFU_GREATER_EXPRESSION_PREFIX = "< + ctime * + %s 1 days atime" + + LFU_LESS_EXPRESSION_INFIX = "ctime + %s * days > atime" + LFU_LESS_EXPRESSION_PREFIX = "> + ctime * %s days atime" + + # pylint: disable=too-few-public-methods + def __init__(self, days, greater_than=False, less_than=False, exclude=False): + # pylint: disable=too-many-arguments + self.lfu_days = days + self.lfu_greater_than = greater_than + self.lfu_less_than = less_than + super(LipeFindUsed, self).__init__(exclude=exclude) + + def lfu_expression_no_exclude(self): + """ + Return the string of expression without exclude + """ + # lipe_find is lazy, it depends on the core to get the systime + # + if self.lfu_greater_than: + return (self.LFU_GREATER_EXPRESSION_PREFIX % + (self.lfu_days)) + elif self.lfu_less_than: + return (self.LFU_LESS_EXPRESSION_PREFIX % + (self.lfu_days)) + else: + return (self.LFU_EXPRESSION_PREFIX % + (self.lfu_days, self.lfu_days)) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + self.lfu_expression_no_exclude())) + else: + return self.lfu_expression_no_exclude() + + +def used_option_parse(log, arg, negative): + """ + Parse the option and return an LipeFindUsed object + + arg is a decimal number. + """ + greater_than = False + less_than = False + if arg[0] == "+": + greater_than = True + days_string = arg[1:] + elif arg[0] == "-": + less_than = True + days_string = arg[1:] + else: + days_string = arg + + try: + days = str(int(days_string, 10)) + except ValueError: + log.cl_error("invalid argument [%s] to [-used]", arg) + return None + + if days < 0: + log.cl_error("negative [%s] to [-used]", arg) + return None + + return LipeFindUsed(days, greater_than=greater_than, less_than=less_than, + exclude=negative) + + +class LipeFindLayout(LipeFindPredicate): + """ + The predicate of checking the layout type of file + """ + # pylint: disable=too-few-public-methods + def __init__(self, arg, exclude=False): + # pylint: disable=too-many-arguments + self.lfl_arg = arg + super(LipeFindLayout, self).__init__(exclude=exclude) + + def lfl_expression_no_exclude(self): + """ + Return the string of expression without exclude + """ + return "%s(\"%s\")" % (lipe_constant.LIPE_POLICY_FUNCTION_LAYOUT, self.lfl_arg) + + def lfp_expression(self): + """ + Return the string of expression + """ + if self.lfp_exclude: + return ('%s %s 0' % + (lipe_constant.LIPE_POLICY_OPERATOR_EQUAL, + self.lfl_expression_no_exclude())) + else: + return self.lfl_expression_no_exclude() + + +def layout_option_parse(log, arg, negative): + """ + Parse the option and return an LipeFindLayout object + + arg is released, raid0, and mdt seperated by "," + """ + patterns = arg.split(",") + for pattern in patterns: + if pattern != "released" and pattern != "raid0" and pattern != "mdt": + log.cl_error("unknown pattern [%s] in argument [%s] of [-layout]", + pattern, arg) + return None + return LipeFindLayout(arg, exclude=negative) + + +def perm_option_parse(log, arg, negative): + """ + Parse the option and return an object with type LipeFindPermReg + + arg is [-/]mode + """ + arg_length = len(arg) + if arg_length == 0: + log.cl_error("too short argument [%s] of option [-perm]", arg) + return None + + mode_index = 0 + if arg[0] == '-': + function = lipe_constant.LIPE_POLICY_FUNCTION_PERM_AT_LEAST + mode_index = 1 + elif arg[0] == '/': + function = lipe_constant.LIPE_POLICY_FUNCTION_PERM_ANY + mode_index = 1 + else: + function = lipe_constant.LIPE_POLICY_FUNCTION_PERM_EQUAL + mode_index = 0 + + if mode_index == arg_length: + log.cl_error("too short argument [%s] of option [-perm]", arg) + return None + + mode_string = arg[mode_index:] + try: + mode = int(mode_string, 8) + except ValueError: + log.cl_error("argument [%s] of option [-perm] is not an octal number", + mode_string) + return None + + if mode > 07777: + log.cl_error("argument [%s] of option [-perm] is larger than 07777", + mode_string) + return None + + return LipeFindPermReg(mode_string, function, exclude=negative) + + +def build_expression_tree(log, start_option, find_arg): + """ + Build the expression tree + """ + # pylint: disable=too-many-branches,too-many-statements,too-many-locals + argc = len(sys.argv) + i = start_option + negative = False + lonely_negative = False + while i < argc: + arg = sys.argv[i] + if arg == "!": + negative = not negative + lonely_negative = True + else: + lonely_negative = False + time_args = ["-atime", "-ctime", "-mtime", "-amin", "-cmin", + "-mmin"] + if arg in time_args: + i += 1 + if i >= argc: + log.cl_error("missing argument to [%s]", arg) + exit_with_tip(log) + option = sys.argv[i] + lipe_find_xtime = xtime_option_parse(log, arg[1:], option, + negative) + if lipe_find_xtime is None: + log.cl_error("unknown argument to [%s]: [%s]", arg, option) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_xtime) + elif arg == "-blocks": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-blocks]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_size = size_option_parse(log, arg, negative, attribute="blocks") + if lipe_find_size is None: + log.cl_error("unknown argument to [-blocks]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_size) + elif arg == "-comp-count" or arg == "-component-count": + i += 1 + if i >= argc: + log.cl_error("missing argument to [%s]", arg) + exit_with_tip(log) + argument = sys.argv[i] + lipe_find_count = count_option_parse(log, argument, negative, + attribute="comp_count") + if lipe_find_count is None: + log.cl_error("unknown argument to [%s]: [%s]", + arg, argument) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_count) + elif arg == "-empty": + lipe_find_empty = LipeFindEmpty(negative) + find_arg.lfa_predicates.append(lipe_find_empty) + elif arg == "-entries": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-entries]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_entries = entries_option_parse(log, arg, negative) + if lipe_find_entries is None: + log.cl_error("unknown argument to [-entries]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_entries) + elif arg == "-gid": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-gid]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_gid = gid_option_parse(log, arg, negative) + if lipe_find_gid is None: + log.cl_error("unknown argument to [-gid]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_gid) + elif arg == "-group": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-group]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_group = group_option_parse(arg, negative) + if lipe_find_group is None: + log.cl_error("unknown argument to [-group]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_group) + elif arg == "-fid": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-fid]") + exit_with_tip(log) + find_arg.lfa_predicates.append(LipeFindFid(sys.argv[i], + exclude=negative)) + elif arg == "-iname": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-iname]") + exit_with_tip(log) + find_arg.lfa_predicates.append(LipeFindFnmatch(sys.argv[i], + case_insensitive=True, + exclude=negative)) + elif arg == "-inum": + i += 1 + arg = sys.argv[i] + lipe_find_inum = inum_option_parse(log, arg, negative) + if lipe_find_inum is None: + log.cl_error("unknown argument to [-inum]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_inum) + elif arg == "-layout": + i += 1 + arg = sys.argv[i] + lipe_find_layout = layout_option_parse(log, arg, negative) + if lipe_find_layout is None: + log.cl_error("unknown argument to [-layout]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_layout) + elif arg == "-links": + i += 1 + arg = sys.argv[i] + lipe_find_links = links_option_parse(log, arg, negative) + if lipe_find_links is None: + log.cl_error("unknown argument to [-links]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_links) + elif arg == "-name": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-name]") + exit_with_tip(log) + find_arg.lfa_predicates.append(LipeFindFnmatch(sys.argv[i], + case_insensitive=False, + exclude=negative)) + elif arg == "-nogroup": + lipe_find_nogroup = LipeFindNogroup(negative) + find_arg.lfa_predicates.append(lipe_find_nogroup) + elif arg == "-nouser": + lipe_find_nouser = LipeFindNouser(negative) + find_arg.lfa_predicates.append(lipe_find_nouser) + elif arg == "-type": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-type]") + exit_with_tip(log) + arg = sys.argv[i] + filetype = file_option2lipe_constant(arg) + if filetype is None: + log.cl_error("unknown argument to [-type]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(LipeFindType(filetype, negative)) + elif arg == "-ost" or arg == "-stripe-index": + i += 1 + if i >= argc: + log.cl_error("missing argument to [%s]", arg) + exit_with_tip(log) + argument = sys.argv[i] + lipe_find_ost = ost_option_parse(log, argument, negative) + if lipe_find_ost is None: + log.cl_error("unknown argument to [%s]: [%s]", arg, + argument) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_ost) + elif arg == "-perm": + i += 1 + arg = sys.argv[i] + lipe_find_perm = perm_option_parse(log, arg, negative) + if lipe_find_perm is None: + log.cl_error("unknown argument to [-perm]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_perm) + elif arg == "-pool-regex": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-pool-regex]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_pool_reg = pool_reg_option_parse(arg, negative) + if lipe_find_pool_reg is None: + log.cl_error("unknown argument to [-pool-regex]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_pool_reg) + elif arg == "-pool": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-pool]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_pool = pool_match_option_parse(arg, negative) + if lipe_find_pool is None: + log.cl_error("unknown argument to [-pool]: [%s]", + arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_pool) + elif arg == "-projid": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-projid]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_count = count_option_parse(log, arg, negative, + attribute="projid") + if lipe_find_count is None: + log.cl_error("unknown argument to [-projid]: [%s]", + arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_count) + elif arg == "-size": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-size]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_size = size_option_parse(log, arg, negative) + if lipe_find_size is None: + log.cl_error("unknown argument to [-size]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_size) + elif arg == "-stripe-count": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-stripe-count]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_count = count_option_parse(log, arg, negative, + attribute="stripe_count") + if lipe_find_count is None: + log.cl_error("unknown argument to [-stripe-count]: [%s]", + arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_count) + elif arg == "-stripe-size": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-stripe-size]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_size = size_option_parse(log, arg, negative, + attribute="stripe_size", + default_unit=None) + if lipe_find_size is None: + log.cl_error("unknown argument to [-stripe-size]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_size) + elif arg == "-uid": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-uid]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_uid = uid_option_parse(log, arg, negative) + if lipe_find_uid is None: + log.cl_error("unknown argument to [-uid]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_uid) + elif arg == "-used": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-used]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_used = used_option_parse(log, arg, negative) + if lipe_find_used is None: + log.cl_error("unknown argument to [-used]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_used) + elif arg == "-user": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-user]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_user = user_option_parse(arg, negative) + if lipe_find_user is None: + log.cl_error("unknown argument to [-user]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_user) + elif arg == "-xattr": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-xattr]") + exit_with_tip(log) + arg = sys.argv[i] + lipe_find_xattr = xattr_option_parse(log, arg, negative) + if lipe_find_xattr is None: + log.cl_error("unknown argument to [-xattr]: [%s]", arg) + exit_with_tip(log) + find_arg.lfa_predicates.append(lipe_find_xattr) + elif arg == "-expr": + i += 1 + if i >= argc: + log.cl_error("missing argument to [-expr]") + exit_with_tip(log) + ori_expr = sys.argv[i] + find_arg.lfa_super_predicate = lipe_find_user_expression_format(sys.argv[i]) + if ori_expr != find_arg.lfa_super_predicate: + log.cl_warning("\nOriginal expression: %s\nFormalized expression: %s\n", + ori_expr, find_arg.lfa_super_predicate) + else: + log.cl_error("unknown argument [%s]", arg) + exit_with_tip(log) + # reset 'negative' flag + negative = False + i += 1 + + if lonely_negative: + log.cl_error("expected an expression after [!]") + exit_with_tip(log) + + +def main(): + """ + Convert expression argument to an expression string + """ + reload(sys) + sys.setdefaultencoding("utf-8") + + argc = len(sys.argv) + find_arg = LipeFindArg() + end_of_leading_options = process_leading_options(argc, sys.argv) + + log = clog.get_log() + if argc < 2: + log.cl_error("missing argument") + exit_with_tip(log) + + build_expression_tree(log, end_of_leading_options, find_arg) + + print(str(find_arg.lfa_expression())) + + sys.exit(0) diff --git a/pylipe/lipe_hotpool_test.py b/pylipe/lipe_hotpool_test.py index 0556a0a..8428fe6 100644 --- a/pylipe/lipe_hotpool_test.py +++ b/pylipe/lipe_hotpool_test.py @@ -343,6 +343,7 @@ def hotpool_test_generate_config(log, config_file, options_dict): fd.write("%s=%s\n" % (k, str(v).lower())) else: fd.write("%s=%s\n" % (k, v)) + fd.write("debug\n") except: log.cl_error("failed to generate config [%s] for options:%s ", config_file, options_dict) @@ -929,6 +930,10 @@ def hotpool_test_lpurge_wait_purged(log, host, pid, stats_file, log.cl_error("timeout (%d s) on waiting for lpurge processed [%d] " "files on host [%s]", timeout, purge_count, host.sh_hostname) + + command = """journalctl -u 'lpurge*'""" + retval = host.sh_run(log, command) + log.cl_error("lpurge logs = [%s], [%s]", retval.cr_stdout, retval.cr_stderr) return -1 return -1 diff --git a/scripts/stratagem-hp-stop.sh b/scripts/stratagem-hp-stop.sh index dba265b..d89cd75 100755 --- a/scripts/stratagem-hp-stop.sh +++ b/scripts/stratagem-hp-stop.sh @@ -2,16 +2,60 @@ # Copyright DDN 2020 # -*- indent-tabs-mode: nil -*- +# Set a default 10+ minute timeout for waitfor() +TIMEOUT=${TIMEOUT:-600} + +NOW=false + FS=$1 -function usage() { - echo "Usage: $(basename $0) [-h] [FILESYSTEM]" +function usage { + echo "Usage: $(basename $0) [-h] [--now] [FILESYSTEM]" echo " Stop stratagem hotpools in HA environment" + echo " --now - Stop mirroring processes and client mounts" +} + +function waitfor { + local ids=$* + + [ -z "$ids" ] && return + + echo -n "Waiting for $ids" + + local deadline=$((SECONDS + TIMEOUT)) + + local done=false + until $done || ((deadline < SECONDS)); do + + done=true + for i in $ids; do + # This finds the host the resource is active on (empty it is stopped) + res=$(clush -qg ha_heads crm_resource -QW -r $i 2> /dev/null) + if [ -n "$res" ]; then + done=false + echo -n "." + sleep 1 + break + fi + done + done + + echo "" + + if ! $done; then + echo "Waiting for $ids TIMED OUT!" + exit 1 + fi } if [ "$FS" = "-h" ] || [ "$FS" = "--help" ]; then usage exit 0 + +elif [ "$FS" = "--now" ]; then + shift + NOW=true + FS=$1 fi if [ -z "$FS" ]; then @@ -29,10 +73,27 @@ if crm_resource -QW -r $FS-hotpool > /dev/null 2>&1; then echo Stopping Hotpools for $FS clush -qS --group=ha_heads crm res stop $FS-hotpool + if $NOW; then + waitfor $FS-hotpool + + echo Killing all lfs mirror commands for $FS + clush -q -g oss,mds,mgs pkill -f ^lfs.mirror + + echo Stopping Client mounts for $FS + clush -qS -g ha_heads crm res stop cl-$FS-client + + waitfor cl-$FS-client + fi + elif clush -Ng ha_heads crm_resource -l|egrep -q "^(lamigo|lpurge)-$FS-"; then echo "Stopping old style Hotpools (see stratagem-hp-convert.sh)" clush -qS -g ha_heads crm res stop cl-$FS-client + if $NOW; then + echo Killing all lfs mirror commands for $FS + clush -q -g oss,mds,mgs pkill -f ^lfs.mirror + fi + else echo "Hotpools not configured" echo " Please run: stratagem-hp-config.sh" diff --git a/src/Makefile.am b/src/Makefile.am index 792c8ff..e8d0df7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -5,9 +5,15 @@ AM_CPPFLAGS = -D_GNU_SOURCE -include config.h AM_CFLAGS = -Wall -Werror -g $(json_c_CFLAGS) $(json_c_LIBS) \ -llustreapi -lpthread -Wno-error=deprecated-declarations -bin_PROGRAMS = lipe_scan lfill ldumpstripe lipe_hsm_remover \ - lipe_hsm_copytool ext4_inode2path lcreatemany - +bin_PROGRAMS = \ + ext4_inode2path \ + lcreatemany \ + ldumpstripe \ + lfill \ + lipe_hsm_copytool \ + lipe_hsm_remover \ + lipe_scan \ + lipe_scan2 noinst_PROGRAMS = lipe_expression_test generate_definition @@ -25,6 +31,7 @@ LIPE_SOURCES = cmd.c cmd.h debug.c debug.h fake_lustre_disk.h \ fake_lustre_idl.h flist.c flist.h misc.c misc.h \ lustre_ea.c lustre_ea.h \ lipe_ldiskfs.c lipe_ldiskfs.h \ + lipe_object_attrs.c \ lipe_object_attrs.h \ lipe_version.c \ lipe_version.h \ @@ -42,10 +49,14 @@ LIPE_LDFLAGS += -lzfs -lzpool -lnvpair LIPE_CFLAGS += $(zfs_CFLAGS) endif -lipe_scan_SOURCES = lipe.c lipe_config.c lipe_config.h lipe_result.c lipe_result.h $(LIPE_SOURCES) +lipe_scan_SOURCES = lipe_scan.c lipe_config.c lipe_config.h lipe_result.c lipe_result.h $(LIPE_SOURCES) lipe_scan_CFLAGS = $(LIPE_CFLAGS) lipe_scan_LDFLAGS = $(LIPE_LDFLAGS) -lssl -lcrypto -lyaml +lipe_scan2_SOURCES = lipe_scan2.c lipe_scan2.h $(LIPE_SOURCES) +lipe_scan2_CFLAGS = $(LIPE_CFLAGS) +lipe_scan2_LDFLAGS = $(LIPE_LDFLAGS) + lipe_hsm_remover_SOURCES = lipe_hsm_remover.c debug.c debug.h lipe_hsm_copytool_SOURCES = lipe_hsm_copytool.c debug.c debug.h lipe_string.h \ diff --git a/src/debug.c b/src/debug.c index 0c802f4..231e5f3 100644 --- a/src/debug.c +++ b/src/debug.c @@ -10,4 +10,5 @@ #include "debug.h" -int debug_level = INFO; +enum debug_level debug_level = INFO; +struct lu_fid debug_watch_fid; diff --git a/src/debug.h b/src/debug.h index 6098894..d612649 100644 --- a/src/debug.h +++ b/src/debug.h @@ -16,22 +16,41 @@ #include #include +#ifdef NEW_USER_HEADER +# include +#else +# include +#endif + enum debug_level { ERROR = 0, INFO, DEBUG, }; -extern int debug_level; +extern enum debug_level debug_level; +extern struct lu_fid debug_watch_fid; -/* Print debug information. This is controlled by the value of the - * global variable 'debug' - */ -static inline void _lipe_logging(int level, bool watch, const char *fmt, ...) +/* FIXME Use lu_fid_eq() when fake_lustre_idl.h is removed. */ +static inline bool lipe_fid_eq(const struct lu_fid *f1, const struct lu_fid *f2) +{ + return f1->f_seq == f2->f_seq && + f1->f_oid == f2->f_oid && + f1->f_ver == f2->f_ver; +} + +static inline bool lipe_logging_enabled(enum debug_level level, const struct lu_fid *fid) +{ + return level <= debug_level || + (fid != NULL && lipe_fid_eq(fid, &debug_watch_fid)); +} + +/* Keeps this to mask the many format errors which would make build fail. */ +static inline void _lipe_logging(int level, const struct lu_fid *fid, const char *fmt, ...) { va_list ap; - if (level > debug_level && !watch) + if (!lipe_logging_enabled(level, fid)) return; va_start(ap, fmt); @@ -39,33 +58,32 @@ static inline void _lipe_logging(int level, bool watch, const char *fmt, ...) va_end(ap); } -#define lipe_logging(level, watch, format, args...) \ - _lipe_logging(level, watch, "["#level"] [%s:%d] [%s()]: " format, \ +#if 0 +#define _lipe_logging(level, fid, format, args...) \ + do { \ + if (lipe_logging_enabled((level), (fid))) \ + fprintf(stderr, format, ##args); \ + } while (0) +#endif /* 0 */ + +#define lipe_logging(level, fid, format, args...) \ + _lipe_logging(level, fid, "["#level"] [%s:%d] [%s()]: " format, \ __FILE__, __LINE__, __func__, ##args) #define LDEBUGL(level, format, args...) \ - lipe_logging(level, false, format, ##args) + lipe_logging(level, NULL, format, ##args) #define LERROR(format, args...) \ - lipe_logging(ERROR, false, format, ##args) + lipe_logging(ERROR, NULL, format, ##args) #define LDEBUG(format, args...) \ - lipe_logging(DEBUG, false, format, ##args) + lipe_logging(DEBUG, NULL, format, ##args) #define LINFO(format, args...) \ - lipe_logging(INFO, false, format, ##args) - -#define _LINFO(format, args...) \ - _lipe_logging(INFO, false, format, ##args) - -#define LERRORW(watch, format, args...) \ - lipe_logging(ERROR, watch, format, ##args) - -#define LDEBUGW(watch, format, args...) \ - lipe_logging(DEBUG, watch, format, ##args) + lipe_logging(INFO, NULL, format, ##args) -#define LINFOW(watch, format, args...) \ - lipe_logging(INFO, watch, format, ##args) +#define LDEBUGW(fid, format, args...) \ + lipe_logging(DEBUG, fid, format, ##args) typedef long long_ptr_t; diff --git a/src/flist.c b/src/flist.c index fb35abe..2a7b9df 100644 --- a/src/flist.c +++ b/src/flist.c @@ -60,13 +60,19 @@ struct lipe_flist *flist_alloc(struct lipe_counter *counter, int size, enum flist_dump_type ftype) { int buffer_size = size + LIPE_FLIST_MAX_SINGLE_SIZE; - struct lipe_flist *flist = malloc(offsetof(typeof(*flist), - lf_buffer[buffer_size])); + struct lipe_flist *flist; + + flist = calloc(1, offsetof(typeof(*flist), lf_buffer[buffer_size])); if (flist == NULL) return NULL; flist->lf_counter = counter; flist->lf_ftype = ftype; + flist->lf_fd = -1; + flist->lf_flush_size = size; + flist->lf_buffer_size = buffer_size; + strncpy(flist->lf_fname, fname, sizeof(flist->lf_fname) - 1); + switch (ftype) { case LDT_UNIX_SOCKET: flist->lf_fd = flist_setup_unix_socket(fname); @@ -82,11 +88,6 @@ struct lipe_flist *flist_alloc(struct lipe_counter *counter, int size, goto out; } - flist->lf_used_size = 0; - flist->lf_flush_size = size; - flist->lf_buffer_size = buffer_size; - strncpy(flist->lf_fname, fname, sizeof(flist->lf_fname) - 1); - return flist; out: free(flist); @@ -95,7 +96,12 @@ out: void flist_free(struct lipe_flist *flist) { - close(flist->lf_fd); + if (flist == NULL) + return; + + if (!(flist->lf_fd < 0)) + close(flist->lf_fd); + if (flist->lf_ftype == LDT_UNIX_SOCKET) unlink(flist->lf_fname); @@ -179,12 +185,8 @@ static int flist_add_fid(struct lipe_flist *flist, struct lu_fid *fid) static void flist_add_json_linkea_entries(struct json_object *obj_top, struct lipe_object_attrs *attrs) { - int i; - int reclen; char fid_buf[FID_LEN + 1]; - struct lu_fid pfid; - struct link_ea_entry *lee; - struct link_ea_header *leh; + struct lipe_link_entry *lle; struct json_object *obj_entry; struct json_object *obj_entries; @@ -194,22 +196,16 @@ static void flist_add_json_linkea_entries(struct json_object *obj_top, obj_entries = json_object_new_array(); json_object_object_add(obj_top, "entries", obj_entries); - leh = (struct link_ea_header *)attrs->loa_leh_buf; - lee = (struct link_ea_entry *)(leh + 1); - - for (i = 0; i < leh->leh_reccount; i++) { + lipe_list_for_each_entry(lle, &attrs->loa_links, lle_linkage) { obj_entry = json_object_new_object(); json_object_array_add(obj_entries, obj_entry); - memcpy(&pfid, &lee->lee_parent_fid, sizeof(pfid)); - snprintf(fid_buf, sizeof(fid_buf), DFID, PFID(&pfid)); + snprintf(fid_buf, sizeof(fid_buf), + DFID, PFID(&lle->lle_parent_fid)); json_object_object_add(obj_entry, "pfid", json_object_new_string(fid_buf)); json_object_object_add(obj_entry, "fname", - json_object_new_string(lee->lee_name)); - - reclen = (lee->lee_reclen[0] << 8) | lee->lee_reclen[1]; - lee = (struct link_ea_entry *)((char *)lee + reclen); + json_object_new_string(lle->lle_name)); } } diff --git a/src/general_policy.c b/src/general_policy.c index 348ed2b..e4b21ba 100644 --- a/src/general_policy.c +++ b/src/general_policy.c @@ -393,6 +393,41 @@ lipe_policy_sysattr_evaluate(enum lipe_policy_sysattr_type type, } static int64_t +lipe_policy_link_regex_evaluate(struct lipe_policy_function *function, + struct lipe_object_attrs *attrs) +{ + regex_t *regex = &function->u.lpv_regex; + struct lipe_link_entry *lle; + + lipe_list_for_each_entry(lle, &attrs->loa_links, lle_linkage) { + /* + * REG_NOSUB is used, so the nmatch and pmatch arguments are + * ignored. + */ + if (regexec(regex, lle->lle_name, 0, NULL, 0) == 0) + return 1; + } + + return 0; +} + +static int64_t +lipe_policy_link_fnmatch_evaluate(struct lipe_policy_function *function, + struct lipe_object_attrs *attrs, + int fnmatch_flags) +{ + const char *pattern = function->lpf_argument; + struct lipe_link_entry *lle; + + lipe_list_for_each_entry(lle, &attrs->loa_links, lle_linkage) { + if (fnmatch(pattern, lle->lle_name, fnmatch_flags) == 0) + return 1; + } + + return 0; +} + +static int64_t lipe_policy_xattr_match_evaluate(struct lipe_policy_function *function, struct lipe_object_attrs *attrs) { @@ -415,6 +450,21 @@ lipe_policy_xattr_match_evaluate(struct lipe_policy_function *function, } static int64_t +lipe_policy_xattr_name_evaluate(struct lipe_policy_function *function, + struct lipe_object_attrs *attrs) +{ + const char *name = function->lpf_argument; + struct lipe_xattr *xattr; + + lipe_list_for_each_entry(xattr, &attrs->loa_xattrs, lx_linkage) { + if (strcmp(name, xattr->lx_name) == 0) + return 1; + } + + return 0; +} + +static int64_t lipe_policy_function_evaluate(struct lipe_policy_function *function, struct lipe_object_attrs *attrs) { @@ -422,9 +472,8 @@ lipe_policy_function_evaluate(struct lipe_policy_function *function, int64_t result = 0; int rc; regex_t *reg; - struct link_ea_header *leh; struct lov_user_md *lum = attrs->loa_lum; - bool case_insensitive = false; + int fnmatch_flags = 0; LENTRY; @@ -436,13 +485,7 @@ lipe_policy_function_evaluate(struct lipe_policy_function *function, LERROR("shouldn't evaluate the fname function on objects without linkea\n"); LRETURN(0); } - leh = (struct link_ea_header *)attrs->loa_leh_buf; - reg = &function->u.lpv_regex; - rc = match_linkea(leh, reg); - if (rc) - result = 0; - else - result = 1; + result = lipe_policy_link_regex_evaluate(function, attrs); break; case LIPE_POLICY_FUNCTION_ONOST: if (!(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_ATTR)) { @@ -492,20 +535,16 @@ lipe_policy_function_evaluate(struct lipe_policy_function *function, result = 1; break; case LIPE_POLICY_FUNCTION_FNAME_IMATCH: - case_insensitive = true; + fnmatch_flags = FNM_CASEFOLD; + /* fallthrough */ case LIPE_POLICY_FUNCTION_FNAME_MATCH: if (!(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_LINKEA)) { LERROR("shouldn't evaluate the [%s] function on objects without linkea\n", lipe_policy_function_names[type]); LRETURN(0); } - leh = (struct link_ea_header *)attrs->loa_leh_buf; - rc = fnmatch_linkea(leh, function->lpf_argument, - case_insensitive); - if (rc) - result = 0; - else - result = 1; + result = lipe_policy_link_fnmatch_evaluate(function, attrs, + fnmatch_flags); break; case LIPE_POLICY_FUNCTION_USER: if (!(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_ATTR)) { @@ -601,7 +640,16 @@ lipe_policy_function_evaluate(struct lipe_policy_function *function, else result = 1; break; + case LIPE_POLICY_FUNCTION_XATTR_NAME: + if (!(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_ALL_XATTR)) { + LERROR("shouldn't evaluate the [%s] function on objects without all xattrs\n", + lipe_policy_function_names[type]); + LRETURN(0); + } + result = lipe_policy_xattr_name_evaluate(function, attrs); + break; } + LRETURN(result); } @@ -1254,6 +1302,9 @@ static int special_function_init(struct lipe_policy_value *value, if (rc) return rc; break; + case LIPE_POLICY_FUNCTION_XATTR_NAME: + *valid = LIPE_OBJECT_ATTR_ALL_XATTR; + break; } value->lpv_type = LIPE_POLICY_VALUE_FUNCTION; value->u.lpv_function.lpf_type = type; @@ -1731,6 +1782,7 @@ static void special_function_fini(struct lipe_policy_value *value) case LIPE_POLICY_FUNCTION_PERM_AT_LEAST: case LIPE_POLICY_FUNCTION_PERM_ANY: case LIPE_POLICY_FUNCTION_LAYOUT: + case LIPE_POLICY_FUNCTION_XATTR_NAME: break; } } @@ -2449,52 +2501,3 @@ int lipe_policy_value_init(struct lipe_list_head *list, expression); return rc; } - -static void lipe_policy_xattr_free(struct lipe_xattr *xattr) -{ - free(xattr->lx_name); - LIPE_FREE(xattr->lx_value, xattr->lx_value_len + 1); - LIPE_FREE_PTR(xattr); -} - -static void lipe_policy_xattrs_fini(struct lipe_object_attrs *attrs) -{ - struct lipe_xattr *xattr, *n; - - lipe_list_for_each_entry_safe(xattr, n, &attrs->loa_xattrs, - lx_linkage) { - lipe_list_del_init(&xattr->lx_linkage); - lipe_policy_xattr_free(xattr); - } - LASSERT(lipe_list_empty(&attrs->loa_xattrs)); -} - -void lipe_object_attrs_reset(struct lipe_object_attrs *attrs) -{ - struct lov_user_md *lum = attrs->loa_lum; - int lum_size = attrs->loa_lum_size; - - lipe_policy_xattrs_fini(attrs); - memset(lum, 0, lum_size); - memset(attrs, 0, sizeof(*attrs)); - attrs->loa_lum = lum; - attrs->loa_lum_size = lum_size; - LIPE_INIT_LIST_HEAD(&attrs->loa_xattrs); -} - -int lipe_object_attrs_init(struct lipe_object_attrs *attrs) -{ - attrs->loa_lum_size = lov_user_md_size(LOV_MAX_STRIPE_COUNT, - LOV_USER_MAGIC_V3); - LIPE_ALLOC(attrs->loa_lum, attrs->loa_lum_size); - if (attrs->loa_lum == NULL) - return -ENOMEM; - LIPE_INIT_LIST_HEAD(&attrs->loa_xattrs); - return 0; -} - -void lipe_object_attrs_fini(struct lipe_object_attrs *attrs) -{ - lipe_policy_xattrs_fini(attrs); - LIPE_FREE(attrs->loa_lum, attrs->loa_lum_size); -} diff --git a/src/general_policy.h b/src/general_policy.h index e4a76d0..89fc5b7 100644 --- a/src/general_policy.h +++ b/src/general_policy.h @@ -551,6 +551,8 @@ enum lipe_policy_function_type { LIPE_POLICY_FUNCTION_PERM_ANY = 15, /* Layout pattern contains given flags */ LIPE_POLICY_FUNCTION_LAYOUT = 16, + /* Whether the file has an xattr with given name */ + LIPE_POLICY_FUNCTION_XATTR_NAME = 17, }; #define LIPE_POLICY_FUNCTION_NAMES { \ @@ -570,6 +572,7 @@ enum lipe_policy_function_type { [LIPE_POLICY_FUNCTION_PERM_AT_LEAST] = "perm_at_least", \ [LIPE_POLICY_FUNCTION_PERM_ANY] = "perm_any", \ [LIPE_POLICY_FUNCTION_LAYOUT] = "layout", \ + [LIPE_POLICY_FUNCTION_XATTR_NAME] = "xattr_name", \ } #define LIPE_POLICY_FUNCTION_EXAMPLES { \ @@ -596,6 +599,7 @@ enum lipe_policy_function_type { [LIPE_POLICY_FUNCTION_PERM_ANY] = "perm_any(01010)", \ [LIPE_POLICY_FUNCTION_LAYOUT] = \ "layout(\"mdt\"), layout(\"mdt,raid0,released\")", \ + [LIPE_POLICY_FUNCTION_XATTR_NAME] = "xattr_name(\"user.foo\")", \ } #define LIPE_POLICY_FUNCTION_ARG_TYPES { \ @@ -615,6 +619,7 @@ enum lipe_policy_function_type { [LIPE_POLICY_FUNCTION_PERM_AT_LEAST] = LIPE_POLICY_FUNCTION_ARG_INT, \ [LIPE_POLICY_FUNCTION_PERM_ANY] = LIPE_POLICY_FUNCTION_ARG_INT, \ [LIPE_POLICY_FUNCTION_LAYOUT] = LIPE_POLICY_FUNCTION_ARG_STR, \ + [LIPE_POLICY_FUNCTION_XATTR_NAME] = LIPE_POLICY_FUNCTION_ARG_STR, \ } #define MAX_FUNCTION_ARGUMENT (4096) @@ -700,9 +705,6 @@ int lipe_policy_rule_postorder_traverse(struct lipe_policy_value *value, bool evaluating, struct lipe_object_attrs *attrs, struct lipe_policy_sysattrs *sysattrs); -int lipe_object_attrs_init(struct lipe_object_attrs *attrs); -void lipe_object_attrs_fini(struct lipe_object_attrs *attrs); -void lipe_object_attrs_reset(struct lipe_object_attrs *attrs); int lipe_policy_attr_parse(const char *field, enum lipe_policy_attribute_type *attr_type, __u64 *attr_bits); diff --git a/src/lamigo.c b/src/lamigo.c index 90fc2e2..2f29566 100644 --- a/src/lamigo.c +++ b/src/lamigo.c @@ -219,6 +219,7 @@ struct options opt = { .o_alr_ofd_interval = 0, .o_alr_hot_fraction = DEF_HOT_FRACTION, .o_alr_hot_after_idle = DEF_HOT_AFTER_IDLE, + .o_src_dom = 0, }; struct history { @@ -431,12 +432,12 @@ void lamigo_usr1_handle(int sig) FILE *f; int i; - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "dump to %s\n", opt.o_dump_file); + llapi_printf(LLAPI_MSG_DEBUG, "dump to %s\n", opt.o_dump_file); if (opt.o_dump_file == NULL) return; f = fopen(opt.o_dump_file, "w"); if (!f) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "can't open dump file\n"); + llapi_error(LLAPI_MSG_ERROR, errno, "can't open dump file"); return; } fprintf(f, @@ -468,14 +469,15 @@ void lamigo_usr1_handle(int sig) " hot_fraction: %d\n" " hot_after_idle: %d\n" " src_free: %d\n" - " tgt_free: %d\n", + " tgt_free: %d\n" + " src_dom: %d\n", opt.o_tgt_pool, opt.o_min_age, opt.o_cache_size, opt.o_rescan, opt.o_num_threads, opt.o_pool_refresh, opt.o_progress_interval, opt.o_alr_periods, opt.o_alr_period_time, opt.o_alr_warmup_k, opt.o_alr_cooldown_k, opt.o_alr_ofd_interval, opt.o_alr_hot_fraction, opt.o_alr_hot_after_idle, - opt.o_src_free, opt.o_tgt_free); + opt.o_src_free, opt.o_tgt_free, opt.o_src_dom); for (pl = src_pools; pl != NULL; pl = pl->pl_next, i++) fprintf(f, "pool %s:\n" " osts: %d\n" @@ -655,9 +657,9 @@ static int lamigo_exec_cmd(struct resync_agent *a, char *cmd) rc = lipe_ssh_exec(&rss->rss_ctx, cmd); if (rc) - llapi_error(LLAPI_MSG_INFO|LLAPI_MSG_NO_ERRNO, 0, - "error executing ssh command '%s' on '%s': rc = %d\n", - cmd, a->rag_hostname, rc); + llapi_error(LLAPI_MSG_INFO, rc, + "error executing ssh command '%s' on '%s'", + cmd, a->rag_hostname); pthread_mutex_lock(&a->rag_ssh_lock); lipe_list_add(&rss->rss_list, &a->rag_ssh_list); @@ -704,7 +706,7 @@ void *lamigo_replicate_one(void *args) agent->rag_mountpoint, PFID(&rj->rj_fid)); } else { - llapi_error(LLAPI_MSG_ERROR|LLAPI_MSG_NO_ERRNO, 0, "unknown resync: %d\n", resync); + llapi_err_noerrno(LLAPI_MSG_ERROR, "unknown resync: %d", resync); rc = -EINVAL; goto out; } @@ -742,13 +744,13 @@ static int lamigo_spawn_replication(struct resync_job *rj) } } if (!a) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "no good agent\n"); + llapi_printf(LLAPI_MSG_DEBUG, "no good agent\n"); return -EBUSY; } rj->rj_agent = a; rj->rj_start = time(NULL); - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "new job %s for "DFID" spawned on %s\n", + llapi_printf(LLAPI_MSG_DEBUG, "new job %s for "DFID" spawned on %s\n", resync == AMIGO_RESYNC_EXTEND ? "extend" : "resync", PFID(&rj->rj_fid), rj->rj_agent->rag_hostname); @@ -956,6 +958,13 @@ static int lamigo_striping_is_in_sync(struct lov_user_md *lum, /* do not reset onsrc as any object matters */ } + if (opt.o_src_dom && v3->lmm_pattern == LOV_PATTERN_MDT) { + llapi_printf(LLAPI_MSG_DEBUG, + "DoM component"); + onsrc++; + continue; + } + rc = lamigo_get_objects(v3, &objects, &stripes); if (rc) { stats.s_skip_unknown++; @@ -1044,7 +1053,7 @@ lamigo_check_user_rules(struct lipe_object_attrs *attrs, rc = lipe_rule_evaluate(rule, attrs, sysattrs, &result); if (rc) { - llapi_error(LLAPI_MSG_ERROR, rc, "rule failed\n"); + llapi_error(LLAPI_MSG_ERROR, rc, "rule failed"); return AMIGO_RESYNC_NONE; } if (!result) @@ -1087,7 +1096,6 @@ static inline void lustre_som_swab(struct lustre_som_attrs *attrs) static char lamigo_lmd_buf[offsetof(typeof(struct lov_user_mds_data_v1), lmd_lmm) + 65536]; -int decode_linkea(char *buf, ssize_t size); static int lamigo_get_attrs(const struct lu_fid *fid, __u64 need_attr, struct lipe_object_attrs *attrs) @@ -1104,9 +1112,8 @@ static int lamigo_get_attrs(const struct lu_fid *fid, rc = ioctl(open_by_fid_fd, IOC_MDC_GETFILEINFO_V1, lamigo_lmd_buf); if (rc) { if (errno != ENOENT) - llapi_error(LLAPI_MSG_DEBUG, errno, - "cannot get striping for "DFID, - PFID(fid)); + llapi_error(LLAPI_MSG_ERROR, errno, + "cannot get striping for "DFID, PFID(fid)); return errno; } lmdp = (struct lov_user_mds_data_v1 *)lamigo_lmd_buf; @@ -1135,7 +1142,7 @@ static int lamigo_get_attrs(const struct lu_fid *fid, snprintf(fidstr, sizeof(fidstr), DFID, PFID(fid)); fd = openat(open_by_fid_fd, fidstr, O_RDONLY); if (fd < 0) { - llapi_printf(LLAPI_MSG_DEBUG, "can't open file: rc=%d\n", errno); + llapi_error(LLAPI_MSG_ERROR, errno, "can't open file"); goto out; } @@ -1159,9 +1166,7 @@ static int lamigo_get_attrs(const struct lu_fid *fid, rc = fgetxattr(fd, XATTR_NAME_LINK, attrs->loa_leh_buf, sizeof(attrs->loa_leh_buf)); if (rc >= 0) - rc = decode_linkea(attrs->loa_leh_buf, rc); - if (rc == 0) - attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_LINKEA; + rc = lipe_object_attrs_set_links(attrs, attrs->loa_leh_buf, rc); } if (need_attr & LIPE_OBJECT_ATTR_HSMEA) { @@ -1241,8 +1246,7 @@ static int lamigo_is_in_sync(struct lu_fid *fid, out: lamigo_hist_add(fid, resync); - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, - "check "DFID" stripes=%d: resync=%d\n", + llapi_printf(LLAPI_MSG_DEBUG, "check "DFID" stripes=%d: resync=%d\n", PFID(fid), mo->mo_stripes, resync); return resync; @@ -1287,8 +1291,8 @@ static void lamigo_check_bad_agents(void) return; rj = calloc(1, sizeof(struct resync_job)); if (rj == NULL) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, - "can't allocate for a test job\n"); + llapi_err_noerrno(LLAPI_MSG_ERROR, + "can't allocate for a test job"); return; } rj->rj_check_job = 1; @@ -1313,9 +1317,7 @@ static int lamigo_submit_job(struct resync_job *rj) rc = lamigo_spawn_replication(rj); if (rc < 0) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, - "can't create a job: rc=%d, %s\n", - rc, strerror(errno)); + llapi_error(LLAPI_MSG_ERROR, errno, "can't create a job"); lipe_list_add_tail(&rj->rj_list, &lamigo_failed_job_list); return 1; } @@ -1344,8 +1346,7 @@ void lamigo_schedule_setprefer(struct resync_job *rj, void *cbdata, int rc) srj = calloc(1, sizeof(*srj)); if (srj == NULL) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, - "can't allocate for a job\n"); + llapi_err_noerrno(LLAPI_MSG_ERROR, "can't allocate for a job"); return; } @@ -1375,7 +1376,7 @@ static int lamigo_update_one(struct fid_rec *f) if (are_agents_busy()) { /* all the agents are busy */ - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "no agents avilable (max: %d)\n", lamigo_max_jobs); + llapi_printf(LLAPI_MSG_DEBUG, "no agents avilable (max: %d)\n", lamigo_max_jobs); return 1; } @@ -1405,7 +1406,7 @@ static int lamigo_update_one(struct fid_rec *f) rj = calloc(1, sizeof(struct resync_job)); if (rj == NULL) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "can't allocate for a job\n"); + llapi_err_noerrno(LLAPI_MSG_ERROR, "can't allocate for a job"); return 1; } rj->rj_fid = f->fr_fh.fh_fid; @@ -1462,9 +1463,8 @@ static int lamigo_check_sync(void) struct resync_job, rj_list); lipe_list_del(&rj->rj_list); rc = lamigo_submit_job(rj); - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, - "tried to resubmit failed job %p: rc=%d\n", - rj, rc); + llapi_printf(LLAPI_MSG_DEBUG, + "tried to resubmit failed job %p: rc=%d\n", rj, rc); if (rc != 0) return rc; } @@ -1538,7 +1538,7 @@ static int lamigo_process_record(struct changelog_rec *rec) if (f == NULL) { rc = -ENOMEM; llapi_error(LLAPI_MSG_ERROR, rc, - "failed to alloc memory for fid_rec"); + "failed to alloc memory for fid_rec"); return rc; } @@ -1636,7 +1636,7 @@ static void lamigo_check_and_clear_changelog(void) index - lamigo_last_cleared_index < opt.o_chlg_clear_frequency) return; - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "CLEAR upto %llu in %s (%llu last)\n", + llapi_printf(LLAPI_MSG_DEBUG, "CLEAR upto %llu in %s (%llu last)\n", index, opt.o_chlg_user, lamigo_last_processed_idx); lamigo_last_cleared_index = index; rc = llapi_changelog_clear(opt.o_mdtname, opt.o_chlg_user, index); @@ -1650,8 +1650,8 @@ static void lamigo_check_and_clear_changelog(void) static void lamigo_job_fini(struct resync_job *rj, intptr_t retval) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, - "job %lu on "DFID" done in %lu: %"PRIdPTR" (%d)\n", + llapi_printf(LLAPI_MSG_DEBUG, + "job %lu on "DFID" done in %lu: %"PRIdPTR" (%d)\n", rj->rj_pid, PFID(&rj->rj_fid), time(NULL) - rj->rj_start, retval, rj->rj_agent->rag_bad); @@ -1662,7 +1662,7 @@ static void lamigo_job_fini(struct resync_job *rj, intptr_t retval) if (retval == 0) { /* the agent is back */ if (rj->rj_agent->rag_bad) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "agent %s is back\n", + llapi_printf(LLAPI_MSG_DEBUG, "agent %s is back\n", rj->rj_agent->rag_hostname); rj->rj_agent->rag_bad = false; lamigo_max_jobs += rj->rj_agent->rag_maxjobs; @@ -1670,7 +1670,7 @@ static void lamigo_job_fini(struct resync_job *rj, intptr_t retval) } else { /* the agent is still bad */ if (rj->rj_agent->rag_bad == false) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "agent %s is bad\n", + llapi_printf(LLAPI_MSG_DEBUG, "agent %s is bad\n", rj->rj_agent->rag_hostname); assert(lamigo_max_jobs >= rj->rj_agent->rag_maxjobs); @@ -1748,8 +1748,8 @@ static void lamigo_add_agent(const char *host, const char *mnt, char *jobs) a = calloc(1, sizeof(*a)); if (!a) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "can't allocate memory for agent\n"); + llapi_err_noerrno(LLAPI_MSG_FATAL, + "can't allocate memory for agent"); exit(1); } @@ -1761,9 +1761,9 @@ static void lamigo_add_agent(const char *host, const char *mnt, char *jobs) a->rag_maxjobs = strtol(jobs, &endptr, 10); if (*endptr != '\0') { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "invalid jobs: '%s' (1-2048 expected)\n", - jobs); + llapi_err_noerrno(LLAPI_MSG_FATAL, + "invalid jobs: '%s' (1-2048 expected)", + jobs); exit(1); } } else { @@ -1771,9 +1771,9 @@ static void lamigo_add_agent(const char *host, const char *mnt, char *jobs) } if (a->rag_maxjobs < 1 || a->rag_maxjobs > 2048) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "invalid jobs per agent: %d (1-2048 expected)\n", - a->rag_maxjobs); + llapi_err_noerrno(LLAPI_MSG_FATAL, + "invalid jobs per agent: %d (1-2048 expected)", + a->rag_maxjobs); exit(1); } lipe_list_add(&a->rag_list, &lamigo_agent_list); @@ -1791,14 +1791,14 @@ static void lamigo_add_agent(const char *host, const char *mnt, char *jobs) int rc; if (!rss) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, + llapi_err_noerrno(LLAPI_MSG_FATAL, "can't allocate memory for agent ssh session\n"); exit(1); } rc = lipe_ssh_context_init(&rss->rss_ctx, a->rag_hostname); if (rc < 0) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, + llapi_err_noerrno(LLAPI_MSG_FATAL, "cannot create SSH context for '%s'\n", a->rag_hostname); exit(1); @@ -1807,7 +1807,7 @@ static void lamigo_add_agent(const char *host, const char *mnt, char *jobs) lipe_list_add(&rss->rss_list, &a->rag_ssh_list); } - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "AGENT: %s %s %d\n", a->rag_hostname, + llapi_printf(LLAPI_MSG_DEBUG, "AGENT: %s %s %d\n", a->rag_hostname, a->rag_mountpoint, a->rag_maxjobs); lamigo_agent_count++; @@ -1829,6 +1829,7 @@ static void lamigo_add_agent(const char *host, const char *mnt, char *jobs) #define LAMIGO_OPT_SRC_FREE 16 #define LAMIGO_OPT_TGT_FREE 17 #define LAMIGO_OPT_VERSION 18 +#define LAMIGO_OPT_SRC_DOM 19 static struct option options[] = { { "ignore-reads", no_argument, NULL, LAMIGO_OPT_IGNORE_READS}, @@ -1860,6 +1861,7 @@ static struct option options[] = { LAMIGO_OPT_PROGRESS_INTV }, { "rescan", no_argument, NULL, 'r'}, { "src", required_argument, NULL, 's'}, + { "src-dom", no_argument, NULL, LAMIGO_OPT_SRC_DOM}, { "src-free", required_argument, NULL, LAMIGO_OPT_SRC_FREE}, { "tgt-free", required_argument, NULL, LAMIGO_OPT_TGT_FREE}, { "statfs-refresh", required_argument, NULL, LAMIGO_OPT_STATFS_REFRESH }, @@ -1915,12 +1917,12 @@ struct pool_list *lamigo_alloc_pool(char *pool) pl = calloc(sizeof(*pl), 1); if (pl == NULL) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, "can't allocate pool\n"); + llapi_err_noerrno(LLAPI_MSG_FATAL, "can't allocate pool"); exit(1); } pl->pl_pool = strdup(pool); if (pl->pl_pool == NULL) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, "can't allocate pool name\n"); + llapi_err_noerrno(LLAPI_MSG_FATAL, "can't allocate pool name"); exit(1); } pl->pl_ostnr = 0; @@ -1960,9 +1962,8 @@ void lamigo_refresh_osts_from_pool(struct pool_list *pl) rc = cfs_get_param_paths(&paths, "lod/%s-*/numobd", fsname); if (rc != 0) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "can't find numobd fs '%s': rc=%d\n", - fsname, errno); + llapi_error(LLAPI_MSG_FATAL, errno, + "can't find numobd fs '%s'", fsname); exit(1); } for (i = 0; i < paths.gl_pathc; i++) { @@ -1972,8 +1973,8 @@ void lamigo_refresh_osts_from_pool(struct pool_list *pl) obdcount = strtol(data, &endptr, 10); if (*endptr != '\0') { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "invalid numobd: '%s'\n", data); + llapi_err_noerrno(LLAPI_MSG_FATAL, + "invalid numobd: '%s'", data); exit(1); } break; @@ -1982,16 +1983,14 @@ void lamigo_refresh_osts_from_pool(struct pool_list *pl) globfree(&paths); if (obdcount < 0) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, "can't find fs '%s': rc=%d\n", - fsname, errno); + llapi_error(LLAPI_MSG_FATAL, errno, "can't find fs '%s'", fsname); exit(1); } bufsize = sizeof(struct obd_uuid) * obdcount; buffer = malloc(bufsize + sizeof(*list) * obdcount); if (buffer == NULL) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "can't get mem for pool members: rc=%d\n", rc); + llapi_err_noerrno(LLAPI_MSG_FATAL, "can't get mem for pool members"); exit(1); } list = (char **) (buffer + bufsize); @@ -2014,8 +2013,7 @@ void lamigo_refresh_osts_from_pool(struct pool_list *pl) if (pl->pl_osts == NULL) pl->pl_osts = malloc(sizeof(int) * nb); if (pl->pl_osts == NULL) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "can't allocate mem for OST ind\n"); + llapi_err_noerrno(LLAPI_MSG_FATAL, "can't allocate mem for OST ind"); exit(1); } @@ -2145,6 +2143,9 @@ void lamigo_process_opt(int c, char *optarg) exit(1); } break; + case LAMIGO_OPT_SRC_DOM: + opt.o_src_dom = 1; + break; case LAMIGO_OPT_TGT_FREE: opt.o_tgt_free = atoi(optarg); if (opt.o_tgt_free < 1 || opt.o_tgt_free > 99) { @@ -2188,8 +2189,7 @@ void lamigo_process_opt(int c, char *optarg) opt.o_cache_size = get_fid_cache_size(cache_size); else opt.o_cache_size = cache_size; - llapi_error(LLAPI_MSG_INFO|LLAPI_MSG_NO_ERRNO, 0, "Cache size: %lu\n", - opt.o_cache_size); + llapi_printf(LLAPI_MSG_INFO, "Cache size: %lu\n", opt.o_cache_size); break; } case 'f': @@ -2200,8 +2200,8 @@ void lamigo_process_opt(int c, char *optarg) mnt = strsep(&optarg, ":"); jobs = strsep(&optarg, ":"); if (!host || !mnt) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "invalid agent definition\n"); + llapi_err_noerrno(LLAPI_MSG_FATAL, + "invalid agent definition"); exit(1); } lamigo_add_agent(host, mnt, jobs); @@ -2215,9 +2215,9 @@ void lamigo_process_opt(int c, char *optarg) } else { opt.o_alr_heat_fn = atoi(optarg); if (opt.o_alr_heat_fn < 0 || opt.o_alr_heat_fn > 1) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, - 0, "invalid heat function '%s'\n", - optarg); + llapi_err_noerrno(LLAPI_MSG_FATAL, + "invalid heat function '%s'", + optarg); exit(1); } } @@ -2375,8 +2375,8 @@ static void load_config(char *name) f = fopen(name, "r"); if (!f) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "can't open config file %s\n", name); + llapi_error(LLAPI_MSG_FATAL, errno, + "can't open config file %s", name); exit(1); } while (!feof(f)) { @@ -2405,13 +2405,12 @@ static void load_config(char *name) } if (strcmp(t, "mountpoint") == 0) - llapi_error(LLAPI_MSG_WARN|LLAPI_MSG_NO_ERRNO, 0, - "'mountpoint' is deprecated, use 'mount' instead\n"); + llapi_err_noerrno(LLAPI_MSG_WARN, + "'mountpoint' is deprecated, use 'mount' instead"); opt = lamigo_keyword_lookup(t); if (!opt) { - llapi_error(LLAPI_MSG_ERROR|LLAPI_MSG_NO_ERRNO, 0, - "unknown tunable: %s\n", t); + llapi_err_noerrno(LLAPI_MSG_ERROR, "unknown tunable: %s", t); continue; } @@ -2424,14 +2423,14 @@ static void load_config(char *name) optarg = strsep(&s, "\n "); if (!optarg && opt->has_arg == required_argument) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "no argument for %s\n", t); + llapi_err_noerrno(LLAPI_MSG_FATAL, + "no argument for %s", t); exit(1); } } else { optarg = NULL; } - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "conf: %s %s\n", t, optarg); + llapi_printf(LLAPI_MSG_DEBUG, "conf: %s %s\n", t, optarg); lamigo_process_opt(opt->val, optarg); } @@ -2455,8 +2454,8 @@ void lamigo_parse_opts(int argc, char **argv) exit(1); } if (strcmp(options[opt_index].name, "mountpoint") == 0) - llapi_error(LLAPI_MSG_WARN|LLAPI_MSG_NO_ERRNO, 0, - "'mountpoint' is deprecated, use 'mount' instead\n"); + llapi_err_noerrno(LLAPI_MSG_WARN, + "'mountpoint' is deprecated, use 'mount' instead"); lamigo_process_opt(c, optarg); } @@ -2475,13 +2474,12 @@ void lamigo_parse_opts(int argc, char **argv) } if (!opt.o_mdtname) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, "no MDT specified\n"); + llapi_err_noerrno(LLAPI_MSG_FATAL, "no MDT specified"); exit(1); } rc = cfs_get_param_paths(&paths, "mdt/%s/uuid", opt.o_mdtname); if (rc != 0) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, "can't find MDT %s\n", - opt.o_mdtname); + llapi_err_noerrno(LLAPI_MSG_FATAL, "can't find MDT %s", opt.o_mdtname); exit(1); } globfree(&paths); @@ -2489,39 +2487,38 @@ void lamigo_parse_opts(int argc, char **argv) snprintf(buf, sizeof(buf), "%s/.lustre/fid", opt.o_mntpt); open_by_fid_fd = open(buf, O_RDONLY); if (open_by_fid_fd < 0) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "can't open '%s': %d\n", buf, errno); + llapi_error(LLAPI_MSG_FATAL, errno, "can't open '%s'", buf); exit(1); } if (src_pools == NULL) { lamigo_parse_pool(DEF_SOURCE_POOL); - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "source pools aren't defined, use %s\n", + llapi_err_noerrno(LLAPI_MSG_FATAL, + "source pools aren't defined, use %s", DEF_SOURCE_POOL); } if (opt.o_tgt_pool == NULL) { opt.o_tgt_pool = DEF_TARGET_POOL; - llapi_error(LLAPI_MSG_INFO|LLAPI_MSG_NO_ERRNO, 0, - "target pool not defined, use %s\n", + llapi_err_noerrno(LLAPI_MSG_INFO, + "target pool not defined, use %s", opt.o_tgt_pool); } opt.o_tgt_pool_len = strlen(opt.o_tgt_pool); if (lamigo_lookup_pool(opt.o_tgt_pool)) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "target pool '%s' cannot also be source pool\n", + llapi_err_noerrno(LLAPI_MSG_FATAL, + "target pool '%s' cannot also be source pool", opt.o_tgt_pool); exit(1); } if (lipe_list_empty(&lamigo_agent_list)) { - llapi_err_noerrno(LLAPI_MSG_ERROR, "no agents configured?\n"); + llapi_err_noerrno(LLAPI_MSG_ERROR, "no agents configured?"); exit(1); } - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "target pool: %s/%d\n", opt.o_tgt_pool, + llapi_printf(LLAPI_MSG_DEBUG, "target pool: %s/%d\n", opt.o_tgt_pool, opt.o_tgt_pool_len); tgt_pools = lamigo_alloc_pool(opt.o_tgt_pool); @@ -2541,7 +2538,7 @@ void lamigo_parse_opts(int argc, char **argv) rc = pipe2(lamigo_sigpipe, O_NONBLOCK); if (rc < 0) { llapi_error(LLAPI_MSG_FATAL, errno, - "cannot create sigpipe\n"); + "cannot create sigpipe"); exit(1); } } @@ -2575,7 +2572,7 @@ static int lamigo_create_job(struct lu_fid *fid, rj = calloc(1, sizeof(struct resync_job)); if (rj == NULL) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "can't allocate for a job\n"); + llapi_err_noerrno(LLAPI_MSG_ERROR, "can't allocate for a job"); return 1; } rj->rj_fid = *fid; @@ -2628,11 +2625,14 @@ int lamigo_lipe_callback(struct lipe_instance *instance, /* Quit on error since rules will only match the files with * valid attrs */ rule.lr_attr_bits = lamigo_rule_attrs; - rc = lipe_rule_read_attrs(NULL, object, &rule, attrs, - &sysattrs, NULL, true, - false); + rc = lipe_rule_read_attrs(NULL /* policy */, + object, + &rule, + attrs, + true /* need_fid */, + false /* quit_on_error */); if (rc) { - llapi_error(LLAPI_MSG_ERROR, rc, "can't get attrs\n"); + llapi_error(LLAPI_MSG_ERROR, rc, "can't get attrs"); return 0; } @@ -2675,9 +2675,8 @@ static int lamigo_find_device(const char *devname) rc = cfs_get_param_paths(&paths, "osd-*/%s/fstype", devname); if (rc != 0) { - llapi_error(LLAPI_MSG_ERROR|LLAPI_MSG_NO_ERRNO, 0, - "can't find device '%s': rc=%d\n", - devname, errno); + llapi_error(LLAPI_MSG_ERROR, errno, + "can't find device '%s'", devname); return rc; } for (i = 0; i < paths.gl_pathc; i++) { @@ -2691,8 +2690,8 @@ static int lamigo_find_device(const char *devname) break; } if (!mdtprefix) { - llapi_error(LLAPI_MSG_ERROR|LLAPI_MSG_NO_ERRNO, 0, - "Can't find device '%s'\n", devname); + llapi_err_noerrno(LLAPI_MSG_ERROR, + "Can't find device '%s'", devname); return -1; } i = strlen(mdtprefix); @@ -2720,7 +2719,7 @@ static int lamigo_read_file(const char *param, char *val, const int vallen) rc = 0; } if (rc < 0) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, "can't read: %d\n", errno); + llapi_error(LLAPI_MSG_FATAL, errno, "can't read"); return -errno; } close(fd); @@ -2735,8 +2734,7 @@ static int lamigo_read_param(const char *param, char *val, const int vallen) snprintf(buf, sizeof(buf), "%s/%s", mdtprefix, param); rc = lamigo_read_file(buf, val, vallen); if (rc) - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "can't open %s: %s\n", param, strerror(rc)); + llapi_error(LLAPI_MSG_FATAL, errno, "can't open %s", param); return rc; } @@ -2750,18 +2748,17 @@ int lamigo_rescan(void) char buf[1024]; bool ldd_err; int rc; - struct lu_fid watch_fid = {0, 0, 0}; rc = lamigo_find_device(opt.o_mdtname); if (rc) { - llapi_error(LLAPI_MSG_ERROR|LLAPI_MSG_NO_ERRNO, 0, "failed to find device '%s'\n", - opt.o_mdtname); + llapi_err_noerrno(LLAPI_MSG_ERROR, "failed to find device '%s'", + opt.o_mdtname); return rc; } rc = lamigo_read_param("fstype", buf, sizeof(buf)); if (rc) { - llapi_error(LLAPI_MSG_ERROR|LLAPI_MSG_NO_ERRNO, 0, "failed to read param fstype\n"); + llapi_err_noerrno(LLAPI_MSG_ERROR, "failed to read param fstype"); return rc; } @@ -2770,13 +2767,13 @@ int lamigo_rescan(void) else if (!strcmp(buf, "zfs")) { instance.li_expected_fstype = LBT_LUSTRE_ON_ZFS; } else { - llapi_error(LLAPI_MSG_ERROR|LLAPI_MSG_NO_ERRNO, 0, "unknown fstype %s\n", buf); + llapi_err_noerrno(LLAPI_MSG_ERROR, "unknown fstype %s", buf); return -EINVAL; } rc = lamigo_read_param("mntdev", mdt_mntdev, sizeof(buf)); if (rc) { - llapi_error(LLAPI_MSG_ERROR|LLAPI_MSG_NO_ERRNO, 0, "failed to read param mntdev\n"); + llapi_err_noerrno(LLAPI_MSG_ERROR, "failed to read param mntdev"); return rc; } strcpy(instance.li_device, mdt_mntdev); @@ -2792,12 +2789,18 @@ int lamigo_rescan(void) memset(&result, 0, sizeof(result)); gettimeofday(&result.sr_time_start, NULL); - rc = lipe_scan(&instance, &policy, &result, NULL, - NULL, opt.o_num_threads, "test", &watch_fid, - true, false, &ldd_err); + rc = lipe_scan(&instance, + &policy, + &result, + NULL /* sum_counter_list */, + NULL /* sum_classify_list */, + opt.o_num_threads, + "test" /* workspace */, + true /* abort_failure */, + &ldd_err); if (rc) { - llapi_error(LLAPI_MSG_ERROR|LLAPI_MSG_NO_ERRNO, 0, "failed to scan %s\n", - instance.li_device); + llapi_err_noerrno(LLAPI_MSG_ERROR, "failed to scan %s", + instance.li_device); return rc; } @@ -2805,7 +2808,7 @@ int lamigo_rescan(void) diff_timevals(&result.sr_time_start, &result.sr_time_end, &result.sr_time_diff); - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "finished scanning in %d.%06u seconds\n", + llapi_printf(LLAPI_MSG_DEBUG, "finished scanning in %d.%06u seconds\n", (int)result.sr_time_diff.tv_sec, (unsigned int)result.sr_time_diff.tv_usec); @@ -2821,15 +2824,15 @@ static void lamigo_changelog_check_and_set_mask(void) rc = cfs_get_param_paths(&paths, "mdd/%s/changelog_mask", opt.o_mdtname); if (rc != 0 || paths.gl_pathc != 1) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, "can't find changelog mask\n"); + llapi_err_noerrno(LLAPI_MSG_FATAL, "can't find changelog mask"); exit(1); } rc = lamigo_read_file(paths.gl_pathv[0], buf, sizeof(buf)); globfree(&paths); if (rc < 0) { - llapi_error(LLAPI_MSG_WARN|LLAPI_MSG_NO_ERRNO, 0, "can't open %s, assume OK: %s\n", - paths.gl_pathv[0], strerror(rc)); + llapi_error(LLAPI_MSG_WARN, rc, "can't open %s, assume OK", + paths.gl_pathv[0]); return; } if (rc >= 0 && strstr(buf, "CLOSE") && strstr(buf, "UNLNK")) { @@ -2837,19 +2840,19 @@ static void lamigo_changelog_check_and_set_mask(void) return; } - llapi_error(LLAPI_MSG_INFO|LLAPI_MSG_NO_ERRNO, 0, "set changelog mask\n"); + llapi_err_noerrno(LLAPI_MSG_INFO, "set changelog mask"); snprintf(buf, sizeof(buf), "lctl set_param -n mdd.%s.changelog_mask=+\"CLOSE UNLNK\"", opt.o_mdtname); rc = system(buf); if (rc < 0) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "can't enable CLOSE/UNLNK in changelog: rc=%d\n", + llapi_err_noerrno(LLAPI_MSG_FATAL, + "can't enable CLOSE/UNLNK in changelog: rc=%d", rc); exit(1); } - llapi_error(LLAPI_MSG_INFO|LLAPI_MSG_NO_ERRNO, 0, "enable CLOSE/UNLNK in changelog\n"); + llapi_err_noerrno(LLAPI_MSG_INFO, "enable CLOSE/UNLNK in changelog"); } /* @@ -2866,12 +2869,12 @@ static int lamigo_check_changelog_user(const char *user) rc = cfs_get_param_paths(&paths, "mdd/%s/changelog_users", opt.o_mdtname); if (rc != 0 || paths.gl_pathc != 1) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, "can't find changelog users\n"); + llapi_err_noerrno(LLAPI_MSG_FATAL, "can't find changelog users"); exit(1); } rc = lamigo_read_file(paths.gl_pathv[0], buf, sizeof(buf)); if (rc < 0) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, "can't get changelog users\n"); + llapi_err_noerrno(LLAPI_MSG_FATAL, "can't get changelog users"); exit(1); } rc = -1; @@ -2915,7 +2918,7 @@ again: if (!rc) { /* found, use it */ opt.o_chlg_user = strdup(user); - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, + llapi_printf(LLAPI_MSG_DEBUG, "found Changelog user '%s' in '%s'\n", user, buf); return; @@ -2924,8 +2927,8 @@ again: if (registered) { /* can't find just registered changelog user */ - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "can't find registered Changelog user '%s'\n", + llapi_err_noerrno(LLAPI_MSG_FATAL, + "can't find registered Changelog user '%s'", user); exit(1); } @@ -2935,22 +2938,22 @@ again: rc = lamigo_check_changelog_user(opt.o_chlg_user); if (!rc) { /* found, use it */ - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, + llapi_printf(LLAPI_MSG_DEBUG, "found Changelog user '%s' from config\n", opt.o_chlg_user); return; } } - llapi_error(LLAPI_MSG_INFO|LLAPI_MSG_NO_ERRNO, 0, "register new Changelog user\n"); + llapi_err_noerrno(LLAPI_MSG_INFO, "register new Changelog user"); /* not found, try to register own */ snprintf(buf, sizeof(buf), "lctl --device %s changelog_register -n >"LAMIGO_USERFILE, opt.o_mdtname, opt.o_mdtname); rc = system(buf); if (rc < 0) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "Changelog user '%s' is not registered\n", + llapi_err_noerrno(LLAPI_MSG_FATAL, + "Changelog user '%s' is not registered", opt.o_chlg_user); exit(1); } @@ -2979,7 +2982,7 @@ void lamigo_show_progress(void) return; progress_last_processed = stats.s_processed; - llapi_error(LLAPI_MSG_INFO|LLAPI_MSG_NO_ERRNO, 0, + llapi_printf(LLAPI_MSG_INFO, "%lu processed, %lu replicated, %lu busy, %lu in queue, " "%lu hot skipped, %lu ro2hot, %lu rw2hot, %lu rw2cold\n", stats.s_processed, stats.s_replicated, stats.s_busy, @@ -2996,9 +2999,7 @@ static void lamigo_lock_pidfile(void) snprintf(buf, sizeof(buf), LAMIGO_PIDFILE, opt.o_mdtname); fd = open(buf, O_RDWR | O_CREAT, 0600); if (fd < 0) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "can't create pidfile: %s\n", - strerror(errno)); + llapi_error(LLAPI_MSG_FATAL, errno, "can't create pidfile"); exit(1); } rc = flock(fd, LOCK_EX | LOCK_NB); @@ -3009,24 +3010,22 @@ static void lamigo_lock_pidfile(void) sz = 0; if (sz > 0) buf[sz] = 0; - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "another lamigo is running, locked by %s\n", + llapi_err_noerrno(LLAPI_MSG_FATAL, + "another lamigo is running, locked by %s", sz > 0 ? buf : "[unknown]"); exit(1); } rc = ftruncate(fd, 0); if (rc < 0) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "cannot truncate pidfile: %s\n", strerror(errno)); + llapi_error(LLAPI_MSG_FATAL, errno, "cannot truncate pidfile"); exit(1); } sz = snprintf(buf, sizeof(buf), "%d\n", getpid()); rc = write(fd, buf, sz); if (rc < 0 || rc != sz) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, - "can't write pidfile: rc=%d\n", rc); + llapi_error(LLAPI_MSG_FATAL, rc, "can't write pidfile"); exit(1); } } @@ -3099,12 +3098,11 @@ again: } while (i++ < 5 && rc != 0); if (rc) { llapi_error(LLAPI_MSG_ERROR, rc, - "unable to reopen changelog of MDT [%s]\n", + "unable to reopen changelog of MDT [%s]", opt.o_mdtname); exit(1); } - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, - "Reopened changelog\n"); + llapi_printf(LLAPI_MSG_DEBUG, "Reopened changelog\n"); goto again; } } @@ -3120,7 +3118,7 @@ void lamigo_parse_rules(const char *rule_str, const char *filename) tok = json_tokener_new(); if (!tok) { llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO , -1, - "cannot allocate json token\n"); + "cannot allocate json token"); exit(1); } @@ -3141,7 +3139,7 @@ void lamigo_parse_rules(const char *rule_str, const char *filename) } if (json_object_get_type(obj_rules) != json_type_array) { llapi_error(LLAPI_MSG_ERROR, -EINVAL, - "rules are not an array in %s\n", filename); + "rules are not an array in %s", filename); exit(1); } @@ -3155,7 +3153,7 @@ void lamigo_parse_rules(const char *rule_str, const char *filename) if (!obj_rule) { llapi_error(LLAPI_MSG_ERROR, -EINVAL, - "failed to get rule #%d\n", i); + "failed to get rule #%d", i); exit(1); } @@ -3163,14 +3161,14 @@ void lamigo_parse_rules(const char *rule_str, const char *filename) &obj_action); if (!rc) { llapi_error(LLAPI_MSG_ERROR, -EINVAL, - "no action in rule %s\n", + "no action in rule %s", json_object_to_json_string(obj_rule)); exit(1); } action = json_object_get_string(obj_action); if (!action) { llapi_error(LLAPI_MSG_ERROR, -EINVAL, - "invalid action in rule %s\n", + "invalid action in rule %s", json_object_to_json_string(obj_rule)); exit(1); } @@ -3178,14 +3176,14 @@ void lamigo_parse_rules(const char *rule_str, const char *filename) &obj_expr); if (!rc) { llapi_error(LLAPI_MSG_ERROR, -EINVAL, - "no expression in rule %s\n", + "no expression in rule %s", json_object_to_json_string(obj_rule)); exit(1); } expr = json_object_get_string(obj_expr); if (!expr) { llapi_error(LLAPI_MSG_ERROR, -EINVAL, - "invalid expression in rule %s\n", + "invalid expression in rule %s", json_object_to_json_string(obj_rule)); exit(1); } @@ -3193,7 +3191,7 @@ void lamigo_parse_rules(const char *rule_str, const char *filename) LIPE_ALLOC_PTR(lr); if (!lr) { llapi_error(LLAPI_MSG_ERROR, rc, - "cannot allocate rule %s\n", + "cannot allocate rule %s", json_object_to_json_string(obj_rule)); exit(1); } @@ -3204,7 +3202,7 @@ void lamigo_parse_rules(const char *rule_str, const char *filename) lr->lr_action.la_action = LAT_SHELL_CMD_FID; else { llapi_error(LLAPI_MSG_ERROR, 0, - "unknown action '%s' in rule %s\n", action, + "unknown action '%s' in rule %s", action, json_object_to_json_string(obj_rule)); exit(1); } @@ -3214,7 +3212,7 @@ void lamigo_parse_rules(const char *rule_str, const char *filename) &valid, expr); if (rc) { llapi_error(LLAPI_MSG_ERROR, rc, - "cannot parse expression in rule %s\n", + "cannot parse expression in rule %s", json_object_to_json_string(obj_rule)); exit(1); } @@ -3273,7 +3271,7 @@ int main(int argc, char **argv) rc = lamigo_init_cache(); if (rc < 0) { - llapi_error(LLAPI_MSG_FATAL|LLAPI_MSG_NO_ERRNO, 0, "can't init cache\n"); + llapi_err_noerrno(LLAPI_MSG_FATAL, "can't init cache\n"); exit(1); } @@ -3283,9 +3281,9 @@ int main(int argc, char **argv) /* wait till the target pool got one OST at least */ lamigo_refresh_osts_from_pool(tgt_pools); while (tgt_pools->pl_ostnr == 0) { - llapi_error(LLAPI_MSG_ERROR|LLAPI_MSG_NO_ERRNO, 0, - "Target pool %s is empty, waiting...\n", - tgt_pools->pl_pool); + llapi_err_noerrno(LLAPI_MSG_ERROR, + "Target pool %s is empty, waiting...", + tgt_pools->pl_pool); sleep(opt.o_pool_refresh); lamigo_refresh_osts_from_pool(tgt_pools); } @@ -3301,7 +3299,7 @@ int main(int argc, char **argv) rc = pthread_create(&pid, NULL, lamigo_refresh_statfs_thread, NULL); if (rc) { llapi_error(LLAPI_MSG_FATAL, rc, - "unable to start statfs thread\n"); + "unable to start statfs thread"); exit(1); } @@ -3310,7 +3308,7 @@ int main(int argc, char **argv) ret = lamigo_rescan(); } - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "Start receiving records\n"); + llapi_printf(LLAPI_MSG_DEBUG, "Start receiving records\n"); rc = llapi_changelog_start(&chglog_hdlr, CHANGELOG_FLAG_FOLLOW | CHANGELOG_FLAG_BLOCK | @@ -3320,13 +3318,13 @@ int main(int argc, char **argv) if (rc) { /* XXX: probably keep trying in some cases? */ llapi_error(LLAPI_MSG_ERROR, rc, - "unable to open changelog of MDT [%s]\n", - opt.o_mdtname); + "unable to open changelog of MDT [%s]", + opt.o_mdtname); ret = rc; goto out; } - llapi_error(LLAPI_MSG_INFO|LLAPI_MSG_NO_ERRNO, 0, "started\n"); + llapi_printf(LLAPI_MSG_INFO, "started\n"); lamigo_register_signal_handlers(); @@ -3392,7 +3390,7 @@ static void lamigo_new_job_for_hot(struct lu_fid *fid, enum amigo_resync_type sy rj = calloc(1, sizeof(struct resync_job)); if (rj == NULL) { - llapi_error(LLAPI_MSG_DEBUG|LLAPI_MSG_NO_ERRNO, 0, "can't allocate for a job\n"); + llapi_err_noerrno(LLAPI_MSG_ERROR, "can't allocate for a job"); return; } rj->rj_fid = *fid; @@ -3599,8 +3597,7 @@ static __u64 lamigo_read_osp_param(const int ostidx, const char *param) fd = open(buf, O_RDONLY); if (fd < 0) { - llapi_printf(LLAPI_MSG_ERROR, - "can't open %s: %d\n", buf, errno); + llapi_error(LLAPI_MSG_ERROR, errno, "can't open %s", buf); /* 0 means non-available OST */ return 0; } @@ -3608,7 +3605,7 @@ static __u64 lamigo_read_osp_param(const int ostidx, const char *param) if (rc > 0) retval = strtoul(buf, NULL, 10); if (rc < 0) - llapi_printf(LLAPI_MSG_ERROR, "can't read: %d\n", errno); + llapi_error(LLAPI_MSG_ERROR, errno, "can't read"); close(fd); /* report zero if something went wrong @@ -3696,8 +3693,7 @@ static void *lamigo_refresh_statfs_thread(void *arg) rc = cfs_get_param_paths(&paths, "osp"); if (rc != 0) { - llapi_printf(LLAPI_MSG_FATAL, - "can't find OSP root: rc=%d\n", rc); + llapi_error(LLAPI_MSG_FATAL, rc, "can't find OSP root"); exit(1); } osproot = strdup(paths.gl_pathv[0]); diff --git a/src/lamigo.h b/src/lamigo.h index bb7421e..87db237 100644 --- a/src/lamigo.h +++ b/src/lamigo.h @@ -99,6 +99,7 @@ struct options { int o_alr_large_io; int o_alr_ignore_reads; int o_alr_ignore_writes; + int o_src_dom; char *o_heat_file; }; extern struct options opt; diff --git a/src/ldiskfs_read_ldd.h b/src/ldiskfs_read_ldd.h index 29a19bc..0818b30 100644 --- a/src/ldiskfs_read_ldd.h +++ b/src/ldiskfs_read_ldd.h @@ -11,7 +11,12 @@ #ifndef _LIPE_LDISKFS_READ_LDD_H_ #define _LIPE_LDISKFS_READ_LDD_H_ +#include +#include "debug.h" #include "fake_lustre_disk.h" + +struct lustre_disk_data; + #define LDISKFS_OPENFS_FLAGS (EXT2_FLAG_64BITS | EXT2_FLAG_SKIP_MMP | \ EXT2_FLAG_IGNORE_SB_ERRORS | EXT2_FLAG_SUPER_ONLY) int ldiskfs_read_ldd(const char *dev, ext2_filsys fs, diff --git a/src/lipe_expression_test.c b/src/lipe_expression_test.c index b8b10e7..44cb7f7 100644 --- a/src/lipe_expression_test.c +++ b/src/lipe_expression_test.c @@ -7,8 +7,9 @@ * * Author: Li Xi */ -#include +#include #include +#include "debug.h" #include "lipe_object_attrs.h" #include "policy.h" diff --git a/src/lipe_ldiskfs.c b/src/lipe_ldiskfs.c index eb22e15..514a118 100644 --- a/src/lipe_ldiskfs.c +++ b/src/lipe_ldiskfs.c @@ -28,7 +28,6 @@ #include "ldiskfs_read_ldd.h" #include "fake_lustre_idl.h" #include "debug.h" -#include "lipe_config.h" #include "lustre_ea.h" #include "cmd.h" #include "lustre_ea_ldiskfs.h" @@ -126,26 +125,16 @@ static int ldiskfs_scan_groups(ext2_inode_scan scan, struct lipe_policy *policy; struct lipe_instance *instance; struct global_info *ginfo; - struct counter_list *counter_list; - struct classify_list *classify_list; - struct scan_result *result; - struct lu_fid *watch_fid; bool lustre; bool abort_failure; - bool all_inode; int rc; instance = info->ti_instance; ginfo = info->ti_global_info; policy = info->ti_policy; - counter_list = &info->ti_counter_list; - classify_list = &info->ti_classify_list; - result = &info->ti_result; - watch_fid = ginfo->gi_watch_fid; lustre = ginfo->gi_lustre; abort_failure = ginfo->gi_abort_failure; - all_inode = ginfo->gi_all_inode; rc = ext2fs_inode_scan_goto_blockgroup(scan, start_group); if (rc) { @@ -184,15 +173,14 @@ static int ldiskfs_scan_groups(ext2_inode_scan scan, continue; } - result->sr_number_inodes++; + info->ti_result.sr_number_inodes++; lipe_object_attrs_reset(attrs); ldiskfs_copy_inode_attrs(fs, attrs, ino, inode); attrs->loa_attr_bits = LIPE_OBJECT_ATTR_ATTR; - rc = lipe_policy_apply(object, policy, instance, attrs, - sysattrs, result, abort_failure, - counter_list, classify_list, watch_fid, - all_inode, lustre); + rc = lipe_policy_apply(info, object, policy, instance, attrs, + sysattrs, abort_failure, + lustre); if (rc) { LDEBUG("failed to apply policy on inode [%d]\n", ino); rc = 0; @@ -207,8 +195,6 @@ void *ldiskfs_scan_thread(void *arg) struct thread_info *info = arg; struct lipe_instance *instance = info->ti_instance; const char *dev = instance->li_device; - struct counter_list *counter_list = &info->ti_counter_list; - struct classify_list *classify_list = &info->ti_classify_list; struct scan_result *result = &info->ti_result; struct lipe_object object; ext2_filsys fs; @@ -229,13 +215,6 @@ void *ldiskfs_scan_thread(void *arg) object.lo_backfs_ops = &ldiskfs_operations; object.lo_backfs_type = instance->li_detected_fstype; - classify_list = &info->ti_classify_list; - - result->sr_number_dirs = 0; - result->sr_number_files = 0; - result->sr_number_inodes = 0; - result->sr_number_groups = 0; - gettimeofday(&result->sr_time_start, NULL); sysattrs.lps_sys_time = result->sr_time_start.tv_sec * 1000 + result->sr_time_start.tv_usec / 1000; @@ -305,12 +284,6 @@ void *ldiskfs_scan_thread(void *arg) result->sr_group_count += (end_group - start_group); LDEBUG("scaned groups [%lu, %lu)\n", start_group, end_group); } - rc = counter_list_flush(counter_list); - if (rc) - LERROR("failed to flush counter list\n"); - rc = classify_list_flush(classify_list); - if (rc) - LERROR("failed to flush classify list\n"); out_free: free(inode); @@ -335,11 +308,10 @@ int ldiskfs_scan(struct lipe_instance *instance, struct scan_result *result, struct counter_list *sum_counter_list, struct classify_list *sum_classify_list, + void **thread_data, int num_threads, const char *workspace, - struct lu_fid *watch_fid, bool abort_failure, - bool all_inode, bool *ldd_error) { const char *dev = instance->li_device; @@ -351,9 +323,7 @@ int ldiskfs_scan(struct lipe_instance *instance, struct global_info global_info; global_info.gi_workspace = workspace; - global_info.gi_watch_fid = watch_fid; global_info.gi_abort_failure = abort_failure; - global_info.gi_all_inode = all_inode; global_info.gi_lustre = (instance->li_detected_fstype == LBT_LUSTRE_ON_LDISKFS ? true : false); global_info.gi_num_threads = num_threads; @@ -392,7 +362,8 @@ int ldiskfs_scan(struct lipe_instance *instance, global_info.gi_group_batch = group_batch; - rc = scan_threads_start(&infos, num_threads, instance, &ldd, policy, + rc = scan_threads_start(&infos, thread_data, num_threads, instance, + &ldd, policy, &global_info, instance->li_detected_fstype); if (rc) { LERROR("failed to start threads\n"); diff --git a/src/lipe_ldiskfs.h b/src/lipe_ldiskfs.h index 2c6c46d..3aa9c33 100644 --- a/src/lipe_ldiskfs.h +++ b/src/lipe_ldiskfs.h @@ -11,6 +11,15 @@ #ifndef _LIPE_LDISKFS_H_ #define _LIPE_LDISKFS_H_ +#include + +struct lipe_instance; +struct lipe_policy; +struct scan_result; +struct counter_list; +struct classify_list; +struct lu_fid; + void *ldiskfs_scan_thread(void *arg); int ldiskfs_scan(struct lipe_instance *instance, @@ -18,10 +27,9 @@ int ldiskfs_scan(struct lipe_instance *instance, struct scan_result *result, struct counter_list *sum_counter_list, struct classify_list *sum_classify_list, + void **thread_data, int num_threads, const char *workspace, - struct lu_fid *watch_fid, bool abort_failure, - bool all_inode, bool *ldd_error); #endif /* _LIPE_LDISKFS_H_ */ diff --git a/src/lipe_object_attrs.c b/src/lipe_object_attrs.c new file mode 100644 index 0000000..fb5e4df --- /dev/null +++ b/src/lipe_object_attrs.c @@ -0,0 +1,533 @@ +#ifndef LUSTRE_UTILS +# define LUSTRE_UTILS /* Prvent #error when including lustre_ioctl.h */ +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "debug.h" +#include "list.h" +#include "lipe_object_attrs.h" + +char *bytes_to_base64(const unsigned char *b, size_t b_size) +{ + char *s = NULL; + size_t s_size; + size_t i; + size_t j; + + static const char E[64] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789" + "+/"; + + s_size = 4 * ((b_size + 2) / 3) + 1; + s = malloc(s_size); + if (s == NULL) + goto out; + + i = 0; + j = 0; + while (i < b_size) { + unsigned int o1 = (i < b_size) ? b[i++] : 0; + unsigned int o2 = (i < b_size) ? b[i++] : 0; + unsigned int o3 = (i < b_size) ? b[i++] : 0; + unsigned int q = (o1 << 16) | (o2 << 8) | (o3 << 0); + + s[j++] = E[(q >> 18) & 63]; + s[j++] = E[(q >> 12) & 63]; + s[j++] = E[(q >> 6) & 63]; + s[j++] = E[(q >> 0) & 63]; + } + + switch (b_size % 3) { + case 1: + s[j - 2] = '='; + case 2: + s[j - 1] = '='; + case 0: + s[j - 0] = '\0'; + } +out: + return s; +} + +static struct json_object * +bytes_to_json_string_base64(const void *b, size_t b_size) +{ + struct json_object *obj = NULL; + char *s = NULL; + + s = bytes_to_base64(b, b_size); + if (s == NULL) + goto out; + + obj = json_object_new_string(s); +out: + free(s); + + return obj; +} + +static struct json_object *fid_to_json(const struct lu_fid *fid) +{ + char fid_buf[FID_LEN + 1]; + + snprintf(fid_buf, sizeof(fid_buf), DFID, PFID(fid)); + + return json_object_new_string(fid_buf); +} + +static struct json_object *lipe_link_entry_list_to_json(const struct lipe_list_head *links) +{ + struct json_object *arr; + const struct lipe_link_entry *lle; + + arr = json_object_new_array(); + + lipe_list_for_each_entry(lle, links, lle_linkage) { + struct json_object *obj = json_object_new_object(); + + json_object_object_add(obj, "parent_fid", + fid_to_json(&lle->lle_parent_fid)); + + json_object_object_add(obj, "name", + json_object_new_string(lle->lle_name)); + + json_object_array_add(arr, obj); + } + + return arr; +} + +static struct json_object *lipe_path_entry_list_to_json(const struct lipe_list_head *paths) +{ + struct json_object *arr; + const struct lipe_path_entry *lpe; + + arr = json_object_new_array(); + + lipe_list_for_each_entry(lpe, paths, lpe_linkage) + json_object_array_add(arr, + json_object_new_string(lpe->lpe_path)); + + return arr; +} + +static struct json_object *lipe_xattr_list_to_json(const struct lipe_list_head *xattrs) +{ + struct json_object *arr; + const struct lipe_xattr *lx; + + arr = json_object_new_array(); + + lipe_list_for_each_entry(lx, xattrs, lx_linkage) { + struct json_object *obj = json_object_new_object(); + + json_object_object_add(obj, "name", + json_object_new_string(lx->lx_name)); + + json_object_object_add(obj, "value", + bytes_to_json_string_base64( + (const unsigned char *)lx->lx_value, + lx->lx_value_len)); + + json_object_array_add(arr, obj); + } + + return arr; +} + +struct json_object * +lipe_object_attrs_to_json(const struct lipe_object_attrs *loa, + __u64 encode_bits) +{ + struct json_object *obj; + __u64 attr_bits = loa->loa_attr_bits & encode_bits; + + obj = json_object_new_object(); + +#define J(key, json) \ + json_object_object_add(obj, (key), (json)) + +#define B(key, val) \ + J(key, json_object_new_boolean((bool)(val))) + +#define S(key, str) \ + J(key, json_object_new_string(str)) + +#define I(key, val) \ + J(key, json_object_new_int64(val)) + + if (attr_bits & LIPE_OBJECT_ATTR_ATTR) { + I("ino", loa->loa_ino); + I("uid", loa->loa_uid); + I("gid", loa->loa_gid); + I("nlink", loa->loa_nlinks); + I("mode", loa->loa_mode); + I("atime_ms", loa->loa_atime_ms); + I("mtime_ms", loa->loa_mtime_ms); + I("ctime_ms", loa->loa_ctime_ms); + } + + if (attr_bits & LIPE_OBJECT_ATTR_LINKEA) + J("links", lipe_link_entry_list_to_json(&loa->loa_links)); + + if (attr_bits & LIPE_OBJECT_ATTR_LMAEA) + J("fid", fid_to_json(&loa->loa_fid)); + + /* TODO _HSMEA, _LOVEA, _SOM, _DOM */ + + if (attr_bits & LIPE_OBJECT_ATTR_SIZE) { + I("size", loa->loa_size); + I("blocks", loa->loa_blocks / 512); + } + + if (attr_bits & LIPE_OBJECT_ATTR_ENTRIES) + I("entries", loa->loa_entries); + + if (attr_bits & LIPE_OBJECT_ATTR_EMPTY) + B("empty", loa->loa_is_empty); + + if (attr_bits & LIPE_OBJECT_ATTR_FILTER_FID) + /* TODO */; + + if (attr_bits & LIPE_OBJECT_ATTR_ALL_XATTR) + J("xattrs", lipe_xattr_list_to_json(&loa->loa_xattrs)); + + /* Skip _NOGROUP, _NOUSER */ + + if (attr_bits & LIPE_OBJECT_ATTR_PROJID) + I("projid", loa->loa_projid); + + if (attr_bits & LIPE_OBJECT_ATTR_STRIPE) { + I("stripe_count", loa->loa_stripe_count); + I("stripe_size", loa->loa_stripe_size); + I("comp_count", loa->loa_comp_count); + } + + if (attr_bits & LIPE_OBJECT_ATTR_PATHS) + J("paths", lipe_path_entry_list_to_json(&loa->loa_paths)); + +#undef J +#undef B +#undef S +#undef I + + return obj; +} + +static void lipe_object_attrs_links_fini(struct lipe_object_attrs *attrs) +{ + struct lipe_link_entry *lle, *n; + + lipe_list_for_each_entry_safe(lle, n, &attrs->loa_links, lle_linkage) { + lipe_list_del_init(&lle->lle_linkage); + free(lle->lle_name); + free(lle); + } + + LASSERT(lipe_list_empty(&attrs->loa_links)); +} + +static void lipe_object_attrs_paths_fini(struct lipe_object_attrs *attrs) +{ + struct lipe_path_entry *lpe, *n; + + lipe_list_for_each_entry_safe(lpe, n, &attrs->loa_paths, lpe_linkage) { + lipe_list_del_init(&lpe->lpe_linkage); + free(lpe->lpe_path); + free(lpe); + } + + LASSERT(lipe_list_empty(&attrs->loa_paths)); +} + +static void lipe_object_attrs_xattrs_fini(struct lipe_object_attrs *attrs) +{ + struct lipe_xattr *lx, *n; + + lipe_list_for_each_entry_safe(lx, n, &attrs->loa_xattrs, lx_linkage) { + lipe_list_del_init(&lx->lx_linkage); + free(lx->lx_name); + LIPE_FREE(lx->lx_value, lx->lx_value_len + 1); + LIPE_FREE_PTR(lx); + } + + LASSERT(lipe_list_empty(&attrs->loa_xattrs)); +} + +void lipe_object_attrs_reset(struct lipe_object_attrs *attrs) +{ + struct lov_user_md *lum = attrs->loa_lum; + int lum_size = attrs->loa_lum_size; + + lipe_object_attrs_links_fini(attrs); + lipe_object_attrs_paths_fini(attrs); + lipe_object_attrs_xattrs_fini(attrs); + memset(lum, 0, lum_size); + memset(attrs, 0, sizeof(*attrs)); + attrs->loa_lum = lum; + attrs->loa_lum_size = lum_size; + LIPE_INIT_LIST_HEAD(&attrs->loa_links); + LIPE_INIT_LIST_HEAD(&attrs->loa_paths); + LIPE_INIT_LIST_HEAD(&attrs->loa_xattrs); +} + +int lipe_object_attrs_init(struct lipe_object_attrs *attrs) +{ + memset(attrs, 0, sizeof(*attrs)); + attrs->loa_lum_size = lov_user_md_size(LOV_MAX_STRIPE_COUNT, + LOV_USER_MAGIC_V3); + LIPE_ALLOC(attrs->loa_lum, attrs->loa_lum_size); + if (attrs->loa_lum == NULL) + return -ENOMEM; + + LIPE_INIT_LIST_HEAD(&attrs->loa_links); + LIPE_INIT_LIST_HEAD(&attrs->loa_paths); + LIPE_INIT_LIST_HEAD(&attrs->loa_xattrs); + + return 0; +} + +void lipe_object_attrs_fini(struct lipe_object_attrs *attrs) +{ + lipe_object_attrs_links_fini(attrs); + lipe_object_attrs_paths_fini(attrs); + lipe_object_attrs_xattrs_fini(attrs); + LIPE_FREE(attrs->loa_lum, attrs->loa_lum_size); +} + +int lipe_object_attrs_add_link(struct lipe_object_attrs *attrs, + const struct lu_fid *parent_fid, + const char *name) +{ + struct lipe_link_entry *lle = NULL; + int rc; + + LIPE_ALLOC_PTR(lle); + if (lle == NULL) { + rc = -ENOMEM; + goto out; + } + + lle->lle_parent_fid = *parent_fid; + + lle->lle_name = strdup(name); + if (lle->lle_name == NULL) { + rc = -ENOMEM; + goto out; + } + + lipe_list_add_tail(&lle->lle_linkage, &attrs->loa_links); + + return 0; +out: + if (lle != NULL) + free(lle->lle_name); + + free(lle); + + return rc; +} + +int lipe_object_attrs_set_links(struct lipe_object_attrs *attrs, + const void *link_xattr, + size_t link_xattr_size) +{ + const struct link_ea_header *leh; + const struct link_ea_entry *lee; + unsigned long len; + unsigned long pos; + unsigned int reccount; + unsigned int i; + int rc; + + leh = link_xattr; + if (leh->leh_magic == LINK_EA_MAGIC) { + reccount = leh->leh_reccount; + len = leh->leh_len; + } else if (leh->leh_magic == __swab32(LINK_EA_MAGIC)) { + reccount = __swab32(leh->leh_reccount); + len = __swab64(leh->leh_len); + } else { + LERROR("LINKEA magic mismatch, expected 0x%lx, got 0x%x\n", + LINK_EA_MAGIC, leh->leh_magic); + return -EINVAL; + } + + if (len > link_xattr_size) { + LERROR("LINKEA invalid length %lu, should smaller than %zu\n", + len, link_xattr_size); + return -ERANGE; + } + + pos = sizeof(*leh); + lee = (const struct link_ea_entry *)(leh + 1); + LDEBUG("LINKEA count %u\n", reccount); + for (i = 0; i < reccount; i++) { + unsigned int reclen = (lee->lee_reclen[0] << 8) | lee->lee_reclen[1]; + struct lu_fid parent_fid; + + pos += reclen; + if (pos > len) { + LERROR("LINKEA length exceeded, expected %lu, got %lu\n", + len, pos); + return -EOVERFLOW; + } + + fid_be_to_cpu(&parent_fid, (const struct lu_fid *)&lee->lee_parent_fid); + + LDEBUG(" %d: parent_fid "DFID", name '%s'\n", + i, PFID(&parent_fid), lee->lee_name); + + rc = lipe_object_attrs_add_link(attrs, &parent_fid, lee->lee_name); + if (rc < 0) + return rc; + + lee = (const struct link_ea_entry *)((const char *)lee + reclen); + } + + if (pos != len) { + LERROR("LINKEA length mismatch, expected %lu, got %lu\n", + len, pos); + return -EOVERFLOW; + } + + attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_LINKEA; + + return 0; +} + +int lipe_object_attrs_add_path(struct lipe_object_attrs *attrs, + const char *path) +{ + struct lipe_path_entry *lpe = NULL; + int rc; + + LIPE_ALLOC_PTR(lpe); + if (lpe == NULL) { + rc = -ENOMEM; + goto out; + } + + lpe->lpe_path = strdup(path); + if (lpe->lpe_path == NULL) { + rc = -ENOMEM; + goto out; + } + + lipe_list_add_tail(&lpe->lpe_linkage, &attrs->loa_paths); + + return 0; +out: + if (lpe != NULL) + free(lpe->lpe_path); + + free(lpe); + + return rc; +} + +int lipe_object_attrs_set_paths(struct lipe_object_attrs *loa, + int client_mount_fd) +{ + struct getinfo_fid2path *gf = NULL; + unsigned int pathlen = PATH_MAX; + unsigned int linkno; + int rc; + + if (client_mount_fd < 0) { + rc = -ENOTTY; + goto out; + } + + if (!(loa->loa_attr_bits & LIPE_OBJECT_ATTR_LMAEA)) { + rc = -EINVAL; + goto out; + } + + gf = malloc(sizeof(*gf) + pathlen); + if (gf == NULL) { + rc = -ENOMEM; + goto out; + } + + linkno = 0; + while (1) { + /* FIXME Use llapi_fid2path_at() when available. */ + + memset(gf, 0, sizeof(*gf)); + gf->gf_fid = loa->loa_fid; + gf->gf_linkno = linkno; + gf->gf_pathlen = pathlen; + gf->gf_u.gf_path[0] = '\0'; + + rc = ioctl(client_mount_fd, OBD_IOC_FID2PATH, gf); + if (rc < 0) { + rc = -errno; + goto out; + } + + /* FIXME // and root. */ + rc = lipe_object_attrs_add_path(loa, gf->gf_u.gf_path); + if (rc < 0) + goto out; + + if (gf->gf_linkno == linkno) + break; + + linkno = gf->gf_linkno; + } + + loa->loa_attr_bits |= LIPE_OBJECT_ATTR_PATHS; +out: + free(gf); + + return rc; +} + +int lipe_object_attrs_add_xattr(struct lipe_object_attrs *attrs, + const char *name, + const void *value, size_t value_len) +{ + int rc; + struct lipe_xattr *xattr; + + LIPE_ALLOC_PTR(xattr); + if (xattr == NULL) { + LERROR("not enough memory\n"); + return -ENOMEM; + } + + xattr->lx_name = strdup(name); + if (xattr->lx_name == NULL) { + LERROR("not enough memory\n"); + rc = -ENOMEM; + goto out_xattr; + } + + LIPE_ALLOC(xattr->lx_value, value_len + 1); + if (xattr->lx_value == NULL) { + LERROR("not enough memory\n"); + rc = -ENOMEM; + goto out_name; + } + memcpy(xattr->lx_value, value, value_len); + xattr->lx_value[value_len] = '\0'; + xattr->lx_value_len = value_len; + lipe_list_add_tail(&xattr->lx_linkage, &attrs->loa_xattrs); + return 0; +out_name: + free(xattr->lx_name); +out_xattr: + LIPE_FREE_PTR(xattr); + return rc; +} diff --git a/src/lipe_object_attrs.h b/src/lipe_object_attrs.h index 62245b3..ea5a353 100644 --- a/src/lipe_object_attrs.h +++ b/src/lipe_object_attrs.h @@ -76,6 +76,7 @@ enum lipe_object_attr_bit { * Stripe count and stripe size */ LIPE_OBJECT_ATTR_STRIPE = 1 << 15, + LIPE_OBJECT_ATTR_PATHS = 1 << 16, }; enum lustre_dom_status { @@ -89,6 +90,17 @@ enum lustre_dom_status { LDS_DOM_OST }; +struct lipe_link_entry { + struct lipe_list_head lle_linkage; + struct lu_fid lle_parent_fid; + char *lle_name; +}; + +struct lipe_path_entry { + struct lipe_list_head lpe_linkage; + char *lpe_path; +}; + struct lipe_xattr { /* Linked to loa_xattrs */ struct lipe_list_head lx_linkage; @@ -152,6 +164,10 @@ struct lipe_object_attrs { */ struct filter_fid loa_filter_fid; int loa_filter_fid_size; + /* Link xattr entries. */ + struct lipe_list_head loa_links; + /* Lipe path entries */ + struct lipe_list_head loa_paths; /* * All of names and values of the EAs. */ @@ -168,4 +184,25 @@ struct lipe_object_attrs { __u32 loa_comp_count; }; +struct json_object * +lipe_object_attrs_to_json(const struct lipe_object_attrs *loa, + __u64 encode_bits); + +int lipe_object_attrs_init(struct lipe_object_attrs *attrs); +void lipe_object_attrs_fini(struct lipe_object_attrs *attrs); +void lipe_object_attrs_reset(struct lipe_object_attrs *attrs); +int lipe_object_attrs_add_link(struct lipe_object_attrs *attrs, + const struct lu_fid *parent_fid, + const char *name); +int lipe_object_attrs_set_links(struct lipe_object_attrs *attrs, + const void *link_xattr, + size_t link_xattr_size); +int lipe_object_attrs_add_path(struct lipe_object_attrs *attrs, + const char *path); +int lipe_object_attrs_set_paths(struct lipe_object_attrs *attrs, + int mnt_dir_fd); +int lipe_object_attrs_add_xattr(struct lipe_object_attrs *attrs, + const char *name, + const void *value, size_t value_len); + #endif /* _LIPE_OBJECT_ATTRS_H_ */ diff --git a/src/lipe.c b/src/lipe_scan.c similarity index 94% rename from src/lipe.c rename to src/lipe_scan.c index 2ac8b96..f0f23bc 100644 --- a/src/lipe.c +++ b/src/lipe_scan.c @@ -30,7 +30,6 @@ static void usage(void) printf("\nUsage: %s [ options ]\n" "Options:\n" " -a, --abort-failure\t\t\t\tAbort on failure\n" - " -A, --all-inode\t\t\t\tAll inode will be checked\n" " -c FILE,--config-file=FILE\t\t\tUse FILE as the configure file\n" " -h,--help\t\t\t\t\tThis message\n" " -l LEVEL,--log-level=LEVEL\t\t\tSet log level to LEVEL\n" @@ -53,8 +52,7 @@ static void usage(void) } while (0) bool abort_failure; -bool all_inode; -struct lu_fid watch_fid = {0, 0, 0}; + #ifndef PATH_MAX #define PATH_MAX (4096) #endif @@ -104,7 +102,6 @@ int main(int argc, char **argv) static struct option options[] = { { "abort-failure", no_argument, NULL, 'a' }, - { "all-inode", no_argument, NULL, 'A' }, { "config-file", required_argument, NULL, 'c' }, { "help", no_argument, NULL, 'h' }, { "log-level", required_argument, NULL, 'l' }, @@ -116,15 +113,12 @@ int main(int argc, char **argv) { NULL } }; - while ((c = getopt_long(argc, argv, "aAc:hl:r:Rt:w:W:", options, NULL)) + while ((c = getopt_long(argc, argv, "ac:hl:r:Rt:w:W:", options, NULL)) != EOF) { switch (c) { case 'a': abort_failure = true; break; - case 'A': - all_inode = true; - break; case 'c': config_file = optarg; break; @@ -156,7 +150,7 @@ int main(int argc, char **argv) } break; case 'w': - rc = parse_fid(&watch_fid, optarg); + rc = parse_fid(&debug_watch_fid, optarg); if (rc) BAD_USAGE("invalid watch FID '%s'\n", optarg); break; @@ -231,7 +225,7 @@ int main(int argc, char **argv) rc = lipe_scan(instance, &policy, &result, &sum_counter_list, &sum_classify_list, num_threads, workspace, - &watch_fid, abort_failure, all_inode, + abort_failure, &ldd_error); if (rc) { LERROR("failed to scan [%s]\n", instance->li_device); diff --git a/src/lipe_scan2.c b/src/lipe_scan2.c new file mode 100644 index 0000000..ff72b90 --- /dev/null +++ b/src/lipe_scan2.c @@ -0,0 +1,446 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "debug.h" +#include "lipe_object_attrs.h" +#include "lipe_scan2.h" +#include "lipe_version.h" + +static bool ls2_debug; + +#define LS2_DEBUG(fmt, args...) \ + do { \ + if (ls2_debug) \ + fprintf(stderr, "DEBUG %s:%d: "fmt, __func__, __LINE__, ##args); \ + } while (0) + +#define LS2_DEBUG_D(x) LS2_DEBUG("%s = %"PRIdMAX"\n", #x, (intmax_t)x) +#define LS2_DEBUG_P(x) LS2_DEBUG("%s = %p\n", #x, x) +#define LS2_DEBUG_S(x) LS2_DEBUG("%s = '%s'\n", #x, x) +#define LS2_DEBUG_U(x) LS2_DEBUG("%s = %"PRIuMAX"\n", #x, (uintmax_t)x) +#define LS2_DEBUG_X(x) LS2_DEBUG("%s = %"PRIxMAX"\n", #x, (uintmax_t)x) + +#define LS2_ERROR(fmt, args...) \ + fprintf(stderr, "%s: "fmt, program_invocation_short_name, ##args) + +#define LS2_FATAL(fmt, args...) \ + do { \ + LS2_ERROR("FATAL: "fmt, ##args); \ + exit(EXIT_FAILURE); \ + } while (0) + +#define TRY_HELP_1() \ + do { \ + fprintf(stderr, \ + "Try '%s --help' for more information.\n", \ + program_invocation_short_name); \ + exit(2); \ + } while (0) + + +#define TRY_HELP(fmt, args...) \ + do { \ + fprintf(stderr, "%s: "fmt, \ + program_invocation_short_name, \ + ##args); \ + TRY_HELP_1(); \ + } while (0) + +struct attr_bit_name { + const char *abn_name; + enum lipe_object_attr_bit abn_bits; +}; + +/* Some are still TODO in lipe_object_attrs_to_json(). */ +enum { + LIPE_SCAN2_ATTR_ALL = + LIPE_OBJECT_ATTR_ATTR | + LIPE_OBJECT_ATTR_LINKEA | + LIPE_OBJECT_ATTR_LMAEA | + LIPE_OBJECT_ATTR_HSMEA | + // LIPE_OBJECT_ATTR_LOVEA | TODO in lipe_object_attrs_to_json() + LIPE_OBJECT_ATTR_SIZE | + // LIPE_OBJECT_ATTR_SOM | TODO in lipe_object_attrs_to_json() + // LIPE_OBJECT_ATTR_DOM | TODO in lipe_object_attrs_to_json() + // LIPE_OBJECT_ATTR_ENTRIES | FIXME Broken for striped dirs + // LIPE_OBJECT_ATTR_EMPTY | FIXME Broken for striped dirs + // LIPE_OBJECT_ATTR_FILTER_FID | TODO in lipe_object_attrs_to_json() + LIPE_OBJECT_ATTR_ALL_XATTR | + LIPE_OBJECT_ATTR_PROJID | + LIPE_OBJECT_ATTR_STRIPE | + LIPE_OBJECT_ATTR_PATHS, +}; + +/* Names are chosen to match JSON field names. See + * lipe_object_attrs_to_json(). */ + +static struct attr_bit_name attr_bit_names[] = { +#define X(bits, name) { .abn_bits = (bits), .abn_name = (name), } + + X(LIPE_SCAN2_ATTR_ALL, "all"), + X(LIPE_OBJECT_ATTR_ATTR, "inode"), /* ino, uid, gid, nlink, mode, [amc]time_ms */ + X(LIPE_OBJECT_ATTR_LINKEA, "links"), + X(LIPE_OBJECT_ATTR_LMAEA, "fid"), + X(LIPE_OBJECT_ATTR_HSMEA, "hsm"), + // X(LIPE_OBJECT_ATTR_LOVEA, "lov"), + X(LIPE_OBJECT_ATTR_SIZE, "size"), + // X(LIPE_OBJECT_ATTR_SOM, "som"), + // X(LIPE_OBJECT_ATTR_DOM, "dom"), + // X(LIPE_OBJECT_ATTR_ENTRIES, "entries"), + // X(LIPE_OBJECT_ATTR_EMPTY, "empty"), + // X(LIPE_OBJECT_ATTR_FILTER_FID, "filter_fid"), + X(LIPE_OBJECT_ATTR_ALL_XATTR, "xattrs"), + X(LIPE_OBJECT_ATTR_PROJID, "projid"), + X(LIPE_OBJECT_ATTR_STRIPE, "stripe"), /* stripe_count, stirpe_size, comp_count */ + X(LIPE_OBJECT_ATTR_PATHS, "paths"), +#undef X +}; + +static int attr_bits_parse1(const char *name, enum lipe_object_attr_bit *val) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(attr_bit_names); i++) { + if (strcmp(attr_bit_names[i].abn_name, name) == 0) { + *val |= attr_bit_names[i].abn_bits; + return 0; + } + } + + return -EINVAL; +} + +/* str should be a comma separated list of bit names as in + * attr_bit_names[]. The bits named in str will be or-ed into + * *val. Callers must initialize *val to 0 for the simple usage. */ +static int attr_bits_parse(const char *str, enum lipe_object_attr_bit *val) +{ + char *dup = NULL; + char *pos; + char *name; + int rc = 0; + + dup = strdup(str); + if (dup == NULL) { + rc = -ENOMEM; + goto out; + } + + pos = dup; + while ((name = strsep(&pos, ",")) != NULL) { + rc = attr_bits_parse1(name, val); + if (rc < 0) + goto out; + } +out: + free(dup); + + return rc; +} + +static void ls2_list_attrs(void) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(attr_bit_names); i++) + printf("%s\n", attr_bit_names[i].abn_name); +} + +static const char *client_mount_path; +static enum lipe_object_attr_bit print_json_attr_bits; +static const char *print_delim = "\n"; +static bool print_null_delim; +static bool print_all_paths; +static bool print_absolute_paths; + +/* XXX Must use a single printf() per file due to + * multithreading. */ + +#define LS2_PRINT_DELIM(fmt, args...) \ + do { \ + if (print_null_delim) \ + printf(fmt "%c", ##args, '\0'); \ + else \ + printf(fmt "%s", ##args, print_delim); \ + } while (0) + +static int print_fid_callback(const struct lipe_object *unused, + struct lipe_object_attrs *loa, + void *rule_data, + void *thread_data) +{ + if (print_absolute_paths) + LS2_PRINT_DELIM("%s/.lustre/fid/"DFID_NOBRACE, + client_mount_path, PFID(&loa->loa_fid)); + else + LS2_PRINT_DELIM(DFID_NOBRACE, PFID(&loa->loa_fid)); + + return 0; +} + +static int print_json_callback(const struct lipe_object *unused, + struct lipe_object_attrs *loa, + void *rule_data, + void *thread_data) +{ + struct json_object *obj; + const char *str; + + /* Attributes printed will be loa->loa_attr_bits & + * print_json_attr_bits. */ + obj = lipe_object_attrs_to_json(loa, print_json_attr_bits); + str = json_object_to_json_string_ext(obj, JSON_C_TO_STRING_PLAIN); + + LS2_PRINT_DELIM("%s", str); + + json_object_put(obj); + + return 0; +} + +static int print_path_callback(const struct lipe_object *unused, + struct lipe_object_attrs *loa, + void *rule_data, + void *thread_data) +{ + const struct lipe_path_entry *lpe; + + lipe_list_for_each_entry(lpe, &loa->loa_paths, lpe_linkage) { + if (print_absolute_paths) + LS2_PRINT_DELIM("%s/%s", client_mount_path, lpe->lpe_path); + else + LS2_PRINT_DELIM("%s", lpe->lpe_path); + + if (!print_all_paths) + break; + } + + return 0; +} + +static void usage(void) +{ + printf( +"Usage: %s [OPTION]... DEVICE [EXPRESSION]\n" +"Scan DEVICE for files matching EXPRESSION (or all files if expression\n" +"is omitted). Print FIDs (default), JSON object descriptions, or paths,\n" +"of matching files to stdout.\n" +"\n" +"Mandatory arguments to long options are mandatory for short options too.\n" +" --client-mount=MOUNT use the Lustre client at MOUNT for FID to path\n" +" --print-fid print FID of each file matching expression\n" +" --print-json[=ATTRS] print a JSON object with atributes specified by ATTRS\n" +" describing each file matching expression. ATTRS must be\n" +" a comma separated list of attribute names.\n" +" use '%s --list-attrs' to see available attributes names\n" +" --print-path[=WHICH] print path(s) of each file matching expression\n" +" WHICH must be 'one' (default) or 'all'\n" +" --absolute-paths prefix paths with MOUNT/ (for --print-path)\n" +" prefix FIDs with MOUNT/.lustre/fid/ (for --print-fid)\n" +" --delimiter=DELIM use DELIM intead of newline to delimit matches\n" +" --null use a NUL byte intead of newline to delimit matches\n" +" --threads=COUNT use COUNT scanning threads\n" +" -h,--help display this help text and exit\n" +" --list-attrs list available attribute names\n" +" --version output version information and exit\n" +"\n" +"If --absolute-paths or --print-path is used then --client-mount is\n" +"required. It is also required if --print-json is used and ATTRS\n" +"includes paths.\n" +, +program_invocation_short_name, +program_invocation_short_name); +} + +enum { + LS2_OPT_CLIENT_MOUNT = 1, + LS2_OPT_PRINT_FID, + LS2_OPT_PRINT_JSON, + LS2_OPT_PRINT_PATH, + LS2_OPT_ABSOLUTE_PATHS, + LS2_OPT_DELIMITER, + LS2_OPT_NULL, + LS2_OPT_THREADS, + LS2_OPT_DEBUG, + LS2_OPT_LIST_ATTRS, + LS2_OPT_VERSION, +}; + +static struct option options[] = { + { "client-mount", required_argument, NULL, LS2_OPT_CLIENT_MOUNT }, + { "print-fid", no_argument, NULL, LS2_OPT_PRINT_FID }, + { "print-json", optional_argument, NULL, LS2_OPT_PRINT_JSON }, + { "print-path", optional_argument, NULL, LS2_OPT_PRINT_PATH }, + { "absolute-paths", no_argument, NULL, LS2_OPT_ABSOLUTE_PATHS }, + { "delimiter", required_argument, NULL, LS2_OPT_DELIMITER }, + { "null", no_argument, NULL, LS2_OPT_NULL }, + { "threads", required_argument, NULL, LS2_OPT_THREADS }, + { "help", no_argument, NULL, 'h' }, + { "debug", no_argument, NULL, LS2_OPT_DEBUG }, + { "list-attrs", no_argument, NULL, LS2_OPT_LIST_ATTRS }, + { "version", no_argument, NULL, LS2_OPT_VERSION }, + { NULL }, +}; + +int main(int argc, char *argv[]) +{ + const char *device; + const char *expression; + int client_mount_fd = -1; + lipe_scan2_callback_t scan_callback = &print_fid_callback; + enum lipe_object_attr_bit attr_bits = LIPE_OBJECT_ATTR_LMAEA; + struct lipe_scan2_options scan2_options; + size_t thread_count = 0; + char *end; + int c; + int rc; + + debug_level = ERROR; + + while ((c = getopt_long(argc, argv, "h", options, NULL)) != EOF) { + switch (c) { + case LS2_OPT_CLIENT_MOUNT: + client_mount_path = optarg; + break; + case LS2_OPT_PRINT_FID: + /* This is the default. */ + break; + case LS2_OPT_PRINT_JSON: + scan_callback = &print_json_callback; + if (optarg != NULL) { + rc = attr_bits_parse(optarg, &print_json_attr_bits); + if (rc < 0) + TRY_HELP("invalid attribute list '%s' for '--print-json'\n", + optarg); + + attr_bits |= print_json_attr_bits; + } else { + attr_bits = LIPE_OBJECT_ATTR_ATTR; + print_json_attr_bits = -1; + } + + break; + case LS2_OPT_PRINT_PATH: + scan_callback = &print_path_callback; + attr_bits = LIPE_OBJECT_ATTR_PATHS; + + if (optarg != NULL) { + if (strcmp(optarg, "one") == 0) + print_all_paths = false; + else if (strcmp(optarg, "all") == 0) + print_all_paths = true; + else + TRY_HELP("invalid argument '%s' for '--print-path'\n", + optarg); + } + break; + case LS2_OPT_ABSOLUTE_PATHS: + print_absolute_paths = true; + break; + case LS2_OPT_NULL: + print_null_delim = true; + break; + case LS2_OPT_DELIMITER: + print_delim = optarg; + break; + case LS2_OPT_THREADS: + errno = 0; + thread_count = strtoul(optarg, &end, 0); + if (errno != 0 || *end != '\0') + TRY_HELP("invalid thread count '%s'\n", optarg); + + break; + case LS2_OPT_DEBUG: + ls2_debug = true; + debug_level = DEBUG; + /* llapi ... */ + break; + case LS2_OPT_LIST_ATTRS: + ls2_list_attrs(); + exit(EXIT_SUCCESS); + case LS2_OPT_VERSION: + lipe_version(); + exit(EXIT_SUCCESS); + case 'h': + usage(); + exit(EXIT_SUCCESS); + case '?': + TRY_HELP_1(); + } + } + + if (optind + 1 == argc) { + device = argv[optind]; + expression = "1"; + } else if (optind + 2 == argc) { + device = argv[optind]; + expression = argv[optind + 1]; + } else { + TRY_HELP("usage: %s [OPTION]... DEVICE [EXPRESSION]\n", + program_invocation_short_name); + } + + if (thread_count == 0) + thread_count = sysconf(_SC_NPROCESSORS_ONLN); + + if (attr_bits & LIPE_OBJECT_ATTR_PATHS) + attr_bits |= LIPE_OBJECT_ATTR_LMAEA | LIPE_OBJECT_ATTR_LINKEA; + + if ((attr_bits & LIPE_OBJECT_ATTR_PATHS) && client_mount_path == NULL) + TRY_HELP("printing paths requires a client mount\n"); + + if (client_mount_path != NULL) { + client_mount_fd = open(client_mount_path, O_RDONLY); + if (client_mount_fd < 0) + LS2_FATAL("cannot open client mount '%s': %s\n", + client_mount_path, strerror(errno)); + + /* stat and check */ + } + + LS2_DEBUG_S(device); + LS2_DEBUG_S(expression); + LS2_DEBUG_S(client_mount_path); + LS2_DEBUG_D(client_mount_fd); + LS2_DEBUG_X(attr_bits); + LS2_DEBUG_X(print_json_attr_bits); + LS2_DEBUG_S(print_delim); + LS2_DEBUG_U(print_null_delim); + LS2_DEBUG_U(print_all_paths); + LS2_DEBUG_U(print_absolute_paths); + LS2_DEBUG_U(thread_count); + + struct lipe_scan2_rule rule = { + .lsr_expression = expression, + .lsr_callback = scan_callback, + .lsr_data = NULL, + .lsr_attr_bits = attr_bits, + }; + + memset(&scan2_options, 0, sizeof(scan2_options)); + + rc = lipe_scan2(device, + client_mount_fd, + &rule, + 1 /* rule_count */, + NULL /* thread_data */, + thread_count, + &scan2_options); + LS2_DEBUG_D(rc); + if (rc < 0) + LS2_ERROR("cannot scan '%s': %s\n", device, strerror(-rc)); + + if (!(client_mount_fd < 0)) + close(client_mount_fd); + + exit(EXIT_SUCCESS); +} diff --git a/src/lipe_scan2.h b/src/lipe_scan2.h new file mode 100644 index 0000000..a33cf76 --- /dev/null +++ b/src/lipe_scan2.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021, DDN Storage Corporation. + */ +#ifndef _LIPE_SCAN2_H_ +#define _LIPE_SCAN2_H_ + +#include +#include "lipe_object_attrs.h" + +struct lipe_object; + +typedef int (*lipe_scan2_callback_t)(const struct lipe_object *, + struct lipe_object_attrs *, + void * /* rule_data */, + void * /* thread_data */); + +struct lipe_scan2_rule { + const char *lsr_expression; + lipe_scan2_callback_t lsr_callback; + void *lsr_data; + /* See comment at definition of lac_attr_bits. */ + enum lipe_object_attr_bit lsr_attr_bits; +}; + +struct lipe_scan2_options { + /* Other options as needed. */ +}; + +/* + * Each rule has an expression (lsr_expression) for example: + * 'type == S_IFREG && atime < sys_time - 31536000000'. + * + * For each inode scanned, walk through the list of rules. If + * lsr_expression matches then call lsr_callback with the object, + * attributes, and lsr_data, and the thread data. Rules after the + * first match will not be evaluated. + * + * If the thread_data argument passed to lipe_scan2() is NULL then the + * callback will be invoked a NULL thread_data argument. Otherwise it + * should be the address of an array of void * pointers and the i-th + * thread will pass thread_data[i] to the callback. + * + * TODO: If thread_count is 0 then lipe_scan2 will choose an + * appropriate thread count. + */ +int lipe_scan2(const char *device, + int client_mount_fd, + const struct lipe_scan2_rule *rules, + size_t rule_count, + void **thread_data, + size_t thread_count, + const struct lipe_scan2_options *options); + +#endif /* _LIPE_SCAN2_H_ */ diff --git a/src/lipe_ssh.c b/src/lipe_ssh.c index 43ac483..d9173f5 100644 --- a/src/lipe_ssh.c +++ b/src/lipe_ssh.c @@ -23,7 +23,7 @@ static int lipe_ssh_session_start_cmd(ssh_session session, const char *cmd, ssh_ channel = ssh_channel_new(session); if (channel == NULL) { - lipe_ssh_error("canot create a new SSH channel: %s\n", + lipe_ssh_error("canot create a new SSH channel: %s", ssh_get_error(session)); rc = SSH_ERROR; goto out; @@ -31,14 +31,14 @@ static int lipe_ssh_session_start_cmd(ssh_session session, const char *cmd, ssh_ rc = ssh_channel_open_session(channel); if (rc != SSH_OK) { - lipe_ssh_error("cannot open SSH session channel: %s\n", + lipe_ssh_error("cannot open SSH session channel: %s", ssh_get_error(session)); goto out; } rc = ssh_channel_request_exec(channel, cmd); if (rc != SSH_OK) { - lipe_ssh_error("cannot execute SSH command: %s\n", + lipe_ssh_error("cannot execute SSH command: %s", ssh_get_error(session)); goto out; } @@ -95,11 +95,11 @@ static int lipe_ssh_session_create(ssh_session *psession, const char *host) assert(*psession == NULL); assert(host != NULL); - lipe_ssh_debug("creating new SSH session for host '%s'\n", host); + lipe_ssh_debug("creating new SSH session for host '%s'", host); session = ssh_new(); if (session == NULL) { - lipe_ssh_error("cannot create a new SSH session: %s\n", + lipe_ssh_error("cannot create a new SSH session: %s", strerror(ENOMEM)); /* Probably. */ rc = SSH_ERROR; goto out; @@ -107,14 +107,14 @@ static int lipe_ssh_session_create(ssh_session *psession, const char *host) rc = ssh_options_set(session, SSH_OPTIONS_HOST, host); if (rc != SSH_OK) { - lipe_ssh_error("cannot set SSH session host to '%s: %s'\n", + lipe_ssh_error("cannot set SSH session host to '%s: %s'", host, ssh_get_error(session)); goto out; } rc = ssh_options_set(session, SSH_OPTIONS_TIMEOUT, &timeout); if (rc != SSH_OK) { - lipe_ssh_error("cannot set SSH timeout to %ld: %s\n", + lipe_ssh_error("cannot set SSH timeout to %ld: %s", timeout, ssh_get_error(session)); goto out; } @@ -122,7 +122,7 @@ static int lipe_ssh_session_create(ssh_session *psession, const char *host) /* Connect to the ssh server */ rc = ssh_connect(session); if (rc != SSH_OK) { - lipe_ssh_error("cannot connect SSH session to host '%s': %s\n", + lipe_ssh_error("cannot connect SSH session to host '%s': %s", host, ssh_get_error(session)); goto out; } @@ -130,7 +130,7 @@ static int lipe_ssh_session_create(ssh_session *psession, const char *host) /* Automatically authenticate with public key */ rc = ssh_userauth_publickey_auto(session, NULL, NULL); if (rc != SSH_AUTH_SUCCESS) { - lipe_ssh_error("cannot authenticate SSH session to host '%s': %s\n", + lipe_ssh_error("cannot authenticate SSH session to host '%s': %s", host, ssh_get_error(session)); goto out; } @@ -139,7 +139,7 @@ static int lipe_ssh_session_create(ssh_session *psession, const char *host) session = NULL; rc = SSH_OK; out: - lipe_ssh_debug("create new SSH session for host '%s': rc = %d\n", host, rc); + lipe_ssh_debug("create new SSH session for host '%s': rc = %d", host, rc); lipe_ssh_session_destroy(&session); return rc; diff --git a/src/lipe_zfs.c b/src/lipe_zfs.c index 2076cd3..06f0748 100644 --- a/src/lipe_zfs.c +++ b/src/lipe_zfs.c @@ -13,6 +13,7 @@ #include #include +#include "lipe_object_attrs.h" #include "lipe_zfs.h" #include "zfs_read_ldd.h" #include "fake_lustre_idl.h" @@ -88,7 +89,8 @@ static int zfs_get_xattr(struct lipe_object *lobject, { int rc; - rc = lipe_attrs_add_xattr(attrs, xattr_name, (const char *)value, cnt); + rc = lipe_object_attrs_add_xattr(attrs, xattr_name, (const char *)value, + cnt); if (rc) { OBJ_ERROR(lobject, "failed to add xattr [%s]\n", xattr_name); return rc; @@ -113,13 +115,13 @@ static int zfs_get_xattr(struct lipe_object *lobject, attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_FILTER_FID; attrs->loa_filter_fid_size = cnt; } else if (strcmp(xattr_name, XATTR_NAME_LINK) == 0) { - rc = decode_linkea((char *)value, cnt); + memcpy(attrs->loa_leh_buf, value, cnt); + + rc = lipe_object_attrs_set_links(attrs, value, cnt); if (rc) { OBJ_ERROR(lobject, "failed to decode linkea\n"); return rc; } - memcpy(attrs->loa_leh_buf, value, cnt); - attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_LINKEA; } else if (strcmp(xattr_name, XATTR_NAME_LMV) == 0) { if (cnt > sizeof(attrs->loa_lmv_buf)) { OBJ_ERROR(lobject, "invalid lmv ea\n"); @@ -295,8 +297,7 @@ out: } int get_dir_entries_zfs(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; objset_t *os = object->u.lo_zfs.loz_objset; @@ -320,10 +321,9 @@ int get_dir_entries_zfs(struct lipe_object *object, return 0; } -static void print_object_prefix_zfs(struct lipe_object *object, int level, - bool watch) +static void print_object_prefix_zfs(struct lipe_object *object, int level) { - _lipe_logging(level, watch, "obj [%d], ", object->u.lo_zfs.loz_object); + _lipe_logging(level, NULL, "obj [%d], ", object->u.lo_zfs.loz_object); } static struct lipe_backfs_operations zfs_operations = { @@ -339,22 +339,17 @@ pthread_key_t kthread_key; void *zfs_scan_thread(void *arg) { - int rc; struct thread_info *info = arg; struct zfs_obj_range *zor = (struct zfs_obj_range *)info->ti_private; struct scan_result *result = &info->ti_result; - struct counter_list *counter_list = &info->ti_counter_list; - struct classify_list *classify_list = &info->ti_classify_list; struct lipe_instance *instance = info->ti_global_info->gi_instance; struct lipe_policy *policy = info->ti_global_info->gi_policy; objset_t *os = info->ti_global_info->gi_objset; - struct lu_fid *watch_fid = info->ti_global_info->gi_watch_fid; uint64_t start = zor->zfs_obj_start; uint64_t end = zor->zfs_obj_end; uint64_t max = zor->zfs_obj_max; uint64_t object_count = 0; uint64_t object; - bool all_inode = info->ti_global_info->gi_all_inode; bool abort_failure = info->ti_global_info->gi_abort_failure; bool lustre = info->ti_global_info->gi_lustre; struct lipe_object_attrs attrs; @@ -371,10 +366,6 @@ void *zfs_scan_thread(void *arg) goto out; } - result->sr_number_dirs = 0; - result->sr_number_files = 0; - result->sr_number_inodes = 0; - gettimeofday(&result->sr_time_start, NULL); sysattrs.lps_sys_time = result->sr_time_start.tv_sec * 1000 + result->sr_time_start.tv_usec / 1000; @@ -446,10 +437,9 @@ void *zfs_scan_thread(void *arg) } result->sr_number_inodes++; - rc = lipe_policy_apply(&lobject, policy, instance, &attrs, - &sysattrs, result, abort_failure, - counter_list, classify_list, watch_fid, - all_inode, lustre); + rc = lipe_policy_apply(info, &lobject, policy, instance, &attrs, + &sysattrs, abort_failure, + lustre); if (rc) { LERROR("failed to apply policy on object [%llu]\n", object); @@ -458,10 +448,6 @@ void *zfs_scan_thread(void *arg) zor->zfs_obj_count = object_count; - rc = counter_list_flush(counter_list); - if (rc) - LERROR("failed to flush counter list\n"); - classify_list_flush(classify_list); lipe_object_attrs_fini(&attrs); gettimeofday(&result->sr_time_end, NULL); diff_timevals(&result->sr_time_start, &result->sr_time_end, @@ -551,11 +537,10 @@ int zfs_scan(struct lipe_instance *instance, struct scan_result *result, struct counter_list *sum_counter_list, struct classify_list *sum_classify_list, + void **thread_data, int num_threads, const char *workspace, - struct lu_fid *watch_fid, bool abort_failure, - bool all_inode, bool *ldd_error) { int rc; @@ -666,9 +651,7 @@ int zfs_scan(struct lipe_instance *instance, } global_info.gi_workspace = workspace; - global_info.gi_watch_fid = watch_fid; global_info.gi_abort_failure = abort_failure; - global_info.gi_all_inode = all_inode; global_info.gi_lustre = lustre; global_info.gi_num_threads = num_threads; pthread_mutex_init(&global_info.gi_mutex, NULL); @@ -679,7 +662,8 @@ int zfs_scan(struct lipe_instance *instance, global_info.gi_objset = os; global_info.gi_instance = instance; - rc = scan_threads_start(&infos, num_threads, instance, &ldd, policy, + rc = scan_threads_start(&infos, thread_data, num_threads, instance, + &ldd, policy, &global_info, instance->li_detected_fstype); if (rc) { LERROR("failed to start threads\n"); diff --git a/src/lipe_zfs.h b/src/lipe_zfs.h index 82d295d..54da33a 100644 --- a/src/lipe_zfs.h +++ b/src/lipe_zfs.h @@ -28,6 +28,15 @@ #include #include "policy.h" +#include + +struct lipe_instance; +struct lipe_policy; +struct scan_result; +struct counter_list; +struct classify_list; +struct lu_fid; + extern int zfs_recover; extern uint64_t zfs_arc_max, zfs_arc_meta_limit; extern int zfs_vdev_async_read_max_active; @@ -46,10 +55,9 @@ int zfs_scan(struct lipe_instance *instance, struct scan_result *result, struct counter_list *sum_counter_list, struct classify_list *sum_classify_list, + void **thread_data, int num_threads, const char *workspace, - struct lu_fid *watch_fid, bool abort_failure, - bool all_inode, bool *ldd_error); #endif /* _LIPE_ZFS_H_ */ diff --git a/src/lpurge.c b/src/lpurge.c index ca4493f..91b670f 100644 --- a/src/lpurge.c +++ b/src/lpurge.c @@ -157,6 +157,7 @@ char *ostname; char *ostprefix; char ost_mntdev[PATH_MAX]; char *ost_mntpt; +bool is_mdt = false; int lustre_fd = -1; int open_by_fid_fd = -1; unsigned long oldest; @@ -312,6 +313,8 @@ static void lpurge_find_device(char *devname) while (i && ostprefix[i] != '/') i--; ostprefix[i] = 0; + if (strstr(ostprefix, "-MDT")) + is_mdt = true; globfree(&paths); } @@ -471,12 +474,21 @@ static int lipe_scan_llite(struct lipe_instance *instance, int num_threads) int rc = 0; memset(&result, 0, sizeof(result)); + memset(&policy, 0, sizeof(policy)); policy.lp_attr_bits = LIPE_OBJECT_ATTR_ATTR | LIPE_OBJECT_ATTR_LMAEA | LIPE_OBJECT_ATTR_FILTER_FID; - rc = lipe_scan(instance, &policy, &result, NULL, - NULL, num_threads, "test", NULL, - true, false, &ldd_err); + if (is_mdt) + policy.lp_attr_bits |= LIPE_OBJECT_ATTR_LOVEA; + rc = lipe_scan(instance, + &policy, + &result, + NULL /* sum_counter_list */, + NULL /* sum_classify_list */, + num_threads, + "test" /* workspace */, + true /* abort_failure */, + &ldd_err); return rc; } @@ -541,6 +553,101 @@ static void lpurge_reclaim_slot(unsigned int index) lpurge_hist[index].ls_age = lpurge_hist[index + 1].ls_age; pthread_mutex_unlock(&lpurge_hist[index].ls_mutex); } +static inline struct lov_user_md * +lov_comp_entry(struct lov_comp_md_v1 *comp_v1, int ent_idx) +{ + return (struct lov_user_md *)((char *)comp_v1 + + comp_v1->lcm_entries[ent_idx].lcme_offset); +} + +static int lpurge_check_mdt_object(struct lpurge_slot *ls, + struct lipe_object_attrs *attrs) +{ + struct lov_user_md_v3 *v3; + struct lov_comp_md_v1 *comp_v1; + struct lov_comp_md_entry_v1 *entry; + unsigned src_comp_id = -1; + unsigned tgt_comp_id = -1; + int i; + + if ((attrs->loa_attr_bits & LIPE_OBJECT_ATTR_LOVEA) == 0) { + ls->ls_nomirror_objs++; + ls->ls_nomirror_space += attrs->loa_blocks >> 10; + return 0; + } + + comp_v1 = (struct lov_comp_md_v1 *)attrs->loa_lum; + if (comp_v1->lcm_magic != LOV_USER_MAGIC_COMP_V1 || + comp_v1->lcm_mirror_count == 0) { + ls->ls_nomirror_objs++; + ls->ls_nomirror_space += attrs->loa_blocks >> 10; + return 0; + } + + for (i = 0; i < comp_v1->lcm_entry_count; i++) { + + entry = &comp_v1->lcm_entries[i]; + + if ((entry->lcme_flags & LCME_FL_INIT) == 0 || + (entry->lcme_flags & LCME_FL_STALE) != 0) + continue; + + v3 = (struct lov_user_md_v3 *)lov_comp_entry(comp_v1, i); + + if (v3->lmm_pattern == LOV_PATTERN_MDT) { + src_comp_id = mirror_id_of(entry->lcme_id); + } else { + tgt_comp_id = mirror_id_of(entry->lcme_id); + } + } + + if (src_comp_id != -1 && tgt_comp_id != -1) { + /* there is MDT component and in-sync replica */ + attrs->loa_filter_fid.ff_parent = attrs->loa_fid; + attrs->loa_filter_fid.ff_layout.ol_comp_id = + src_comp_id << MIRROR_ID_SHIFT; + attrs->loa_filter_fid_size = sizeof(struct filter_fid); + return 1; + } + + ls->ls_nomirror_objs++; + ls->ls_nomirror_space += attrs->loa_blocks >> 10; + return 0; +} + +static int lpurge_check_ost_object(struct lpurge_slot *ls, + struct lipe_object_attrs *attrs) +{ + if ((attrs->loa_attr_bits & LIPE_OBJECT_ATTR_FILTER_FID) == 0) { + ls->ls_nopfid_objs++; + ls->ls_nopfid_space += attrs->loa_blocks >> 10; + return 0; + } + + /* to avoid N OSTs to 1 MDT scalability issue we only consider + * objects which store 1st stripe + */ + if (attrs->loa_filter_fid.ff_parent.f_ver != 0) { + ls->ls_notfirst_objs++; + ls->ls_notfirst_space += attrs->loa_blocks >> 10; + return 0; + } + + /* if the object has got ost_layout structure which encodes + * whether the object has a mirror, then we can skip objects + * with mirror_id=0 (no mirror) + */ + if (attrs->loa_filter_fid_size >= sizeof(struct filter_fid)) { + if (mirror_id_of(attrs->loa_filter_fid.ff_layout.ol_comp_id) + == 0) { + ls->ls_nomirror_objs++; + ls->ls_nomirror_space += attrs->loa_blocks >> 10; + return 0; + } + } + + return 1; +} /* * so this is a callback for object scanner @@ -568,7 +675,7 @@ int lpurge_lipe_callback(struct lipe_instance *instance, struct lpurge_slot *ls = NULL; struct lpurge_object *lo = NULL; time_t age, last_used; - int index; + int index, rc; /* no attr, ignore */ if ((attrs->loa_attr_bits & LIPE_OBJECT_ATTR_ATTR) == 0) @@ -604,33 +711,12 @@ int lpurge_lipe_callback(struct lipe_instance *instance, pthread_mutex_lock(&ls->ls_mutex); - if ((attrs->loa_attr_bits & LIPE_OBJECT_ATTR_FILTER_FID) == 0) { - ls->ls_nopfid_objs++; - ls->ls_nopfid_space += attrs->loa_blocks >> 10; - goto out_ls_mutex; - } - - /* to avoid N OSTs to 1 MDT scalability issue we only consider - * objects which store 1st stripe - */ - if (attrs->loa_filter_fid.ff_parent.f_ver != 0) { - ls->ls_notfirst_objs++; - ls->ls_notfirst_space += attrs->loa_blocks >> 10; + if (is_mdt) + rc = lpurge_check_mdt_object(ls, attrs); + else + rc = lpurge_check_ost_object(ls, attrs); + if (!rc) goto out_ls_mutex; - } - - /* if the object has got ost_layout structure which encodes - * whether the object has a mirror, then we can skip objects - * with mirror_id=0 (no mirror) - */ - if (attrs->loa_filter_fid_size >= sizeof(struct filter_fid)) { - if (mirror_id_of(attrs->loa_filter_fid.ff_layout.ol_comp_id) - == 0) { - ls->ls_nomirror_objs++; - ls->ls_nomirror_space += attrs->loa_blocks >> 10; - goto out_ls_mutex; - } - } llapi_printf(LLAPI_MSG_DEBUG, "found under "DFID": size %ld block %ld age %ld slot %d\n", diff --git a/src/lustre_ea.c b/src/lustre_ea.c index fd674bf..e3147e0 100644 --- a/src/lustre_ea.c +++ b/src/lustre_ea.c @@ -14,113 +14,6 @@ #include "lustre_ea.h" #include "debug.h" -int decode_linkea(char *buf, ssize_t size) -{ - struct link_ea_header *leh; - struct link_ea_entry *lee; - int i; - __u64 length; - int reclen; - struct lu_fid pfid; - - leh = (struct link_ea_header *)buf; - if (leh->leh_magic == __swab32(LINK_EA_MAGIC)) { - leh->leh_magic = LINK_EA_MAGIC; - leh->leh_reccount = __swab32(leh->leh_reccount); - leh->leh_len = __swab64(leh->leh_len); - } - if (leh->leh_magic != LINK_EA_MAGIC) { - LERROR("LINKEA magic mismatch, expected 0x%lx, got 0x%x\n", - LINK_EA_MAGIC, leh->leh_magic); - return -EINVAL; - } - if (leh->leh_reccount == 0) { - LERROR("LINKEA empty record count\n"); - return -ENODATA; - } - if (leh->leh_len > size) { - LERROR("LINKEA invalid length %llu, should smaller than %zd\n", - leh->leh_len, size); - return -ERANGE; - } - - length = sizeof(struct link_ea_header); - lee = (struct link_ea_entry *)(leh + 1); - LDEBUG("LINKEA count %u\n", leh->leh_reccount); - for (i = 0; i < leh->leh_reccount; i++) { - reclen = (lee->lee_reclen[0] << 8) | lee->lee_reclen[1]; - length += reclen; - if (length > leh->leh_len) { - LERROR("LINKEA length exceeded, expected %lld, got %lld\n", - leh->leh_len, length); - return -EOVERFLOW; - } - fid_be_to_cpu(&pfid, (struct lu_fid *)&lee->lee_parent_fid); - memcpy(&lee->lee_parent_fid, &pfid, sizeof(pfid)); - - LDEBUG(" %d: pfid "DFID", name '%s'\n", i, - PFID(&pfid), lee->lee_name); - lee = (struct link_ea_entry *)((char *)lee + reclen); - } - - if (length != leh->leh_len) { - LERROR("LINKEA length mismatch, expected %lld, got %lld\n", - leh->leh_len, length); - return -EOVERFLOW; - } - - return 0; -} - -int match_linkea(struct link_ea_header *leh, regex_t *reg) -{ - struct link_ea_entry *lee; - int i; - int reclen; - int rc; - - lee = (struct link_ea_entry *)(leh + 1); - for (i = 0; i < leh->leh_reccount; i++) { - reclen = (lee->lee_reclen[0] << 8) | lee->lee_reclen[1]; - /* - * REG_NOSUB is used, so the nmatch and pmatch arguments are - * ignored. - */ - rc = regexec(reg, lee->lee_name, 0, NULL, 0); - if (rc == 0) - return 0; - - lee = (struct link_ea_entry *)((char *)lee + reclen); - } - - return -1; -} - -int fnmatch_linkea(struct link_ea_header *leh, const char *pattern, - bool case_insensitive) -{ - struct link_ea_entry *lee; - int i; - int reclen; - int rc; - int flags = 0; - - if (case_insensitive) - flags |= FNM_CASEFOLD; - - lee = (struct link_ea_entry *)(leh + 1); - for (i = 0; i < leh->leh_reccount; i++) { - reclen = (lee->lee_reclen[0] << 8) | lee->lee_reclen[1]; - rc = fnmatch(pattern, lee->lee_name, flags); - if (rc == 0) - return 0; - - lee = (struct link_ea_entry *)((char *)lee + reclen); - } - - return -1; -} - /* Copied from Lustre/utils/liblustreapi.c */ static inline struct lov_user_md * lov_comp_entry(struct lov_comp_md_v1 *comp_v1, int ent_idx) diff --git a/src/lustre_ea.h b/src/lustre_ea.h index 64f1ab2..5ab7612 100644 --- a/src/lustre_ea.h +++ b/src/lustre_ea.h @@ -14,10 +14,6 @@ #include "general_policy.h" #include "lipe_object_attrs.h" -int decode_linkea(char *buf, ssize_t size); -int match_linkea(struct link_ea_header *leh, regex_t *reg); -int fnmatch_linkea(struct link_ea_header *leh, const char *pattern, - bool case_insensitive); int match_lum_ost(struct lov_user_md *lum, int64_t ost_index); int regular_expression_match_lum_pool(struct lov_user_md *lum, regex_t *reg); int shell_pattern_match_lum_pool(struct lov_user_md *lum, char *pattern); diff --git a/src/lustre_ea_ldiskfs.c b/src/lustre_ea_ldiskfs.c index dbf0bf1..197e7cd 100644 --- a/src/lustre_ea_ldiskfs.c +++ b/src/lustre_ea_ldiskfs.c @@ -8,6 +8,7 @@ * Author: Li Xi */ #include "fake_lustre_idl.h" +#include "lipe_object_attrs.h" #include "lustre_ea.h" #include "lustre_ea_ldiskfs.h" #include "debug.h" @@ -18,32 +19,27 @@ int get_link_ea_ldiskfs(struct lipe_object *object, { int size; int rc; - char *buf; ext2_filsys fs = object->u.lo_ldiskfs.lol_fs; ext2_ino_t ino = object->u.lo_ldiskfs.lol_ino; struct ext2_inode_large *inode = object->u.lo_ldiskfs.lol_inode; - buf = attrs->loa_leh_buf; - /* - * Need to clear the buffer, otherwise lee->lee_name will keep dirty - * data of last inode - */ - memset(buf, 0, MAX_LINKEA_SIZE); rc = ext2fs_attr_get(fs, (struct ext2_inode *)inode, EXT2_ATTR_INDEX_TRUSTED, XATTR_NAME_LINK + strlen("trusted."), - buf, MAX_LINKEA_SIZE, &size); + attrs->loa_leh_buf, sizeof(attrs->loa_leh_buf), + &size); if (rc) { LDEBUG("inode [%d] has no [%s] xattr, ignoring rc = %d\n", ino, XATTR_NAME_LINK, rc); return -ENODATA; } - rc = decode_linkea(buf, size); + rc = lipe_object_attrs_set_links(attrs, attrs->loa_leh_buf, size); if (rc) { LERROR("failed to decode linkea for inode [%d]\n", ino); return rc; } + attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_LINKEA; return 0; } @@ -126,8 +122,7 @@ int get_lma_ea_ldiskfs(struct lipe_object *object, } int get_hsm_ea_ldiskfs(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { char buf[MAX_LINKEA_SIZE]; struct hsm_attrs *hsm; @@ -145,7 +140,7 @@ int get_hsm_ea_ldiskfs(struct lipe_object *object, XATTR_NAME_HSM + strlen("trusted."), buf, MAX_LINKEA_SIZE, &size); if (xattr_name_not_found(rc)) { - LDEBUGW(watch, "inode [%d] has no [%s] xattr\n", + LDEBUGW(&attrs->loa_fid, "inode [%d] has no [%s] xattr\n", ino, XATTR_NAME_HSM); hus->hus_states = 0; hus->hus_archive_id = 0; @@ -195,8 +190,7 @@ int get_fid_ea_ldiskfs(struct lipe_object *object, } int get_lum_ea_ldiskfs(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; int size; @@ -210,7 +204,7 @@ int get_lum_ea_ldiskfs(struct lipe_object *object, XATTR_NAME_LOV + strlen("trusted."), (char *)lov, attrs->loa_lum_size, &size); if (rc) { - LDEBUGW(watch, "inode [%d] has no [%s] xattr, rc = %d\n", + LDEBUGW(&attrs->loa_fid, "inode [%d] has no [%s] xattr, rc = %d\n", ino, XATTR_NAME_LOV, rc); return -ENODATA; } @@ -231,8 +225,7 @@ int get_lum_ea_ldiskfs(struct lipe_object *object, * so no need to duplicate the macros. */ int get_som_ea_ldiskfs(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; int size; @@ -246,7 +239,7 @@ int get_som_ea_ldiskfs(struct lipe_object *object, XATTR_NAME_SOM + strlen("trusted."), (char *)som, sizeof(*som), &size); if (rc) { - LDEBUGW(watch, "inode [%d] has no [%s] xattr, rc = %d\n", + LDEBUGW(&attrs->loa_fid, "inode [%d] has no [%s] xattr, rc = %d\n", ino, XATTR_NAME_SOM, rc); return -ENODATA; } @@ -291,8 +284,7 @@ static int check_empty_proc(ext2_ino_t dir EXT2FS_ATTR((unused)), } int check_dir_empty_ldiskfs(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { ext2_filsys fs = object->u.lo_ldiskfs.lol_fs; ext2_ino_t ino = object->u.lo_ldiskfs.lol_ino; @@ -338,8 +330,7 @@ static int entries_proc(ext2_ino_t dir EXT2FS_ATTR((unused)), } int get_dir_entries_ldiskfs(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { ext2_filsys fs = object->u.lo_ldiskfs.lol_fs; ext2_ino_t ino = object->u.lo_ldiskfs.lol_ino; @@ -364,12 +355,11 @@ static int ldiskfs_copy_xattr(char *name, char *value, size_t value_len, { struct lipe_object_attrs *attrs = (struct lipe_object_attrs *)data; - return lipe_attrs_add_xattr(attrs, name, value, value_len); + return lipe_object_attrs_add_xattr(attrs, name, value, value_len); } static int ldiskfs_get_all_xattrs(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; ext2_filsys fs = object->u.lo_ldiskfs.lol_fs; @@ -401,8 +391,7 @@ out_close: } static int ldiskfs_get_projid(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { struct ext2_inode_large *inode = object->u.lo_ldiskfs.lol_inode; @@ -411,10 +400,9 @@ static int ldiskfs_get_projid(struct lipe_object *object, return 0; } -static void print_object_prefix_ldiskfs(struct lipe_object *object, int level, - bool watch) +static void print_object_prefix_ldiskfs(struct lipe_object *object, int level) { - _lipe_logging(level, watch, "ino [%d], ", object->u.lo_ldiskfs.lol_ino); + _lipe_logging(level, NULL, "ino [%d], ", object->u.lo_ldiskfs.lol_ino); } struct lipe_backfs_operations ldiskfs_operations = { diff --git a/src/policy.c b/src/policy.c index ccc2927..5ff27ac 100644 --- a/src/policy.c +++ b/src/policy.c @@ -7,21 +7,24 @@ * * Author: Li Xi */ -#include -#include +#include #include #include -#ifdef HAVE_ZFS -#include "lipe_zfs.h" -#endif +#include +#include #include -#include "lustre_ea.h" -#include "policy.h" #include "debug.h" -#include "lustre_ea_ldiskfs.h" #include "fake_lustre_idl.h" -#include "lipe_ldiskfs.h" +#include "general_policy.h" #include "ldiskfs_read_ldd.h" +#include "lipe_ldiskfs.h" +#include "lipe_scan2.h" +#ifdef HAVE_ZFS +# include "lipe_zfs.h" +#endif +#include "lustre_ea.h" +#include "lustre_ea_ldiskfs.h" +#include "policy.h" #include "posix.h" const char * const lipe_action_names[] = @@ -268,10 +271,17 @@ static int counter_init(struct lipe_counter *counter, const char *dirname, return 0; } -static void counter_fini(struct lipe_counter *counter) +static int counter_fini(struct lipe_counter *counter) { - if (counter->lc_flist) - flist_free(counter->lc_flist); + int rc = 0; + + if (counter->lc_flist != NULL) + rc = flist_write(counter->lc_flist, true); + + flist_free(counter->lc_flist); + counter->lc_flist = NULL; + + return rc; } static int lipe_policy_action_type_parse(const char *action_string, @@ -660,6 +670,7 @@ int lipe_policy_init(struct lipe_policy *policy) int rc = 0; LENTRY; + memset(policy, 0, sizeof(*policy)); LIPE_INIT_LIST_HEAD(&policy->lp_groups); LIPE_INIT_LIST_HEAD(&policy->lp_classifys); @@ -764,7 +775,7 @@ out: } static int get_link_ea(struct lipe_object *object, - struct lipe_object_attrs *attrs) + struct lipe_object_attrs *attrs) { int rc; @@ -777,6 +788,7 @@ static int get_link_ea(struct lipe_object *object, OBJ_DEBUG(object, "failed to get LINKEA\n"); return rc; } + return 0; } @@ -847,8 +859,7 @@ int get_fid_ea(struct lipe_object *object, struct lipe_object_attrs *attrs) } static int get_hsm_ea(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; @@ -856,7 +867,7 @@ static int get_hsm_ea(struct lipe_object *object, object->lo_backfs_ops->get_hsm_ea == NULL) return -ENOTSUP; - rc = object->lo_backfs_ops->get_hsm_ea(object, attrs, watch); + rc = object->lo_backfs_ops->get_hsm_ea(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get HSM status\n"); return rc; @@ -865,8 +876,7 @@ static int get_hsm_ea(struct lipe_object *object, } static int get_lum_ea(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; @@ -874,7 +884,7 @@ static int get_lum_ea(struct lipe_object *object, object->lo_backfs_ops->get_lum_ea == NULL) return -ENOTSUP; - rc = object->lo_backfs_ops->get_lum_ea(object, attrs, watch); + rc = object->lo_backfs_ops->get_lum_ea(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get LUM\n"); return rc; @@ -887,8 +897,7 @@ static int get_lum_ea(struct lipe_object *object, * SOM support */ int get_som_ea(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; @@ -896,7 +905,7 @@ int get_som_ea(struct lipe_object *object, object->lo_backfs_ops->get_som_ea == NULL) return -ENOTSUP; - rc = object->lo_backfs_ops->get_som_ea(object, attrs, watch); + rc = object->lo_backfs_ops->get_som_ea(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get SoM\n"); return rc; @@ -905,13 +914,12 @@ int get_som_ea(struct lipe_object *object, } static int get_dom_status(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; if (!(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_LOVEA)) { - rc = get_lum_ea(object, attrs, watch); + rc = get_lum_ea(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get LOVEA\n"); return rc; @@ -931,8 +939,7 @@ static int get_dom_status(struct lipe_object *object, } static int get_size(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; #ifdef HAVE_LAZY_SIZE_ON_MDT @@ -946,7 +953,7 @@ static int get_size(struct lipe_object *object, return 0; } - rc = get_dom_status(object, attrs, watch); + rc = get_dom_status(object, attrs); if (rc) return rc; @@ -955,7 +962,7 @@ static int get_size(struct lipe_object *object, #ifdef HAVE_LAZY_SIZE_ON_MDT if (!(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_SOM)) { - rc = get_som_ea(object, attrs, watch); + rc = get_som_ea(object, attrs); if (rc) return rc; } @@ -974,41 +981,8 @@ static int get_size(struct lipe_object *object, #endif /* !HAVE_LAZY_SIZE_ON_MDT */ } -static bool lipe_need_read_attr(struct lipe_policy *policy, - struct lipe_rule *rule, - struct lipe_object_attrs *attrs, - enum lipe_object_attr_bit bit) -{ - if (attrs->loa_attr_bits & bit) - return false; - if (policy && policy->lp_attr_bits & bit) - return true; - if (rule == NULL) - return false; - return rule->lr_attr_bits & bit; -} - -static bool lipe_need_read_lma(struct lipe_policy *policy, - struct lipe_rule *rule, - struct lipe_object_attrs *attrs, - bool need_fid) -{ - enum lipe_object_attr_bit bit = LIPE_OBJECT_ATTR_LMAEA; - - if (attrs->loa_attr_bits & bit) - return false; - if (policy->lp_attr_bits & bit) - return true; - if (need_fid) - return true; - if (rule == NULL) - return false; - return rule->lr_attr_bits & bit; -} - static int check_empty(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; @@ -1021,8 +995,7 @@ static int check_empty(struct lipe_object *object, object->lo_backfs_ops->check_dir_empty == NULL) return -ENOTSUP; - rc = object->lo_backfs_ops->check_dir_empty(object, attrs, - watch); + rc = object->lo_backfs_ops->check_dir_empty(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to check whether dir is empty\n"); @@ -1030,7 +1003,7 @@ static int check_empty(struct lipe_object *object, } } else if ((attrs->loa_mode & S_IFMT) == S_IFREG) { if (!(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_SIZE)) { - rc = get_size(object, attrs, watch); + rc = get_size(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get size\n"); return rc; @@ -1051,8 +1024,7 @@ static int check_empty(struct lipe_object *object, } static int get_entries(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; @@ -1068,7 +1040,7 @@ static int get_entries(struct lipe_object *object, object->lo_backfs_ops->get_dir_entries == NULL) return -ENOTSUP; - rc = object->lo_backfs_ops->get_dir_entries(object, attrs, watch); + rc = object->lo_backfs_ops->get_dir_entries(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get entry numbe\n"); return rc; @@ -1077,8 +1049,7 @@ static int get_entries(struct lipe_object *object, } static int get_all_xattrs(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; @@ -1086,7 +1057,7 @@ static int get_all_xattrs(struct lipe_object *object, object->lo_backfs_ops->get_all_xattrs == NULL) return -ENOTSUP; - rc = object->lo_backfs_ops->get_all_xattrs(object, attrs, watch); + rc = object->lo_backfs_ops->get_all_xattrs(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get all xattrs\n"); return rc; @@ -1095,8 +1066,7 @@ static int get_all_xattrs(struct lipe_object *object, } static int get_projid(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; @@ -1104,7 +1074,7 @@ static int get_projid(struct lipe_object *object, object->lo_backfs_ops->get_projid == NULL) return -ENOTSUP; - rc = object->lo_backfs_ops->get_projid(object, attrs, watch); + rc = object->lo_backfs_ops->get_projid(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get projid\n"); return rc; @@ -1112,46 +1082,8 @@ static int get_projid(struct lipe_object *object, return 0; } -int lipe_attrs_add_xattr(struct lipe_object_attrs *attrs, const char *name, - const char *value, size_t value_len) -{ - int rc; - struct lipe_xattr *xattr; - - LIPE_ALLOC_PTR(xattr); - if (xattr == NULL) { - LERROR("not enough memory\n"); - return -ENOMEM; - } - - xattr->lx_name = strdup(name); - if (xattr->lx_name == NULL) { - LERROR("not enough memory\n"); - rc = -ENOMEM; - goto out_xattr; - } - - LIPE_ALLOC(xattr->lx_value, value_len + 1); - if (xattr->lx_value == NULL) { - LERROR("not enough memory\n"); - rc = -ENOMEM; - goto out_name; - } - memcpy(xattr->lx_value, value, value_len); - xattr->lx_value[value_len] = '\0'; - xattr->lx_value_len = value_len; - lipe_list_add_tail(&xattr->lx_linkage, &attrs->loa_xattrs); - return 0; -out_name: - free(xattr->lx_name); -out_xattr: - LIPE_FREE_PTR(xattr); - return rc; -} - static int check_nogroup(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { LASSERT(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_ATTR); @@ -1161,8 +1093,7 @@ static int check_nogroup(struct lipe_object *object, } static int check_nouser(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { LASSERT(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_ATTR); @@ -1171,20 +1102,22 @@ static int check_nouser(struct lipe_object *object, return 0; } -/* rule could be NULL */ -int lipe_rule_read_attrs(struct lipe_policy *policy, - struct lipe_object *object, - struct lipe_rule *rule, - struct lipe_object_attrs *attrs, - struct lipe_policy_sysattrs *sysattrs, - struct lu_fid *watch_fid, - bool need_fid, - bool quit_on_error) +static bool lipe_need_read_attr(const struct lipe_object_attrs *attrs, + enum lipe_object_attr_bit need_bits, + enum lipe_object_attr_bit bit) +{ + return ~attrs->loa_attr_bits & need_bits & bit; +} + +static int lipe_read_attrs(struct lipe_policy *policy, + struct lipe_object *object, + struct lipe_object_attrs *attrs, + enum lipe_object_attr_bit need_bits, + bool quit_on_error) { int rc, rc2 = 0; - bool watch = false; - if (lipe_need_read_attr(policy, rule, attrs, LIPE_OBJECT_ATTR_LINKEA)) { + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_LINKEA)) { rc = get_link_ea(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get LINKEA\n"); @@ -1195,8 +1128,7 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, } } - if (lipe_need_read_lma(policy, rule, attrs, - fid_is_sane(watch_fid) || need_fid)) { + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_LMAEA)) { rc = get_lma_ea(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get LMAEA\n"); @@ -1205,12 +1137,9 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, if (!rc2) rc2 = rc; } - if (watch_fid && lu_fid_eq(&attrs->loa_fid, watch_fid)) - watch = true; } - if (lipe_need_read_attr(policy, rule, attrs, - LIPE_OBJECT_ATTR_FILTER_FID)) { + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_FILTER_FID)) { rc = get_fid_ea(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get FIDEA\n"); @@ -1221,8 +1150,8 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, } } - if (lipe_need_read_attr(policy, rule, attrs, LIPE_OBJECT_ATTR_HSMEA)) { - rc = get_hsm_ea(object, attrs, watch); + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_HSMEA)) { + rc = get_hsm_ea(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get HSMEA\n"); if (quit_on_error) @@ -1232,8 +1161,9 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, } } - if (lipe_need_read_attr(policy, rule, attrs, LIPE_OBJECT_ATTR_LOVEA)) { - rc = get_lum_ea(object, attrs, watch); + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_LOVEA) || + lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_STRIPE)) { + rc = get_lum_ea(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get LOVEA\n"); if (quit_on_error) @@ -1243,9 +1173,8 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, } } - if (lipe_need_read_attr(policy, rule, attrs, - LIPE_OBJECT_ATTR_STRIPE)) { - LASSERT(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_LOVEA); + if ((attrs->loa_attr_bits & LIPE_OBJECT_ATTR_LOVEA) && + lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_STRIPE)) { rc = lum_get_stripe(attrs->loa_lum, &attrs->loa_stripe_count, &attrs->loa_stripe_size, &attrs->loa_comp_count); @@ -1259,8 +1188,8 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_STRIPE; } - if (lipe_need_read_attr(policy, rule, attrs, LIPE_OBJECT_ATTR_DOM)) { - rc = get_dom_status(object, attrs, watch); + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_DOM)) { + rc = get_dom_status(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get DoM\n"); if (quit_on_error) @@ -1270,8 +1199,8 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, } } - if (lipe_need_read_attr(policy, rule, attrs, LIPE_OBJECT_ATTR_SIZE)) { - rc = get_size(object, attrs, watch); + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_SIZE)) { + rc = get_size(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get size\n"); if (quit_on_error) @@ -1287,9 +1216,8 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, * want to iterate the dir twice on Ext4 (even though checking * whether a directory is empty or not on ZFS is much less expensive). */ - if (rule && (rule->lr_attr_bits & LIPE_OBJECT_ATTR_ENTRIES) && - (!(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_ENTRIES))) { - rc = get_entries(object, attrs, watch); + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_ENTRIES)) { + rc = get_entries(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get entry number\n"); if (quit_on_error) @@ -1299,9 +1227,8 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, } } - if (rule && (rule->lr_attr_bits & LIPE_OBJECT_ATTR_EMPTY) && - (!(attrs->loa_attr_bits & LIPE_OBJECT_ATTR_EMPTY))) { - rc = check_empty(object, attrs, watch); + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_EMPTY)) { + rc = check_empty(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to check whether is empty or not\n"); @@ -1316,9 +1243,8 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, * We could avoid some I/Os by use this as a cache for other Lustre * internal EAs, but not sure how much performance difference. */ - if (lipe_need_read_attr(policy, rule, attrs, - LIPE_OBJECT_ATTR_ALL_XATTR)) { - rc = get_all_xattrs(object, attrs, watch); + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_ALL_XATTR)) { + rc = get_all_xattrs(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get all xattr\n"); if (quit_on_error) @@ -1328,9 +1254,8 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, } } - if (lipe_need_read_attr(policy, rule, attrs, - LIPE_OBJECT_ATTR_NOGROUP)) { - rc = check_nogroup(object, attrs, watch); + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_NOGROUP)) { + rc = check_nogroup(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to check whether it is nogroup or not\n"); @@ -1341,9 +1266,8 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, } } - if (lipe_need_read_attr(policy, rule, attrs, - LIPE_OBJECT_ATTR_NOUSER)) { - rc = check_nouser(object, attrs, watch); + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_NOUSER)) { + rc = check_nouser(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to check whether it is nouser or not\n"); @@ -1354,9 +1278,8 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, } } - if (lipe_need_read_attr(policy, rule, attrs, - LIPE_OBJECT_ATTR_PROJID)) { - rc = get_projid(object, attrs, watch); + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_PROJID)) { + rc = get_projid(object, attrs); if (rc) { OBJ_DEBUG(object, "failed to get projid\n"); @@ -1366,20 +1289,52 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, rc2 = rc; } } + + if (lipe_need_read_attr(attrs, need_bits, LIPE_OBJECT_ATTR_PATHS)) { + rc = lipe_object_attrs_set_paths(attrs, policy->lp_client_mount_fd); + if (rc < 0) { + OBJ_DEBUG(object, + "cannot get paths for "DFID": %s\n", + PFID(&attrs->loa_fid), strerror(-rc)); + if (quit_on_error) + return rc; + if (!rc2) + rc2 = rc; + } + } + return rc2; } -static int lipe_rule_group_apply(struct lipe_object *object, +/* rule could be NULL */ +int lipe_rule_read_attrs(struct lipe_policy *policy, + struct lipe_object *object, + struct lipe_rule *rule, + struct lipe_object_attrs *attrs, + bool need_fid, + bool quit_on_error) +{ + enum lipe_object_attr_bit need_bits = 0; + + if (policy != NULL) + need_bits |= policy->lp_attr_bits; + + if (rule != NULL) + need_bits |= rule->lr_attr_bits; + + if (need_fid) + need_bits |= LIPE_OBJECT_ATTR_LMAEA; + + return lipe_read_attrs(policy, object, attrs, need_bits, quit_on_error); +} + +static int lipe_rule_group_apply(struct thread_info *info, + struct lipe_object *object, struct lipe_policy *policy, struct lipe_rule_group *group, struct lipe_object_attrs *attrs, struct lipe_policy_sysattrs *sysattrs, - struct scan_result *scan_result, - bool abort_failure, - struct counter_list *counter_list, - struct classify_list *classify_list, - struct lu_fid *watch_fid, - bool all_object) + bool abort_failure) { struct lipe_rule *rule; int64_t result; @@ -1391,6 +1346,9 @@ static int lipe_rule_group_apply(struct lipe_object *object, struct lu_fid *pfid = &fid; struct lipe_classify *classify; int64_t value; + struct scan_result *scan_result = &info->ti_result; + struct counter_list *counter_list = &info->ti_counter_list; + struct classify_list *classify_list = &info->ti_classify_list; LENTRY; @@ -1402,9 +1360,12 @@ static int lipe_rule_group_apply(struct lipe_object *object, * Quit on error since rules will only match the files with * valid attrs. */ - rc = lipe_rule_read_attrs(policy, object, rule, attrs, - sysattrs, watch_fid, !all_object, - true); + rc = lipe_rule_read_attrs(policy, + object, + rule, + attrs, + true /* need_fid */, + true /* quit_on_error */); if (rc) continue; @@ -1459,29 +1420,44 @@ static int lipe_rule_group_apply(struct lipe_object *object, index = action->la_counter.lac_type->lct_index; counter = &counter_list->cl_counters[index]; break; + case LAT_CALLBACK: + /* Some attributes don't make sense for all file + * types. So just return whatever we could get. */ + (void)lipe_read_attrs(policy, + object, + attrs, + action->u.la_callback.lac_attr_bits, + false /* quit_on_error */); + rc = (*action->u.la_callback.lac_callback)( + object, + attrs, + action->u.la_callback.lac_rule_data, + info->ti_data); + if (rc < 0) + LRETURN(rc); + + break; } - LASSERT(counter); - rc = counter_add(counter, object, pfid, attrs, action, - policy->lp_summarize_size); - if (rc) { - LERROR("failed to increase counter\n"); - LRETURN(rc); + if (counter != NULL) { + rc = counter_add(counter, object, pfid, attrs, action, + policy->lp_summarize_size); + if (rc) { + LERROR("failed to increase counter\n"); + LRETURN(rc); + } } + LRETURN(0); } -int lipe_policy_apply(struct lipe_object *object, +int lipe_policy_apply(struct thread_info *info, + struct lipe_object *object, struct lipe_policy *policy, struct lipe_instance *instance, struct lipe_object_attrs *attrs, struct lipe_policy_sysattrs *sysattrs, - struct scan_result *scan_result, bool abort_failure, - struct counter_list *counter_list, - struct classify_list *classify_list, - struct lu_fid *watch_fid, - bool all_object, bool lustre) { struct lipe_rule_group *group; @@ -1490,23 +1466,23 @@ int lipe_policy_apply(struct lipe_object *object, bool quit_on_error = true; LENTRY; - /* Even LMAEA is missing, don't quit */ - if (lustre && all_object) - quit_on_error = false; + /* If have callback, don't quit, since other attrs are needed */ if (instance->li_callback) quit_on_error = false; /* Ignore the error for the case like optional LMAEA */ - lipe_rule_read_attrs(policy, object, NULL, attrs, sysattrs, - watch_fid, lustre, quit_on_error); + lipe_rule_read_attrs(policy, + object, + NULL /* rule */, + attrs, + lustre, + quit_on_error); /* * For Lustre, skip the object with no LMAEA or invalid LMEA. - * That means old objects from old Lustre version won't be found unless - * all_object is enabled. */ - if (lustre && (!all_object) && + if (lustre && ((attrs->loa_attr_bits & LIPE_OBJECT_ATTR_LMAEA) == 0 || (!fid_is_norm(&attrs->loa_fid) && !fid_is_idif(&attrs->loa_fid)))) goto out; @@ -1518,12 +1494,9 @@ int lipe_policy_apply(struct lipe_object *object, for (i = 0; i < instance->li_group_number; i++) { group = instance->li_groups[i]; LDEBUG("evaluating group [%s]\n", group->lrg_name); - rc = lipe_rule_group_apply(object, policy, group, - attrs, sysattrs, scan_result, - abort_failure, counter_list, - classify_list, watch_fid, - /* If not lustre, search objects without FID */ - all_object || (!lustre)); + rc = lipe_rule_group_apply(info, object, policy, group, + attrs, sysattrs, + abort_failure); if (rc) { LDEBUG("failed to apply group\n"); rc2 = rc2 == 0 ? rc : rc2; @@ -1545,24 +1518,23 @@ out: LRETURN(rc2); } -struct lipe_instance *lipe_instance_alloc(int group_number, const char *name) +struct lipe_instance *lipe_instance_alloc(int group_count, const char *name) { struct lipe_instance *instance; - LIPE_ALLOC_PTR(instance); + instance = calloc(1, sizeof(*instance)); if (instance == NULL) return NULL; strncpy(instance->li_device, name, sizeof(instance->li_device) - 1); - LIPE_ALLOC(instance->li_groups, - sizeof(*instance->li_groups) * group_number); + instance->li_groups = calloc(group_count, sizeof(instance->li_groups[0])); if (instance->li_groups == NULL) goto out; instance->li_expected_fstype = LBT_UNKNOWN; instance->li_detected_fstype = LBT_UNKNOWN; - instance->li_group_number = group_number; + instance->li_group_number = group_count; instance->li_callback = NULL; return instance; out: @@ -1572,6 +1544,9 @@ out: void lipe_instance_free(struct lipe_instance *instance) { + if (instance == NULL) + return; + LIPE_FREE(instance->li_groups, sizeof(*instance->li_groups) * instance->li_group_number); LIPE_FREE_PTR(instance); @@ -1678,53 +1653,48 @@ void counter_list_sum(struct counter_list *sum, counter_sum(&sum->cl_counters[i], &list->cl_counters[i]); } -int counter_list_flush(struct counter_list *list) +int counter_list_fini(struct counter_list *list) { + int rc = 0; int i; - int rc; - struct lipe_counter *counter; - if (list->cl_counter_number == 0) - return 0; for (i = 0; i < list->cl_counter_number; i++) { - counter = &list->cl_counters[i]; - if (counter->lc_flist) { - rc = flist_write(counter->lc_flist, true); - if (rc) { - LERROR("failed to write flist\n"); - return rc; - } - } - } - return 0; -} - -void counter_list_fini(struct counter_list *list) -{ - int i; - struct lipe_counter *counter; + int rc2; - if (list->cl_counter_number == 0) - return; - for (i = 0; i < list->cl_counter_number; i++) { - counter = &list->cl_counters[i]; - counter_fini(counter); + rc2 = counter_fini(&list->cl_counters[i]); + if (rc2 < 0) + rc = rc2; } + free(list->cl_counters); + + list->cl_counters = NULL; + list->cl_counter_number = 0; + + return rc; } -static void lipe_classify_fini(struct lipe_classify *classify) +static int lipe_classify_fini(struct lipe_classify *classify) { struct lipe_counter *counter, *n; + int rc = 0; lipe_list_for_each_entry_safe(counter, n, &classify->lc_counters, lc_linkage) { + int rc2; + lipe_list_del_init(&counter->lc_linkage); - counter_fini(counter); + rc2 = counter_fini(counter); + if (rc2 < 0) + rc = rc2; + LIPE_FREE_PTR(counter); } + LASSERT(lipe_list_empty(&classify->lc_counters)); classify->lc_counter_number = 0; + + return rc; } int classify_list_init(struct lipe_policy *policy, @@ -1795,58 +1765,30 @@ out_fini: classify = &list->cl_classifys[i]; lipe_classify_fini(classify); } - free(list->cl_classifys); - return rc; -} -void classify_list_fini(struct classify_list *list) -{ - int i; - struct lipe_classify *classify; - - if (list->cl_classify_number == 0) - return; - for (i = 0; i < list->cl_classify_number; i++) { - classify = &list->cl_classifys[i]; - lipe_classify_fini(classify); - } free(list->cl_classifys); -} - -static int lipe_classify_flush(struct lipe_classify *classify) -{ - int rc; - struct lipe_counter *counter; + list->cl_classifys = NULL; + list->cl_classify_number = 0; - lipe_list_for_each_entry(counter, &classify->lc_counters, lc_linkage) { - if (counter->lc_flist) { - rc = flist_write(counter->lc_flist, true); - if (rc) { - LERROR("failed to write flist for classify\n"); - return rc; - } - } - } - return 0; + return rc; } -int classify_list_flush(struct classify_list *list) +int classify_list_fini(struct classify_list *list) { + int rc = 0; int i; - int rc; - struct lipe_classify *classify; - if (list->cl_classify_number == 0) - return 0; for (i = 0; i < list->cl_classify_number; i++) { - classify = &list->cl_classifys[i]; - rc = lipe_classify_flush(classify); - if (rc) { - LERROR("failed to flush classify\n"); - return rc; - } + int rc2 = lipe_classify_fini(&list->cl_classifys[i]); + if (rc2 < 0) + rc = rc2; } - return 0; + + free(list->cl_classifys); + list->cl_classifys = NULL; + list->cl_classify_number = 0; + + return rc; } static int @@ -1945,7 +1887,8 @@ int scan_threads_abort(struct global_info *ginfo, struct thread_info *infos, return ret; } -int scan_threads_start(struct thread_info **pinfo, int num_threads, +int scan_threads_start(struct thread_info **pinfo, + void **thread_data, int num_threads, struct lipe_instance *instance, struct lustre_disk_data *ldd, struct lipe_policy *policy, @@ -2013,6 +1956,9 @@ int scan_threads_start(struct thread_info **pinfo, int num_threads, tmp_pinfo->ti_stopping = false; tmp_pinfo->ti_started = false; + if (thread_data != NULL) + tmp_pinfo->ti_data = thread_data[i]; + #ifdef HAVE_ZFS if (backfs_type == LBT_LUSTRE_ON_ZFS || backfs_type == LBT_ZFS) { @@ -2030,24 +1976,22 @@ int scan_threads_start(struct thread_info **pinfo, int num_threads, zor[i] = NULL; } #endif - if (instance->li_group_number) { - rc = counter_list_init(policy, - &tmp_pinfo->ti_counter_list, - workspace, FLIST_TYPE_INVALID, - i); - if (rc) { - LERROR("failed to init counter list\n"); - break; - } + rc = counter_list_init(policy, + &tmp_pinfo->ti_counter_list, + workspace, FLIST_TYPE_INVALID, + i); + if (rc) { + LERROR("failed to init counter list\n"); + break; + } - rc = classify_list_init(policy, - &tmp_pinfo->ti_classify_list, - workspace, i); - if (rc) { - LERROR("failed to init classify list\n"); - counter_list_fini(&tmp_pinfo->ti_counter_list); - break; - } + rc = classify_list_init(policy, + &tmp_pinfo->ti_classify_list, + workspace, i); + if (rc) { + LERROR("failed to init classify list\n"); + counter_list_fini(&tmp_pinfo->ti_counter_list); + break; } switch (backfs_type) { @@ -2143,7 +2087,8 @@ int scan_threads_join(struct thread_info *infos, int num_threads, t_result->sr_number_files, t_result->sr_number_dirs); - result->sr_number_groups += t_result->sr_number_groups; + if (result != NULL) + result->sr_number_groups += t_result->sr_number_groups; break; #ifdef HAVE_ZFS case LBT_ZFS: @@ -2163,22 +2108,38 @@ int scan_threads_join(struct thread_info *infos, int num_threads, break; } - result->sr_number_inodes += t_result->sr_number_inodes; - result->sr_number_files += t_result->sr_number_files; - result->sr_number_dirs += t_result->sr_number_dirs; + if (result != NULL) { + result->sr_number_inodes += t_result->sr_number_inodes; + result->sr_number_files += t_result->sr_number_files; + result->sr_number_dirs += t_result->sr_number_dirs; + } - if (pinfo->ti_instance->li_group_number) { - counter_list_sum(sum_counter_list, - &pinfo->ti_counter_list); - counter_list_fini(&pinfo->ti_counter_list); + if (sum_counter_list != NULL) + counter_list_sum(sum_counter_list, + &pinfo->ti_counter_list); + + rc = counter_list_fini(&pinfo->ti_counter_list); + if (rc < 0) { + LERROR("cannot flush counter list: rc = %d\n", rc); + if (ret == 0) + ret = rc; + } + + if (sum_classify_list != NULL) { rc = classify_list_sum(sum_classify_list, &pinfo->ti_classify_list); if (rc) { - LERROR("failed to sum classify list\n"); + LERROR("cannot sum classify list: rc = %d\n", rc); if (ret == 0) ret = rc; } - classify_list_fini(&pinfo->ti_classify_list); + } + + rc = classify_list_fini(&pinfo->ti_classify_list); + if (rc < 0) { + LERROR("cannot flush classify list: rc = %d\n", rc); + if (ret == 0) + ret = rc; } } @@ -2201,7 +2162,7 @@ const char *lipe_backfs_type_name[] = LIPE_BACKFS_TYPES; * Return 0 if not * Return negative value if error */ -static int lipe_detect_directory(char *dev) +static int lipe_detect_directory(const char *dev) { int rc; struct stat s; @@ -2221,7 +2182,7 @@ static int lipe_detect_directory(char *dev) return 0; } -static enum lipe_backfs_type lipe_detect_fs_type(char *dev) +enum lipe_backfs_type lipe_detect_fs_type(const char *dev) { ext2_filsys fs; int rc; @@ -2310,9 +2271,7 @@ int lipe_scan(struct lipe_instance *instance, struct classify_list *sum_classify_list, int num_threads, const char *workspace, - struct lu_fid *watch_fid, bool abort_failure, - bool all_inode, bool *ldd_error) { int rc; @@ -2341,22 +2300,22 @@ int lipe_scan(struct lipe_instance *instance, switch (fstype) { case LBT_POSIX: rc = posix_scan(instance, policy, result, sum_counter_list, - sum_classify_list, num_threads, workspace, - watch_fid, abort_failure); + sum_classify_list, NULL, num_threads, workspace, + abort_failure); break; case LBT_LUSTRE_ON_LDISKFS: case LBT_EXT4: rc = ldiskfs_scan(instance, policy, result, sum_counter_list, - sum_classify_list, num_threads, workspace, - watch_fid, abort_failure, all_inode, + sum_classify_list, NULL, num_threads, workspace, + abort_failure, ldd_error); break; #ifdef HAVE_ZFS case LBT_LUSTRE_ON_ZFS: case LBT_ZFS: rc = zfs_scan(instance, policy, result, sum_counter_list, - sum_classify_list, num_threads, workspace, - watch_fid, abort_failure, all_inode, + sum_classify_list, NULL, num_threads, workspace, + abort_failure, ldd_error); break; #endif @@ -2368,3 +2327,165 @@ int lipe_scan(struct lipe_instance *instance, return rc; } + +/* Based on lipe_rule_add() from policy.c. */ +static int lipe_scan2_rule_add(struct lipe_rule_group *group, + const char *expression, + lipe_scan2_callback_t callback, + void *rule_data, + enum lipe_object_attr_bit attr_bits) +{ + struct lipe_rule *lr; + int rc = 0; + + lr = calloc(1, sizeof(*lr)); + if (lr == NULL) { + rc = -ENOMEM; + goto out_err; + } + + lr->lr_action.la_action = LAT_CALLBACK; + lr->lr_action.u.la_callback.lac_callback = callback; + lr->lr_action.u.la_callback.lac_rule_data = rule_data; + lr->lr_action.u.la_callback.lac_attr_bits = attr_bits; + + LIPE_INIT_LIST_HEAD(&lr->lr_values); + LIPE_INIT_LIST_HEAD(&lr->lr_linkage); + strncpy(lr->lr_expression_string, expression, + sizeof(lr->lr_expression_string) - 1); + + lr->lr_group_index = group->lrg_rule_number; + group->lrg_rule_number++; + + rc = lipe_policy_value_init(&lr->lr_values, + &lr->lr_expression, + &lr->lr_attr_bits, + expression); + if (rc != 0) { + /* ... */ + goto out_err; + } + + lipe_list_add_tail(&lr->lr_linkage, &group->lrg_rules); + LRETURN(0); + +out_err: + free(lr); + + LRETURN(rc); +} + +int lipe_scan2(const char *device, + int client_mount_fd, + const struct lipe_scan2_rule *rules, + size_t rule_count, + void **thread_data, + size_t thread_count, + const struct lipe_scan2_options *options) +{ + struct lipe_instance *instance = NULL; + struct lipe_policy policy; + enum lipe_backfs_type fstype; + struct lipe_rule_group *group = NULL; + bool ldd_error = false; + size_t i; + int rc; + + instance = lipe_instance_alloc(1 /* group_count */, device); + if (instance == NULL) { + rc = -ENOMEM; + goto out; + } + + rc = lipe_policy_init(&policy); + if (rc < 0) { + /* ... */ + goto out; + } + + policy.lp_client_mount_fd = client_mount_fd; + + group = lipe_rule_group_add(&policy, "LIPE_SCAN2" /* group_name */); + if (group == NULL) { + rc = -EINVAL; + goto out; + } + + instance->li_groups[0] = group; + + for (i = 0; i < rule_count; i++) { + const struct lipe_scan2_rule *lsr = &rules[i]; + + rc = lipe_scan2_rule_add(group, + lsr->lsr_expression, + lsr->lsr_callback, + lsr->lsr_data, + lsr->lsr_attr_bits); + if (rc < 0) { + /* ... */ + goto out; + } + } + + fstype = lipe_detect_fs_type(device); + if (fstype == LBT_UNKNOWN) { + LERROR("unknown format on device '%s'\n", device); + rc = -EINVAL; + goto out; + } + + LDEBUG("scanning '%s' with format '%s'\n", device, + lipe_backfs_type_name[fstype]); + + instance->li_detected_fstype = fstype; + + switch (fstype) { + case LBT_POSIX: + rc = posix_scan(instance, + &policy, + NULL /* result */, + NULL /* sum_counter_list */, + NULL /* sum_classify_list */, + thread_data, thread_count, + NULL /* workspace */, + false /* abort_failure */); + break; + case LBT_LUSTRE_ON_LDISKFS: + case LBT_EXT4: + rc = ldiskfs_scan(instance, + &policy, + NULL /* result */, + NULL /* sum_counter_list */, + NULL /* sum_classify_list */, + thread_data, thread_count, + NULL /* workspace */, + false /* abort_failure */, + &ldd_error); + break; +#ifdef HAVE_ZFS + case LBT_LUSTRE_ON_ZFS: + case LBT_ZFS: + rc = zfs_scan(instance, + &policy, + NULL /* result */, + NULL /* sum_counter_list */, + NULL /* sum_classify_list */, + thread_data, thread_count, + NULL /* workspace */, + false /* abort_failure */, + &ldd_error); + break; +#endif + default: + LERROR("unknown format on device '%s'\n", device); + rc = -EINVAL; + break; + } + +out: + lipe_instance_free(instance); + + lipe_policy_fini(&policy); + + return rc; +} diff --git a/src/policy.h b/src/policy.h index 33a0b98..37064f4 100644 --- a/src/policy.h +++ b/src/policy.h @@ -22,6 +22,7 @@ #include #endif +#include "lipe_scan2.h" #include "general_policy.h" #include "flist.h" #include "debug.h" @@ -54,6 +55,10 @@ enum lipe_action_type { * implementing customized action handlers in batch mode. */ LAT_SHELL_CMD_FID_LIST, + /* + * + */ + LAT_CALLBACK, }; #define LIPE_ACTION_NAMES { \ @@ -62,6 +67,7 @@ enum lipe_action_type { [LAT_SHELL_CMD_FPATH] = "LAT_SHELL_CMD_FPATH", \ [LAT_SHELL_CMD_FID] = "LAT_SHELL_CMD_FID", \ [LAT_SHELL_CMD_FID_LIST] = "LAT_SHELL_CMD_FID_LIST", \ + [LAT_CALLBACK] = "LAT_CALLBACK", \ } #define LIPE_ACTION_HUMANIZED_NAMES { \ @@ -70,6 +76,7 @@ enum lipe_action_type { [LAT_SHELL_CMD_FPATH] = "exec", \ [LAT_SHELL_CMD_FID] = "exec-fid", \ [LAT_SHELL_CMD_FID_LIST] = "exec-fid-list", \ + [LAT_CALLBACK] = "callback", \ } struct lipe_counter { @@ -181,12 +188,27 @@ struct lipe_action_command { char lac_command[MAX_ACTION_ARGUMENT_LENGTH]; }; +struct lipe_action_callback { + lipe_scan2_callback_t lac_callback; + void *lac_rule_data; + + /* We have separate attribute bits for the rule (lr_attr_bits) + * and the callback (lac_attr_bits) since getting some + * attributes may be expensive (especially + * LIPE_OBJECT_ATTR_STRIPE). lr_attr_bits will contain just + * the bits needed to evaluate the expression. If the + * expression is true then we get any additional bits in + * lac_attr_bits before invoking the callback. */ + enum lipe_object_attr_bit lac_attr_bits; +}; + struct lipe_action { enum lipe_action_type la_action; struct lipe_action_counter la_counter; union { struct lipe_action_command la_command; struct lipe_classify_type la_classify; + struct lipe_action_callback la_callback; } u; }; @@ -256,6 +278,8 @@ struct lipe_policy { __u64 lp_attr_bits; /** Whether summarize matched file sizes during scanning */ bool lp_summarize_size; + /** Open FD on a Lustre client mount point. */ + int lp_client_mount_fd; }; struct lipe_object_ldiskfs { @@ -332,8 +356,7 @@ struct scan_result { struct lipe_backfs_operations { int (*get_fid_ea)(struct lipe_object *object, struct lipe_object_attrs *attrs); - void (*print_object_prefix)(struct lipe_object *object, int level, - bool watch); + void (*print_object_prefix)(struct lipe_object *object, int level); int (*get_lma_ea)(struct lipe_object *object, struct lipe_object_attrs *attrs); int (*get_link_ea)(struct lipe_object *object, @@ -341,39 +364,32 @@ struct lipe_backfs_operations { int (*get_lmv_ea)(struct lipe_object *object, struct lipe_object_attrs *attrs); int (*get_hsm_ea)(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch); + struct lipe_object_attrs *attrs); int (*get_lum_ea)(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch); + struct lipe_object_attrs *attrs); int (*get_som_ea)(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch); + struct lipe_object_attrs *attrs); /* * Check whether the directory is empty, i.e. has no sub-entry * except . and .. */ int (*check_dir_empty)(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch); + struct lipe_object_attrs *attrs); /* * Get the entry number of a directory, not including . and .. */ int (*get_dir_entries)(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch); + struct lipe_object_attrs *attrs); /* * Get all EA names and values */ int (*get_all_xattrs)(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch); + struct lipe_object_attrs *attrs); /* * Get projid */ int (*get_projid)(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch); + struct lipe_object_attrs *attrs); }; struct posix_scan_work { @@ -383,10 +399,8 @@ struct posix_scan_work { struct global_info { /* common fields for both ldiskfs and ZFS */ - struct lu_fid *gi_watch_fid; const char *gi_workspace; bool gi_abort_failure; - bool gi_all_inode; bool gi_lustre; /* Number of threads */ unsigned long gi_num_threads; @@ -437,12 +451,16 @@ struct thread_info { bool ti_stopping; struct counter_list ti_counter_list; struct classify_list ti_classify_list; - void *ti_private; + void *ti_private; /* Used internall by scan. */ + void *ti_data; /* Data for lipe_scan2() callback. */ #ifdef HAVE_ZFS_SPECIAL_KTHREAD_T kthread_t ti_kthread; #endif }; +extern const char *lipe_backfs_type_name[]; +enum lipe_backfs_type lipe_detect_fs_type(const char *dev); + int lipe_policy_init(struct lipe_policy *policy); struct lipe_rule_group *lipe_rule_group_find(struct lipe_policy *policy, const char *name); @@ -459,22 +477,16 @@ int lipe_rule_read_attrs(struct lipe_policy *policy, struct lipe_object *object, struct lipe_rule *rule, struct lipe_object_attrs *attrs, - struct lipe_policy_sysattrs *sysattrs, - struct lu_fid *watch_fid, bool need_fid, bool quit_on_error); void lipe_policy_fini(struct lipe_policy *policy); -int lipe_policy_apply(struct lipe_object *object, +int lipe_policy_apply(struct thread_info *info, + struct lipe_object *object, struct lipe_policy *policy, struct lipe_instance *instance, struct lipe_object_attrs *attrs, struct lipe_policy_sysattrs *sysattrs, - struct scan_result *scan_result, bool abort_failure, - struct counter_list *counter_list, - struct classify_list *classify_list, - struct lu_fid *watch_fid, - bool all_inode, bool lustre); struct lipe_instance *lipe_instance_alloc(int group_number, const char *name); void lipe_instance_free(struct lipe_instance *instance); @@ -492,19 +504,18 @@ int counter_list_init(struct lipe_policy *policy, int thread_id); void counter_list_sum(struct counter_list *sum, struct counter_list *list); -int counter_list_flush(struct counter_list *list); -void counter_list_fini(struct counter_list *list); +int counter_list_fini(struct counter_list *list); int classify_list_init(struct lipe_policy *policy, struct classify_list *list, const char *workspace, int thread_id); -void classify_list_fini(struct classify_list *list); -int classify_list_flush(struct classify_list *list); +int classify_list_fini(struct classify_list *list); int classify_list_sum(struct classify_list *sum, struct classify_list *list); int scan_threads_abort(struct global_info *ginfo, struct thread_info *infos, int num_threads); -int scan_threads_start(struct thread_info **pinfo, int num_threads, +int scan_threads_start(struct thread_info **pinfo, + void **thread_data, int num_threads, struct lipe_instance *instance, struct lustre_disk_data *ldd, struct lipe_policy *policy, @@ -535,30 +546,26 @@ int lipe_scan(struct lipe_instance *instance, struct classify_list *sum_classify_list, int num_threads, const char *workspace, - struct lu_fid *watch_fid, bool abort_failure, - bool all_inode, bool *ldd_error); -int lipe_attrs_add_xattr(struct lipe_object_attrs *attrs, const char *name, - const char *value, size_t value_len); -#define OBJ_LOG(object, level, watch, format, args...) \ +#define OBJ_LOG(object, level, format, args...) \ do { \ if (object->lo_backfs_ops == NULL || \ object->lo_backfs_ops->print_object_prefix == NULL) { \ - lipe_logging(level, watch, format, ##args); \ + lipe_logging(level, NULL, format, ##args); \ } else { \ - lipe_logging(level, watch, ""); \ + lipe_logging(level, NULL, ""); \ object->lo_backfs_ops->print_object_prefix(object, \ - level, watch); \ - _lipe_logging(level, watch, format, ##args); \ + level); \ + _lipe_logging(level, NULL, format, ##args); \ } \ } while (0) #define OBJ_DEBUG(object, format, args...) \ - OBJ_LOG(object, DEBUG, false, format, ##args) + OBJ_LOG(object, DEBUG, format, ##args) #define OBJ_ERROR(object, format, args...) \ - OBJ_LOG(object, ERROR, false, format, ##args) + OBJ_LOG(object, ERROR, format, ##args) #endif /* _LIPE_POLICY_H_ */ diff --git a/src/posix.c b/src/posix.c index 957da91..87304dc 100644 --- a/src/posix.c +++ b/src/posix.c @@ -153,14 +153,9 @@ static int posix_scan_match_policy(struct global_info *ginfo, int rc; struct stat sb; bool lustre = ginfo->gi_lustre; - bool all_inode = ginfo->gi_all_inode; bool abort_failure = ginfo->gi_abort_failure; - struct lu_fid *watch_fid = ginfo->gi_watch_fid; - struct scan_result *result = &info->ti_result; struct lipe_policy *policy = info->ti_policy; struct lipe_instance *instance = info->ti_instance; - struct counter_list *counter_list = &info->ti_counter_list; - struct classify_list *classify_list = &info->ti_classify_list; rc = lstat(path, &sb); if (rc) { @@ -172,9 +167,8 @@ static int posix_scan_match_policy(struct global_info *ginfo, posix_scan_copy_attrs(attrs, &sb); object->u.lo_posix.lop_path = path; - rc = lipe_policy_apply(object, policy, instance, attrs, - sysattrs, result, abort_failure, counter_list, - classify_list, watch_fid, all_inode, + rc = lipe_policy_apply(info, object, policy, instance, attrs, + sysattrs, abort_failure, lustre); if (rc) LDEBUG("failed to apply policy on [%s]\n", path); @@ -269,8 +263,6 @@ static int posix_scan_traverse(struct global_info *ginfo, void *posix_scan_thread(void *arg) { struct thread_info *info = arg; - struct counter_list *counter_list = &info->ti_counter_list; - struct classify_list *classify_list = &info->ti_classify_list; struct scan_result *result = &info->ti_result; struct lipe_object object; struct lipe_object_attrs attrs; @@ -285,12 +277,6 @@ void *posix_scan_thread(void *arg) LERROR("failed to init atttrs: not enough memory\n"); goto out; } - classify_list = &info->ti_classify_list; - - result->sr_number_dirs = 0; - result->sr_number_files = 0; - result->sr_number_inodes = 0; - result->sr_number_groups = 0; gettimeofday(&result->sr_time_start, NULL); sysattrs.lps_sys_time = result->sr_time_start.tv_sec * 1000 + @@ -321,14 +307,6 @@ void *posix_scan_thread(void *arg) LIPE_FREE_PTR(work); } - rc = counter_list_flush(counter_list); - if (rc) - LERROR("failed to flush counter list\n"); - - rc = classify_list_flush(classify_list); - if (rc) - LERROR("failed to flush classify list\n"); - lipe_object_attrs_fini(&attrs); out: gettimeofday(&result->sr_time_end, NULL); @@ -343,9 +321,9 @@ int posix_scan(struct lipe_instance *instance, struct scan_result *result, struct counter_list *sum_counter_list, struct classify_list *sum_classify_list, + void **thread_data, int num_threads, const char *workspace, - struct lu_fid *watch_fid, bool abort_failure) { int rc; @@ -354,9 +332,7 @@ int posix_scan(struct lipe_instance *instance, struct posix_scan_work *work; global_info.gi_workspace = workspace; - global_info.gi_watch_fid = watch_fid; global_info.gi_abort_failure = abort_failure; - global_info.gi_all_inode = true; global_info.gi_lustre = true; global_info.gi_num_threads = num_threads; pthread_mutex_init(&global_info.gi_mutex, NULL); @@ -374,7 +350,8 @@ int posix_scan(struct lipe_instance *instance, } posix_scan_add_global_work(&global_info, work); - rc = scan_threads_start(&infos, num_threads, instance, NULL, policy, + rc = scan_threads_start(&infos, thread_data, num_threads, + instance, NULL, policy, &global_info, LBT_POSIX); if (rc) { LERROR("failed to start threads\n"); diff --git a/src/posix.h b/src/posix.h index 75bf3d1..981b3e8 100644 --- a/src/posix.h +++ b/src/posix.h @@ -11,6 +11,15 @@ #ifndef _LIPE_POSIX_H_ #define _LIPE_POSIX_H_ +#include + +struct lipe_instance; +struct lipe_policy; +struct scan_result; +struct counter_list; +struct classify_list; +struct lu_fid; + void *posix_scan_thread(void *arg); int posix_scan(struct lipe_instance *instance, @@ -18,8 +27,8 @@ int posix_scan(struct lipe_instance *instance, struct scan_result *result, struct counter_list *sum_counter_list, struct classify_list *sum_classify_list, + void **thread_data, int num_threads, const char *workspace, - struct lu_fid *watch_fid, bool abort_failure); #endif /* _LIPE_POSIX_H_ */ diff --git a/src/posix_ea.c b/src/posix_ea.c index a6e61e4..db81ece 100644 --- a/src/posix_ea.c +++ b/src/posix_ea.c @@ -16,6 +16,7 @@ #include #include #include "fake_lustre_idl.h" +#include "lipe_object_attrs.h" #include "lustre_ea.h" #include "posix_ea.h" #include "debug.h" @@ -27,17 +28,10 @@ static int get_link_ea_posix(struct lipe_object *object, { int rc; ssize_t size; - char *buf = attrs->loa_leh_buf; const char *path = object->u.lo_posix.lop_path; - /* - * Need to clear the buffer, otherwise lee->lee_name will keep dirty - * data of last inode - */ - memset(buf, 0, MAX_LINKEA_SIZE); - - size = lgetxattr(path, XATTR_NAME_LINK, buf, - MAX_LINKEA_SIZE); + size = lgetxattr(path, XATTR_NAME_LINK, attrs->loa_leh_buf, + sizeof(attrs->loa_leh_buf)); if (size < 0) { if (errno == ENOATTR) LDEBUG("path [%s] has no [%s] xattr, ignoring\n", @@ -48,12 +42,12 @@ static int get_link_ea_posix(struct lipe_object *object, return -errno; } - rc = decode_linkea(buf, size); + rc = lipe_object_attrs_set_links(attrs, attrs->loa_leh_buf, size); if (rc) { LERROR("failed to decode linkea for path [%s]\n", path); return -1; } - attrs->loa_attr_bits |= LIPE_OBJECT_ATTR_LINKEA; + return 0; } @@ -111,8 +105,7 @@ static int get_lma_ea_posix(struct lipe_object *object, } static int get_hsm_ea_posix(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { char buf[MAX_LINKEA_SIZE]; struct hsm_attrs *hsm; @@ -126,7 +119,7 @@ static int get_hsm_ea_posix(struct lipe_object *object, MAX_LINKEA_SIZE); if (size < 0) { if (errno == ENOATTR) { - LDEBUGW(watch, + LDEBUGW(&attrs->loa_fid, "path [%d] has no [%s] xattr, ignoring\n", path, XATTR_NAME_HSM); hus->hus_states = 0; @@ -150,8 +143,7 @@ static int get_hsm_ea_posix(struct lipe_object *object, } static int get_lum_ea_posix(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; int size; @@ -185,8 +177,7 @@ static int get_lum_ea_posix(struct lipe_object *object, * so no need to duplicate the macros. */ static int get_som_ea_posix(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int size; struct lustre_som_attrs *som = &attrs->loa_som; @@ -217,8 +208,7 @@ static int get_som_ea_posix(struct lipe_object *object, #endif /* HAVE_LAZY_SIZE_ON_MDT */ static int check_dir_empty_posix(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { DIR *d; struct dirent64 *dent; @@ -245,8 +235,7 @@ static int check_dir_empty_posix(struct lipe_object *object, } static int get_dir_entries_posix(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { DIR *d; struct dirent64 *dent; @@ -274,10 +263,9 @@ static int get_dir_entries_posix(struct lipe_object *object, return 0; } -void print_object_prefix_posix(struct lipe_object *object, int level, - bool watch) +void print_object_prefix_posix(struct lipe_object *object, int level) { - _lipe_logging(level, watch, "path [%s], ", object->u.lo_posix.lop_path); + _lipe_logging(level, false, "path [%s], ", object->u.lo_posix.lop_path); } static int posix_get_xattr(struct lipe_object *object, @@ -321,8 +309,7 @@ out: } static int posix_get_all_xattrs(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int rc; char *name; @@ -356,8 +343,7 @@ out: } static int posix_get_projid(struct lipe_object *object, - struct lipe_object_attrs *attrs, - bool watch) + struct lipe_object_attrs *attrs) { int fd; int rc; -- 1.8.3.1