From 8d271a51500a567ff10b6efc2f96220ccc7ca945 Mon Sep 17 00:00:00 2001 From: Sebastien Buisson Date: Tue, 7 Nov 2023 17:00:37 +0100 Subject: [PATCH] EX-8543 tools: remove laudit/laudit-report laudit/laudit-report is a demonstration tool for what is possible in terms of Lustre audit. It is not meant to be used in production because it stores the audit data as plaintext flat files, which is both not secure and not scalable. And it is largely untested at scale. So remove laudit/laudit-report from lipe sources, and fix build and packaging mechanisms accordingly. Test-Parameters: trivial Signed-off-by: Sebastien Buisson Change-Id: I36fbd50cd4485f2cc7b0ee91922e58f92e008255 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53015 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- lipe/.gitignore | 2 - lipe/Makefile.am | 7 - lipe/configure.ac | 44 - lipe/laudit.conf.example | 12 - lipe/lipe.spec.in | 25 - lipe/man/laudit-report.1 | 71 -- lipe/man/laudit.1 | 68 -- lipe/man/laudit.conf.5 | 69 -- lipe/src/Makefile.am | 10 - lipe/src/laudit-report.c | 1039 ----------------------- lipe/src/laudit.c | 1579 ----------------------------------- lustre/tests/Makefile.am | 1 - lustre/tests/sanity-laudit.sh | 194 ----- lustre/tests/test-groups/regression | 1 - 14 files changed, 3122 deletions(-) delete mode 100644 lipe/laudit.conf.example delete mode 100644 lipe/man/laudit-report.1 delete mode 100644 lipe/man/laudit.1 delete mode 100644 lipe/man/laudit.conf.5 delete mode 100644 lipe/src/laudit-report.c delete mode 100644 lipe/src/laudit.c delete mode 100644 lustre/tests/sanity-laudit.sh diff --git a/lipe/.gitignore b/lipe/.gitignore index 07d48e8..da4922f 100644 --- a/lipe/.gitignore +++ b/lipe/.gitignore @@ -12,8 +12,6 @@ /src/ext4_inode2path /src/generate_definition /src/lamigo -/src/laudit -/src/laudit-report /src/ldumpstripe /src/lfill /src/lipe_expression_test diff --git a/lipe/Makefile.am b/lipe/Makefile.am index 6398579..b98a119 100644 --- a/lipe/Makefile.am +++ b/lipe/Makefile.am @@ -6,12 +6,6 @@ rpmbuild_opt = AUTOMAKE_OPTIONS = -Wall foreign ACLOCAL_AMFLAGS = ${ALOCAL_FLAGS} -if BUILD_LAUDIT -rpmbuild_opt += --with laudit -else -rpmbuild_opt += --without laudit -endif - if BUILD_SERVER rpmbuild_opt += --with server else @@ -58,7 +52,6 @@ EXTRA_DIST= \ lipe_purge \ lipe-func.sh \ lpcc.conf \ - laudit.conf.example \ pybuild/*.py \ pylipe/.pylintrc \ pylipe/*.py \ diff --git a/lipe/configure.ac b/lipe/configure.ac index c90947f..3851f00 100644 --- a/lipe/configure.ac +++ b/lipe/configure.ac @@ -130,50 +130,6 @@ LIBS=$saved_libs AC_MSG_RESULT([$have_llapi_changelog_in_buf]) AM_CONDITIONAL([BUILD_HOTPOOL_UTILS], [test x$have_llapi_changelog_in_buf = xyes]) -# -------- check for build laudit -------- -BUILD_LAUDIT="yes" -NIDSTR="" -AC_CHECK_HEADER([lnet/nidstr.h], NIDSTR="lnet", [], [ -AC_INCLUDES_DEFAULT -#include -]) -AS_IF([test "x$NIDSTR" = "xlnet" ], - [AC_DEFINE([HAVE_LNET_NIDSTR], [1], [lnet/nidstr.h exists]) lustre_user_dir="lustre"], - [AC_CHECK_HEADER([linux/lnet/nidstr.h], NIDSTR="linux", [], [ -AC_INCLUDES_DEFAULT -#include -])]) -AS_IF([test "x$NIDSTR" = "xlinux" ], - [AC_DEFINE([HAVE_LINUX_NIDSTR], [1], [linux/lnet/nidstr.h exists])] lustre_user_dir="linux/lustre") -if test "x$NIDSTR" = "x"; then - AC_MSG_WARN([]) - AC_MSG_WARN([nidstr include is needed for laudit. laudit will not be built.]) - AC_MSG_WARN([]) - BUILD_LAUDIT="no" -fi - -if test "x$BUILD_LAUDIT" = "xyes"; then -AC_MSG_CHECKING([whether Lustre is audit-capable]) -AC_COMPILE_IFELSE([AC_LANG_SOURCE([ - #include <$lustre_user_dir/lustre_user.h> - - int main(void) { - enum changelog_rec_type type1 = CL_GETXATTR; - enum changelog_rec_type type2 = CL_DN_OPEN; - } -])],[audit_capable="yes"],[audit_capable="no"]) -AC_MSG_RESULT([$audit_capable]) -if test "x$audit_capable" = "xno"; then - AC_MSG_WARN([]) - AC_MSG_WARN([Lustre must be audit-capable for laudit. laudit will not be built.]) - AC_MSG_WARN([]) - BUILD_LAUDIT="no" -fi -fi - -AC_SUBST(BUILD_LAUDIT) -AM_CONDITIONAL(BUILD_LAUDIT, test "$BUILD_LAUDIT" = "yes") - # -------- check for distro version -------- AC_MSG_CHECKING([for distro version]) DISTRO=$(sh detect-distro.sh) diff --git a/lipe/laudit.conf.example b/lipe/laudit.conf.example deleted file mode 100644 index aca7eb8..0000000 --- a/lipe/laudit.conf.example +++ /dev/null @@ -1,12 +0,0 @@ -{ - "fs_name": "testfs", - "mount": "/mnt/lustre/testfs", - "dump_path": "/mnt/audit", - "mdts": [ - {"mdt_idx": 0, "reader_id": "cl1"}, - {"mdt_idx": 1, "reader_id": "cl1"} - ], - "parse_interval_sec": 10, - "sync_every_n_entries": 1000, - "max_syncs_before_sleep": 10 -} diff --git a/lipe/lipe.spec.in b/lipe/lipe.spec.in index ccaf3f6..9f2074a 100644 --- a/lipe/lipe.spec.in +++ b/lipe/lipe.spec.in @@ -1,7 +1,6 @@ # LIPE specfile # Declare rpmbuild --with/--without parameters -%bcond_with laudit %bcond_with server %bcond_with zfs %bcond_with hotpool @@ -232,12 +231,6 @@ cp \ cp -a src/lipe_find3/lipe $RPM_BUILD_ROOT%{guile_site_dir} cp -a src/lipe_scan3/lipe $RPM_BUILD_ROOT%{guile_site_dir} -%if %{with laudit} -cp src/laudit \ - src/laudit-report \ - $RPM_BUILD_ROOT%{_bindir} -%endif # laudit - %if %{with hotpool} cp src/lamigo \ src/lpurge \ @@ -257,11 +250,6 @@ cp -a \ lipe.conf \ $RPM_BUILD_ROOT%{_sysconfdir} -%if %{with laudit} -mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/laudit -cp -a laudit.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/laudit -%endif # laudit - %if %{with hotpool} cp -a example_configs/hotpool/* $RPM_BUILD_ROOT%{_sysconfdir}/ %endif # hotpool @@ -293,11 +281,6 @@ install -m 0644 man/lipe_find.1 $RPM_BUILD_ROOT%{_mandir}/man1/ install -m 0644 man/lipe_scan.1 $RPM_BUILD_ROOT%{_mandir}/man1/ install -m 0644 man/lipe_find3.1 $RPM_BUILD_ROOT%{_mandir}/man1/ install -m 0644 man/lfill.1 $RPM_BUILD_ROOT%{_mandir}/man1/ -%if %{with laudit} -install -m 0644 man/laudit.1 $RPM_BUILD_ROOT%{_mandir}/man1/ -install -m 0644 man/laudit-report.1 $RPM_BUILD_ROOT%{_mandir}/man1/ -install -m 0644 man/laudit.conf.5 $RPM_BUILD_ROOT%{_mandir}/man5/ -%endif # laudit %endif # server @@ -348,14 +331,6 @@ rm -rf $RPM_BUILD_ROOT %files client %defattr(-,root,root) %{_bindir}/lipe_run_action -%if %{with laudit} -%{_bindir}/laudit -%{_bindir}/laudit-report -%{_sysconfdir}/laudit/laudit.conf.example -%{_mandir}/man1/laudit.1* -%{_mandir}/man1/laudit-report.1* -%{_mandir}/man5/laudit.conf.5* -%endif # laudit %files scan %defattr(-,root,root) diff --git a/lipe/man/laudit-report.1 b/lipe/man/laudit-report.1 deleted file mode 100644 index 435d806..0000000 --- a/lipe/man/laudit-report.1 +++ /dev/null @@ -1,71 +0,0 @@ -.\" Manpage for laudit-report. -.TH laudit 1 "7 March 2018" "1.0" "laudit-report man page" - -.SH NAME -laudit-report \- Lustre audit report - -.SH SYNOPSIS -.B laudit-report -[ options ] - -.SH DESCRIPTION -.B laudit-report -is a command-line utility for Lustre audit report. More specifically, -.B laudit-report -is used to trigger typical queries on audit logs generated by laudit, like: -.RS -- What is the history of a given file: who created, then accessed the file? -.PP -- What data is manipulated by a given user: which files did the user access? -.RE -.PP -.B laudit-report -gives the ability to filter results by date, to speed up queries and limit the amount of information returned. -.PP -config file describes how to retrieve audit records to query. An example can be found at -.I /etc/laudit/laudit.conf.example -Please see the -.BR laudit.conf (1) -man page for details. - -.SH OPTIONS -.TP -.B -f, --file FILE -Build report for given FILE -.PP -.RS -You can provide file path or FID, and also simple file or directory name. -.RE -.TP -.B -u, --user UID:GID -Build report for given UID:GID. -.TP -.B -b, --before TIME -Only consider events occurring before TIME, included. -.PP -.RS -Time expected in the form 'yyyy.mm.dd HH:MM'. -.RE -.TP -.B -a, --after TIME -Only consider events occurring after TIME, included. -.PP -.RS -Time expected in the form 'yyyy.mm.dd HH:MM'. -.RE -.TP -.B -v, --verbose -Output some debug info; repeat for more details. -.TP -.B -h,--help -Print the usage - -.SH "SEE ALSO" -.BR laudit (1) -.BR laudit.conf (5) - -.SH BUGS -No known bugs. - -.SH AUTHOR -Sebastien Buisson diff --git a/lipe/man/laudit.1 b/lipe/man/laudit.1 deleted file mode 100644 index a816693..0000000 --- a/lipe/man/laudit.1 +++ /dev/null @@ -1,68 +0,0 @@ -.\" Manpage for laudit. -.TH laudit 1 "7 March 2018" "1.0" "laudit man page" - -.SH NAME -laudit \- Lustre audit - -.SH SYNOPSIS -.B laudit -[ options ] - -.SH DESCRIPTION -.B laudit -is a command-line utility for Lustre audit. More specifically, -.B laudit -is a dedicated Lustre Changelogs consumer, specific for auditing. -.B laudit -has to be launched from a Lustre client, and can run in the background. It processes Changelog entries (freeing space on Lustre metadata targets), and stores relevant audit logs under a predefined local directory, as flat files. So -.B laudit -does not rely on any complicated database schema. -.PP -Information is organized to ease two typical audit queries: -.RS -- What is the history of a given file: who created, then accessed the file? -.PP -- What data is manipulated by a given user: which files did the user access? -.RE -.PP -.B laudit -config file describes how to reach Lustre, as well as how to store audit records. An example can be found at -.I /etc/laudit/laudit.conf.example -Please see the -.BR laudit.conf (1) -man page for details. - -.SH OPTIONS -.TP -.B -d, --daemon -Launch in daemon mode -.PP -.RS -If specified, parse Changelogs regularly in the background, according to the interval specified in the config file. -If not specified, just parse Changelogs once in the foreground. -.RE -.TP -.B -p, --path -Build path-to-fid info -.PP -.RS -Resolve FID to path when parsing Changelogs, and dump info along with entry. -.PP -WARNING! This can cost extra time to process. -.RE -.TP -.B -v, --verbose -Output some debug info; repeat for more details. -.TP -.B -h,--help -Print the usage - -.SH "SEE ALSO" -.BR laudit-report (1) -.BR laudit.conf (5) - -.SH BUGS -No known bugs. - -.SH AUTHOR -Sebastien Buisson diff --git a/lipe/man/laudit.conf.5 b/lipe/man/laudit.conf.5 deleted file mode 100644 index 2f4388e..0000000 --- a/lipe/man/laudit.conf.5 +++ /dev/null @@ -1,69 +0,0 @@ -.\" Manpage for laudit.conf. -.TH laudit 1 "7 March 2018" "1.0" "laudit.conf man page" - -.SH NAME -laudit.conf \- Lustre audit configuration file - -.SH DESCRIPTION -.B laudit.conf -is the laudit utility configuration file. It describes how to reach Lustre, as well as how to store audit records. -Please see the -.BR laudit (1) -man page for more information. - -.SH SYNTAX -.B laudit.conf -is in JSON format. -.PP -.B fs_name -gives the name of the file system to retrieve audit information from. -.PP -.B mount -gives the mount path of the file system to retrieve audit information from. -.PP -.B dump_path -is the path where to dump audit records. -.PP -.B mdts -is a list of Lustre MDTs for the file system fs_name. -.RS -.B mdt_idx -is the MDT index -.PP -.B reader_id -is the Changelogs consumer name for this MDT. -.RE -.PP -.B parse_interval_sec -defines the interval in seconds at which the Changelogs are consumed when laudit is run in daemon mode. -.PP -.B sync_every_n_entries -is the maximum number of Changelog entries that are processed before syncing to disk audit records stored under dump_path. -.PP -.B max_syncs_before_sleep -limits the number of entries processed before interrupting the daemon. Process will resume after parse_interval_sec. - -.SH EXAMPLE -Here is an example of laudit configuration file: -.RS -{ - "fs_name": "testfs", - "mount": "/mnt/lustre/testfs", - "dump_path": "/mnt/audit", - "mdts": [ - {"mdt_idx": 0, "reader_id": "cl1"}, - {"mdt_idx": 1, "reader_id": "cl1"} - ], - "parse_interval_sec": 10, - "sync_every_n_entries": 1000, - "max_syncs_before_sleep": 10 - -} -.RE - -.SH "SEE ALSO" -.BR laudit (1) -.BR laudit-report (1) - -.SH AUTHOR -Sebastien Buisson diff --git a/lipe/src/Makefile.am b/lipe/src/Makefile.am index 2be68ed..c27c8d8 100644 --- a/lipe/src/Makefile.am +++ b/lipe/src/Makefile.am @@ -27,10 +27,6 @@ bin_PROGRAMS += \ noinst_PROGRAMS += lipe_expression_test -if BUILD_LAUDIT -bin_PROGRAMS += laudit laudit-report -endif - if BUILD_HOTPOOL_UTILS bin_PROGRAMS += lamigo lpurge endif @@ -90,12 +86,6 @@ lfill_LDADD = $(LIPE_LDADD) ldumpstripe_SOURCES = ldumpstripe.c lustre_ea.h lustre_ea.c debug.c debug.h ldumpstripe_LDADD = $(liblustre_LIBS) -laudit_SOURCES = laudit.c -laudit_LDADD = $(json_c_LIBS) $(liblustre_LIBS) - -laudit_report_SOURCES = laudit-report.c -laudit_report_LDADD = $(json_c_LIBS) $(liblustre_LIBS) - lipe_ssh_SOURCES = lipe_ssh.c lipe_ssh.h lamigo_SOURCES = lx_log.h lamigo.c lamigo.h lamigo_alr.c lamigo_hash.c lamigo_hash.h \ diff --git a/lipe/src/laudit-report.c b/lipe/src/laudit-report.c deleted file mode 100644 index a21ca1b..0000000 --- a/lipe/src/laudit-report.c +++ /dev/null @@ -1,1039 +0,0 @@ -/* - * Copyright (c) 2018 DDN Storage, Inc - * - * Author: Sebastien Buisson sbuisson@ddn.com - */ - -/* - * lustre/utils/laudit-report.c - * Report tool for audit. - */ -#include -#include -#include -#include -#include -#include - -#include - -#include - -#ifndef ARRAY_SIZE -# define ARRAY_SIZE(a) ((sizeof(a)) / (sizeof((a)[0]))) -#endif /* !ARRAY_SIZE */ - -#define MDD_NAME_LEN 16 - -struct mdt_desc { - char reader_id[8]; - char mdd_name[MDD_NAME_LEN + 1]; -}; - -#define MAX_MDTS 256 -struct audit_config { - char fs_name[9]; - char mount[PATH_MAX + 1]; - char dump_path[PATH_MAX + 1]; - int parse_interval_sec; - int sync_every_n_entries; - int max_syncs_before_sleep; - int nb_mdts; - struct mdt_desc mdts[MAX_MDTS]; -}; - -static const char *progname; -static struct audit_config cfg = { .mdts = { { .reader_id = "" } } }; -static bool config_init; -static int verbose; -static char config_file[PATH_MAX + 1]; - -static unsigned int dir_level; -static char *file_to_audit, *user_to_audit; -static struct tm ts_before, ts_after; -static bool check_before, check_after; - -void laudit_report_usage(void) -{ - printf("usage: %s [options] \n\n", progname); - printf("\toptions:\n"); - printf("\t\t-f|--file : build report for given file\n"); - printf("\t\t\tYou can provide file path or FID, and also simple"); - printf("file or directory name.\n"); - printf("\t\t-u|--user : build report for given user\n"); - printf("\t\t-b|--before