EX-8130 lipe: Link a new statistics module

This patch links the collection of statistics for
directories with the main collection of statistics
about file sizes. This way we collect all the statistics
at one time.

This patch also adds 2 new options for collecting statistics
via lipe_find3.

The -depth option serves as a limiter on the collection and
output of directory statistics to a file.
The -top-rating option allows adjustment of the size of the
table ranking the largest directories by size.

A mechanism has also been added that aallows a copy of the
path from the filter for correct data processing.

Test-Parameters: trivial testlist=sanity-lipe-scan3,sanity-lipe-find3
Signed-off-by: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
Change-Id: Ia61f57536575fd17e40b6e34a0c4b9b5db9111c5
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53990
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Alexandre Ioffe <aioffe@ddn.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lipe/src/lipe_find3/lf3_lexer.l b/lipe/src/lipe_find3/lf3_lexer.l
index 93fabf3..233ae48 100644 (file)
--- a/lipe/src/lipe_find3/lf3_lexer.l
+++ b/lipe/src/lipe_find3/lf3_lexer.l
@@ -59,7 +59,7 @@ static int expr_arg_type;
        return TOKEN_TEST;
 }
 
--(amin|atime|blocks|cmin|crtime|ctime|gid|group|iname|inum|ipath|links|mirror-count|mmin|mtime|name|path|perm|pool|projid|size|stripe-count|type|uid|user|xattr)\0 {
+-(amin|atime|blocks|cmin|crtime|ctime|gid|group|iname|inum|ipath|links|mirror-count|mmin|mtime|name|path|perm|pool|projid|size|stripe-count|type|uid|user|xattr|depth|top-rating)\0 {
        LF3_DEBUG("unary '%s'\n", yytext);
        expr_arg_begin = lf3_arg_index;
        expr_arg_end = lf3_arg_index + 2;

diff --git a/lipe/src/lipe_find3/lf3_parse.y b/lipe/src/lipe_find3/lf3_parse.y
index c9abddf..9a421f1 100644 (file)
--- a/lipe/src/lipe_find3/lf3_parse.y
+++ b/lipe/src/lipe_find3/lf3_parse.y
@@ -45,6 +45,12 @@ static const char LF3_SCAN_COMMAND[] = LF3_SCAN_PROGRAM " --script=/dev/stdin";
 static const char LF3_SCM_SCAN_PROC[] = "lipe-scan";
 static char *lf3_policy_body;
 static bool lf3_policy_body_has_action;
+static bool lf3_policy_has_depth;
+static char *lf3_depth_arg_val;
+static bool lf3_policy_has_top_rating;
+static char *lf3_top_rating_arg_val;
+static bool lf3_stats_has_start_path;
+static char *lf3_stats_start_path;
 
 typedef unsigned long long __ull;
 
@@ -440,6 +446,22 @@ static char *lf3_numeric_expr(const char *thunk, int begin, int end)
 
        assert(unit == 1);
 
+       if (strcmp(thunk, "depth") == 0) {
+               lf3_policy_has_depth = true;
+               lf3_depth_arg_val = xsprintf("%llu", val);
+               /* skip it */
+               return xsprintf("");
+       } else if (strcmp(thunk, "top-rating") == 0) {
+               lf3_policy_has_top_rating = true;
+               lf3_top_rating_arg_val = xsprintf("%llu", val);
+               /* skip it */
+               return xsprintf("");
+       } else if (strcmp(thunk, "spath") == 0) {
+               /* value doesn't make sense here */
+               lf3_stats_has_start_path = true;
+               return xsprintf("");
+       }
+
        return xsprintf("(%c (%s) %llu)", cmp, thunk, val);
 }
 
@@ -737,6 +759,12 @@ static char *lf3_path_expr(int begin, int end)
 {
        assert(begin + 2 == end);
 
+       /* copy path for size statistics, since it will
+        * no longer be available later. */
+       lf3_stats_has_start_path = true;
+       lf3_stats_start_path = xsprintf("\"%s\"", lf3_arg[begin + 1]);
+       LF3_DEBUG_S(lf3_stats_start_path);
+
        return lf3_fnmatch_expr(LF3_FNMATCH, lf3_arg[begin + 1], "call-with-relative-path");
 }
 
@@ -1166,6 +1194,8 @@ static char *lf3_simple_expr(int begin, int end)
        X2("gid",   lf3_numeric_expr("gid",   begin, end));
        X2("projid", lf3_numeric_expr("projid",   begin, end));
        X2("links", lf3_numeric_expr("nlink", begin, end));
+       X2("depth", lf3_numeric_expr("depth", begin, end));
+       X2("top-rating", lf3_numeric_expr("top-rating", begin, end));
        X2("mirror-count", lf3_numeric_expr("lov-mirror-count", begin, end));
        X2("stripe-count", lf3_numeric_expr("lov-stripe-count", begin, end));
 
@@ -1489,6 +1519,15 @@ int main(int argc, char *argv[])
        LF3_DEBUG_S(lf3_policy_body);
        LF3_DEBUG_B(lf3_policy_body_has_action);
 
+       if (!lf3_policy_has_depth)
+               lf3_depth_arg_val = "(lipe-getopt-stats-depth)";
+
+       if (!lf3_policy_has_top_rating)
+               lf3_top_rating_arg_val = "(lipe-getopt-top-rating)";
+
+       if (!lf3_stats_has_start_path)
+               lf3_stats_start_path = "(lipe-getopt-spath)";
+
        if (rc != 0 || lf3_policy_body == NULL)
                LF3_FATAL("internal argument parsing error\n");
 
@@ -1547,6 +1586,9 @@ int main(int argc, char *argv[])
                "        %s\n"
                "        %s\n"
                "        %s\n"
+               "        %s\n"
+               "        %s\n"
+               "        %s\n"
                "        %s))\n"
                "    (lambda () %K)))\n",
                &lf3_genvar_list,
@@ -1557,6 +1599,9 @@ int main(int argc, char *argv[])
                        policy_thunk,
                        "(lipe-getopt-required-attrs)",
                        thread_count,
+                       lf3_depth_arg_val,
+                       lf3_top_rating_arg_val,
+                       lf3_stats_start_path,
                &lf3_fini_list);
        LF3_DEBUG_S(scm_code);
 

diff --git a/lipe/src/lipe_find3/lf3_parse_format.c b/lipe/src/lipe_find3/lf3_parse_format.c
index f015004..5d40f1e 100644 (file)
--- a/lipe/src/lipe_find3/lf3_parse_format.c
+++ b/lipe/src/lipe_find3/lf3_parse_format.c
@@ -268,6 +268,12 @@ width:
        case 'n': /* Number of hard links to file. */
                LF3_EMIT_N('d', "(nlink)");
                break;
+       case 'X': /* Number of depth for stats dir. */
+               LF3_EMIT_N('d', "(depth)");
+               break;
+       case 'V': /* TOP X raiting for stats dir. */
+               LF3_EMIT_N('d', "(top-rating)");
+               break;
        case 'p': /* File's name with 'starting-point'. */
                LF3_EMIT_S('a', "(absolute-path)");
                break;

diff --git a/lipe/src/lipe_find3/lipe_scan3_mock.scm b/lipe/src/lipe_find3/lipe_scan3_mock.scm
index fdab34b..3306fe5 100755 (executable)
--- a/lipe/src/lipe_find3/lipe_scan3_mock.scm
+++ b/lipe/src/lipe_find3/lipe_scan3_mock.scm
@@ -216,6 +216,12 @@
 (define (lipe-getopt-thread-count)
   1)
 
+(define (lipe-getopt-stats-depth)
+  4)
+
+(define (lipe-getopt-top-rating)
+  15)
+
 (define (lipe-scan device client-mount-path policy required-attrs thread-count)
   (policy))
 

diff --git a/lipe/src/lipe_scan3/Makefile.am b/lipe/src/lipe_scan3/Makefile.am
index 0f62166..637d444 100644 (file)
--- a/lipe/src/lipe_scan3/Makefile.am
+++ b/lipe/src/lipe_scan3/Makefile.am
@@ -24,6 +24,8 @@ lipe_scan3_SOURCES = \
        ls3_main.c \
        ls3_object_attrs.c \
        ls3_object_attrs.h \
+       ls3_dir_stats.c \
+       ls3_dir_stats.h \
        ls3_stats.c \
        ls3_stats.h \
        ls3_scan.c \

diff --git a/lipe/src/lipe_scan3/ls3_main.c b/lipe/src/lipe_scan3/ls3_main.c
index a083510..099d5ca 100644 (file)
--- a/lipe/src/lipe_scan3/ls3_main.c
+++ b/lipe/src/lipe_scan3/ls3_main.c
@@ -29,6 +29,7 @@
 #include "ls3_object_attrs.h"
 #include "ls3_scan.h"
 #include "ls3_stats.h"
+#include "ls3_dir_stats.h"
 
 #define LS3_MODULE_NAME "lipe"
 #define LS3_SCAN "lipe-scan"
@@ -44,6 +45,9 @@
 #define LS3_GETOPT_CLIENT_MOUNT_PATH "lipe-getopt-client-mount-path"
 #define LS3_GETOPT_REQUIRED_ATTRS "lipe-getopt-required-attrs"
 #define LS3_GETOPT_THREAD_COUNT "lipe-getopt-thread-count"
+#define LS3_GETOPT_STATS_DEPTH "lipe-getopt-stats-depth"
+#define LS3_GETOPT_TOP_RATING "lipe-getopt-top-rating"
+#define LS3_GETOPT_SPATH "lipe-getopt-spath"
 
 static const char *ls3_device_path;
 static const char *ls3_client_mount_path = LS3_CSTR_AUTO;
@@ -430,7 +434,8 @@ SCM_DEFINE(ls3_scm_warn_get_attr_enable, "lipe-warnings-get-attr-enable", 0, 1,
        return scm_from_bool(old_warn);
 }
 
-SCM_DEFINE(ls3_scm_gettid, "lipe-gettid", 0, 0, 0, (), "print caller's thread ID (not pthread id)")
+SCM_DEFINE(ls3_scm_gettid, "lipe-gettid", 0, 0, 0, (),
+          "print caller's thread ID (not pthread id)")
 {
        return scm_from_ulong(syscall(SYS_gettid));
 }
@@ -459,7 +464,8 @@ SCM_DEFINE(ls3_scm_string_to_json_attrs, "lipe-string->json-attrs", 1, 0, 0,
 }
 #undef FUNC_NAME
 
-SCM_DEFINE(ls3_scm_getopt_device_path, LS3_GETOPT_DEVICE_PATH, 0, 0, 0, (), "return device path from options or #f")
+SCM_DEFINE(ls3_scm_getopt_device_path, LS3_GETOPT_DEVICE_PATH, 0, 0, 0, (),
+          "return device path from options or #f")
 {
        if (ls3_device_path != NULL)
                return scm_from_latin1_string(ls3_device_path);
@@ -468,7 +474,8 @@ SCM_DEFINE(ls3_scm_getopt_device_path, LS3_GETOPT_DEVICE_PATH, 0, 0, 0, (), "ret
 }
 
 /* Global command line options accessors. */
-SCM_DEFINE(ls3_scm_getopt_client_mount_path, LS3_GETOPT_CLIENT_MOUNT_PATH, 0, 0, 0, (), "return client mount path from options")
+SCM_DEFINE(ls3_scm_getopt_client_mount_path, LS3_GETOPT_CLIENT_MOUNT_PATH, 0, 0,
+          0, (), "return client mount path from options")
 {
        /* Normally this should just return #t to tell lipe-scan to
         * try infer the client mount from the device.
@@ -483,16 +490,36 @@ SCM_DEFINE(ls3_scm_getopt_client_mount_path, LS3_GETOPT_CLIENT_MOUNT_PATH, 0, 0,
                return scm_from_latin1_string(ls3_client_mount_path);
 }
 
-SCM_DEFINE(ls3_scm_getopt_required_attrs, LS3_GETOPT_REQUIRED_ATTRS, 0, 0, 0, (), "return required object attrs from options")
+SCM_DEFINE(ls3_scm_getopt_required_attrs, LS3_GETOPT_REQUIRED_ATTRS, 0, 0, 0, (),
+          "return required object attrs from options")
 {
        return scm_from_uint(ls3_required_attrs);
 }
 
-SCM_DEFINE(ls3_scm_getopt_thread_count, LS3_GETOPT_THREAD_COUNT, 0, 0, 0, (), "return thread count from options")
+SCM_DEFINE(ls3_scm_getopt_thread_count, LS3_GETOPT_THREAD_COUNT, 0, 0, 0, (),
+          "return thread count from options")
 {
        return scm_from_int(ls3_thread_count);
 }
 
+SCM_DEFINE(ls3_scm_getopt_stats_depth, LS3_GETOPT_STATS_DEPTH, 0, 0, 0, (),
+          "return depth for stats from options")
+{
+               return scm_from_int(0);
+}
+
+SCM_DEFINE(ls3_scm_getopt_top_rating, LS3_GETOPT_TOP_RATING, 0, 0, 0, (),
+          "return N for top rating directory in stats from options")
+{
+               return scm_from_int(0);
+}
+
+SCM_DEFINE(ls3_scm_getopt_spath, LS3_GETOPT_SPATH, 0, 0, 0, (),
+          "return filtr 'path' for directory stats")
+{
+               return scm_from_latin1_string("");
+}
+
 /* Scanning context (instance and tread info) accessors. */
 
 /* To be called when scheme procedures that depend on the scanning
@@ -1147,21 +1174,27 @@ SCM_DEFINE(ls3_scm_print_relative_path, LS3_PRINT_RELATIVE_PATH, 0, 0, 0,
        return SCM_UNSPECIFIED;
 }
 
-SCM_DEFINE(ls3_scm_scan, LS3_SCAN, 5, 0, 0,
+SCM_DEFINE(ls3_scm_scan, LS3_SCAN, 8, 0, 0,
           (SCM s_device_path,
            SCM s_client_mount_path,
            SCM s_policy_thunk,
            SCM s_required_attrs,
-           SCM s_thread_count),
+           SCM s_thread_count,
+           SCM s_stats_depth,
+           SCM s_stats_top_rating,
+           SCM s_stats_spath),
           "scan a device")
 #define FUNC_NAME s_ls3_scm_scan
 {
        char *c_device_path = NULL;
+       char *c_stats_spath = NULL;
        const char *c_client_mount_path = NULL;
        char *c_client_mount_path_1 = NULL;
 
        unsigned int c_required_attrs;
        int c_thread_count;
+       int c_stats_depth;
+       int c_stats_top_rating;
        int rc;
 
        SCM_VALIDATE_STRING(1, s_device_path);
@@ -1179,11 +1212,40 @@ SCM_DEFINE(ls3_scm_scan, LS3_SCAN, 5, 0, 0,
        SCM_VALIDATE_THUNK(3, s_policy_thunk);
        SCM_VALIDATE_UINT_COPY(4, s_required_attrs, c_required_attrs);
        SCM_VALIDATE_INT_COPY(5, s_thread_count, c_thread_count);
+       SCM_VALIDATE_INT_COPY(6, s_stats_depth, c_stats_depth);
+       SCM_VALIDATE_INT_COPY(7, s_stats_top_rating, c_stats_top_rating);
+       SCM_VALIDATE_STRING(8, s_stats_spath);
+       c_stats_spath = scm_to_latin1_string(s_stats_spath);
 
        LS3_DEBUG_S(c_device_path);
        LS3_DEBUG_S(c_client_mount_path);
        LS3_DEBUG_X(c_required_attrs);
        LS3_DEBUG_D(c_thread_count);
+       LS3_DEBUG_D(c_stats_depth);
+       LS3_DEBUG_D(c_stats_top_rating);
+       LS3_DEBUG_D(c_stats_spath);
+
+       if (c_stats_spath[0] != '\0') {
+               size_t len = strlen(c_stats_spath);
+
+               if (len > 0 && c_stats_spath[len - 1] == '*') {
+                       c_stats_spath[len - 1] = '\0';
+                       dir_stats->lsdg_start_path = xstrdup(c_stats_spath);
+               }
+       } else {
+               dir_stats->lsdg_start_path = xstrdup("");
+       }
+
+       if (c_stats_depth == 0)
+               dir_stats->lsdg_max_depth = LS3_STATS_DEPTH_BY_DEFAULT;
+       else
+               dir_stats->lsdg_max_depth = c_stats_depth;
+
+       if (c_stats_top_rating == 0)
+               dir_stats->lsdg_top_rating_limit =
+                       LS3_STATS_TOP_RATING_BY_DEFAULT;
+       else
+               dir_stats->lsdg_top_rating_limit = c_stats_top_rating;
 
        rc = ls3_scan(c_device_path,
                      c_client_mount_path,
@@ -1415,6 +1477,9 @@ static void ls3_module_init(void *unused)
        "lipe-getopt-device-path",
        "lipe-getopt-required-attrs",
        "lipe-getopt-thread-count",
+       "lipe-getopt-stats-depth",
+       "lipe-getopt-top-rating",
+       "lipe-getopt-spath",
        "lipe-warnings-get-attr-enable",
        "lipe-scan",
        "lipe-scan-break",
@@ -1594,6 +1659,7 @@ static void ls3_main_scm(void *data, int argc, char *argv[])
        int exit_status = EXIT_SUCCESS;
 
        ls3_stats_init();
+       ls3_stats_dir_init();
 
        if (policy != NULL) {
                int i;
@@ -1617,7 +1683,10 @@ static void ls3_main_scm(void *data, int argc, char *argv[])
                                               scm_call_0(X(LS3_GETOPT_CLIENT_MOUNT_PATH)),
                                               X(policy),
                                               scm_call_0(X(LS3_GETOPT_REQUIRED_ATTRS)),
-                                              scm_call_0(X(LS3_GETOPT_THREAD_COUNT)));
+                                              scm_call_0(X(LS3_GETOPT_THREAD_COUNT)),
+                                              scm_call_0(X(LS3_GETOPT_STATS_DEPTH)),
+                                              scm_call_0(X(LS3_GETOPT_TOP_RATING)),
+                                              scm_call_0(X(LS3_GETOPT_SPATH)));
 #undef X
                        scan_status = ls3_scm_to_exit_status(scan_rc);
                        LS3_DEBUG_D(scan_status);
@@ -1642,6 +1711,7 @@ static void ls3_main_scm(void *data, int argc, char *argv[])
                ls3_counters_print();
 
        ls3_stats_destroy();
+       ls3_stats_dir_destroy();
        LS3_DEBUG_D(exit_status);
        exit(exit_status);
 }

diff --git a/lipe/src/lipe_scan3/ls3_stats.c b/lipe/src/lipe_scan3/ls3_stats.c
index c60c93e..37fe149 100644 (file)
--- a/lipe/src/lipe_scan3/ls3_stats.c
+++ b/lipe/src/lipe_scan3/ls3_stats.c
@@ -17,6 +17,7 @@
 #include <lustre/lustreapi.h>
 #include <json-c/json.h>
 #include "ls3_stats.h"
+#include "ls3_dir_stats.h"
 
 
 /* Function to calculate the range index of a number.
@@ -1360,6 +1361,8 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time)
                }
        }
 
+       ls3_stats_fprintf_out(out_fd);
+
        fclose(out_fd);
        printf("  Path to out: %s\n", reports_with_stats->format_report_file_name);
 }
@@ -2128,6 +2131,9 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all)
                                       LS3_STATS_EMPTY_VALUE);
        }
 
+       /* Size statistics for directories */
+       ls3_stats_dir_distributor(loa_all);
+
        /* Reports only for regular file's */
        if (!(loa_all->loa_mode & S_IFREG))
                /* TODO: Directory processing will be added here */

diff --git a/lipe/src/lipe_scan3/scripts/lipe-scan-break.scm b/lipe/src/lipe_scan3/scripts/lipe-scan-break.scm
index 3f79c0e..b1cb407 100755 (executable)
--- a/lipe/src/lipe_scan3/scripts/lipe-scan-break.scm
+++ b/lipe/src/lipe_scan3/scripts/lipe-scan-break.scm
@@ -9,5 +9,7 @@
                             (lipe-scan-break 7))
                         (format #t "~a ~a\n" (lipe-scan-thread-index) (ino)))
                       (lipe-getopt-required-attrs)
-                      (lipe-getopt-thread-count)))
+                      (lipe-getopt-thread-count)
+                      (lipe-getopt-stats-depth)
+                      (lipe-getopt-top-rating)))
          (cdr (program-arguments)))

diff --git a/lipe/src/lipe_scan3/scripts/lipe-scan-continue.scm b/lipe/src/lipe_scan3/scripts/lipe-scan-continue.scm
index f3f5432..b54e143 100755 (executable)
--- a/lipe/src/lipe_scan3/scripts/lipe-scan-continue.scm
+++ b/lipe/src/lipe_scan3/scripts/lipe-scan-continue.scm
@@ -9,5 +9,7 @@
                         (lipe-scan-continue)
                         (format #t "REACHED\n"))
                       (lipe-getopt-required-attrs)
-                      (lipe-getopt-thread-count)))
+                      (lipe-getopt-thread-count)
+                      (lipe-getopt-stats-depth)
+                      (lipe-getopt-top-rating)))
          (cdr (program-arguments)))

diff --git a/lipe/src/lipe_scan3/scripts/lipe-scan-format-ino.scm b/lipe/src/lipe_scan3/scripts/lipe-scan-format-ino.scm
index 5a62021..6c21334 100755 (executable)
--- a/lipe/src/lipe_scan3/scripts/lipe-scan-format-ino.scm
+++ b/lipe/src/lipe_scan3/scripts/lipe-scan-format-ino.scm
@@ -6,5 +6,7 @@
                       (lipe-getopt-client-mount-path)
                       (lambda () (format #t "~a ~a\n" (lipe-scan-device-name) (ino)))
                       (lipe-getopt-required-attrs)
-                      (lipe-getopt-thread-count)))
+                      (lipe-getopt-thread-count)
+                      (lipe-getopt-stats-depth)
+                      (lipe-getopt-top-rating)))
          (cdr (program-arguments)))

diff --git a/lipe/src/lipe_scan3/scripts/lipe-scan-print-json.scm b/lipe/src/lipe_scan3/scripts/lipe-scan-print-json.scm
index 0900161..99aa56c 100755 (executable)
--- a/lipe/src/lipe_scan3/scripts/lipe-scan-print-json.scm
+++ b/lipe/src/lipe_scan3/scripts/lipe-scan-print-json.scm
@@ -6,5 +6,7 @@
                       (lipe-getopt-client-mount-path)
                       print-json
                       (lipe-getopt-required-attrs)
-                      (lipe-getopt-thread-count)))
+                      (lipe-getopt-thread-count)
+                      (lipe-getopt-stats-depth)
+                      (lipe-getopt-top-rating)))
          (cdr (program-arguments)))

diff --git a/lipe/src/lipe_scan3/scripts/lipe-scan-repl.scm b/lipe/src/lipe_scan3/scripts/lipe-scan-repl.scm
index d51404d..8430f8e 100755 (executable)
--- a/lipe/src/lipe_scan3/scripts/lipe-scan-repl.scm
+++ b/lipe/src/lipe_scan3/scripts/lipe-scan-repl.scm
@@ -37,5 +37,7 @@
                       (lipe-getopt-client-mount-path)
                       lipe-repl-policy
                       (lipe-getopt-required-attrs)
-                      (lipe-getopt-thread-count)))
+                      (lipe-getopt-thread-count)
+                      (lipe-getopt-stats-depth)
+                      (lipe-getopt-top-rating)))
          (cdr (program-arguments)))

diff --git a/lipe/src/lipe_scan3/tests/lipe.scm b/lipe/src/lipe_scan3/tests/lipe.scm
index 4de0ab9..97ca965 100644 (file)
--- a/lipe/src/lipe_scan3/tests/lipe.scm
+++ b/lipe/src/lipe_scan3/tests/lipe.scm
@@ -49,6 +49,8 @@
           ctime
           size
           blocks
+          depth
+          top-rating
 
           file-fid
           self-fid