EX-9121 lipe: Add entry point for report merging option

This patch adds a new option for lipe_scan3 to merge
statistics reports.

This option will work like this:
lipe_scan3 --merge-reports=/dir_with_reports
File with the results:
  Path to out: merged_report.out
  Path to yaml: merged_report.yaml
  Path to json: merged_report.json
  Path to csv: merged_report.csv

This patch is the first in a series of patches to implement the
functionality for merging reports on size statistics and includes
functionality for initialization and the first entry point.

Test-Parameters: trivial testlist=sanity-lipe-scan3,sanity-lipe-find3
Signed-off-by: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
Change-Id: Ia02c28811922e0abba52a9c2d6408da8df9ae4c2
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/55046
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lipe/src/lipe_scan3/Makefile.am b/lipe/src/lipe_scan3/Makefile.am
index 637d444..1ba549e 100644 (file)
--- a/lipe/src/lipe_scan3/Makefile.am
+++ b/lipe/src/lipe_scan3/Makefile.am
@@ -28,6 +28,8 @@ lipe_scan3_SOURCES = \
        ls3_dir_stats.h \
        ls3_stats.c \
        ls3_stats.h \
+       ls3_merge_stats.c \
+       ls3_merge_stats.h \
        ls3_scan.c \
        ls3_scan.h
 

diff --git a/lipe/src/lipe_scan3/ls3_main.c b/lipe/src/lipe_scan3/ls3_main.c
index 1eea156..515d744 100644 (file)
--- a/lipe/src/lipe_scan3/ls3_main.c
+++ b/lipe/src/lipe_scan3/ls3_main.c
@@ -30,6 +30,7 @@
 #include "ls3_scan.h"
 #include "ls3_stats.h"
 #include "ls3_dir_stats.h"
+#include "ls3_merge_stats.h"
 
 #define LS3_MODULE_NAME "lipe"
 #define LS3_SCAN "lipe-scan"
@@ -1364,6 +1365,7 @@ enum {
        LS3_OPT_THREAD_COUNT,
        LS3_OPT_SHOW_COUNTERS,
        LS3_OPT_COLLECT_STATS,
+       LS3_OPT_MERGE_STATS,
 
        LS3_OPT_HELP = 'h',
        LS3_OPT_INTERACTIVE = 'i',
@@ -1386,6 +1388,7 @@ static struct option options[] = {
        { "print-json", optional_argument, NULL, LS3_OPT_PRINT_JSON },
        { LS3_SHOW_COUNTERS, no_argument, NULL, LS3_OPT_SHOW_COUNTERS},
        { LS3_COLLECT_STATS, optional_argument, NULL, LS3_OPT_COLLECT_STATS},
+       { "merge-reports", required_argument, NULL, LS3_OPT_MERGE_STATS},
        { "print-absolute-path", no_argument, NULL, LS3_OPT_PRINT_ABSOLUTE_PATH },
        { "print-relative-path", no_argument, NULL, LS3_OPT_PRINT_RELATIVE_PATH },
        { "all-paths", no_argument, NULL, LS3_OPT_ALL_PATHS },
@@ -1583,6 +1586,10 @@ static void ls3_main_scm(void *data, int argc, char *argv[])
                        if (optarg)
                                ls3_report_path = optarg;
                        break;
+               case LS3_OPT_MERGE_STATS:
+                       if (optarg)
+                               ls3_stats_merge_reports(optarg);
+                       exit(EXIT_SUCCESS);
                case LS3_OPT_PRINT_SELF_FID:
                        policy = LS3_PRINT_SELF_FID;
                        break;

diff --git a/lipe/src/lipe_scan3/ls3_merge_stats.c b/lipe/src/lipe_scan3/ls3_merge_stats.c
new file mode 100644 (file)
index 0000000..6a37f7f
--- /dev/null
+++ b/lipe/src/lipe_scan3/ls3_merge_stats.c
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2024, DDN Storage Corporation.
+ *
+ * Author: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
+ */
+
+#include "ls3_merge_stats.h"
+
+static void ls3_m_parsing_json(struct fstats_report *stats, FILE *fd,
+                              const char *path)
+{
+
+}
+
+static FILE *ls3_m_open_json_file(const char *filename)
+{
+       FILE *file = fopen(filename, "r");
+
+       if (!file)
+               LS3_ERROR("error with read report [%s]: %s\n",
+                         filename, strerror(errno));
+
+       return file;
+}
+
+static bool ls3_m_is_json_file(const char *filename)
+{
+       const char *extension;
+       int len = strlen(filename);
+
+       if (len < 5)
+               return false;   /* ".json" format equal 5 char */
+
+       extension = &filename[len - 5];
+       if (strcmp(extension, ".json") == 0)
+               return true;
+
+       /* TODO: Add version check in report */
+       return false;
+}
+
+void ls3_m_read_reports(struct fstats_report *stats,
+                       const char *dir_with_reports)
+{
+       struct dirent *entry;
+       DIR *dir;
+
+       /* TODO: Add support current dir by default */
+       dir = opendir(dir_with_reports);
+       if (!dir)
+               LS3_FATAL("error with read target dir: %s\n", dir_with_reports);
+
+       /* Read all reports in json format */
+       while ((entry = readdir(dir))) {
+               FILE *fd;
+               char full_path[4096];
+
+               if (strcmp(entry->d_name, ".") == 0 &&
+                   strcmp(entry->d_name, "..") == 0)
+                       continue;       /* Skip it */
+
+               if (!ls3_m_is_json_file(entry->d_name))
+                       continue;       /* Skip if not JSON file */
+
+               sprintf(full_path, "%s/%s", dir_with_reports, entry->d_name);
+               fd = ls3_m_open_json_file(full_path);
+               ls3_m_parsing_json(stats, fd, full_path);
+               fclose(fd);
+       }
+}
+
+static struct fstats_report * ls3_m_report_init(void)
+{
+       struct fstats_report *stats;
+       int i;
+
+       stats = xcalloc(1, sizeof(struct fstats_report));
+       for (i = 0; i < LS3_STATS_TYPE_TOTAL_COUNT_REPORT; i++) {
+               stats->reports[i] = ls3_stats_get_new_report_ptr();
+       }
+
+       stats->max_count_users_in_report = LS3_STATS_ARRAY_SIZE_BY_DEFAULT;
+       stats->last_user_idx_in_array = 0;
+       stats->users_reports = (struct ls3_stats_user_report_template**)xcalloc(
+                               stats->max_count_users_in_report,
+                               sizeof(struct ls3_stats_user_report_template*));
+
+       pthread_mutex_init(&stats->user_rt_mutex, NULL);
+       return stats;
+}
+
+void ls3_stats_merge_reports(const char *dir_with_reports)
+{
+       struct fstats_report *main_report;
+
+       main_report = ls3_m_report_init();
+       main_report->merging = true;
+
+       ls3_m_read_reports(main_report, dir_with_reports);
+       ls3_stats_printf(main_report);
+       ls3_stats_destroy(main_report);
+}

diff --git a/lipe/src/lipe_scan3/ls3_merge_stats.h b/lipe/src/lipe_scan3/ls3_merge_stats.h
new file mode 100644 (file)
index 0000000..4605aff
--- /dev/null
+++ b/lipe/src/lipe_scan3/ls3_merge_stats.h
@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2024, DDN Storage Corporation.
+ *
+ * Author: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
+ */
+
+#ifndef _LS3_MERGE_H_
+#define _LS3_MERGE_H_
+
+#include <stdio.h>
+#include <float.h>
+#include <dirent.h>
+#include <json-c/json.h>
+#include <pthread.h>
+#include "ls3_stats.h"
+
+void ls3_stats_merge_reports(const char *dir_with_reports);
+
+#endif /* _LS3_MERGE_H_ */