From: Vitaliy Kuznetsov Date: Wed, 8 May 2024 13:04:40 +0000 (+0200) Subject: EX-9121 lipe: Add entry point for report merging option X-Git-Url: https://git.whamcloud.com/gitweb?a=commitdiff_plain;h=486a310dddaef58d4a5a5a7b041ed0d6dd9fecea;p=fs%2Flustre-release.git EX-9121 lipe: Add entry point for report merging option This patch adds a new option for lipe_scan3 to merge statistics reports. This option will work like this: lipe_scan3 --merge-reports=/dir_with_reports File with the results: Path to out: merged_report.out Path to yaml: merged_report.yaml Path to json: merged_report.json Path to csv: merged_report.csv This patch is the first in a series of patches to implement the functionality for merging reports on size statistics and includes functionality for initialization and the first entry point. Test-Parameters: trivial testlist=sanity-lipe-scan3,sanity-lipe-find3 Signed-off-by: Vitaliy Kuznetsov Change-Id: Ia02c28811922e0abba52a9c2d6408da8df9ae4c2 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/55046 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- diff --git a/lipe/src/lipe_scan3/Makefile.am b/lipe/src/lipe_scan3/Makefile.am index 637d444..1ba549e 100644 --- a/lipe/src/lipe_scan3/Makefile.am +++ b/lipe/src/lipe_scan3/Makefile.am @@ -28,6 +28,8 @@ lipe_scan3_SOURCES = \ ls3_dir_stats.h \ ls3_stats.c \ ls3_stats.h \ + ls3_merge_stats.c \ + ls3_merge_stats.h \ ls3_scan.c \ ls3_scan.h diff --git a/lipe/src/lipe_scan3/ls3_main.c b/lipe/src/lipe_scan3/ls3_main.c index 1eea156..515d744 100644 --- a/lipe/src/lipe_scan3/ls3_main.c +++ b/lipe/src/lipe_scan3/ls3_main.c @@ -30,6 +30,7 @@ #include "ls3_scan.h" #include "ls3_stats.h" #include "ls3_dir_stats.h" +#include "ls3_merge_stats.h" #define LS3_MODULE_NAME "lipe" #define LS3_SCAN "lipe-scan" @@ -1364,6 +1365,7 @@ enum { LS3_OPT_THREAD_COUNT, LS3_OPT_SHOW_COUNTERS, LS3_OPT_COLLECT_STATS, + LS3_OPT_MERGE_STATS, LS3_OPT_HELP = 'h', LS3_OPT_INTERACTIVE = 'i', @@ -1386,6 +1388,7 @@ static struct option options[] = { { "print-json", optional_argument, NULL, LS3_OPT_PRINT_JSON }, { LS3_SHOW_COUNTERS, no_argument, NULL, LS3_OPT_SHOW_COUNTERS}, { LS3_COLLECT_STATS, optional_argument, NULL, LS3_OPT_COLLECT_STATS}, + { "merge-reports", required_argument, NULL, LS3_OPT_MERGE_STATS}, { "print-absolute-path", no_argument, NULL, LS3_OPT_PRINT_ABSOLUTE_PATH }, { "print-relative-path", no_argument, NULL, LS3_OPT_PRINT_RELATIVE_PATH }, { "all-paths", no_argument, NULL, LS3_OPT_ALL_PATHS }, @@ -1583,6 +1586,10 @@ static void ls3_main_scm(void *data, int argc, char *argv[]) if (optarg) ls3_report_path = optarg; break; + case LS3_OPT_MERGE_STATS: + if (optarg) + ls3_stats_merge_reports(optarg); + exit(EXIT_SUCCESS); case LS3_OPT_PRINT_SELF_FID: policy = LS3_PRINT_SELF_FID; break; diff --git a/lipe/src/lipe_scan3/ls3_merge_stats.c b/lipe/src/lipe_scan3/ls3_merge_stats.c new file mode 100644 index 0000000..6a37f7f --- /dev/null +++ b/lipe/src/lipe_scan3/ls3_merge_stats.c @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2024, DDN Storage Corporation. + * + * Author: Vitaliy Kuznetsov + */ + +#include "ls3_merge_stats.h" + +static void ls3_m_parsing_json(struct fstats_report *stats, FILE *fd, + const char *path) +{ + +} + +static FILE *ls3_m_open_json_file(const char *filename) +{ + FILE *file = fopen(filename, "r"); + + if (!file) + LS3_ERROR("error with read report [%s]: %s\n", + filename, strerror(errno)); + + return file; +} + +static bool ls3_m_is_json_file(const char *filename) +{ + const char *extension; + int len = strlen(filename); + + if (len < 5) + return false; /* ".json" format equal 5 char */ + + extension = &filename[len - 5]; + if (strcmp(extension, ".json") == 0) + return true; + + /* TODO: Add version check in report */ + return false; +} + +void ls3_m_read_reports(struct fstats_report *stats, + const char *dir_with_reports) +{ + struct dirent *entry; + DIR *dir; + + /* TODO: Add support current dir by default */ + dir = opendir(dir_with_reports); + if (!dir) + LS3_FATAL("error with read target dir: %s\n", dir_with_reports); + + /* Read all reports in json format */ + while ((entry = readdir(dir))) { + FILE *fd; + char full_path[4096]; + + if (strcmp(entry->d_name, ".") == 0 && + strcmp(entry->d_name, "..") == 0) + continue; /* Skip it */ + + if (!ls3_m_is_json_file(entry->d_name)) + continue; /* Skip if not JSON file */ + + sprintf(full_path, "%s/%s", dir_with_reports, entry->d_name); + fd = ls3_m_open_json_file(full_path); + ls3_m_parsing_json(stats, fd, full_path); + fclose(fd); + } +} + +static struct fstats_report * ls3_m_report_init(void) +{ + struct fstats_report *stats; + int i; + + stats = xcalloc(1, sizeof(struct fstats_report)); + for (i = 0; i < LS3_STATS_TYPE_TOTAL_COUNT_REPORT; i++) { + stats->reports[i] = ls3_stats_get_new_report_ptr(); + } + + stats->max_count_users_in_report = LS3_STATS_ARRAY_SIZE_BY_DEFAULT; + stats->last_user_idx_in_array = 0; + stats->users_reports = (struct ls3_stats_user_report_template**)xcalloc( + stats->max_count_users_in_report, + sizeof(struct ls3_stats_user_report_template*)); + + pthread_mutex_init(&stats->user_rt_mutex, NULL); + return stats; +} + +void ls3_stats_merge_reports(const char *dir_with_reports) +{ + struct fstats_report *main_report; + + main_report = ls3_m_report_init(); + main_report->merging = true; + + ls3_m_read_reports(main_report, dir_with_reports); + ls3_stats_printf(main_report); + ls3_stats_destroy(main_report); +} diff --git a/lipe/src/lipe_scan3/ls3_merge_stats.h b/lipe/src/lipe_scan3/ls3_merge_stats.h new file mode 100644 index 0000000..4605aff --- /dev/null +++ b/lipe/src/lipe_scan3/ls3_merge_stats.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2024, DDN Storage Corporation. + * + * Author: Vitaliy Kuznetsov + */ + +#ifndef _LS3_MERGE_H_ +#define _LS3_MERGE_H_ + +#include +#include +#include +#include +#include +#include "ls3_stats.h" + +void ls3_stats_merge_reports(const char *dir_with_reports); + +#endif /* _LS3_MERGE_H_ */