From e2301821901e3764472a11af87961e17743b596d Mon Sep 17 00:00:00 2001 From: Vitaliy Kuznetsov Date: Wed, 29 May 2024 14:26:29 +0200 Subject: [PATCH] EX-9121 lipe: Add statistics merging for directories This patch adds the ability to merge statistics for directories. This is the first of two patches and contains the basic collection of information from json as well as basic output. Test-Parameters: trivial testlist=sanity-lipe-scan3,sanity-lipe-find3 Signed-off-by: Vitaliy Kuznetsov Change-Id: I253f7606b66921eddf52709931cc1c880e66a997 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/55233 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Alexandre Ioffe Reviewed-by: Andreas Dilger --- lipe/src/lipe_scan3/ls3_merge_stats.c | 341 +++++++++++++++++++++++++++++++--- lipe/src/lipe_scan3/ls3_merge_stats.h | 4 + 2 files changed, 321 insertions(+), 24 deletions(-) diff --git a/lipe/src/lipe_scan3/ls3_merge_stats.c b/lipe/src/lipe_scan3/ls3_merge_stats.c index 62ddbe0..f021675 100644 --- a/lipe/src/lipe_scan3/ls3_merge_stats.c +++ b/lipe/src/lipe_scan3/ls3_merge_stats.c @@ -6,6 +6,22 @@ #include "ls3_merge_stats.h" +static void ls3_m_upd_str2fid(const char *fid_str, struct lu_fid *fid) +{ + char *end_ptr; + + /* if was not deleted, in one of the reports FID will be 100% */ + if (strcmp(fid_str, LS3_M_EMPTY_FID_STR) == 0 || + strcmp(fid_str, "") == 0) + return; /* Skip */ + + fid->f_seq = strtoull(fid_str, &end_ptr, 16); + if (*end_ptr != '\0') { + fid->f_oid = strtoul(end_ptr + 1, &end_ptr, 16); + if (*end_ptr != '\0') + fid->f_ver = strtoul(end_ptr + 1, NULL, 16); + } +} static struct range_report_template *ls3_m_alloc_new_id_range( struct report_template *report_ptr, @@ -58,7 +74,6 @@ static struct range_report_template *ls3_m_alloc_new_range( return range_ptr; } - static void ls3_m_parsing_range(struct range_report_template *range, json_object *json, const char *dev_name, @@ -72,7 +87,6 @@ static void ls3_m_parsing_range(struct range_report_template *range, double min; double max; uint64_t l_atime; - int last_dev_idx; if (user_report) { json_object_object_get_ex(json, "CountFilesInRange", &j_count); @@ -92,10 +106,6 @@ static void ls3_m_parsing_range(struct range_report_template *range, min = json_object_get_int64(j_min); max = json_object_get_int64(j_max); - last_dev_idx = range->rrt_dev_cnt + 1; - range->rrt_dev[last_dev_idx] = xstrdup(dev_name); - range->rrt_dev_cnt++; - if (range->rrt_min > min) range->rrt_min = min; @@ -168,8 +178,8 @@ static int64_t ls3_m_get_index_for_range(double range_start, int64_t range_id, } static void ls3_m_parsing_report(struct report_template **r_template_ptr, - json_object *report, const char *dev_name, - bool user_report) + json_object *report, + const char *dev_name, bool user_report) { struct report_template *mem_report; json_object *general_info; @@ -204,7 +214,7 @@ static void ls3_m_parsing_report(struct report_template **r_template_ptr, table_idx = json_object_get_int(table_id); mem_report = r_template_ptr[table_idx]; if (!mem_report) - LS3_FATAL("Table %d not found in memory\n", table_idx); + return; if (user_report) { range_types = LS3_STATS_RANGE_TYPE_INT; @@ -332,18 +342,268 @@ static void ls3_m_parsing_user_report(struct fstats_report *stats, } } -static void ls3_m_parsing_json(struct fstats_report *stats, FILE *fd, - const char *path) +static struct ls3_stats_dir_obj *ls3_m_get_child_dir_ptr( + struct ls3_stats_dir_general *m_dir_stats, + struct ls3_stats_dir_obj *parent_ptr, const char *dir_name, int depth) +{ + struct ls3_stats_dir_obj *dir_ptr; + uint64_t current_index = 0; + bool was_found = false; + int i; + + if (!parent_ptr) + LS3_FATAL("parent directory is not defined (NULL)\n"); + + + for (i = 0; i <= parent_ptr->lsdo_last_child; i++) { + dir_ptr = parent_ptr->lsdo_child[i]; + + if (dir_ptr == NULL) + continue; + + if (strcmp(dir_ptr->lsdo_name, dir_name) == 0) { + was_found = true; + break; + } + } + + if (was_found) + return dir_ptr; + + /* Get exactly index in the array since it's a new catalog + * Expand memory if necessary and create/allocate a new directory. + */ + current_index = parent_ptr->lsdo_last_child + 1; + if (current_index == 1 && parent_ptr->lsdo_child[0] == NULL) + current_index = 0; /* This is first catalog with 0 index */ + + if (current_index == parent_ptr->lsdo_child_max) + ls3_expand_num_of_dirs(parent_ptr); + + dir_ptr = parent_ptr->lsdo_child[current_index]; + if (dir_ptr == NULL) { + dir_ptr = ls3_stats_allocate_new_dir(m_dir_stats, NULL, + dir_name, depth); + parent_ptr->lsdo_child[current_index] = dir_ptr; + parent_ptr->lsdo_last_child = current_index; + } + + return dir_ptr; +} + +static void ls3_m_read_dir_in_tree(struct ls3_stats_dir_general *m_dir_stats, + json_object *j_dir, + struct ls3_stats_dir_obj *dir) +{ + json_object *j_total_size; + json_object *j_total_alloc_size; + json_object *j_uid; + json_object *j_projid; + json_object *j_gid; + json_object *j_fid; + json_object *j_files_cnt; + json_object *j_dirs_cnt; + json_object *j_child_dirs; + json_object *j_mtime; + json_object *j_atime; + json_object *j_ctime; + json_object *j_crtime; + int cnt_child = 0; + int i; + + if (!j_dir) + return; + + if (json_object_object_get_ex(j_dir, "SizeBytes", &j_total_size)) + dir->lsdo_size += json_object_get_int64(j_total_size); + + if (json_object_object_get_ex(j_dir, "AllocatedSizeBytes", + &j_total_alloc_size)) { + uint64_t alloc_size_in_sec = + json_object_get_int64(j_total_alloc_size) / 512; + + dir->lsdo_alloc_size_in_sectors += alloc_size_in_sec; + } + + if (json_object_object_get_ex(j_dir, "UserID", &j_uid)) + dir->lsdo_uid = json_object_get_int64(j_uid); + + if (json_object_object_get_ex(j_dir, "GroupID", &j_gid)) + dir->lsdo_gid = json_object_get_int64(j_gid); + + if (json_object_object_get_ex(j_dir, "ProjID", &j_projid)) + dir->lsdo_projid = json_object_get_int64(j_gid); + + if (json_object_object_get_ex(j_dir, "FilesCount", &j_files_cnt)) + dir->lsdo_files_count += json_object_get_int64(j_files_cnt); + + if (json_object_object_get_ex(j_dir, "DirsCount", &j_dirs_cnt)) + dir->lsdo_dirs_count += json_object_get_int64(j_dirs_cnt); + + if (json_object_object_get_ex(j_dir, "Atime", &j_atime)) { + if (dir->lsdo_atime < json_object_get_int64(j_atime)) + dir->lsdo_atime = json_object_get_int64(j_atime); + } + + if (json_object_object_get_ex(j_dir, "Mtime", &j_mtime)) { + if (dir->lsdo_mtime < json_object_get_int64(j_mtime)) + dir->lsdo_mtime = json_object_get_int64(j_mtime); + } + + if (json_object_object_get_ex(j_dir, "Ctime", &j_ctime)) { + if (dir->lsdo_ctime < json_object_get_int64(j_ctime)) + dir->lsdo_ctime = json_object_get_int64(j_ctime); + } + + if (json_object_object_get_ex(j_dir, "Crtime", &j_crtime)) { + if (dir->lsdo_crtime < json_object_get_int64(j_crtime)) + dir->lsdo_crtime = json_object_get_int64(j_crtime); + } + + if (json_object_object_get_ex(j_dir, "FID", &j_fid)) { + const char *fid_str = json_object_get_string(j_fid); + + ls3_m_upd_str2fid(fid_str, &dir->lsdo_fid); + } + + if (json_object_object_get_ex(j_dir, "ChildDirectories", &j_child_dirs)) + cnt_child = json_object_array_length(j_child_dirs); + + for (i = 0; i < cnt_child; i++) { + struct ls3_stats_dir_obj *child_dir_ptr; + json_object *j_dir_name; + json_object *j_depth; + json_object *j_child; + const char *dir_name; + + j_child = json_object_array_get_idx(j_child_dirs, i); + if (!j_child) { + continue; + } + + json_object_object_get_ex(j_child, "DirectoryName", &j_dir_name); + dir_name = json_object_get_string(j_dir_name); + + json_object_object_get_ex(j_child, "Depth", &j_depth); + child_dir_ptr = + ls3_m_get_child_dir_ptr(m_dir_stats, dir, dir_name, + json_object_get_int(j_depth)); + /* recursion */ + ls3_m_read_dir_in_tree(m_dir_stats, j_child, child_dir_ptr); + } + +} + +static int ls3_m_check_first_dir(struct ls3_stats_dir_general *m_dir_stats, + json_object *j_dirs) +{ + struct ls3_stats_dir_obj *new_dir_ptr; + json_object *j_first_dir; + json_object *j_total_size; + json_object *j_total_alloc_size; + json_object *j_r_limit; + json_object *j_max_depth; + const char *dir_name; + uint64_t alloc_size_in_sec; + + json_object_object_get_ex(j_dirs, "SourceDirectory", &j_first_dir); + json_object_object_get_ex(j_dirs, "TotalSizeBytes", &j_total_size); + json_object_object_get_ex(j_dirs, "TotalAllocatedSizeBytes", + &j_total_alloc_size); + + dir_name = json_object_get_string(j_first_dir); + alloc_size_in_sec = json_object_get_int64(j_total_alloc_size) / 512; + + if (m_dir_stats->lsdg_start_dir == NULL) { + new_dir_ptr = xcalloc(1, sizeof(struct ls3_stats_dir_obj)); + new_dir_ptr->lsdo_child_max = LS3_STATS_DIR_COUNT_BY_DEFAULT; + new_dir_ptr->lsdo_depth = -1; + new_dir_ptr->lsdo_size = json_object_get_int64(j_total_size); + new_dir_ptr->lsdo_name = xstrdup(dir_name); + new_dir_ptr->lsdo_alloc_size_in_sectors = alloc_size_in_sec; + new_dir_ptr->lsdo_child = (struct ls3_stats_dir_obj**)xcalloc( + new_dir_ptr->lsdo_child_max, + sizeof(struct ls3_stats_dir_obj*)); + + m_dir_stats->lsdg_start_dir = new_dir_ptr; + m_dir_stats->lsdg_start_path = xstrdup(""); + } else { + /* This is not the first report we have read. Just update */ + new_dir_ptr = m_dir_stats->lsdg_start_dir; + + if (strcmp(new_dir_ptr->lsdo_name, dir_name) != 0) { + LS3_ERROR("starting directory entry points do not match. expect:[%s] got:[%s]\n", + new_dir_ptr->lsdo_name, dir_name); + return -EINVAL; + } + + new_dir_ptr->lsdo_size += json_object_get_int64(j_total_size); + new_dir_ptr->lsdo_alloc_size_in_sectors += alloc_size_in_sec; + } + + if (json_object_object_get_ex(j_dirs, "TopRatingLimit", &j_r_limit)) + m_dir_stats->lsdg_top_rating_limit = + json_object_get_int(j_r_limit); + + if (json_object_object_get_ex(j_dirs, "MaxDepth", &j_max_depth)) + m_dir_stats->lsdg_max_depth = json_object_get_int(j_max_depth); + + return 0; +} + +static void ls3_m_parsing_dirs_tree(struct fstats_report *stats, + struct ls3_stats_dir_general *m_dir_stats, + json_object *j_dirs, const char *dev_name) +{ + json_object *j_main_tree; + json_object *j_child_dirs; + int dirs_cnt; + int rc; + + rc = ls3_m_check_first_dir(m_dir_stats, j_dirs); + if (rc) + return; /* Skip this report */ + + if (json_object_object_get_ex(j_dirs, "MainTree", &j_main_tree)) { + if (!json_object_object_get_ex(j_main_tree, "ChildDirectories", + &j_child_dirs)) + return; + + /* Get the number of directories with depth equal to 0. */ + dirs_cnt = json_object_array_length(j_child_dirs); + if (dirs_cnt == 0) { + LS3_WARNING("no directories found. device: %s", + dev_name); + /* It’s better to know this before we get into recursion + * Skip it. + */ + return; + } + + /* Recourse reading child dirs starting from the first dir*/ + ls3_m_read_dir_in_tree(m_dir_stats, j_main_tree, + m_dir_stats->lsdg_start_dir); + } +} + +static int ls3_m_parsing_json(struct fstats_report *stats, + struct ls3_stats_dir_general *m_dirs_stats, + FILE *fd, const char *path) { json_object *json; json_object *j_reports; json_object *ju_reports; + json_object *j_dir_stats; json_object *j_dev_name; + json_object *j_mount_point; const char *dev_name; - char *ftext; long length; + char *ftext; int i; + if (!fd || !stats || !m_dirs_stats) + return -1; + fseek(fd, 0, SEEK_END); length = ftell(fd); fseek(fd, 0, SEEK_SET); @@ -359,6 +619,15 @@ static void ls3_m_parsing_json(struct fstats_report *stats, FILE *fd, json_object_object_get_ex(json, "DeviceName", &j_dev_name); dev_name = json_object_get_string(j_dev_name); + if (json_object_object_get_ex(json, "ClientMountPath", &j_mount_point)) { + const char *mount_point; + + mount_point = json_object_get_string(j_mount_point); + stats->client_mount_path = xstrdup(mount_point); + } else { + stats->client_mount_path = xstrdup("MergedReport"); + } + if (json_object_object_get_ex(json, "Reports", &j_reports)) { int reports_cnt = json_object_array_length(j_reports); @@ -387,6 +656,12 @@ static void ls3_m_parsing_json(struct fstats_report *stats, FILE *fd, ls3_m_parsing_user_report(stats, user, dev_name); } } + + if (json_object_object_get_ex(json, "DirectoriesStats", &j_dir_stats)) + ls3_m_parsing_dirs_tree(stats, m_dirs_stats, j_dir_stats, + dev_name); + + return 0; } static FILE *ls3_m_open_json_file(const char *filename) @@ -416,8 +691,9 @@ static bool ls3_m_is_json_file(const char *filename) return false; } -void ls3_m_read_reports(struct fstats_report *stats, - const char *dir_with_reports) +static void ls3_m_read_reports(struct fstats_report *stats, + struct ls3_stats_dir_general *dirs_stats, + const char *dir_with_reports) { struct dirent *entry; DIR *dir; @@ -441,7 +717,10 @@ void ls3_m_read_reports(struct fstats_report *stats, sprintf(full_path, "%s/%s", dir_with_reports, entry->d_name); fd = ls3_m_open_json_file(full_path); - ls3_m_parsing_json(stats, fd, full_path); + if (!fd) + continue; /* Skip if NULL */ + + ls3_m_parsing_json(stats, dirs_stats, fd, full_path); fclose(fd); } } @@ -466,15 +745,29 @@ static struct fstats_report * ls3_m_report_init(void) return stats; } +static struct ls3_stats_dir_general *ls3_m_dirs_report_init(void) +{ + struct ls3_stats_dir_general *dirs_tree; + + /* This is the first and only place with initialization. */ + dirs_tree = xcalloc(1, sizeof(struct ls3_stats_dir_general)); + + return dirs_tree; +} + void ls3_stats_merge_reports(const char *dir_with_reports) { - struct fstats_report *main_report; - struct ls3_stats_dir_general *dirs_report = NULL; - - main_report = ls3_m_report_init(); - main_report->merging = true; - main_report->report_file_name = xstrdup("merged_report.all"); - ls3_m_read_reports(main_report, dir_with_reports); - ls3_stats_printf(main_report, dirs_report); - ls3_stats_destroy(main_report); + struct fstats_report *files_report; + struct ls3_stats_dir_general *dirs_report; + + files_report = ls3_m_report_init(); + dirs_report = ls3_m_dirs_report_init(); + + files_report->merging = true; + files_report->device_name = xstrdup("MergedReport"); + files_report->report_file_name = xstrdup("merged_report.all"); + dirs_report->lsdg_files_report = files_report; + ls3_m_read_reports(files_report, dirs_report, dir_with_reports); + ls3_stats_printf(files_report, dirs_report); + ls3_stats_destroy(files_report); } diff --git a/lipe/src/lipe_scan3/ls3_merge_stats.h b/lipe/src/lipe_scan3/ls3_merge_stats.h index 4605aff..a68d06c 100644 --- a/lipe/src/lipe_scan3/ls3_merge_stats.h +++ b/lipe/src/lipe_scan3/ls3_merge_stats.h @@ -10,9 +10,13 @@ #include #include #include +#include #include #include #include "ls3_stats.h" +#include "ls3_dir_stats.h" + +#define LS3_M_EMPTY_FID_STR "0:0x0:0x0" void ls3_stats_merge_reports(const char *dir_with_reports); -- 1.8.3.1