From 8bb38eef17b66d714efd869bd9aa6971fc891ae4 Mon Sep 17 00:00:00 2001 From: Vitaliy Kuznetsov Date: Wed, 8 May 2024 13:54:03 +0200 Subject: [PATCH] EX-9121 lipe: Add functionality for parsing ranges from JSON This patch is the third in a series of patches that implement functionality for combining size statistics reports and includes functionality for reading and saving the resulting ranges in tables obtained from different reports in JSON format. Only affects file size statistics and this patch is the final one for reports on file size statistics. Test-Parameters: trivial testlist=sanity-lipe-scan3,sanity-lipe-find3 Signed-off-by: Vitaliy Kuznetsov Change-Id: I9714f5ea970b103652f7714c93d76be2549ad3b8 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/55048 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- lipe/src/lipe_scan3/ls3_merge_stats.c | 83 +++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 4 deletions(-) diff --git a/lipe/src/lipe_scan3/ls3_merge_stats.c b/lipe/src/lipe_scan3/ls3_merge_stats.c index 0fe6f6c..b64a413 100644 --- a/lipe/src/lipe_scan3/ls3_merge_stats.c +++ b/lipe/src/lipe_scan3/ls3_merge_stats.c @@ -11,8 +11,29 @@ static struct range_report_template *ls3_m_alloc_new_id_range( struct report_template *report_ptr, int range_index, int64_t range_id) { - struct range_report_template *range_ptr = NULL; + struct range_report_template *range_ptr; + uint64_t current_index; + + pthread_mutex_lock(&report_ptr->report_template_mutex); + current_index = report_ptr->last_idx_in_fs_ranges + 1; + if (current_index == 1 && report_ptr->fs_ranges[0] == NULL) + current_index = 0; /* This is first UID/PID/PRJID */ + + if (current_index == report_ptr->count_ranges) + ls3_stats_expand_num_of_ranges(report_ptr, + report_ptr->count_ranges + + LS3_STATS_COUNT_RANGE_BY_DEFAULT); + + range_ptr = report_ptr->fs_ranges[current_index]; + if (!range_ptr) { + range_ptr = xcalloc(1, sizeof(struct range_report_template)); + range_ptr->rrt_min = DBL_MAX; + range_ptr->rrt_id = range_id; + report_ptr->fs_ranges[current_index] = range_ptr; + } + report_ptr->last_idx_in_fs_ranges = current_index; + pthread_mutex_unlock(&report_ptr->report_template_mutex); return range_ptr; } @@ -20,7 +41,19 @@ static struct range_report_template *ls3_m_alloc_new_range( struct report_template *report_ptr, int range_index, double start, double end) { - struct range_report_template *range_ptr = NULL; + struct range_report_template *range_ptr; + + pthread_mutex_lock(&report_ptr->report_template_mutex); + range_ptr = report_ptr->fs_ranges[range_index]; + if (!range_ptr) { + range_ptr = xcalloc(1, sizeof(struct range_report_template)); + range_ptr->rrt_min = DBL_MAX; + range_ptr->range_start = start; + range_ptr->range_end = end; + range_ptr->rrt_dev_cnt = -1; + report_ptr->fs_ranges[range_index] = range_ptr; + } + pthread_mutex_unlock(&report_ptr->report_template_mutex); return range_ptr; } @@ -31,7 +64,50 @@ static void ls3_m_parsing_range(struct range_report_template *range, const char *dev_name, bool user_report) { + json_object *j_count; + json_object *j_total; + json_object *j_min; + json_object *j_max; + json_object *j_atime; + double min; + double max; + uint64_t l_atime; + int last_dev_idx; + + if (user_report) { + json_object_object_get_ex(json, "CountFilesInRange", &j_count); + json_object_object_get_ex(json, "TotalSizeInRangeKB", &j_total); + json_object_object_get_ex(json, "MinSizeKB", &j_min); + json_object_object_get_ex(json, "MaxSizeKB", &j_max); + json_object_object_get_ex(json, "LastTimeAccess", &j_atime); + } else { + json_object_object_get_ex(json, "CountInRange", &j_count); + json_object_object_get_ex(json, "TotalInRange", &j_total); + json_object_object_get_ex(json, "MinValueInRange", &j_min); + json_object_object_get_ex(json, "MaxValueInRange", &j_max); + } + + range->count_in_range += json_object_get_int64(j_count); + range->total_in_range += json_object_get_int64(j_total); + min = json_object_get_int64(j_min); + max = json_object_get_int64(j_max); + last_dev_idx = range->rrt_dev_cnt + 1; + range->rrt_dev[last_dev_idx] = xstrdup(dev_name); + range->rrt_dev_cnt++; + + if (range->rrt_min > min) + range->rrt_min = min; + + if (range->rrt_max < max) + range->rrt_max = max; + + if (!user_report) + return; + + l_atime = json_object_get_int64(j_atime); + if (range->last_time_access < l_atime) + range->last_time_access = l_atime; } static ls3_stats_range_type ls3_m_get_range_type(ls3_stats_report_type r_type) @@ -91,7 +167,6 @@ static int64_t ls3_m_get_index_for_range(double range_start, int64_t range_id, return -1; } - static void ls3_m_parsing_report(struct report_template **r_template_ptr, json_object *report, const char *dev_name, bool user_report) @@ -397,7 +472,7 @@ void ls3_stats_merge_reports(const char *dir_with_reports) main_report = ls3_m_report_init(); main_report->merging = true; - + main_report->report_file_name = xstrdup("merged_report.all"); ls3_m_read_reports(main_report, dir_with_reports); ls3_stats_printf(main_report); ls3_stats_destroy(main_report); -- 1.8.3.1