From 1830cebc6616b8aacfa3dd916ea315369beed115 Mon Sep 17 00:00:00 2001 From: Vitaliy Kuznetsov Date: Wed, 8 May 2024 17:00:41 +0200 Subject: [PATCH] EX-9121 lipe: Add functionality for parsing tables from JSON This patch is the second in a series of patches that implement functionality for combining reports on size statistics and includes functionality for reading and recording tables obtained from different reports in JSON format. Only affects file size statistics. Test-Parameters: trivial testlist=sanity-lipe-scan3,sanity-lipe-find3 Signed-off-by: Vitaliy Kuznetsov Change-Id: I742879e61049e00b98d5f9defd7dabaea85fba0a Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/55047 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- lipe/src/lipe_scan3/ls3_merge_stats.c | 302 ++++++++++++++++++++++++++++++++++ 1 file changed, 302 insertions(+) diff --git a/lipe/src/lipe_scan3/ls3_merge_stats.c b/lipe/src/lipe_scan3/ls3_merge_stats.c index 6a37f7f..0fe6f6c 100644 --- a/lipe/src/lipe_scan3/ls3_merge_stats.c +++ b/lipe/src/lipe_scan3/ls3_merge_stats.c @@ -6,10 +6,312 @@ #include "ls3_merge_stats.h" + +static struct range_report_template *ls3_m_alloc_new_id_range( + struct report_template *report_ptr, + int range_index, int64_t range_id) +{ + struct range_report_template *range_ptr = NULL; + + return range_ptr; +} + +static struct range_report_template *ls3_m_alloc_new_range( + struct report_template *report_ptr, + int range_index, double start, double end) +{ + struct range_report_template *range_ptr = NULL; + + return range_ptr; +} + + +static void ls3_m_parsing_range(struct range_report_template *range, + json_object *json, + const char *dev_name, + bool user_report) +{ + +} + +static ls3_stats_range_type ls3_m_get_range_type(ls3_stats_report_type r_type) +{ + + switch (r_type) + { + case LS3_STATS_TYPE_FILES_SIZE: + case LS3_STATS_TYPE_CAPACITY_USED: + case LS3_STATS_TYPE_EQUAL_OVERHEAD: + case LS3_STATS_TYPE_POSITIVE_OVERHEAD: + case LS3_STATS_TYPE_NEGATIVE_OVERHEAD: + case LS3_STATS_TYPE_DIRECTORY_SIZE_ENTRIES: + case LS3_STATS_TYPE_DIRECTORY_SIZE_KB: + case LS3_STATS_TYPE_LINK_COUNT: + case LS3_STATS_TYPE_FILENAME_LENGTH: + case LS3_STATS_TYPE_TIME_SINCE_LAST_MOD_RF: + case LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF: + case LS3_STATS_TYPE_TIME_SINCE_LAST_CREATION_RF: + case LS3_STATS_TYPE_TIME_SINCE_LAST_ACCESS_RF: + return LS3_STATS_RANGE_TYPE_INT; + case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_USER: + case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_GROUP: + case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_PROJID: + case LS3_STATS_TYPE_STRIPE_COUNT: + case LS3_STATS_TYPE_STRIPE_SIZE: + case LS3_STATS_TYPE_MIRROR_COUNT: + return LS3_STATS_RANGE_TYPE_ID; + case LS3_STATS_TYPE_COMPRESSION_RATIO: + return LS3_STATS_RANGE_TYPE_DOUBLE; + default: + return LS3_STATS_RANGE_TYPE_UNKNOWN; + } +} + +static int64_t ls3_m_get_index_for_range(double range_start, int64_t range_id, + ls3_stats_range_type range_types) +{ + switch (range_types) + { + case LS3_STATS_RANGE_TYPE_INT: + if (range_start <= 0) + return 0; + + return (int64_t)log2(range_start); + case LS3_STATS_RANGE_TYPE_DOUBLE: + if (range_start <= 0.0) + return 0; + + return (int64_t)floor(range_start / 0.2); + case LS3_STATS_RANGE_TYPE_ID: + return range_id; + case LS3_STATS_RANGE_TYPE_UNKNOWN: + return -1; + } + + return -1; +} + + +static void ls3_m_parsing_report(struct report_template **r_template_ptr, + json_object *report, const char *dev_name, + bool user_report) +{ + struct report_template *mem_report; + json_object *general_info; + json_object *table_id; + json_object *ranges; + json_object *f_count; + json_object *min; + json_object *max; + json_object *total_value; + ls3_stats_range_type range_types; + int ranges_cnt = 0; + int table_idx; + int i; + + if (user_report) { + json_object_object_get_ex(report, "UsersReportsTableId", + &table_id); + } else { + json_object_object_get_ex(report, "GeneralInfo", &general_info); + json_object_object_get_ex(general_info, "TableId", &table_id); + json_object_object_get_ex(general_info, "Count", &f_count); + json_object_object_get_ex(general_info, "TotalValue", + &total_value); + json_object_object_get_ex(general_info, "Min", &min); + json_object_object_get_ex(general_info, "Max", &max); + } + + json_object_object_get_ex(report, "Ranges", &ranges); + ranges_cnt = json_object_array_length(ranges); + + /* Update general info */ + table_idx = json_object_get_int(table_id); + mem_report = r_template_ptr[table_idx]; + if (!mem_report) + LS3_FATAL("Table %d not found in memory\n", table_idx); + + if (user_report) { + range_types = LS3_STATS_RANGE_TYPE_INT; + } else { + range_types = ls3_m_get_range_type(table_idx); + mem_report->files_count += json_object_get_int64(f_count); + mem_report->total_value += json_object_get_int64(total_value); + + if (json_object_get_double(min) < mem_report->rt_min) + mem_report->rt_min = json_object_get_double(min); + + if (json_object_get_double(max) > mem_report->rt_max) + mem_report->rt_max = json_object_get_double(max); + } + + /* Update ranges */ + for (i = 0; i < ranges_cnt; i++) { + struct range_report_template *range_ptr; + json_object *json_range; + json_object *json_r_start; + json_object *json_r_end; + json_object *json_r_id; + double range_start; + double range_end; + uint64_t range_id = 0; + int64_t range_index; + bool ptr_found = false; + int j; + + json_range = json_object_array_get_idx(ranges, i); + if (user_report) { + json_object_object_get_ex(json_range, "RangeDayStart", + &json_r_start); + json_object_object_get_ex(json_range, "RangeDayEnd", + &json_r_end); + + } else { + json_object_object_get_ex(json_range, + "RangeStart", &json_r_start); + json_object_object_get_ex(json_range, "RangeEnd", + &json_r_end); + json_object_object_get_ex(json_range, "RangeID", + &json_r_id); + } + + range_start = json_object_get_double(json_r_start); + range_end = json_object_get_double(json_r_end); + range_index = ls3_m_get_index_for_range(range_start, range_id, + range_types); + + for(j = 0; j < mem_report->count_ranges; j++) { + if (range_types == LS3_STATS_RANGE_TYPE_ID) { + range_ptr = + ls3_stats_find_id_in_ranges(range_id, + mem_report); + + if (!range_ptr) + break; + } else { + if (range_index > mem_report->count_ranges - 1) + ls3_stats_expand_num_of_ranges(mem_report, + range_index); + + range_ptr = mem_report->fs_ranges[range_index]; + } + + if (!range_ptr) + continue; + + ls3_m_parsing_range(range_ptr, json_range, + dev_name, user_report); + ptr_found = true; + break; + } + + if (ptr_found) /* All updated */ + continue; + + if (range_types == LS3_STATS_RANGE_TYPE_ID) + range_ptr = ls3_m_alloc_new_id_range(mem_report, + range_index, + range_id); + else + range_ptr = ls3_m_alloc_new_range(mem_report, + range_index, + range_start, + range_end); + + ls3_m_parsing_range(range_ptr, json_range, + dev_name, user_report); + } +} + +static void ls3_m_parsing_user_report(struct fstats_report *stats, + json_object *user, const char *dev_name) +{ + struct ls3_stats_user_report_template *mem_user_report; + json_object *j_tables; + json_object *j_uid; + uint64_t uid; + int i; + + json_object_object_get_ex(user, "UserUID", &j_uid); + uid = json_object_get_int64(j_uid); + mem_user_report = ls3_stats_find_user_reports(stats, uid); + if (!mem_user_report) { + pthread_mutex_lock(&stats->user_rt_mutex); + mem_user_report = ls3_stats_add_new_user(stats, uid); + pthread_mutex_unlock(&stats->user_rt_mutex); + } + + if (json_object_object_get_ex(user, "Tables", &j_tables)) { + int reports_cnt = json_object_array_length(j_tables); + + for (i = 0; i < reports_cnt; i++) { + json_object *report; + + report = json_object_array_get_idx(j_tables, i); + if (!report) + continue; + + ls3_m_parsing_report(mem_user_report->user_reports, + report, dev_name, true); + } + } +} + static void ls3_m_parsing_json(struct fstats_report *stats, FILE *fd, const char *path) { + json_object *json; + json_object *j_reports; + json_object *ju_reports; + json_object *j_dev_name; + const char *dev_name; + char *ftext; + long length; + int i; + + fseek(fd, 0, SEEK_END); + length = ftell(fd); + fseek(fd, 0, SEEK_SET); + ftext = (char *)xcalloc(length + 1, sizeof(char)); + fread(ftext, 1, length, fd); + ftext[length] = '\0'; + + json = json_tokener_parse(ftext); + if (!json) + LS3_FATAL("error with parse JSON: %s\n", path); + + free(ftext); + json_object_object_get_ex(json, "DeviceName", &j_dev_name); + dev_name = json_object_get_string(j_dev_name); + if (json_object_object_get_ex(json, "Reports", &j_reports)) { + int reports_cnt = json_object_array_length(j_reports); + + for (i = 0; i < reports_cnt; i++) { + json_object *report; + + report = json_object_array_get_idx(j_reports, i); + if (!report) + continue; + + ls3_m_parsing_report(stats->reports, report, dev_name, + false); + } + } + + if (json_object_object_get_ex(json, "UserTimeReports", &ju_reports)) { + int users_cnt = json_object_array_length(ju_reports); + + for (i = 0; i < users_cnt; i++) { + json_object *user; + + user = json_object_array_get_idx(ju_reports, i); + if (!user) + continue; + + ls3_m_parsing_user_report(stats, user, dev_name); + } + } } static FILE *ls3_m_open_json_file(const char *filename) -- 1.8.3.1