Whamcloud - gitweb
EX-9121 lipe: Add functionality for parsing tables from JSON
authorVitaliy Kuznetsov <vkuznetsov@ddn.com>
Wed, 8 May 2024 15:00:41 +0000 (17:00 +0200)
committerAndreas Dilger <adilger@whamcloud.com>
Mon, 13 May 2024 22:34:39 +0000 (22:34 +0000)
This patch is the second in a series of patches that implement
functionality for combining reports on size statistics and
includes functionality for reading and recording tables obtained
from different reports in JSON format.
Only affects file size statistics.

Test-Parameters: trivial testlist=sanity-lipe-scan3,sanity-lipe-find3
Signed-off-by: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
Change-Id: I742879e61049e00b98d5f9defd7dabaea85fba0a
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/55047
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
lipe/src/lipe_scan3/ls3_merge_stats.c

index 6a37f7f..0fe6f6c 100644 (file)
 
 #include "ls3_merge_stats.h"
 
+
+static struct range_report_template *ls3_m_alloc_new_id_range(
+               struct report_template *report_ptr,
+               int range_index, int64_t range_id)
+{
+       struct range_report_template *range_ptr = NULL;
+
+       return range_ptr;
+}
+
+static struct range_report_template *ls3_m_alloc_new_range(
+               struct report_template *report_ptr,
+               int range_index, double start, double end)
+{
+       struct range_report_template *range_ptr = NULL;
+
+       return range_ptr;
+}
+
+
+static void ls3_m_parsing_range(struct range_report_template *range,
+                              json_object *json,
+                              const char *dev_name,
+                              bool user_report)
+{
+
+}
+
+static ls3_stats_range_type ls3_m_get_range_type(ls3_stats_report_type r_type)
+{
+
+       switch (r_type)
+       {
+       case LS3_STATS_TYPE_FILES_SIZE:
+       case LS3_STATS_TYPE_CAPACITY_USED:
+       case LS3_STATS_TYPE_EQUAL_OVERHEAD:
+       case LS3_STATS_TYPE_POSITIVE_OVERHEAD:
+       case LS3_STATS_TYPE_NEGATIVE_OVERHEAD:
+       case LS3_STATS_TYPE_DIRECTORY_SIZE_ENTRIES:
+       case LS3_STATS_TYPE_DIRECTORY_SIZE_KB:
+       case LS3_STATS_TYPE_LINK_COUNT:
+       case LS3_STATS_TYPE_FILENAME_LENGTH:
+       case LS3_STATS_TYPE_TIME_SINCE_LAST_MOD_RF:
+       case LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF:
+       case LS3_STATS_TYPE_TIME_SINCE_LAST_CREATION_RF:
+       case LS3_STATS_TYPE_TIME_SINCE_LAST_ACCESS_RF:
+               return LS3_STATS_RANGE_TYPE_INT;
+       case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_USER:
+       case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_GROUP:
+       case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_PROJID:
+       case LS3_STATS_TYPE_STRIPE_COUNT:
+       case LS3_STATS_TYPE_STRIPE_SIZE:
+       case LS3_STATS_TYPE_MIRROR_COUNT:
+               return LS3_STATS_RANGE_TYPE_ID;
+       case LS3_STATS_TYPE_COMPRESSION_RATIO:
+               return LS3_STATS_RANGE_TYPE_DOUBLE;
+       default:
+               return LS3_STATS_RANGE_TYPE_UNKNOWN;
+       }
+}
+
+static int64_t ls3_m_get_index_for_range(double range_start, int64_t range_id,
+                                        ls3_stats_range_type range_types)
+{
+       switch (range_types)
+       {
+       case LS3_STATS_RANGE_TYPE_INT:
+               if (range_start <= 0)
+                               return 0;
+
+               return (int64_t)log2(range_start);
+       case LS3_STATS_RANGE_TYPE_DOUBLE:
+               if (range_start <= 0.0)
+                       return 0;
+
+               return (int64_t)floor(range_start / 0.2);
+       case LS3_STATS_RANGE_TYPE_ID:
+               return range_id;
+       case  LS3_STATS_RANGE_TYPE_UNKNOWN:
+               return -1;
+       }
+
+       return -1;
+}
+
+
+static void ls3_m_parsing_report(struct report_template **r_template_ptr,
+                                json_object *report, const char *dev_name,
+                                bool user_report)
+{
+       struct report_template *mem_report;
+       json_object *general_info;
+       json_object *table_id;
+       json_object *ranges;
+       json_object *f_count;
+       json_object *min;
+       json_object *max;
+       json_object *total_value;
+       ls3_stats_range_type range_types;
+       int ranges_cnt = 0;
+       int table_idx;
+       int i;
+
+       if (user_report) {
+               json_object_object_get_ex(report, "UsersReportsTableId",
+                                         &table_id);
+       } else {
+               json_object_object_get_ex(report, "GeneralInfo", &general_info);
+               json_object_object_get_ex(general_info, "TableId", &table_id);
+               json_object_object_get_ex(general_info, "Count", &f_count);
+               json_object_object_get_ex(general_info, "TotalValue",
+                                         &total_value);
+               json_object_object_get_ex(general_info, "Min", &min);
+               json_object_object_get_ex(general_info, "Max", &max);
+       }
+
+       json_object_object_get_ex(report, "Ranges", &ranges);
+       ranges_cnt = json_object_array_length(ranges);
+
+       /* Update general info */
+       table_idx = json_object_get_int(table_id);
+       mem_report = r_template_ptr[table_idx];
+       if (!mem_report)
+               LS3_FATAL("Table %d not found in memory\n", table_idx);
+
+       if (user_report) {
+               range_types = LS3_STATS_RANGE_TYPE_INT;
+       } else {
+               range_types = ls3_m_get_range_type(table_idx);
+               mem_report->files_count += json_object_get_int64(f_count);
+               mem_report->total_value += json_object_get_int64(total_value);
+
+               if (json_object_get_double(min) < mem_report->rt_min)
+                       mem_report->rt_min = json_object_get_double(min);
+
+               if (json_object_get_double(max) > mem_report->rt_max)
+                       mem_report->rt_max = json_object_get_double(max);
+       }
+
+       /* Update ranges */
+       for (i = 0; i < ranges_cnt; i++) {
+               struct range_report_template *range_ptr;
+               json_object *json_range;
+               json_object *json_r_start;
+               json_object *json_r_end;
+               json_object *json_r_id;
+               double range_start;
+               double range_end;
+               uint64_t range_id = 0;
+               int64_t range_index;
+               bool ptr_found = false;
+               int j;
+
+               json_range = json_object_array_get_idx(ranges, i);
+               if (user_report) {
+                       json_object_object_get_ex(json_range, "RangeDayStart",
+                                                 &json_r_start);
+                       json_object_object_get_ex(json_range, "RangeDayEnd",
+                                                 &json_r_end);
+
+               } else {
+                       json_object_object_get_ex(json_range,
+                                                 "RangeStart", &json_r_start);
+                       json_object_object_get_ex(json_range, "RangeEnd",
+                                                 &json_r_end);
+                       json_object_object_get_ex(json_range, "RangeID",
+                                                 &json_r_id);
+               }
+
+               range_start = json_object_get_double(json_r_start);
+               range_end = json_object_get_double(json_r_end);
+               range_index = ls3_m_get_index_for_range(range_start, range_id,
+                                                       range_types);
+
+               for(j = 0; j < mem_report->count_ranges; j++) {
+                       if (range_types == LS3_STATS_RANGE_TYPE_ID) {
+                               range_ptr =
+                                       ls3_stats_find_id_in_ranges(range_id,
+                                                                   mem_report);
+
+                               if (!range_ptr)
+                                       break;
+                       } else {
+                               if (range_index > mem_report->count_ranges - 1)
+                                       ls3_stats_expand_num_of_ranges(mem_report,
+                                                               range_index);
+
+                               range_ptr = mem_report->fs_ranges[range_index];
+                       }
+
+                       if (!range_ptr)
+                               continue;
+
+                       ls3_m_parsing_range(range_ptr, json_range,
+                                           dev_name, user_report);
+                       ptr_found = true;
+                       break;
+               }
+
+               if (ptr_found) /* All updated */
+                       continue;
+
+               if (range_types == LS3_STATS_RANGE_TYPE_ID)
+                       range_ptr = ls3_m_alloc_new_id_range(mem_report,
+                                                            range_index,
+                                                            range_id);
+               else
+                       range_ptr = ls3_m_alloc_new_range(mem_report,
+                                                         range_index,
+                                                         range_start,
+                                                         range_end);
+
+               ls3_m_parsing_range(range_ptr, json_range,
+                                   dev_name, user_report);
+       }
+}
+
+static void ls3_m_parsing_user_report(struct fstats_report *stats,
+                                     json_object *user, const char *dev_name)
+{
+       struct ls3_stats_user_report_template *mem_user_report;
+       json_object *j_tables;
+       json_object *j_uid;
+       uint64_t uid;
+       int i;
+
+       json_object_object_get_ex(user, "UserUID", &j_uid);
+       uid = json_object_get_int64(j_uid);
+       mem_user_report = ls3_stats_find_user_reports(stats, uid);
+       if (!mem_user_report) {
+               pthread_mutex_lock(&stats->user_rt_mutex);
+               mem_user_report = ls3_stats_add_new_user(stats, uid);
+               pthread_mutex_unlock(&stats->user_rt_mutex);
+       }
+
+       if (json_object_object_get_ex(user, "Tables", &j_tables)) {
+               int reports_cnt = json_object_array_length(j_tables);
+
+               for (i = 0; i < reports_cnt; i++) {
+                       json_object *report;
+
+                       report = json_object_array_get_idx(j_tables, i);
+                       if (!report)
+                               continue;
+
+                       ls3_m_parsing_report(mem_user_report->user_reports,
+                                            report, dev_name, true);
+               }
+       }
+}
+
 static void ls3_m_parsing_json(struct fstats_report *stats, FILE *fd,
                               const char *path)
 {
+       json_object *json;
+       json_object *j_reports;
+       json_object *ju_reports;
+       json_object *j_dev_name;
+       const char *dev_name;
+       char *ftext;
+       long length;
+       int i;
+
+       fseek(fd, 0, SEEK_END);
+       length = ftell(fd);
+       fseek(fd, 0, SEEK_SET);
+       ftext = (char *)xcalloc(length + 1, sizeof(char));
+       fread(ftext, 1, length, fd);
+       ftext[length] = '\0';
+
+       json = json_tokener_parse(ftext);
+       if (!json)
+               LS3_FATAL("error with parse JSON: %s\n", path);
+
+       free(ftext);
+       json_object_object_get_ex(json, "DeviceName", &j_dev_name);
+       dev_name = json_object_get_string(j_dev_name);
 
+       if (json_object_object_get_ex(json, "Reports", &j_reports)) {
+               int reports_cnt = json_object_array_length(j_reports);
+
+               for (i = 0; i < reports_cnt; i++) {
+                       json_object *report;
+
+                       report = json_object_array_get_idx(j_reports, i);
+                       if (!report)
+                               continue;
+
+                       ls3_m_parsing_report(stats->reports, report, dev_name,
+                                            false);
+               }
+       }
+
+       if (json_object_object_get_ex(json, "UserTimeReports", &ju_reports)) {
+               int users_cnt = json_object_array_length(ju_reports);
+
+               for (i = 0; i < users_cnt; i++) {
+                       json_object *user;
+
+                       user = json_object_array_get_idx(ju_reports, i);
+                       if (!user)
+                               continue;
+
+                       ls3_m_parsing_user_report(stats, user, dev_name);
+               }
+       }
 }
 
 static FILE *ls3_m_open_json_file(const char *filename)