EX-9121 lipe: Trivial improvements for report merging

Minor changes that do not affect functionality.
Fixed memory leaks when merging reports, and excessive
memory allocation when generating JSON type reports when
there are a large number of ranges.

Test-Parameters: trivial testlist=sanity-lipe-scan3,sanity-lipe-find3
Signed-off-by: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
Change-Id: I75c580f53246301d6262c6f5c5db271d36b2ec75
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/55393
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lipe/src/lipe_scan3/ls3_merge_stats.c b/lipe/src/lipe_scan3/ls3_merge_stats.c
index f021675..a4f4361 100644 (file)
--- a/lipe/src/lipe_scan3/ls3_merge_stats.c
+++ b/lipe/src/lipe_scan3/ls3_merge_stats.c
@@ -66,7 +66,6 @@ static struct range_report_template *ls3_m_alloc_new_range(
                range_ptr->rrt_min = DBL_MAX;
                range_ptr->range_start = start;
                range_ptr->range_end = end;
-               range_ptr->rrt_dev_cnt = -1;
                report_ptr->fs_ranges[range_index] = range_ptr;
        }
        pthread_mutex_unlock(&report_ptr->report_template_mutex);
@@ -769,5 +768,6 @@ void ls3_stats_merge_reports(const char *dir_with_reports)
        dirs_report->lsdg_files_report = files_report;
        ls3_m_read_reports(files_report, dirs_report, dir_with_reports);
        ls3_stats_printf(files_report, dirs_report);
+       ls3_stats_dir_destroy(dirs_report);
        ls3_stats_destroy(files_report);
 }

diff --git a/lipe/src/lipe_scan3/ls3_stats.c b/lipe/src/lipe_scan3/ls3_stats.c
index 21df12a..c0436e8 100644 (file)
--- a/lipe/src/lipe_scan3/ls3_stats.c
+++ b/lipe/src/lipe_scan3/ls3_stats.c
@@ -343,16 +343,16 @@ static const ls3_stats_out_mapping ls3_out_mappings[] = {
                        "of total size of dirs;\n"
         "min_size    - Minimum directory size in range;\n"
         "max_size    - Maximum directory size in range.\n",
-        "____directory_size_range_____ ___number___ pct_num cpt_num total_size"
+        "____dir_size_range_____ ___number___ pct_num cpt_num total_size"
         " pct_rng cpt_rng _min_size_ _max_size_\n",
         "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n",
         "General information:\n"
-        "Total number of directory: %lu\n"
+        "Total number of directory: %lu (Within the specified depth)\n"
         "Number of skipped directory: %u\n"
         "Minimum directory size (in blocks): %s\n"
         "Maximum directory size (in blocks): %s\n"
         "Average directory size (in blocks): %s\n"
-        "Total capacity used of all directory: %s\n",
+        "Total capacity used of all directory (in blocks): %s\n",
         LS3_STATS_TYPE_DIRECTORY_SIZE_KB},
        {"Link count (of regular files)",
         "Description:\n"
@@ -818,6 +818,8 @@ static uint32_t ls3_stats_get_count_skipped_obj(struct fstats_report *report,
                                           LS3_STATS_ERROR_EMPTY_BLOCK_SIZE);
        case LS3_STATS_TYPE_FILENAME_LENGTH:
                return LS3_GET_ERROR_COUNT(report, LS3_STATS_ERROR_WITHOUT_SIZE);
+       case LS3_STATS_TYPE_DIRECTORY_SIZE_KB:
+               return 0;
        default:
                return LS3_GET_ERROR_COUNT(report,
                                           LS3_STATS_ERROR_EMPTY_BLOCK_SIZE);
@@ -1039,8 +1041,10 @@ static char *ls3_stats_get_str_value_type(ls3_stats_val_type num_type)
        return "";
 }
 
-static void ls3_stats_get_range_str(struct range_report_template *range_ptr,
-       char *str, size_t strlen, ls3_stats_report_type r_type)
+static void ls3_stats_get_range_str(struct fstats_report *report,
+                                   struct range_report_template *range_ptr,
+                                   char *str, size_t strlen,
+                                   ls3_stats_report_type r_type)
 {
        /* Different type of "range" field */
        switch (r_type)
@@ -1091,6 +1095,7 @@ static void ls3_stats_get_range_str(struct range_report_template *range_ptr,
                         range_ptr->range_end);
                break;
        case LS3_STATS_TYPE_FILES_SIZE:
+       case LS3_STATS_TYPE_DIRECTORY_SIZE_KB:
        case LS3_STATS_TYPE_POSITIVE_OVERHEAD:
        case LS3_STATS_TYPE_NEGATIVE_OVERHEAD:
        case LS3_STATS_TYPE_EQUAL_OVERHEAD:
@@ -1262,8 +1267,7 @@ static void ls3_stats_print_to_json(struct fstats_report *report,
                                               second_value_ts : value_ts));
 
                for (j = 0; j < report_ptr->count_ranges; j++) {
-                       struct json_object *jobj_range =
-                               json_object_new_object();
+                       struct json_object *jobj_range;
                        char range_t[30];       /* size in 30 byte enough */
 
                        range_ptr = report_ptr->fs_ranges[j];
@@ -1273,7 +1277,8 @@ static void ls3_stats_print_to_json(struct fstats_report *report,
                        if (range_ptr->count_in_range == 0)
                                continue;       /* Nothing for print */
 
-                       ls3_stats_get_range_str(range_ptr, range_t,
+                       jobj_range = json_object_new_object();
+                       ls3_stats_get_range_str(report, range_ptr, range_t,
                                sizeof(range_t), i);
 
                        json_object_object_add(jobj_range, "RangePosition",
@@ -1354,8 +1359,8 @@ static void ls3_stats_print_to_json(struct fstats_report *report,
                                       json_object_new_int64(u_report_ptr->uid));
 
                for (j = 0; j < LS3_STATS_USERS_REPORTS_COUNTER; j++) {
-                       struct json_object *jobj_table = json_object_new_object();
-                       struct json_object *jobj_ranges = json_object_new_array();
+                       struct json_object *jobj_table;
+                       struct json_object *jobj_ranges;
                        char *title_text;
 
                        title_text = ls3_stats_get_title_for_user_report(j);
@@ -1363,20 +1368,22 @@ static void ls3_stats_print_to_json(struct fstats_report *report,
                        if (report_ptr == NULL)
                                LS3_FATAL("Unable to access the user report data: %d\n", j);
 
+                       jobj_table = json_object_new_object();
                        json_object_object_add(jobj_table, "Title",
                                               json_object_new_string(title_text));
 
                        json_object_object_add(jobj_table, "UsersReportsTableId",
                                               json_object_new_int(j));
 
+                       jobj_ranges = json_object_new_array();
                        for (k = 0; k < report_ptr->count_ranges; k++) {
-                               struct json_object *jobj_range =
-                                       json_object_new_object();
+                               struct json_object *jobj_range;
 
                                range_ptr = report_ptr->fs_ranges[k];
                                if (range_ptr == NULL)
                                        continue;
 
+                               jobj_range = json_object_new_object();
                                json_object_object_add(jobj_range,
                                                       "RangeDayStart",
                                                       json_object_new_int64(
@@ -1523,7 +1530,7 @@ static void ls3_stats_print_to_yaml(struct fstats_report *report,
                                continue;       /* Nothing for print */
 
                        /* Different type of "range" field */
-                       ls3_stats_get_range_str(range_ptr, range_t,
+                       ls3_stats_get_range_str(report, range_ptr, range_t,
                                                sizeof(range_t), i);
 
                        fprintf(yaml_fd,
@@ -1688,7 +1695,7 @@ static void ls3_stats_print_to_out(struct fstats_report *report,
                                continue;       /* Nothing for print */
 
                        /* Different type of "range" field */
-                       ls3_stats_get_range_str(range_ptr, range_t,
+                       ls3_stats_get_range_str(report, range_ptr, range_t,
                                                sizeof(range_t), i);
                        srv_str = ls3_stats_get_range_val_str(report_ptr,
                                                              range_ptr,
@@ -1856,7 +1863,7 @@ static void ls3_stats_print_to_csv(struct fstats_report *report,
                                continue;       /* Nothing for print */
 
                        /* Different type of "range" field */
-                       ls3_stats_get_range_str(range_ptr, range_t,
+                       ls3_stats_get_range_str(report, range_ptr, range_t,
                                                sizeof(range_t), i);
                        fprintf(csv_fd, "%s,%lu,%.2f,%.2f,%lu %s,%.2f,%.2f,"
                                "%.2f %s,%.2f %s\n",
@@ -2048,7 +2055,6 @@ struct range_report_template *ls3_stats_get_new_range_ptr(uint64_t res)
 
        range_ptr = xcalloc(1, sizeof(struct range_report_template));
        range_ptr->rrt_min = DBL_MAX;
-       range_ptr->rrt_dev_cnt = -1;
        if (res >= 2) {
                range_ptr->range_start = (int)pow(2, floor(log2(res)));
                range_ptr->range_end = range_ptr->range_start * 2;
@@ -2066,7 +2072,6 @@ static struct range_report_template *ls3_stats_get_new_range_double_ptr(double r
 
        range_ptr = xcalloc(1, sizeof(struct range_report_template));
        range_ptr->rrt_min = DBL_MAX;
-       range_ptr->rrt_dev_cnt = -1;
        /* 0.001 it's rounding for exception handling with integers */
        range_ptr->range_start = floor((res + 0.001) / 0.2) * 0.2;
        range_ptr->range_end = range_ptr->range_start + 0.2;
@@ -2153,17 +2158,18 @@ static void ls3_stats_update_range_double(ls3_stats_report_type report_type,
        pthread_mutex_unlock(&report_ptr->report_template_mutex);
 }
 
-static void ls3_stats_update_range(ls3_stats_report_type report_type,
-       int64_t range_value, int64_t second_value)
+void ls3_stats_update_range(struct fstats_report *g_report,
+                           ls3_stats_report_type report_type,
+                           int64_t range_value, int64_t second_value)
 {
-       int index_range;
        struct report_template *report_ptr;
        struct range_report_template *range_ptr;
        ls3_stats_val_type value_type;
        uint64_t result_range_value;
        uint64_t result_second_value = 0;
+       int index_range;
 
-       report_ptr = reports_with_stats->reports[report_type];
+       report_ptr = g_report->reports[report_type];
        if (!report_ptr)
                LS3_FATAL("Unable to access the report data: %d\n", report_type);
 
@@ -2567,7 +2573,7 @@ int llapi_layout_compress_get(struct llapi_layout *layout,
 int ls3_stats_update_info(struct ls3_object_attrs *loa_all)
 {
        struct lipe_path_entry *lpe;
-       const char *file_name;
+       const char *file_name = NULL;
        uint64_t allocate_file_size = 0;
        int64_t cmps_fsize = 0;
        double cmps_ratio = 0;
@@ -2608,13 +2614,14 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all)
         * and special files) */
        lipe_list_for_each_entry(lpe, &loa_all->loa_paths, lpe_linkage) {
                file_name = ls3_stats_get_filename_from_path(lpe->lpe_path);
-               ls3_stats_update_range(LS3_STATS_TYPE_FILENAME_LENGTH,
+               ls3_stats_update_range(reports_with_stats,
+                                      LS3_STATS_TYPE_FILENAME_LENGTH,
                                       (int16_t)strlen(file_name),
                                       LS3_STATS_EMPTY_VALUE);
        }
 
-       ls3_stats_update_range(LS3_STATS_TYPE_FILES_SIZE, loa_all->loa_size,
-                              LS3_STATS_EMPTY_VALUE);
+       ls3_stats_update_range(reports_with_stats, LS3_STATS_TYPE_FILES_SIZE,
+                              loa_all->loa_size, LS3_STATS_EMPTY_VALUE);
 
        if (allocate_file_size == 0)
                return 0;
@@ -2622,32 +2629,42 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all)
        if (loa_all->loa_size != 0 &&
            allocate_file_size >= loa_all->loa_size - 4096 &&
            allocate_file_size <= loa_all->loa_size + 4096)
-               ls3_stats_update_range(LS3_STATS_TYPE_EQUAL_OVERHEAD,
-                                      allocate_file_size, LS3_STATS_EMPTY_VALUE);
+               ls3_stats_update_range(reports_with_stats,
+                                      LS3_STATS_TYPE_EQUAL_OVERHEAD,
+                                      allocate_file_size,
+                                      LS3_STATS_EMPTY_VALUE);
 
        /* Get positive overhead (Files whose capacity used is
         * larger than to its size. */
-       else if (loa_all->loa_size != 0 && allocate_file_size > loa_all->loa_size)
-               ls3_stats_update_range(LS3_STATS_TYPE_POSITIVE_OVERHEAD,
-                                       allocate_file_size - loa_all->loa_size,
-                                       LS3_STATS_EMPTY_VALUE);
+       else if (loa_all->loa_size != 0 &&
+                allocate_file_size > loa_all->loa_size)
+               ls3_stats_update_range(reports_with_stats,
+                                      LS3_STATS_TYPE_POSITIVE_OVERHEAD,
+                                      allocate_file_size - loa_all->loa_size,
+                                      LS3_STATS_EMPTY_VALUE);
        /* Get negative overhead (Files whose size is larger than
         * capacity used. Includes directories, symlinks and
         * regular files). */
        else if (loa_all->loa_size != 0)
-               ls3_stats_update_range(LS3_STATS_TYPE_NEGATIVE_OVERHEAD,
+               ls3_stats_update_range(reports_with_stats,
+                                      LS3_STATS_TYPE_NEGATIVE_OVERHEAD,
                                       loa_all->loa_size - allocate_file_size,
                                       LS3_STATS_EMPTY_VALUE);
 
-       ls3_stats_update_range(LS3_STATS_TYPE_CAPACITY_USED, allocate_file_size,
+       ls3_stats_update_range(reports_with_stats,
+                              LS3_STATS_TYPE_CAPACITY_USED, allocate_file_size,
                               LS3_STATS_EMPTY_VALUE);
-       ls3_stats_update_range(LS3_STATS_TYPE_TIME_SINCE_LAST_MOD_RF,
+       ls3_stats_update_range(reports_with_stats,
+                              LS3_STATS_TYPE_TIME_SINCE_LAST_MOD_RF,
                               loa_all->loa_mtime, allocate_file_size);
-       ls3_stats_update_range(LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF,
+       ls3_stats_update_range(reports_with_stats,
+                              LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF,
                               loa_all->loa_ctime, allocate_file_size);
-       ls3_stats_update_range(LS3_STATS_TYPE_TIME_SINCE_LAST_CREATION_RF,
+       ls3_stats_update_range(reports_with_stats,
+                              LS3_STATS_TYPE_TIME_SINCE_LAST_CREATION_RF,
                               loa_all->loa_crtime, allocate_file_size);
-       ls3_stats_update_range(LS3_STATS_TYPE_TIME_SINCE_LAST_ACCESS_RF,
+       ls3_stats_update_range(reports_with_stats,
+                              LS3_STATS_TYPE_TIME_SINCE_LAST_ACCESS_RF,
                               loa_all->loa_atime, allocate_file_size);
        ls3_stats_update_range_with_id(LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_USER,
                                       loa_all->loa_uid,
@@ -2743,7 +2760,8 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all)
                ptr_filter_fid = loa_all->loa_filter_fid;
                layout_compr = ptr_filter_fid.ff_layout_compr;
                ptr_ost_layout = ptr_filter_fid.ff_layout;
-               ls3_stats_update_range(LS3_STATS_TYPE_STRIPE_COUNT,
+               ls3_stats_update_range(reports_with_stats,
+                                      LS3_STATS_TYPE_STRIPE_COUNT,
                                       ptr_ost_layout.ol_stripe_count,
                                       LS3_STATS_EMPTY_VALUE);
 
@@ -2813,14 +2831,6 @@ void ls3_stats_destroy(struct fstats_report *report)
                        if (range_ptr == NULL)
                                continue;
 
-                       if (range_ptr->rrt_dev_cnt >= 0) {
-                               for(k = 0; k < LS3_STATS_ARRAY_SIZE_BY_DEFAULT;
-                                   k++) {
-                                       if (range_ptr->rrt_dev[k])
-                                               free(range_ptr->rrt_dev[k]);
-                               }
-                       }
-
                        free(range_ptr);
                }
                pthread_mutex_destroy(&report_ptr->report_template_mutex);

diff --git a/lipe/src/lipe_scan3/ls3_stats.h b/lipe/src/lipe_scan3/ls3_stats.h
index 11e1424..ba9c661 100644 (file)
--- a/lipe/src/lipe_scan3/ls3_stats.h
+++ b/lipe/src/lipe_scan3/ls3_stats.h
@@ -149,8 +149,6 @@ struct range_report_template {
        double cumulative_percentage;
        double percent_in_range;
        double percent_cumulative_in_range;
-       int32_t rrt_dev_cnt;
-       char *rrt_dev[LS3_STATS_ARRAY_SIZE_BY_DEFAULT];
 };
 
 struct report_template {
@@ -219,5 +217,8 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all);
 void ls3_stats_calculate_values(struct fstats_report *report);
 void ls3_stats_printf(struct fstats_report *report,
                      struct ls3_stats_dir_general *dir_report);
+void ls3_stats_update_range(struct fstats_report *g_report,
+                           ls3_stats_report_type report_type,
+                           int64_t range_value, int64_t second_value);
 
 #endif /* _LS3_STATS_H_ */