EX-8130 lipe: Add output for dir sizes stats

This patch adds functions for displaying size statistics
for directories in the general report.
This patch adds support for *.out format only.

Test-Parameters: trivial
Signed-off-by: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
Change-Id: Iaf70aa4d84295f1a1a297b00fa45f12fb98c7625
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53983
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Alexandre Ioffe <aioffe@ddn.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.c b/lipe/src/lipe_scan3/ls3_dir_stats.c
index 693c8a2..77ec5ec 100644 (file)
--- a/lipe/src/lipe_scan3/ls3_dir_stats.c
+++ b/lipe/src/lipe_scan3/ls3_dir_stats.c
@@ -222,6 +222,177 @@ static const char *ls3_stats_fmt_size_units(uint64_t bytes)
        return buffer;
 }
 
+/* ls3_stats_fprintf_out_recursive - Recursively traverse a directory tree
+ * in memory and prints statistics for each directory to a
+ * file if all conditions are met.
+ */
+static void ls3_stats_fprintf_out_recursive(FILE *out_fd,
+                                           struct ls3_stats_dir_obj *dir_ptr,
+                                           bool top_rating_only,
+                                           uint64_t *total_size,
+                                           uint64_t *total_alloc_size)
+{
+       struct ls3_stats_dir_obj *child_ptr;
+       uint64_t allocate_dir_size;
+       char fmt_str_for_fid[30];
+       char *tabulation;
+       int i;
+
+       if (dir_ptr->lsdo_depth >= 0) {
+               *total_size += dir_ptr->lsdo_size;
+               *total_alloc_size += dir_ptr->lsdo_alloc_size_in_512blocks * 512;
+       }
+
+       /* dir_stats->lsdg_min_size_value_for_print is initialized
+        * to 0 by default unless the user has specified this
+        * option with a value in Lipe_find3. This is necessary
+        * when there are a lot of directories and it won't be
+        * very good to store it in a file.
+        *
+        * If at this stage the FID is still empty
+        * (dir_ptr->lsdo_fid.f_seq == 0), it means that lipe did not
+        * scan this directory, in other words, while lipe was scanning/working,
+        * this directory was deleted =(. Or data is stored on another MDT.
+        * But OST no have FID and type "dir" and should print anyway
+        *
+        * And skip first dir.
+        */
+       if (dir_ptr->lsdo_depth >= 0 &&
+           dir_ptr->lsdo_size >= dir_stats->lsdg_min_size_value_for_print &&
+           !top_rating_only && out_fd) {
+               allocate_dir_size = dir_ptr->lsdo_alloc_size_in_512blocks * 512;
+               tabulation = ls3_stats_get_dots(dir_ptr->lsdo_depth);
+
+               snprintf(fmt_str_for_fid, sizeof(fmt_str_for_fid),
+                        DFID_NOBRACE, PFID(&dir_ptr->lsdo_fid));
+
+               fprintf(out_fd, "%12s %12s %15u %15u %30s %s%s\n",
+                       ls3_stats_fmt_size_units(dir_ptr->lsdo_size),
+                       ls3_stats_fmt_size_units(allocate_dir_size),
+                       dir_ptr->lsdo_files_count, dir_ptr->lsdo_dirs_count,
+                       fmt_str_for_fid, tabulation, dir_ptr->lsdo_name);
+               free(tabulation);
+       }
+
+       /* Trying to add a directory to the rating table */
+       if (dir_stats->lsdg_top_size)
+               ls3_stats_insert_heap(dir_stats->lsdg_top_size, dir_ptr);
+
+       for (i = 0; i < dir_ptr->lsdo_last_child + 1; i++) {
+               child_ptr = dir_ptr->lsdo_child[i];
+               if (child_ptr == NULL)
+                       continue;
+
+               ls3_stats_fprintf_out_recursive(out_fd, child_ptr,
+                                               top_rating_only, total_size,
+                                               total_alloc_size);
+       }
+}
+
+static void ls3_stats_printf_rating_out(FILE *out_fd,
+                                       struct ls3_stats_dir_rating *heap)
+{
+       int i;
+
+       fprintf(out_fd, "\n\nTop %d largest directories:\n",
+               dir_stats->lsdg_top_rating_limit);
+       fprintf(out_fd, "____Size____ _Alloc_Size_ __Files_Count__"
+               " ___Dirs_Count__ __User_ID___ _____________FID______________"
+               " ____________________Directory_name_______________________\n");
+
+       for (i = 0; i < heap->lsdr_current_size_heap; i++) {
+               uint64_t allocate_dir_size;
+               char fid_str[64];
+               char path[PATH_MAX] = "";
+               int rc;
+
+               if (heap->lsdr_array[i]->lsdo_depth < 0)
+                       continue;       /* skip first dir */
+
+               allocate_dir_size =
+                       heap->lsdr_array[i]->lsdo_alloc_size_in_512blocks * 512;
+               /* prepare FID */
+               snprintf(fid_str, sizeof(fid_str), DFID_NOBRACE,
+                        PFID(&heap->lsdr_array[i]->lsdo_fid));
+
+               if (heap->lsdr_array[i]->lsdo_fid.f_seq != 0) {
+                       rc = llapi_fid2path(reports_with_stats->client_mount_path,
+                                           fid_str, path, PATH_MAX, NULL, NULL);
+                       if (rc != 0)
+                               llapi_error(LLAPI_MSG_ERROR, rc,
+                                           "cannot get path of fid %s", fid_str);
+               }
+
+               fprintf(out_fd, "%12s %12s %15u %15u %12u %30s %s\n",
+                       ls3_stats_fmt_size_units(heap->lsdr_array[i]->lsdo_size),
+                       ls3_stats_fmt_size_units(allocate_dir_size),
+                       heap->lsdr_array[i]->lsdo_files_count,
+                       heap->lsdr_array[i]->lsdo_dirs_count,
+                       heap->lsdr_array[i]->lsdo_uid,
+                       fid_str,
+                       strlen(path) > 0 ? path : heap->lsdr_array[i]->lsdo_name);
+       }
+
+       fprintf(out_fd, "____________ ____________ _______________"
+               " _______________ ____________ ______________________________"
+               " _________________________________________________________\n");
+}
+
+/* ls3_stats_fprintf_out - Initiates the process of printing directory
+ * size statistics in an easy-to-read format into a general stats report
+ * with the *.out extension.
+ */
+void ls3_stats_fprintf_out(FILE *out_fd)
+{
+       uint64_t total_size;
+       uint64_t total_alloc_size;
+
+       if (!dir_stats)
+               return;
+
+       if (!dir_stats->lsdg_start_dir)
+               return;
+
+       ls3_stats_init_heap();
+
+       fprintf(out_fd,
+               "\n\nDirectory statistics for: '%s' [files count:%u size:%s]\n",
+               dir_stats->lsdg_start_path,
+               dir_stats->lsdg_start_dir->lsdo_files_count,
+               ls3_stats_fmt_size_units(dir_stats->lsdg_start_dir->lsdo_size));
+
+       fprintf(out_fd,
+               "____Size____ _Alloc_Size_ __Files_Count__ ___Dirs_Count__"
+               " _____________FID______________"
+               " _____________Structure_and_directory_name________________\n");
+
+       ls3_stats_fprintf_out_recursive(out_fd, dir_stats->lsdg_start_dir,
+                                       false, &total_size, &total_alloc_size);
+       fprintf(out_fd,
+               "____________ ____________ _______________ _______________"
+               " ______________________________"
+               " _________________________________________________________\n");
+
+       fprintf(out_fd,
+               "Total size of directories on %s: %s | allocate size: %s",
+               reports_with_stats->device_name,
+               ls3_stats_fmt_size_units(total_size),
+               ls3_stats_fmt_size_units(total_alloc_size));
+
+       /* Update total size counters for next reports (*.all) */
+       dir_stats->lsdg_total_size = total_size;
+       dir_stats->lsdg_total_alloc_size = total_alloc_size;
+
+       if (dir_stats->lsdg_top_size) {
+               ls3_stats_heap_sort(dir_stats->lsdg_top_size);
+               ls3_stats_printf_rating_out(out_fd, dir_stats->lsdg_top_size);
+       } else {
+               LS3_WARNING("the size of the rating table has not been "
+                           "specified or is equal to 0, the rating table "
+                           "will not be built\n");
+       }
+}
+
 /* ls3_stats_dir_incr_counters - Increases directory size */
 static void ls3_stats_dir_incr_counters(struct ls3_object_attrs *loa_all,
                                        struct ls3_stats_dir_obj *dir_ptr)

diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.h b/lipe/src/lipe_scan3/ls3_dir_stats.h
index 5c3c3a4..b8f6fc4 100644 (file)
--- a/lipe/src/lipe_scan3/ls3_dir_stats.h
+++ b/lipe/src/lipe_scan3/ls3_dir_stats.h
@@ -80,6 +80,7 @@ struct ls3_stats_dir_general {
 
 extern struct ls3_stats_dir_general *dir_stats;
 
+void ls3_stats_fprintf_out(FILE *out_fd);
 void ls3_stats_dir_init(void);
 void ls3_stats_dir_destroy(void);
 void ls3_stats_dir_distributor(struct ls3_object_attrs *loa_all);