From: Vitaliy Kuznetsov Date: Tue, 5 Mar 2024 14:49:39 +0000 (+0100) Subject: EX-8130 lipe: Add output for dir sizes stats X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=4f48489167420988b2ce836810ecb0ffe71df148;p=fs%2Flustre-release.git EX-8130 lipe: Add output for dir sizes stats This patch adds functions for displaying size statistics for directories in the general report. This patch adds support for *.out format only. Test-Parameters: trivial Signed-off-by: Vitaliy Kuznetsov Change-Id: Iaf70aa4d84295f1a1a297b00fa45f12fb98c7625 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53983 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Alexandre Ioffe Reviewed-by: Andreas Dilger --- diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.c b/lipe/src/lipe_scan3/ls3_dir_stats.c index 693c8a20..77ec5ec 100644 --- a/lipe/src/lipe_scan3/ls3_dir_stats.c +++ b/lipe/src/lipe_scan3/ls3_dir_stats.c @@ -222,6 +222,177 @@ static const char *ls3_stats_fmt_size_units(uint64_t bytes) return buffer; } +/* ls3_stats_fprintf_out_recursive - Recursively traverse a directory tree + * in memory and prints statistics for each directory to a + * file if all conditions are met. + */ +static void ls3_stats_fprintf_out_recursive(FILE *out_fd, + struct ls3_stats_dir_obj *dir_ptr, + bool top_rating_only, + uint64_t *total_size, + uint64_t *total_alloc_size) +{ + struct ls3_stats_dir_obj *child_ptr; + uint64_t allocate_dir_size; + char fmt_str_for_fid[30]; + char *tabulation; + int i; + + if (dir_ptr->lsdo_depth >= 0) { + *total_size += dir_ptr->lsdo_size; + *total_alloc_size += dir_ptr->lsdo_alloc_size_in_512blocks * 512; + } + + /* dir_stats->lsdg_min_size_value_for_print is initialized + * to 0 by default unless the user has specified this + * option with a value in Lipe_find3. This is necessary + * when there are a lot of directories and it won't be + * very good to store it in a file. + * + * If at this stage the FID is still empty + * (dir_ptr->lsdo_fid.f_seq == 0), it means that lipe did not + * scan this directory, in other words, while lipe was scanning/working, + * this directory was deleted =(. Or data is stored on another MDT. + * But OST no have FID and type "dir" and should print anyway + * + * And skip first dir. + */ + if (dir_ptr->lsdo_depth >= 0 && + dir_ptr->lsdo_size >= dir_stats->lsdg_min_size_value_for_print && + !top_rating_only && out_fd) { + allocate_dir_size = dir_ptr->lsdo_alloc_size_in_512blocks * 512; + tabulation = ls3_stats_get_dots(dir_ptr->lsdo_depth); + + snprintf(fmt_str_for_fid, sizeof(fmt_str_for_fid), + DFID_NOBRACE, PFID(&dir_ptr->lsdo_fid)); + + fprintf(out_fd, "%12s %12s %15u %15u %30s %s%s\n", + ls3_stats_fmt_size_units(dir_ptr->lsdo_size), + ls3_stats_fmt_size_units(allocate_dir_size), + dir_ptr->lsdo_files_count, dir_ptr->lsdo_dirs_count, + fmt_str_for_fid, tabulation, dir_ptr->lsdo_name); + free(tabulation); + } + + /* Trying to add a directory to the rating table */ + if (dir_stats->lsdg_top_size) + ls3_stats_insert_heap(dir_stats->lsdg_top_size, dir_ptr); + + for (i = 0; i < dir_ptr->lsdo_last_child + 1; i++) { + child_ptr = dir_ptr->lsdo_child[i]; + if (child_ptr == NULL) + continue; + + ls3_stats_fprintf_out_recursive(out_fd, child_ptr, + top_rating_only, total_size, + total_alloc_size); + } +} + +static void ls3_stats_printf_rating_out(FILE *out_fd, + struct ls3_stats_dir_rating *heap) +{ + int i; + + fprintf(out_fd, "\n\nTop %d largest directories:\n", + dir_stats->lsdg_top_rating_limit); + fprintf(out_fd, "____Size____ _Alloc_Size_ __Files_Count__" + " ___Dirs_Count__ __User_ID___ _____________FID______________" + " ____________________Directory_name_______________________\n"); + + for (i = 0; i < heap->lsdr_current_size_heap; i++) { + uint64_t allocate_dir_size; + char fid_str[64]; + char path[PATH_MAX] = ""; + int rc; + + if (heap->lsdr_array[i]->lsdo_depth < 0) + continue; /* skip first dir */ + + allocate_dir_size = + heap->lsdr_array[i]->lsdo_alloc_size_in_512blocks * 512; + /* prepare FID */ + snprintf(fid_str, sizeof(fid_str), DFID_NOBRACE, + PFID(&heap->lsdr_array[i]->lsdo_fid)); + + if (heap->lsdr_array[i]->lsdo_fid.f_seq != 0) { + rc = llapi_fid2path(reports_with_stats->client_mount_path, + fid_str, path, PATH_MAX, NULL, NULL); + if (rc != 0) + llapi_error(LLAPI_MSG_ERROR, rc, + "cannot get path of fid %s", fid_str); + } + + fprintf(out_fd, "%12s %12s %15u %15u %12u %30s %s\n", + ls3_stats_fmt_size_units(heap->lsdr_array[i]->lsdo_size), + ls3_stats_fmt_size_units(allocate_dir_size), + heap->lsdr_array[i]->lsdo_files_count, + heap->lsdr_array[i]->lsdo_dirs_count, + heap->lsdr_array[i]->lsdo_uid, + fid_str, + strlen(path) > 0 ? path : heap->lsdr_array[i]->lsdo_name); + } + + fprintf(out_fd, "____________ ____________ _______________" + " _______________ ____________ ______________________________" + " _________________________________________________________\n"); +} + +/* ls3_stats_fprintf_out - Initiates the process of printing directory + * size statistics in an easy-to-read format into a general stats report + * with the *.out extension. + */ +void ls3_stats_fprintf_out(FILE *out_fd) +{ + uint64_t total_size; + uint64_t total_alloc_size; + + if (!dir_stats) + return; + + if (!dir_stats->lsdg_start_dir) + return; + + ls3_stats_init_heap(); + + fprintf(out_fd, + "\n\nDirectory statistics for: '%s' [files count:%u size:%s]\n", + dir_stats->lsdg_start_path, + dir_stats->lsdg_start_dir->lsdo_files_count, + ls3_stats_fmt_size_units(dir_stats->lsdg_start_dir->lsdo_size)); + + fprintf(out_fd, + "____Size____ _Alloc_Size_ __Files_Count__ ___Dirs_Count__" + " _____________FID______________" + " _____________Structure_and_directory_name________________\n"); + + ls3_stats_fprintf_out_recursive(out_fd, dir_stats->lsdg_start_dir, + false, &total_size, &total_alloc_size); + fprintf(out_fd, + "____________ ____________ _______________ _______________" + " ______________________________" + " _________________________________________________________\n"); + + fprintf(out_fd, + "Total size of directories on %s: %s | allocate size: %s", + reports_with_stats->device_name, + ls3_stats_fmt_size_units(total_size), + ls3_stats_fmt_size_units(total_alloc_size)); + + /* Update total size counters for next reports (*.all) */ + dir_stats->lsdg_total_size = total_size; + dir_stats->lsdg_total_alloc_size = total_alloc_size; + + if (dir_stats->lsdg_top_size) { + ls3_stats_heap_sort(dir_stats->lsdg_top_size); + ls3_stats_printf_rating_out(out_fd, dir_stats->lsdg_top_size); + } else { + LS3_WARNING("the size of the rating table has not been " + "specified or is equal to 0, the rating table " + "will not be built\n"); + } +} + /* ls3_stats_dir_incr_counters - Increases directory size */ static void ls3_stats_dir_incr_counters(struct ls3_object_attrs *loa_all, struct ls3_stats_dir_obj *dir_ptr) diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.h b/lipe/src/lipe_scan3/ls3_dir_stats.h index 5c3c3a4..b8f6fc4 100644 --- a/lipe/src/lipe_scan3/ls3_dir_stats.h +++ b/lipe/src/lipe_scan3/ls3_dir_stats.h @@ -80,6 +80,7 @@ struct ls3_stats_dir_general { extern struct ls3_stats_dir_general *dir_stats; +void ls3_stats_fprintf_out(FILE *out_fd); void ls3_stats_dir_init(void); void ls3_stats_dir_destroy(void); void ls3_stats_dir_distributor(struct ls3_object_attrs *loa_all);