From ae4f377d82a2e608c46e6d511e8b8546339ade59 Mon Sep 17 00:00:00 2001 From: Vitaliy Kuznetsov Date: Sat, 6 Apr 2024 11:08:34 +0200 Subject: [PATCH] EX-9136 lipe: Update the display format for fstats MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit This patch affects only .out format report types. The data output format has been updated according to the request in the ticket. Also: 1. Fixed incorrect information display in some tables; 2. Expanded additional information for each table; 3. The size in tables is now displayed in various formats, not just in KB; 4. Except for the “File Size” table, the size obtained from block_size is now used everywhere; 5. Fixed an issue with displaying the allocated size for directories; 6. Fixed the total size calculation for all directories; 7. Removed tables that are not yet available; 8. Added additional information about the number of missed files for each table; 9. All txt information for working with reports in the .out format is combined into a single array to simplify code maintenance. Test-Parameters: trivial testlist=sanity-lipe-scan3,sanity-lipe-find3 Signed-off-by: Vitaliy Kuznetsov Change-Id: Id75b3af12ea00761850a9009848621539c016446 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/54658 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- lipe/src/lipe_scan3/ls3_dir_stats.c | 26 +- lipe/src/lipe_scan3/ls3_dir_stats.h | 1 + lipe/src/lipe_scan3/ls3_stats.c | 1357 +++++++++++++++++++++-------------- lipe/src/lipe_scan3/ls3_stats.h | 21 +- lustre/tests/sanity-lipe-scan3.sh | 42 ++ 5 files changed, 904 insertions(+), 543 deletions(-) diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.c b/lipe/src/lipe_scan3/ls3_dir_stats.c index 77ec5ec..fea22bc 100644 --- a/lipe/src/lipe_scan3/ls3_dir_stats.c +++ b/lipe/src/lipe_scan3/ls3_dir_stats.c @@ -205,7 +205,7 @@ static char* ls3_stats_get_dots(int depth) { /* ls3_stats_fmt_size_units convert size in bytes to * size in human readable output. */ -static const char *ls3_stats_fmt_size_units(uint64_t bytes) +const char *ls3_stats_fmt_size_units(uint64_t bytes) { const char *sizes[] = { "B", "KB", "MB", "GB", "TB", "Pb" }; int order = 0; @@ -235,10 +235,11 @@ static void ls3_stats_fprintf_out_recursive(FILE *out_fd, struct ls3_stats_dir_obj *child_ptr; uint64_t allocate_dir_size; char fmt_str_for_fid[30]; + char tmp_alocte_size_str[LS3_STATS_NUM_STR_MAX_WITH_NULL]; char *tabulation; int i; - if (dir_ptr->lsdo_depth >= 0) { + if (dir_ptr->lsdo_depth == 0) { *total_size += dir_ptr->lsdo_size; *total_alloc_size += dir_ptr->lsdo_alloc_size_in_512blocks * 512; } @@ -266,11 +267,14 @@ static void ls3_stats_fprintf_out_recursive(FILE *out_fd, snprintf(fmt_str_for_fid, sizeof(fmt_str_for_fid), DFID_NOBRACE, PFID(&dir_ptr->lsdo_fid)); + snprintf(tmp_alocte_size_str, sizeof(tmp_alocte_size_str), + "%s", ls3_stats_fmt_size_units(allocate_dir_size)); + fprintf(out_fd, "%12s %12s %15u %15u %30s %s%s\n", ls3_stats_fmt_size_units(dir_ptr->lsdo_size), - ls3_stats_fmt_size_units(allocate_dir_size), - dir_ptr->lsdo_files_count, dir_ptr->lsdo_dirs_count, - fmt_str_for_fid, tabulation, dir_ptr->lsdo_name); + tmp_alocte_size_str, dir_ptr->lsdo_files_count, + dir_ptr->lsdo_dirs_count, fmt_str_for_fid, tabulation, + dir_ptr->lsdo_name); free(tabulation); } @@ -304,6 +308,7 @@ static void ls3_stats_printf_rating_out(FILE *out_fd, uint64_t allocate_dir_size; char fid_str[64]; char path[PATH_MAX] = ""; + char tmp_alocte_size_str[LS3_STATS_NUM_STR_MAX_WITH_NULL]; int rc; if (heap->lsdr_array[i]->lsdo_depth < 0) @@ -315,6 +320,9 @@ static void ls3_stats_printf_rating_out(FILE *out_fd, snprintf(fid_str, sizeof(fid_str), DFID_NOBRACE, PFID(&heap->lsdr_array[i]->lsdo_fid)); + snprintf(tmp_alocte_size_str, sizeof(tmp_alocte_size_str), + "%s", ls3_stats_fmt_size_units(allocate_dir_size)); + if (heap->lsdr_array[i]->lsdo_fid.f_seq != 0) { rc = llapi_fid2path(reports_with_stats->client_mount_path, fid_str, path, PATH_MAX, NULL, NULL); @@ -325,7 +333,7 @@ static void ls3_stats_printf_rating_out(FILE *out_fd, fprintf(out_fd, "%12s %12s %15u %15u %12u %30s %s\n", ls3_stats_fmt_size_units(heap->lsdr_array[i]->lsdo_size), - ls3_stats_fmt_size_units(allocate_dir_size), + tmp_alocte_size_str, heap->lsdr_array[i]->lsdo_files_count, heap->lsdr_array[i]->lsdo_dirs_count, heap->lsdr_array[i]->lsdo_uid, @@ -346,6 +354,7 @@ void ls3_stats_fprintf_out(FILE *out_fd) { uint64_t total_size; uint64_t total_alloc_size; + char tmp_alocte_size_str[LS3_STATS_NUM_STR_MAX_WITH_NULL]; if (!dir_stats) return; @@ -373,11 +382,14 @@ void ls3_stats_fprintf_out(FILE *out_fd) " ______________________________" " _________________________________________________________\n"); + snprintf(tmp_alocte_size_str, sizeof(tmp_alocte_size_str), + "%s", ls3_stats_fmt_size_units(total_alloc_size)); + fprintf(out_fd, "Total size of directories on %s: %s | allocate size: %s", reports_with_stats->device_name, ls3_stats_fmt_size_units(total_size), - ls3_stats_fmt_size_units(total_alloc_size)); + tmp_alocte_size_str); /* Update total size counters for next reports (*.all) */ dir_stats->lsdg_total_size = total_size; diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.h b/lipe/src/lipe_scan3/ls3_dir_stats.h index b8f6fc4..e9c8fd0 100644 --- a/lipe/src/lipe_scan3/ls3_dir_stats.h +++ b/lipe/src/lipe_scan3/ls3_dir_stats.h @@ -80,6 +80,7 @@ struct ls3_stats_dir_general { extern struct ls3_stats_dir_general *dir_stats; +const char *ls3_stats_fmt_size_units(uint64_t bytes); void ls3_stats_fprintf_out(FILE *out_fd); void ls3_stats_dir_init(void); void ls3_stats_dir_destroy(void); diff --git a/lipe/src/lipe_scan3/ls3_stats.c b/lipe/src/lipe_scan3/ls3_stats.c index b995a18..a9051f0 100644 --- a/lipe/src/lipe_scan3/ls3_stats.c +++ b/lipe/src/lipe_scan3/ls3_stats.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -152,6 +153,681 @@ static const ls3_stats_extension_mapping ext_mappings[] = { {NULL, 0} /* End marker */ }; +static const ls3_stats_out_mapping ls3_out_mappings[] = { + {"Files Size", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a percent of total number " + "of files;\n" + "cpt_num - Number of files in this range or smaller as a % of " + "total # of files;\n" + "total_size - Total size of files in range;\n" + "pct_rng - Total size of files in range as a % of total size " + "of files;\n" + "cpt_rng - Total size of files in this range or smaller as a % of " + "total size of files;\n" + "min_size - Minimum file size in range;\n" + "max_size - Maximum file size in range.\n", + "____file_size_range____ ___number___ pct_num cpt_num total_size " + "pct_rng cpt_rng _min_size_ _max_size_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum file size: %s\n" + "Maximum file size: %s\n" + "Average file size: %s\n" + "Total size of all files: %s\n", + LS3_STATS_TYPE_FILES_SIZE}, + {"Capacity used (Regular files space used on disk)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a percent of total " + "number of files;\n" + "cpt_num - Number of files in this range or smaller as " + "a % of total # of file;\n" + "total_used - Total capacity used in range;\n" + "pct_rng - Total capacity used in range as a % of total " + "capacity used;\n" + "cpt_rng - Total capacity used in this range or smaller as " + "a % of total capacity;\n" + "min_size - Minimum file size (in blocks) in range;\n" + "max_size - Maximum file size (in blocks) in range.\n", + "__file_capacity_range__ ___number___ pct_num cpt_num total_used " + "pct_rng cpt_rng _min_size_ _max_size_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum file size (in blocks): %s\n" + "Maximum file size (in blocks): %s\n" + "Average file size (in blocks): %s\n" + "Total capacity used of all files: %s\n", + LS3_STATS_TYPE_CAPACITY_USED}, + {"Equal overhead (Files whose capacity used is equal to their size)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a percent of total number " + "of files;\n" + "cpt_num - Number of files in this range or smaller as a % of " + "total # of files;\n" + "total_size - Total size that overflowed in range;\n" + "pct_rng - Total kilobytes that overflowed in this range as " + "a % of total kilobytes that overflowed;\n" + "cpt_rng - Total kilobytes that overflowed in this range or " + "smaller as a % of total kilobytes that overflowed;\n" + "min_size - Minimum file size in range;\n" + "max_size - Maximum file size in range.\n", + "______size_range_______ ___number___ pct_num cpt_num total_size " + "pct_rng cpt_rng _min_size_ _max_size_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum file size (in blocks): %s\n" + "Maximum file size (in blocks): %s\n" + "Average file size (in blocks): %s\n" + "Total capacity used of all files: %s\n", + LS3_STATS_TYPE_EQUAL_OVERHEAD}, + {"Positive overhead (Files whose capacity used is larger than " + "to it's size)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a percent of total " + "number of files;\n" + "cpt_num - Number of files in this range or smaller " + "as a % of total # of files;\n" + "total_size - Total size that overflowed in range;\n" + "pct_rng - Total bytes that overflowed in this range " + "as a % of total bytes that overflowed;\n" + "cpt_rng - Total bytes that overflowed in this range or " + "smaller as a % of total bytes that overflowed;\n" + "min_size - Minimum file size in range;\n" + "max_size - Maximum file size in range.\n", + "__range_of_diff_sizes__ ___number___ pct_num cpt_num total_size " + "pct_rng cpt_rng _min_size_ _max_size_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum file size (in blocks): %s\n" + "Maximum file size (in blocks): %s\n" + "Average file size (in blocks): %s\n" + "Total capacity used of all files: %s\n", + LS3_STATS_TYPE_POSITIVE_OVERHEAD}, + {"Negative overhead (Files whose size is larger than capacity used.)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a percent of total " + "number of files;\n" + "cpt_num - Number of files in this range or smaller as a % of " + "total # of files;\n" + "total_size - Total size that underflowed in range;\n" + "pct_rng - Total bytes that underflowed in this range as a % " + "of total bytes that underflowed;\n" + "cpt_rng - Total bytes that underflowed in this range or " + "smaller as a % of total bytes that underflowed;\n" + "min_size - Minimum value in range;\n" + "max_size - Maximum value in range.\n", + "__range_of_diff_sizes__ ___number___ pct_num cpt_num total_size " + "pct_rng cpt_rng _min_size_ _max_size_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum file size (in blocks): %s\n" + "Maximum file size (in blocks): %s\n" + "Average file size (in blocks): %s\n" + "Total capacity used of all files: %s\n", + LS3_STATS_TYPE_NEGATIVE_OVERHEAD}, + {"Compression ratio (of regular files)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a percent of total " + "number of files;\n" + "cpt_num - Number of files in this range or smaller as a % " + "of total # of files;\n" + "total_size - Total compression size of files in range;\n" + "pct_rng - Total compression size of files in range as a % of " + "total compression size of files;\n" + "cpt_rng - Total compression size of files in this range or " + "smaller as a % of total compression size of files;\n" + "min - Minimum compression ratio in range;\n" + "max - Maximum compression ratio in range.\n", + "compression_ratio ___number___ pct_num cpt_num total_size " + "pct_rng cpt_rng __min__ __max__\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %7s %7s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum compression ratio: %s\n" + "Maximum compression ratio: %s\n" + "Average compression ratio: %s\n" + "Total compression size of all files: %s\n", + LS3_STATS_TYPE_COMPRESSION_RATIO}, + {"Directory size (entries)", + "Description:\n" + "number - Count of directories in range;\n" + "pct_num - Number of dirs in range as a % of total " + "num of dirs;\n" + "cpt_num - Number of dirs in this range or smaller as " + "a % total number of dirs;\n" + "tot_entries - Total entries in range;\n" + "pct_rng - Number of entries in range as a % of total number " + "of entries;\n" + "cpt_rng - Number of entries in this range or smaller as a % " + "of total number of entries;\n" + "min - Minimum entries in range;\n" + "max - Maximum entries in range.\n", + "______range_of_entries_______ ___number___ pct_num cpt_num " + "tot_entries pct_rng cpt_rng ____min____ ____max____\n", + "%s %12lu %6.2f%% %6.2f%% %11s %6.2f%% %6.2f%% %11s %11s\n", + "General information:\n" + "Total number of entries: %lu\n" + "Number of skipped directory: %u\n" + "Minimum entries value in ranges: %s\n" + "Maximum entries value in ranges: %s\n" + "Average entries value in ranges: %s\n" + "Total entries of all directory in rnages: %s\n", + LS3_STATS_TYPE_DIRECTORY_SIZE_ENTRIES}, + {"Directory size", + "Description:\n" + "number - Count of directories in range;\n" + "pct_num - Number of files in dirs as a percent of total " + "number of dirs;\n" + "cpt_num - Number of dirs in this range or smaller as a % of " + "total # of dirs;\n" + "total_size - Total size of dirs in range;\n" + "pct_rng - Total size of dirs in range as a % of total size " + "of dirs;\n" + "cpt_rng - Total size of dirs in this range or smaller as a % " + "of total size of dirs;\n" + "min_size - Minimum directory size in range;\n" + "max_size - Maximum directory size in range.\n", + "____directory_size_range_____ ___number___ pct_num cpt_num total_size" + " pct_rng cpt_rng _min_size_ _max_size_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n", + "General information:\n" + "Total number of directory: %lu\n" + "Number of skipped directory: %u\n" + "Minimum directory size (in blocks): %s\n" + "Maximum directory size (in blocks): %s\n" + "Average directory size (in blocks): %s\n" + "Total capacity used of all directory: %s\n", + LS3_STATS_TYPE_DIRECTORY_SIZE_KB}, + {"Link count (of regular files)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a % of total files;\n" + "cpt_num - Number of files in this range or smaller as a % of " + "total files;\n" + "tot_links - Total links in range;\n" + "pct_rng - Total links in range as a % of total links;\n" + "cpt_rng - Total links in this range or smaller as a % of " + "total links;\n" + "min - Minimum link count in range;\n" + "max - Maximum link count in range.\n", + "____rnage_of_link_counts_____ ___number___ pct_num cpt_num tot_links " + "pct_rng cpt_rng ____min____ ____max____\n", + "%s %12lu %6.2f%% %6.2f%% %11s %6.2f%% %6.2f%% %11s %11s\n", + "General information:\n" + "Total number of links: %lu\n" + "Number of skipped links: %u\n" + "Minimum links in ranges: %s\n" + "Maximum links in ranges: %s\n" + "Average links in ranges: %s\n" + "Total links in rnages: %s\n", + LS3_STATS_TYPE_LINK_COUNT}, + {"Filename length (Includes regular files, dirs, symlinks and " + "special files)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Count of files in range as a % of total number " + "of files;\n" + "cpt_num - Number of files in this range or smaller as a % " + "of total # of files;\n" + "total_chars - Total chars of filenames in range;\n" + "pct_rng - Total chars of filenames in range as a % of total " + "number of chars of filenames;\n" + "cpt_rng - Total number of chars of filenames in this range or " + "smaller as a % of total number bytes of filenames;\n" + "min - Minimum chars in range;\n" + "max - Maximum chars in range.\n", + "fname_length_range ___number___ pct_num cpt_num total_chars " + "pct_rng cpt_rng _min_ _max_\n", + "%s %12lu %6.2f%% %6.2f%% %11s %6.2f%% %6.2f%% %5s %5s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum chars in ranges: %s\n" + "Maximum chars in ranges: %s\n" + "Average chars in ranges: %s\n" + "Total chars in rnages: %s\n", + LS3_STATS_TYPE_FILENAME_LENGTH}, + {"Time since last modification [mtime](Regular files)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a % of total number " + "of files;\n" + "cpt_num - Number of files in this range or smaller as a % " + "of total number of files;\n" + "total_size - Total size that underflowed in range;\n" + "pct_rng - Total of age in range, total age in range as a % " + "of total age of all files;\n" + "cpt_rng - total age in this range or smaller as a % of " + "total age of all files;\n" + "min - Minimum value in range;\n" + "max - Maximum value in range.\n", + "__range_in_days_ ___number___ pct_num cpt_num total_size " + "pct_rng cpt_rng _min_ _max_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %5s %5s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum days in ranges: %s\n" + "Maximum days in ranges: %s\n" + "Average days in ranges: %s\n" + "Total capacity size of all days: %s\n", + LS3_STATS_TYPE_TIME_SINCE_LAST_MOD_RF}, + {"Time since last metadata modification [ctime](Regular files)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a % of total number " + "of files;\n" + "cpt_num - Number of files in this range or smaller as a % " + "of total number of files;\n" + "total_size - Total size that underflowed in range;\n" + "pct_rng - Total of age in range, total age in range as a % " + "of total age of all files;\n" + "cpt_rng - Total age in this range or smaller as a % of total " + "age of all files;\n" + "min - Minimum days in range;\n" + "max - Maximum days in range.\n", + "__range_in_days_ ___number___ pct_num cpt_num total_size " + "pct_rng cpt_rng _min_ _max_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %5s %5s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum days in ranges: %s\n" + "Maximum days in ranges: %s\n" + "Average days in ranges: %s\n" + "Total capacity size of all days: %s\n", + LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF}, + {"Time since creation [crtime](Regular files)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a % of total number " + "of files;\n" + "cpt_num - Number of files in this range or smaller as a % " + "of total number of files;\n" + "total_size - Total size that underflowed in range;\n" + "pct_rng - Total of age in range, total age in range as a % " + "of total age of all files;\n" + "cpt_rng - Total age in this range or smaller as a % of total " + "age of all files;\n" + "min - Minimum days in range;\n" + "max - Maximum days in range.\n", + "__range_in_days_ ___number___ pct_num cpt_num total_size " + "pct_rng cpt_rng _min_ _max_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %5s %5s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum days in ranges: %s\n" + "Maximum days in ranges: %s\n" + "Average days in ranges: %s\n" + "Total capacity size of all days: %s\n", + LS3_STATS_TYPE_TIME_SINCE_LAST_CREATION_RF}, + {"Time since last access [atime](Regular files)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a % of total number " + "of files;\n" + "cpt_num - Number of files in this range or smaller as a % " + "of total number of files;\n" + "total_size - Total size that underflowed in range;\n" + "pct_rng - Total of age in range, total age in range as a % " + "of total age of all files;\n" + "cpt_rng - Total age in this range or smaller as a % of total " + "age of all files;\n" + "min - Minimum days in range;\n" + "max - Maximum days in range.\n", + "__range_in_days_ ___number___ pct_num cpt_num total_size " + "pct_rng cpt_rng _min_ _max_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %5s %5s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum days in ranges: %s\n" + "Maximum days in ranges: %s\n" + "Average days in ranges: %s\n" + "Total capacity size of all days: %s\n", + LS3_STATS_TYPE_TIME_SINCE_LAST_ACCESS_RF}, + {"Storage size (Regular files) by user", + "Description:\n" + "number - Count of files for user;\n" + "pct_num - Number of files for user as a percent of total " + "number of files;\n" + "cpt_num - Number of files for user as a % of total # of files;\n" + "total_size - Total size of files for user;\n" + "pct_rng - Total size of files for user as a % of total size " + "of files;\n" + "cpt_rng - Total size of files for user or smaller as a % of " + "total size of files;\n" + "min_size - Minimum file size for the user;\n" + "max_size - Maximum file size for the user.\n", + "________UID_and_name_______ ___number___ pct_num cpt_num total_size " + "pct_rng cpt_rng _min_size_ _max_size_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum file size (in blocks): %s\n" + "Maximum file size (in blocks): %s\n" + "Average file size (in blocks): %s\n" + "Total capacity used of all files: %s\n", + LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_USER}, + {"Storage size (Regular files) by group", + "Description:\n" + "number - Count of files for group;\n" + "pct_num - Number of files for group as a percent of total " + "number of files;\n" + "cpt_num - Number of files for group as a % of total # of files;\n" + "total_size - Total size of files for group;\n" + "pct_rng - Total size of files for group as a % of total " + "size of files;\n" + "cpt_rng - Total size of files for group or smaller as " + "a % of total size of files;\n" + "min_size - Minimum file size for the group;\n" + "max_size - Maximum file size for the group.\n", + "_______GUID_and_name_______ ___number___ pct_num cpt_num total_size " + "pct_rng cpt_rng _min_size_ _max_size_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum file size (in blocks): %s\n" + "Maximum file size (in blocks): %s\n" + "Average file size (in blocks): %s\n" + "Total capacity used of all files: %s\n", + LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_GROUP}, + {"Storage size (Regular files) by project ID", + "Description:\n" + "number - Count of files for projid;\n" + "pct_num - Number of files for projid as a percent of total " + "number of files;\n" + "cpt_num - Number of files for projid as a % of " + "total # of files;\n" + "total_size - Total size of files for projid;\n" + "pct_rng - Total size of files for projid as a % of total " + "size of files;\n" + "cpt_rng - Total size of files for projid or smaller as a % " + "of total size of files;\n" + "min_size - Minimum file size for the projid;\n" + "max_size - Maximum file size for the projid.\n", + "_project_ID_ ___number___ pct_num cpt_num total_size " + "pct_rng cpt_rng _min_size_ _max_size_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum file size (in blocks): %s\n" + "Maximum file size (in blocks): %s\n" + "Average file size (in blocks): %s\n" + "Total capacity used of all files: %s\n", + LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_PROJID}, + {"Stripe count (of regular files)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a % of total files;\n" + "cpt_num - Number of files in this range as a % of total files;\n" + "total_size - Total size of files with this stripe count;\n" + "pct_rng - Total size in range as a % of total stripe;\n" + "cpt_rng - Total size in this range as a % of total stripe;\n" + "min_size - Minimum size of files with this stripe count;\n" + "max_size - Maximum size of files with this stripe count.\n", + "stripe_count ___number___ pct_num cpt_num total_size pct_rng cpt_rng " + "_min_size_ _max_size_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum file size (in blocks): %s\n" + "Maximum file size (in blocks): %s\n" + "Average file size (in blocks): %s\n" + "Total capacity used of all files: %s\n", + LS3_STATS_TYPE_STRIPE_COUNT}, + {"Stripe size (of regular files)", + "Description:\n" + "number - Count of files with stripe size;\n" + "pct_num - Number of files with stripe size as a % of " + "total files;\n" + "cpt_num - Number of files with this stripe size as a % of " + "total files;\n" + "total_size - Total files size with this stripe size;\n" + "pct_rng - Total file size with stripe size as a % " + "of total size;\n" + "cpt_rng - Total file size with this stripe size or smaller as " + "a % of total file;\n" + "min_size - Minimum size of files with this stripe size;\n" + "max_size - Maximum size of files with this stripe size.\n", + "stripe_size ___number___ pct_num cpt_num total_size pct_rng cpt_rng " + "_min_size_ _max_size_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum file size (in blocks): %s\n" + "Maximum file size (in blocks): %s\n" + "Average file size (in blocks): %s\n" + "Total capacity used of all files: %s\n", + LS3_STATS_TYPE_STRIPE_SIZE}, + {"Mirror count (of regular files)", + "Description:\n" + "number - Count of files in range;\n" + "pct_num - Number of files in range as a % of total files;\n" + "cpt_num - Number of files in this range as a % " + "of total files;\n" + "total_size - Total size of files with this mirror count;\n" + "pct_rng - Total mirror in range as a % of total mirror;\n" + "cpt_rng - Total mirror in this range as a % of " + "total mirror;\n" + "min_size - Minimum size of files with this mirror count;\n" + "max_size - Maximum size of files with this mirror count.\n", + "mirror_cnt ___number___ pct_num cpt_num total_size pct_rng cpt_rng " + "_min_size_ _max_size_\n", + "%s %12lu %6.2f%% %6.2f%% %10s %6.2f%% %6.2f%% %10s %10s\n", + "General information:\n" + "Total number of files: %lu\n" + "Number of skipped files: %u\n" + "Minimum file size (in blocks): %s\n" + "Maximum file size (in blocks): %s\n" + "Average file size (in blocks): %s\n" + "Total capacity used of all files: %s\n", + LS3_STATS_TYPE_MIRROR_COUNT}, + /* end marker (always last) */ + {"", "", "", "", "", LS3_STATS_TYPE_TOTAL_COUNT_REPORT} +}; + + +static struct ls3_stats_range_val ls3_stats_get_range_val_str( + struct report_template *report_ptr, + struct range_report_template *range_ptr, + ls3_stats_report_type report_type, bool b_range) +{ + struct ls3_stats_range_val result; + + /* Everything regarding size will be multiplied + * by 1024 to convert to bytes + */ + switch (report_type) + { + case LS3_STATS_TYPE_FILES_SIZE: + case LS3_STATS_TYPE_CAPACITY_USED: + case LS3_STATS_TYPE_POSITIVE_OVERHEAD: + case LS3_STATS_TYPE_NEGATIVE_OVERHEAD: + case LS3_STATS_TYPE_DIRECTORY_SIZE_KB: + case LS3_STATS_TYPE_EQUAL_OVERHEAD: + case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_USER: + case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_GROUP: + case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_PROJID: + case LS3_STATS_TYPE_STRIPE_SIZE: + case LS3_STATS_TYPE_STRIPE_COUNT: + case LS3_STATS_TYPE_MIRROR_COUNT: { + /* Here, explicit indication of the (uint64_t) type is not + * critical, since for these tables integers (number of bytes) + * are stored here. + */ + if (b_range) { + /* Fill for range_ptr. */ + snprintf(result.srv_min, sizeof(result.srv_min), + "%s", + ls3_stats_fmt_size_units(range_ptr->rrt_min * + 1024)); + snprintf(result.srv_max, sizeof(result.srv_max), + "%s", + ls3_stats_fmt_size_units(range_ptr->rrt_max * + 1024)); + snprintf(result.srv_total, sizeof(result.srv_total), + "%s", + ls3_stats_fmt_size_units(1024 * + range_ptr->total_in_range)); + } else { + /* Fill for report_ptr */ + snprintf(result.srv_min, sizeof(result.srv_min), + "%s", + ls3_stats_fmt_size_units(report_ptr->rt_min * + 1024)); + snprintf(result.srv_max, sizeof(result.srv_max), + "%s", + ls3_stats_fmt_size_units(report_ptr->rt_max * + 1024)); + snprintf(result.srv_avg, sizeof(result.srv_avg), + "%s", + ls3_stats_fmt_size_units(report_ptr->rt_avg * + 1024)); + snprintf(result.srv_total, sizeof(result.srv_total), + "%s", + ls3_stats_fmt_size_units(1024 * + report_ptr->total_value)); + } + return result; + } + case LS3_STATS_TYPE_TIME_SINCE_LAST_MOD_RF: + case LS3_STATS_TYPE_TIME_SINCE_LAST_CREATION_RF: + case LS3_STATS_TYPE_TIME_SINCE_LAST_ACCESS_RF: + case LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF: { + /* min and max store the number of days here */ + if (b_range) { + snprintf(result.srv_min, sizeof(result.srv_min), + "%.0f", range_ptr->rrt_min); + snprintf(result.srv_max, sizeof(result.srv_max), + "%.0f", range_ptr->rrt_max); + snprintf(result.srv_total, sizeof(result.srv_total), + "%s", ls3_stats_fmt_size_units(1024 * + range_ptr->total_in_range)); + } else { + snprintf(result.srv_min, sizeof(result.srv_min), + "%.0f", report_ptr->rt_min); + snprintf(result.srv_max, sizeof(result.srv_max), + "%.0f", report_ptr->rt_max); + snprintf(result.srv_avg, sizeof(result.srv_avg), + "%.0f", report_ptr->rt_avg); + snprintf(result.srv_total, sizeof(result.srv_total), + "%s", ls3_stats_fmt_size_units(1024 * + report_ptr->total_value)); + } + + return result; + } + case LS3_STATS_TYPE_DIRECTORY_SIZE_ENTRIES: + case LS3_STATS_TYPE_FILENAME_LENGTH: + case LS3_STATS_TYPE_LINK_COUNT: { + /* min and max store the number of chars/entries/links here */ + if (b_range) { + snprintf(result.srv_min, sizeof(result.srv_min), + "%.0f", range_ptr->rrt_min); + snprintf(result.srv_max, sizeof(result.srv_max), + "%.0f", range_ptr->rrt_max); + snprintf(result.srv_total, sizeof(result.srv_total), + "%11lu", range_ptr->total_in_range); + } else { + snprintf(result.srv_min, sizeof(result.srv_min), + "%.0f", report_ptr->rt_min); + snprintf(result.srv_max, sizeof(result.srv_max), + "%.0f", report_ptr->rt_max); + snprintf(result.srv_avg, sizeof(result.srv_avg), + "%.0f", report_ptr->rt_avg); + snprintf(result.srv_total, sizeof(result.srv_total), + "%lu", report_ptr->total_value); + } + + return result; + } + case LS3_STATS_TYPE_COMPRESSION_RATIO: { + if (b_range) { + snprintf(result.srv_min, sizeof(result.srv_min), + "%6.2f%%", range_ptr->rrt_min); + snprintf(result.srv_max, sizeof(result.srv_max), + "%6.2f%%", range_ptr->rrt_max); + snprintf(result.srv_total, sizeof(result.srv_total), + "%s", ls3_stats_fmt_size_units(1024 * + range_ptr->total_in_range)); + } else { + snprintf(result.srv_min, sizeof(result.srv_min), + "%.2f%%", report_ptr->rt_min); + snprintf(result.srv_max, sizeof(result.srv_max), + "%.2f%%", report_ptr->rt_max); + snprintf(result.srv_avg, sizeof(result.srv_avg), + "%.2f%%", report_ptr->rt_avg); + snprintf(result.srv_total, sizeof(result.srv_total), + "%s", ls3_stats_fmt_size_units(1024 * + report_ptr->total_value)); + } + + return result; + } + case LS3_STATS_TYPE_TOTAL_COUNT_REPORT: + default: + /* init by default, nothing critical */ + strncpy(result.srv_min, "N/A", sizeof(result.srv_min)); + strncpy(result.srv_max, "N/A", sizeof(result.srv_max)); + strncpy(result.srv_avg, "N/A", sizeof(result.srv_avg)); + strncpy(result.srv_total, "N/A", sizeof(result.srv_total)); + LS3_ERROR_ONCE("unknown report type: %d\n", report_type); + break; + } + + return result; +} + +static uint32_t ls3_stats_get_count_skipped_obj( + ls3_stats_report_type report_type) +{ + switch (report_type) + { + case LS3_STATS_TYPE_FILES_SIZE: + return LS3_GET_ERROR_COUNT(reports_with_stats, + LS3_STATS_ERROR_EMPTY_SIZE) + + LS3_GET_ERROR_COUNT(reports_with_stats, + LS3_STATS_ERROR_WITHOUT_SIZE); + case LS3_STATS_TYPE_COMPRESSION_RATIO: + return LS3_GET_ERROR_COUNT(reports_with_stats, + LS3_STATS_ERROR_EMPTY_SIZE) + + LS3_GET_ERROR_COUNT(reports_with_stats, + LS3_STATS_ERROR_EMPTY_BLOCK_SIZE); + case LS3_STATS_TYPE_FILENAME_LENGTH: + return LS3_GET_ERROR_COUNT(reports_with_stats, + LS3_STATS_ERROR_WITHOUT_SIZE); + default: + return LS3_GET_ERROR_COUNT(reports_with_stats, + LS3_STATS_ERROR_EMPTY_BLOCK_SIZE); + } + + return 0; +} + static const char* ls3_stats_get_filename_from_path(const char *path) { const char *last_slash = strrchr(path, '/'); @@ -318,7 +994,6 @@ static ls3_stats_val_type ls3_stats_get_value_type( case LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF: return LS3_STATS_VALUE_TYPE_DAYS; case LS3_STATS_TYPE_FILENAME_LENGTH: - case LS3_STATS_TYPE_SYMLINK_TARGET_LENGTH: return LS3_STATS_VALUE_TYPE_CHARS; case LS3_STATS_TYPE_DIRECTORY_SIZE_ENTRIES: return LS3_STATS_VALUE_TYPE_ENTRIES; @@ -330,8 +1005,6 @@ static ls3_stats_val_type ls3_stats_get_value_type( return LS3_STATS_VALUE_TYPE_MIRROR; case LS3_STATS_TYPE_COMPRESSION_RATIO: return LS3_STATS_VALUE_TYPE_COMP_RATIO; - case LS3_STATS_TYPE_FILES_EMPLOYING_DOM: - case LS3_STATS_TYPE_COMPRESSION_FLAG: case LS3_STATS_TYPE_TOTAL_COUNT_REPORT: return LS3_STATS_VALUE_TYPE_ERROR; } @@ -370,384 +1043,8 @@ static char *ls3_stats_get_str_value_type(ls3_stats_val_type num_type) return ""; } -static char *ls3_stats_get_title_text(ls3_stats_report_type report_type) -{ - switch (report_type) - { - case LS3_STATS_TYPE_FILES_SIZE: - return "Files Size"; - case LS3_STATS_TYPE_CAPACITY_USED: - return "Capacity used (Regular files space used on disk)"; - case LS3_STATS_TYPE_EQUAL_OVERHEAD: - return "Equal overhead (Files whose capacity used is " - "equal to their size)"; - case LS3_STATS_TYPE_POSITIVE_OVERHEAD: - return "Positive overhead (Files whose capacity used is " - "larger than or equal to its size)"; - case LS3_STATS_TYPE_NEGATIVE_OVERHEAD: - return "Negative overhead (Files whose size is larger " - "than capacity used.)"; - case LS3_STATS_TYPE_DIRECTORY_SIZE_KB: - return "Directory size (KB)"; - case LS3_STATS_TYPE_TIME_SINCE_LAST_CREATION_RF: - return "Time since creation [crtime](Regular files)"; - case LS3_STATS_TYPE_TIME_SINCE_LAST_MOD_RF: - return "Time since last modification [mtime](Regular files)"; - case LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF: - return "Time since last metadata modification [ctime]"; - case LS3_STATS_TYPE_TIME_SINCE_LAST_ACCESS_RF: - return "Time since last access [atime](Regular files)"; - case LS3_STATS_TYPE_FILENAME_LENGTH: - return "Filename length (Includes regular files, " - "dirs, symlinks and special files)"; - case LS3_STATS_TYPE_DIRECTORY_SIZE_ENTRIES: - return "Directory size (entries)"; - case LS3_STATS_TYPE_FILES_EMPLOYING_DOM: - return "Files employing DoM"; - case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_USER: - return "Storage size (all directories and files) by user"; - case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_GROUP: - return "Storage size (all directories and files) by group"; - case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_PROJID: - return "Storage size (all directories and files) by project ID"; - case LS3_STATS_TYPE_LINK_COUNT: - return "Link count (of regular files)"; - case LS3_STATS_TYPE_STRIPE_COUNT: - return "Stripe count (of regular files)"; - case LS3_STATS_TYPE_STRIPE_SIZE: - return "Stripe size (of regular files)"; - case LS3_STATS_TYPE_MIRROR_COUNT: - return "Mirror count (of regular files)"; - case LS3_STATS_TYPE_COMPRESSION_RATIO: - return "Compression ratio (of regular files)"; - case LS3_STATS_TYPE_SYMLINK_TARGET_LENGTH: - return "Symlink target length"; - case LS3_STATS_TYPE_COMPRESSION_FLAG: - case LS3_STATS_TYPE_TOTAL_COUNT_REPORT: - return ""; - } - - return ""; -} - -static char *ls3_stats_get_header_text(ls3_stats_report_type report_type) -{ - switch (report_type) - { - case LS3_STATS_TYPE_FILES_SIZE: - return "\n\nFiles Size:\n" - "0 - Size range;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a percent of total " - "number of files;\n" - "3 - Number of files in this range or smaller as a % " - "of total # of files;\n" - "4 - Total size of files in range;\n" - "5 - Total size of files in range as a % of total " - "size of files;\n" - "6 - Total size of files in this range or smaller as " - "a % of total size of files;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_CAPACITY_USED: - return "\n\nCapacity used (Regular files space used on disk):\n" - "0 - Size range;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a percent of total " - "number of files;\n" - "3 - Number of files in this range or smaller as a % of " - "total # of file;\n" - "4 - Total capacity used in range;\n" - "5 - Total capacity used in range as a % of total " - "capacity used;\n" - "6 - Total capacity used in this range or smaller as " - "a % of total capacity;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_EQUAL_OVERHEAD: - return "\n\nEqual overhead (Files whose usable capacity is " - "equal to their size):\n" - "0 - Size range;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a percent of total " - "number of files;\n" - "3 - Number of files in this range or smaller as a % of " - "total # of files;\n" - "4 - Total kilobytes that overflowed in range;\n" - "5 - Total kilobytes that overflowed in this range as " - "a % of total kilobytes that overflowed;\n" - "6 - Total kilobytes that overflowed in this range or " - "smaller as a % of total kilobytes " - "that overflowed;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_POSITIVE_OVERHEAD: - return "\n\nPositive overhead (Files whose capacity used is " - "larger than to its size):\n" - "0 - Range of difference sizes;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a percent of total " - "number of files;\n" - "3 - Number of files in this range or smaller as a % of " - "total # of files;\n" - "4 - Total bytes that overflowed in range;\n" - "5 - Total bytes that overflowed in this range as a % " - "of total bytes that overflowed;\n" - "6 - Total bytes that overflowed in this range or " - "smaller as a % of total bytes that overflowed;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_NEGATIVE_OVERHEAD: - return "\n\nNegative overhead (Files whose size is larger than " - "capacity used.):\n" - "0 - Range of difference sizes;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a percent of total " - "number of files;\n" - "3 - Number of files in this range or smaller as a % of " - "total # of files;\n" - "4 - Total bytes that underflowed in range;\n" - "5 - Total bytes that underflowed in this range as a % " - "of total bytes that underflowed;\n" - "6 - Total bytes that underflowed in this range or " - "smaller as a % of total bytes that underflowed;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_DIRECTORY_SIZE_KB: - return "\n\nDirectory size (KB):\n" - "0 - Range of dir sizes;\n" - "1 - Count of directories in range;\n" - "2 - Number of files in dirs as a percent of total " - "number of dirs;\n" - "3 - Number of dirs in this range or smaller as a % of " - "total # of dirs;\n" - "4 - Total size of dirs in range;\n" - "5 - Total size of dirs in range as a % of total size " - "of dirs;\n" - "6 - Total size of dirs in this range or smaller as a % " - "of total size of dirs;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_TIME_SINCE_LAST_CREATION_RF: - return "\n\nTime since creation [crtime](Regular files):\n" - "0 - Range in days;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a % of total number " - "of files;\n" - "3 - Number of files in this range or smaller as a % " - "of total number of files;\n" - "4 - Total kilobytes that underflowed in range;\n" - "5 - Total of age in range, total age in range as a % " - "of total age of all files;\n" - "6 - total age in this range or smaller as a % of " - "total age of all files;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_TIME_SINCE_LAST_MOD_RF: - return "\n\nTime since last modification [mtime] " - "(Regular files):\n" - "0 - Range in days;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a % of total number " - "of files;\n" - "3 - Number of files in this range or smaller as a % " - "of total number of files;\n" - "4 - Total kilobytes that underflowed in range;\n" - "5 - Total of age in range, total age in range as a % " - "of total age of all files;\n" - "6 - total age in this range or smaller as a % of total " - "age of all files;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF: - return "\n\nTime since last metadata modification [ctime] " - "(Regular files):\n" - "0 - Range in days;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a % of total number " - "of files;\n" - "3 - Number of files in this range or smaller as a % " - "of total number of files;\n" - "4 - Total kilobytes that underflowed in range;\n" - "5 - Total of age in range, total age in range as a % " - "of total age of all files;\n" - "6 - total age in this range or smaller as a % of total " - "age of all files;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_TIME_SINCE_LAST_ACCESS_RF: - return "\n\nTime since last access [atime](Regular files):\n" - "0 - Range in days;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a % of total number " - "of files;\n" - "3 - Number of files in this range or smaller as a % " - "of total number of files;\n" - "4 - Total kilobytes that underflowed in range;\n" - "5 - Total of age in range, total age in range as a % " - "of total age of all files;\n" - "6 - total age in this range or smaller as a % of total " - "age of all files;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_FILENAME_LENGTH: - return "\n\nFilename length (Includes regular files, dirs, " - "symlinks and special files):\n" - "0 - Range of filename lengths;\n" - "1 - Count of files in range;\n" - "2 - Count of files in range as a % of total number " - "of files;\n" - "3 - Number of files in this range or smaller as a % " - "of total # of files;\n" - "4 - Total chars of filenames in range;\n" - "5 - Total chars of filenames in range as a % of total " - "number of chars of filenames;\n" - "6 - Total number of chars of filenames in this range " - "or smaller as a % of total number bytes " - "of filenames;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_DIRECTORY_SIZE_ENTRIES: - return "\n\nDirectory size (entries):\n" - "0 - Range of entries;\n" - "1 - Count of directories in range;\n" - "2 - Number of dirs in range as a % of total " - "num of dirs;\n" - "3 - Number of dirs in this range or smaller as " - "a % total number of dirs;\n" - "4 - Total entries in range;\n" - "5 - Number of entries in range as a % of total number " - "of entries;\n" - "6 - Number of entries in this range or smaller as a % " - "of total number of entries;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_USER: - return "\n\nStorage size by user (Regular files):\n" - "0 - User ID (name);\n" - "1 - Count of files for user;\n" - "2 - Number of files for user as a percent of total " - "number of files;\n" - "3 - Number of files for user as a % of total # " - "of files;\n" - "4 - Total size of files for user;\n" - "5 - Total size of files for user as a % of total size " - "of files;\n" - "6 - Total size of files for user or smaller as a % of " - "total size of files;\n" - "7 - Minimum size for user;\n" - "8 - Maximum size for user.\n"; - case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_GROUP: - return "\n\nStorage size by group (Regular files):\n" - "0 - Group ID (name);\n" - "1 - Count of files for group;\n" - "2 - Number of files for group as a percent of total " - "number of files;\n" - "3 - Number of files for group as a % of " - "total # of files;\n" - "4 - Total size of files for group;\n" - "5 - Total size of files for group as a % of total " - "size of files;\n" - "6 - Total size of files for group or smaller as " - "a % of total size of files;\n" - "7 - Minimum size for group;\n" - "8 - Maximum size for group.\n"; - case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_PROJID: - return "\n\nStorage size by projid (Regular files):\n" - "0 - Project ID;\n" - "1 - Count of files for projid;\n" - "2 - Number of files for projid as a percent of total " - "number of files;\n" - "3 - Number of files for projid as a % of " - "total # of files;\n" - "4 - Total size of files for projid;\n" - "5 - Total size of files for projid as a % of total " - "size of files;\n" - "6 - Total size of files for projid or smaller as a % " - "of total size of files;\n" - "7 - Minimum size for projid;\n" - "8 - Maximum size for projid.\n"; - case LS3_STATS_TYPE_LINK_COUNT: - return "\n\nLink count (of regular files):\n" - "0 - Range of link counts;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a % of total files;\n" - "3 - Number of files in this range or smaller as " - "a % of total files;\n" - "4 - Total links in range;\n" - "5 - Total links in range as a % of total links;\n" - "6 - Total links in this range or smaller as " - "a % of total links;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_STRIPE_COUNT: - return "\n\nStripe count (of regular files):\n" - "0 - Range of stripe count;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a % of total files;\n" - "3 - Number of files in this range as a % of " - "total files;\n" - "4 - Total stripe in range;\n" - "5 - Total stripe in range as a % of total stripe;\n" - "6 - Total stripe in this range as a % " - "of total stripe;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_STRIPE_SIZE: - return "\n\nStripe size (of regular files):\n" - "0 - Stripe size;\n" - "1 - Count of files with stripe size;\n" - "2 - Number of files with stripe size as a % " - "of total files;\n" - "3 - Number of files with this stripe size as a % " - "of total files;\n" - "4 - Total file size with this stripe size;\n" - "5 - Total file size with stripe size as a % " - "of total size;\n" - "6 - Total file size with this stripe size or smaller " - "as a % of total file;\n" - "7 - Minimum value with this stripe size;\n" - "8 - Maximum value with this stripe size.\n"; - case LS3_STATS_TYPE_MIRROR_COUNT: - return "\n\nMirror count (of regular files):\n" - "0 - Range of mirror count;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a % of total files;\n" - "3 - Number of files in this range as a % " - "of total files;\n" - "4 - Total mirror in range;\n" - "5 - Total mirror in range as a % of total mirror;\n" - "6 - Total mirror in this range as a % of " - "total mirror;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_COMPRESSION_RATIO: - return "\n\nCompression ratio (of regular files):\n" - "0 - Compression ratio range;\n" - "1 - Count of files in range;\n" - "2 - Number of files in range as a percent of total " - "number of files;\n" - "3 - Number of files in this range or smaller as a % " - "of total # of files;\n" - "4 - Total compression size of files in range;\n" - "5 - Total compression size of files in range as a % of " - "total compression size of files;\n" - "6 - Total compression size of files in this range or " - "smaller as a % of total compression size of files;\n" - "7 - Minimum value in range;\n" - "8 - Maximum value in range.\n"; - case LS3_STATS_TYPE_FILES_EMPLOYING_DOM: - case LS3_STATS_TYPE_SYMLINK_TARGET_LENGTH: - case LS3_STATS_TYPE_COMPRESSION_FLAG: - case LS3_STATS_TYPE_TOTAL_COUNT_REPORT: - return ""; - } - return ""; -} - static void ls3_stats_get_range_str(struct range_report_template *range_ptr, - char *str, size_t strlen, int range, ls3_stats_report_type r_type, - char *value_ts) + char *str, size_t strlen, ls3_stats_report_type r_type) { /* Different type of "range" field */ switch (r_type) @@ -763,46 +1060,58 @@ static void ls3_stats_get_range_str(struct range_report_template *range_ptr, ls3_stats_get_groupname_from_gid(range_ptr->rrt_id)); break; case LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_PROJID: - snprintf(str, strlen, "%26ld ", range_ptr->rrt_id); + snprintf(str, strlen, "%12ld", range_ptr->rrt_id); break; case LS3_STATS_TYPE_STRIPE_COUNT: - snprintf(str, strlen, "%26ld ", range_ptr->rrt_id); + snprintf(str, strlen, "%12ld", range_ptr->rrt_id); break; case LS3_STATS_TYPE_STRIPE_SIZE: - snprintf(str, strlen, "%18ld %7s ", range_ptr->rrt_id, - value_ts); + snprintf(str, strlen, "%11s", + ls3_stats_fmt_size_units(range_ptr->rrt_id * 1024)); break; case LS3_STATS_TYPE_MIRROR_COUNT: - snprintf(str, strlen, "%26ld ", range_ptr->rrt_id); + snprintf(str, strlen, "%10ld", range_ptr->rrt_id); + break; + case LS3_STATS_TYPE_FILENAME_LENGTH: + snprintf(str, strlen, " %4.0f - %4.0f chars", + range_ptr->range_start, range_ptr->range_end); break; case LS3_STATS_TYPE_TIME_SINCE_LAST_ACCESS_RF: case LS3_STATS_TYPE_TIME_SINCE_LAST_CREATION_RF: case LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF: case LS3_STATS_TYPE_TIME_SINCE_LAST_MOD_RF: - if (range_ptr->range_start >= 16384) + if (range_ptr->range_start >= 8192) snprintf(str, strlen, - " Never changed "); + " Never changed "); else snprintf(str, strlen, - "%8.0f - %-8.0f %7s", + "%4.0f - %4.0f days", range_ptr->range_start, - range_ptr->range_end, value_ts); + range_ptr->range_end); break; case LS3_STATS_TYPE_COMPRESSION_RATIO: snprintf(str, strlen, - "%11.2f - %-11.2f %%", range_ptr->range_start, + "%6.2f - %6.2f %%", range_ptr->range_start, range_ptr->range_end); break; case LS3_STATS_TYPE_FILES_SIZE: - case LS3_STATS_TYPE_CAPACITY_USED: - snprintf(str, strlen, - "%10.0f - %-10.0f %2s", range_ptr->range_start, - range_ptr->range_end, value_ts); + case LS3_STATS_TYPE_POSITIVE_OVERHEAD: + case LS3_STATS_TYPE_NEGATIVE_OVERHEAD: + case LS3_STATS_TYPE_EQUAL_OVERHEAD: + case LS3_STATS_TYPE_CAPACITY_USED: { + char *range_start = xstrdup(ls3_stats_fmt_size_units( + range_ptr->range_start * 1024)); + char *range_end = xstrdup(ls3_stats_fmt_size_units( + range_ptr->range_end * 1024)); + + snprintf(str, strlen, "%10s - %10s", range_start, range_end); + free(range_start); + free(range_end); break; - default: + } default: snprintf(str, strlen, - "%8.0f - %-8.0f %7s", range_ptr->range_start, - range_ptr->range_end, value_ts); + "%8.0f - %8.0f", range_ptr->range_start, + range_ptr->range_end); break; } } @@ -857,7 +1166,6 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) struct json_object *jobj_errors = json_object_new_object(); const ls3_stats_err_mapping *err; const char *json_str; - char *title_text; char *value_ts; char *second_value_ts; int i, j, k; @@ -909,6 +1217,7 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) struct json_object *jobj_report; struct json_object *jobj_general; struct json_object *jobj_ranges; + const ls3_stats_out_mapping *text_tmpl = &ls3_out_mappings[i]; report_ptr = reports_with_stats->reports[i]; if (report_ptr == NULL) @@ -921,14 +1230,13 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) ls3_stats_get_str_value_type(report_ptr->value_type); second_value_ts = ls3_stats_get_str_value_type(report_ptr->second_value_type); - title_text = ls3_stats_get_title_text(i); jobj_report = json_object_new_object(); jobj_general = json_object_new_object(); jobj_ranges = json_object_new_array(); json_object_object_add(jobj_general, "Title", - json_object_new_string(title_text)); + json_object_new_string(text_tmpl->som_table_title)); json_object_object_add(jobj_general, "Count", json_object_new_int64(report_ptr->files_count)); @@ -965,7 +1273,7 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) continue; /* Nothing for print */ ls3_stats_get_range_str(range_ptr, range_t, - sizeof(range_t), j, i, value_ts); + sizeof(range_t), i); json_object_object_add(jobj_range, "RangeInStr", json_object_new_string(range_t)); @@ -1041,6 +1349,7 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) for (j = 0; j < LS3_STATS_USERS_REPORTS_COUNTER; j++) { struct json_object *jobj_table = json_object_new_object(); struct json_object *jobj_ranges = json_object_new_array(); + char *title_text; title_text = ls3_stats_get_title_for_user_report(j); report_ptr = u_report_ptr->user_reports[j]; @@ -1161,7 +1470,7 @@ static void ls3_stats_print_to_yaml(const char *f_time, double e_time) fprintf(yaml_fd, "\n"); for (i = 0; i < LS3_STATS_TYPE_TOTAL_COUNT_REPORT; i++) { - char *title_text; + const ls3_stats_out_mapping *text_tmpl = &ls3_out_mappings[i]; report_ptr = reports_with_stats->reports[i]; if (report_ptr == NULL) @@ -1173,7 +1482,6 @@ static void ls3_stats_print_to_yaml(const char *f_time, double e_time) value_ts = ls3_stats_get_str_value_type(report_ptr->value_type); second_value_ts = ls3_stats_get_str_value_type(report_ptr->second_value_type); - title_text = ls3_stats_get_title_text(i); fprintf(yaml_fd, "\n%s:\n" @@ -1184,7 +1492,7 @@ static void ls3_stats_print_to_yaml(const char *f_time, double e_time) " Avg: %.2f %s\n" " Total: %lu %s\n" " Ranges:\n", - title_text, report_ptr->files_count, + text_tmpl->som_table_title, report_ptr->files_count, report_ptr->rt_min, value_ts, report_ptr->rt_max, value_ts, report_ptr->rt_avg, value_ts, @@ -1204,8 +1512,7 @@ static void ls3_stats_print_to_yaml(const char *f_time, double e_time) /* Different type of "range" field */ ls3_stats_get_range_str(range_ptr, range_t, - sizeof(range_t), - j, i, value_ts); + sizeof(range_t), i); fprintf(yaml_fd, " - RangeID: %i\n" @@ -1296,8 +1603,6 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time) struct report_template *report_ptr; struct range_report_template *range_ptr; const ls3_stats_err_mapping *err; - char *second_value_ts; - char *value_ts; FILE *out_fd; int i, j, k; @@ -1332,7 +1637,8 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time) fprintf(out_fd, "\n"); for (i = 0; i < LS3_STATS_TYPE_TOTAL_COUNT_REPORT; i++) { - char *header_text; + struct ls3_stats_range_val srv_basic_str; + const ls3_stats_out_mapping *text_tmpl; report_ptr = reports_with_stats->reports[i]; if (report_ptr == NULL) @@ -1341,26 +1647,24 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time) if (report_ptr->files_count == 0) continue; /* Nothing for print */ - value_ts = ls3_stats_get_str_value_type(report_ptr->value_type); - second_value_ts = - ls3_stats_get_str_value_type(report_ptr->second_value_type); - header_text = ls3_stats_get_header_text(i); - - fprintf(out_fd, "%s\nGeneral: Count: %lu | Min value in ranges:" - " %.2f %s | Max value in ranges: %.2f %s |" - " Average value in ranges: %.2f %s |" - " Total value: %lu %s\n\n", - header_text, report_ptr->files_count, report_ptr->rt_min, - value_ts, report_ptr->rt_max, value_ts, - report_ptr->rt_avg, value_ts, report_ptr->total_value, - (second_value_ts[0] != '\0') ? second_value_ts : value_ts); - - fprintf(out_fd, "[------------- 0 ------------] [---- 1 -----]" - " [-- 2 --] [------- 3 --------] [-------- 4 --------]" - " [-- 5 --] [------- 6 --------] [-------- 7 --------]" - " [-------- 8 --------]\n"); + text_tmpl = &ls3_out_mappings[i]; + srv_basic_str = ls3_stats_get_range_val_str(report_ptr, + NULL, i, false); + + /* fprintf table title */ + fprintf(out_fd, "\n\n%s\n", text_tmpl->som_table_title); + fprintf(out_fd, text_tmpl->som_fmt_basic_str, + report_ptr->files_count, + ls3_stats_get_count_skipped_obj(i), + srv_basic_str.srv_min, + srv_basic_str.srv_max, + srv_basic_str.srv_avg, + srv_basic_str.srv_total); + fprintf(out_fd, "\n%s\n", text_tmpl->som_table_description); + fprintf(out_fd, text_tmpl->som_table_header); for (j = 0; j < report_ptr->count_ranges; j++) { + struct ls3_stats_range_val srv_str; char range_t[30]; /* size in 30 byte enough */ range_ptr = report_ptr->fs_ranges[j]; @@ -1372,28 +1676,21 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time) /* Different type of "range" field */ ls3_stats_get_range_str(range_ptr, range_t, - sizeof(range_t), - j, i, value_ts); - - fprintf(out_fd, "[ %s] [ %11lu] [%6.2f%%]" - " [%6.2f%% cumulative] [%11lu %7s] [%6.2f%%]" - " [%6.2f%% cumulative] [%11.2f %7s] [%11.2f %7s]\n", + sizeof(range_t), i); + srv_str = ls3_stats_get_range_val_str(report_ptr, + range_ptr, + i, true); + fprintf(out_fd, text_tmpl->som_fmt_range_str, range_t, range_ptr->count_in_range, range_ptr->percentage, range_ptr->cumulative_percentage, - range_ptr->total_in_range, - (second_value_ts[0] != '\0') ? - second_value_ts : value_ts, - range_ptr->percent_in_range, + srv_str.srv_total, range_ptr->percent_in_range, range_ptr->percent_cumulative_in_range, - range_ptr->rrt_min, value_ts, - range_ptr->rrt_max, value_ts); + srv_str.srv_min, srv_str.srv_max); } - fprintf(out_fd, "[----------------------------] [------------]" - " [-------] [------------------] [-------------------]" - " [-------] [------------------] [-------------------]" - " [-------------------]\n\n"); + fprintf(out_fd, text_tmpl->som_table_header); + fprintf(out_fd, "\n\n"); } @@ -1404,9 +1701,12 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time) if (!u_report_ptr) continue; - fprintf(out_fd, "\n\n[ user:%s uid:%lu ]\n", + fprintf(out_fd, "\n\n[ user:%s uid:%lu ]\n%s", ls3_stats_get_username_from_uid(u_report_ptr->uid), - u_report_ptr->uid); + u_report_ptr->uid, + "_____________________________________________" + "_______________________________________\n"); + for (j = 0; j < LS3_STATS_USERS_REPORTS_COUNTER; j++) { char *title = ls3_stats_get_title_for_user_report(j); @@ -1414,45 +1714,46 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time) if (report_ptr == NULL) LS3_FATAL("Unable to access the user report data: %d\n", j); - value_ts = - ls3_stats_get_str_value_type(report_ptr->value_type); - second_value_ts = - ls3_stats_get_str_value_type(report_ptr->second_value_type); - fprintf(out_fd, "[ %-50s %79s ]\n", title, " " ); - fprintf(out_fd, "[-----------------------------" - "--------------------------------------" - "--------------------------------------" - "----------------------------]\n"); - fprintf(out_fd, "[--------Range by day--------]" - " [-Count files-] [------Min size-----]" - " [------Max size-----] [----Total size-----]" - " [--Last time access--]\n"); + fprintf(out_fd, "\n%-50s:\n_range_by_days_ " + "_count_files_ _min_size_ _max_size_ " + "total_size __last_time_access__\n", title); for (k = 0; k < report_ptr->count_ranges; k++) { - char buffer[20]; + char date_str[20]; + char tmp_min[LS3_STATS_NUM_STR_MAX_WITH_NULL]; + char tmp_max[LS3_STATS_NUM_STR_MAX_WITH_NULL]; + char tmp_total[LS3_STATS_NUM_STR_MAX_WITH_NULL]; range_ptr = report_ptr->fs_ranges[k]; if (range_ptr == NULL) continue; - ls3_stats_get_time_str(buffer, sizeof(buffer), + snprintf(tmp_total, sizeof(tmp_total), "%s", + ls3_stats_fmt_size_units(1024 * + range_ptr->total_in_range)); + + snprintf(tmp_min, sizeof(tmp_min), "%s", + ls3_stats_fmt_size_units(1024 * + range_ptr->rrt_min)); + + snprintf(tmp_max, sizeof(tmp_max), "%s", + ls3_stats_fmt_size_units(1024 * + range_ptr->rrt_max)); + + ls3_stats_get_time_str(date_str, + sizeof(date_str), range_ptr->last_time_access); - fprintf(out_fd, "[ %8.0f - %8.0f %7s] [ %11lu ]" - " [%11.0f %7s] [%11.0f %7s]" - " [%11lu %7s] [%20s]\n", + fprintf(out_fd, "%6.0f - %6.0f %13lu %10s %10s" + "%10s %20s\n", range_ptr->range_start, - range_ptr->range_end, value_ts, + range_ptr->range_end, range_ptr->count_in_range, - range_ptr->rrt_min, second_value_ts, - range_ptr->rrt_max, second_value_ts, - range_ptr->total_in_range, - second_value_ts, buffer); + tmp_min, tmp_max, tmp_total, date_str); } - fprintf(out_fd, "[----------------------------]" - " [-------------] [-------------------]" - " [-------------------] [-------------------]" - " [--------------------]\n\n"); + fprintf(out_fd, "_range_by_days_ _count_files_ " + "_min_size_ _max_size_ total_size " + "__last_time_access__\n"); } } @@ -1501,8 +1802,8 @@ static void ls3_stats_print_to_csv(const char *f_time, double e_time) fprintf(csv_fd, "\n"); for (i = 0; i < LS3_STATS_TYPE_TOTAL_COUNT_REPORT; i++) { + const ls3_stats_out_mapping *text_tmpl = &ls3_out_mappings[i]; char *value_ts; - char *title_text; char *second_value_ts; report_ptr = reports_with_stats->reports[i]; @@ -1515,9 +1816,8 @@ static void ls3_stats_print_to_csv(const char *f_time, double e_time) value_ts = ls3_stats_get_str_value_type(report_ptr->value_type); second_value_ts = ls3_stats_get_str_value_type(report_ptr->second_value_type); - title_text = ls3_stats_get_title_text(i); - fprintf(csv_fd, "Histogram,%s\n", title_text); + fprintf(csv_fd, "Histogram,%s\n", text_tmpl->som_table_title); fprintf(csv_fd, "General,\nCount,%lu\nMinValueInRanges,%.2f,%s\n" "MaxValueInRanges,%.2f,%s\nAverageValueInRanges,%.2f,%s\n" "TotalValue,%lu,%s\n\n", @@ -1544,8 +1844,7 @@ static void ls3_stats_print_to_csv(const char *f_time, double e_time) /* Different type of "range" field */ ls3_stats_get_range_str(range_ptr, range_t, - sizeof(range_t), - j, i, value_ts); + sizeof(range_t), i); fprintf(csv_fd, "%s,%lu,%.2f,%.2f,%lu %s,%.2f,%.2f," "%.2f %s,%.2f %s\n", range_t, range_ptr->count_in_range, @@ -1730,7 +2029,7 @@ static struct range_report_template *ls3_stats_get_new_range_ptr(uint64_t res) struct range_report_template *range_ptr; range_ptr = xcalloc(1, sizeof(struct range_report_template)); - range_ptr->rrt_min = UINT64_MAX; + range_ptr->rrt_min = DBL_MAX; if (res >= 2) { range_ptr->range_start = (int)pow(2, floor(log2(res))); range_ptr->range_end = range_ptr->range_start * 2; @@ -1747,7 +2046,7 @@ static struct range_report_template *ls3_stats_get_new_range_double_ptr(double r struct range_report_template *range_ptr; range_ptr = xcalloc(1, sizeof(struct range_report_template)); - range_ptr->rrt_min = UINT64_MAX; + range_ptr->rrt_min = DBL_MAX; /* 0.001 it's rounding for exception handling with integers */ range_ptr->range_start = floor((res + 0.001) / 0.2) * 0.2; @@ -2209,30 +2508,12 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all) LS3_STATS_ERROR_EMPTY_SIZE); } - /* In most cases loa_all->loa_blocks is not available on MDT. */ - if (!reports_with_stats->device_is_mdt && allocate_file_size != 0) { - /* Get equal overhead (Files whose size is equal - * than capacity used. */ - if (allocate_file_size == loa_all->loa_size) - ls3_stats_update_range(LS3_STATS_TYPE_EQUAL_OVERHEAD, - loa_all->loa_size, - LS3_STATS_EMPTY_VALUE); - /* Get positive overhead (Files whose capacity used is - * larger than to its size. */ - else if (allocate_file_size > loa_all->loa_size) - ls3_stats_update_range(LS3_STATS_TYPE_POSITIVE_OVERHEAD, - allocate_file_size - - loa_all->loa_size, - LS3_STATS_EMPTY_VALUE); - /* Get negative overhead (Files whose size is larger than - * capacity used. Includes directories, symlinks and - * regular files). */ - else - ls3_stats_update_range(LS3_STATS_TYPE_NEGATIVE_OVERHEAD, - loa_all->loa_size - - allocate_file_size, - LS3_STATS_EMPTY_VALUE); - } + /* Size statistics for directories */ + ls3_stats_dir_distributor(loa_all); + + /* Reports only for regular file's */ + if (!(loa_all->loa_mode & S_IFREG)) + return 0; /* Get filename length (Includes regular files, dirs, symlinks * and special files) */ @@ -2243,58 +2524,65 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all) LS3_STATS_EMPTY_VALUE); } - /* Size statistics for directories */ - ls3_stats_dir_distributor(loa_all); + ls3_stats_update_range(LS3_STATS_TYPE_FILES_SIZE, loa_all->loa_size, + LS3_STATS_EMPTY_VALUE); - /* Reports only for regular file's */ - if (!(loa_all->loa_mode & S_IFREG)) + if (allocate_file_size == 0) return 0; - if (allocate_file_size != 0) { - ls3_stats_update_range(LS3_STATS_TYPE_CAPACITY_USED, - allocate_file_size, + if (loa_all->loa_size != 0 && allocate_file_size == loa_all->loa_size) + ls3_stats_update_range(LS3_STATS_TYPE_EQUAL_OVERHEAD, + allocate_file_size, LS3_STATS_EMPTY_VALUE); + + /* Get positive overhead (Files whose capacity used is + * larger than to its size. */ + else if (loa_all->loa_size != 0 && allocate_file_size > loa_all->loa_size) + ls3_stats_update_range(LS3_STATS_TYPE_POSITIVE_OVERHEAD, + allocate_file_size - loa_all->loa_size, + LS3_STATS_EMPTY_VALUE); + /* Get negative overhead (Files whose size is larger than + * capacity used. Includes directories, symlinks and + * regular files). */ + else if (loa_all->loa_size != 0) + ls3_stats_update_range(LS3_STATS_TYPE_NEGATIVE_OVERHEAD, + loa_all->loa_size - allocate_file_size, LS3_STATS_EMPTY_VALUE); - /* For compression statistics */ - cmps_ratio = (double)loa_all->loa_size / - (double)allocate_file_size; - cmps_fsize = loa_all->loa_size - allocate_file_size; - } - ls3_stats_update_range(LS3_STATS_TYPE_FILES_SIZE, loa_all->loa_size, + ls3_stats_update_range(LS3_STATS_TYPE_CAPACITY_USED, allocate_file_size, LS3_STATS_EMPTY_VALUE); ls3_stats_update_range(LS3_STATS_TYPE_TIME_SINCE_LAST_MOD_RF, - loa_all->loa_mtime, loa_all->loa_size); + loa_all->loa_mtime, allocate_file_size); ls3_stats_update_range(LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF, - loa_all->loa_ctime, loa_all->loa_size); + loa_all->loa_ctime, allocate_file_size); ls3_stats_update_range(LS3_STATS_TYPE_TIME_SINCE_LAST_CREATION_RF, - loa_all->loa_crtime, loa_all->loa_size); + loa_all->loa_crtime, allocate_file_size); ls3_stats_update_range(LS3_STATS_TYPE_TIME_SINCE_LAST_ACCESS_RF, - loa_all->loa_atime, loa_all->loa_size); + loa_all->loa_atime, allocate_file_size); ls3_stats_update_range_with_id(LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_USER, loa_all->loa_uid, - loa_all->loa_size); + allocate_file_size); ls3_stats_update_range_with_id(LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_GROUP, loa_all->loa_gid, - loa_all->loa_size); + allocate_file_size); ls3_stats_update_range_with_id(LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_PROJID, loa_all->loa_projid, - loa_all->loa_size); + allocate_file_size); ls3_stats_update_user_report_range(LS3_STATS_USERS_REPORTS_MOD_RF, loa_all->loa_uid, loa_all->loa_mtime, - loa_all->loa_size); + allocate_file_size); ls3_stats_update_user_report_range(LS3_STATS_USERS_REPORTS_MD_MOD_RF, loa_all->loa_uid, loa_all->loa_ctime, - loa_all->loa_size); + allocate_file_size); ls3_stats_update_user_report_range(LS3_STATS_USERS_REPORTS_CREATION_RF, loa_all->loa_uid, loa_all->loa_crtime, - loa_all->loa_size); + allocate_file_size); ls3_stats_update_user_report_range(LS3_STATS_USERS_REPORTS_ACCESS_RF, loa_all->loa_uid, loa_all->loa_atime, - loa_all->loa_size); + allocate_file_size); if (reports_with_stats->device_is_mdt) { uint16_t mirror_count = 0; @@ -2322,7 +2610,7 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all) } else { ls3_stats_update_range_with_id(LS3_STATS_TYPE_MIRROR_COUNT, mirror_count, - loa_all->loa_size); + allocate_file_size); } rc = llapi_layout_stripe_count_get(loa_all->loa_layout, @@ -2334,7 +2622,7 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all) ls3_stats_update_range_with_id( LS3_STATS_TYPE_STRIPE_COUNT, stripe_count, - loa_all->loa_size); + allocate_file_size); } rc = llapi_layout_stripe_size_get(loa_all->loa_layout, @@ -2345,7 +2633,7 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all) } else { ls3_stats_update_range_with_id(LS3_STATS_TYPE_STRIPE_SIZE, stripe_size, - loa_all->loa_size); + allocate_file_size); } rc = llapi_layout_compress_get(loa_all->loa_layout, @@ -2370,7 +2658,7 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all) ls3_stats_update_range_with_id(LS3_STATS_TYPE_STRIPE_SIZE, ptr_ost_layout.ol_stripe_size, - loa_all->loa_size); + allocate_file_size); if (layout_compr.ol_compr_type == LL_COMPR_TYPE_NONE) return 0; /* skip compression statistics */ @@ -2379,10 +2667,13 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all) * with using struct ost_layout_compr */ } - if (allocate_file_size == 0 || - loa_all->loa_size == 0 || allocate_file_size > loa_all->loa_size) + if (loa_all->loa_size == 0 || allocate_file_size > loa_all->loa_size) return 0; /* skip compression statistics */ + cmps_ratio = (double)loa_all->loa_size / + (double)allocate_file_size; + cmps_fsize = loa_all->loa_size - allocate_file_size; + /* Сompression statistics by ratio MDT/OST */ ls3_stats_update_range_double(LS3_STATS_TYPE_COMPRESSION_RATIO, cmps_ratio, cmps_fsize); diff --git a/lipe/src/lipe_scan3/ls3_stats.h b/lipe/src/lipe_scan3/ls3_stats.h index 4e12352..d730823 100644 --- a/lipe/src/lipe_scan3/ls3_stats.h +++ b/lipe/src/lipe_scan3/ls3_stats.h @@ -29,6 +29,7 @@ #define LS3_STATS_EMPTY_VALUE 0 #define LS3_STATS_COUNT_RANGE_BY_DEFAULT 10 #define LS3_STATS_ARRAY_SIZE_BY_DEFAULT 100 +#define LS3_STATS_NUM_STR_MAX_WITH_NULL 20 #define DEFAULT_DIR_FOR_REPORTS "/var/log/lustre/lipe/reports/" /* ls3_stats_file_extension - bitmap to select report extension */ @@ -81,22 +82,36 @@ typedef enum { LS3_STATS_TYPE_DIRECTORY_SIZE_KB, LS3_STATS_TYPE_LINK_COUNT, LS3_STATS_TYPE_FILENAME_LENGTH, - LS3_STATS_TYPE_SYMLINK_TARGET_LENGTH, LS3_STATS_TYPE_TIME_SINCE_LAST_MOD_RF, LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF, LS3_STATS_TYPE_TIME_SINCE_LAST_CREATION_RF, LS3_STATS_TYPE_TIME_SINCE_LAST_ACCESS_RF, - LS3_STATS_TYPE_FILES_EMPLOYING_DOM, LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_USER, LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_GROUP, LS3_STATS_TYPE_STORAGE_SET_SIZE_BY_PROJID, LS3_STATS_TYPE_STRIPE_COUNT, LS3_STATS_TYPE_STRIPE_SIZE, LS3_STATS_TYPE_MIRROR_COUNT, - LS3_STATS_TYPE_COMPRESSION_FLAG, LS3_STATS_TYPE_TOTAL_COUNT_REPORT /* Should always be the last */ } ls3_stats_report_type; +typedef struct { + const char *som_table_title; + const char *som_table_description; + const char *som_table_header; + const char *som_fmt_range_str; + const char *som_fmt_basic_str; + ls3_stats_report_type som_table_type; +} ls3_stats_out_mapping; + +struct ls3_stats_range_val { + uint32_t srv_skipped; + char srv_min[LS3_STATS_NUM_STR_MAX_WITH_NULL]; + char srv_max[LS3_STATS_NUM_STR_MAX_WITH_NULL]; + char srv_avg[LS3_STATS_NUM_STR_MAX_WITH_NULL]; + char srv_total[LS3_STATS_NUM_STR_MAX_WITH_NULL]; +}; + typedef enum { LS3_STATS_VALUE_TYPE_ERROR, LS3_STATS_VALUE_TYPE_KB, diff --git a/lustre/tests/sanity-lipe-scan3.sh b/lustre/tests/sanity-lipe-scan3.sh index 7c413a7..ef5674e 100644 --- a/lustre/tests/sanity-lipe-scan3.sh +++ b/lustre/tests/sanity-lipe-scan3.sh @@ -1150,6 +1150,48 @@ test_306() { } run_test 306 "lipe_scan3 --collect-fsize-stats works correctly" +# Test for --collect-fsize-stats +# This test does not require any checks, and serves only to show the +# contents of the report with statistics on file sizes/directories +# obtained through the option --collect-fsize-stats +test_307() { + local facet=mds1 + local main_dir=$DIR/$tdir + local report_path="$MOUNT/$tfile.out" + local file=$MOUNT/$tfile + init_lipe_scan3_env "$file" + + mkdir -p $main_dir + cd $main_dir + + for i in {1..10}; do + local dir_name="$main_dir"/tmp_dir"$i" + local num_files=$((RANDOM % 21 + 20)) + + mkdir -p "$dir_name" + cd "$dir_name" + for ((j = 1; j <= num_files; j++)); do + local f_name="$tfile".file"$j" + local file_size=$((RANDOM % 10 + 1)) + + fallocate -l "$file_size"M "$f_name" + lfs somsync "$f_name" + done + cd .. + done + + fallocate -l 200G "$file" + lfs somsync "$file" + sync + + out=$(lipe_scan3_facet "$facet" --collect-fsize-stats="$report_path") + [[ $? -ne 0 ]] && error "lipe_scan3 collect-fsize-stats failed" + + cat "$report_path" + rm -rf $main_dir +} +run_test 307 "lipe_scan3 --collect-fsize-stats report.out" + # loading and scripts test_400() { -- 1.8.3.1