From d918739be34d9cbef2908dd9ba0cbf70f5eb8cab Mon Sep 17 00:00:00 2001 From: Vitaliy Kuznetsov Date: Wed, 8 May 2024 15:02:57 +0200 Subject: [PATCH] EX-9121 lipe: Trivial improvements for report merging Small changes that do not affect the functionality, but allow to reuse some functions in other parts of lipe3, for example in the utility for merging different stats reports. Test-Parameters: trivial testlist=sanity-lipe-scan3,sanity-lipe-find3 Signed-off-by: Vitaliy Kuznetsov Change-Id: I1bc2d4b22e57a369acea86bf60d8f460c5b3b093 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/55045 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- lipe/src/lipe_scan3/ls3_main.c | 6 +- lipe/src/lipe_scan3/ls3_stats.c | 370 ++++++++++++++++++++-------------------- lipe/src/lipe_scan3/ls3_stats.h | 26 ++- 3 files changed, 207 insertions(+), 195 deletions(-) diff --git a/lipe/src/lipe_scan3/ls3_main.c b/lipe/src/lipe_scan3/ls3_main.c index 0416a9b..1eea156 100644 --- a/lipe/src/lipe_scan3/ls3_main.c +++ b/lipe/src/lipe_scan3/ls3_main.c @@ -1094,7 +1094,7 @@ SCM_DEFINE(ls3_scm_collect_fsize_stats, LS3_COLLECT_STATS, 0, 1, 0, rc = ls3_stats_update_info(loa_all); if (rc) { - ls3_stats_destroy(); + ls3_stats_destroy(reports_with_stats); LS3_FATAL("Error: %d updating file size stats\n", rc); } @@ -1705,12 +1705,12 @@ static void ls3_main_scm(void *data, int argc, char *argv[]) } if (save_fsize_stats) - ls3_stats_printf(); + ls3_stats_printf(reports_with_stats); if (show_counters) ls3_counters_print(); - ls3_stats_destroy(); + ls3_stats_destroy(reports_with_stats); ls3_stats_dir_destroy(); LS3_DEBUG_D(exit_status); exit(exit_status); diff --git a/lipe/src/lipe_scan3/ls3_stats.c b/lipe/src/lipe_scan3/ls3_stats.c index d0b0458..c5dd321 100644 --- a/lipe/src/lipe_scan3/ls3_stats.c +++ b/lipe/src/lipe_scan3/ls3_stats.c @@ -803,26 +803,23 @@ static struct ls3_stats_range_val ls3_stats_get_range_val_str( return result; } -static uint32_t ls3_stats_get_count_skipped_obj( +static uint32_t ls3_stats_get_count_skipped_obj(struct fstats_report *report, ls3_stats_report_type report_type) { switch (report_type) { case LS3_STATS_TYPE_FILES_SIZE: - return LS3_GET_ERROR_COUNT(reports_with_stats, - LS3_STATS_ERROR_EMPTY_SIZE) + - LS3_GET_ERROR_COUNT(reports_with_stats, + return LS3_GET_ERROR_COUNT(report, LS3_STATS_ERROR_EMPTY_SIZE) + + LS3_GET_ERROR_COUNT(report, LS3_STATS_ERROR_WITHOUT_SIZE); case LS3_STATS_TYPE_COMPRESSION_RATIO: - return LS3_GET_ERROR_COUNT(reports_with_stats, - LS3_STATS_ERROR_EMPTY_SIZE) + - LS3_GET_ERROR_COUNT(reports_with_stats, + return LS3_GET_ERROR_COUNT(report, LS3_STATS_ERROR_EMPTY_SIZE) + + LS3_GET_ERROR_COUNT(report, LS3_STATS_ERROR_EMPTY_BLOCK_SIZE); case LS3_STATS_TYPE_FILENAME_LENGTH: - return LS3_GET_ERROR_COUNT(reports_with_stats, - LS3_STATS_ERROR_WITHOUT_SIZE); + return LS3_GET_ERROR_COUNT(report, LS3_STATS_ERROR_WITHOUT_SIZE); default: - return LS3_GET_ERROR_COUNT(reports_with_stats, + return LS3_GET_ERROR_COUNT(report, LS3_STATS_ERROR_EMPTY_BLOCK_SIZE); } @@ -848,7 +845,8 @@ static void ls3_stats_remove_extension_in_file(char *file_path) *last_dot = '\0'; } -static void ls3_stats_change_extension_in_file(int bit_extension) +static void ls3_stats_change_extension_in_file(struct fstats_report *report, + int bit_extension) { const ls3_stats_extension_mapping *ext = ext_mappings; @@ -857,20 +855,20 @@ static void ls3_stats_change_extension_in_file(int bit_extension) char buffer[256]; snprintf(buffer, sizeof(buffer), "%s.%s", - reports_with_stats->report_file_name, + report->report_file_name, ext->extension_name); - if (reports_with_stats->format_report_file_name) - free(reports_with_stats->format_report_file_name); + if (report->format_report_file_name) + free(report->format_report_file_name); - reports_with_stats->format_report_file_name = - xstrdup(buffer); + report->format_report_file_name = xstrdup(buffer); return; } } } -static void ls3_stats_get_report_extension(char *fileName) +static void ls3_stats_get_report_extension(struct fstats_report *report, + char *fileName) { const ls3_stats_extension_mapping *ext = ext_mappings; char *dot = strrchr(fileName, '.'); @@ -880,8 +878,7 @@ static void ls3_stats_get_report_extension(char *fileName) if(!dot || dot == fileName) { /* By default if filename not have extension */ - reports_with_stats->report_extension |= - LS3_STATS_FILE_EXTENSION_ALL; + report->report_extension |= LS3_STATS_FILE_EXTENSION_ALL; return; } @@ -892,8 +889,7 @@ static void ls3_stats_get_report_extension(char *fileName) for (ext = ext_mappings; ext->extension_name != NULL; ++ext) { if (strcmp(extension, ext->extension_name) == 0) { - reports_with_stats->report_extension |= - ext->extension_value; + report->report_extension |= ext->extension_value; marker = true; break; } @@ -901,8 +897,7 @@ static void ls3_stats_get_report_extension(char *fileName) /* By default if filename have no supported extension */ if (!marker) - reports_with_stats->report_extension |= - LS3_STATS_FILE_EXTENSION_ALL; + report->report_extension |= LS3_STATS_FILE_EXTENSION_ALL; } static FILE* ls3_stats_open_file(char *file_name) @@ -921,7 +916,7 @@ static FILE* ls3_stats_open_file(char *file_name) * 2. Generates a unique file name if the user didn't explicitly specify one. * 3. Determines the user-requested file extension for the report. */ -static void ls3_stats_prepare_file(void) +static void ls3_stats_prepare_file(struct fstats_report *report) { char tmp[256]; char *p = NULL; @@ -954,23 +949,23 @@ static void ls3_stats_prepare_file(void) } /* when using lipe_scan3 --collect-fsize-stats without path to file */ - if (!reports_with_stats->report_file_name) { + if (!report->report_file_name) { char format_buffer[512]; char file_name_buffer[512]; - struct tm *timeinfo = localtime(&reports_with_stats->start_time); + struct tm *timeinfo = localtime(&report->start_time); sprintf(format_buffer, DEFAULT_DIR_FOR_REPORTS "files_sizes_report_%s.%%Y-%%m-%%d-%%H:%%M:%%S.out", - reports_with_stats->device_name); + report->device_name); strftime(file_name_buffer, sizeof(file_name_buffer), format_buffer, timeinfo); - reports_with_stats->report_file_name = xstrdup(file_name_buffer); + report->report_file_name = xstrdup(file_name_buffer); } - ls3_stats_get_report_extension(reports_with_stats->report_file_name); - ls3_stats_remove_extension_in_file(reports_with_stats->report_file_name); + ls3_stats_get_report_extension(report, report->report_file_name); + ls3_stats_remove_extension_in_file(report->report_file_name); } static ls3_stats_val_type ls3_stats_get_value_type( @@ -1155,7 +1150,8 @@ static char* ls3_stats_get_title_for_user_report(ls3_stats_users_reports rtype) } } -static void ls3_stats_print_to_json(const char *f_time, double e_time) +static void ls3_stats_print_to_json(struct fstats_report *report, + const char *f_time, double e_time) { FILE *json_fd; struct report_template *report_ptr; @@ -1178,41 +1174,39 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) json_object_new_string(f_time)); json_object_object_add(jobj_main, "ProcessedFiles", - json_object_new_int64(reports_with_stats->total_count_files)); + json_object_new_int64(report->total_count_files)); json_object_object_add(jobj_main, "ProcessingTimeSecs", json_object_new_double(e_time)); - if (reports_with_stats->device_path) + if (report->device_path) json_object_object_add(jobj_main, "DevicePath", - json_object_new_string(reports_with_stats->device_path)); + json_object_new_string(report->device_path)); - if (reports_with_stats->device_name) + if (report->device_name) json_object_object_add(jobj_main, "DeviceName", - json_object_new_string(reports_with_stats->device_name)); + json_object_new_string(report->device_name)); json_object_object_add(jobj_main, "DeviceType", - json_object_new_string(reports_with_stats->device_is_mdt ? - "MDT" : "OST")); + json_object_new_string(report->device_is_mdt ? "MDT" : "OST")); json_object_object_add(jobj_main, "InodesCountOnDevice", - json_object_new_int(reports_with_stats->inodes_count)); + json_object_new_int(report->inodes_count)); json_object_object_add(jobj_main, "FreeInodesCountOnDevice", - json_object_new_int(reports_with_stats->free_inodes_count)); + json_object_new_int(report->free_inodes_count)); - if (reports_with_stats->client_mount_path) { + if (report->client_mount_path) { json_object_object_add(jobj_main, "ClientMountPath", json_object_new_string( - reports_with_stats->client_mount_path)); + report->client_mount_path)); } for (err = ls3_err_mappings; err->counter_num != LS3_STATS_ERROR_END_MARKER; ++err) { json_object_object_add(jobj_errors, err->json_err_header, - json_object_new_int( - LS3_GET_ERROR_COUNT(reports_with_stats, - err->counter_num))); + json_object_new_int(LS3_GET_ERROR_COUNT(report, + err->counter_num))); } json_object_object_add(jobj_main, "ErrorCounters", jobj_errors); @@ -1222,7 +1216,7 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) struct json_object *jobj_ranges; const ls3_stats_out_mapping *text_tmpl = &ls3_out_mappings[i]; - report_ptr = reports_with_stats->reports[i]; + report_ptr = report->reports[i]; if (report_ptr == NULL) LS3_FATAL("Unable to access the report data: %d\n", i); @@ -1340,12 +1334,12 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) json_object_object_add(jobj_main, "Reports", jobj_reports); /* Printf user time reports */ - for (i = 0; i <= reports_with_stats->last_user_idx_in_array; i++) { + for (i = 0; i <= report->last_user_idx_in_array; i++) { struct json_object *jobj_user_report; struct json_object *jobj_tables; const char *user_name; - u_report_ptr = reports_with_stats->users_reports[i]; + u_report_ptr = report->users_reports[i]; if (!u_report_ptr) continue; @@ -1368,11 +1362,6 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) if (report_ptr == NULL) LS3_FATAL("Unable to access the user report data: %d\n", j); - value_ts = - ls3_stats_get_str_value_type(report_ptr->value_type); - second_value_ts = - ls3_stats_get_str_value_type(report_ptr->second_value_type); - json_object_object_add(jobj_table, "Title", json_object_new_string(title_text)); @@ -1382,14 +1371,11 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) for (k = 0; k < report_ptr->count_ranges; k++) { struct json_object *jobj_range = json_object_new_object(); - char buffer[20]; range_ptr = report_ptr->fs_ranges[k]; if (range_ptr == NULL) continue; - ls3_stats_get_time_str(buffer, sizeof(buffer), - range_ptr->last_time_access); json_object_object_add(jobj_range, "RangeDayStart", json_object_new_int64( @@ -1418,10 +1404,6 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) "LastTimeAccess", json_object_new_int64( range_ptr->last_time_access)); - json_object_object_add(jobj_range, - "LastTimeAccessStr", - json_object_new_string(buffer)); - json_object_array_add(jobj_ranges, jobj_range); } json_object_object_add(jobj_table, "Ranges", jobj_ranges); @@ -1434,7 +1416,7 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) json_object_object_add(jobj_main, "UserTimeReports", jobj_users_reports); /* Try add dirs stats (TOP N rating table + tree) */ - if (reports_with_stats->client_mount_path) { + if (report->client_mount_path) { struct json_object *jobj_dirs_rating; jobj_dirs_rating = ls3_stats_fprintf_dirs_json(); @@ -1447,18 +1429,17 @@ static void ls3_stats_print_to_json(const char *f_time, double e_time) json_str = json_object_to_json_string_ext(jobj_main, JSON_C_TO_STRING_PRETTY); - ls3_stats_change_extension_in_file(LS3_STATS_FILE_EXTENSION_JSON); - - json_fd = - ls3_stats_open_file(reports_with_stats->format_report_file_name); + ls3_stats_change_extension_in_file(report, LS3_STATS_FILE_EXTENSION_JSON); + json_fd = ls3_stats_open_file(report->format_report_file_name); fprintf(json_fd, "%s", json_str); json_object_put(jobj_main); fclose(json_fd); - printf(" Path to json: %s\n", reports_with_stats->format_report_file_name); + printf(" Path to json: %s\n", report->format_report_file_name); } -static void ls3_stats_print_to_yaml(const char *f_time, double e_time) +static void ls3_stats_print_to_yaml(struct fstats_report *report, + const char *f_time, double e_time) { struct report_template *report_ptr; struct range_report_template *range_ptr; @@ -1468,8 +1449,8 @@ static void ls3_stats_print_to_yaml(const char *f_time, double e_time) FILE *yaml_fd; int i, j, k; - ls3_stats_change_extension_in_file(LS3_STATS_FILE_EXTENSION_YAML); - yaml_fd = ls3_stats_open_file(reports_with_stats->format_report_file_name); + ls3_stats_change_extension_in_file(report, LS3_STATS_FILE_EXTENSION_YAML); + yaml_fd = ls3_stats_open_file(report->format_report_file_name); fprintf(yaml_fd, "GeneratedBy: LIPE stats v%.2f\n" @@ -1483,27 +1464,26 @@ static void ls3_stats_print_to_yaml(const char *f_time, double e_time) "FreeInodesCountOnDevice: %u\n" "DeviceType: %s\n", LS3_STATS_VERSION, f_time, - reports_with_stats->total_count_files, e_time, - reports_with_stats->device_path, - reports_with_stats->device_name, - reports_with_stats->client_mount_path, - reports_with_stats->inodes_count, - reports_with_stats->free_inodes_count, - reports_with_stats->device_is_mdt ? "MDT" : "OST"); + report->total_count_files, e_time, + report->device_path, + report->device_name, + report->client_mount_path, + report->inodes_count, + report->free_inodes_count, + report->device_is_mdt ? "MDT" : "OST"); fprintf(yaml_fd, "\nError counters:\n"); for (err = ls3_err_mappings; err->counter_num != LS3_STATS_ERROR_END_MARKER; ++err) { fprintf(yaml_fd, " %s: %u\n", err->json_err_header, - LS3_GET_ERROR_COUNT(reports_with_stats, - err->counter_num)); + LS3_GET_ERROR_COUNT(report, err->counter_num)); } fprintf(yaml_fd, "\n"); for (i = 0; i < LS3_STATS_TYPE_TOTAL_COUNT_REPORT; i++) { const ls3_stats_out_mapping *text_tmpl = &ls3_out_mappings[i]; - report_ptr = reports_with_stats->reports[i]; + report_ptr = report->reports[i]; if (report_ptr == NULL) LS3_FATAL("Unable to access the report data: %d\n", i); @@ -1570,9 +1550,9 @@ static void ls3_stats_print_to_yaml(const char *f_time, double e_time) } fprintf(yaml_fd, "\nTime frame reports for all users:\n"); - for (i = 0; i <= reports_with_stats->last_user_idx_in_array; i++) { + for (i = 0; i <= report->last_user_idx_in_array; i++) { struct ls3_stats_user_report_template *u_report_ptr = - reports_with_stats->users_reports[i]; + report->users_reports[i]; if (!u_report_ptr) continue; @@ -1626,10 +1606,11 @@ static void ls3_stats_print_to_yaml(const char *f_time, double e_time) } fclose(yaml_fd); - printf(" Path to yaml: %s\n", reports_with_stats->format_report_file_name); + printf(" Path to yaml: %s\n", report->format_report_file_name); } -static void ls3_stats_print_to_out(const char *f_time, double e_time) +static void ls3_stats_print_to_out(struct fstats_report *report, + const char *f_time, double e_time) { struct report_template *report_ptr; struct range_report_template *range_ptr; @@ -1637,8 +1618,8 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time) FILE *out_fd; int i, j, k; - ls3_stats_change_extension_in_file(LS3_STATS_FILE_EXTENSION_OUT); - out_fd = ls3_stats_open_file(reports_with_stats->format_report_file_name); + ls3_stats_change_extension_in_file(report, LS3_STATS_FILE_EXTENSION_OUT); + out_fd = ls3_stats_open_file(report->format_report_file_name); fprintf(out_fd, "Generated by LIPE stats v%.2f\n" "Create Report Time: %s\n" @@ -1650,20 +1631,19 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time) "Free inodes count on device: %u\n" "Client mount path: %s\n\n", LS3_STATS_VERSION, f_time, - reports_with_stats->total_count_files, e_time, - reports_with_stats->device_path, - reports_with_stats->device_name, - reports_with_stats->device_is_mdt ? "MDT" : "OST", - reports_with_stats->inodes_count, - reports_with_stats->free_inodes_count, - reports_with_stats->client_mount_path); + report->total_count_files, e_time, + report->device_path, + report->device_name, + report->device_is_mdt ? "MDT" : "OST", + report->inodes_count, + report->free_inodes_count, + report->client_mount_path); fprintf(out_fd, "Error counters:\n"); for (err = ls3_err_mappings; err->counter_num != LS3_STATS_ERROR_END_MARKER; ++err) { fprintf(out_fd, "%s: %u\n", err->str_err_header, - LS3_GET_ERROR_COUNT(reports_with_stats, - err->counter_num)); + LS3_GET_ERROR_COUNT(report, err->counter_num)); } fprintf(out_fd, "\n"); @@ -1671,7 +1651,7 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time) struct ls3_stats_range_val srv_basic_str; const ls3_stats_out_mapping *text_tmpl; - report_ptr = reports_with_stats->reports[i]; + report_ptr = report->reports[i]; if (report_ptr == NULL) LS3_FATAL("Unable to access the report data: %d\n", i); @@ -1686,7 +1666,7 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time) fprintf(out_fd, "\n\n%s\n", text_tmpl->som_table_title); fprintf(out_fd, text_tmpl->som_fmt_basic_str, report_ptr->files_count, - ls3_stats_get_count_skipped_obj(i), + ls3_stats_get_count_skipped_obj(report, i), srv_basic_str.srv_min, srv_basic_str.srv_max, srv_basic_str.srv_avg, @@ -1725,9 +1705,9 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time) } - for (i = 0; i <= reports_with_stats->last_user_idx_in_array; i++) { + for (i = 0; i <= report->last_user_idx_in_array; i++) { struct ls3_stats_user_report_template *u_report_ptr = - reports_with_stats->users_reports[i]; + report->users_reports[i]; if (!u_report_ptr) continue; @@ -1791,10 +1771,11 @@ static void ls3_stats_print_to_out(const char *f_time, double e_time) ls3_stats_fprintf_out(out_fd); fclose(out_fd); - printf(" Path to out: %s\n", reports_with_stats->format_report_file_name); + printf(" Path to out: %s\n", report->format_report_file_name); } -static void ls3_stats_print_to_csv(const char *f_time, double e_time) +static void ls3_stats_print_to_csv(struct fstats_report *report, + const char *f_time, double e_time) { struct report_template *report_ptr; struct range_report_template *range_ptr; @@ -1802,8 +1783,8 @@ static void ls3_stats_print_to_csv(const char *f_time, double e_time) FILE *csv_fd; int i, j, k; - ls3_stats_change_extension_in_file(LS3_STATS_FILE_EXTENSION_CSV); - csv_fd = ls3_stats_open_file(reports_with_stats->format_report_file_name); + ls3_stats_change_extension_in_file(report, LS3_STATS_FILE_EXTENSION_CSV); + csv_fd = ls3_stats_open_file(report->format_report_file_name); fprintf(csv_fd, "Generated by LIPE stats v%.2f\n" "Create Report Time [%s]\n" @@ -1815,20 +1796,19 @@ static void ls3_stats_print_to_csv(const char *f_time, double e_time) "Free inodes count on device [%u]\n" "Client mount path [%s]\n\n", LS3_STATS_VERSION, f_time, - reports_with_stats->total_count_files, e_time, - reports_with_stats->device_path, - reports_with_stats->device_name, - reports_with_stats->device_is_mdt ? "MDT" : "OST", - reports_with_stats->inodes_count, - reports_with_stats->free_inodes_count, - reports_with_stats->client_mount_path); + report->total_count_files, e_time, + report->device_path, + report->device_name, + report->device_is_mdt ? "MDT" : "OST", + report->inodes_count, + report->free_inodes_count, + report->client_mount_path); fprintf(csv_fd, "Error counters:\n"); for (err = ls3_err_mappings; err->counter_num != LS3_STATS_ERROR_END_MARKER; ++err) { fprintf(csv_fd, "%s [%u]\n", err->str_err_header, - LS3_GET_ERROR_COUNT(reports_with_stats, - err->counter_num)); + LS3_GET_ERROR_COUNT(report, err->counter_num)); } fprintf(csv_fd, "\n"); @@ -1837,7 +1817,7 @@ static void ls3_stats_print_to_csv(const char *f_time, double e_time) char *value_ts; char *second_value_ts; - report_ptr = reports_with_stats->reports[i]; + report_ptr = report->reports[i]; if (report_ptr == NULL) LS3_FATAL("Unable to access the report data: %d\n", i); @@ -1895,9 +1875,9 @@ static void ls3_stats_print_to_csv(const char *f_time, double e_time) "---------------------------------\n"); } - for (i = 0; i <= reports_with_stats->last_user_idx_in_array; i++) { + for (i = 0; i <= report->last_user_idx_in_array; i++) { struct ls3_stats_user_report_template *u_report_ptr = - reports_with_stats->users_reports[i]; + report->users_reports[i]; if (!u_report_ptr) continue; @@ -1942,48 +1922,51 @@ static void ls3_stats_print_to_csv(const char *f_time, double e_time) } fclose(csv_fd); - printf(" Path to csv: %s\n", reports_with_stats->format_report_file_name); + printf(" Path to csv: %s\n", report->format_report_file_name); } /* ls3_stats_printf - Generates a report file for the * user with the required extension */ -void ls3_stats_printf(void) +void ls3_stats_printf(struct fstats_report *report) { time_t current_time = time(NULL); double elapsed_time; struct tm *time_info; char formatted_time[20]; - elapsed_time = difftime(current_time, reports_with_stats->start_time); + if (!report) + LS3_FATAL("filed read report. pointer is NULL\n"); + + elapsed_time = difftime(current_time, report->start_time); time_info = localtime(¤t_time); strftime(formatted_time, sizeof(formatted_time), "%Y-%m-%d %H:%M:%S", time_info); /* Do all the necessary calculations */ - ls3_stats_calculate_values(); - ls3_stats_prepare_file(); + ls3_stats_calculate_values(report); + ls3_stats_prepare_file(report); printf("File with the results:\n"); - if (reports_with_stats->report_extension & LS3_STATS_FILE_EXTENSION_OUT) - ls3_stats_print_to_out(formatted_time, elapsed_time); + if (report->report_extension & LS3_STATS_FILE_EXTENSION_OUT) + ls3_stats_print_to_out(report, formatted_time, elapsed_time); - if (reports_with_stats->report_extension & LS3_STATS_FILE_EXTENSION_YAML) - ls3_stats_print_to_yaml(formatted_time, elapsed_time); + if (report->report_extension & LS3_STATS_FILE_EXTENSION_YAML) + ls3_stats_print_to_yaml(report, formatted_time, elapsed_time); - if (reports_with_stats->report_extension & LS3_STATS_FILE_EXTENSION_JSON) - ls3_stats_print_to_json(formatted_time, elapsed_time); + if (report->report_extension & LS3_STATS_FILE_EXTENSION_JSON) + ls3_stats_print_to_json(report, formatted_time, elapsed_time); - if (reports_with_stats->report_extension & LS3_STATS_FILE_EXTENSION_CSV) - ls3_stats_print_to_csv(formatted_time, elapsed_time); + if (report->report_extension & LS3_STATS_FILE_EXTENSION_CSV) + ls3_stats_print_to_csv(report, formatted_time, elapsed_time); } /* ls3_stats_calculate_values - makes additional calculations based on all * collected data for each table. */ -void ls3_stats_calculate_values(void) +void ls3_stats_calculate_values(struct fstats_report *report) { int i, j; struct report_template *report_ptr; @@ -1994,7 +1977,7 @@ void ls3_stats_calculate_values(void) double percent_cumulative_in_range = 0; double total_weighted_sum = 0; - report_ptr = reports_with_stats->reports[i]; + report_ptr = report->reports[i]; if (report_ptr == NULL) LS3_FATAL("Unable to access the report data: %d\n", i); @@ -2002,8 +1985,7 @@ void ls3_stats_calculate_values(void) continue; if (i == LS3_STATS_TYPE_FILES_SIZE) - reports_with_stats->total_count_files = - report_ptr->files_count; + report->total_count_files = report_ptr->files_count; for (j = 0; j < report_ptr->count_ranges; j++) { range_ptr = report_ptr->fs_ranges[j]; @@ -2055,12 +2037,13 @@ void ls3_stats_calculate_values(void) } } -static struct range_report_template *ls3_stats_get_new_range_ptr(uint64_t res) +struct range_report_template *ls3_stats_get_new_range_ptr(uint64_t res) { struct range_report_template *range_ptr; range_ptr = xcalloc(1, sizeof(struct range_report_template)); range_ptr->rrt_min = DBL_MAX; + range_ptr->rrt_dev_cnt = -1; if (res >= 2) { range_ptr->range_start = (int)pow(2, floor(log2(res))); range_ptr->range_end = range_ptr->range_start * 2; @@ -2078,14 +2061,14 @@ static struct range_report_template *ls3_stats_get_new_range_double_ptr(double r range_ptr = xcalloc(1, sizeof(struct range_report_template)); range_ptr->rrt_min = DBL_MAX; - + range_ptr->rrt_dev_cnt = -1; /* 0.001 it's rounding for exception handling with integers */ range_ptr->range_start = floor((res + 0.001) / 0.2) * 0.2; range_ptr->range_end = range_ptr->range_start + 0.2; return range_ptr; } -static struct report_template *ls3_stats_get_new_report_ptr(void) +struct report_template *ls3_stats_get_new_report_ptr(void) { struct report_template *report_ptr = xcalloc(1, sizeof(struct report_template)); @@ -2100,7 +2083,7 @@ static struct report_template *ls3_stats_get_new_report_ptr(void) return report_ptr; } -static void ls3_stats_expand_num_of_ranges(struct report_template *report_ptr, +void ls3_stats_expand_num_of_ranges(struct report_template *report_ptr, int index_range) { struct range_report_template **new_ranges_array; @@ -2249,7 +2232,7 @@ static void ls3_stats_update_range(ls3_stats_report_type report_type, } -static struct range_report_template *ls3_stats_find_id_in_ranges(uint64_t id, +struct range_report_template *ls3_stats_find_id_in_ranges(uint64_t id, struct report_template *report_ptr) { struct range_report_template *range_ptr; @@ -2337,54 +2320,52 @@ static void ls3_stats_update_range_with_id(ls3_stats_report_type report_type, /* Adds a new user to the stats reporting structure, allocating and expanding * resources as needed. * + * @param stats The general structure with all stats. * @param user_id The user ID to add. * @return Pointer to the ls3_stats_user_report_template. */ -static struct ls3_stats_user_report_template - *ls3_stats_add_new_user(uint64_t user_id) +struct ls3_stats_user_report_template + *ls3_stats_add_new_user(struct fstats_report *stats, uint64_t uid) { struct ls3_stats_user_report_template *user_ptr; uint64_t current_index = 0; int i; /* Get exactly index in the array since it's a new user */ - current_index = reports_with_stats->last_user_idx_in_array + 1; - if (current_index == 1 && reports_with_stats->users_reports[0] == NULL) + current_index = stats->last_user_idx_in_array + 1; + if (current_index == 1 && stats->users_reports[0] == NULL) current_index = 0; /* This is first user with 0 index */ - if (current_index == reports_with_stats->max_count_users_in_report) { + if (current_index == stats->max_count_users_in_report) { struct ls3_stats_user_report_template **new_ranges_array; - unsigned int old_size = - reports_with_stats->max_count_users_in_report; + unsigned int old_size = stats->max_count_users_in_report; size_t num_bytes_to_clear; size_t tmp_sizeof_urt = sizeof(struct ls3_stats_user_report_template*); /* current max value + default size (step 100 entries) */ - reports_with_stats->max_count_users_in_report = - reports_with_stats->max_count_users_in_report + + stats->max_count_users_in_report = + stats->max_count_users_in_report + LS3_STATS_ARRAY_SIZE_BY_DEFAULT; - new_ranges_array = - xrealloc(reports_with_stats->users_reports, - reports_with_stats->max_count_users_in_report * - tmp_sizeof_urt); + new_ranges_array = xrealloc(stats->users_reports, + stats->max_count_users_in_report * + tmp_sizeof_urt); - reports_with_stats->users_reports = new_ranges_array; + stats->users_reports = new_ranges_array; num_bytes_to_clear = - (reports_with_stats->max_count_users_in_report - - old_size) * - sizeof(struct ls3_stats_user_report_template*); + (stats->max_count_users_in_report - old_size) * + sizeof(struct ls3_stats_user_report_template*); - memset(reports_with_stats->users_reports + old_size, 0, + memset(stats->users_reports + old_size, 0, num_bytes_to_clear); } - user_ptr = reports_with_stats->users_reports[current_index]; + user_ptr = stats->users_reports[current_index]; if (!user_ptr) { user_ptr = xcalloc(1, sizeof(struct ls3_stats_user_report_template)); - user_ptr->uid = user_id; + user_ptr->uid = uid; for (i = 0; i < LS3_STATS_USERS_REPORTS_COUNTER; i++) { struct report_template *tmp_report_ptr = ls3_stats_get_new_report_ptr(); @@ -2396,21 +2377,21 @@ static struct ls3_stats_user_report_template } } - reports_with_stats->users_reports[current_index] = user_ptr; - reports_with_stats->last_user_idx_in_array = current_index; - + stats->users_reports[current_index] = user_ptr; + stats->last_user_idx_in_array = current_index; return user_ptr; } /* Finds and returns the user report data for a given user_id. * Returns NULL if the user_report for user_id is not found. * + * @param stats The general structure with all stats. * @param user_id The user ID to search for. * @return Pointer to the ls3_stats_user_report_template data if found, - * otherwise NULL. + * otherwise NULL. */ -static struct ls3_stats_user_report_template - *ls3_stats_find_user_reports(uint64_t user_id) +struct ls3_stats_user_report_template + *ls3_stats_find_user_reports(struct fstats_report *stats, uint64_t uid) { struct ls3_stats_user_report_template *user_ptr; int i; @@ -2423,15 +2404,15 @@ static struct ls3_stats_user_report_template * Bitmap - 0.000005 sec | Array 0.000009 sec). * But this saves 125MB of RAM on bitmap. * But such cases when > 3000 users must probably be very rare. */ - for (i = 0; i <= reports_with_stats->last_user_idx_in_array; i++) { - user_ptr = reports_with_stats->users_reports[i]; + for (i = 0; i <= stats->last_user_idx_in_array; i++) { + user_ptr = stats->users_reports[i]; if (user_ptr == NULL && i != 0) LS3_FATAL("Unable to access the user %lu report data\n", - user_id); + uid); else if (user_ptr == NULL && i == 0) return NULL; /* When first file */ - if (user_ptr->uid == user_id) + if (user_ptr->uid == uid) return user_ptr; } @@ -2452,10 +2433,12 @@ static void ls3_stats_update_user_report_range(ls3_stats_users_reports report_ty LS3_FATAL("Defunct report type LS3_STATS_USERS_REPORTS_%d\n", report_type); - user_report_ptr = ls3_stats_find_user_reports(user_id); + user_report_ptr = ls3_stats_find_user_reports(reports_with_stats, + user_id); if (!user_report_ptr) { pthread_mutex_lock(&reports_with_stats->user_rt_mutex); - user_report_ptr = ls3_stats_add_new_user(user_id); + user_report_ptr = ls3_stats_add_new_user(reports_with_stats, + user_id); pthread_mutex_unlock(&reports_with_stats->user_rt_mutex); } @@ -2811,28 +2794,37 @@ void ls3_stats_init(void) LS3_INIT_ERRORS_COUNT(reports_with_stats); } -void ls3_stats_destroy(void) +void ls3_stats_destroy(struct fstats_report *report) { int i, j, k; for (i = 0; i < LS3_STATS_TYPE_TOTAL_COUNT_REPORT; i++) { - struct report_template *report_ptr = - reports_with_stats->reports[i]; + struct report_template *report_ptr = report->reports[i]; for (j = 0; j < report_ptr->count_ranges; j++) { struct range_report_template *range_ptr = report_ptr->fs_ranges[j]; - if (range_ptr != NULL) - free(range_ptr); + if (range_ptr == NULL) + continue; + + if (range_ptr->rrt_dev_cnt >= 0) { + for(k = 0; k < LS3_STATS_ARRAY_SIZE_BY_DEFAULT; + k++) { + if (range_ptr->rrt_dev[k]) + free(range_ptr->rrt_dev[k]); + } + } + + free(range_ptr); } pthread_mutex_destroy(&report_ptr->report_template_mutex); free(report_ptr->fs_ranges); } - for (i = 0; i <= reports_with_stats->last_user_idx_in_array; i++) { + for (i = 0; i <= report->last_user_idx_in_array; i++) { struct ls3_stats_user_report_template *tmp_user_reports = - reports_with_stats->users_reports[i]; + report->users_reports[i]; if (tmp_user_reports == NULL) continue; @@ -2855,23 +2847,23 @@ void ls3_stats_destroy(void) free(tmp_user_reports); } - if (reports_with_stats->report_file_name) - free(reports_with_stats->report_file_name); + if (report->report_file_name) + free(report->report_file_name); - if (reports_with_stats->device_path) - free(reports_with_stats->device_path); + if (report->device_path) + free(report->device_path); - if (reports_with_stats->format_report_file_name) - free(reports_with_stats->format_report_file_name); + if (report->format_report_file_name) + free(report->format_report_file_name); - if (reports_with_stats->device_name) - free(reports_with_stats->device_name); + if (report->device_name) + free(report->device_name); - if (reports_with_stats->client_mount_path) - free(reports_with_stats->client_mount_path); + if (report->client_mount_path) + free(report->client_mount_path); - pthread_mutex_destroy(&reports_with_stats->user_rt_mutex); - LS3_DESTROY_ERRORS_COUNT(reports_with_stats); - free(reports_with_stats->users_reports); - free(reports_with_stats); + pthread_mutex_destroy(&report->user_rt_mutex); + LS3_DESTROY_ERRORS_COUNT(report); + free(report->users_reports); + free(report); } diff --git a/lipe/src/lipe_scan3/ls3_stats.h b/lipe/src/lipe_scan3/ls3_stats.h index d730823..47da62f 100644 --- a/lipe/src/lipe_scan3/ls3_stats.h +++ b/lipe/src/lipe_scan3/ls3_stats.h @@ -50,6 +50,13 @@ typedef enum { LS3_STATS_USERS_REPORTS_COUNTER } ls3_stats_users_reports; +typedef enum { + LS3_STATS_RANGE_TYPE_INT = 0, + LS3_STATS_RANGE_TYPE_ID, + LS3_STATS_RANGE_TYPE_DOUBLE, + LS3_STATS_RANGE_TYPE_UNKNOWN +} ls3_stats_range_type; + typedef struct { const char *extension_name; ls3_stats_file_extension extension_value; @@ -141,6 +148,8 @@ struct range_report_template { double cumulative_percentage; double percent_in_range; double percent_cumulative_in_range; + int32_t rrt_dev_cnt; + char *rrt_dev[LS3_STATS_ARRAY_SIZE_BY_DEFAULT]; }; struct report_template { @@ -186,16 +195,27 @@ struct fstats_report { char *device_name; char *client_mount_path; bool device_is_mdt; + bool merging; struct report_template *reports[LS3_STATS_TYPE_TOTAL_COUNT_REPORT]; struct ls3_stats_user_report_template **users_reports; }; extern struct fstats_report *reports_with_stats; void ls3_stats_init(void); -void ls3_stats_destroy(void); +void ls3_stats_destroy(struct fstats_report *report); int ls3_stats_get_range_index(uint64_t num); +struct report_template *ls3_stats_get_new_report_ptr(void); +struct range_report_template *ls3_stats_get_new_range_ptr(uint64_t res); +void ls3_stats_expand_num_of_ranges(struct report_template *report_ptr, + int index_range); +struct range_report_template *ls3_stats_find_id_in_ranges(uint64_t id, + struct report_template *report_ptr); +struct ls3_stats_user_report_template + *ls3_stats_add_new_user(struct fstats_report *stats, uint64_t uid); +struct ls3_stats_user_report_template + *ls3_stats_find_user_reports(struct fstats_report *stats, uint64_t uid); int ls3_stats_update_info(struct ls3_object_attrs *loa_all); -void ls3_stats_calculate_values(void); -void ls3_stats_printf(void); +void ls3_stats_calculate_values(struct fstats_report *report); +void ls3_stats_printf(struct fstats_report *report); #endif /* _LS3_STATS_H_ */ -- 1.8.3.1