From ceeb6b07941d55662e84a35ae4ffcd5730de4428 Mon Sep 17 00:00:00 2001 From: Vitaliy Kuznetsov Date: Tue, 5 Mar 2024 15:21:21 +0100 Subject: [PATCH] EX-8130 lipe: Add directory scan structs This patch adds new structures to lipe_scan3 for collecting and storing directory statistics, as well as initialization and destroy functions. This patch is the first in a series of patches that add functionality for collecting directory statistics in lipe_find3. Test-Parameters: trivial Signed-off-by: Vitaliy Kuznetsov Change-Id: Ib14ce13677d93d1a53299501138e78c7b290793c Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53958 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Alexandre Ioffe Reviewed-by: Andreas Dilger --- lipe/src/lipe_scan3/ls3_dir_stats.c | 70 ++++++++++++++++++++++++++++++ lipe/src/lipe_scan3/ls3_dir_stats.h | 86 +++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 lipe/src/lipe_scan3/ls3_dir_stats.c create mode 100644 lipe/src/lipe_scan3/ls3_dir_stats.h diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.c b/lipe/src/lipe_scan3/ls3_dir_stats.c new file mode 100644 index 0000000..9a276d9 --- /dev/null +++ b/lipe/src/lipe_scan3/ls3_dir_stats.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2024, DDN Storage Corporation. + * + * Author: Vitaliy Kuznetsov + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ls3_fid2path.h" +#include "ls3_stats.h" +#include "ls3_dir_stats.h" + +struct ls3_stats_dir_general *dir_stats; + +void ls3_stats_dir_init(void) +{ + int rc; + + /* This is the first and only place with initialization. */ + dir_stats = xcalloc(1, sizeof(struct ls3_stats_dir_general)); + rc = pthread_mutex_init(&dir_stats->lsdg_mutex, NULL); + if (rc != 0) + LS3_FATAL("failed to initialize mutex\n"); + +} + +static void ls3_stats_destroy_dirs(struct ls3_stats_dir_obj *dir_ptr) +{ + struct ls3_stats_dir_obj *new_dir_ptr; + int i; + + for (i = 0; i < dir_ptr->lsdo_child_max; i++) { + new_dir_ptr = dir_ptr->lsdo_child[i]; + if (new_dir_ptr != NULL) + ls3_stats_destroy_dirs(new_dir_ptr); + } + + pthread_mutex_destroy(&dir_ptr->lsdo_mutex); + + if (dir_ptr->lsdo_name) + free(dir_ptr->lsdo_name); + + free(dir_ptr); +} + +void ls3_stats_dir_destroy(void) +{ + if (dir_stats->lsdg_start_dir) + ls3_stats_destroy_dirs(dir_stats->lsdg_start_dir); + + if (dir_stats->lsdg_top_size) { + free(dir_stats->lsdg_top_size->lsdr_array); + free(dir_stats->lsdg_top_size); + } + + pthread_mutex_destroy(&dir_stats->lsdg_mutex); + if (dir_stats->lsdg_start_path) + free(dir_stats->lsdg_start_path); + + free(dir_stats); +} diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.h b/lipe/src/lipe_scan3/ls3_dir_stats.h new file mode 100644 index 0000000..dc0c26f --- /dev/null +++ b/lipe/src/lipe_scan3/ls3_dir_stats.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2024, DDN Storage Corporation. + * + * Author: Vitaliy Kuznetsov + */ + +#ifndef _LS3_DIR_STATS_H_ +#define _LS3_DIR_STATS_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ls3_debug.h" +#include "ls3_scan.h" +#include "ls3_object_attrs.h" + +#define LS3_STATS_DIR_COUNT_BY_DEFAULT 10 +#define LS3_STATS_DEPTH_BY_DEFAULT 4 +#define LS3_STATS_TOP_RATING_BY_DEFAULT 100 +/* In bytes. For count the size of empty directories and dir size on OST */ +#define LS3_STATS_DIR_SIZE 4096 + +/* ls3_stats_dir_obj - Stores basic information about the directory. + * Current size by sizeof == 112 + */ +struct ls3_stats_dir_obj { + pthread_mutex_t lsdo_mutex; + struct lu_fid lsdo_fid; /* Only for MDT */ + uint64_t lsdo_size; /* Total dir size */ + /* Total allocate size (number of 512B blocks) from inodes */ + uint64_t lsdo_alloc_size_in_512blocks; + uint32_t lsdo_files_count; + uint32_t lsdo_dirs_count; + uint32_t lsdo_child_max; + uint32_t lsdo_last_child; + uint32_t lsdo_uid; /* Owner */ + uint16_t lsdo_compress_type; + int16_t lsdo_depth; /* Current object depth */ + char *lsdo_name; + struct ls3_stats_dir_obj **lsdo_child; +}; + +/* ls3_stats_dir_rating - This structure stores a ranking of the + * top N (like TOP-100) largest directories by size. The default + * object count is equivalent to LS3_STATS_TOP_RATING_BY_DEFAULT, + * but the number of objects in this rating can be specified by + * the user using the -top-rating option. Work this struct + * occurs according to the principle of a minimal heap. + */ +struct ls3_stats_dir_rating { + uint64_t lsdr_min_size; + uint64_t lsdr_max_size; + uint64_t lsdr_total_size; + uint32_t lsdr_current_size_heap; + uint32_t lsdr_capacity_heap; + struct ls3_stats_dir_obj **lsdr_array; +}; + +struct ls3_stats_dir_general { + pthread_mutex_t lsdg_mutex; + uint64_t lsdg_total_size; + uint64_t lsdg_total_alloc_size; + uint32_t lsdg_max_depth; + uint32_t lsdg_top_rating_limit; + uint32_t lsdg_empty_dirs_count; /* Only for MDT */ + uint32_t lsdg_min_size_value_for_print; + uint32_t lsdg_dirs_count; + char *lsdg_start_path; + struct ls3_stats_dir_obj *lsdg_start_dir; + struct ls3_stats_dir_rating *lsdg_top_size; +}; + +extern struct ls3_stats_dir_general *dir_stats; + +void ls3_stats_dir_init(void); +void ls3_stats_dir_destroy(void); + +#endif /* _LS3_DIR_STATS_H_ */ -- 1.8.3.1