From 2d7942e45778a2cd893f81a538b0b7bc9437192f Mon Sep 17 00:00:00 2001 From: Vitaliy Kuznetsov Date: Mon, 19 Feb 2024 16:37:42 +0100 Subject: [PATCH] EX-8130 lipe: Add key func for work with tree This patch adds two key functions to collect directory size statistics, which contain the basic logic for adding directories to memory and incrementing size counters. Test-Parameters: trivial Signed-off-by: Vitaliy Kuznetsov Change-Id: I06e03a6be1052b7178274835169cc41d044ca1ab Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53963 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Alexandre Ioffe Reviewed-by: Andreas Dilger --- lipe/src/lipe_scan3/ls3_dir_stats.c | 135 ++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.c b/lipe/src/lipe_scan3/ls3_dir_stats.c index 4eb4c58..7150033 100644 --- a/lipe/src/lipe_scan3/ls3_dir_stats.c +++ b/lipe/src/lipe_scan3/ls3_dir_stats.c @@ -509,6 +509,141 @@ static void ls3_stats_rm_first_dir(char ***dirs, int *count) (*dirs)[(*count)] = NULL; } +/* ls3_stats_check_dir_in_mem - Checks whether such a directory + * exists in memory. If not, it creates one. Forces counters to update. + * + * This is a key function and it works like this: + * When we first enter, we always start from the top of the tree. + * We check the first directory specified in the path for its presence + * in memory, if it is not there, we create it, and if the directory exists, + * we increment the size counter and move on to the next one. + * ***dirs == ["dir1", "dir2", "dir3", ..., "max_depth"] + * ["dir1"] counters++; + * ["dir2"] counters++; + * ... + * ["max_depth"] create and/or counters++; + * It's important to note that we are always limited by depth, but even so, + * all counters up to ["max_depth"] will be incremented, allowing the entire + * size to be taken into account without storing a data structure. + * For example, with a depth of 4 and a file /dir1/.../dir20/fileX, + * the resulting array would be ["dir1", "dir2", "dir3", "dir4"], but the + * entire size including / dir1/.../dir20/fileX + */ +static void ls3_stats_check_dir_in_mem(struct ls3_object_attrs *loa_all, + struct ls3_stats_dir_obj *dir_ptr, + char ***dirs, int *count, + int original_depth) +{ + struct ls3_stats_dir_obj *child_ptr; + uint64_t current_index = 0; + int i; + + /* Check array with dir */ + if ((*count) == 0 || (*dirs)[0] == NULL) + return; /* catalogs are out */ + + /* Checking the current directory for matching name */ + if ((dir_ptr != NULL) && + (strcmp(dir_ptr->lsdo_name, (*dirs)[0]) == 0)) { + ls3_stats_dir_incr_counters(loa_all, dir_ptr); + /* last catalog, try update FID */ + if ((*count) == 1 && dir_ptr->lsdo_fid.f_oid == 0) + ls3_stats_dir_update_fid(loa_all, dir_ptr, + original_depth); + + ls3_stats_rm_first_dir(dirs, count); + ls3_stats_check_dir_in_mem(loa_all, dir_ptr, + dirs, count, original_depth); + return; + } + + pthread_mutex_lock(&dir_ptr->lsdo_mutex); + /* Checking child directories to match the name */ + for (i = 0; i <= dir_ptr->lsdo_last_child; i++) { + child_ptr = dir_ptr->lsdo_child[i]; + + if (child_ptr == NULL) + continue; + + if (strcmp(child_ptr->lsdo_name, (*dirs)[0]) != 0) + continue; + + pthread_mutex_unlock(&dir_ptr->lsdo_mutex); + + /* last catalog, try update FID */ + if ((*count) == 1 && child_ptr->lsdo_fid.f_oid == 0) + ls3_stats_dir_update_fid(loa_all, child_ptr, + original_depth); + + ls3_stats_dir_incr_counters(loa_all, child_ptr); + ls3_stats_rm_first_dir(dirs, count); + ls3_stats_check_dir_in_mem(loa_all, child_ptr, + dirs, count, original_depth); + return; + } + + /* Get exactly index in the array since it's a new catalog + * Expand memory if necessary and create/allocate a new directory. + */ + current_index = dir_ptr->lsdo_last_child + 1; + if (current_index == 1 && dir_ptr->lsdo_child[0] == NULL) + current_index = 0; /* This is first catalog with 0 index */ + + if (current_index == dir_ptr->lsdo_child_max) + ls3_expand_num_of_dirs(dir_ptr); + + child_ptr = dir_ptr->lsdo_child[current_index]; + if (child_ptr == NULL) { + child_ptr = ls3_stats_allocate_new_dir(loa_all, (*dirs)[0], + dir_ptr->lsdo_depth + 1); + dir_ptr->lsdo_child[current_index] = child_ptr; + dir_ptr->lsdo_last_child = current_index; + } + + pthread_mutex_unlock(&dir_ptr->lsdo_mutex); + ls3_stats_dir_incr_counters(loa_all, child_ptr); + + /* last catalog, try update FID */ + if ((*count) == 1 && child_ptr->lsdo_fid.f_oid == 0) + ls3_stats_dir_update_fid(loa_all, child_ptr, original_depth); + + ls3_stats_rm_first_dir(dirs, count); + + /* Let's move on, since still maybe have other catalogs left. */ + ls3_stats_check_dir_in_mem(loa_all, child_ptr, dirs, + count, original_depth); +} + +/* ls3_stats_upd_dir_info - An intermediate link between the distributor and + * directly accounting for the directory in memory. + */ +static void ls3_stats_upd_dir_info(struct ls3_object_attrs *loa_all, char* path, + int original_depth) +{ + char **directories; + int count_dir; + + /* Only get here if the directory depth is >= 1 */ + directories = ls3_stats_dir_split_path(path, &count_dir); + + if (count_dir != 0) + /* The starting point of entry into the recursion until + * increment the counters in each directory. + */ + ls3_stats_check_dir_in_mem(loa_all, dir_stats->lsdg_start_dir, + &directories, &count_dir, + original_depth); + else + LS3_FATAL("failed to count directories correctly\n"); + + /* In the previous step we must decrement the count_dir to 0 */ + if (count_dir != 0) + LS3_FATAL("error processing directories\n"); + + if (!directories) + free(directories); +} + void ls3_stats_dir_init(void) { int rc; -- 1.8.3.1