From 594f3bc58140f6aa17242252a3af6615c795572c Mon Sep 17 00:00:00 2001 From: Vitaliy Kuznetsov Date: Tue, 5 Mar 2024 15:52:10 +0100 Subject: [PATCH] EX-8130 lipe: Directory scan size stats This patch adds functionality for creating new directories and expanding memory for new child directories in memory. Adds a function to initialize the starting directory. Test-Parameters: trivial Signed-off-by: Vitaliy Kuznetsov Change-Id: I3ff6a62ffd9d6535ed4434f517d1c93d6ae01b34 Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53960 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger --- lipe/src/lipe_scan3/ls3_dir_stats.c | 144 ++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.c b/lipe/src/lipe_scan3/ls3_dir_stats.c index fc293bf..000f001 100644 --- a/lipe/src/lipe_scan3/ls3_dir_stats.c +++ b/lipe/src/lipe_scan3/ls3_dir_stats.c @@ -177,6 +177,150 @@ static void ls3_stats_init_heap(void) ls3_stats_create_heap(dir_stats->lsdg_top_rating_limit); } +/* ls3_stats_dir_incr_counters - Increases directory size */ +static void ls3_stats_dir_incr_counters(struct ls3_object_attrs *loa_all, + struct ls3_stats_dir_obj *dir_ptr) +{ + pthread_mutex_lock(&dir_ptr->lsdo_mutex); + dir_ptr->lsdo_size += loa_all->loa_size; + + /* loa_all->loa_blocks is taken from the inode and reflects the . + * number of 512B blocks. Since in the blocks_from_inode() + * function we get the value from inode->i_blocks which is equal + * to the number of blocks with a size of 512B + */ + dir_ptr->lsdo_alloc_size_in_512blocks += loa_all->loa_blocks; + /* It is likely that loa_all->loa_size is also aligned to a + * block of 512 bytes somewhere in EXT2_I_SIZE(inode)... + * For this reason, we do not align the size in future. + */ + + /* TODO?: other counters in future? */ + if (loa_all->loa_mode & S_IFDIR) + dir_ptr->lsdo_dirs_count++; + else if (loa_all->loa_mode & S_IFREG) + dir_ptr->lsdo_files_count++; + + pthread_mutex_unlock(&dir_ptr->lsdo_mutex); +} + +/* ls3_expand_num_of_dirs expanding the number of child directories + * for a directory in statistics. + */ +static void ls3_expand_num_of_dirs(struct ls3_stats_dir_obj *dir_ptr) +{ + struct ls3_stats_dir_obj **new_dirs_array; + unsigned int old_size = dir_ptr->lsdo_child_max; + size_t num_bytes_to_clear; + + /* Check after mutex unlock */ + if (dir_ptr->lsdo_child_max > dir_ptr->lsdo_last_child + 1) + return; + + dir_ptr->lsdo_child_max = old_size * 2 + 1; + new_dirs_array = xrealloc(dir_ptr->lsdo_child, dir_ptr->lsdo_child_max * + sizeof(struct ls3_stats_dir_obj*)); + + dir_ptr->lsdo_child = new_dirs_array; + num_bytes_to_clear = (dir_ptr->lsdo_child_max - old_size) * + sizeof(struct ls3_stats_dir_obj*); + + memset(dir_ptr->lsdo_child + old_size, 0, num_bytes_to_clear); +} + +static void ls3_stats_dir_update_fid(struct ls3_object_attrs *loa_all, + struct ls3_stats_dir_obj *dir_ptr, + int original_depth) +{ + if (!(loa_all->loa_mode & S_IFDIR)) + return; + + /* Check to ensure that the directory is not will + * assigned an FID from a child element. + */ + if (dir_ptr->lsdo_depth != original_depth - 1) + return; + + dir_ptr->lsdo_fid.f_oid = loa_all->loa_file_fid.f_oid; + dir_ptr->lsdo_fid.f_seq = loa_all->loa_file_fid.f_seq; + dir_ptr->lsdo_fid.f_ver = loa_all->loa_file_fid.f_ver; + /* Also update UID */ + dir_ptr->lsdo_uid = loa_all->loa_uid; +} + +/* ls3_stats_allocate_new_dir allocates memory for a new + * directory in statistics. + */ +static struct ls3_stats_dir_obj + *ls3_stats_allocate_new_dir(struct ls3_object_attrs *loa_all, + char *dir_name, int depth) +{ + struct ls3_stats_dir_obj *new_dir_ptr; + int rc; + + new_dir_ptr = xcalloc(1, sizeof(struct ls3_stats_dir_obj)); + new_dir_ptr->lsdo_child_max = LS3_STATS_DIR_COUNT_BY_DEFAULT; + new_dir_ptr->lsdo_depth = depth; + new_dir_ptr->lsdo_name = xstrdup(dir_name); + rc = pthread_mutex_init(&new_dir_ptr->lsdo_mutex, NULL); + if (rc != 0) + LS3_FATAL("failed to initialize mutex\n"); + + new_dir_ptr->lsdo_child = (struct ls3_stats_dir_obj**)xcalloc( + new_dir_ptr->lsdo_child_max, + sizeof(struct ls3_stats_dir_obj*)); + + if (!loa_all) /* skip first dir from ls3_stats_init_first_dir() */ + return new_dir_ptr; + + dir_stats->lsdg_dirs_count++; + + return new_dir_ptr; +} + +/* ls3_stats_init_first_dir - Initializes the first directory. + * If a "path" filter is specified, the first directory will be + * the value from that filter, but assigned later. If no value is + * specified, "/" will be assigned. + */ +static void ls3_stats_init_first_dir(char *path) +{ + char *last_slash; + char *tmp_path; + + if (path) + tmp_path = xstrdup(path); + else { + /* init lsdg_start_path */ + dir_stats->lsdg_start_path = xstrdup(""); + tmp_path = xstrdup(dir_stats->lsdg_start_path); + } + + /* Check after mutex unlock */ + if (dir_stats->lsdg_start_dir != NULL) + return; + + /* Remove unnecessary slash for correct processing in the future */ + if (*tmp_path != '\0') { + last_slash = strrchr(tmp_path, '/'); + if (last_slash != NULL) + *last_slash = '\0'; + + } else { + last_slash = ""; + } + + last_slash = strrchr(tmp_path, '/'); + /* Send NULL ptr in ls3_stats_allocate_new_dir since we don't + * have attributes for the first directory. + */ + dir_stats->lsdg_start_dir = + ls3_stats_allocate_new_dir(NULL, + (last_slash ? + last_slash + 1 : tmp_path), -1); + free(tmp_path); +} + void ls3_stats_dir_init(void) { int rc; -- 1.8.3.1