EX-8130 lipe: Add directory scan structs

This patch adds new structures to lipe_scan3 for collecting
and storing directory statistics, as well as initialization
and destroy functions. This patch is the first in a series
of patches that add functionality for collecting directory
statistics in lipe_find3.

Test-Parameters: trivial
Signed-off-by: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
Change-Id: Ib14ce13677d93d1a53299501138e78c7b290793c
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53958
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Alexandre Ioffe <aioffe@ddn.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.c b/lipe/src/lipe_scan3/ls3_dir_stats.c
new file mode 100644 (file)
index 0000000..9a276d9
--- /dev/null
+++ b/lipe/src/lipe_scan3/ls3_dir_stats.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2024, DDN Storage Corporation.
+ *
+ * Author: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
+ */
+
+#include <libgen.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <inttypes.h>
+#include <ctype.h>
+#include <limits.h>
+#include <assert.h>
+#include <lustre/lustreapi.h>
+#include "ls3_fid2path.h"
+#include "ls3_stats.h"
+#include "ls3_dir_stats.h"
+
+struct ls3_stats_dir_general *dir_stats;
+
+void ls3_stats_dir_init(void)
+{
+       int rc;
+
+       /* This is the first and only place with initialization. */
+       dir_stats = xcalloc(1, sizeof(struct ls3_stats_dir_general));
+       rc = pthread_mutex_init(&dir_stats->lsdg_mutex, NULL);
+       if (rc != 0)
+               LS3_FATAL("failed to initialize mutex\n");
+
+}
+
+static void ls3_stats_destroy_dirs(struct ls3_stats_dir_obj *dir_ptr)
+{
+       struct ls3_stats_dir_obj *new_dir_ptr;
+       int i;
+
+       for (i = 0; i < dir_ptr->lsdo_child_max; i++) {
+               new_dir_ptr = dir_ptr->lsdo_child[i];
+               if (new_dir_ptr != NULL)
+                       ls3_stats_destroy_dirs(new_dir_ptr);
+       }
+
+       pthread_mutex_destroy(&dir_ptr->lsdo_mutex);
+
+       if (dir_ptr->lsdo_name)
+               free(dir_ptr->lsdo_name);
+
+       free(dir_ptr);
+}
+
+void ls3_stats_dir_destroy(void)
+{
+       if (dir_stats->lsdg_start_dir)
+               ls3_stats_destroy_dirs(dir_stats->lsdg_start_dir);
+
+       if (dir_stats->lsdg_top_size) {
+               free(dir_stats->lsdg_top_size->lsdr_array);
+               free(dir_stats->lsdg_top_size);
+       }
+
+       pthread_mutex_destroy(&dir_stats->lsdg_mutex);
+       if (dir_stats->lsdg_start_path)
+               free(dir_stats->lsdg_start_path);
+
+       free(dir_stats);
+}

diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.h b/lipe/src/lipe_scan3/ls3_dir_stats.h
new file mode 100644 (file)
index 0000000..dc0c26f
--- /dev/null
+++ b/lipe/src/lipe_scan3/ls3_dir_stats.h
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2024, DDN Storage Corporation.
+ *
+ * Author: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
+ */
+
+#ifndef _LS3_DIR_STATS_H_
+#define _LS3_DIR_STATS_H_
+
+#include <math.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <linux/fs.h>
+#include <errno.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <time.h>
+#include <stdbool.h>
+#include <json-c/json.h>
+#include "ls3_debug.h"
+#include "ls3_scan.h"
+#include "ls3_object_attrs.h"
+
+#define LS3_STATS_DIR_COUNT_BY_DEFAULT 10
+#define LS3_STATS_DEPTH_BY_DEFAULT 4
+#define LS3_STATS_TOP_RATING_BY_DEFAULT 100
+/* In bytes. For count the size of empty directories and dir size on OST */
+#define LS3_STATS_DIR_SIZE 4096
+
+/* ls3_stats_dir_obj - Stores basic information about the directory.
+ * Current size by sizeof == 112
+ */
+struct ls3_stats_dir_obj {
+       pthread_mutex_t                 lsdo_mutex;
+       struct lu_fid                   lsdo_fid;       /* Only for MDT */
+       uint64_t                        lsdo_size;      /* Total dir size */
+       /* Total allocate size (number of 512B blocks) from inodes */
+       uint64_t                        lsdo_alloc_size_in_512blocks;
+       uint32_t                        lsdo_files_count;
+       uint32_t                        lsdo_dirs_count;
+       uint32_t                        lsdo_child_max;
+       uint32_t                        lsdo_last_child;
+       uint32_t                        lsdo_uid;               /* Owner */
+       uint16_t                        lsdo_compress_type;
+       int16_t                         lsdo_depth; /* Current object depth */
+       char                            *lsdo_name;
+       struct ls3_stats_dir_obj        **lsdo_child;
+};
+
+/* ls3_stats_dir_rating - This structure stores a ranking of the
+ * top N (like TOP-100) largest directories by size. The default
+ * object count is equivalent to LS3_STATS_TOP_RATING_BY_DEFAULT,
+ * but the number of objects in this rating can be specified by
+ * the user using the -top-rating option. Work this struct
+ * occurs according to the principle of a minimal heap.
+ */
+struct ls3_stats_dir_rating {
+       uint64_t                        lsdr_min_size;
+       uint64_t                        lsdr_max_size;
+       uint64_t                        lsdr_total_size;
+       uint32_t                        lsdr_current_size_heap;
+       uint32_t                        lsdr_capacity_heap;
+       struct ls3_stats_dir_obj        **lsdr_array;
+};
+
+struct ls3_stats_dir_general {
+       pthread_mutex_t                 lsdg_mutex;
+       uint64_t                        lsdg_total_size;
+       uint64_t                        lsdg_total_alloc_size;
+       uint32_t                        lsdg_max_depth;
+       uint32_t                        lsdg_top_rating_limit;
+       uint32_t                        lsdg_empty_dirs_count; /* Only for MDT */
+       uint32_t                        lsdg_min_size_value_for_print;
+       uint32_t                        lsdg_dirs_count;
+       char                            *lsdg_start_path;
+       struct ls3_stats_dir_obj        *lsdg_start_dir;
+       struct ls3_stats_dir_rating     *lsdg_top_size;
+};
+
+extern struct ls3_stats_dir_general *dir_stats;
+
+void ls3_stats_dir_init(void);
+void ls3_stats_dir_destroy(void);
+
+#endif /* _LS3_DIR_STATS_H_ */