EX-8130 lipe: Add functions to create a rating

This patch adds new functionality to directory statistics
for working with a ranking table for the largest
directories (like TOP 100).

The creation of a structure for storing the rating occurs
after the lipe_scan3 scan is completed. The number of
objects in the structure is determined before lipe_scan3
is launched by the default value or by the user in lipe_find3
via the -top-rating option and is not expanded while lipe_scan3
is running. Adding new objects to the heap works by the logic
of replacing the object with the smallest size in the heap with
a new object if its size is larger. Adding objects to the heap
occurs when printing the results about the directory sizes,
since only in this case do we know the final sizes of the directories.

Test-Parameters: trivial
Signed-off-by: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
Change-Id: Ie4a449fe69022716232638e0f856a10850403831
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/53959
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Alexandre Ioffe <aioffe@ddn.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lipe/src/lipe_scan3/ls3_dir_stats.c b/lipe/src/lipe_scan3/ls3_dir_stats.c
index 9a276d9..fc293bf 100644 (file)
--- a/lipe/src/lipe_scan3/ls3_dir_stats.c
+++ b/lipe/src/lipe_scan3/ls3_dir_stats.c
@@ -21,6 +21,162 @@
 
 struct ls3_stats_dir_general *dir_stats;
 
+/* ls3_sd_heap_parent - Get ls3_sd_heap_parent index of a obj. */
+static inline int ls3_sd_heap_parent(int i)
+{
+       assert(i > 0);
+       return (i - 1) / 2;
+}
+
+/* ls3_sd_heap_left - Get ls3_sd_heap_left child index of a obj. */
+static inline int ls3_sd_heap_left(int i)
+{
+       return (2 * i + 1);
+}
+
+/* ls3_sd_heap_right - Get ls3_sd_heap_right child index of a obj. */
+static inline int ls3_sd_heap_right(int i)
+{
+       return (2 * i + 2);
+}
+
+/* ls3_sd_swap_heap_obj - Swap two obj in the heap. */
+static void ls3_sd_swap_heap_obj(struct ls3_stats_dir_obj** a,
+                                struct ls3_stats_dir_obj** b)
+{
+       struct ls3_stats_dir_obj *temp = *a;
+
+       *a = *b;
+       *b = temp;
+}
+
+/* ls3_sd_heapify - Ensure the heap property by percolating down. */
+static void ls3_sd_heapify(struct ls3_stats_dir_rating* heap, int idx)
+{
+       int smallest = idx;
+       int left;
+       int right;
+
+       left = ls3_sd_heap_left(idx);
+       right = ls3_sd_heap_right(idx);
+
+       if (left < heap->lsdr_current_size_heap &&
+           heap->lsdr_array[left]->lsdo_size <
+           heap->lsdr_array[smallest]->lsdo_size)
+               smallest = left;
+
+       if (right < heap->lsdr_current_size_heap &&
+           heap->lsdr_array[right]->lsdo_size <
+           heap->lsdr_array[smallest]->lsdo_size)
+               smallest = right;
+
+       if (smallest != idx) {
+               ls3_sd_swap_heap_obj(&heap->lsdr_array[smallest],
+                                    &heap->lsdr_array[idx]);
+               ls3_sd_heapify(heap, smallest);
+       }
+}
+
+/* ls3_sd_extract_min - Extract and return the obj with the
+ * smallest value from the heap.
+ */
+static struct ls3_stats_dir_obj*
+       ls3_sd_extract_min(struct ls3_stats_dir_rating* heap)
+{
+       struct ls3_stats_dir_obj *root;
+
+       if (heap->lsdr_current_size_heap <= 0)
+               return NULL;
+
+       if (heap->lsdr_current_size_heap == 1) {
+               heap->lsdr_current_size_heap--;
+               return heap->lsdr_array[0];
+       }
+
+       root = heap->lsdr_array[0];
+       heap->lsdr_array[0] = heap->lsdr_array[heap->lsdr_current_size_heap - 1];
+       heap->lsdr_current_size_heap--;
+       ls3_sd_heapify(heap, 0);
+
+       return root;
+}
+
+/* ls3_stats_create_heap - Create a new heap with given capacity. */
+static struct ls3_stats_dir_rating* ls3_stats_create_heap(int capacity)
+{
+       struct ls3_stats_dir_rating* heap;
+
+       heap = (struct ls3_stats_dir_rating*)xcalloc(1,
+               sizeof(struct ls3_stats_dir_rating));
+
+       heap->lsdr_capacity_heap = capacity;
+       heap->lsdr_array = (struct ls3_stats_dir_obj**)xcalloc(capacity,
+                          sizeof(struct ls3_stats_dir_obj*));
+       heap->lsdr_min_size = UINT64_MAX;
+
+       return heap;
+}
+
+/* ls3_stats_heap_sort - Sorts the minimum heap by maximum size */
+static void ls3_stats_heap_sort(struct ls3_stats_dir_rating* heap)
+{
+       int original_size = heap->lsdr_current_size_heap;
+       int i;
+
+       for (i = heap->lsdr_current_size_heap - 1; i >= 0; i--) {
+               ls3_sd_swap_heap_obj(&heap->lsdr_array[0], &heap->lsdr_array[i]);
+               heap->lsdr_current_size_heap--;
+               ls3_sd_heapify(heap, 0);
+       }
+
+       heap->lsdr_current_size_heap = original_size;
+}
+
+/* ls3_stats_insert_heap - Insert a new obj into the heap */
+static void ls3_stats_insert_heap(struct ls3_stats_dir_rating* heap,
+                                  struct ls3_stats_dir_obj* dir_obj)
+{
+       int i;
+
+       assert(heap != NULL);
+       assert(heap->lsdr_capacity_heap > 0);
+
+       if (heap->lsdr_current_size_heap == heap->lsdr_capacity_heap) {
+               if (dir_obj->lsdo_size <= heap->lsdr_array[0]->lsdo_size)
+                       return;
+
+               ls3_sd_extract_min(heap);
+       }
+
+       heap->lsdr_total_size += dir_obj->lsdo_size;
+       if (dir_obj->lsdo_size < heap->lsdr_min_size)
+               heap->lsdr_min_size = dir_obj->lsdo_size;
+
+       if (dir_obj->lsdo_size > heap->lsdr_max_size)
+               heap->lsdr_max_size = dir_obj->lsdo_size;
+
+       i = heap->lsdr_current_size_heap++;
+       heap->lsdr_array[i] = dir_obj;
+
+       while (i != 0 && heap->lsdr_array[ls3_sd_heap_parent(i)]->lsdo_size >
+              heap->lsdr_array[i]->lsdo_size) {
+               ls3_sd_swap_heap_obj(&heap->lsdr_array[i],
+                                    &heap->lsdr_array[ls3_sd_heap_parent(i)]);
+               i = ls3_sd_heap_parent(i);
+       }
+}
+
+static void ls3_stats_init_heap(void)
+{
+       if (!dir_stats)
+               return;
+
+       if (dir_stats->lsdg_top_size == NULL)
+               /* Create/init HEAP */
+               dir_stats->lsdg_top_size =
+                       ls3_stats_create_heap(dir_stats->lsdg_top_rating_limit);
+}
+
 void ls3_stats_dir_init(void)
 {
        int rc;