EX-8191 lipe: Fix test for --collect-fsize-stats in lipe3

This patch modifies the test for collecting statistics
in lipe3 and corrects:
1. Error getting a username if it doesnt already exist.
2. Error comparing file sizes after changing table
generation rules.
3. Converts the test from reading yaml to reading json
4. Now many files of different sizes are generated
for the test.
5. Now the data for comparison is retrieved from
the ls utility.
6. The test has added a check for creating a user with a
large UID, GID, which checks the availability of reports
for this user.

Test-Parameters: trivial testlist=sanity-lipe-scan3
Signed-off-by: Vitaliy Kuznetsov <vkuznetsov@ddn.com>
Change-Id: I7d0bdcc407bc0d27441c4204511dab2e6a421a5f
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/52424
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lipe/src/lipe_scan3/ls3_stats.c b/lipe/src/lipe_scan3/ls3_stats.c
index b33ce23..975785c 100644 (file)
--- a/lipe/src/lipe_scan3/ls3_stats.c
+++ b/lipe/src/lipe_scan3/ls3_stats.c
@@ -60,14 +60,14 @@ static const char* ls3_stats_get_username_from_uid(uid_t uid)
 {
        struct passwd *pw = getpwuid(uid);
 
-       return pw ? pw->pw_name : NULL;
+       return pw ? pw->pw_name : "NULL";
 }
 
 static const char* ls3_stats_get_groupname_from_gid(gid_t gid)
 {
        struct group *grp = getgrgid(gid);
 
-       return grp ? grp->gr_name : NULL;
+       return grp ? grp->gr_name : "NULL";
 }
 
 static const ls3_stats_extension_mapping ext_mappings[] = {
@@ -299,7 +299,7 @@ static char *ls3_stats_get_title_text(ls3_stats_report_type report_type)
        case LS3_STATS_TYPE_FILES_SIZE:
                return "Files Size";
        case LS3_STATS_TYPE_CAPACITY_USED:
-               return "Capacity used (Regular file's space used on disk)";
+               return "Capacity used (Regular files space used on disk)";
        case LS3_STATS_TYPE_EQUAL_OVERHEAD:
                return "Equal overhead (Files whose capacity used is "
                       "equal to their size)";
@@ -2031,6 +2031,8 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all)
 
        ls3_stats_update_range(LS3_STATS_TYPE_FILES_SIZE, loa_all->loa_size,
                               LS3_STATS_EMPTY_VALUE);
+       ls3_stats_update_range(LS3_STATS_TYPE_CAPACITY_USED, allocate_file_size,
+                              LS3_STATS_EMPTY_VALUE);
        ls3_stats_update_range(LS3_STATS_TYPE_TIME_SINCE_LAST_MOD_RF,
                               loa_all->loa_mtime, loa_all->loa_size);
        ls3_stats_update_range(LS3_STATS_TYPE_TIME_SINCE_LAST_MD_MOD_RF,
@@ -2117,10 +2119,6 @@ int ls3_stats_update_info(struct ls3_object_attrs *loa_all)
                ls3_stats_update_range_with_id(LS3_STATS_TYPE_STRIPE_SIZE,
                                               ptr_ost_layout.ol_stripe_size,
                                               loa_all->loa_size);
-
-               ls3_stats_update_range(LS3_STATS_TYPE_CAPACITY_USED,
-                                      allocate_file_size,
-                                      LS3_STATS_EMPTY_VALUE);
        }
 
        return 0;

diff --git a/lustre/tests/sanity-lipe-scan3.sh b/lustre/tests/sanity-lipe-scan3.sh
index 319b012..031710f 100644 (file)
--- a/lustre/tests/sanity-lipe-scan3.sh
+++ b/lustre/tests/sanity-lipe-scan3.sh
@@ -1071,20 +1071,43 @@ test_305() {
 }
 run_test 305 "print-json prints the right paths"
 
+create_files() {
+       local num_files=$1
+       local file=$2
+
+       # Create one large sparse file on 1T == 1099511627776 byte
+       truncate "$file" 1099511627776
+       [[ $? -ne 0 ]] && error "truncate ended with an error"
+       sync
+
+       # Create many small files of different sizes
+       for ((i = 0; i < $num_files; i++)); do
+               fallocate -l $((RANDOM % 125 + 1))Kb $file"_"$i
+               [[ $? -ne 0 ]] && error "fallocate ended with an error"
+       done
+       sync
+}
+
 # Test for --collect-fsize-stats policy
 test_306() {
        local facet=mds1
-       local report_path="$MOUNT/files_size_report.yaml"
+       local report_path="$MOUNT/$tfile.json"
        local file=$MOUNT/$tfile
-       local no_of_files
+       local tmp_id=1000000123
+       local count_files_ls
+       local total_size_ls
+       local id_from_report
+       local count_files
        local total_size
 
        init_lipe_scan3_env "$file"
-       fallocate -l 10M $file
-
-       [[ $? -ne 0 ]] && error "dd ended with an error"
+       create_files 3000 "$file"
+       chown "$tmp_id:$tmp_id" $file
+       [[ $? -ne 0 ]] && error "chown ended with an error"
 
        sync
+       count_files_ls=$(ls -1 "$MOUNT" | grep -vE '^total' | wc -l)
+       total_size_ls=$(ls -l "$MOUNT" | grep '^total' | awk '{print $2}')
 
        out=$(lipe_scan3_facet "$facet" --collect-fsize-stats="$report_path")
 
@@ -1092,16 +1115,29 @@ test_306() {
 
        [[ -f "$report_path" ]] || error "File with report not found."
 
-       verify_yaml $report_path || error "$report_path not yaml formatted"
+       count_files=$(cat "$report_path" |
+                     jq -r '.Reports[] | select(.GeneralInfo.Title ==
+                     "Capacity used (Regular files space used on disk)") |
+                     .GeneralInfo | .Count' | grep -oE '[0-9]+')
+
+       total_size=$(cat "$report_path" |
+                    jq '.Reports[] | select(.GeneralInfo.Title ==
+                    "Capacity used (Regular files space used on disk)") |
+                    .GeneralInfo.Total' | grep -oE '[0-9]+')
+
+       id_from_report=$(cat "$report_path" |
+                        jq '.UserTimeReports[] |
+                        select(.UserName == "NULL") |
+                        .UserUID' | grep -oE '[0-9]+')
 
-       no_of_files=$(awk '/Count:/ {print $2; exit}' "$report_path")
-       total_size=$(awk '/Total:/ {print $2; exit}' "$report_path")
+       (( count_files == count_files_ls )) ||
+               error "expected no_of_files value '$count_files_ls', got '$count_files'"
 
-       (( no_of_files == 1 )) ||
-               error "expected no_of_files value 1, got '$no_of_files'"
+       (( total_size == total_size_ls )) ||
+               error "expected total_size value '$total_size_ls', got '$total_size'"
 
-       [[ "$total_size" != "10240" ]] &&
-               error "expected total_size value 10240, got '$total_size'"
+       (( tmp_id == id_from_report )) ||
+               error "expected UID value '$tmp_id', got '$id_from_report'"
 
        echo "--collect-fsize-stats works"
 }