LU-17251 test: improve parallel-scale rr_alloc test

Added checking for pre-created OST objects and waiting
(maximum 60 seconds) before executing the rr_alloc test.

Lustre-change: https://review.whamcloud.com/52940
Lustre-commit: TBD (from 3f1f70264e1ed9ba77094435fc598bc9abbbc044)

Test-Parameters: trivial
Test-Parameters: testlist=parallel-scale env=ONLY=rr_alloc,ONLY_REPEAT=8
Test-Parameters: testlist=parallel-scale env=ONLY=rr_alloc,ONLY_REPEAT=8
Test-Parameters: testlist=parallel-scale env=ONLY=rr_alloc,ONLY_REPEAT=8
Test-Parameters: testlist=parallel-scale env=ONLY=rr_alloc,ONLY_REPEAT=8
Test-Parameters: testlist=parallel-scale env=ONLY=rr_alloc,ONLY_REPEAT=8
Test-Parameters: testlist=parallel-scale env=ONLY=rr_alloc,ONLY_REPEAT=8
Signed-off-by: Alex Deiter <adeiter@tintri.com>
Change-Id: Ib604b99138ceccf384476ad2876d9df7cd7d524b
Reviewed-on: https://review.whamcloud.com/c/ex/lustre-release/+/52999
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lustre/tests/functions.sh b/lustre/tests/functions.sh
index 6f284bc..93771eb 100644 (file)
--- a/lustre/tests/functions.sh
+++ b/lustre/tests/functions.sh
@@ -1047,9 +1047,9 @@ run_rr_alloc() {
        # Make sure that every osp has enough precreated objects for the file
        # creation app
 
-       # create_count is always set to the power of 2 only, so if the files
-       # per OST are not multiple of that then it will be set to nearest
-       # lower power of 2. So set 'create_count' to the upper power of 2.
+       # The MDS does not precreate objects if there are at least
+       # create_count / 2 precreated objects available for the OST.
+       # Set 'create_count' to 2x required number to force creation.
 
        # foeo = file on each ost. calc = calculated.
        local foeo_calc=$((rr_alloc_NFILES * total_MNTPTS / OSTCOUNT))
@@ -1058,31 +1058,34 @@ run_rr_alloc() {
        # create_count accepted values:
        #   [OST_MIN_PRECREATE=32, OST_MAX_PRECREATE=20000]
        # values exceeding OST_MAX_PRECREATE are lowered to the maximum.
-       [[ $create_count -lt 32 ]] && create_count=32
-       local i
-       for i in $(seq $MDSCOUNT); do
-               do_facet mds$i "$LCTL set_param -n \
-                       lod.$FSNAME-MDT*.qos_threshold_rr=100 \
-                       osp.$FSNAME-OST*-osc-MDT*.create_count=$create_count"
-       done
+       (( create_count >= 32 )) || create_count=32
+
+       local mdts=$(comma_list $(mdts_nodes))
 
-       # Create few temporary files in order to increase the precreated objects
-       # to a desired value, before starting 'rr_alloc' app. Due to default
-       # value 32 of precreation count (OST_MIN_PRECREATE=32), precreated
-       # objects available are 32 initially, these gets exhausted very soon,
-       # which causes skip of some osps when very large number of files
-       # is created per OSTs.
-       createmany -o $DIR/$tdir/foo- $(((old_create_count + 1) * OSTCOUNT)) \
-               > /dev/null
-       unlinkmany $DIR/$tdir/foo- $(((old_create_count + 1)  * OSTCOUNT))
+       do_nodes $mdts "$LCTL set_param lod.*.qos_threshold_rr=100 \
+               osp.*.create_count=$create_count"
 
        # Check for enough precreated objects... We should not
        # fail here because code(osp_precreate.c) also takes care of it.
        # So we have good chances of passing test even if this check fails.
-       local mdt_idx=0
-       for ((ost_idx = 0; ost_idx < $OSTCOUNT; ost_idx++ )); do
-               (($(precreated_ost_obj_count $mdt_idx $ost_idx) >= foeo_calc))||
-               echo "Warning: test may fail from too few objs on OST$ost_idx"
+       local stop=$((SECONDS + 60))
+
+       while ((SECONDS < stop)); do
+               local sleep=0
+
+               for ((mdt_idx = 0; mdt_idx < $MDSCOUNT; mdt_idx++)); do
+                       for ((ost_idx = 0; ost_idx < $OSTCOUNT; ost_idx++)); do
+                               local count=$(precreated_ost_obj_count \
+                                       $mdt_idx $ost_idx)
+                               if ((count < foeo_calc)); then
+                                       sleep=1
+                               fi
+                       done
+               done
+
+               (( sleep > 0 )) || break
+
+               sleep $sleep
        done
 
        local cmd="$RR_ALLOC $mntpt_root/$tdir/f $rr_alloc_NFILES $num_clients"

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 25d5e89..f873593 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -9571,7 +9571,14 @@ precreated_ost_obj_count()
                        osp.$proc_path.prealloc_last_id)
        local next_id=$(do_facet mds$((mdt_idx + 1)) lctl get_param -n \
                        osp.$proc_path.prealloc_next_id)
-       echo $((last_id - next_id + 1))
+       local ost_obj_count=$((last_id - next_id + 1))
+
+       echo " - precreated_ost_obj_count $proc_path" \
+            "prealloc_last_id: $last_id" \
+            "prealloc_next_id: $next_id" \
+            "count: $ost_obj_count"
+
+       echo $ost_obj_count
 }
 
 check_file_in_pool()