LU-16110 lprocfs: make job_stats and rename_stats valid YAML

Adjust the format of job_stats and rename_stats to make
them valid YAML.

Lustre-change: https://review.whamcloud.com/48417
Lustre-commit: e96cb6ff1fea7a2bc62a6c0786fb0e07cbfda81a
Test-Parameters: testlist=sanity env=ONLY=907
Signed-off-by: Lei Feng <flei@whamcloud.com>
Change-Id: Idca36621241e97ff87f8ab0448f3c5604057a460
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49215
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lustre.spec.in b/lustre.spec.in
index 13ec7c0..b67ca28 100644 (file)
--- a/lustre.spec.in
+++ b/lustre.spec.in
@@ -350,6 +350,9 @@ Provides: lustre-tests = %{version}
 Requires: lustre-iokit
 %endif
 Requires: lustre-devel = %{version}
+%if 0%{?rhel} >= 8 || 0%{?suse_version} >= 1500
+Requires: python3 >= 3.6.0, python3-PyYAML
+%endif
 %if %{with lustre_modules}
 Requires: %{requires_kmod_name} = %{requires_kmod_version}
 Requires: %{requires_kmod_tests_name} = %{requires_kmod_version}

diff --git a/lustre/include/lprocfs_status.h b/lustre/include/lprocfs_status.h
index 85fab90..0b2edaf 100644 (file)
--- a/lustre/include/lprocfs_status.h
+++ b/lustre/include/lprocfs_status.h
@@ -595,7 +595,7 @@ extern int lprocfs_obd_seq_create(struct obd_device *obd, const char *name,
                                  void *data);
 extern void lprocfs_stats_header(struct seq_file *seq, ktime_t now,
                                 ktime_t ts_init, int width, const char *colon,
-                                bool show_units);
+                                bool show_units, const char *prefix);
 
 /* Generic callbacks */
 extern int lprocfs_uuid_seq_show(struct seq_file *m, void *data);

diff --git a/lustre/llite/lproc_llite.c b/lustre/llite/lproc_llite.c
index d80eb8e..ddcbc79 100644 (file)
--- a/lustre/llite/lproc_llite.c
+++ b/lustre/llite/lproc_llite.c
@@ -2016,7 +2016,8 @@ static int ll_rw_extents_stats_pp_seq_show(struct seq_file *seq, void *v)
        }
 
        spin_lock(&sbi->ll_pp_extent_lock);
-       lprocfs_stats_header(seq, ktime_get(), rw_extents->pp_init, 25, ":", 1);
+       lprocfs_stats_header(seq, ktime_get(), rw_extents->pp_init, 25, ":",
+                            true, "");
        seq_printf(seq, "%15s %19s       | %20s\n", " ", "read", "write");
        seq_printf(seq, "%13s   %14s %4s %4s  | %14s %4s %4s\n",
                   "extents", "calls", "%", "cum%", "calls", "%", "cum%");
@@ -2157,7 +2158,8 @@ static int ll_rw_extents_stats_seq_show(struct seq_file *seq, void *v)
        }
 
        spin_lock(&sbi->ll_lock);
-       lprocfs_stats_header(seq, ktime_get(), rw_extents->pp_init, 25, ":", 1);
+       lprocfs_stats_header(seq, ktime_get(), rw_extents->pp_init, 25, ":",
+                            true, "");
 
        seq_printf(seq, "%15s %19s       | %20s\n", " ", "read", "write");
        seq_printf(seq, "%13s   %14s %4s %4s  | %14s %4s %4s\n",
@@ -2340,7 +2342,7 @@ static int ll_rw_offset_stats_seq_show(struct seq_file *seq, void *v)
 
        spin_lock(&sbi->ll_process_lock);
        lprocfs_stats_header(seq, ktime_get(), sbi->ll_process_stats_init, 25,
-                            ":", true);
+                            ":", true, "");
        seq_printf(seq, "%3s %10s %14s %14s %17s %17s %14s\n",
                   "R/W", "PID", "RANGE START", "RANGE END",
                   "SMALLEST EXTENT", "LARGEST EXTENT", "OFFSET");

diff --git a/lustre/mdc/lproc_mdc.c b/lustre/mdc/lproc_mdc.c
index 506f06f..dfa02fb 100644 (file)
--- a/lustre/mdc/lproc_mdc.c
+++ b/lustre/mdc/lproc_mdc.c
@@ -518,7 +518,8 @@ static int mdc_stats_seq_show(struct seq_file *seq, void *v)
        struct obd_device *obd = seq->private;
        struct osc_stats *stats = &obd2osc_dev(obd)->od_stats;
 
-       lprocfs_stats_header(seq, ktime_get(), stats->os_init, 25, ":", true);
+       lprocfs_stats_header(seq, ktime_get(), stats->os_init, 25, ":", true,
+                            "");
        seq_printf(seq, "lockless_write_bytes\t\t%llu\n",
                   stats->os_lockless_writes);
        seq_printf(seq, "lockless_read_bytes\t\t%llu\n",

diff --git a/lustre/mdt/mdt_lproc.c b/lustre/mdt/mdt_lproc.c
index e3bfce8..954bc94 100644 (file)
--- a/lustre/mdt/mdt_lproc.c
+++ b/lustre/mdt/mdt_lproc.c
@@ -86,7 +86,7 @@ static void display_rename_stats(struct seq_file *seq, char *name,
 
        tot = lprocfs_oh_sum(rs_hist);
        if (tot > 0)
-               seq_printf(seq, "- %s\n", name);
+               seq_printf(seq, "- %s:\n", name);
 
        for (i = 0; i < OBD_HIST_MAX; i++) {
                t = rs_hist->oh_buckets[i];
@@ -113,9 +113,9 @@ static void rename_stats_show(struct seq_file *seq,
                              struct rename_stats *rename_stats)
 {
        /* this sampling races with updates */
-       seq_puts(seq, "rename_stats:\n- ");
+       seq_puts(seq, "rename_stats:\n-\n");
        lprocfs_stats_header(seq, ktime_get(), rename_stats->rs_init, 15, ":",
-                            false);
+                            false, "  ");
 
        display_rename_stats(seq, "same_dir",
                             &rename_stats->rs_hist[RENAME_SAMEDIR_SIZE]);

diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c
index 5f3edde..2450ccf 100644 (file)
--- a/lustre/obdclass/genops.c
+++ b/lustre/obdclass/genops.c
@@ -2199,7 +2199,7 @@ int obd_mod_rpc_stats_seq_show(struct client_obd *cli,
 
        spin_lock(&cli->cl_mod_rpcs_lock);
        lprocfs_stats_header(seq, ktime_get(), cli->cl_mod_rpcs_init, 25,
-                            ":", true);
+                            ":", true, "");
        seq_printf(seq, "modify_RPCs_in_flight:  %hu\n",
                   cli->cl_mod_rpcs_in_flight);
 

diff --git a/lustre/obdclass/lprocfs_jobstats.c b/lustre/obdclass/lprocfs_jobstats.c
index 5eca82e..5583462 100644 (file)
--- a/lustre/obdclass/lprocfs_jobstats.c
+++ b/lustre/obdclass/lprocfs_jobstats.c
@@ -469,7 +469,7 @@ static int lprocfs_jobstats_seq_show(struct seq_file *p, void *v)
        seq_printf(p, "- %-16s %s%*s%s\n",
                   "job_id:", quote, joblen, escaped, quote);
        lprocfs_stats_header(p, job->js_time_latest, job->js_time_init, 16,
-                            ":", true);
+                            ":", true, "  ");
 
        s = job->js_stats;
        for (i = 0; i < s->ls_num; i++) {

diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c
index ca0234d..a2e7abf 100644 (file)
--- a/lustre/obdclass/lprocfs_status.c
+++ b/lustre/obdclass/lprocfs_status.c
@@ -1361,21 +1361,40 @@ static void *lprocfs_stats_seq_next(struct seq_file *p, void *v, loff_t *pos)
        return lprocfs_stats_seq_start(p, pos);
 }
 
+/**
+ * print header of stats including snapshot_time, start_time and elapsed_time.
+ *
+ * \param seq          the file to print content to
+ * \param now          end time to calculate elapsed_time
+ * \param ts_init      start time to calculate elapsed_time
+ * \param width                the width of key to align them well
+ * \param colon                "" or ":"
+ * \param show_units   show units or not
+ * \param prefix       prefix (indent) before printing each line of header
+ *                     to align them with other content
+ */
 void lprocfs_stats_header(struct seq_file *seq, ktime_t now, ktime_t ts_init,
-                         int width, const char *colon, bool show_units)
+                         int width, const char *colon, bool show_units,
+                         const char *prefix)
 {
        const char *units = show_units ? " secs.nsecs" : "";
        struct timespec64 ts;
+       const char *field;
 
+       field = (colon && colon[0]) ? "snapshot_time:" : "snapshot_time";
        ts = ktime_to_timespec64(now);
-       seq_printf(seq, "%-*s%s %llu.%09lu%s\n", width,
-                  "snapshot_time", colon, (s64)ts.tv_sec, ts.tv_nsec, units);
+       seq_printf(seq, "%s%-*s %llu.%09lu%s\n", prefix, width, field,
+                  (s64)ts.tv_sec, ts.tv_nsec, units);
+
+       field = (colon && colon[0]) ? "start_time:" : "start_time";
        ts = ktime_to_timespec64(ts_init);
-       seq_printf(seq, "%-*s%s %llu.%09lu%s\n", width,
-                  "start_time", colon, (s64)ts.tv_sec, ts.tv_nsec, units);
+       seq_printf(seq, "%s%-*s %llu.%09lu%s\n", prefix, width, field,
+                  (s64)ts.tv_sec, ts.tv_nsec, units);
+
+       field = (colon && colon[0]) ? "elapsed_time:" : "elapsed_time";
        ts = ktime_to_timespec64(ktime_sub(now, ts_init));
-       seq_printf(seq, "%-*s%s %llu.%09lu%s\n", width,
-                  "elapsed_time", colon, (s64)ts.tv_sec, ts.tv_nsec, units);
+       seq_printf(seq, "%s%-*s %llu.%09lu%s\n", prefix, width, field,
+                  (s64)ts.tv_sec, ts.tv_nsec, units);
 }
 EXPORT_SYMBOL(lprocfs_stats_header);
 
@@ -1388,7 +1407,8 @@ static int lprocfs_stats_seq_show(struct seq_file *p, void *v)
        int idx = *(loff_t *)v;
 
        if (idx == 0)
-               lprocfs_stats_header(p, ktime_get(), stats->ls_init, 25, "", 1);
+               lprocfs_stats_header(p, ktime_get(), stats->ls_init, 25, "",
+                                    true, "");
 
        hdr = &stats->ls_cnt_header[idx];
        lprocfs_stats_collect(stats, idx, &ctr);

diff --git a/lustre/obdclass/lprocfs_status_server.c b/lustre/obdclass/lprocfs_status_server.c
index afdcb51..a437310 100644 (file)
--- a/lustre/obdclass/lprocfs_status_server.c
+++ b/lustre/obdclass/lprocfs_status_server.c
@@ -776,7 +776,8 @@ static int brw_stats_seq_show(struct seq_file *seq, void *v)
        int i;
 
        /* this sampling races with updates */
-       lprocfs_stats_header(seq, ktime_get(), brw_stats->bs_init, 25, ":", 1);
+       lprocfs_stats_header(seq, ktime_get(), brw_stats->bs_init, 25, ":",
+                            true, "");
 
        for (i = 0; i < ARRAY_SIZE(brw_stats->bs_props); i++) {
                if (!brw_stats->bs_props[i].bsp_name)

diff --git a/lustre/osc/lproc_osc.c b/lustre/osc/lproc_osc.c
index b93dcb1..10de69a 100644 (file)
--- a/lustre/osc/lproc_osc.c
+++ b/lustre/osc/lproc_osc.c
@@ -713,7 +713,8 @@ static int osc_rpc_stats_seq_show(struct seq_file *seq, void *v)
 
        spin_lock(&cli->cl_loi_list_lock);
 
-       lprocfs_stats_header(seq, ktime_get(), cli->cl_stats_init, 25, ":", 1);
+       lprocfs_stats_header(seq, ktime_get(), cli->cl_stats_init, 25, ":",
+                            true, "");
        seq_printf(seq, "read RPCs in flight:  %d\n",
                   cli->cl_r_in_flight);
        seq_printf(seq, "write RPCs in flight: %d\n",
@@ -822,7 +823,8 @@ static int osc_stats_seq_show(struct seq_file *seq, void *v)
        struct obd_device *obd = seq->private;
        struct osc_stats *stats = &obd2osc_dev(obd)->od_stats;
 
-       lprocfs_stats_header(seq, ktime_get(), stats->os_init, 25, ":", true);
+       lprocfs_stats_header(seq, ktime_get(), stats->os_init, 25, ":", true,
+                            "");
        seq_printf(seq, "lockless_write_bytes\t\t%llu\n",
                   stats->os_lockless_writes);
        seq_printf(seq, "lockless_read_bytes\t\t%llu\n",

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index aa322a7..1533e55 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -1171,6 +1171,7 @@ test_24s() {
        $CHECKSTAT -t dir $DIR/R15a/b/c || error "$DIR/R15a/b/c missing"
 }
 run_test 24s "mkdir .../R15a/b/c; rename .../R15a .../R15a/b/c ="
+
 test_24t() {
        test_mkdir $DIR/R16a
        test_mkdir $DIR/R16a/b
@@ -28271,6 +28272,42 @@ test_904() {
 }
 run_test 904 "virtual project ID xattr"
 
+test_907() {
+       local file=$DIR/$tdir/$tfile
+
+       (( $MDS1_VERSION >= $(version_code 2.15.1) )) ||
+               skip "need lustre >= 2.15.51"
+       (( $OST1_VERSION >= $(version_code 2.15.1) )) ||
+               skip "need lustre >= 2.15.51"
+       verify_yaml_available || skip_env "YAML verification not installed"
+
+       test_mkdir $DIR/$tdir
+       $LFS setstripe -E 1M -L mdt -E -1 $file || error "setstripe failed"
+
+       dd if=/dev/zero of=$file bs=1M count=10 conv=sync ||
+               error "failed to write data to $file"
+       mv $file $file.2
+
+       echo -n 'verify rename_stats...'
+       output=$(do_facet mds1 \
+                "$LCTL get_param -n mdt.$FSNAME-MDT0000.rename_stats")
+       verify_yaml "$output" || error "rename_stats is not valid YAML"
+       echo " OK"
+
+       echo -n 'verify mdt job_stats...'
+       output=$(do_facet mds1 \
+                "$LCTL get_param -n mdt.$FSNAME-MDT0000.job_stats")
+       verify_yaml "$output" || error "job_stats on mds1 is not valid YAML"
+       echo " OK"
+
+       echo -n 'verify ost job_stats...'
+       output=$(do_facet ost1 \
+                "$LCTL get_param -n obdfilter.$FSNAME-OST0000.job_stats")
+       verify_yaml "$output" || error "job_stats on ost1 is not valid YAML"
+       echo " OK"
+}
+run_test 907 "verify the format of some stats files"
+
 complete $SECONDS
 [ -f $EXT2_DEV ] && rm $EXT2_DEV || true
 check_and_cleanup_lustre

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 266b96a..860b1fc 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -10838,3 +10838,11 @@ exhaust_all_precreations() {
        done
        sleep_maxage
 }
+
+verify_yaml_available() {
+       python3 -c "import yaml; yaml.safe_load_all('''a: b''')"
+}
+
+verify_yaml() {
+       python3 -c "import yaml; yaml.safe_load_all('''$1''')"
+}