LU-15938 llog: more checks in llog_reader

Add more correctness checks and reports in llog_reader:
- better report wrong record length and chunk skipping case
- add tail check: tail id and len should be the same as in head
- better report for gap in record indeces
- test case with two corruption types:
  1) llog has bits set in bitmap beyond file end
  2) corruption in the middle

Signed-off-by: Mikhail Pershin <mpershin@whamcloud.com>
Change-Id: I0c2af6ae2592c94e14e90ead12e28104409313b2
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/48112
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Alexander <alexander.boyko@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 11a3442..db35b4d 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -8786,6 +8786,70 @@ test_60i() {
 }
 run_test 60i "llog: new record vs reader race"
 
+test_60j() {
+       (( $MDS1_VERSION >= $(version_code 2.15.50) )) ||
+               skip "need MDS version at least 2.15.50"
+       [[ $PARALLEL != "yes" ]] || skip "skip parallel run"
+       remote_mds_nodsh && skip "remote MDS with nodsh"
+       [[ "$mds1_FSTYPE" == "ldiskfs" ]] || skip "ldiskfs only test"
+
+       changelog_users $SINGLEMDS | grep "^cl" &&
+               skip "active changelog user"
+
+       local llog_reader=$(do_facet $SINGLEMDS "which llog_reader 2> /dev/null")
+
+       [[ -z $(do_facet $SINGLEMDS ls -d $llog_reader 2> /dev/null) ]] &&
+               skip_env "missing llog_reader"
+
+       mkdir_on_mdt0 $DIR/$tdir
+
+       local f=$DIR/$tdir/$tfile
+       local mdt_dev
+       local tmpfile
+       local plain
+
+       changelog_register || error "cannot register changelog user"
+
+       # set changelog_mask to ALL
+       changelog_chmask "ALL"
+       changelog_clear
+
+       createmany -o ${f}- 100 || error "createmany failed as $RUNAS_ID"
+       unlinkmany ${f}- 100 || error "unlinkmany failed"
+
+       tmpfile="$(mktemp --tmpdir -u $tfile.XXXXXX)"
+       mdt_dev=$(facet_device $SINGLEMDS)
+
+       do_facet $SINGLEMDS sync
+       plain=$(do_facet $SINGLEMDS "$DEBUGFS -c -R 'dump changelog_catalog \
+               $tmpfile' $mdt_dev; $llog_reader $tmpfile" |
+               awk '{match($0,"path=([^ ]+)",a)}END{print a[1]}')
+
+       stack_trap "do_facet $SINGLEMDS rm -f $tmpfile"
+
+       # if $tmpfile is not on EXT3 filesystem for some reason
+       [[ ${plain:0:1} == 'O' ]] ||
+               skip "path $plain is not in 'O/1/d<n>/<n>' format"
+
+       size=$(do_facet $SINGLEMDS "$DEBUGFS -c -R 'dump $plain $tmpfile' \
+               $mdt_dev; stat -c %s $tmpfile")
+       echo "Truncate llog from $size to $((size - size % 8192))"
+       size=$((size - size % 8192))
+       do_facet $SINGLEMDS $TRUNCATE $tmpfile $size
+       errs=$(do_facet $SINGLEMDS "$llog_reader $tmpfile" |
+               grep -c 'in bitmap only')
+       (( $errs > 0 )) || error "llog_reader didn't find lost records"
+
+       size=$((size - 9000))
+       echo "Corrupt llog in the middle at $size"
+       do_facet $SINGLEMDS dd if=/dev/urandom of=$tmpfile bs=1 seek=$size \
+               count=333 conv=notrunc
+       errs=$(do_facet $SINGLEMDS "$llog_reader $tmpfile" |
+               grep -c 'next chunk')
+       (( $errs > 0 )) || error "llog_reader didn't skip bad chunk"
+}
+run_test 60j "llog_reader reports corruptions"
+
 test_61a() {
        [ $PARALLEL == "yes" ] && skip "skip parallel run"
 

diff --git a/lustre/utils/llog_reader.c b/lustre/utils/llog_reader.c
index c218af2..dac7b5d 100644 (file)
--- a/lustre/utils/llog_reader.c
+++ b/lustre/utils/llog_reader.c
@@ -307,7 +307,8 @@ int llog_pack_buffer(int fd, struct llog_log_hdr **llog,
        last_idx = 0;
        while (ptr < (file_buf + file_size)) {
                struct llog_rec_hdr *cur_rec;
-               int idx;
+               struct llog_rec_tail *cur_tail;
+               unsigned int idx, len, tail_idx, tail_len;
                unsigned long offset;
 
                offset = (unsigned long)ptr - (unsigned long)file_buf;
@@ -319,31 +320,53 @@ int llog_pack_buffer(int fd, struct llog_log_hdr **llog,
                }
                cur_rec = (struct llog_rec_hdr *)ptr;
                idx = __le32_to_cpu(cur_rec->lrh_index);
-               if (cur_rec->lrh_len == 0 ||
-                   cur_rec->lrh_len > (*llog)->llh_hdr.lrh_len) {
-                       cur_rec->lrh_len = (*llog)->llh_hdr.lrh_len -
-                               offset % (*llog)->llh_hdr.lrh_len;
-                       printf("off %lu skip %u to next chunk.\n", offset,
-                              cur_rec->lrh_len);
-               } else if (ext2_test_bit(idx, LLOG_HDR_BITMAP(*llog))) {
+               len = __le32_to_cpu(cur_rec->lrh_len);
+
+               if (len == 0 || len + offset % (*llog)->llh_hdr.lrh_len >
+                               (*llog)->llh_hdr.lrh_len) {
+                       printf("error: rec #%d type=%x has wrong len=%u @%lu\n",
+                              idx, cur_rec->lrh_type, len, offset);
+                       errors++;
+                       len = (*llog)->llh_hdr.lrh_len -
+                             offset % (*llog)->llh_hdr.lrh_len;
+                       printf("skip %u bytes to the next chunk at off %lu.\n",
+                              len, offset + len);
+                       ptr += len;
+                       continue;
+               }
+
+               if (ext2_test_bit(idx, LLOG_HDR_BITMAP(*llog))) {
                        printf("rec #%d type=%x len=%u offset %lu\n", idx,
-                              cur_rec->lrh_type, cur_rec->lrh_len, offset);
+                              cur_rec->lrh_type, len, offset);
                        recs_pr[i] = cur_rec;
                        i++;
                } else {
                        cur_rec->lrh_id = CANCELLED;
                        if (cur_rec->lrh_type == LLOG_PAD_MAGIC &&
-                          ((offset + cur_rec->lrh_len) & 0x7) != 0) {
+                          ((offset + len) & 0x7) != 0) {
                                printf("error: rec #%d wrong padding len=%u offset %lu to 0x%lx\n",
-                                      idx, cur_rec->lrh_len, offset,
-                                      offset + cur_rec->lrh_len);
+                                      idx, len, offset, offset + len);
                                errors++;
                        }
                        /* The header counts only set records */
                }
 
+               cur_tail = (struct llog_rec_tail *)(ptr + len - sizeof(*cur_tail));
+               tail_idx = __le32_to_cpu(cur_tail->lrt_index);
+               tail_len = __le32_to_cpu(cur_tail->lrt_len);
+               if (idx != tail_idx || len != tail_len) {
+                       printf("error: rec #%d len=%u has tail #%d len=%u%s\n",
+                              idx, len, tail_idx, tail_len,
+                              cur_rec->lrh_id == CANCELLED ? " (unset)" : "");
+                       errors++;
+               }
+
+               if (idx > last_idx + 1)
+                       printf("error: rec #%d len=%u has gap in %d recs\n",
+                              idx, len, idx - last_idx);
+
                while (++last_idx < idx) {
-                       printf("error: rec #%d is missing%s set in bitmap\n",
+                       printf("error: -> rec #%d is lost%s set in bitmap\n",
                               last_idx,
                               ext2_test_bit(last_idx, LLOG_HDR_BITMAP(*llog)) ?
                               " but" : ", not");
@@ -357,7 +380,7 @@ int llog_pack_buffer(int fd, struct llog_log_hdr **llog,
                        last_idx = idx;
                }
 
-               ptr += __le32_to_cpu(cur_rec->lrh_len);
+               ptr += len;
                if ((ptr - file_buf) > file_size) {
                        printf("error: rec #%d is trimmed by EOF, offset %lu\n",
                               idx, offset);